Showing preview only (6,297K chars total). Download the full file or copy to clipboard to get everything.
Repository: juicedata/juicefs
Branch: main
Commit: acb45e1224c5
Files: 755
Total size: 5.9 MB
Directory structure:
gitextract_vo3z05bo/
├── .autocorrectrc
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug-report.md
│ │ ├── enhancement.md
│ │ └── support.md
│ ├── actions/
│ │ ├── build/
│ │ │ └── action.yml
│ │ ├── cancel-outdate-runs/
│ │ │ └── action.yml
│ │ ├── mount-coverage-dir/
│ │ │ └── action.yml
│ │ ├── upload-coverage/
│ │ │ └── action.yml
│ │ └── upload-total-coverage/
│ │ └── action.yml
│ ├── scripts/
│ │ ├── apt_install.sh
│ │ ├── cache.sh
│ │ ├── chaos/
│ │ │ ├── dynamic.yaml
│ │ │ ├── juicefs-csi-driver.Dockerfile
│ │ │ ├── juicefs.Dockerfile
│ │ │ ├── minio.yaml
│ │ │ ├── pvc.yaml
│ │ │ ├── redis.yaml
│ │ │ ├── sc.yaml
│ │ │ └── workflow.yaml
│ │ ├── check_juicefs_log.sh
│ │ ├── cmptree.py
│ │ ├── command/
│ │ │ ├── acl.sh
│ │ │ ├── clone.sh
│ │ │ ├── config.sh
│ │ │ ├── debug.sh
│ │ │ ├── dump_load.sh
│ │ │ ├── dump_load_bench.sh
│ │ │ ├── dump_load_cross_meta.sh
│ │ │ ├── format.sh
│ │ │ ├── fsck.sh
│ │ │ ├── gateway-random.sh
│ │ │ ├── gateway.sh
│ │ │ ├── gc.sh
│ │ │ ├── graceful_upgrade.sh
│ │ │ ├── info.sh
│ │ │ ├── interface.sh
│ │ │ ├── mount.sh
│ │ │ ├── quota.sh
│ │ │ └── random.sh
│ │ ├── command-win/
│ │ │ ├── acl.sh
│ │ │ ├── clone.sh
│ │ │ ├── debug.sh
│ │ │ ├── dump_load.sh
│ │ │ ├── fsck.sh
│ │ │ ├── gateway.sh
│ │ │ ├── gc.sh
│ │ │ ├── profile.sh
│ │ │ └── quota.sh
│ │ ├── common/
│ │ │ ├── common.sh
│ │ │ ├── common_win.sh
│ │ │ └── run_test.sh
│ │ ├── compare_results.sh
│ │ ├── copyFile.js
│ │ ├── fio.sh
│ │ ├── flush_meta.py
│ │ ├── fsrand.py
│ │ ├── hypo/
│ │ │ ├── command.py
│ │ │ ├── command_op.py
│ │ │ ├── command_test.py
│ │ │ ├── common.py
│ │ │ ├── context.py
│ │ │ ├── file.py
│ │ │ ├── file_op.py
│ │ │ ├── file_test.py
│ │ │ ├── fs.py
│ │ │ ├── fs_acl_test.py
│ │ │ ├── fs_op.py
│ │ │ ├── fs_sdk_test.py
│ │ │ ├── fs_test.py
│ │ │ ├── readme.md
│ │ │ ├── s3.py
│ │ │ ├── s3_contant.py
│ │ │ ├── s3_op.py
│ │ │ ├── s3_strategy.py
│ │ │ ├── s3_test.py
│ │ │ ├── stats.py
│ │ │ ├── strategy.py
│ │ │ ├── sync.py
│ │ │ └── sync_test.py
│ │ ├── mutate/
│ │ │ ├── check_coverage.py
│ │ │ ├── check_skip_by_comment.py
│ │ │ ├── how_to_use_mutate_test.md
│ │ │ ├── modify_sdk_pom.py
│ │ │ ├── mutest.sh
│ │ │ ├── mutesting.py
│ │ │ ├── parse_black_list.py
│ │ │ ├── parse_job_total.py
│ │ │ ├── parse_mutate_log.py
│ │ │ ├── parse_test_cases.py
│ │ │ ├── query_report.py
│ │ │ └── save_report.py
│ │ ├── perf/
│ │ │ ├── ai.sh
│ │ │ ├── ai_format_benchmark.py
│ │ │ ├── compare_ai.sh
│ │ │ ├── compare_mdtest_fio.sh
│ │ │ └── mdtest_fio.sh
│ │ ├── prepare_db.sh
│ │ ├── pysdk/
│ │ │ ├── bench.py
│ │ │ └── pysdk_test.py
│ │ ├── random_read_write.py
│ │ ├── save_benchmark.sh
│ │ ├── setup-hdfs.sh
│ │ ├── ssh/
│ │ │ ├── Dockerfile
│ │ │ └── docker-compose.yml
│ │ ├── start_meta_engine.sh
│ │ ├── sync/
│ │ │ ├── sync.sh
│ │ │ ├── sync_cluster.sh
│ │ │ ├── sync_fsrand.sh
│ │ │ └── sync_minio.sh
│ │ ├── test-mac/
│ │ │ ├── mac_commands.sh
│ │ │ └── start_meta_engine.sh
│ │ ├── testVersionCompatible.py
│ │ ├── upload_coverage_report.sh
│ │ ├── utils.py
│ │ └── wins_fs_test.py
│ └── workflows/
│ ├── bash/
│ │ ├── rm_fs
│ │ ├── rm_list.sh
│ │ └── rm_syscalls
│ ├── cache.yml
│ ├── cancel_outdate_runs.yml
│ ├── chaos.yml
│ ├── check-doc.yaml
│ ├── codeql-analysis.yml
│ ├── command-win.yml
│ ├── command.yml
│ ├── command2.yml
│ ├── compile.yml
│ ├── coverage-report.yml
│ ├── dependency-review.yml
│ ├── dockerfile-sftp
│ ├── dump_load.yml
│ ├── dump_load_bench.yml
│ ├── dump_load_cross_meta.yml
│ ├── fsrand.yml
│ ├── fsspec.yml
│ ├── gateway-random.yml
│ ├── gateway.yml
│ ├── integrationtests.yml
│ ├── ltpfs.yml
│ ├── ltpsyscalls.yml
│ ├── mutate-test-sdk.yml
│ ├── mutate-test.yml
│ ├── perf-test.yml
│ ├── permission-check.yaml
│ ├── pjdfstest.yml
│ ├── pysdk.yml
│ ├── random-test.yml
│ ├── release.yml
│ ├── resources/
│ │ ├── core-site.xml
│ │ ├── load-balancer.conf
│ │ ├── sync-options.txt
│ │ ├── tpcds_datagen.scala
│ │ ├── tpcds_run.scala
│ │ ├── vdbench_big_file.conf
│ │ ├── vdbench_long_run.conf
│ │ └── vdbench_small_file.conf
│ ├── rmfiles.yml
│ ├── sdktest.yml
│ ├── sync.yml
│ ├── unit-random-tests.yml
│ ├── unittests.yml
│ ├── vdbench.yml
│ ├── verify.yml
│ ├── version_compatible_hypo.yml
│ ├── wintest.yml
│ └── xattr.yml
├── .gitignore
├── .golangci.yml
├── .goreleaser.yml
├── .markdownlint-cli2.jsonc
├── .pre-commit-config.yaml
├── ADOPTERS.md
├── ADOPTERS_CN.md
├── CODEOWNERS
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── Makefile
├── README.md
├── README_CN.md
├── check-changed.sh
├── cmd/
│ ├── bench.go
│ ├── bench_test.go
│ ├── clone.go
│ ├── compact.go
│ ├── compact_test.go
│ ├── config.go
│ ├── config_test.go
│ ├── debug.go
│ ├── debug_test.go
│ ├── debug_unix.go
│ ├── debug_windows.go
│ ├── destroy.go
│ ├── dump.go
│ ├── dump_test.go
│ ├── flags.go
│ ├── flags_test.go
│ ├── format.go
│ ├── format_test.go
│ ├── fsck.go
│ ├── fsck_test.go
│ ├── gateway.go
│ ├── gateway_noop.go
│ ├── gc.go
│ ├── gc_test.go
│ ├── info.go
│ ├── info_test.go
│ ├── integration_test.go
│ ├── load.go
│ ├── main.go
│ ├── main_test.go
│ ├── mdtest.go
│ ├── mount.go
│ ├── mount_test.go
│ ├── mount_unix.go
│ ├── mount_windows.go
│ ├── objbench.go
│ ├── object.go
│ ├── object_test.go
│ ├── passfd.go
│ ├── printsid.go
│ ├── profile.go
│ ├── quota.go
│ ├── restore.go
│ ├── restore_test.go
│ ├── rmr.go
│ ├── rmr_test.go
│ ├── stats.go
│ ├── status.go
│ ├── status_test.go
│ ├── summary.go
│ ├── sync.go
│ ├── sync_test.go
│ ├── umount.go
│ ├── version.go
│ ├── warmup.go
│ ├── warmup_test.go
│ ├── webdav.go
│ └── webdav_noop.go
├── codecov.yml
├── deploy/
│ └── juicefs-s3-gateway.yaml
├── docs/
│ ├── README.md
│ ├── en/
│ │ ├── administration/
│ │ │ ├── destroy.md
│ │ │ ├── fault_diagnosis_and_analysis.md
│ │ │ ├── metadata/
│ │ │ │ ├── _category_.yml
│ │ │ │ ├── etcd_best_practices.md
│ │ │ │ ├── fdb_best_practices.md
│ │ │ │ ├── mysql_best_practices.md
│ │ │ │ ├── postgresql_best_practices.md
│ │ │ │ ├── redis_best_practices.md
│ │ │ │ └── tikv_best_practices.md
│ │ │ ├── metadata_dump_load.md
│ │ │ ├── monitoring.md
│ │ │ ├── mount_at_boot.md
│ │ │ ├── status_check_and_maintenance.md
│ │ │ ├── sync_accounts_between_multiple_hosts.md
│ │ │ ├── troubleshooting.md
│ │ │ └── upgrade.md
│ │ ├── benchmark/
│ │ │ ├── benchmark.md
│ │ │ ├── fio.md
│ │ │ ├── mdtest.md
│ │ │ ├── metadata_engines_benchmark.md
│ │ │ └── performance_evaluation_guide.md
│ │ ├── community/
│ │ │ ├── _roadmap.md
│ │ │ ├── adopters.md
│ │ │ ├── articles.md
│ │ │ ├── integrations.md
│ │ │ └── usage_tracking.md
│ │ ├── deployment/
│ │ │ ├── _share_via_nfs.md
│ │ │ ├── _share_via_smb.md
│ │ │ ├── automation.md
│ │ │ ├── hadoop_java_sdk.md
│ │ │ ├── how_to_use_on_kubernetes.md
│ │ │ ├── juicefs_on_docker.md
│ │ │ ├── nfs.md
│ │ │ ├── production_deployment_recommendations.md
│ │ │ ├── python_sdk.md
│ │ │ ├── samba.md
│ │ │ └── webdav.md
│ │ ├── development/
│ │ │ ├── contributing_guide.md
│ │ │ └── internals.md
│ │ ├── faq.md
│ │ ├── getting-started/
│ │ │ ├── for_distributed.md
│ │ │ ├── installation.md
│ │ │ └── standalone.md
│ │ ├── grafana_template.json
│ │ ├── guide/
│ │ │ ├── cache.md
│ │ │ ├── clone.md
│ │ │ ├── dir-stats.md
│ │ │ ├── gateway.md
│ │ │ ├── quota.md
│ │ │ └── sync.md
│ │ ├── introduction/
│ │ │ ├── README.md
│ │ │ ├── architecture.md
│ │ │ ├── comparison/
│ │ │ │ ├── _category_.yml
│ │ │ │ ├── juicefs_vs_3fs.md
│ │ │ │ ├── juicefs_vs_alluxio.md
│ │ │ │ ├── juicefs_vs_cephfs.md
│ │ │ │ ├── juicefs_vs_glusterfs.md
│ │ │ │ ├── juicefs_vs_lustre.md
│ │ │ │ ├── juicefs_vs_s3fs.md
│ │ │ │ ├── juicefs_vs_s3ql.md
│ │ │ │ └── juicefs_vs_seaweedfs.md
│ │ │ └── io_processing.md
│ │ ├── reference/
│ │ │ ├── _common_options.mdx
│ │ │ ├── command_reference.mdx
│ │ │ ├── fuse_mount_options.md
│ │ │ ├── how_to_set_up_metadata_engine.md
│ │ │ ├── how_to_set_up_object_storage.md
│ │ │ ├── p8s_metrics.md
│ │ │ ├── posix_compatibility.md
│ │ │ ├── redis-csc.md
│ │ │ └── spec-limits.md
│ │ ├── release_notes.md
│ │ ├── security/
│ │ │ ├── encryption.md
│ │ │ ├── posix_acl.md
│ │ │ └── trash.md
│ │ └── tutorials/
│ │ ├── aliyun.md
│ │ ├── aws.md
│ │ ├── digitalocean.md
│ │ ├── juicefs_on_colab.md
│ │ ├── juicefs_on_k3s.md
│ │ ├── juicefs_on_kubesphere.md
│ │ ├── juicefs_on_rancher.md
│ │ ├── juicefs_on_wsl.md
│ │ ├── qcloud.md
│ │ └── windows.md
│ └── zh_cn/
│ ├── administration/
│ │ ├── destroy.md
│ │ ├── fault_diagnosis_and_analysis.md
│ │ ├── metadata/
│ │ │ ├── _category_.yml
│ │ │ ├── etcd_best_practices.md
│ │ │ ├── fdb_best_practices.md
│ │ │ ├── mysql_best_practices.md
│ │ │ ├── postgresql_best_practices.md
│ │ │ ├── redis_best_practices.md
│ │ │ └── tikv_best_practices.md
│ │ ├── metadata_dump_load.md
│ │ ├── monitoring.md
│ │ ├── mount_at_boot.md
│ │ ├── status_check_and_maintenance.md
│ │ ├── sync_accounts_between_multiple_hosts.md
│ │ ├── troubleshooting.md
│ │ └── upgrade.md
│ ├── benchmark/
│ │ ├── benchmark.md
│ │ ├── fio.md
│ │ ├── mdtest.md
│ │ ├── metadata_engines_benchmark.md
│ │ └── performance_evaluation_guide.md
│ ├── community/
│ │ ├── _roadmap.md
│ │ ├── adopters.md
│ │ ├── articles.md
│ │ ├── integrations.md
│ │ └── usage_tracking.md
│ ├── deployment/
│ │ ├── _share_via_nfs.md
│ │ ├── _share_via_smb.md
│ │ ├── automation.md
│ │ ├── hadoop_java_sdk.md
│ │ ├── how_to_use_on_kubernetes.md
│ │ ├── juicefs_on_docker.md
│ │ ├── nfs.md
│ │ ├── production_deployment_recommendations.md
│ │ ├── python_sdk.md
│ │ ├── samba.md
│ │ └── webdav.md
│ ├── development/
│ │ ├── contributing_guide.md
│ │ └── internals.md
│ ├── faq.md
│ ├── getting-started/
│ │ ├── for_distributed.md
│ │ ├── installation.md
│ │ └── standalone.md
│ ├── guide/
│ │ ├── cache.md
│ │ ├── clone.md
│ │ ├── dir-stats.md
│ │ ├── gateway.md
│ │ ├── quota.md
│ │ └── sync.md
│ ├── introduction/
│ │ ├── README.md
│ │ ├── architecture.md
│ │ ├── comparison/
│ │ │ ├── _category_.yml
│ │ │ ├── juicefs_vs_3fs.md
│ │ │ ├── juicefs_vs_alluxio.md
│ │ │ ├── juicefs_vs_cephfs.md
│ │ │ ├── juicefs_vs_glusterfs.md
│ │ │ ├── juicefs_vs_lustre.md
│ │ │ ├── juicefs_vs_s3fs.md
│ │ │ ├── juicefs_vs_s3ql.md
│ │ │ └── juicefs_vs_seaweedfs.md
│ │ └── io_processing.md
│ ├── reference/
│ │ ├── _common_options.mdx
│ │ ├── command_reference.mdx
│ │ ├── fuse_mount_options.md
│ │ ├── how_to_set_up_metadata_engine.md
│ │ ├── how_to_set_up_object_storage.md
│ │ ├── p8s_metrics.md
│ │ ├── posix_compatibility.md
│ │ └── spec-limits.md
│ ├── release_notes.md
│ ├── security/
│ │ ├── encryption.md
│ │ ├── posix_acl.md
│ │ └── trash.md
│ └── tutorials/
│ ├── aliyun.md
│ ├── aws.md
│ ├── digitalocean.md
│ ├── juicefs_on_colab.md
│ ├── juicefs_on_k3s.md
│ ├── juicefs_on_kubesphere.md
│ ├── juicefs_on_rancher.md
│ ├── juicefs_on_wsl.md
│ ├── qcloud.md
│ └── windows.md
├── go.mod
├── go.sum
├── hack/
│ ├── autocomplete/
│ │ ├── bash_autocomplete
│ │ └── zsh_autocomplete
│ ├── builder/
│ │ ├── Dockerfile
│ │ └── sdk.Dockerfile
│ └── winfsp_headers/
│ ├── fuse.h
│ ├── fuse_common.h
│ ├── fuse_opt.h
│ └── winfsp_fuse.h
├── integration/
│ ├── Makefile
│ ├── ioctl_test.sh
│ └── s3gateway_test.sh
├── main.go
├── package.json
├── pkg/
│ ├── acl/
│ │ ├── acl.go
│ │ ├── cache.go
│ │ └── cache_test.go
│ ├── chunk/
│ │ ├── cache_eviction.go
│ │ ├── cached_store.go
│ │ ├── cached_store_test.go
│ │ ├── chunk.go
│ │ ├── disk_cache.go
│ │ ├── disk_cache_state.go
│ │ ├── disk_cache_state_test.go
│ │ ├── disk_cache_test.go
│ │ ├── mem_cache.go
│ │ ├── metrics.go
│ │ ├── page.go
│ │ ├── page_test.go
│ │ ├── prefetch.go
│ │ ├── prefetch_test.go
│ │ ├── singleflight.go
│ │ ├── singleflight_test.go
│ │ ├── utils_darwin.go
│ │ ├── utils_linux.go
│ │ ├── utils_unix.go
│ │ ├── utils_unix_test.go
│ │ └── utils_windows.go
│ ├── compress/
│ │ ├── compress.go
│ │ └── compress_test.go
│ ├── fs/
│ │ ├── fs.go
│ │ ├── fs_test.go
│ │ ├── http.go
│ │ ├── http_test.go
│ │ └── metrics.go
│ ├── fuse/
│ │ ├── context.go
│ │ ├── device_darwin.go
│ │ ├── device_linux.go
│ │ ├── fuse.go
│ │ ├── fuse_darwin.go
│ │ ├── fuse_linux.go
│ │ ├── fuse_test.go
│ │ ├── gidcache.go
│ │ └── utils.go
│ ├── gateway/
│ │ ├── gateway.go
│ │ └── gateway_test.go
│ ├── meta/
│ │ ├── backup.go
│ │ ├── base.go
│ │ ├── base_test.go
│ │ ├── benchmarks_test.go
│ │ ├── config.go
│ │ ├── config_test.go
│ │ ├── context.go
│ │ ├── dump.go
│ │ ├── info.go
│ │ ├── info_test.go
│ │ ├── interface.go
│ │ ├── interface_test.go
│ │ ├── load_dump_test.go
│ │ ├── lua_scripts.go
│ │ ├── metadata-sub.sample
│ │ ├── metadata.sample
│ │ ├── openfile.go
│ │ ├── pb/
│ │ │ ├── backup.pb.go
│ │ │ └── backup.proto
│ │ ├── quota.go
│ │ ├── random_test.go
│ │ ├── redis.go
│ │ ├── redis_bak.go
│ │ ├── redis_csc.go
│ │ ├── redis_csc_test.go
│ │ ├── redis_lock.go
│ │ ├── slice.go
│ │ ├── sql.go
│ │ ├── sql_bak.go
│ │ ├── sql_lock.go
│ │ ├── sql_mysql.go
│ │ ├── sql_pg.go
│ │ ├── sql_sqlite.go
│ │ ├── sql_test.go
│ │ ├── status.go
│ │ ├── tkv.go
│ │ ├── tkv_badger.go
│ │ ├── tkv_bak.go
│ │ ├── tkv_etcd.go
│ │ ├── tkv_fdb.go
│ │ ├── tkv_fdb_test.go
│ │ ├── tkv_lock.go
│ │ ├── tkv_mem.go
│ │ ├── tkv_prefix.go
│ │ ├── tkv_test.go
│ │ ├── tkv_tikv.go
│ │ ├── utils.go
│ │ ├── utils_darwin.go
│ │ ├── utils_linux.go
│ │ ├── utils_test.go
│ │ └── utils_windows.go
│ ├── metric/
│ │ └── metrics.go
│ ├── object/
│ │ ├── azure.go
│ │ ├── b2.go
│ │ ├── bos.go
│ │ ├── bunny.go
│ │ ├── ceph.go
│ │ ├── checksum.go
│ │ ├── checksum_test.go
│ │ ├── cifs.go
│ │ ├── cos.go
│ │ ├── dragonfly.go
│ │ ├── encrypt.go
│ │ ├── encrypt_test.go
│ │ ├── eos.go
│ │ ├── etcd.go
│ │ ├── file.go
│ │ ├── file_darwin.go
│ │ ├── file_linux.go
│ │ ├── file_unix.go
│ │ ├── file_unix_test.go
│ │ ├── file_windows.go
│ │ ├── filesystem_test.go
│ │ ├── gluster.go
│ │ ├── gluster_test.go
│ │ ├── gs.go
│ │ ├── hdfs.go
│ │ ├── hdfs_kerberos.go
│ │ ├── ibmcos.go
│ │ ├── interface.go
│ │ ├── ks3.go
│ │ ├── mem.go
│ │ ├── minio.go
│ │ ├── nfs.go
│ │ ├── object_storage.go
│ │ ├── object_storage_test.go
│ │ ├── obs.go
│ │ ├── oos.go
│ │ ├── oss.go
│ │ ├── prefix.go
│ │ ├── qingstor.go
│ │ ├── qiniu.go
│ │ ├── redis.go
│ │ ├── response_attrs.go
│ │ ├── response_attrs_test.go
│ │ ├── restful.go
│ │ ├── restful_test.go
│ │ ├── s3.go
│ │ ├── s3_test.go
│ │ ├── scw.go
│ │ ├── sftp.go
│ │ ├── sharding.go
│ │ ├── space.go
│ │ ├── sql.go
│ │ ├── sql_mysql.go
│ │ ├── sql_pg.go
│ │ ├── sql_sqlite.go
│ │ ├── swift.go
│ │ ├── tikv.go
│ │ ├── tos.go
│ │ ├── ufile.go
│ │ ├── wasabi.go
│ │ └── webdav.go
│ ├── sync/
│ │ ├── cluster.go
│ │ ├── cluster_test.go
│ │ ├── config.go
│ │ ├── download.go
│ │ ├── download_test.go
│ │ ├── sync.go
│ │ └── sync_test.go
│ ├── usage/
│ │ ├── usage.go
│ │ └── usage_test.go
│ ├── utils/
│ │ ├── alloc.go
│ │ ├── alloc_test.go
│ │ ├── buffer.go
│ │ ├── buffer_test.go
│ │ ├── clock_test.go
│ │ ├── clock_unix.go
│ │ ├── clock_windows.go
│ │ ├── cond.go
│ │ ├── cond_test.go
│ │ ├── errors.go
│ │ ├── general.go
│ │ ├── humanize.go
│ │ ├── logger.go
│ │ ├── logger_syslog.go
│ │ ├── logger_test.go
│ │ ├── logger_windows.go
│ │ ├── memusage.go
│ │ ├── memusage_test.go
│ │ ├── memusage_windows.go
│ │ ├── proc_title.go
│ │ ├── proc_title_noop.go
│ │ ├── progress.go
│ │ ├── progress_test.go
│ │ ├── rusage.go
│ │ ├── rusage_test.go
│ │ ├── rusage_windows.go
│ │ ├── utils.go
│ │ ├── utils_darwin.go
│ │ ├── utils_linux.go
│ │ ├── utils_test.go
│ │ ├── utils_unix.go
│ │ └── utils_windows.go
│ ├── version/
│ │ ├── .gitattributes
│ │ ├── version.go
│ │ └── version_test.go
│ ├── vfs/
│ │ ├── accesslog.go
│ │ ├── accesslog_test.go
│ │ ├── backup.go
│ │ ├── backup_test.go
│ │ ├── compact.go
│ │ ├── compact_test.go
│ │ ├── fill.go
│ │ ├── fill_test.go
│ │ ├── handle.go
│ │ ├── helpers.go
│ │ ├── helpers_test.go
│ │ ├── internal.go
│ │ ├── reader.go
│ │ ├── vfs.go
│ │ ├── vfs_test.go
│ │ ├── vfs_unix.go
│ │ ├── vfs_windows.go
│ │ └── writer.go
│ ├── win/
│ │ ├── ldap.go
│ │ └── sid.go
│ └── winfsp/
│ ├── log.go
│ └── winfs.go
├── rfcs/
│ └── 1-dir-used-statistics.md
└── sdk/
├── java/
│ ├── .gitignore
│ ├── Makefile
│ ├── conf/
│ │ ├── contract/
│ │ │ └── juicefs.xml
│ │ ├── core-site.xml
│ │ └── log4j.properties
│ ├── kerberos.sh
│ ├── libjfs/
│ │ ├── Makefile
│ │ ├── bridge.go
│ │ ├── bridge_test.go
│ │ ├── callback.c
│ │ ├── guid.go
│ │ ├── guid_unix.go
│ │ ├── guid_windows.go
│ │ ├── kerberos.go
│ │ ├── main.go
│ │ ├── remote_write.go
│ │ └── remote_write_test.go
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── io/
│ │ │ └── juicefs/
│ │ │ ├── FlinkFileSystemFactory.java
│ │ │ ├── JuiceFS.java
│ │ │ ├── JuiceFileSystem.java
│ │ │ ├── JuiceFileSystemImpl.java
│ │ │ ├── KiteDataLoader.java
│ │ │ ├── Main.java
│ │ │ ├── bench/
│ │ │ │ ├── AccumulatingReducer.java
│ │ │ │ ├── IOMapperBase.java
│ │ │ │ ├── NNBench.java
│ │ │ │ └── TestDFSIO.java
│ │ │ ├── exception/
│ │ │ │ └── QuotaExceededException.java
│ │ │ ├── kerberos/
│ │ │ │ ├── AuthCredential.java
│ │ │ │ ├── JuiceFSDelegationTokenIdentifier.java
│ │ │ │ ├── JuiceFSTokenRenewer.java
│ │ │ │ └── KerberosUtil.java
│ │ │ ├── metrics/
│ │ │ │ └── JuiceFSInstrumentation.java
│ │ │ ├── permission/
│ │ │ │ ├── RangerAdminRefresher.java
│ │ │ │ ├── RangerConfig.java
│ │ │ │ ├── RangerJfsAccessRequest.java
│ │ │ │ ├── RangerJfsPlugin.java
│ │ │ │ ├── RangerJfsResource.java
│ │ │ │ ├── RangerPermissionChecker.java
│ │ │ │ ├── RangerPermissionContext.java
│ │ │ │ ├── RangerPluginCfg.java
│ │ │ │ └── RangerRules.java
│ │ │ ├── tools/
│ │ │ │ └── RangerDownloader.java
│ │ │ └── utils/
│ │ │ ├── AclTransformation.java
│ │ │ ├── BgTaskUtil.java
│ │ │ ├── BufferPool.java
│ │ │ ├── CallerContextUtil.java
│ │ │ ├── ConsistentHash.java
│ │ │ ├── FsNodesFetcher.java
│ │ │ ├── FsPermissionExtension.java
│ │ │ ├── NodesFetcher.java
│ │ │ ├── NodesFetcherBuilder.java
│ │ │ ├── PatchUtil.java
│ │ │ ├── PrestoNodesFetcher.java
│ │ │ ├── RedefineClassAgent.java
│ │ │ ├── ReflectionUtil.java
│ │ │ ├── SparkNodesFetcher.java
│ │ │ ├── SparkThriftNodesFetcher.java
│ │ │ └── YarnNodesFetcher.java
│ │ └── resources/
│ │ └── META-INF/
│ │ └── services/
│ │ ├── org.apache.flink.core.fs.FileSystemFactory
│ │ ├── org.apache.hadoop.security.token.TokenIdentifier
│ │ ├── org.apache.hadoop.security.token.TokenRenewer
│ │ └── org.kitesdk.data.spi.Loadable
│ └── test/
│ ├── java/
│ │ └── io/
│ │ └── juicefs/
│ │ ├── JuiceFileSystemBgTaskTest.java
│ │ ├── JuiceFileSystemTest.java
│ │ ├── acl/
│ │ │ └── TestAclCLI.java
│ │ ├── contract/
│ │ │ ├── JuiceFSContract.java
│ │ │ ├── TestAppend.java
│ │ │ ├── TestConcat.java
│ │ │ ├── TestCreate.java
│ │ │ ├── TestDelete.java
│ │ │ ├── TestGetFileStatus.java
│ │ │ ├── TestJuiceFileSystemContract.java
│ │ │ ├── TestMkdir.java
│ │ │ ├── TestOpen.java
│ │ │ ├── TestRename.java
│ │ │ ├── TestSeek.java
│ │ │ └── TestSetTimes.java
│ │ ├── kerberos/
│ │ │ └── KerberosTest.java
│ │ ├── permission/
│ │ │ ├── RangerAdminClientImpl.java
│ │ │ └── RangerPermissionCheckerTest.java
│ │ └── utils/
│ │ ├── BgTaskUtilTest.java
│ │ └── HashTest.java
│ ├── resources/
│ │ ├── hdfs-policies-tag.json
│ │ ├── hdfs-policies.json
│ │ ├── kerberos.cfg
│ │ ├── log4j.properties
│ │ └── testAclCLI.xml
│ └── test-spark.sh
└── python/
├── .gitignore
├── Dockerfile.builder
├── Dockerfile.builder.arm
├── Makefile
├── examples/
│ ├── ffrecord/
│ │ ├── dataloader.py
│ │ ├── dataset.py
│ │ ├── filereader.py
│ │ ├── filereader_dio.py
│ │ ├── main.py
│ │ └── readme.md
│ └── fsspec/
│ ├── main.py
│ └── readme.md
└── juicefs/
├── juicefs/
│ ├── __init__.py
│ ├── juicefs.py
│ └── spec.py
├── setup.py
└── tests/
├── __init__.py
└── test.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .autocorrectrc
================================================
rules:
# Default rules: https://github.com/huacnlee/autocorrect/raw/main/autocorrect/.autocorrectrc.default
spellcheck: 1
textRules:
# Config some special rule for some texts
# For example, if we wants to let "Hello你好" just warning, and "Hi你好" to ignore
# "Hello你好": 2
# "Hi你好": 0
fileTypes:
# Config the files associations, you config is higher priority than default.
# "rb": ruby
# "Rakefile": ruby
# "*.js": javascript
# ".mdx": markdown
spellcheck:
words:
# Please do not add a general English word (eg. apple, python) here.
# Users can add their special words to their .autocorrectrc file by their need.
- Digital Ocean = DigitalOcean
- JucieFS = JuiceFS
- JueicFS = JuiceFS
- JuiecFS = JuiceFS
- filesystem = file system
- mountpoint = mount point
================================================
FILE: .github/ISSUE_TEMPLATE/bug-report.md
================================================
---
name: Bug Report
about: Report a bug encountered while operating JuiceFS
labels: kind/bug
---
<!--
Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!
-->
**What happened**:
**What you expected to happen**:
**How to reproduce it (as minimally and precisely as possible)**:
**Anything else we need to know?**
**Environment**:
- JuiceFS version (use `juicefs --version`) or Hadoop Java SDK version:
- Cloud provider or hardware configuration running JuiceFS:
- OS (e.g `cat /etc/os-release`):
- Kernel (e.g. `uname -a`):
- Object storage (cloud provider and region, or self maintained):
- Metadata engine info (version, cloud provider managed or self maintained):
- Network connectivity (JuiceFS to metadata engine, JuiceFS to object storage):
- Others:
================================================
FILE: .github/ISSUE_TEMPLATE/enhancement.md
================================================
---
name: Enhancement Request
about: Suggest an enhancement to the JuiceFS project
labels: kind/feature
---
<!-- Please only use this template for submitting enhancement requests -->
**What would you like to be added**:
**Why is this needed**:
================================================
FILE: .github/ISSUE_TEMPLATE/support.md
================================================
---
name: Support Request
about: Support request or question relating to JuiceFS
labels: kind/question
---
<!--
STOP -- PLEASE READ!
GitHub issue is not the right place for support requests.
If you're looking for help, check the Discussions (https://github.com/juicedata/juicefs/discussions).
You can also post your question on the Discussions or the JuiceFS Slack channel (https://juicefs.slack.com).
-->
================================================
FILE: .github/actions/build/action.yml
================================================
name: 'Build Action'
description: 'Build action'
inputs:
target:
description: 'build target: juicefs, juicefs.fdb etc'
required: true
default: 'juicefs'
beta:
description: 'beta version for the following test'
required: false
runs:
using: "composite"
steps:
- uses: actions/setup-go@v3
with:
go-version: 'oldstable'
cache: true
- name: Change go version for root user
shell: bash
run: |
go_path=`which go`
echo $go_path
root_go_path=`sudo which go`
echo $root_go_path
sudo rm -f $root_go_path
sudo ln -s $go_path $root_go_path
go version
sudo go version
- name: Install tools
shell: bash
run: |
if [ "${{inputs.target}}" == "juicefs.fdb" ]; then
wget -q https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb
sudo dpkg -i foundationdb-clients_6.3.23-1_amd64.deb
elif [ "${{inputs.target}}" == "juicefs.gluster" ]; then
sudo .github/scripts/apt_install.sh uuid-dev libglusterfs-dev
fi
- name: Build linux target
shell: bash
run: |
if [[ -n "${{ inputs.beta }}" ]]; then
echo "use beta version of juicefs: ${{inputs.beta}}"
wget -q https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/juicefs_beta/${{inputs.beta}} -O juicefs
chmod +x juicefs
./juicefs version
else
echo "start to build ${{inputs.target}}"
make ${{inputs.target}}.cover
[ "${{inputs.target}}" != "juicefs" ] && mv ${{inputs.target}} juicefs
./juicefs version
echo "build ${{inputs.target}} succeed"
fi
================================================
FILE: .github/actions/cancel-outdate-runs/action.yml
================================================
name: 'Cancel Outdate Runs'
description: 'Cancel Outdate Runs'
inputs:
head_sha:
description: 'head_sha triggers the workflow runs'
required: true
type: string
per_page:
description: 'Page size of runs to cancel'
required: true
type: number
default: 5
page:
description: 'Page number of runs to cancel'
required: true
type: number
default: 1
github_token:
description: 'GITHUB_TOKEN'
required: true
type: string
runs:
using: "composite"
steps:
- name: display parameters
shell: bash
run: |
echo "head_sha is ${{inputs.head_sha}}"
echo "per_page is ${{inputs.per_page}}"
echo "page is ${{inputs.page}}"
- uses: octokit/request-action@v2.x
id: get_active_workflows
with:
route: GET /repos/${{github.repository}}/actions/runs?status=in_progress&event=pull_request&per_page=${{inputs.per_page}}&page=${{inputs.page}}&head_sha=${{inputs.head_sha}}
env:
GITHUB_TOKEN: ${{inputs.github_token}}
- name: display active workflows
shell: bash
env:
data: ${{ steps.get_active_workflows.outputs.data }}
run: |
echo "$data" | jq '.workflow_runs | map({id, head_sha, pull_request_number:.pull_requests[0].number})'
- name: Extract workflow ids
shell: bash
id: extract_workflow_ids
env:
data: ${{ steps.get_active_workflows.outputs.data }}
run: |
echo pull_request_number is ${{ github.event.pull_request.number }}
echo head_sha is ${{ github.event.pull_request.head.sha }}
workflow_ids=$(echo "$data" | \
jq '.workflow_runs | map({id, head_sha, pull_request_number:.pull_requests[0].number})' | \
jq 'map(select( .pull_request_number == ${{ github.event.pull_request.number }} and .head_sha != "${{ github.event.pull_request.head.sha }}")) | map(.id)' | \
jq 'join(",")')
echo workflow_ids is $workflow_ids
echo 'WORKFLOW_IDS='$(echo $workflow_ids | tr -d '"') >> $GITHUB_ENV
- name: Cancel active workflows
shell: bash
run: |
for i in ${WORKFLOW_IDS//,/ }
do
echo "Cancelling workflow with id: $i"
# use curl here as I have no idea how to use a github action in a loop
curl \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{inputs.github_token}}" \
https://api.github.com/repos/${{ github.repository }}/actions/runs/$i/cancel
done
================================================
FILE: .github/actions/mount-coverage-dir/action.yml
================================================
name: 'mount_coverage_dir'
description: 'mount coverage directory'
inputs:
mount_point:
description: 'mount point'
required: true
type: string
subdir:
description: 'subdir'
required: false
type: string
token:
description: 'token of jfs'
required: true
type: string
access_key:
description: 'access key of object storage service'
required: true
type: string
secret_key:
description: 'secret key of object storage service'
required: true
type: string
runs:
using: "composite"
steps:
- name: set subdir
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs)
job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}") | select(.status=="in_progress") | .id')
echo Job ID is: ${job_id}
if [ "${{ github.event_name }}" == "pull_request" ]; then
branch=${GITHUB_BASE_REF} # 目标分支
elif [ "${{ github.event_name }}" == "push" ]; then
branch=${GITHUB_REF#refs/heads/} # 当前分支
else
branch=${GITHUB_REF#refs/heads/} # 对于 schedule 和 workflow_dispatch
fi
echo input.subdir is ${{inputs.subdir}}
if [ -n "${{inputs.subdir}}" ]; then
subdir=${{inputs.subdir}}
elif [[ "${{github.event_name}}" == "schedule" ]]; then
subdir=juicefs/schedule/$(date +"%Y%m%d")/${{github.workflow}}
elif [[ "${{github.job}}" == "success-all-test" ]]; then
subdir=juicefs/pr/$branch/${{github.workflow}}/${{github.run_id}}
else
subdir=juicefs/pr/$branch/${{github.workflow}}/${{github.run_id}}/${job_id}
fi
echo "subdir=$subdir"
echo "subdir=$subdir" >> $GITHUB_ENV
- name: mount coverage dir
shell: bash
run: |
sudo mkdir -p /root/.juicefs
if ! sudo test -f /root/.juicefs/jfsmount; then
sudo wget -q s.juicefs.com/static/Linux/mount -O /root/.juicefs/jfsmount
sudo chmod +x /root/.juicefs/jfsmount
fi
sudo curl -s -L https://juicefs.com/static/juicefs -o /usr/local/bin/juicefs && sudo chmod +x /usr/local/bin/juicefs
if [[ -n "${{inputs.access_key}}" && -n "${{inputs.secret_key}}" && -n "${{inputs.token}}" ]]; then
sudo juicefs auth ci-coverage --access-key ${{ inputs.access_key }} --secret-key ${{ inputs.secret_key }} --token ${{inputs.token}} --encrypt-keys
sudo juicefs mount ci-coverage --subdir ${subdir} ${{inputs.mount_point}} --allow-other
else
echo "no jfs secrets provided, use local dir instead of jfs"
mkdir -p ${{inputs.mount_point}}
fi
================================================
FILE: .github/actions/upload-coverage/action.yml
================================================
name: 'upload_coverage_report'
description: 'upload coverage report of one job'
inputs:
UPLOAD_TOKEN:
description: 'upload token'
required: true
type: string
runs:
using: "composite"
steps:
- name: umount juicefs
shell: bash
run: |
sudo umount /tmp/jfs || true
sleep 3s
- name: get job id
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs)
job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}") | select(.status=="in_progress") | .id')
echo Job ID is: ${job_id}
echo "job_id=$job_id" >> $GITHUB_ENV
- name: generate mount coverage report
shell: bash
run: |
echo "generate coverage percentage report"
sudo go tool covdata percent -i=cover/ | sudo tee cover/percent.txt
echo "generate coverage text report"
sudo go tool covdata textfmt -i=cover/ -o cover/cover.txt
echo "generate coverage html report"
sudo go tool cover -html=cover/cover.txt -o cover/cover.html
[[ -z "${{inputs.UPLOAD_TOKEN}}" ]] && echo "no upload token, skip upload" && exit 0 || true
.github/scripts/upload_coverage_report.sh cover/cover.html juicefs_${{github.workflow}}_${{github.run_id}}_${job_id}.html ${{inputs.UPLOAD_TOKEN}}
================================================
FILE: .github/actions/upload-total-coverage/action.yml
================================================
name: 'upload_total_coverage_report'
description: 'upload total coverage report of all jobs in workflow'
inputs:
UPLOAD_TOKEN:
description: 'upload token'
required: true
type: string
runs:
using: "composite"
steps:
- name: generate total coverage report
shell: bash
run: |
echo "current dir is $(pwd)"
if [[ "${{github.event_name}}" == "schedule" ]]; then
coverdirs="cover,"
else
for dir in $(find cover -mindepth 1 -maxdepth 1 -type d -exec basename {} \;); do
coverdirs+="cover/$dir/,"
done
fi
coverdirs=${coverdirs%,}
echo coverdirs is $coverdirs
[[ -z "$coverdirs" ]] && echo -e "\e[31m no coverage dir found\e[0m" && exit 0
sudo go tool covdata percent -i=$coverdirs | sudo tee cover/cover.percent
echo "generated coverage percent report:" $(realpath cover/cover.percent)
sudo go tool covdata textfmt -o cover/cover.txt -i=$coverdirs
echo "generated coverage report in text format:" $(realpath cover/cover.txt)
sudo go tool cover -html=cover/cover.txt -o cover/cover.html
echo "generated coverage report in html format:" $(realpath cover/cover.html)
ls -l cover/cover*
- name: upload coverage report
shell: bash
run: |
[[ -z "${{inputs.UPLOAD_TOKEN}}" ]] && echo -e "\e[31m no upload token, skip upload \e[0m" && exit 0 || true
if [[ -f cover/cover.html ]]; then
.github/scripts/upload_coverage_report.sh cover/cover.html ${{github.workflow}}_${{github.run_id}}.html ${{inputs.UPLOAD_TOKEN}}
else
echo -e "\e[31m no coverage report found\e[0m" && exit 0
fi
================================================
FILE: .github/scripts/apt_install.sh
================================================
#!/bin/bash
set -e
# Set the maximum number of retries
MAX_RETRIES=3
# Define a function to run a command and check the return code
# The function takes two arguments: the command to run and a description of the command
function run_command() {
local cmd=$1
local retries=0
local retry_cmd="$cmd"
while true; do
# Run the command and capture the return code
$retry_cmd 2>&1 | tee /tmp/install.log || true
local ret=$?
# If the command succeeded, break out of the loop
if [[ $ret -eq 0 ]]; then
break
fi
# If the command failed and we have retries left, print a warning and retry
if [[ $retries -lt $MAX_RETRIES ]]; then
retries=$((retries + 1))
echo "WARNING: $cmd failed with return code $ret. Retrying ($retries/$MAX_RETRIES)..."
# If the error message indicates missing packages, retry with --fix-missing
if [[ $cmd == "apt-get update"* ]] && grep -q 'Failed to fetch' /tmp/install.log; then
retry_cmd="apt-get update -y --fix-missing"
elif [[ $cmd == "apt-get install"* ]] && grep -q 'Unable to fetch some archives' /tmp/install.log; then
retry_cmd="apt-get install -y --fix-missing $package_name"
fi
else
# If we've exhausted all retries, exit with an error
echo "ERROR: $cmd failed with return code $ret after $MAX_RETRIES retries."
exit 1
fi
done
}
# Run apt-get update and check the return code
run_command "apt-get update -y"
package_name=$@
# Run apt-get install and check the return code
run_command "apt-get install -y $package_name"
================================================
FILE: .github/scripts/cache.sh
================================================
#!/bin/bash -e
dpkg -s redis-server || .github/scripts/apt_install.sh redis-tools redis-server
dpkg -s fio || .github/scripts/apt_install.sh fio
source .github/scripts/common/common.sh
source .github/scripts/start_meta_engine.sh
[[ -z "$META" ]] && META=sqlite3
start_meta_engine $META minio
META_URL=$(get_meta_url $META)
if [[ "$META" == "sqlite3" ]]; then
META_URL="sqlite3:///tmp/test.db"
fi
test_warmup_in_background(){
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount $META_URL /tmp/jfs -d
dd if=/dev/zero of=/tmp/jfs/test bs=1M count=1024
./juicefs warmup /tmp/jfs/test --evict
./juicefs warmup /tmp/jfs/test --background
wait_warmup_finish /tmp/jfs/test 100
./juicefs warmup /tmp/jfs/test --background --evict
wait_warmup_finish /tmp/jfs/test 0
}
test_batch_warmup(){
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount $META_URL /tmp/jfs -d
rm -f file.list
file_count=11000
time seq 1 $file_count | xargs -P 8 -I {} sh -c 'echo {} > /tmp/jfs/test_{}; echo /tmp/jfs/test_{} >> file.list'
# time for i in $(seq 1 $file_count); do echo $i > /tmp/jfs/test_$i; echo /tmp/jfs/test_$i >> file.list; done
./juicefs warmup -f file.list 2>&1 | tee warmup.log
files=$(get_cache_file_count)
[[ $files -ne $file_count ]] && echo "warmup failed, expect $file_count files, actual $files" && exit 1 || true
./juicefs warmup -f file.list --check 2>&1 | tee warmup.log
files=$(get_cache_file_count)
[[ $files -ne $file_count ]] && echo "warmup failed, expect $file_count files, actual $files" && exit 1 || true
grep "(100.0%)" warmup.log || (echo "warmup failed, expect 100.0% warmup" && exit 1)
./juicefs warmup -f file.list --evict 2>&1 | tee warmup.log
files=$(get_cache_file_count)
[[ $files -ne $file_count ]] && echo "warmup evict failed, expect $file_count files, actual $files" && exit 1 || true
./juicefs warmup -f file.list --check 2>&1 | tee warmup.log
files=$(get_cache_file_count)
[[ $files -ne $file_count ]] && echo "warmup evict failed, expect $file_count files, actual $files" && exit 1 || true
grep "(0.0%)" warmup.log || (echo "warmup failed, expect 0.0% warmup" && exit 1)
./juicefs warmup /tmp/jfs/test* 2>&1 | tee warmup.log
files=$(get_cache_file_count)
[[ $files -ne $file_count ]] && echo "warmup failed, expect $file_count files, actual $files" && exit 1 || true
}
test_kernel_writeback_cache(){
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount $META_URL /tmp/jfs -d -o writeback_cache
mkdir /tmp/jfs/fio
runtime=15
cat /tmp/jfs/.stats | grep fuse | grep 'juicefs_fuse_written_size_bytes_sum\|juicefs_fuse_ops_total_write'
fio --name=seq_write_test --rw=write --bs=10 --size=4M --numjobs=8 --nrfiles=1 --runtime=$runtime --time_based --group_reporting --directory=/tmp/jfs/fio | tee fio.log
cat /tmp/jfs/.stats | grep fuse | grep 'juicefs_fuse_written_size_bytes_sum\|juicefs_fuse_ops_total_write'
bytes=$(cat /tmp/jfs/.stats | grep juicefs_fuse_written_size_bytes_sum | awk '{print $2}')
ops=$(cat /tmp/jfs/.stats | grep juicefs_fuse_ops_total_write | awk '{print $2}')
[[ $((bytes/ops)) -lt 10240 ]] && echo "writeback_cache may not enabled" && exit 1 || true
}
test_o_tmpfile(){
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount $META_URL /tmp/jfs -d -o writeback_cache
TEST_DIR="/tmp/jfs/tmp"
mkdir -p "$TEST_DIR"
cat > /tmp/test_otmp.c << 'EOF'
#define _GNU_SOURCE
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
int main() {
int fd = openat(AT_FDCWD, "/tmp/jfs/tmp", O_RDWR|O_EXCL|O_CLOEXEC|O_TMPFILE, 0600);
if (fd < 0) {
perror("openat");
return 1;
}
puts("openat ok");
if (write(fd, "x", 1) < 0) perror("write");
if (close(fd) < 0) {
printf("close: %s\n", strerror(errno));
return 1;
}
puts("close ok");
return 0;
}
EOF
gcc -o /tmp/test_otmp /tmp/test_otmp.c
/tmp/test_otmp
result=$?
if [ $result -ne 0 ]; then
echo "TEST FAILED: close fail"
exit 1
else
echo "TEST PASSED"
fi
}
test_cache_items(){
do_test_cache_items 2-random
}
test_cache_items_lru(){
do_test_cache_items lru
}
do_test_cache_items(){
cache_eviction=$1
prepare_test
./juicefs format $META_URL myjfs
cache_items=500
./juicefs mount $META_URL /tmp/jfs -d --cache-items $cache_items --cache-eviction $cache_eviction
seq 1 $((cache_items*2)) | xargs -P 8 -I {} sh -c 'echo {} > /tmp/jfs/test_{};'
./juicefs warmup /tmp/jfs/
./juicefs warmup /tmp/jfs/ --check 2>&1 | tee warmup.log
ratio=$(get_warmup_ratio)
[[ $ratio -lt 55 ]] || (echo "ratio should less than 55%" && exit 1)
}
test_evict_on_writeback(){
prepare_test
./juicefs format $META_URL myjfs --compress zstd
./juicefs mount $META_URL /tmp/jfs -d --writeback --upload-delay 3s
dd if=/dev/urandom of=/tmp/test bs=1M count=200
cp /tmp/test /tmp/jfs/test
sleep 3
stageBlocks=$(grep "juicefs_staging_blocks" /tmp/jfs/.stats | awk '{print $2}')
[[ $stageBlocks -eq 0 ]] && echo "stage blocks should not be 0" && exit 1 || true
./juicefs warmup /tmp/jfs/test --evict
wait_stage_uploaded
compare_md5sum /tmp/test /tmp/jfs/test
}
test_remount_on_writeback(){
prepare_test
./juicefs format $META_URL myjfs --compress lz4
./juicefs mount $META_URL /tmp/jfs -d --writeback --upload-delay 3s
dd if=/dev/urandom of=/tmp/test bs=1M count=200
cp /tmp/test /tmp/jfs/test
umount_jfs /tmp/jfs $META_URL
./juicefs mount $META_URL /tmp/jfs -d --writeback
sleep 3
stage_size=$(du -shm $(get_rawstaging_dir) | awk '{print $1}')
[[ $stage_size -gt 2 ]] && echo "stage size should not great than 2M" && exit 1 || true
./juicefs warmup /tmp/jfs/test --evict
compare_md5sum /tmp/test /tmp/jfs/test
}
test_memory_cache_none(){
do_test_memory_cache none
}
test_memory_cache_2_random(){
do_test_memory_cache 2-random
}
test_memory_cache_lru_fallback(){
prepare_test
./juicefs format $META_URL myjfs --compress lz4
./juicefs mount $META_URL /tmp/jfs -d --cache-dir memory --cache-size 100M --cache-eviction lru
eviction=$(get_cache_eviction)
[[ "$eviction" == "2-random" ]] || (echo "memory cache should fallback to 2-random, actual is $eviction" && exit 1)
dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200
./juicefs warmup /tmp/jfs/test
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
ratio=$(get_warmup_ratio)
[[ "$ratio" -gt 40 && "$ratio" -lt 60 ]] || (echo "ratio($ratio) should between 40% and 60% after lru fallback" && exit 1)
}
test_cache_eviction_invalid_fallback(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount $META_URL /tmp/jfs -d --cache-size 100M --cache-eviction invalid-policy
eviction=$(get_cache_eviction)
[[ "$eviction" == "2-random" ]] || (echo "invalid cache-eviction should fallback to 2-random, actual is $eviction" && exit 1)
}
do_test_memory_cache(){
cache_eviction=$1
prepare_test
./juicefs format $META_URL myjfs --compress lz4
./juicefs mount $META_URL /tmp/jfs -d --cache-dir memory --cache-size 100M --cache-eviction $cache_eviction
dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200
./juicefs warmup /tmp/jfs/test
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
ratio=$(get_warmup_ratio)
if [[ $cache_eviction == "2-random" ]]; then
[[ "$ratio" -gt 40 && "$ratio" -lt 60 ]] || (echo "ratio($ratio) should between 40% and 60%" && exit 1)
elif [[ $cache_eviction == "none" ]]; then
[[ "$ratio" -gt 40 && "$ratio" -lt 60 ]] || (echo "ratio($ratio) should between 40% and 60%" && exit 1)
fi
./juicefs warmup /tmp/jfs/test --evict
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
ratio=$(get_warmup_ratio)
[[ "$ratio" = 0 ]] || (echo "ratio($ratio) should less than 0" && exit 1)
}
test_cache_expired(){
do_test_cache_expired /var/jfsCache/myjfs 2-random
}
test_cache_expired_memory(){
do_test_cache_expired memory 2-random
}
test_cache_expired_lru(){
do_test_cache_expired /var/jfsCache/myjfs lru
}
do_test_cache_expired(){
cache_dir=$1
cache_eviction=$2
[[ -z $cache_eviction ]] && cache_eviction=2-random
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount $META_URL /tmp/jfs -d --cache-dir $cache_dir --cache-expire 3s --cache-eviction $cache_eviction
dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200
for i in $(seq 1 1100); do
dd if=/dev/zero of=/tmp/jfs/test$i bs=32k count=1 status=none
done
./juicefs warmup /tmp/jfs/ 2>&1 | tee warmup.log
sleep 15
./juicefs warmup /tmp/jfs/ --check 2>&1 | tee warmup.log
grep "(0.0%)" warmup.log || (echo "cache should expired" && exit 1)
}
test_cache_large_write(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount $META_URL /tmp/jfs -d -v
dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
ratio=$(get_warmup_ratio)
[[ "$ratio" = 0 ]] || (echo "ratio($ratio) should less than 0" && exit 1)
./juicefs mount $META_URL /tmp/jfs -d --cache-large-write
dd if=/dev/zero of=/tmp/jfs/test1 bs=1M count=200
./juicefs warmup /tmp/jfs/test1 --check 2>&1 | tee warmup.log
# TODO: should check the ratio
check_warmup_log 90
}
test_cache_mode(){
prepare_test
./juicefs format $META_URL myjfs
cache_mode=$(printf "%03o" $((RANDOM % 512)))
echo "cache mode is $cache_mode"
./juicefs mount $META_URL /tmp/jfs -d --cache-mode $cache_mode --writeback --upload-delay 3s
dd if=/dev/zero of=/tmp/jfs/test bs=1M count=32
./juicefs warmup /tmp/jfs/test
find $(get_raw_dir) -type f ! -perm $cache_mode -exec echo "perm of {} is incorrect" \; -exec false {} +
find $(get_rawstaging_dir) -type f ! -perm $cache_mode -exec echo "perm of {} is incorrect" \; -exec false {} +
sleep 5s
find $(get_raw_dir) -type f ! -perm $cache_mode -exec echo "perm of {} is incorrect" \; -exec false {} +
find $(get_rawstaging_dir) -type f ! -perm $cache_mode -exec echo "perm of {} is incorrect" \; -exec false {} +
}
test_cache_compressed(){
prepare_test
./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test \
--access-key minioadmin --secret-key minioadmin --compress lz4 --hash-prefix
./juicefs mount $META_URL /tmp/jfs -d
dd if=/dev/urandom of=/tmp/test bs=1M count=200
cp /tmp/test /tmp/jfs/test
./juicefs warmup /tmp/jfs/test --evict
./juicefs warmup /tmp/jfs/test
docker stop minio
compare_md5sum /tmp/test /tmp/jfs/test
docker start minio
}
test_cache_checksum_none(){
do_test_cache_checksum none
}
test_cache_checksum_full(){
do_test_cache_checksum full
}
test_cache_checksum_shrink(){
do_test_cache_checksum shrink
}
test_cache_checksum_extend(){
do_test_cache_checksum extend
}
do_test_cache_checksum(){
checksum_level=$1
prepare_test
./juicefs format $META_URL myjfs --compress lz4
./juicefs mount $META_URL /tmp/jfs -d --verify-cache-checksum $checksum_level
mkdir -p /tmp/jfs/rand-rw
fio --name=seq_rw --rw=readwrite --bsrange=1k-4k --size=80M --numjobs=4 --runtime=5 --time_based --group_reporting --filename=/tmp/jfs/req-rw
fio --name=rand_rw --rw=randrw --bsrange=1k-4k --size=80M --numjobs=4 --runtime=5 --time_based --group_reporting --directory=/tmp/jfs/rand-rw --nrfiles=1000 --filesize=4k
}
test_disk_full_2_random(){
do_test_disk_full 2-random
}
test_disk_full_lru(){
do_test_disk_full lru
}
test_disk_full_none(){
do_test_disk_full none
}
do_test_disk_full(){
cache_eviction=$1
prepare_test
mount_jfsCache1 1G
./juicefs format $META_URL myjfs
./juicefs mount $META_URL /tmp/jfs -d --cache-dir /var/jfsCache1 --cache-eviction $cache_eviction --free-space-ratio 0.2
dd if=/dev/zero of=/tmp/test bs=1M count=1200
cp /tmp/test /tmp/jfs/test
./juicefs warmup /tmp/jfs/test
sleep 3 # wait to free space
df -h /var/jfsCache1
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
used_percent=$(df /var/jfsCache1 | tail -1 | awk '{print $5}' | tr -d %)
echo "used percent is $used_percent"
if [[ $cache_eviction == "2-random" || $cache_eviction == "lru" ]]; then
[[ $used_percent -gt 80 ]] && echo "used percent($used_percent) should not more than 80%" && exit 1 || true
elif [[ $cache_eviction == "none" ]]; then
# cache will not evict even reach the free-space-ratio.
[[ $used_percent -lt 80 ]] && echo "used percent($used_percent) should not less than 80%" && exit 1 || true
fi
}
test_lru_hotset_prefer_recent(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount $META_URL /tmp/jfs -d --cache-size 128M --cache-items 40 --cache-eviction lru
mkdir -p /tmp/jfs/lru
for i in $(seq 1 80); do
dd if=/dev/zero of=/tmp/jfs/lru/f_$i bs=64k count=1 status=none
done
for i in $(seq 1 40); do
./juicefs warmup /tmp/jfs/lru/f_$i > /dev/null
done
sleep 2
for i in $(seq 1 10); do
cat /tmp/jfs/lru/f_$i > /dev/null
done
sleep 2
for i in $(seq 41 70); do
./juicefs warmup /tmp/jfs/lru/f_$i > /dev/null
done
rm -f hot.list cold.list
for i in $(seq 1 10); do
echo /tmp/jfs/lru/f_$i >> hot.list
done
for i in $(seq 11 40); do
echo /tmp/jfs/lru/f_$i >> cold.list
done
./juicefs warmup -f hot.list --check 2>&1 | tee warmup.log
hot_ratio=$(get_warmup_ratio)
[[ "$hot_ratio" -eq 100 ]] || (echo "hot set ratio($hot_ratio) should be 100% for lru" && exit 1)
./juicefs warmup -f cold.list --check 2>&1 | tee warmup.log
cold_ratio=$(get_warmup_ratio)
[[ "$cold_ratio" -lt 20 ]] || (echo "cold set ratio($cold_ratio) should be less than 20% for lru" && exit 1)
}
test_inode_full(){
prepare_test
mount_jfsCache1 100G 1000
./juicefs format $META_URL myjfs
./juicefs mount $META_URL /tmp/jfs -d --cache-dir /var/jfsCache1 --free-space-ratio 0.2
seq 1 1000 | xargs -P 8 -I {} sh -c 'echo {} > /tmp/jfs/test_{};'
./juicefs warmup /tmp/jfs/
./juicefs warmup /tmp/jfs/ --check 2>&1 | tee warmup.log
sleep 3
used_percent=$(df -i /var/jfsCache1 | tail -1 | awk '{print $5}' | tr -d %)
[[ $used_percent -gt 85 ]] && echo "used percent($used_percent) should less than 85%" && exit 1 || true
}
test_disk_full_with_writeback(){
prepare_test
mount_jfsCache1 1G
./juicefs format $META_URL myjfs --compress zstd
./juicefs mount $META_URL /tmp/jfs -d --cache-dir /var/jfsCache1 --writeback --free-space-ratio 0.2 --upload-delay 5s
dd if=/dev/urandom of=/tmp/test bs=1M count=1400
cp /tmp/test /tmp/jfs/test
wait_stage_uploaded
sleep 3
used_percent=$(df /var/jfsCache1 | tail -1 | awk '{print $5}' | tr -d %)
[[ $used_percent -gt 80 ]] && echo "used percent($used_percent) should less than 80%" && exit 1 || true
echo 3 > /proc/sys/vm/drop_caches
./juicefs warmup /tmp/jfs/test --evict
compare_md5sum /tmp/test /tmp/jfs/test
}
test_disk_failover()
{
prepare_test
mount_jfsCache1
rm -rf /var/log/juicefs.log
rm -rf /var/jfsCache2 /var/jfsCache3
./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
JFS_MAX_DURATION_TO_DOWN=10s JFS_MAX_IO_DURATION=3s ./juicefs mount $META_URL /tmp/jfs -d \
--cache-dir=/var/jfsCache1:/var/jfsCache2:/var/jfsCache3 --io-retries 1
dd if=/dev/urandom of=/tmp/test bs=1M count=1024
cp /tmp/test /tmp/jfs/test
/etc/init.d/redis-server stop
./juicefs warmup /tmp/jfs/test
./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log
check_warmup_log 50
wait_disk_down 60
./juicefs warmup /tmp/jfs/test
./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log
check_warmup_log 98
check_cache_distribute 1024 /var/jfsCache2 /var/jfsCache3
echo stop minio && docker stop minio
compare_md5sum /tmp/test /tmp/jfs/test
docker start minio && sleep 3
}
test_disk_failover_lru()
{
prepare_test
mount_jfsCache1
rm -rf /var/log/juicefs.log
rm -rf /var/jfsCache2 /var/jfsCache3
./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
JFS_MAX_DURATION_TO_DOWN=10s JFS_MAX_IO_DURATION=3s ./juicefs mount $META_URL /tmp/jfs -d \
--cache-dir=/var/jfsCache1:/var/jfsCache2:/var/jfsCache3 --io-retries 1 --cache-eviction lru
dd if=/dev/urandom of=/tmp/test bs=1M count=1024
cp /tmp/test /tmp/jfs/test
/etc/init.d/redis-server stop
./juicefs warmup /tmp/jfs/test
./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log
check_warmup_log 50
wait_disk_down 60
./juicefs warmup /tmp/jfs/test
./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log
check_warmup_log 98
check_cache_distribute 1024 /var/jfsCache2 /var/jfsCache3
echo stop minio && docker stop minio
compare_md5sum /tmp/test /tmp/jfs/test
docker start minio && sleep 3
}
test_manual_delete_cache_data_lru()
{
prepare_test
./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
./juicefs mount $META_URL /tmp/jfs -d --cache-eviction lru --cache-size 1G --cache-scan-interval -1
dd if=/dev/urandom of=/tmp/test bs=1M count=256
cp /tmp/test /tmp/jfs/test
./juicefs warmup /tmp/jfs/test
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
check_warmup_log 95
raw_dir=$(get_raw_dir)
find "$raw_dir" -type f | head -n 200 | xargs rm -f
sync
echo 3 > /proc/sys/vm/drop_caches || true
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
ratio=$(get_warmup_ratio)
[[ "$ratio" -lt 90 ]] || (echo "after manually deleting cache data, warmup ratio($ratio) should be less than 90%" && exit 1)
compare_md5sum /tmp/test /tmp/jfs/test
./juicefs warmup /tmp/jfs/test
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
check_warmup_log 95
}
test_disk_failure_on_writeback()
{
prepare_test
mount_jfsCache1
rm -rf /var/log/juicefs.log
rm -rf /var/jfsCache2 /var/jfsCache3
mkdir -p /var/jfsCache2 /var/jfsCache3
./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
JFS_MAX_DURATION_TO_DOWN=5s JFS_MAX_IO_DURATION=3s ./juicefs mount $META_URL /tmp/jfs -d \
--cache-dir=/var/jfsCache? --io-retries 1 --writeback -v
dd if=/dev/urandom of=/tmp/test bs=1M count=1024
cp /tmp/test /tmp/jfs/test
dd if=/dev/urandom of=/tmp/jfs/test2 bs=1M count=10
/etc/init.d/redis-server stop
./juicefs warmup /tmp/jfs/test2 &
sleep 15
grep -q "state change from unstable to down" /var/log/juicefs.log && echo "disk should not down" && exit 1 || true
/etc/init.d/redis-server start
./juicefs warmup /tmp/jfs/test
./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log
# TODO: the ratio should be 100%
check_warmup_log 60
check_cache_distribute 1024 /var/jfsCache1 /var/jfsCache2 /var/jfsCache3
compare_md5sum /tmp/test /tmp/jfs/test
}
prepare_test()
{
df -h /
umount_jfs /tmp/jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
rm -rf /var/jfs/myjfs || true
rm -rf /var/jfsCache/myjfs || true
[[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
mc alias set myminio http://localhost:9000 minioadmin minioadmin
mc rm --force --recursive myminio/test || true
}
wait_warmup_finish(){
path=$1
expected_ratio=$2
timeout=30
for i in $(seq 1 $timeout); do
./juicefs warmup $path --check 2>&1 |tee warmup.log
ratio=$(get_warmup_ratio)
if [[ "$ratio" == "$expected_ratio" ]]; then
echo "warmup finished after $i seconds, ratio is $ratio, expected ratio is $expected_ratio"
break
else
echo "wait warmup finish $i"
sleep 1
fi
if [[ $i -eq $timeout ]]; then
echo "wait warmup finish timeout after $timeout seconds" && exit 1
fi
done
}
wait_stage_uploaded()
{
echo "wait stage upload"
for i in {1..30}; do
stageBlocks=$(grep "juicefs_staging_blocks" /tmp/jfs/.stats | awk '{print $2}')
if [[ "$stageBlocks" -eq 0 ]]; then
echo "stageBlocks is now 0"
break
fi
echo "wait stage upload $i" && sleep 1
done
if [[ "$stageBlocks" -ne 0 ]]; then
echo "stage blocks have not uploaded: $stageBlocks" && exit 1
fi
}
mount_jfsCache1(){
capacity=$1
[[ -z $capacity ]] && capacity=100G
inodes=$2
[[ -z $inodes ]] && inodes=10000000
/etc/init.d/redis-server start
timeout 30s bash -c 'until nc -zv localhost 6379; do sleep 1; done'
umount -l /var/jfsCache1 || true
rm -rf /var/jfsCache1
redis-cli flushall
rm -rf /var/jfs/test
./juicefs format "redis://localhost/1?read-timeout=3&write-timeout=1&max-retry-backoff=3" test --trash-days 0 --capacity $capacity --inodes $inodes
./juicefs mount redis://localhost/1 /var/jfsCache1 -d --log /tmp/juicefs.log
# trap "echo umount /var/jfsCache1 && umount -l /var/jfsCache1" EXIT
}
get_cache_dir(){
grep CacheDir /tmp/jfs/.config | awk -F'"' '{print $4}'
}
get_cache_eviction(){
grep CacheEviction /tmp/jfs/.config | awk -F'"' '{print $4}'
}
get_raw_dir(){
echo $(get_cache_dir)/raw/
}
get_rawstaging_dir(){
echo $(get_cache_dir)/rawstaging/
}
check_evict_log(){
ratio=$(get_warmup_ratio)
if [[ "$ratio" -gt 0 ]]; then
echo "cache ratio($ratio) should be 0 after evict"
exit 1
fi
}
check_warmup_log(){
expected_ratio=$1
ratio=$(get_warmup_ratio)
if [[ "$ratio" -lt "$expected_ratio" ]]; then
echo "cache ratio($ratio) should be more than expected_ratio($expected_ratio) after warmup"
exit 1
fi
}
get_cache_file_count(){
sed -n 's/.* \([0-9]\+\) files.*/\1/p' warmup.log
}
get_cache_file_size(){
sed -n 's/.* \([0-9]*\) MiB of.*/\1/p' warmup.log
}
get_warmup_ratio(){
sed -n 's/.*(\([0-9]*\.[0-9]*%\)).*/\1/p' warmup.log | sed 's/%//' | awk '{print int($1)}'
}
check_cache_distribute() {
max_total_size=$(echo "$1 * 1024" | bc | awk '{printf "%.0f", $1}')
echo check_cache_distribute, max_total_size is $max_total_size
shift
total_weight=0
declare -A weights
declare -A sizes
# Parse directory names and weights
for arg in "$@"; do
dir=$(echo "$arg" | awk -F: '{print $1}')
weight=$(echo "$arg" | awk -F: '{print $2}')
if [[ -z $weight ]]; then
weight=1
fi
weights["$dir"]=$weight
total_weight=$((total_weight + weight))
done
# Calculate total size and sizes of each directory
for dir in "${!weights[@]}"; do
echo dir is $dir
du -sh "$dir" || true
size=$(du -s "$dir" | awk '{print $1}')
echo size is $size
sizes["$dir"]=$size
done
# Check if total size exceeds max limit
total_size=0
for dir in "${!sizes[@]}"; do
size=${sizes["$dir"]}
total_size=$((total_size + size))
done
echo "total size is $total_size, max_total_size is $max_total_size"
if [[ $total_size -gt $((max_total_size + max_total_size/10)) ]]; then
echo "Total size of directories exceeds max limit"
return 1
fi
# Check if each directory is evenly distributed based on its weight
for dir in "${!sizes[@]}"; do
size=${sizes["$dir"]}
weight=${weights["$dir"]}
avg_size=$((total_size * weight / total_weight))
min_size=$((avg_size * 5 / 10))
max_size=$((avg_size * 20 / 10))
if [[ $size -lt $min_size || $size -gt $max_size ]]; then
echo "$dir is not evenly distributed, size: $size, weight: $weight, ave_size: $avg_size, min_size: $min_size, max_size: $max_size"
exit 1
else
echo "$dir is evenly distributed"
fi
done
}
wait_disk_down()
{
timeout=$1
for i in $(seq 1 $timeout); do
if grep -q "state change from unstable to down" /var/log/juicefs.log; then
echo "state changed from unstable to down after $i seconds"
return
else
echo "\rWait for state change to down, $i"
sleep 1
count=$((count+1))
fi
done
echo "Wait for state change to down timeout after $timeout seconds" && exit 1
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/chaos/dynamic.yaml
================================================
apiVersion: apps/v1
kind: Deployment
metadata:
name: dynamic-ce
labels:
juicefs-app-type: dynamic-ce
spec:
replicas: 1
selector:
matchLabels:
juicefs-app-type: dynamic-ce
template:
metadata:
labels:
juicefs-app-type: dynamic-ce
spec:
containers:
- name: vdbench
image: zwwhdlsdocker/vdbench:latest
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /data
name: data
- mountPath: /vdbench/config
name: vdbench-cfg
- mountPath: /vdbench/output
name: output
command: ["sh", "-c", "./vdbench -f /vdbench/config/vdbench.vdb -v"]
volumes:
- name: data
persistentVolumeClaim:
claimName: dynamic-ce
- name: output
hostPath:
path: /root/vdbench/output
- name: vdbench-cfg
configMap:
name: dynamic-ce
items:
- key: "vdbench.vdb"
path: "vdbench.vdb"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: dynamic-ce
data:
vdbench.vdb: |
messagescan=no
fsd=fsd1,anchor=/data,depth=1,width=1,files=20000,size=4k,openflags=o_direct
fwd=fwd1,fsd=fsd1,operation=write,xfersize=4k,fileio=random,fileselect=random,threads=1
rd=rd1,fwd=fwd1,fwdrate=max,format=yes,elapsed=60,interval=2
================================================
FILE: .github/scripts/chaos/juicefs-csi-driver.Dockerfile
================================================
FROM golang:1.20-buster as builder
ARG GOPROXY
# refs/remotes/pull/3056/merge
ARG GITHUB_REF
# 4ac69613b5919142d87f21a64ca744ae537192d6
ARG GITHUB_SHA
ARG JUICEFS_REPO_URL=https://github.com/juicedata/juicefs
WORKDIR /workspace
ENV GOPROXY=${GOPROXY:-https://proxy.golang.org}
ENV STATIC=1
RUN apt-get update && apt-get install -y musl-tools upx-ucl && \
cd /workspace && git clone --depth=1 $JUICEFS_REPO_URL && \
cd juicefs && git fetch --no-tags --prune origin +$GITHUB_SHA:$GITHUB_REF && \
git checkout $GITHUB_REF && \
make juicefs
FROM juicedata/juicefs-csi-driver:nightly
WORKDIR /app
COPY --from=builder /workspace/juicefs/juicefs /usr/local/bin/
RUN ls -l /usr/local/bin/juicefs
RUN /usr/local/bin/juicefs --version
RUN echo GITHUB_REF is $GITHUB_REF
RUN echo GITHUB_SHA is $GITHUB_SHA
# ENTRYPOINT ["/tini", "--", "/bin/juicefs-csi-driver"]
================================================
FILE: .github/scripts/chaos/juicefs.Dockerfile
================================================
FROM juicedata/mount:nightly
COPY ./juicefs /usr/local/bin/juicefs
# RUN apt-get update && apt-get install -y musl-tools upx-ucl && STATIC=1 make
# RUN cp -f juicefs /usr/local/bin/juicefs
RUN /usr/local/bin/juicefs version
================================================
FILE: .github/scripts/chaos/minio.yaml
================================================
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: minio-server
namespace: kube-system
labels:
app: minio-server
spec:
replicas: 1
selector:
matchLabels:
app: minio-server
serviceName: minio
template:
metadata:
labels:
app: minio-server
spec:
containers:
- name: minio
image: minio/minio
resources:
limits:
memory: "500Mi"
cpu: "500m"
limits:
memory: "100Mi"
cpu: "100m"
env:
- name: MINIO_ROOT_USER
value: minioadmin
- name: MINIO_ROOT_PASSWORD
value: minioadmin
args:
- server
- /data
volumeMounts:
- mountPath: /data
name: minio-data
ports:
- containerPort: 9000
name: sever
volumes:
- name: minio-data
hostPath:
path: /data/minio-data
---
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: kube-system
spec:
type: NodePort
selector:
app: minio-server
ports:
- protocol: TCP
port: 9000
targetPort: 9000
nodePort: 31275
name: server
================================================
FILE: .github/scripts/chaos/pvc.yaml
================================================
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: dynamic-ce
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Pi
storageClassName: dynamic-ce
================================================
FILE: .github/scripts/chaos/redis.yaml
================================================
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis-server
namespace: kube-system
labels:
app: redis-server
spec:
replicas: 1
selector:
matchLabels:
app: redis-server
serviceName: redis
template:
metadata:
labels:
app: redis-server
spec:
containers:
- name: redis
image: redis
volumeMounts:
- mountPath: /data
name: redis-data
resources:
limits:
memory: "500Mi"
cpu: "500m"
limits:
memory: "100Mi"
cpu: "100m"
ports:
- containerPort: 6379
volumes:
- name: redis-data
hostPath:
path: /data/redis
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: kube-system
spec:
type: NodePort
selector:
app: redis-server
ports:
- protocol: TCP
port: 6379
targetPort: 6379
nodePort: 31274
================================================
FILE: .github/scripts/chaos/sc.yaml
================================================
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: dynamic-ce
parameters:
csi.storage.k8s.io/node-publish-secret-name: dynamic-ce
csi.storage.k8s.io/node-publish-secret-namespace: kube-system
csi.storage.k8s.io/provisioner-secret-name: dynamic-ce
csi.storage.k8s.io/provisioner-secret-namespace: kube-system
juicefs/mount-cpu-limit: 5000m
juicefs/mount-memory-limit: 1Gi
juicefs/mount-cpu-request: 100m
juicefs/mount-memory-request: 500Mi
juicefs/mount-image: juicedata/mount:ci
#mountOptions:
# - cache-dir=/var/foo:/var/foo1:/var/foo2
provisioner: csi.juicefs.com
reclaimPolicy: Delete
volumeBindingMode: Immediate
---
apiVersion: v1
stringData:
access-key: minioadmin
bucket: http://minio.kube-system:9000/minio/dynamic-ce
name: dynamic-ce
metaurl: redis://redis.kube-system:6379/0
secret-key: minioadmin
storage: minio
format-options: trash-days=0,block-size=4096
kind: Secret
metadata:
name: dynamic-ce
namespace: kube-system
type: Opaque
================================================
FILE: .github/scripts/chaos/workflow.yaml
================================================
apiVersion: chaos-mesh.org/v1alpha1
kind: Workflow
metadata:
name: juicefs-workflow
spec:
entry: the-entry
templates:
- name: the-entry
templateType: Parallel
children:
# - minio-delay
# - minio-io
# - minio-memory
# - minio-cpu
# - minio-bandwidth
# - redis-bandwidth
# - redis-io
# - redis-delay
# - redis-memory
# - redis-cpu
# - juicefs-bandwidth
# - juicefs-memory
# - juicefs-cpu
# - juicefs-delay
# minio 带宽
- name: minio-bandwidth
templateType: NetworkChaos
deadline: 20s
networkChaos:
action: bandwidth
mode: all
selector:
namespaces:
- kube-system
labelSelectors:
app: minio-server
bandwidth:
rate: '500bps'
limit: 100
buffer: 10000
# minio 网络延迟
- name: minio-delay
templateType: NetworkChaos
networkChaos:
action: delay
mode: all
selector:
namespaces:
- kube-system
labelSelectors:
app: minio-server
delay:
latency: '500ms'
correlation: '50'
jitter: '500ms'
# minio 磁盘读写延迟
- name: minio-io
templateType: IOChaos
ioChaos:
action: latency
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app: minio-server
volumePath: /data
delay: '50ms'
# minio 内存压力
- name: minio-memory
templateType: StressChaos
stressChaos:
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app: minio-server
stressors:
memory:
workers: 4
size: '128MB'
# minio cpu 压力
- name: minio-cpu
templateType: StressChaos
stressChaos:
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app: minio-server
stressors:
cpu:
workers: 4
load: 100
# redis 带宽
- name: redis-bandwidth
templateType: NetworkChaos
networkChaos:
action: bandwidth
mode: all
selector:
namespaces:
- kube-system
labelSelectors:
app: redis-server
bandwidth:
rate: '200mbps'
limit: 100
buffer: 10000
- name: redis-delay
templateType: NetworkChaos
networkChaos:
action: delay
mode: all
selector:
namespaces:
- kube-system
labelSelectors:
app: redis-server
delay:
latency: '100ms'
correlation: '50'
jitter: '500ms'
# redis 磁盘读写延迟
- name: redis-io
templateType: IOChaos
ioChaos:
action: latency
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app: redis-server
volumePath: /redis
delay: '1s'
# redis 内存压力
- name: redis-memory
templateType: StressChaos
stressChaos:
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app: redis-server
stressors:
memory:
workers: 4
size: '2GB'
# redis cpu 压力
- name: redis-cpu
templateType: StressChaos
stressChaos:
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app: redis-server
stressors:
cpu:
workers: 4
load: 100
# 客户端带宽
- name: juicefs-bandwidth
templateType: NetworkChaos
deadline: 20s
networkChaos:
action: bandwidth
mode: all
selector:
namespaces:
- kube-system
labelSelectors:
app.kubernetes.io/name: juicefs-mount
bandwidth:
rate: '100bps'
limit: 100
buffer: 10000
- name: juicefs-delay
templateType: NetworkChaos
networkChaos:
action: delay
mode: all
selector:
namespaces:
- kube-system
labelSelectors:
app.kubernetes.io/name: juicefs-mount
delay:
latency: '100ms'
correlation: '50'
jitter: '500ms'
# 客户端内存压力
- name: juicefs-memory
templateType: StressChaos
stressChaos:
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app.kubernetes.io/name: juicefs-mount
stressors:
memory:
workers: 4
size: '1GB'
# 客户端cpu压力
- name: juicefs-cpu
templateType: StressChaos
stressChaos:
mode: one
selector:
namespaces:
- kube-system
labelSelectors:
app.kubernetes.io/name: juicefs-mount
stressors:
cpu:
workers: 4
load: 100
================================================
FILE: .github/scripts/check_juicefs_log.sh
================================================
#!/bin/bash -e
for log_file in /var/log/juicefs.log $HOME/.juicefs/juicefs.log; do
if [ -f $log_file ]; then
break
fi
done
echo "tail -1000 $log_file"
tail -1000 $log_file
grep -i "<FATAL>\|panic" $log_file && exit 1 || true
================================================
FILE: .github/scripts/cmptree.py
================================================
#!/usr/bin/env python
# Copyright (c) 2015, Bill Zissimopoulos. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import subprocess
try:
__import__("xattr")
except ImportError:
subprocess.check_call(["pip", "install", "xattr"])
import filecmp, os
import xattr
class TreeComparator(object):
def __init__(self, dir1, dir2):
self.dir1 = dir1
self.dir2 = dir2
self.left_only = []
self.right_only = []
self.common_funny = []
self.funny_files = []
self.diff_files = []
def compare(self, p=""):
d1 = os.path.join(self.dir1, p)
d2 = os.path.join(self.dir2, p)
print(f'compare {d1} with {d2}')
dcmp = filecmp.dircmp(d1, d2, ignore=[])
self.left_only.extend(os.path.join(p, n) for n in dcmp.left_only)
self.right_only.extend(os.path.join(p, n) for n in dcmp.right_only)
self.common_funny.extend(os.path.join(p, n) for n in dcmp.common_funny)
self.funny_files.extend(os.path.join(p, n) for n in dcmp.funny_files)
#(match, mismatch, errors) = filecmp.cmpfiles(d1, d2, dcmp.common_files, shallow=False)
#self.diff_files.extend(os.path.join(p, n) for n in mismatch)
#self.funny_files.extend(os.path.join(p, n) for n in errors)
(match, mismatch, errors) = self.compare_files(d1, d2, dcmp.common_files)
self.diff_files.extend(os.path.join(p, n) for n in mismatch)
self.funny_files.extend(os.path.join(p, n) for n in errors)
for d in dcmp.common_dirs:
self.compare(os.path.join(p, d))
def compare_files(self, d1, d2, files):
match = []
mismatch = []
errors = []
for f in files:
f1 = os.path.join(d1, f)
f2 = os.path.join(d2, f)
try:
s1 = os.stat(f1)
s2 = os.stat(f2)
for attr in ['st_mode', 'st_nlink', 'st_uid', 'st_gid', 'st_size']:
if getattr(s1, attr) != getattr(s2, attr):
print(f'{attr} mismatch with {f1}:{getattr(s1, attr)} and {f2}:{getattr(s2, attr)}')
mismatch.append(f)
continue
if not filecmp.cmp(f1, f2):
print(f'content mismatch with {f1} and {f2}')
mismatch.append(f)
continue
if not self.compare_xattr(f1, f2):
print(f'xattr mismatch with {f1} and {f2}')
mismatch.append(f)
continue
match.append(f)
except:
print(f'error: {f}')
errors.append(f)
return match, mismatch, errors
def compare_xattr(self, f1, f2):
for attr in xattr.listxattr(f1):
a1 = xattr.getxattr(f1, attr)
a2 = xattr.getxattr(f2, attr)
if a1 != a2:
return False
return True
if "__main__" == __name__:
import argparse, sys
def info(s):
print ("%s: %s" % (os.path.basename(sys.argv[0]), s))
def warn(s):
print ("%s: %s" % (os.path.basename(sys.argv[0]), s))
def fail(s, exitcode = 1):
warn(s)
sys.exit(exitcode)
def main():
p = argparse.ArgumentParser()
p.add_argument("-q", "--quiet", action="store_true")
p.add_argument("dir1")
p.add_argument("dir2")
args = p.parse_args(sys.argv[1:])
print('start compare tree')
tcmp = TreeComparator(args.dir1, args.dir2)
tcmp.compare()
res = len(tcmp.left_only) + len(tcmp.right_only) + \
len(tcmp.funny_files) + len(tcmp.diff_files)
# res = len(tcmp.left_only) + len(tcmp.right_only) + \
# len(tcmp.common_funny) + len(tcmp.funny_files) + len(tcmp.diff_files)
if not args.quiet:
if tcmp.left_only:
print ("Left only:")
for n in tcmp.left_only:
print( " %s" % n)
if tcmp.right_only:
print ("Right only:")
for n in tcmp.right_only:
print( " %s" % n)
if tcmp.funny_files:
print ("Funny files:")
for n in tcmp.funny_files:
print( " %s" % n)
# if tcmp.common_funny:
# print ("Differing stats:")
# for n in tcmp.common_funny:
# print (" %s" % n)
if tcmp.diff_files:
print ("Differing files:")
for n in tcmp.diff_files:
print (" %s" % n)
sys.exit(int(0 < res))
def __entry():
try:
main()
except EnvironmentError as ex:
fail(ex)
except KeyboardInterrupt:
fail("interrupted", 130)
__entry()
================================================
FILE: .github/scripts/command/acl.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
prepare_test()
{
umount_jfs /tmp/jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
rm -rf /var/jfs/myjfs || true
rm -rf /var/jfsCache/myjfs || true
}
test_acl_with_kernel_check()
{
prepare_test
./juicefs format $META_URL myjfs --enable-acl --trash-days 0
./juicefs mount -d $META_URL /tmp/jfs
python3 .github/scripts/hypo/fs_acl_test.py
}
test_acl_with_user_space_check()
{
prepare_test
./juicefs format $META_URL myjfs --enable-acl --trash-days 0
./juicefs mount -d $META_URL /tmp/jfs --non-default-permission
python3 .github/scripts/hypo/fs_acl_test.py
}
test_modify_acl_config()
{
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount -d $META_URL /tmp/jfs
touch /tmp/jfs/test
setfacl -m u:root:rw /tmp/jfs/test && echo "setfacl should failed" && exit 1
./juicefs config $META_URL --enable-acl=true
./juicefs mount -d $META_URL /tmp/jfs
setfacl -m u:root:rw /tmp/jfs/test
./juicefs config $META_URL --enable-acl
umount_jfs /tmp/jfs $META_URL
./juicefs mount -d $META_URL /tmp/jfs
setfacl -m u:root:rw /tmp/jfs/test
./juicefs config $META_URL --enable-acl=false && echo "should not disable acl" && exit 1 || true
./juicefs config $META_URL | grep EnableACL | grep "true" || (echo "EnableACL should be true" && exit 1)
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/clone.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
test_clone_preserve_with_file()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
id -u juicefs && sudo userdel juicefs
sudo useradd -u 1101 juicefs
sudo -u juicefs touch /jfs/test
for mode in 777 755 644; do
sudo -u juicefs chmod $mode /jfs/test
check_guid_after_clone true
check_guid_after_clone false
done
}
test_clone_preserve_with_dir()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
id -u juicefs && sudo userdel juicefs
sudo useradd -u 1101 juicefs
sudo -u juicefs mkdir /jfs/test
for mode in 777 755 644; do
sudo -u juicefs chmod $mode /jfs/test
check_guid_after_clone true
check_guid_after_clone false
done
}
test_clone_with_jfs_source()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
[[ ! -d /jfs/juicefs ]] && git clone https://github.com/juicedata/juicefs.git /jfs/juicefs --depth 1
do_clone true
do_clone false
}
skip_test_clone_with_fsrand()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
seed=$(date +%s)
python3 .github/scripts/fsrand.py -a -c 2000 -s $seed /jfs/juicefs
do_clone true
do_clone false
}
do_clone()
{
is_preserve=$1
rm -rf /jfs/juicefs1
rm -rf /jfs/juicefs2
[[ "$is_preserve" == "true" ]] && preserve="--preserve" || preserve=""
cp -r /jfs/juicefs /jfs/juicefs1 $preserve
./juicefs clone /jfs/juicefs /jfs/juicefs2 $preserve
diff -ur /jfs/juicefs1 /jfs/juicefs2 --no-dereference
cd /jfs/juicefs1/ && find . -printf "%m\t%u\t%g\t%p\n" | sort -k4 >/tmp/log1 && cd -
cd /jfs/juicefs2/ && find . -printf "%m\t%u\t%g\t%p\n" | sort -k4 >/tmp/log2 && cd -
diff -u /tmp/log1 /tmp/log2
}
test_clone_with_big_file()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
dd if=/dev/urandom of=/tmp/test bs=1M count=1000
cp /tmp/test /jfs/test
./juicefs clone /jfs/test /jfs/test1
rm /jfs/test -rf
diff /tmp/test /jfs/test1
}
test_clone_with_big_file2()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
dd if=/dev/urandom of=/tmp/test bs=1M count=1000
echo "a" | tee -a /tmp/test
cp /tmp/test /jfs/test
./juicefs clone /jfs/test /jfs/test1
rm /jfs/test -rf
diff /tmp/test /jfs/test1
}
test_clone_with_random_write(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
PATH1=/tmp/test PATH2=/jfs/test python3 .github/scripts/random_read_write.py
./juicefs clone /jfs/test /jfs/test1
rm /jfs/test -rf
diff /tmp/test /jfs/test1
}
test_clone_with_sparse_file()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
fallocate -l 1.0001g /jfs/test
./juicefs clone /jfs/test /jfs/test1
diff /jfs/test /jfs/test1
}
test_clone_with_sparse_file2()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
fallocate -l 1.1T /jfs/test
./juicefs clone /jfs/test /jfs/test1
}
test_clone_with_small_files(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
mkdir /jfs/test
for i in $(seq 1 2000); do
echo $i > /jfs/test/$i
done
./juicefs clone /jfs/test /jfs/test1
diff -ur /jfs/test1 /jfs/test1
}
skip_test_clone_with_mdtest1()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
./juicefs mdtest $META_URL /test --depth 2 --dirs 10 --files 10 --threads 100 --write 8192
./juicefs clone /jfs/test /jfs/test1
./juicefs rmr /jfs/test
./juicefs rmr /jfs/test1
}
skip_test_clone_with_mdtest2()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
./juicefs mdtest $META_URL /test --depth 1 --dirs 1 --files 1000 --threads 100 --write 8192
./juicefs clone /jfs/test /jfs/test1
./juicefs rmr /jfs/test
./juicefs rmr /jfs/test1
}
check_guid_after_clone(){
is_preserve=$1
echo "check_guid_after_clone, is_preserve: $is_preserve"
[[ "$is_preserve" == "true" ]] && preserve="--preserve" || preserve=""
rm /jfs/test1 -rf
sleep 3
ls /jfs/test1 && echo "test1 should not exist" && exit 1 || echo "/jfs/test1 not exist"
rm /jfs/test2 -rf
./juicefs clone /jfs/test /jfs/test1 $preserve
cp /jfs/test /jfs/test2 -rf $preserve
uid1=$(stat -c %u /jfs/test1)
gid1=$(stat -c %g /jfs/test1)
mode1=$(stat -c %a /jfs/test1)
uid2=$(stat -c %u /jfs/test2)
gid2=$(stat -c %g /jfs/test2)
mode2=$(stat -c %a /jfs/test2)
if [[ "$uid1" != "$uid2" ]] || [[ "$gid1" != "$gid2" ]] || [[ "$mode1" != "$mode2" ]]; then
echo >&2 "<FATAL>: clone does not same as cp: uid1: $uid1, uid2: $uid2, gid1: $gid1, gid2: $gid2, mode1: $mode1, mode2: $mode2"
exit 1
fi
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/config.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META minio
META_URL=$(get_meta_url $META)
# version lower than 1.3.0 does not support parameter max_open_conns
if [[ $META_URL == *"?max_open_conns="* ]]; then
META_URL=${META_URL%%\?*}
fi
LEGACY_META_URL=$META_URL
if [[ "$META" == "redis" ]]; then
LEGACY_META_URL=${META_URL%%\?*}
fi
[ ! -x mc ] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x mc
download_juicefs_client(){
version=$1
wget -q https://github.com/juicedata/juicefs/releases/download/v$version/juicefs-$version-linux-amd64.tar.gz
tar -xzf juicefs-$version-linux-amd64.tar.gz -C /tmp/
sudo cp /tmp/juicefs juicefs-$version
./juicefs-$version version
}
test_config_min_client_version()
{
prepare_test
download_juicefs_client 1.0.0
./juicefs format $META_URL myjfs
./juicefs-1.0.0 mount $LEGACY_META_URL /jfs -d && exit 1 || true
./juicefs config $META_URL --min-client-version 1.0.1
./juicefs-1.0.0 mount $LEGACY_META_URL /jfs -d && exit 1 || true
./juicefs config $META_URL --min-client-version 1.0.0
./juicefs-1.0.0 mount $LEGACY_META_URL /jfs -d
}
test_config_max_client_version()
{
prepare_test
current_version=$(./juicefs version | awk '{print $3}')
download_juicefs_client 1.0.0
./juicefs-1.0.0 format $LEGACY_META_URL myjfs
./juicefs-1.0.0 config $LEGACY_META_URL --max-client-version 1.0.1
./juicefs mount $META_URL /jfs -d && exit 1 || true
./juicefs config $META_URL --max-client-version $current_version
./juicefs mount $META_URL /jfs -d
}
test_config_secret_key(){
# # Consider command as failed when any component of the pipe fails:
# https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
prepare_test
set -o pipefail
./mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin
./mc admin user add minio juicedata juicedata
./mc admin policy attach minio consoleAdmin --user juicedata
./juicefs format --storage minio --bucket http://localhost:9000/jfs-test --access-key juicedata --secret-key juicedata $META_URL myjfs
./juicefs mount $META_URL /jfs -d --io-retries 1 --no-usage-report --heartbeat 3
./mc admin user remove minio juicedata
./mc admin user add minio juicedata1 juicedata1
./mc admin policy attach minio consoleAdmin --user juicedata1
./juicefs config $META_URL --access-key juicedata1 --secret-key juicedata1
sleep 6
echo abc | tee /jfs/abc.txt && echo "write success"
cat /jfs/abc.txt | grep abc && echo "read success"
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/debug.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
check_debug_file(){
files=("system-info.log" "juicefs.log" "config.txt" "stats.txt" "stats.5s.txt" "pprof")
debug_dir="debug"
if [ ! -d "$debug_dir" ]; then
echo "error:no debug dir"
exit 1
fi
all_files_exist=true
for file in "${files[@]}"; do
exist=`find "$debug_dir" -name $file | wc -l`
if [ "$exist" == 0 ]; then
echo "no $file"
all_files_exist=false
fi
done
if [ "$all_files_exist" = true ]; then
echo "pass"
else
exit 1
fi
}
test_debug_juicefs(){
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
dd if=/dev/urandom of=/jfs/bigfile bs=1M count=128
./juicefs debug /jfs/
check_debug_file
./juicefs rmr /jfs/bigfile
}
test_debug_abnormal_juicefs(){
rm -rf debug | true
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
dd if=/dev/urandom of=/jfs/bigfile bs=1M count=128
killall -9 redis-server | true
./juicefs debug /jfs/
# check_debug_file
./juicefs rmr /jfs/bigfile
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/dump_load.sh
================================================
#!/bin/bash -ex
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
META_URL2=$(get_meta_url2 $META)
[[ -z "$SEED" ]] && SEED=$(date +%s)
HEARTBEAT_INTERVAL=2
DIR_QUOTA_FLUSH_INTERVAL=4
# [[ -z "$SEED" ]] && SEED=1711594639
[[ -z "$BINARY" ]] && BINARY=false
[[ -z "$FAST" ]] && FAST=false
trap "echo random seed is $SEED" EXIT
if ! docker ps | grep -q minio; then
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ACCESS_KEY=minioadmin" \
-e "MINIO_SECRET_KEY=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
minio/minio server /data
fi
[[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
sleep 3s
mc alias set myminio http://localhost:9000 minioadmin minioadmin
python3 -c "import xattr" || sudo pip install xattr
test_dump_load_sustained_file(){
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount -d $META_URL /jfs
file_count=100
for i in $(seq 1 $file_count); do
touch /jfs/file$i
exec {fd}<>/jfs/file$i
echo fd is $fd
fds[$i]=$fd
rm /jfs/file$i
done
./juicefs dump $META_URL dump.json $(get_dump_option)
for i in $(seq 1 $file_count); do
fd=${fds[$i]}
exec {fd}>&-
done
if [[ "$BINARY" == "true" ]]; then
sustained=$(./juicefs load dump.json --binary --stat | grep sustained | awk -F"|" '{print $2}')
else
sustained=$(jq '.Sustained[].inodes | length' dump.json)
fi
echo "sustained file count: $sustained"
# TODO: uncomment this line
# [[ "$sustained" -eq "$file_count" ]] || (echo "sustained file count($sustained) should be $file_count" && exit 1)
umount_jfs /jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
./juicefs load $META_URL dump.json $(get_load_option)
./juicefs mount -d $META_URL /jfs
}
test_dump_load_with_copy_file_range(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
rm -rf /tmp/test
dd if=/dev/zero of=/tmp/test bs=1M count=1024
cp /tmp/test /jfs/test
node .github/scripts/copyFile.js /jfs/test /jfs/test1
./juicefs dump $META_URL dump.json $(get_dump_option)
umount_jfs /jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
./juicefs load $META_URL dump.json $(get_load_option)
./juicefs mount -d $META_URL /jfs
compare_md5sum /tmp/test /jfs/test1
}
test_dump_load_with_quota(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1
./juicefs dump --log-level error $META_URL $(get_dump_option) > dump.json
umount_jfs /jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
./juicefs load $META_URL dump.json $(get_load_option)
./juicefs mount $META_URL /jfs -d --heartbeat $HEARTBEAT_INTERVAL
./juicefs quota get $META_URL --path /d
dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
}
test_dump_load_with_iflag(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --enable-ioctl
echo "hello" > /jfs/hello.txt
chattr +i /jfs/hello.txt
./juicefs dump $META_URL dump.json $(get_dump_option)
umount_jfs /jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
./juicefs load $META_URL dump.json $(get_load_option)
./juicefs mount -d $META_URL /jfs --enable-ioctl
echo "hello" > /jfs/hello.txt && echo "write should fail" && exit 1 || true
chattr -i /jfs/hello.txt
echo "world" > /jfs/hello.txt
cat /jfs/hello.txt | grep world
}
test_dump_load_with_keep_secret_key()
{
option=$@
prepare_test
./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
./juicefs dump --keep-secret-key $META_URL dump.json $(get_dump_option)
python3 .github/scripts/flush_meta.py $META_URL
./juicefs load $META_URL dump.json $(get_load_option)
./juicefs mount -d $META_URL /jfs
echo "hello" > /jfs/hello.txt
cat /jfs/hello.txt | grep hello
umount_jfs /jfs $META_URL
./juicefs dump $META_URL dump.json $(get_dump_option)
python3 .github/scripts/flush_meta.py $META_URL
./juicefs load $META_URL dump.json $(get_load_option)
./juicefs mount -d $META_URL /jfs && echo "mount should fail" && exit 1 || true
./juicefs config --secret-key minioadmin $META_URL
./juicefs mount -d $META_URL /jfs
echo "hello" > /jfs/hello.txt
cat /jfs/hello.txt | grep hello
}
test_load_encrypted_meta_backup()
{
prepare_test
[[ ! -f my-priv-key.pem ]] && openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048
export JFS_RSA_PASSPHRASE=12345678
./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem
./juicefs mount -d $META_URL /jfs
SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=50 STEP_COUNT=50 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true
umount /jfs
SKIP_BACKUP_META_CHECK=true ./juicefs mount -d --backup-meta 10s $META_URL /jfs
sleep 10s
backup_file=$(ls -l /var/jfs/myjfs/meta/ |tail -1 | awk '{print $NF}')
backup_path=/var/jfs/myjfs/meta/$backup_file
ls -l $backup_path
./juicefs load sqlite3://test2.db $backup_path --encrypt-rsa-key my-priv-key.pem --encrypt-algo aes256gcm-rsa
./juicefs mount -d sqlite3://test2.db /jfs2
diff -ur /jfs/test /jfs2/test --no-dereference
umount_jfs /jfs2 sqlite3://test2.db
rm test2.db -rf
}
test_dump_load_with_random_test()
{
prepare_test
./juicefs format $META_URL myjfs --enable-acl
./juicefs mount -d $META_URL /jfs
./random-test runOp -baseDir /jfs/test -files 500000 -ops 5000000 -threads 50 -dirSize 100 -duration 30s -createOp 30,uniform -deleteOp 5,end --linkOp 10,uniform --symlinkOp 20,uniform --setXattrOp 10,uniform --truncateOp 10,uniform
./juicefs dump $META_URL dump.json $(get_dump_option)
create_database $META_URL2
./juicefs load $META_URL2 dump.json $(get_load_option)
./juicefs dump $META_URL2 dump2.json $(get_dump_option)
./juicefs mount -d $META_URL2 /jfs2
diff -ur /jfs/test /jfs2/test --no-dereference
diff -ur /jfs/.trash /jfs2/.trash --no-dereference
# compare_stat_acl_xattr /jfs/test /jfs2/test
umount_jfs /jfs2 $META_URL2
./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '"' -f 4)
./juicefs destroy --yes $META_URL2 $UUID
}
test_dump_load_with_fsrand()
{
prepare_test
./juicefs format $META_URL myjfs --trash-days 0 --enable-acl
./juicefs mount -d $META_URL /jfs --enable-xattr
rm -rf /tmp/test
SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true
./juicefs dump $META_URL dump.json $(get_dump_option)
create_database $META_URL2
./juicefs load $META_URL2 dump.json $(get_load_option)
./juicefs dump $META_URL2 dump2.json $(get_dump_option)
# if [[ "$BINARY" == "false" ]]; then
# compare_dump_json
# fi
./juicefs mount -d $META_URL2 /jfs2
diff -ur /jfs/test /jfs2/test --no-dereference
compare_stat_acl_xattr /jfs/test /jfs2/test
umount_jfs /jfs2 $META_URL2
./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '"' -f 4)
./juicefs destroy --yes $META_URL2 $UUID
}
compare_dump_json(){
cp dump.json dump.json.bak
cp dump2.json dump2.json.bak
sed -i '/usedSpace/d' dump*.json.bak
sed -i '/usedInodes/d' dump*.json.bak
sed -i '/nextInodes/d' dump*.json.bak
sed -i '/nextChunk/d' dump*.json.bak
sed -i '/nextTrash/d' dump*.json.bak
sed -i '/nextSession/d' dump*.json.bak
sed -i 's/"inode":[0-9]\+/"inode":0/g' dump*.json.bak
diff -ur dump.json.bak dump2.json.bak
}
compare_stat_acl_xattr(){
dir1=$1
dir2=$2
files1=($(find "$dir1" -type f -o -type d -exec stat -c "%n" {} + | sort))
files2=($(find "$dir2" -type f -o -type d -exec stat -c "%n" {} + | sort))
[[ ${#files1[@]} -ne ${#files2[@]} ]] && echo "compare_stat_acl: number of files differs" && exit 1
for i in "${!files1[@]}"; do
stat1=$(stat -c "%F %a %s %h %U %G" "${files1[$i]}")
stat2=$(stat -c "%F %a %s %h %U %G" "${files2[$i]}")
acl1=$(getfacl -p "${files1[$i]}" | tail -n +2)
acl2=$(getfacl -p "${files2[$i]}" | tail -n +2)
xattr1=$(getfattr -d -m . -e hex "${files1[$i]}" 2>/dev/null | tail -n +2 | sort)
xattr2=$(getfattr -d -m . -e hex "${files2[$i]}" 2>/dev/null | tail -n +2 | sort)
[[ "$stat1" != "$stat2" ]] && echo "compare_stat_acl: stat for ${files1[$i]} and ${files2[$i]} differs" && echo $stat1 && echo $stat2 && exit 1
[[ "$acl1" != "$acl2" ]] && echo "compare_stat_acl: ACLs for ${files1[$i]} and ${files2[$i]} differs" && echo $acl1 && echo $acl2 && exit 1
[[ "$xattr1" != "$xattr2" ]] && echo "compare_stat_acl: xattrs for ${files1[$i]} and ${files2[$i]} differs" && echo $xattr1 && echo $xattr2 && exit 1
done
echo "compare_stat_acl: ACLs and stats are the same"
}
get_dump_option(){
if [[ "$BINARY" == "true" ]]; then
option="--binary"
elif [[ "$FAST" == "true" ]]; then
option="--fast"
else
option=""
fi
echo $option
}
get_load_option(){
if [[ "$BINARY" == "true" ]]; then
option="--binary"
else
option=""
fi
echo $option
}
prepare_test(){
umount_jfs /jfs $META_URL
umount_jfs /jfs2 sqlite3://test2.db
python3 .github/scripts/flush_meta.py $META_URL
rm test2.db -rf
rm -rf /var/jfs/myjfs || true
mc rm --force --recursive myminio/test || true
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/dump_load_bench.sh
================================================
#!/bin/bash -ex
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
[[ -z "$START_META" ]] && START_META=true
source .github/scripts/start_meta_engine.sh
META_URL=$(get_meta_url $META)
META_URL2=$(get_meta_url2 $META)
FILE_COUNT_IN_BIGDIR=100000
prepare_test_data(){
umount_jfs /tmp/jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
rm -rf /var/jfs/myjfs || true
create_database $META_URL
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /tmp/jfs
threads=10
./juicefs mdtest $META_URL /bigdir --depth=1 --dirs=0 --files=$((FILE_COUNT_IN_BIGDIR/threads)) --threads=$threads --write=8192
./juicefs mdtest $META_URL /smalldir --depth=3 --dirs=10 --files=10 --threads=10 --write=8192
}
if [[ "$START_META" == "true" ]]; then
start_meta_engine $META
prepare_test_data
fi
test_dump_load(){
do_dump_load dump.json
}
test_dump_load_fast(){
do_dump_load dump.json.gz --fast
}
test_dump_load_in_binary(){
do_dump_load dump.bin --binary
}
do_dump_load(){
dump_file=$1
shift
options=$@
./juicefs dump $META_URL $dump_file $options --threads=50
# python3 .github/scripts/flush_meta.py $META_URL2
create_database $META_URL2
if [[ "$options" == *"--binary"* ]]; then
./juicefs load $META_URL2 $dump_file $options
else
./juicefs load $META_URL2 $dump_file
fi
./juicefs mount $META_URL2 /tmp/jfs2 -d
df -i /tmp/jfs /tmp/jfs2
iused1=$(df -i /tmp/jfs | tail -1 | awk '{print $3}')
iused2=$(df -i /tmp/jfs2 | tail -1 | awk '{print $3}')
[[ "$iused1" == "$iused2" ]] || (echo "<FATAL>: iused error: $iused1 $iused2" && exit 1)
./juicefs summary /tmp/jfs/ --csv
./juicefs summary /tmp/jfs2/ --csv
summary1=$(./juicefs summary /tmp/jfs/ --csv | head -n +2 | tail -n 1)
summary2=$(./juicefs summary /tmp/jfs2/ --csv | head -n +2 | tail -n 1)
[[ "$summary1" == "$summary2" ]] || (echo "<FATAL>: summary error: $summary1 $summary2" && exit 1)
file_count=$(ls -l /tmp/jfs2/bigdir/test-dir.0-0/mdtest_tree.0/ | wc -l)
file_count=$((file_count-1))
if [[ "$file_count" -ne "$FILE_COUNT_IN_BIGDIR" ]]; then
echo "<FATAL>: file_count error: $file_count"
exit 1
fi
./juicefs rmr /tmp/jfs2/smalldir
ls /tmp/jfs2/smalldir && echo "<FATAL>: ls should fail" && exit 1 || true
umount_jfs /tmp/jfs2 $META_URL2
./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '"' -f 4)
./juicefs destroy --yes $META_URL2 $UUID
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/dump_load_cross_meta.sh
================================================
#!/bin/bash -ex
source .github/scripts/common/common.sh
[[ -z "$META1" ]] && META1=sqlite3
[[ -z "$META2" ]] && META2=redis
source .github/scripts/start_meta_engine.sh
start_meta_engine $META1
start_meta_engine $META2
META_URL1=$(get_meta_url $META1)
META_URL2=$(get_meta_url $META2)
[[ -z "$SEED" ]] && SEED=$(date +%s)
# [[ -z "$SEED" ]] && SEED=1711594639
[[ -z "$BINARY" ]] && BINARY=false
[[ -z "$FAST" ]] && FAST=false
trap "echo random seed is $SEED" EXIT
if ! docker ps | grep -q minio; then
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ACCESS_KEY=minioadmin" \
-e "MINIO_SECRET_KEY=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
minio/minio server /data
fi
[[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
sleep 3s
mc alias set myminio http://localhost:9000 minioadmin minioadmin
[[ ! -x random-test ]] && wget -q https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/random-test/random-test -O random-test && chmod +x random-test
python3 -c "import xattr" || sudo pip install xattr
test_dump_load_with_rmr()
{
# ref: https://github.com/juicedata/juicefs/pull/6188
prepare_test
./juicefs format $META_URL1 myjfs --trash-days 0 --enable-acl
./juicefs mount -d $META_URL1 /jfs --enable-xattr
dd if=/dev/urandom of=/jfs/file1 bs=1M count=1024
./juicefs dump $META_URL1 dump1.json
./juicefs dump $META_URL1 dump1 $(get_dump_option)
create_database $META_URL2
./juicefs load $META_URL2 dump1 $(get_load_option)
./juicefs dump $META_URL2 dump2.json
compare_dump_json dump1.json dump2.json
./juicefs mount -d $META_URL2 /jfs2 --no-bgjob
./juicefs rmr --skip-trash /jfs2/file1
JFS_GC_SKIPPEDTIME=1 ./juicefs gc $META_URL2 2>&1| tee gc.log
count=$(sed -n 's/.*\([0-9]\+\) leaked.*/\1/p' gc.log)
[[ "$count" -ne 0 ]] && echo "Expected 0 leaked file, but got $count" && exit 1 || true
}
skip_test_dump_load_with_fsrand()
{
# unskip the test after fix: https://github.com/juicedata/juicefs/issues/6230
prepare_test
./juicefs format $META_URL1 myjfs --trash-days 0 --enable-acl
./juicefs mount -d $META_URL1 /jfs --enable-xattr
rm -rf /tmp/test
SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true
for i in {1..60}; do
JFS_GC_SKIPPEDTIME=1 ./juicefs gc -v $META_URL1 2>&1| tee gc.log
count=$(sed -n 's/.*\([0-9]\+\) leaked.*/\1/p' gc.log)
if [[ "$count" -eq 0 ]]; then
echo "Expected 0 leaked file after rmr /jfs2/test, got $count"
break
else
echo "Expected 0 leaked file after rmr /jfs2/test, got $count, retrying..."
sleep 1s
fi
[[ $i -eq 60 ]] && echo "Expected 0 leaked file after rmr /jfs2/test, but got $count" && exit 1 || true
done
./juicefs dump $META_URL1 dump1.json
./juicefs dump $META_URL1 dump1 $(get_dump_option)
create_database $META_URL2
./juicefs load $META_URL2 dump1 $(get_load_option)
./juicefs dump $META_URL2 dump2.json $(get_dump_option)
# compare_dump_json
./juicefs mount -d $META_URL2 /jfs2 --no-bgjob
diff -ur /jfs/test /jfs2/test --no-dereference
compare_stat_acl_xattr /jfs/test /jfs2/test
./juicefs rmr --skip-trash /jfs2/test
for i in {1..60}; do
JFS_GC_SKIPPEDTIME=1 ./juicefs gc -v $META_URL2 2>&1| tee gc.log
count=$(sed -n 's/.*\([0-9]\+\) leaked.*/\1/p' gc.log)
if [[ "$count" -eq 0 ]]; then
echo "Expected 0 leaked file after rmr /jfs2/test, got $count"
break
else
echo "Expected 0 leaked file after rmr /jfs2/test, got $count, retrying..."
sleep 1s
fi
[[ $i -eq 60 ]] && echo "Expected 0 leaked file after rmr /jfs2/test, but got $count" && exit 1 || true
done
}
skip_test_dump_load_with_random_test()
{
# unskip the test after fix: https://github.com/juicedata/juicefs/issues/6230
prepare_test
./juicefs format $META_URL1 myjfs --trash-days 0 --enable-acl
./juicefs mount -d $META_URL1 /jfs --enable-xattr
./random-test runOp --baseDir /jfs/test --logDir random-test-log --withData --writeSize 1,10240 \
--duration 30s --files 10000000 --ops 100000000 --threads 200 --dirSize 100 \
--mkdirOp 10,uniform -createOp 10,uniform -readOp 1,uniform -lsOp 1,uniform -deleteOp 0.01,uniform -rmrOp 0.01,end -renameOp 1,uniform -linkOp 3,uniform
./juicefs clone /jfs/test /jfs/test_clone
./juicefs dump $META_URL1 dump1.json
./juicefs dump $META_URL1 dump1 $(get_dump_option)
create_database $META_URL2
./juicefs load $META_URL2 dump1 $(get_load_option)
./juicefs dump $META_URL2 dump2.json $(get_dump_option)
./juicefs mount -d $META_URL2 /jfs2 --no-bgjob
diff -ur /jfs/test /jfs2/test --no-dereference
diff -ur /jfs/test_clone /jfs2/test_clone --no-dereference
./juicefs clone /jfs2/test /jfs2/test_clone2
for dir in /jfs2/test_clone /jfs2/test /jfs2/test_clone2; do
./juicefs rmr --skip-trash $dir
JFS_GC_SKIPPEDTIME=1 ./juicefs gc -v $META_URL2 2>&1| tee gc.log
count=$(sed -n 's/.*\([0-9]\+\) leaked.*/\1/p' gc.log)
[[ "$count" -ne 0 ]] && echo "Expected 0 leaked file after rmr $dir, but got $count" && exit 1 || true
done
}
compare_dump_json(){
cat dump1.json
cat dump2.json
cp dump1.json dump1.json.bak
cp dump2.json dump2.json.bak
sed -i '/usedSpace/d' dump*.json.bak
sed -i '/usedInodes/d' dump*.json.bak
sed -i '/nextInodes/d' dump*.json.bak
sed -i '/nextChunk/d' dump*.json.bak
sed -i '/nextTrash/d' dump*.json.bak
sed -i '/nextSession/d' dump*.json.bak
sed -i 's/"inode":[0-9]\+/"inode":0/g' dump*.json.bak
diff -ur dump1.json.bak dump2.json.bak
echo "compare_dump_json: dump json files are the same"
}
compare_stat_acl_xattr(){
dir1=$1
dir2=$2
files1=($(find "$dir1" -type f -o -type d -exec stat -c "%n" {} + | sort))
files2=($(find "$dir2" -type f -o -type d -exec stat -c "%n" {} + | sort))
[[ ${#files1[@]} -ne ${#files2[@]} ]] && echo "compare_stat_acl: number of files differs" && exit 1
for i in "${!files1[@]}"; do
stat1=$(stat -c "%F %a %s %h %U %G" "${files1[$i]}")
stat2=$(stat -c "%F %a %s %h %U %G" "${files2[$i]}")
acl1=$(getfacl -p "${files1[$i]}" | tail -n +2)
acl2=$(getfacl -p "${files2[$i]}" | tail -n +2)
xattr1=$(getfattr -d -m . -e hex "${files1[$i]}" 2>/dev/null | tail -n +2 | sort)
xattr2=$(getfattr -d -m . -e hex "${files2[$i]}" 2>/dev/null | tail -n +2 | sort)
[[ "$stat1" != "$stat2" ]] && echo "compare_stat_acl: stat for ${files1[$i]} and ${files2[$i]} differs" && echo $stat1 && echo $stat2 && exit 1
[[ "$acl1" != "$acl2" ]] && echo "compare_stat_acl: ACLs for ${files1[$i]} and ${files2[$i]} differs" && echo $acl1 && echo $acl2 && exit 1
[[ "$xattr1" != "$xattr2" ]] && echo "compare_stat_acl: xattrs for ${files1[$i]} and ${files2[$i]} differs" && echo $xattr1 && echo $xattr2 && exit 1
done
echo "compare_stat_acl: ACLs and stats are the same"
}
get_dump_option(){
if [[ "$BINARY" == "true" ]]; then
option="--binary"
elif [[ "$FAST" == "true" ]]; then
option="--fast"
else
option=""
fi
echo $option
}
get_load_option(){
if [[ "$BINARY" == "true" ]]; then
option="--binary"
else
option=""
fi
echo $option
}
prepare_test(){
umount_jfs /jfs $META_URL1
umount_jfs /jfs2 $META_URL2
python3 .github/scripts/flush_meta.py $META_URL1
python3 .github/scripts/flush_meta.py $META_URL2
rm -rf /var/jfs/myjfs || true
mc rm --force --recursive myminio/test || true
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/format.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
SMB_CONTAINER_NAME="juicefs-ci-smb"
SMB_USER="juicefs"
SMB_PASSWORD="juicefs"
SMB_SHARE="share"
cleanup_smb_container()
{
docker rm -f "$SMB_CONTAINER_NAME" >/dev/null 2>&1 || true
rm -rf /tmp/${SMB_CONTAINER_NAME}-data >/dev/null 2>&1 || true
}
start_smb_container()
{
cleanup_smb_container
mkdir -p /tmp/${SMB_CONTAINER_NAME}-data
chmod 0777 /tmp/${SMB_CONTAINER_NAME}-data
if [[ "$(uname)" == "Darwin" ]]; then
docker run -d --name "$SMB_CONTAINER_NAME" -p 1445:445 \
-v /tmp/${SMB_CONTAINER_NAME}-data:/mount \
dperson/samba \
-u "$SMB_USER;$SMB_PASSWORD" \
-s "$SMB_SHARE;/mount;yes;no;no;$SMB_USER" >/dev/null
wait_tcp_ready 127.0.0.1 1445 40
SMB_ENDPOINT="127.0.0.1:1445/${SMB_SHARE}"
export SMB_ENDPOINT
return
fi
docker run -d --name "$SMB_CONTAINER_NAME" \
-v /tmp/${SMB_CONTAINER_NAME}-data:/mount \
dperson/samba \
-u "$SMB_USER;$SMB_PASSWORD" \
-s "$SMB_SHARE;/mount;yes;no;no;$SMB_USER" >/dev/null
local container_ip
container_ip=$(docker container inspect "$SMB_CONTAINER_NAME" --format '{{ .NetworkSettings.IPAddress }}')
wait_tcp_ready "$container_ip" 445 40
SMB_ENDPOINT="${container_ip}/${SMB_SHARE}"
export SMB_ENDPOINT
}
assert_objbench_result()
{
local log_file=$1
local test_name=$2
local expected=$3
if ! grep -E "${test_name}.*${expected}" "$log_file" >/dev/null; then
echo "objbench assertion failed: test=${test_name}, expected=${expected}"
echo "--- objbench log ---"
cat "$log_file"
exit 1
fi
}
kill_gateway_by_port()
{
local port=$1
lsof -t -i :$port | xargs -r kill -9 >/dev/null 2>&1 || true
}
wait_tcp_ready()
{
local host=$1
local port=$2
local timeout=${3:-30}
for _ in $(seq 1 "$timeout"); do
if (echo > /dev/tcp/${host}/${port}) >/dev/null 2>&1; then
return
fi
sleep 1
done
echo "tcp ${host}:${port} is not ready in ${timeout} seconds"
exit 1
}
ensure_mc_binary()
{
if [[ -x ./mc ]]; then
return
fi
local os_arch
local cpu_arch
cpu_arch=$(uname -m)
if [[ "$(uname)" == "Darwin" ]]; then
if [[ "$cpu_arch" == "arm64" ]]; then
os_arch="darwin-arm64"
else
os_arch="darwin-amd64"
fi
else
if [[ "$cpu_arch" == "aarch64" || "$cpu_arch" == "arm64" ]]; then
os_arch="linux-arm64"
else
os_arch="linux-amd64"
fi
fi
wget -q "https://dl.min.io/client/mc/release/${os_arch}/mc" -O ./mc
chmod +x ./mc
}
generate_sha_manifest()
{
local root_dir=$1
local output_file=$2
rm -f "$output_file"
if [[ "$(uname)" == "Darwin" ]]; then
while IFS= read -r rel; do
sum=$(shasum -a 256 "$root_dir/$rel" | awk '{print $1}')
echo "$sum $rel" >> "$output_file"
done < <(cd "$root_dir" && find . -type f | sort | sed 's#^\./##')
else
while IFS= read -r rel; do
sum=$(sha256sum "$root_dir/$rel" | awk '{print $1}')
echo "$sum $rel" >> "$output_file"
done < <(cd "$root_dir" && find . -type f | sort | sed 's#^\./##')
fi
}
prepare_sync_source_tree()
{
local src_dir=$1
mkdir -p "$src_dir/dir1/dir2"
echo "hello-juicefs" > "$src_dir/plain.txt"
echo "with space" > "$src_dir/dir1/file with space.txt"
echo "cifs-中文文件" > "$src_dir/dir1/中文文件.txt"
: > "$src_dir/empty.file"
dd if=/dev/urandom of="$src_dir/dir1/dir2/binary.bin" bs=1M count=4 >/dev/null 2>&1
}
skip_test_mount_process_exit_on_format()
{
prepare_test
echo "round $i"
./juicefs format $META_URL volume-$i
./juicefs mount -d $META_URL /tmp/myjfs$i_$j --no-usage-report
cd /tmp/myjfs$i_$j
bash -c 'for k in {1..300}; do echo abc>$k; sleep 0.2; done' || true &
cd -
sleep 3
uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4)
./juicefs destroy --force $META_URL $uuid
./juicefs format $META_URL new-volume-$i
sleep 15
ps -ef | grep juicefs
# TODO: fix the bug and remove the following line
# SEE https://github.com/juicedata/juicefs/issues/4534
pidof juicefs && exit 1
uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4)
./juicefs destroy --force $META_URL $uuid
}
test_format_sftp_object()
{
docker run -d --name sftp -p 2222:22 juicedata/ci-sftp
prepare_test
CONTAINER_IP=$(docker container inspect sftp --format '{{ .NetworkSettings.IPAddress }}')
echo "round $i"
./juicefs format $META_URL volume-$i --storage sftp \
--bucket $CONTAINER_IP:myjfs/ \
--access-key testUser1 \
--secret-key password
./juicefs mount -d $META_URL /tmp/jfs --no-usage-report --cache-size 0
cd /tmp/jfs
bash -c 'for k in {1..100}; do echo abc>$k; sleep 0.1; done' || true &
bg_pid=$!
cd -
sleep 1
docker stop sftp
sleep 10
docker start sftp
sleep 2
wait $bg_pid
echo "Checking JuiceFS read/write"
echo abc > /tmp/jfs/101
for k in {1..100}; do
if [[ $(cat /tmp/jfs/$k) != "abc" ]]; then
echo "ERROR: File $k corrupted after SFTP restart!"
exit 1
fi
done
uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4)
./juicefs destroy --force $META_URL $uuid
./juicefs format $META_URL new-volume-$i
}
test_format_cifs_objbench_matrix()
{
prepare_test
start_smb_container
local log_raw=/tmp/objbench-cifs-raw.log
local log_plain=/tmp/objbench-cifs.log
./juicefs objbench --storage cifs \
--access-key "$SMB_USER" \
--secret-key "$SMB_PASSWORD" \
--threads 2 \
--small-objects 5 \
--small-object-size 4K \
--block-size 1M \
--big-object-size 8M \
"$SMB_ENDPOINT" 2>&1 | tee "$log_raw"
sed -E 's/\x1B\[[0-9;]*[mK]//g' "$log_raw" > "$log_plain"
assert_objbench_result "$log_plain" "create a bucket" "pass"
assert_objbench_result "$log_plain" "put an object" "pass"
assert_objbench_result "$log_plain" "get an object" "pass"
assert_objbench_result "$log_plain" "get non-exist" "pass"
assert_objbench_result "$log_plain" "get partial object" "pass"
assert_objbench_result "$log_plain" "head an object" "pass"
assert_objbench_result "$log_plain" "delete an object" "pass"
assert_objbench_result "$log_plain" "delete non-exist" "pass"
assert_objbench_result "$log_plain" "list objects" "pass"
assert_objbench_result "$log_plain" "special key" "put encode file failed"
assert_objbench_result "$log_plain" "put a big object" "pass"
assert_objbench_result "$log_plain" "put an empty object" "pass"
assert_objbench_result "$log_plain" "multipart upload" "not support"
assert_objbench_result "$log_plain" "change owner/group" "failed to chown object"
assert_objbench_result "$log_plain" "change permission" "expect mode 777 but got"
assert_objbench_result "$log_plain" "change mtime" "pass"
cleanup_smb_container
}
test_format_smb_object_alias()
{
prepare_test
start_smb_container
local volume_name="smb-alias-$RANDOM"
local mount_point="/tmp/jfs-smb-$RANDOM"
./juicefs format $META_URL "$volume_name" --storage smb \
--bucket "$SMB_ENDPOINT" \
--access-key "$SMB_USER" \
--secret-key "$SMB_PASSWORD"
mkdir -p "$mount_point"
./juicefs mount -d $META_URL "$mount_point" --no-usage-report --cache-size 0
echo "smb-alias-ok" > "$mount_point/smb-alias.txt"
read_content=$(cat "$mount_point/smb-alias.txt")
[[ "$read_content" != "smb-alias-ok" ]] && echo "smb alias read/write check failed" && exit 1
./juicefs umount "$mount_point" || true
rm -rf "$mount_point"
uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4)
./juicefs destroy --force $META_URL $uuid
cleanup_smb_container
}
test_format_cifs_sync_consistency()
{
prepare_test
start_smb_container
local volume_name="cifs-sync-$RANDOM"
local mount_point="/tmp/jfs-cifs-sync-$RANDOM"
local mount_data_dir
local src_dir="/tmp/cifs-sync-src-$RANDOM"
local dst_dir="/tmp/cifs-sync-dst-$RANDOM"
local src_manifest="/tmp/cifs-sync-src-$RANDOM.sha256"
local dst_manifest="/tmp/cifs-sync-dst-$RANDOM.sha256"
./juicefs format $META_URL "$volume_name" --storage cifs \
--bucket "$SMB_ENDPOINT" \
--access-key "$SMB_USER" \
--secret-key "$SMB_PASSWORD"
mkdir -p "$mount_point"
./juicefs mount -d $META_URL "$mount_point" --no-usage-report --cache-size 0
mount_data_dir="$mount_point/sync-data"
mkdir -p "$mount_data_dir"
rm -rf "$src_dir" "$dst_dir"
mkdir -p "$src_dir" "$dst_dir"
prepare_sync_source_tree "$src_dir"
./juicefs sync "$src_dir/" "$mount_data_dir/" --threads 8 --dirs
./juicefs sync "$mount_data_dir/" "$dst_dir/" --threads 8 --dirs
generate_sha_manifest "$src_dir" "$src_manifest"
generate_sha_manifest "$dst_dir" "$dst_manifest"
diff "$src_manifest" "$dst_manifest"
src_count=$(find "$src_dir" -type f | wc -l | tr -d ' ')
dst_count=$(find "$dst_dir" -type f | wc -l | tr -d ' ')
[[ "$src_count" != "$dst_count" ]] && echo "sync file count mismatch: $src_count vs $dst_count" && exit 1
./juicefs umount "$mount_point" || true
rm -rf "$mount_point" "$src_dir" "$dst_dir"
uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4)
./juicefs destroy --force $META_URL $uuid
cleanup_smb_container
}
test_format_cifs_object_recovery()
{
prepare_test
start_smb_container
local volume_name="cifs-recovery-$RANDOM"
local mount_point="/tmp/jfs-cifs-recovery-$RANDOM"
./juicefs format $META_URL "$volume_name" --storage cifs \
--bucket "$SMB_ENDPOINT" \
--access-key "$SMB_USER" \
--secret-key "$SMB_PASSWORD"
mkdir -p "$mount_point"
./juicefs mount -d $META_URL "$mount_point" --no-usage-report --cache-size 0
for k in {1..20}; do
echo "before-restart-$k" > "$mount_point/before-$k.txt"
done
docker stop "$SMB_CONTAINER_NAME"
sleep 8
docker start "$SMB_CONTAINER_NAME"
container_ip=$(docker container inspect "$SMB_CONTAINER_NAME" --format '{{ .NetworkSettings.IPAddress }}')
wait_tcp_ready "$container_ip" 445 40
sleep 3
for k in {1..20}; do
content=$(cat "$mount_point/before-$k.txt")
[[ "$content" != "before-restart-$k" ]] && echo "file check failed after restart: before-$k.txt" && exit 1
done
echo "after-restart" > "$mount_point/after-restart.txt"
[[ "$(cat "$mount_point/after-restart.txt")" != "after-restart" ]] && echo "write/read failed after cifs restart" && exit 1
./juicefs umount "$mount_point" || true
rm -rf "$mount_point"
uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4)
./juicefs destroy --force $META_URL $uuid
cleanup_smb_container
}
test_format_cifs_gateway_read_write()
{
prepare_test
start_smb_container
ensure_mc_binary
local volume_name="cifs-gateway-$RANDOM"
local gateway_port=9015
./juicefs format $META_URL "$volume_name" --storage cifs \
--bucket "$SMB_ENDPOINT" \
--access-key "$SMB_USER" \
--secret-key "$SMB_PASSWORD"
kill_gateway_by_port $gateway_port
export MINIO_ROOT_USER=admin
export MINIO_ROOT_PASSWORD=admin123
./juicefs gateway $META_URL 127.0.0.1:${gateway_port} --multi-buckets --keep-etag --object-tag -background
wait_tcp_ready 127.0.0.1 $gateway_port 30
./mc alias set cifsgw http://127.0.0.1:${gateway_port} admin admin123 --api S3v4
./mc mb cifsgw/test-cifs-gw
echo "gateway-cifs-ok" > /tmp/cifs-gateway-file.txt
./mc cp /tmp/cifs-gateway-file.txt cifsgw/test-cifs-gw/cifs-gateway-file.txt
./mc cat cifsgw/test-cifs-gw/cifs-gateway-file.txt | grep "gateway-cifs-ok"
./mc rm cifsgw/test-cifs-gw/cifs-gateway-file.txt
./mc rb cifsgw/test-cifs-gw --force
kill_gateway_by_port $gateway_port
uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4)
./juicefs destroy --force $META_URL $uuid
cleanup_smb_container
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/fsck.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
test_fix_nlink(){
if [[ "$META" == "sqlite3" ]]; then
do_fix_nlink_sqlite3
fi
}
do_fix_nlink_sqlite3(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
mkdir /jfs/a
mkdir /jfs/a/b
touch /jfs/a/c
sleep 4s # to wait dir stat update
./juicefs fsck $META_URL --path / -r
sqlite3 test.db "update jfs_node set nlink=100 where inode=2"
sqlite3 test.db "select nlink from jfs_node where inode=2"
./juicefs fsck $META_URL --path / -r && exit 1 || true
./juicefs fsck $META_URL --path / -r --repair
./juicefs fsck $META_URL --path / -r
}
test_sync_dir_stat()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
./juicefs mdtest $META_URL /d --depth 15 --dirs 2 --files 100 --threads 10 &
pid=$!
sleep 15s
kill -9 $pid
./juicefs info -r /jfs/d
./juicefs info -r /jfs/d --strict
./juicefs fsck $META_URL --path /d --sync-dir-stat --repair -r
./juicefs info -r /jfs/d | tee info1.log
./juicefs info -r /jfs/d --strict | tee info2.log
diff info1.log info2.log
rm info*.log
./juicefs fsck $META_URL --path / --sync-dir-stat --repair -r
./juicefs info -r /jfs | tee info1.log
./juicefs info -r /jfs --strict | tee info2.log
diff info1.log info2.log
}
test_fsck_with_random_test()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
./random-test runOp -baseDir /jfs/test -files 500000 -ops 5000000 -threads 50 -dirSize 100 -duration 30s -createOp 30,uniform -deleteOp 5,end --linkOp 10,uniform --symlinkOp 20,uniform --setXattrOp 10,uniform --truncateOp 10,uniform
./juicefs fsck $META_URL --path /test --sync-dir-stat --repair -r
./juicefs info -r /jfs | tee info1.log
./juicefs info -r /jfs --strict | tee info2.log
diff info1.log info2.log || true
}
test_fsck_delete_object()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
echo "test" > /jfs/test.txt
sleep 1
object=$(./juicefs info /jfs/test.txt | grep chunks | awk '{print $4}')
rm /var/jfs/$object
./juicefs fsck $META_URL 2>&1 | tee fsck.log
grep -q "1 objects are lost" fsck.log || exit 1
rm fsck.log
# ./juicefs fsck $META_URL --path / --sync-dir-stat --repair -r 2>&1 | tee fsck.log
# grep -q "1 objects are lost" fsck.log || exit 1
# rm fsck.log
./juicefs rmr /jfs/test.txt --skip-trash
./juicefs fsck $META_URL || { echo "files is deleted, fsck should success"; exit 1; }
}
test_sync_dir_df()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
./juicefs mdtest $META_URL /d --depth 15 --dirs 2 --files 100 --threads 10 &
pid=$!
sleep 60s
kill -9 $pid
./juicefs info -r /jfs/d --strict
#df -h /jfs的Used和
df -h /jfs
./juicefs fsck $META_URL --path /d --sync-dir-stat --repair -r
./juicefs info -r /jfs/d | tee info1.log
./juicefs info -r /jfs/d --strict | tee info2.log
diff info1.log info2.log
rm info*.log
./juicefs fsck $META_URL --path / --sync-dir-stat --repair -r
./juicefs info -r /jfs | tee info1.log
./juicefs info -r /jfs --strict | tee info2.log
diff info1.log info2.log
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/gateway-random.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
[[ -z "$SUBDIR" ]] && SUBDIR=false
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
[[ ! -x /usr/local/bin/mc ]] && wget -q https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
# docker ps -aq --filter "status=exited" --filter "name=minio_old" | xargs -r docker rm -v
if ! docker ps --filter "name=minio_old$" | grep minio_old; then
echo start minio_old
docker run -d -p 9000:9000 --name minio_old -e "MINIO_ACCESS_KEY=minioadmin" -e "MINIO_SECRET_KEY=minioadmin" minio/minio:RELEASE.2021-04-22T15-44-28Z server /tmp/minio_old
while ! curl -s http://localhost:9000/minio/health/live > /dev/null; do
echo "Waiting for MinIO to be ready..."
sleep 1
done
echo "MinIO is ready."
fi
timeout 30 bash -c 'counter=0; until lsof -i:9000; do echo -ne "wait port ready in $counter\r" && ((counter++)) && sleep 1; done'
[[ -n $CI ]] && trap 'kill_gateway 9005;' EXIT
kill_gateway() {
port=$1
lsof -i:$port || true
lsof -t -i :$port | xargs -r kill -9 || true
}
prepare_test()
{
umount_jfs /tmp/jfs $META_URL
kill_gateway 9005
python3 .github/scripts/flush_meta.py $META_URL
rm -rf /var/jfs/myjfs || true
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount -d $META_URL /tmp/jfs
if [ "$SUBDIR" = true ]; then
echo "start gateway with subdir"
mkdir /tmp/jfs/subdir
MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ./juicefs gateway \
$META_URL localhost:9005 --multi-buckets --keep-etag -d --subdir /subdir
else
MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ./juicefs gateway \
$META_URL localhost:9005 --multi-buckets --keep-etag -d
fi
}
test_run_example()
{
prepare_test
python3 .github/scripts/hypo/s3_test.py
}
test_run_all()
{
prepare_test
python3 .github/scripts/hypo/s3.py
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/gateway.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=redis
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
wget https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O mc
chmod +x mc
export MINIO_ROOT_USER=admin
export MINIO_ROOT_PASSWORD=admin123
export MINIO_REFRESH_IAM_INTERVAL=3s
prepare_test()
{
umount_jfs /tmp/jfs $META_URL
kill_gateway 9001
kill_gateway 9002
python3 .github/scripts/flush_meta.py $META_URL
rm -rf /var/jfs/myjfs || true
rm -rf /var/jfsCache/myjfs || true
}
kill_gateway() {
port=$1
lsof -i:$port || true
lsof -t -i :$port | xargs -r kill -9 || true
}
trap 'kill_gateway 9001; kill_gateway 9002; kill_gateway 9003' EXIT
start_two_gateway()
{
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount -d $META_URL /tmp/jfs
export MINIO_ROOT_USER=admin
export MINIO_ROOT_PASSWORD=admin123
./juicefs gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag -background
sleep 1
./juicefs gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag -background
sleep 2
./mc alias set gateway1 http://127.0.0.1:9001 admin admin123
./mc alias set gateway2 http://127.0.0.1:9002 admin admin123
}
test_user_management()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
sleep 5
user=$(./mc admin user list gateway2 | grep user1) || true
if [ -z "$user" ]
then
echo "user synchronization error"
exit 1
fi
./mc mb gateway1/test1
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
if ./mc cp mc gateway1_user1/test1/file1
then
echo "By default, the user has no read and write permission"
exit 1
fi
./mc admin policy set gateway1 readwrite user=user1
if ./mc cp mc gateway1_user1/test1/file1
then
echo "readwrite policy can read and write objects"
else
echo "set readwrite policy fail"
exit 1
fi
./mc cp gateway2/test1/file1 .
compare_md5sum file1 mc
./mc admin user disable gateway1 user1
./mc admin user remove gateway2 user1
sleep 5
user=$(./mc admin user list gateway1 | grep user1) || true
if [ ! -z "$user" ]
then
echo "remove user user1 fail"
echo $user
exit 1
fi
}
test_group_management()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin user add gateway1 user2 admin123
./mc admin user add gateway1 user3 admin123
./mc admin group add gateway1 testcents user1 user2 user3
result=$(./mc admin group info gateway1 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3" ]
then
echo "error,result is '$result'"
exit 1
fi
./mc admin policy set gateway1 readwrite group=testcents
sleep 5
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
./mc mb gateway1/test1
if ./mc cp mc gateway1_user1/test1/file1
then
echo "readwrite policy can read write"
else
echo "the readwrite group has no read and write permission"
exit 1
fi
./mc admin policy set gateway1 readonly group=testcents
sleep 5
if ./mc cp mc gateway1_user1/test1/file1
then
echo "readonly group policy can not write"
exit 1
else
echo "the readonly group has no write permission"
fi
./mc admin group remove gateway1 testcents user1 user2 user3
./mc admin group remove gateway1 testcents
}
test_mult_gateways_set_group()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin user add gateway1 user2 admin123
./mc admin user add gateway1 user3 admin123
./mc admin group add gateway1 testcents user1 user2 user3
./mc admin group disable gateway2 testcents
sleep 5
result=$(./mc admin group info gateway2 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3" ]
then
echo "error,result is '$result'"
exit 1
fi
./mc admin group enable gateway1 testcents
./mc admin user add gateway1 user4 admin123
./mc admin group add gateway1 testcents user4
sleep 1
./mc admin group disable gateway2 testcents
sleep 5
result=$(./mc admin group info gateway2 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3,user4" ]
then
echo "error,result is '$result'"
exit 1
fi
}
test_user_svcacct_add()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin policy set gateway1 consoleAdmin user=user1
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
./mc admin user svcacct add gateway1_user1 user1 --access-key 12345678 --secret-key 12345678
./mc admin user svcacct info gateway1_user1 12345678
./mc admin user svcacct set gateway1_user1 12345678 --secret-key 123456789
./mc alias set svcacct1 http://127.0.0.1:9001 12345678 123456789
./mc mb svcacct1/test1
if ./mc cp mc svcacct1/test1/file1
then
echo "svcacct user consoleAdmin policy can read write"
else
echo "the svcacct user has no read and write permission"
exit 1
fi
./mc admin user svcacct disable gateway1_user1 12345678
./mc admin user svcacct rm gateway1_user1 12345678
}
test_user_admin_svcacct_add()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin policy set gateway1 readwrite user=user1
./mc admin user svcacct add gateway1 user1 --access-key 12345678 --secret-key 12345678
./mc admin user svcacct info gateway1 12345678
./mc admin user svcacct set gateway1 12345678 --secret-key 12345678910
./mc alias set svcacct1 http://127.0.0.1:9001 12345678 12345678910
./mc mb svcacct1/test1
if ./mc cp mc svcacct1/test1/file1
then
echo "amdin user can do svcacct "
else
echo "the svcacct user has no read and write permission"
exit 1
fi
./mc admin user svcacct disable gateway1 12345678
./mc admin user svcacct rm gateway1 12345678
}
test_user_sts()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin policy set gateway1 consoleAdmin user=user1
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
git clone https://github.com/juicedata/minio.git -b gateway-1.1
./mc mb gateway1_user1/test1
./mc cp mc gateway1_user1/test1/mc
cd minio
go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1 -d
go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1
cd -
./mc admin user remove gateway1 user1
}
skip_test_change_credentials()
{
prepare_test
start_two_gateway
./mc mb gateway1/test1
./mc cp mc gateway1/test1/file1
lsof -i :9001 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
lsof -i :9002 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
export MINIO_ROOT_USER=newadmin
export MINIO_ROOT_PASSWORD=newadmin123
export MINIO_ROOT_USER_OLD=admin
export MINIO_ROOT_PASSWORD_OLD=admin123
./juicefs gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag -background
./juicefs gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag -background
sleep 5
./mc alias set gateway1 http://127.0.0.1:9001 newadmin newadmin123
./mc alias set gateway2 http://127.0.0.1:9002 newadmin newadmin123
./mc cp gateway1/test1/file1 file1
./mc cp gateway2/test1/file1 file2
compare_md5sum file1 mc
compare_md5sum file2 mc
}
test_ro_gateway()
{
prepare_test
start_two_gateway
./juicefs gateway $META_URL 127.0.0.1:9003 --read-only --multi-buckets --keep-etag --object-tag -background
./mc alias set gateway3 http://127.0.0.1:9003 admin admin123
./mc mb gateway1/test1
./mc cp mc gateway1/test1/file1
./mc admin user add gateway1 user1 admin123
sleep 4
user=$(./mc admin user list gateway3 | grep user1) || true
[[ -z "$user" ]] && echo "user synchronization error" && exit 1 || true
./mc mb gateway3/test3 && echo "By default, the ro has no write permission for creating buckets" && exit 1 || true
./mc cp mc gateway3/test1/file1 && echo "By default, the ro has no write permission for copying files" && exit 1 || true
./mc cp gateway3/test1/file1 .
diff mc file1
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/gc.sh
================================================
#!/bin/bash -e
python3 -c "import xattr" || pip install xattr
dpkg -s redis-tools || .github/scripts/apt_install.sh redis-tools
dpkg -s fio || .github/scripts/apt_install.sh fio
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
test_delay_delete_slice_after_compaction(){
if [[ "$META" != redis* ]]; then
echo "this test only runs for redis meta engine"
return
fi
prepare_test
./juicefs format $META_URL myjfs --trash-days 1
./juicefs mount -d $META_URL /jfs --no-usage-report
fio --name=abc --rw=randwrite --refill_buffers --size=500M --bs=256k --directory=/jfs
redis-cli save
# don't skip files when gc compact
export JFS_SKIPPED_TIME=1
./juicefs gc --compact --delete $META_URL
killall -9 redis-server
sleep 3
./juicefs fsck $META_URL
}
test_gc_trash_slices(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
PATH1=/tmp/test PATH2=/jfs/test python3 .github/scripts/random_read_write.py
./juicefs status --more $META_URL
./juicefs config $META_URL --trash-days 0 --yes
./juicefs gc $META_URL
./juicefs gc $META_URL --delete
./juicefs status --more $META_URL
}
test_gc_trash_files(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
python3 .github/scripts/fsrand.py -c 1000 /jfs/fsrand
rm -rf /jfs/fsrand
./juicefs status --more $META_URL
./juicefs config $META_URL --trash-days 0 --yes
./juicefs gc $META_URL
./juicefs gc $META_URL --delete
./juicefs status --more $META_URL
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/graceful_upgrade.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
LEGACY_META_URL=$META_URL
if [[ "$META" == "redis" ]]; then
LEGACY_META_URL=${META_URL%%\?*}
fi
echo meta_url is $META_URL
dpkg -s fio >/dev/null 2>&1 || .github/scripts/apt_install.sh fio
dpkg -s attr >/dev/null 2>&1 || .github/scripts/apt_install.sh attr
if [[ ! -x "./juicefs-1.1" ]]; then
wget -q https://github.com/juicedata/juicefs/releases/download/v1.1.0/juicefs-1.1.0-linux-amd64.tar.gz
rm /tmp/juicefs -rf && mkdir -p /tmp/juicefs
tar -xzvf juicefs-1.1.0-linux-amd64.tar.gz -C /tmp/juicefs
mv /tmp/juicefs/juicefs juicefs-1.1 && chmod +x juicefs-1.1
rm /tmp/juicefs -rf && rm juicefs-1.1.0-linux-amd64.tar.gz
./juicefs-1.1 version | grep "version 1.1"
fi
[[ ! -f my-priv-key.pem ]] && openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048
test_kill_mount_process()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount $META_URL /tmp/jfs -d
wait_process_started 1
force_kill_child_process
sleep 3
wait_process_started 2
kill_parent_process
wait_command_success "ps -ef | grep "mount" | grep "/tmp/jfs" | grep -v grep | wc -l" 0
./juicefs mount $META_URL /tmp/jfs -d
kill_child_process
wait_command_success "ps -ef | grep "mount" | grep "/tmp/jfs" | grep -v grep | wc -l" 0
./juicefs mount $META_URL /tmp/jfs -d
./juicefs umount /tmp/jfs
wait_command_success "ps -ef | grep "mount" | grep "/tmp/jfs" | grep -v grep | wc -l" 0
}
skip_test_update_with_flock(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /tmp/jfs
ps -ef | grep mount
cat /tmp/jfs/.config | grep -i sid
echo abc | tee /tmp/jfs/test
sleep 1s
flock -x /tmp/jfs/test -c cat &
sleep 1s
flock -s /tmp/jfs/test -c "echo abc" > flock.log 2>&1 &
sleep 1s
exit 1
./juicefs mount -d $META_URL /tmp/jfs
ps -ef | grep mount
cat /tmp/jfs/.config | grep -i sid
cat flock.log
count=$(ps -ef | grep flock | grep -v grep | wc -l)
[[ $count -ne 2 ]] && echo "flock process should be 2, count=$count" && exit 1 || true
}
test_update_non_fuse_option(){
prepare_test
JFS_RSA_PASSPHRASE=12345678 ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem
JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs
echo abc | tee /tmp/jfs/test
JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs --read-only
echo abc | tee /tmp/jfs/test && (echo "should not write read-only file system" && exit 1) || true
JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs
echo abc | tee /tmp/jfs/test
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 2 ]] && echo "mount process count should be 2, count=$count" && exit 1 || true
umount /tmp/jfs
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 0 ]] && echo "mount process count should be 0, count=$count" && exit 1 || true
}
test_update_on_failure(){
prepare_test
JFS_RSA_PASSPHRASE=12345678 ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem
JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs
echo abc | tee /tmp/jfs/test
JFS_RSA_PASSPHRASE=abc123xx ./juicefs mount -d $META_URL /tmp/jfs || true
echo abc | tee /tmp/jfs/test
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 2 ]] && echo "mount process count should be 2, count=$count" && exit 1 || true
umount /tmp/jfs
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 0 ]] && echo "mount process count should be 0, count=$count" && exit 1 || true
}
#TODO: fio test failed on database locked.
test_update_on_fio(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /tmp/jfs --buffer-size 300
fio -name=fio -filename=/tmp/jfs/testfile -direct=1 -iodepth 16 -ioengine=libaio \
-rw=randwrite -bs=4k -size=100M -numjobs=4 -runtime=30 -group_reporting >fio.log 2>&1 &
fio_pid=$!
trap "kill -9 $fio_pid > /dev/null || true" EXIT
for i in {1..5}; do
echo "update buffer-size to $((i+300))"
./juicefs mount -d $META_URL /tmp/jfs --buffer-size $((i+300))
wait_command_success "ps -ef | grep juicefs | grep mount | grep \"buffer-size $((i+300))\" | wc -l" 2
echo abc | tee /tmp/jfs/test
done
kill -9 $fio_pid > /dev/null 2>&1 || true
# umount_jfs /tmp/jfs $META_URL
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 2 ]] && echo "mount process count should be 2, count=$count" && exit 1 || true
}
test_update_fuse_option(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /tmp/jfs --enable-xattr
setfattr -n user.test -v "juicedata" /tmp/jfs
getfattr -n user.test /tmp/jfs | grep juicedata
./juicefs mount -d $META_URL /tmp/jfs
getfattr -n user.test /tmp/jfs && exit 1 || true
./juicefs mount -d $META_URL /tmp/jfs --enable-xattr
getfattr -n user.test /tmp/jfs | grep juicedata
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 4 ]] && echo "mount process count should be 4, count=$count" && exit 1 || true
umount /tmp/jfs
getfattr -n user.test /tmp/jfs && exit 1 || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 2 ]] && echo "mount process count should be 2, count=$count" && exit 1 || true
umount /tmp/jfs
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 0 ]] && echo "mount process count should be 0, count=$count" && exit 1 || true
}
test_update_from_old_version(){
prepare_test
./juicefs-1.1 format $LEGACY_META_URL myjfs
./juicefs-1.1 mount -d $LEGACY_META_URL /tmp/jfs
echo hello |tee /tmp/jfs/test
./juicefs mount -d $META_URL /tmp/jfs
count=$(ps -ef | grep juicefs | grep mount | wc -l)
[[ $count -ne 3 ]] && echo "mount process count should be 3" && exit 1 || true
version=$(./juicefs version | awk '{print $3,$4,$5}')
grep Version /tmp/jfs/.config | grep $version
grep "hello" /tmp/jfs/test
echo world | tee /tmp/jfs/test
./juicefs umount /tmp/jfs
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 1 ]] && echo "mount process count should be 1" && exit 1 || true
./juicefs umount /tmp/jfs
ps -ef | grep juicefs | grep mount | grep -v grep || true
count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)
[[ $count -ne 0 ]] && echo "mount process count should be 0" && exit 1 || true
}
test_update_on_fstab(){
prepare_test
./juicefs format $META_URL myjfs
umount_jfs /tmp/jfs $META_URL
rm /sbin/mount.juicefs -rf
./juicefs mount --update-fstab $META_URL /tmp/jfs -d \
-o debug,allow_other,writeback_cache \
--max-uploads 20 --prefetch 3 --upload-limit 3 \
--download-limit 100 --get-timeout 60 --put-timeout 60
grep /tmp/jfs /etc/fstab
ls /sbin/mount.juicefs -l
umount /tmp/jfs
for i in {1..5}; do
mount /tmp/jfs
wait_command_success "ps -ef | grep juicefs | grep /tmp/jfs | grep -v grep | wc -l" 2
# cat /tmp/jfs/.config
done
}
prepare_test(){
umount_jfs /tmp/jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
rm -rf /var/jfs/myjfs || true
}
kill_child_process()
{
echo "kill_child_process"
child_pid=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | awk '$3 != 1 {print $2}')
kill $child_pid
}
force_kill_child_process()
{
echo "force_kill_child_process"
child_pid=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | awk '$3 != 1 {print $2}')
kill -9 $child_pid
}
kill_parent_process()
{
echo "kill_parent_process"
parent_pid=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | awk '$3 == 1 {print $2}')
kill $parent_pid
}
wait_process_started()
{
echo "wait_process_to_start $1"
wait_seconds=15
for i in $(seq 1 $wait_seconds); do
if check_process_is_alive ; then
echo "mount process is started"
break
fi
if [ $i -eq $wait_seconds ]; then
ps -ef | grep "juicefs" | grep "mount" | grep -v grep
echo "mount process is not started after $wait_seconds"
exit 1
fi
echo "wait process to start" && sleep 1
done
}
check_process_is_alive()
{
echo >&2 "check_process_is_alive $1"
count=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | wc -l)
if [ $count -ne 2 ]; then
ps -ef | grep "juicefs" | grep -v "grep"
echo >&2 "mount process is not equal 2"
return 1
fi
child_count=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | awk '$3 != 1 {print $2}' | wc -l)
if [[ $child_count -ne 1 ]]; then
ps -ef | grep "juicefs" | grep -v "grep"
echo >&2 "mount child process is not equal 1"
return 1
fi
parent_count=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | awk '$3 == 1 {print $2}' | wc -l)
if [ $parent_count -ne 1 ]; then
ps -ef | grep "juicefs" | grep -v "grep"
echo >&2 "mount parent process is not equal 1"
return 1
fi
ppid1=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | awk '$3 == 1 {print $2}')
ppid2=$(ps -ef | grep "juicefs" | grep "mount" | grep -v grep | awk '$3 != 1 {print $3}')
if [ $ppid1 -ne $ppid2 ]; then
ps -ef | grep "juicefs" | grep "mount" | grep -v "grep"
echo >&2 "mount parent process is not equal child process's ppid"
return 1
fi
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/info.sh
================================================
#!/bin/bash -e
sudo dpkg -s redis-tools || sudo .github/scripts/apt_install.sh redis-tools
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
test_info_big_file(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
dd if=/dev/zero of=/jfs/bigfile bs=1M count=4096
./juicefs info /jfs/bigfile
./juicefs rmr /jfs/bigfile
df -h /jfs
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/interface.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
test_list_large_dir()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
local files_count=100000
if [[ "$META_URL" == redis://* ]]; then
files_count=1300000
fi
./juicefs mdtest $META_URL /test --depth 0 --dirs 1 --files $files_count --threads 1
du /jfs/test & du_pid=$!
sleep 2
kill -INT $du_pid || true
wait $du_pid || true
if ! [ -d "/jfs/test" ]; then
echo >&2 "<FATAL>: directory /jfs/test is not accessible after ls interruption"
exit 1
fi
}
test_deep_nested_dirs() {
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
dir="/juicefs1/test"
for i in $(seq 1 100); do
dir="$dir/dir$i"
mkdir -p "$dir"
echo "content$i" > "$dir/file$i"
done
max_jobs=10
for i in $(seq 1 50); do
nested_dir="/juicefs1/test"
for j in $(seq 1 $i); do
nested_dir="$nested_dir/dir$j"
done
ls "$nested_dir" > /dev/null 2>&1 &
if (( $(jobs -p | wc -l) >= max_jobs )); then
wait -n
fi
done
wait
file_count=$(find /juicefs1/test -type f | wc -l)
if [[ $file_count -ne 100 ]]; then
echo "File number error: $file_count"
return 1
fi
for i in $(seq 1 100); do
nested_dir="/juicefs1/test"
for j in $(seq 1 $i); do
nested_dir="$nested_dir/dir$j"
done
expected_content="content$i"
actual_content=$(cat "$nested_dir/file$i" 2>/dev/null)
if [[ "$actual_content" != "$expected_content" ]]; then
echo "expect: '$expected_content',actual: '$actual_content'"
return 1
fi
done
return 0
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/mount.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
test_sort_dir(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --sort-dir
for i in {1..1000}; do
touch "/jfs/file_$i"
done
mkdir -p /jfs/subdir
for i in {1..1000}; do
touch "/jfs/subdir/file_$i"
done
ls -lh /jfs > /tmp/sorted_no_u
ls -U -lh /jfs > /tmp/sorted_with_u
diff /tmp/sorted_no_u /tmp/sorted_with_u
ls -lh /jfs/subdir > /tmp/subdir_sorted_no_u
ls -U -lh /jfs/subdir > /tmp/subdir_sorted_with_u
diff /tmp/subdir_sorted_no_u /tmp/subdir_sorted_with_u
rm -f /tmp/sorted_*
rm -f /tmp/subdir_sorted_*
}
measure_lookup_time() {
local start_time end_time elapsed
start_time=$(date +%s.%N)
for file in "${FILE_LIST[@]}"; do
if [[ -e "$file" ]]; then
echo "Error: $file exists!" >&2
exit 1
fi
done
end_time=$(date +%s.%N)
elapsed=$(echo "$end_time - $start_time" | bc)
echo "$elapsed"
}
test_negative_dir(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --negative-entry-cache 5
TEST_DIR="/jfs/test_dir_$$"
mkdir -p "${TEST_DIR}"
FILE_LIST=()
for i in {1..1000}; do
FILE_LIST+=("${TEST_DIR}/nonexistent_file_$(printf "%04d" $i)")
done
echo -e "\n=== First lookup (uncached) ==="
time1=$(measure_lookup_time)
echo "Time taken: ${time1} seconds"
echo -e "\n=== Second lookup (cached) ==="
time2=$(measure_lookup_time)
echo "Time taken: ${time2} seconds"
echo -e "\n=== Waiting for cache to expire... ==="
sleep 6
echo -e "\n=== Third lookup (after cache expiry) ==="
time3=$(measure_lookup_time)
echo "Time taken: ${time3} seconds"
echo -e "\n=== Test Result ==="
if (( $(echo "$time1 > 2 * $time2" | bc -l) )) && \
(( $(echo "$time3 > 2 * $time2" | bc -l) )) && \
(( $(echo "$time1 - $time3 < 0.5" | bc -l) )); then
echo "PASS: Caching behavior matches expectations:"
else
echo "FAIL: Caching behavior does NOT match expectations:"
echo "Expected: First ≈ Third > 2 x Second"
exit 1
fi
rm -rf "${TEST_DIR}"
echo -e "\nTest directory removed: ${TEST_DIR}"
}
test_redis_client_cache()
{
if [[ "$META" != "redis" ]]; then
echo "Skip redis client cache test for META=$META"
return 0
fi
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
mkdir /jfs2 || true
./juicefs mount -d $META_URL /jfs2
mkdir -p /jfs/redis_csc
for i in {1..100}; do
echo "v$i" > "/jfs/redis_csc/file_$i"
done
wait_command_success "ls /jfs2/redis_csc | wc -l" "100" 30
echo "cache-sync" > /jfs/redis_csc/shared_file
wait_command_success "cat /jfs2/redis_csc/shared_file" "cache-sync" 30
./juicefs umount /jfs2 || umount -l /jfs2 || true
}
test_check_storage(){
start_meta_engine $META minio
prepare_test
sleep 2
./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test \
--access-key minioadmin --secret-key minioadmin --compress lz4 --hash-prefix
docker stop minio
./juicefs mount $META_URL /tmp/jfs --check-storage || echo "PASS: Mount failed as expected when storage is not accessible"
docker start minio
sleep 2
./juicefs mount $META_URL /tmp/jfs -d
./juicefs umount /tmp/jfs
docker stop minio && docker rm minio
}
test_capabilities()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --enable-xattr --enable-cap
cp /bin/ls /jfs/test_ls
cp /bin/ping /jfs/test_ping
chmod +x /jfs/test_ls /jfs/test_ping
setcap "cap_net_raw+ep" /jfs/test_ping
setcap "cap_dac_override+ep" /jfs/test_ls
sleep 1
getcap /jfs/test_ping | grep -E "cap_net_raw[+=]ep" || {
echo "FAIL: capability not set correctly on test_ping"
exit 1
}
getcap /jfs/test_ls | grep -E "cap_dac_override[+=]ep" || {
echo "FAIL: capability not set correctly on test_ls"
exit 1
}
capsh --print | grep "Current:" || {
echo "FAIL: cannot get current capabilities"
exit 1
}
setcap -r /jfs/test_ping
setcap -r /jfs/test_ls
getcap /jfs/test_ping | grep -E "cap_net_raw[+=]ep" && {
echo "FAIL: capability not removed from test_ping"
exit 1
}
getcap /jfs/test_ls | grep -E "cap_dac_override[+=]ep" && {
echo "FAIL: capability not removed from test_ls"
exit 1
}
rm -f /jfs/test_ls /jfs/test_ping
echo "PASS: Capabilities test completed successfully"
}
test_all_squash()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --all-squash 1101:1101
mkdir -p /jfs/test_dir
touch /jfs/test_dir/test_file
uid1=$(stat -c %u /jfs/test_dir)
gid1=$(stat -c %g /jfs/test_dir)
uid2=$(stat -c %u /jfs/test_dir/test_file)
gid2=$(stat -c %g /jfs/test_dir/test_file)
if [[ "$uid1" != "1101" ]] || [[ "$gid1" != "1101" ]] || [[ "$uid2" != "1101" ]] || [[ "$gid2" != "1101" ]]; then
echo >&2 "<FATAL>: uid/gid does not same as squash: uid1: $uid1, uid2: $uid2, gid1: $gid1, gid2: $gid2"
exit 1
fi
}
test_umask()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --umask 0027
mkdir -p /jfs/test_dir
dir_perms=$(stat -c %a /jfs/test_dir)
if [[ "$dir_perms" != "750" ]]; then
echo >&2 "<FATAL>: Directory permissions incorrect. Expected: 750, Got: $dir_perms"
exit 1
fi
touch /jfs/test_file
file_perms=$(stat -c %a /jfs/test_file)
if [[ "$file_perms" != "640" ]]; then
echo >&2 "<FATAL>: File permissions incorrect. Expected: 640, Got: $file_perms"
exit 1
fi
touch /jfs/test_dir/nested_file
nested_perms=$(stat -c %a /jfs/test_dir/nested_file)
if [[ "$nested_perms" != "640" ]]; then
echo >&2 "<FATAL>: Nested file permissions incorrect. Expected: 640, Got: $nested_perms"
exit 1
fi
echo "PASS: Umask test completed successfully"
}
test_close_to_open1()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
mkdir /jfs2 || true
./juicefs mount -d $META_URL /jfs2
file1="/jfs/testfile.tmp"
file2="/jfs2/testfile.tmp"
rm $file1 || true
openssl rand -base64 -out $file1 512000
sleep 3
ls -ls $file2
echo "#########################"
echo "hello" > $file1
hex_file2=$(cat $file2 | hexdump -C)
echo "#########################"
hex_file2_2=$(cat $file2 | hexdump -C)
hex_file1=$(cat $file1 | hexdump -C)
[[ "$hex_file2" != "$hex_file1" ]] && echo "Content of $hex_file2 and $hex_file1 do not match" && exit 1 || true
[[ "$hex_file2_2" != "$hex_file1" ]] && echo "Content of $hex_file2_2 and $hex_file1 do not match" && exit 1 || true
}
test_colse_to_open2()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs
mkdir /jfs2 || true
./juicefs mount -d $META_URL /jfs2
file1="/jfs/testfile.tmp"
file2="/jfs2/testfile.tmp"
rm $file1 || true
python3 -c "
for i in range(1, 101):
with open('$file1', 'a') as f:
f.write(f'{i}\\n')
with open('$file2', 'a') as f:
f.write(f'{i}\\n')
"
line_count1=$(cat $file1 | wc -l)
line_count2=$(cat $file2 | wc -l)
[[ $line_count1 -ne 200 ]] && cat $file1 && echo "Error: $file1 should have 200 lines but has $line_count1" && exit 1 || true
[[ $line_count2 -ne 200 ]] && cat $file2 && echo "Error: $file2 should have 200 lines but has $line_count2" && exit 1 || true
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/quota.sh
================================================
#!/bin/bash -e
[[ -z "$META" ]] && META=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
HEARTBEAT_INTERVAL=3
HEARTBEAT_SLEEP=3
DIR_QUOTA_FLUSH_INTERVAL=4
VOLUME_QUOTA_FLUSH_INTERVAL=2
source .github/scripts/common/common.sh
test_total_capacity()
{
prepare_test
./juicefs format $META_URL myjfs --capacity 1
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL --debug
dd if=/dev/zero of=/jfs/test1 bs=1G count=1
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/test1 2>error.log && echo "echo should fail on out of space" && exit 1 || true
grep "No space left on device" error.log
./juicefs config $META_URL --capacity 2
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/jfs/test2 bs=1G count=1
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/test2 2>error.log && echo "echo should fail on out of space" && exit 1 || true
grep "No space left on device" error.log
rm /jfs/test1 -rf
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/test3 2>error.log && echo "echo should fail on out of space" && exit 1 || true
./juicefs rmr /jfs/.trash
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/test3
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
ln /jfs/test2 /jfs/test4
ln /jfs/test2 /jfs/test5
}
test_total_inodes(){
prepare_test
./juicefs format $META_URL myjfs --inodes 1000
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
set +x
for i in {1..1000}; do
echo $i | tee /jfs/test$i > /dev/null
done
set -x
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee /jfs/test1001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
grep "No space left on device" error.log
./juicefs config $META_URL --inodes 2000
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
set +x
for i in {1001..2000}; do
echo $i | tee /jfs/test$i > /dev/null || (df -i /jfs && ls /jfs/ -l | wc -l && exit 1)
done
set -x
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee /jfs/test2001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
}
test_nested_dir(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
file_count=1000
mkdir -p /jfs/d1/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}
dir_count=$(find /jfs/d1 -type d | wc -l)
echo "dir_count: $dir_count"
./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count-1))
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
for i in $(seq 1 $file_count); do
subdir=$(find /jfs/d1/ -type d | shuf -n 1)
echo "touch $subdir/test$i" && touch $subdir/test$i
done
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
subdir=$(find /jfs/d1/ -type d | shuf -n 1)
touch $subdir/test 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count))
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
subdir=$(find /jfs/d1/ -type d | shuf -n 1)
touch $subdir/test
}
test_remove_and_restore(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
./juicefs quota set $META_URL --path /d --capacity 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
echo "remove test1" && rm /jfs/d/test1 -rf
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "0%" ]] && echo "used should be 0%" && exit 1 || true
trash_dir=$(ls /jfs/.trash)
./juicefs restore $META_URL $trash_dir --put-back
./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
echo "remove test1" && rm /jfs/d/test1 -rf
dd if=/dev/zero of=/jfs/d/test2 bs=1M count=1
trash_dir=$(ls /jfs/.trash)
./juicefs restore $META_URL $trash_dir --put-back 2>&1 | tee restore.log
grep "disk quota exceeded" restore.log || (echo "check restore log failed" && exit 1)
}
test_dir_capacity(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
./juicefs quota set $META_URL --path /d --capacity 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs quota get $META_URL --path /d
used=$(./juicefs quota get $META_URL --path /d 2>&1 | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
echo a | tee -a /jfs/d/test1 2>error.log && echo "echo should fail on out of space" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
./juicefs quota set $META_URL --path /d --capacity 2
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/jfs/d/test2 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/d/test2 2>error.log && echo "echo should fail on out of space" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
rm -rf /jfs/d/test1
sleep $DIR_QUOTA_FLUSH_INTERVAL
used=$(./juicefs quota get $META_URL --path /d 2>&1 | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "50%" ]] && echo "used should be 50%" && exit 1 || true
dd if=/dev/zero of=/jfs/d/test3 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs quota check $META_URL --path /d --strict
}
test_dir_inodes(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
./juicefs quota set $META_URL --path /d --inodes 1000
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
set +x
for i in {1..1000}; do
echo $i > /jfs/d/test$i > /dev/null
done
set -x
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee /jfs/d/test1001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
rm -rf error.log
./juicefs quota set $META_URL --path /d --inodes 2000
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
set +x
for i in {1001..2000}; do
echo $i | tee /jfs/d/test$i > /dev/null
done
set -x
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee /jfs/d/test2001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
rm /jfs/d/test1 -rf
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee /jfs/d/test2001
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs quota check $META_URL --path /d --strict
}
test_sub_dir(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
umount_jfs /jfs $META_URL
./juicefs mount -d $META_URL --subdir /d /jfs --heartbeat 2
size=$(df -h /jfs | grep "JuiceFS" | awk '{print $2}')
[[ $size != "1.0G" ]] && echo "size should be 1.0G" && exit 1 || true
inodes=$(df -ih /jfs | grep "JuiceFS" | awk '{print $2}')
[[ $inodes != "1000" ]] && echo "inodes should be 1000" && exit 1 || true
dd if=/dev/zero of=/jfs/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
rm /jfs/test1 -rf
sleep $DIR_QUOTA_FLUSH_INTERVAL
set +x
for i in {1..1000}; do
echo $i | tee /jfs/test$i > /dev/null
done
set -x
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo $i | tee /jfs/test1001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
./juicefs quota check $META_URL --path /d --strict
}
test_dump_load(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
./juicefs dump --log-level error $META_URL --fast > dump.json
umount_jfs /jfs $META_URL
python3 .github/scripts/flush_meta.py $META_URL
./juicefs load $META_URL dump.json
./juicefs mount $META_URL /jfs -d --heartbeat 5
dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
rm /jfs/d/test1 -rf
sleep $DIR_QUOTA_FLUSH_INTERVAL
set +x
for i in {1..1000}; do
echo $i | tee /jfs/d/test$i > /dev/null
done
set -x
sleep 3s
echo a | tee /jfs/d/test1001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
./juicefs quota check $META_URL --path /d --strict
}
test_hard_link(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
dd if=/dev/zero of=/jfs/file bs=1G count=1
./juicefs quota set $META_URL --path /d --capacity 2
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
ln /jfs/file /jfs/d/test2
sleep $DIR_QUOTA_FLUSH_INTERVAL
ln /jfs/file /jfs/d/test3 2>error.log && echo "hard link should fail on out of space" && exit 1 || true
grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs quota check $META_URL --path /d --strict
}
test_check_and_repair_quota(){
prepare_test
./juicefs format $META_URL myjfs
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /jfs/d
./juicefs quota set $META_URL --path /d --capacity 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
pid=$(ps -ef | grep "juicefs mount" | grep -v grep | awk '{print $2}')
kill -9 $pid
sleep $DIR_QUOTA_FLUSH_INTERVAL
# ./juicefs quota check $META_URL --path /d --strict && echo "quota check should fail" && exit 1 || true
./juicefs quota check $META_URL --path /d --strict --repair
./juicefs quota check $META_URL --path /d --strict
}
wait_until()
{
key=$1
value=$2
echo "wait until $key becomes $value"
wait_seconds=15
for i in $(seq 1 $wait_seconds); do
if [ "$key" == "ifree" ]; then
expect_value=$(df -ih /jfs | grep JuiceFS | awk '{print $4}')
elif [ "$key" == "avail_size" ]; then
expect_value=$(df h /jfs | grep JuiceFS | awk '{print $4}')
fi
if [ "$expect_value" == "$value" ]; then
echo "$key becomes $value" && return 0
fi
echo "wait until $key becomes $value" && sleep 1s
done
echo "wait until $key becomes $value failed after $wait_seconds seconds" && exit 1
}
prepare_ug_quota_test()
{
prepare_test
./juicefs format $META_URL myjfs
./juicefs config $META_URL --user-group-quota
./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
}
resolve_test_users()
{
if [[ -n "$TEST_USER_1" ]] && [[ -n "$TEST_USER_2" ]]; then
return 0
fi
TEST_USER_1=""
TEST_USER_2=""
for candidate in nobody daemon bin; do
if id "$candidate" >/dev/null 2>&1; then
candidate_uid=$(id -u "$candidate")
candidate_gid=$(id -g "$candidate")
if [[ "$candidate_uid" == "0" ]] || [[ "$candidate_gid" == "0" ]]; then
continue
fi
if [[ -z "$TEST_USER_1" ]]; then
TEST_USER_1="$candidate"
TEST_UID_1=$candidate_uid
TEST_GID_1=$candidate_gid
elif [[ "$candidate_uid" != "$TEST_UID_1" ]]; then
TEST_USER_2="$candidate"
TEST_UID_2=$candidate_uid
TEST_GID_2=$candidate_gid
break
fi
fi
done
create_temp_user()
{
idx=$1
if ! command -v useradd >/dev/null 2>&1; then
return 1
fi
name="jfs-quota-test-${idx}-${RANDOM}"
if ! useradd -M -s /usr/sbin/nologin "$name" >/dev/null 2>&1; then
return 1
fi
uid=$(id -u "$name" 2>/dev/null || echo 0)
gid=$(id -g "$name" 2>/dev/null || echo 0)
if [[ "$uid" == "0" ]] || [[ "$gid" == "0" ]]; then
userdel -f "$name" >/dev/null 2>&1 || true
return 1
fi
echo "$name:$uid:$gid"
return 0
}
if [[ -z "$TEST_USER_1" ]] || [[ -z "$TEST_USER_2" ]]; then
if [[ "$(id -u)" != "0" ]]; then
echo "cannot find two non-root users for user/group quota tests"
return 1
fi
for i in 1 2 3 4; do
info=$(create_temp_user "$i") || continue
name=$(echo "$info" | cut -d: -f1)
uid=$(echo "$info" | cut -d: -f2)
gid=$(echo "$info" | cut -d: -f3)
if [[ -z "$TEST_USER_1" ]]; then
TEST_USER_1="$name"
TEST_UID_1=$uid
TEST_GID_1=$gid
elif [[ -z "$TEST_USER_2" ]] && [[ "$uid" != "$TEST_UID_1" ]]; then
TEST_USER_2="$name"
TEST_UID_2=$uid
TEST_GID_2=$gid
break
fi
done
fi
if [[ -z "$TEST_USER_1" ]] || [[ -z "$TEST_USER_2" ]]; then
echo "cannot find two non-root users for user/group quota tests"
return 1
fi
echo "test users: $TEST_USER_1($TEST_UID_1:$TEST_GID_1), $TEST_USER_2($TEST_UID_2:$TEST_GID_2)"
}
run_as_user_cmd()
{
user=$1
shift
cmd="$*"
if [[ "$(id -un)" == "$user" ]]; then
bash -c "$cmd"
return $?
fi
if command -v sudo >/dev/null 2>&1; then
sudo -n -u "$user" bash -c "$cmd" && return 0 || true
fi
if command -v runuser >/dev/null 2>&1; then
runuser -u "$user" -- bash -c "$cmd" && return 0 || true
fi
if command -v su >/dev/null 2>&1; then
su -s /bin/bash "$user" -c "$cmd" && return 0 || true
fi
echo "cannot run command as user $user"
return 1
}
set_quota_by_username()
{
username=$1
capacity=$2
inodes=$3
uid=$(id -u "$username")
./juicefs quota set $META_URL --uid "$uid" --capacity "$capacity" --inodes "$inodes"
}
test_user_group_quota_set_get_list_delete(){
prepare_ug_quota_test
resolve_test_users || return 0
./juicefs quota set $META_URL --uid "$TEST_UID_1" --capacity 1 --inodes 20
./juicefs quota set $META_URL --gid "$TEST_GID_1" --capacity 1 --inodes 20
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
./juicefs quota get $META_URL --uid "$TEST_UID_1" 2>&1 | tee uid_quota.log
grep "UID:$TEST_UID_1" uid_quota.log || (echo "uid quota should exist" && exit 1)
./juicefs quota get $META_URL --gid "$TEST_GID_1" 2>&1 | tee gid_quota.log
grep "GID:$TEST_GID_1" gid_quota.log || (echo "gid quota should exist" && exit 1)
./juicefs quota list $META_URL 2>&1 | tee quota_list.log
grep "UID:$TEST_UID_1" quota_list.log || (echo "uid quota should be listed" && exit 1)
grep "GID:$TEST_GID_1" quota_list.log || (echo "gid quota should be listed" && exit 1)
./juicefs quota delete $META_URL --uid "$TEST_UID_1"
./juicefs quota delete $META_URL --gid "$TEST_GID_1"
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
./juicefs quota list $META_URL 2>&1 | tee quota_list_after_delete.log
grep "UID:$TEST_UID_1" quota_list_after_delete.log && echo "uid quota should be deleted" && exit 1 || true
grep "GID:$TEST_GID_1" quota_list_after_delete.log && echo "gid quota should be deleted" && exit 1 || true
}
test_uid_quota_check_on_write_and_truncate(){
prepare_ug_quota_test
resolve_test_users || return 0
mkdir -p /jfs/uidq
chmod 777 /jfs/uidq
./juicefs quota set $META_URL --uid "$TEST_UID_2" --capacity 1 --inodes 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
run_as_user_cmd "$TEST_USER_2" "touch /jfs/uidq/inode1"
sleep $DIR_QUOTA_FLUSH_INTERVAL
run_as_user_cmd "$TEST_USER_2" "touch /jfs/uidq/inode2" 2>error.log && echo "second inode should fail for uid quota" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "uid inode quota check failed" && exit 1)
./juicefs quota set $META_URL --uid "$TEST_UID_2" --capacity 1 --inodes 10
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
rm -f /jfs/uidq/inode1
sleep $DIR_QUOTA_FLUSH_INTERVAL
run_as_user_cmd "$TEST_USER_2" "truncate -s 900M /jfs/uidq/space1"
sleep $DIR_QUOTA_FLUSH_INTERVAL
run_as_user_cmd "$TEST_USER_2" "truncate -s 1100M /jfs/uidq/space1" 2>error.log && echo "truncate should fail for uid capacity quota" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "uid capacity quota check failed" && exit 1)
}
test_gid_quota_check_on_write(){
prepare_ug_quota_test
resolve_test_users || return 0
mkdir -p /jfs/gidq
chmod 777 /jfs/gidq
./juicefs quota set $META_URL --gid "$TEST_GID_2" --inodes 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
run_as_user_cmd "$TEST_USER_2" "touch /jfs/gidq/file1"
sleep $DIR_QUOTA_FLUSH_INTERVAL
run_as_user_cmd "$TEST_USER_2" "touch /jfs/gidq/file2" 2>error.log && echo "second inode should fail for gid quota" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "gid inode quota check failed" && exit 1)
}
test_chown_transfer_user_group_quota(){
prepare_ug_quota_test
resolve_test_users || return 0
mkdir -p /jfs/chownq
chmod 777 /jfs/chownq
./juicefs quota set $META_URL --uid "$TEST_UID_1" --inodes 1
./juicefs quota set $META_URL --uid "$TEST_UID_2" --inodes 1
./juicefs quota set $META_URL --gid "$TEST_GID_1" --inodes 1
./juicefs quota set $META_URL --gid "$TEST_GID_2" --inodes 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
run_as_user_cmd "$TEST_USER_1" "touch /jfs/chownq/src_file"
sleep $DIR_QUOTA_FLUSH_INTERVAL
run_as_user_cmd "$TEST_USER_1" "touch /jfs/chownq/src_file2" 2>error.log && echo "user1 should exceed inode quota before chown" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "user1 pre-chown quota check failed" && exit 1)
chown "$TEST_UID_2:$TEST_GID_2" /jfs/chownq/src_file
sleep $DIR_QUOTA_FLUSH_INTERVAL
run_as_user_cmd "$TEST_USER_1" "touch /jfs/chownq/src_file2"
sleep $DIR_QUOTA_FLUSH_INTERVAL
run_as_user_cmd "$TEST_USER_2" "touch /jfs/chownq/dst_file" 2>error.log && echo "user2 should exceed inode quota after chown transfer" && exit 1 || true
grep -i "Disk quota exceeded" error.log || (echo "user2 post-chown quota check failed" && exit 1)
}
test_set_quota_by_username(){
prepare_ug_quota_test
resolve_test_users || return 0
set_quota_by_username "$TEST_USER_2" 1 10
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
uid=$(id -u "$TEST_USER_2")
./juicefs quota get $META_URL --uid "$uid" 2>&1 | tee username_quota.log
grep "UID:$uid" username_quota.log || (echo "quota set by username should be visible in uid quota" && exit 1)
./juicefs quota list $META_URL 2>&1 | tee username_quota_list.log
grep "UID:$uid" username_quota_list.log || (echo "quota set by username should be listed" && exit 1)
}
test_quota_list_uid_filter_regression(){
prepare_ug_quota_test
resolve_test_users || return 0
./juicefs quota set $META_URL --uid "$TEST_UID_1" --capacity 1 --inodes 3
./juicefs quota set $META_URL --uid "$TEST_UID_2" --capacity 1 --inodes 7
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
./juicefs quota list $META_URL --uid "$TEST_UID_1" 2>&1 | tee uid_filter_1.log
grep "UID:$TEST_UID_1" uid_filter_1.log || (echo "uid filter should show requested uid quota" && exit 1)
grep "UID:$TEST_UID_2" uid_filter_1.log && echo "uid filter should not include other uid quota" && exit 1 || true
uid_rows=$(grep -c "UID:" uid_filter_1.log || true)
[[ "$uid_rows" -ne 1 ]] && echo "uid filter should only return one UID row" && exit 1 || true
inodes_value=$(grep "UID:$TEST_UID_1" uid_filter_1.log | head -n1 | awk -F'|' '{gsub(/[[:space:]]/,"",$6); print $6}')
[[ "$inodes_value" != "3" ]] && echo "uid filter should return uid1 inodes=3" && exit 1 || true
./juicefs quota list $META_URL --uid "$TEST_UID_2" 2>&1 | tee uid_filter_2.log
grep "UID:$TEST_UID_2" uid_filter_2.log || (echo "uid filter should show requested uid quota" && exit 1)
grep "UID:$TEST_UID_1" uid_filter_2.log && echo "uid filter should not include other uid quota" && exit 1 || true
uid_rows=$(grep -c "UID:" uid_filter_2.log || true)
[[ "$uid_rows" -ne 1 ]] && echo "uid filter should only return one UID row" && exit 1 || true
inodes_value=$(grep "UID:$TEST_UID_2" uid_filter_2.log | head -n1 | awk -F'|' '{gsub(/[[:space:]]/,"",$6); print $6}')
[[ "$inodes_value" != "7" ]] && echo "uid filter should return uid2 inodes=7" && exit 1 || true
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command/random.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common.sh
[[ -z "$MAX_EXAMPLE" ]] && MAX_EXAMPLE=100
[[ -z "$STEP_COUNT" ]] && STEP_COUNT=50
[[ -z "$META1" ]] && META1=sqlite3
source .github/scripts/start_meta_engine.sh
start_meta_engine $META1
META_URL1=$(get_meta_url $META1)
[[ -z "$META2" ]] && META2=redis
source .github/scripts/start_meta_engine.sh
start_meta_engine $META2
META_URL2=$(get_meta_url $META2)
prepare_test()
{
meta_url=$1
mp=$2
volume=$3
shift 3
options=$@
umount_jfs $mp $meta_url
python3 .github/scripts/flush_meta.py $meta_url
rm -rf /var/jfs/$volume || true
rm -rf /var/jfsCache/$volume || true
./juicefs format $meta_url $volume $options
./juicefs mount -d $meta_url $mp
}
test_run_examples()
{
prepare_test $META_URL1 /tmp/jfs1 myjfs1 --enable-acl --trash-days 0
prepare_test $META_URL2 /tmp/jfs2 myjfs2 --enable-acl --trash-days 0
python3 .github/scripts/hypo/command_test.py
}
test_run_all()
{
prepare_test $META_URL1 /tmp/jfs1 myjfs1
prepare_test $META_URL2 /tmp/jfs2 myjfs2
CHECK_NLINK=false MAX_EXAMPLE=$MAX_EXAMPLE STEP_COUNT=$STEP_COUNT python3 .github/scripts/hypo/command.py
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/acl.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
test_modify_acl_config()
{
prepare_win_test
./juicefs.exe format $META_URL myjfs --trash-days 0
./juicefs.exe mount -d $META_URL z:
touch z:test
cmd.exe /c "icacls z:\test /grant Everyone:(R,W)" && echo "setfacl should failed" && exit 1
./juicefs.exe config $META_URL --enable-acl=true
./juicefs.exe umount z:
./juicefs.exe mount -d $META_URL z:
cmd.exe /c "icacls z:\test /grant Everyone:(R,W)"
./juicefs.exe config $META_URL --enable-acl=false && echo "should not disable acl" && exit 1 || true
./juicefs.exe config $META_URL | grep EnableACL | grep "true" || (echo "EnableACL should be true" && exit 1)
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/clone.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
test_clone_with_jfs_source()
{
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z:
ls /z
[[ ! -d /z/juicefs ]] && git clone https://github.com/juicedata/juicefs.git /z/juicefs --depth 1
ls /z/juicefs
do_clone true
echo "test clone without --preserve"
# do_clone false
}
do_clone()
{
is_preserve=$1
cmd.exe /c "taskkill /F /IM git.exe 2>nul || ver>nul"
cmd.exe /c "rmdir /s /q z:\juicefs1 2>nul || ver>nul"
cmd.exe /c "rmdir /s /q z:\juicefs2 2>nul || ver>nul"
sleep 1
[[ "$is_preserve" == "true" ]] && preserve="--preserve" || preserve=""
cp -r /z/juicefs /z/juicefs1 $preserve
./juicefs.exe clone /z/juicefs /z/juicefs2 $preserve
diff -ur /z/juicefs1 /z/juicefs2 --no-dereference
# CURRENT_DIR=$(pwd)
# cmd.exe /c "dir /s /b /a z:\juicefs1" > "${CURRENT_DIR}/log1"
# cmd.exe /c "dir /s /b /a z:\juicefs2" > "${CURRENT_DIR}/log2"
# diff -u "${CURRENT_DIR}/log1" "${CURRENT_DIR}/log2"
# rm -f "${CURRENT_DIR}/log1" "${CURRENT_DIR}/log2"
}
test_clone_with_small_files(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z:
mkdir /z/test
for i in $(seq 1 2000); do
echo $i > /z/test/$i
done
./juicefs.exe clone /z/test /z/test1
diff -ur /z/test1 /z/test1
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/debug.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META=redis://127.0.0.1:6379/1
check_debug_file(){
files=("system-info.log" "juicefs.log" "config.txt" "stats.txt" "stats.5s.txt")
debug_dir="debug"
if [ ! -d "$debug_dir" ]; then
echo "error:no debug dir"
exit 1
fi
all_files_exist=true
for file in "${files[@]}"; do
exist=`find "$debug_dir" -name $file | wc -l`
if [ "$exist" == 0 ]; then
echo "no $file"
all_files_exist=false
fi
done
if [ "$all_files_exist" = true ]; then
echo "pass"
else
exit 1
fi
}
test_debug_juicefs(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z:
dd if=/dev/urandom of=/z/bigfile bs=1M count=1024
./juicefs.exe debug z:
check_debug_file
find debug -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'
./juicefs.exe rmr /z/bigfile
}
test_debug_abnormal_juicefs(){
rm -rf debug | true
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z:
dd if=/dev/urandom of=/z/bigfile bs=1M count=1024
killall -9 redis-server | true
./juicefs.exe debug z:
check_debug_file
./juicefs.exe rmr /z/bigfile
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/dump_load.sh
================================================
#!/bin/bash -ex
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
[[ -z "$SEED" ]] && SEED=$(date +%s)
HEARTBEAT_INTERVAL=2
DIR_QUOTA_FLUSH_INTERVAL=4
[[ -z "$BINARY" ]] && BINARY=false
[[ -z "$FAST" ]] && FAST=false
trap "echo random seed is $SEED" EXIT
test_dump_load_sustained_file(){
prepare_win_test
./juicefs.exe format $META_URL myjfs --trash-days 0
./juicefs.exe mount -d $META_URL z:
file_count=100
for i in $(seq 1 $file_count); do
touch /z/file$i
exec {fd}<>/z/file$i
echo fd is $fd
fds[$i]=$fd
rm /z/file$i
done
./juicefs.exe dump $META_URL dump.json $(get_dump_option)
for i in $(seq 1 $file_count); do
fd=${fds[$i]}
exec {fd}>&-
done
if [[ "$BINARY" == "true" ]]; then
sustained=$(./juicefs.exe load dump.json --binary --stat | grep sustained | awk -F"|" '{print $2}')
else
sustained=$(jq '.Sustained[].inodes | length' dump.json)
fi
echo "sustained file count: $sustained"
./juicefs.exe umount z:
prepare_win_test
./juicefs.exe load $META_URL dump.json $(get_load_option)
./juicefs.exe mount -d $META_URL z:
}
test_dump_load_with_copy_file_range(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z:
rm -rf /tmp/test
dd if=/dev/zero of=/tmp/test bs=1M count=1024
cp /tmp/test /z/test
node .github/scripts/copyFile.js /z/test /z/test1
./juicefs.exe dump $META_URL dump.json $(get_dump_option)
./juicefs.exe umount z:
redis-cli -h 127.0.0.1 -p 6379 -n 1 FLUSHDB
./juicefs.exe load $META_URL dump.json $(get_load_option)
./juicefs.exe mount -d $META_URL z:
compare_md5sum /tmp/test /z/test1
}
test_dump_load_with_quota(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /z/d
./juicefs.exe quota set $META_URL --path //d --inodes 1000 --capacity 1
./juicefs.exe dump --log-level error $META_URL $(get_dump_option) > dump.json
./juicefs.exe umount z:
redis-cli -h 127.0.0.1 -p 6379 -n 1 FLUSHDB
./juicefs.exe load $META_URL dump.json $(get_load_option)
./juicefs.exe mount $META_URL z: -d --heartbeat $HEARTBEAT_INTERVAL
./juicefs.exe quota get $META_URL --path //d
dd if=/dev/zero of=/z/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
dd if=/dev/zero of=/z/d/test2 bs=1G count=1 2>error.log && echo "write should fail on out of space" && exit 1 || true
}
get_dump_option(){
if [[ "$BINARY" == "true" ]]; then
option="--binary"
elif [[ "$FAST" == "true" ]]; then
option="--fast"
else
option=""
fi
echo $option
}
get_load_option(){
if [[ "$BINARY" == "true" ]]; then
option="--binary"
else
option=""
fi
echo $option
}
prepare_test(){
umount_jfs /jfs $META_URL
umount_jfs /jfs2 sqlite3://test2.db
python3 .github/scripts/flush_meta.py $META_URL
rm test2.db -rf
rm -rf /var/jfs/myjfs || true
mc rm --force --recursive myminio/test || true
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/fsck.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
test_sync_dir_stat()
{
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z:
./juicefs.exe mdtest $META_URL //d --depth 15 --dirs 2 --files 100 --threads 10 &
pid=$!
sleep 15s
kill -9 $pid
./juicefs.exe info -r /z/d
./juicefs.exe info -r /z/d --strict
./juicefs.exe fsck $META_URL --path //d --sync-dir-stat --repair -r
./juicefs.exe info -r /z/d | tee info1.log
./juicefs.exe info -r /z/d --strict | tee info2.log
diff info1.log info2.log
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/gateway.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
wget https://dl.min.io/client/mc/release/windows-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O mc.exe
chmod +x mc.exe
export MINIO_ROOT_USER=admin
export MINIO_ROOT_PASSWORD=admin123
export MINIO_REFRESH_IAM_INTERVAL=3s
prepare_test()
{
kill_gateway 9001 || true
kill_gateway 9002 || true
prepare_win_test
}
kill_gateway() {
port=$1
for pid in $(netstat -ano | findstr ":$port" | findstr "LISTENING" | awk '{print $5}'); do
taskkill //F //PID $pid
done
}
trap 'kill_gateway 9001; kill_gateway 9002' EXIT
start_two_gateway()
{
prepare_test
./juicefs.exe format $META_URL myjfs --trash-days 0
./juicefs.exe mount -d $META_URL z:
export MINIO_ROOT_USER=admin
export MINIO_ROOT_PASSWORD=admin123
nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag --log=gateway1.log &
sleep 1
nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag --log=gateway2.log &
sleep 2
./mc.exe alias set gateway1 http://127.0.0.1:9001 admin admin123
./mc.exe alias set gateway2 http://127.0.0.1:9002 admin admin123
}
test_user_management()
{
prepare_test
start_two_gateway
./mc.exe admin user add gateway1 user1 admin123
sleep 5
user=$(./mc.exe admin user list gateway2 | grep user1) || true
if [ -z "$user" ]
then
echo "user synchronization error"
exit 1
fi
./mc.exe mb gateway1/test1
./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
if ./mc.exe cp mc.exe gateway1_user1/test1/file1
then
echo "By default, the user has no read and write permission"
exit 1
fi
./mc.exe admin policy set gateway1 readwrite user=user1
if ./mc.exe cp mc.exe gateway1_user1/test1/file1
then
echo "readwrite policy can read and write objects"
else
echo "set readwrite policy fail"
exit 1
fi
./mc.exe cp gateway2/test1/file1 .
compare_md5sum file1 mc.exe
./mc.exe admin user disable gateway1 user1
./mc.exe admin user remove gateway2 user1
sleep 5
user=$(./mc.exe admin user list gateway1 | grep user1) || true
if [ ! -z "$user" ]
then
echo "remove user user1 fail"
echo $user
exit 1
fi
}
test_group_management()
{
prepare_test
start_two_gateway
./mc.exe admin user add gateway1 user1 admin123
./mc.exe admin user add gateway1 user2 admin123
./mc.exe admin user add gateway1 user3 admin123
./mc.exe admin group add gateway1 testcents user1 user2 user3
result=$(./mc.exe admin group info gateway1 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3" ]
then
echo "error,result is '$result'"
exit 1
fi
./mc.exe admin policy set gateway1 readwrite group=testcents
sleep 5
./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
./mc.exe mb gateway1/test1
if ./mc.exe cp mc.exe gateway1_user1/test1/file1
then
echo "readwrite policy can read write"
else
echo "the readwrite group has no read and write permission"
exit 1
fi
./mc.exe admin policy set gateway1 readonly group=testcents
sleep 5
if ./mc.exe cp mc.exe gateway1_user1/test1/file1
then
echo "readonly group policy can not write"
exit 1
else
echo "the readonly group has no write permission"
fi
./mc.exe admin group remove gateway1 testcents user1 user2 user3
./mc.exe admin group remove gateway1 testcents
}
test_mult_gateways_set_group()
{
prepare_test
start_two_gateway
./mc.exe admin user add gateway1 user1 admin123
./mc.exe admin user add gateway1 user2 admin123
./mc.exe admin user add gateway1 user3 admin123
./mc.exe admin group add gateway1 testcents user1 user2 user3
./mc.exe admin group disable gateway2 testcents
sleep 5
result=$(./mc.exe admin group info gateway2 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3" ]
then
echo "error,result is '$result'"
exit 1
fi
./mc.exe admin group enable gateway1 testcents
./mc.exe admin user add gateway1 user4 admin123
./mc.exe admin group add gateway1 testcents user4
sleep 1
./mc.exe admin group disable gateway2 testcents
sleep 5
result=$(./mc.exe admin group info gateway2 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3,user4" ]
then
echo "error,result is '$result'"
exit 1
fi
}
test_user_svcacct_add()
{
prepare_test
start_two_gateway
./mc.exe admin user add gateway1 user1 admin123
./mc.exe admin policy set gateway1 consoleAdmin user=user1
./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
./mc.exe admin user svcacct add gateway1_user1 user1 --access-key 12345678 --secret-key 12345678
./mc.exe admin user svcacct info gateway1_user1 12345678
./mc.exe admin user svcacct set gateway1_user1 12345678 --secret-key 123456789
./mc.exe alias set svcacct1 http://127.0.0.1:9001 12345678 123456789
./mc.exe mb svcacct1/test1
if ./mc.exe cp mc.exe svcacct1/test1/file1
then
echo "svcacct user consoleAdmin policy can read write"
else
echo "the svcacct user has no read and write permission"
exit 1
fi
./mc.exe admin user svcacct disable gateway1_user1 12345678
./mc.exe admin user svcacct rm gateway1_user1 12345678
}
test_user_admin_svcacct_add()
{
prepare_test
start_two_gateway
./mc.exe admin user add gateway1 user1 admin123
./mc.exe admin policy set gateway1 readwrite user=user1
./mc.exe admin user svcacct add gateway1 user1 --access-key 12345678 --secret-key 12345678
./mc.exe admin user svcacct info gateway1 12345678
./mc.exe admin user svcacct set gateway1 12345678 --secret-key 12345678910
./mc.exe alias set svcacct1 http://127.0.0.1:9001 12345678 12345678910
./mc.exe mb svcacct1/test1
if ./mc.exe cp mc.exe svcacct1/test1/file1
then
echo "amdin user can do svcacct "
else
echo "the svcacct user has no read and write permission"
exit 1
fi
./mc.exe admin user svcacct disable gateway1 12345678
./mc.exe admin user svcacct rm gateway1 12345678
}
test_user_sts()
{
prepare_test
start_two_gateway
./mc.exe admin user add gateway1 user1 admin123
./mc.exe admin policy set gateway1 consoleAdmin user=user1
./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
git clone https://github.com/juicedata/minio.git -b gateway-1.1
./mc.exe mb gateway1_user1/test1
./mc.exe cp mc.exe gateway1_user1/test1/mc
cd minio
go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1 -d
go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1
cd -
./mc.exe admin user remove gateway1 user1
}
test_change_credentials()
{
prepare_test
start_two_gateway
./mc.exe mb gateway1/test1
./mc.exe cp mc.exe gateway1/test1/file1
kill_gateway 9001 || true
kill_gateway 9002 || true
export MINIO_ROOT_USER=newadmin
export MINIO_ROOT_PASSWORD=newadmin123
export MINIO_ROOT_USER_OLD=admin
export MINIO_ROOT_PASSWORD_OLD=admin123
nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag --log=gateway1.log &
nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag --log=gateway2.log &
sleep 5
./mc.exe alias set gateway1 http://127.0.0.1:9001 newadmin newadmin123
./mc.exe alias set gateway2 http://127.0.0.1:9002 newadmin newadmin123
./mc.exe cp gateway1/test1/file1 file1
./mc.exe cp gateway2/test1/file1 file2
compare_md5sum file1 mc.exe
compare_md5sum file2 mc.exe
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/gc.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
test_delay_delete_slice_after_compaction(){
if [[ "$META_URL" != redis* ]]; then
echo "this test only runs for redis meta engine"
return
fi
prepare_win_test
./juicefs.exe format $META_URL myjfs --trash-days 1
./juicefs.exe mount -d $META_URL z: --no-usage-report
redis-cli save
# don't skip files when gc compact
export JFS_SKIPPED_TIME=1
./juicefs.exe gc --compact --delete $META_URL
./juicefs.exe fsck $META_URL
}
test_gc_trash_slices(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z: --no-usage-report
PATH1=test PATH2=z:\\test python3 .github/scripts/random_read_write.py
./juicefs.exe status --more $META_URL
./juicefs.exe config $META_URL --trash-days 0 --yes
./juicefs.exe gc $META_URL
./juicefs.exe gc $META_URL --delete
./juicefs.exe status --more $META_URL
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/profile.sh
================================================
#!/bin/bash -e
source .github/scripts/common/common_win.sh
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
test_profile()
{
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z:
./juicefs.exe mdtest $META_URL //d --depth 3 --dirs 3 --files 10 --threads 5
timeout 5s ./juicefs profile /z/.accesslog || EXIT_CODE=$?
if [ "$EXIT_CODE" = "124" ]; then
echo "juicefs profile success"
else
echo "juicefs profile failed"
exit 1
fi
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/command-win/quota.sh
================================================
#!/bin/bash -e
[[ -z "$META_URL" ]] && META_URL=redis://127.0.0.1:6379/1
HEARTBEAT_INTERVAL=3
HEARTBEAT_SLEEP=3
DIR_QUOTA_FLUSH_INTERVAL=4
VOLUME_QUOTA_FLUSH_INTERVAL=2
source .github/scripts/common/common_win.sh
test_total_capacity()
{
prepare_win_test
./juicefs.exe format $META_URL myjfs --capacity 1
./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL --debug
dd if=/dev/zero of=/z/test1 bs=1G count=1
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
dd if=/dev/zero of=/z/test2 bs=1G count=1 && echo "dd should fail on out of space" && exit 1 || true
rm /z/test1 -rf
./juicefs.exe rmr /z/.trash
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
dd if=/dev/zero of=/z/test2 bs=104857600 count=1
}
test_total_inodes(){
prepare_win_test
./juicefs.exe format $META_URL myjfs --inodes 1000
./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL
set +x
for i in {1..1000}; do
echo $i | tee /z/test$i > /dev/null
done
set -x
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee /z/test1001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
# grep "No space left on device" error.log
./juicefs.exe config $META_URL --inodes 2000
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
set +x
for i in {1001..2000}; do
echo $i | tee /z/test$i > /dev/null || (df -i /z && ls /z/ -l | wc -l && exit 1)
done
set -x
sleep $VOLUME_QUOTA_FLUSH_INTERVAL
echo a | tee /z/test2001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
}
test_remove_and_restore(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /z/d
./juicefs.exe quota set $META_URL --path //d --capacity 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/z/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs.exe quota get $META_URL --path //d 2>&1 | tee quota.log
used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
dd if=/dev/zero of=/z/d/test2 bs=1G count=1 && echo "write should fail on out of space" && exit 1 || true
echo "remove test1" && rm /z/d/test* -rf
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs.exe quota get $META_URL --path //d 2>&1 | tee quota.log
used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "0%" ]] && echo "used should be 0%" && exit 1 || true
trash_dir=$(ls /z/.trash)
./juicefs.exe restore $META_URL $trash_dir --put-back
./juicefs.exe quota get $META_URL --path //d 2>&1 | tee quota.log
used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/z/d/test2 bs=1G count=1 && echo "write should fail on out of space" && exit 1 || true
echo "remove test1" && rm /z/d/test1 -rf
}
test_dir_capacity(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /z/d
./juicefs.exe quota set $META_URL --path //d --capacity 1
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/z/d/test1 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs.exe quota get $META_URL --path //d
used=$(./juicefs.exe quota get $META_URL --path //d 2>&1 | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
dd if=/dev/zero of=/z/d/test2 bs=1G count=1 && echo "echo should fail on out of space" && exit 1 || true
./juicefs.exe quota set $META_URL --path //d --capacity 2
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
dd if=/dev/zero of=/z/d/test2 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
dd if=/dev/zero of=/z/d/test3 bs=1G count=1 && echo "echo should fail on out of space" && exit 1 || true
rm -rf /z/d/test1
sleep $DIR_QUOTA_FLUSH_INTERVAL
used=$(./juicefs.exe quota get $META_URL --path //d 2>&1 | grep "/d" | awk -F'|' '{print $5}' | tr -d '[:space:]')
[[ $used != "50%" ]] && echo "used should be 50%" && exit 1 || true
dd if=/dev/zero of=/z/d/test3 bs=1G count=1
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs.exe quota check $META_URL --path //d --strict
}
test_dir_inodes(){
prepare_win_test
./juicefs.exe format $META_URL myjfs
./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL
mkdir -p /z/d
./juicefs.exe quota set $META_URL --path //d --inodes 1000
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
set +x
for i in {1..1000}; do
echo $i > /z/d/test$i > /dev/null
done
set -x
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee /jfs/d/test1001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
#grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
rm -rf error.log
./juicefs.exe quota set $META_URL --path //d --inodes 2000
sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))
set +x
for i in {1001..2000}; do
echo $i | tee /z/d/test$i > /dev/null
done
set -x
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee /z/d/test2001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
#grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
rm /z/d/test1 -rf
sleep $DIR_QUOTA_FLUSH_INTERVAL
echo a | tee /z/d/test2001
sleep $DIR_QUOTA_FLUSH_INTERVAL
./juicefs.exe quota check $META_URL --path //d --strict
}
source .github/scripts/common/run_test.sh && run_test $@
================================================
FILE: .github/scripts/common/common.sh
================================================
#!/bin/bash -e
# Common variables and initialization
init_platform() {
case "$(uname -s)" in
Darwin*) PLATFORM="mac";;
Linux*) PLATFORM="linux";;
*) PLATFORM="unknown"
esac
# Install jq if missing
if ! command -v jq &> /dev/null; then
case "$PLATFORM" in
mac) brew install jq;;
linux) .github/scripts/apt_install.sh jq;;
*) echo "Unsupported platform"; exit 1
esac
fi
}
# Platform-agnostic functions with internal branching
prepare_test() {
case "$PLATFORM" in
mac)
./juicefs umount ~/jfs || true
umount_jfs ~/jfs "$META_URL"
sleep 1
python3 .github/scripts/flush_meta.py "$META_URL"
rm -rf ~/.juicefs/local/myjfs/ || true
rm -rf ~/.juicefs/cache || true
;;
linux)
umount_jfs /jfs "$META_URL"
python3 .github/scripts/flush_meta.py "$META_URL"
rm -rf /var/jfs/myjfs || true
rm -rf /var/jfsCache/myjfs || true
;;
esac
}
umount_jfs() {
local mp=$1
local meta_url=$2
[[ -z "$mp" ]] && echo "mount point is empty" && exit 1
[[ -z "$meta_url" ]] && echo "meta url is empty" && exit 1
echo "umount_jfs $mp $meta_url"
[[ ! -f "$mp/.config" ]] && return
ls -l "$mp/.config"
local status_log="status.log"
./juicefs status --log-level error "$meta_url" 2>/dev/null | tee "$status_log"
local pids
pids=$(jq --arg mp "$mp" '.Sessions[] | select(.MountPoint == $mp) | .ProcessID' "$status_log")
[[ -z "$pids" ]] && cat "$status_log" && echo "pid is empty" && return
echo "umount is $mp, pids are $pids"
for pid in $pids; do
case "$PLATFORM" in
mac)
if mount | grep -q "$mp"; then
diskutil unmount "$mp" || umount "$mp"
fi
;;
linux)
umount -l "$mp"
;;
esac
done
for pid in $pids; do
wait_mount_process_killed "$pid" 60
done
}
wait_mount_process_killed() {
local pid=$1
local wait_seconds=$2
[[ -z "$pid" ]] && echo "pid is empty" && exit 1
[[ -z "$wait_seconds" ]] && echo "wait_seconds is empty" && exit 1
echo "waiting for mount process $pid to exit within $wait_seconds seconds"
for i in $(seq 1 "$wait_seconds"); do
case "$PLATFORM" in
mac)
if ! ps -p "$pid" > /dev/null; then
echo "mount process is killed"
break
fi
;;
linux)
count=$(ps -ef | grep "juicefs mount" | awk '{print $2}' | grep "^$pid$" | wc -l)
if [ "$count" -eq 0 ]; then
echo "mount process is killed"
break
fi
;;
esac
if [ "$i" -eq "$wait_seconds" ]; then
case "$PLATFORM" in
mac) ps -p "$pid";;
linux) ps -ef | grep "juicefs mount" | grep -v "grep";;
esac
echo "<FATAL>: mount process is not killed after $wait_seconds"
exit 1
fi
sleep 1
done
}
compare_md5sum() {
local file1=$1
local file2=$2
case "$PLATFORM" in
mac)
md51=$(md5 -q "$file1")
md52=$(md5 -q "$file2")
;;
linux)
md51=$(md5sum "$file1" | awk '{print $1}')
md52=$(md5sum "$file2" | awk '{print $1}')
;;
esac
if [ "$md51" != "$md52" ]; then
echo "md5 are different: $file1 ($md51) vs $file2 ($md52)"
exit 1
fi
}
wait_command_success() {
local command=$1
local expected=$2
local timeout=${3:-30}
echo "waiting for command success: cmd='$command', expected='$expected', timeout=$timeout"
for i in $(seq 1 "$timeout"); do
result=$(eval "$command" 2>/dev/null | tr -d ' ')
echo "attempt $i: result=$result"
if [[ "$result" == "$expected" ]]; then
echo "command succeeded"
return 0
fi
if [ "$i" -eq "$timeout" ]; then
eval "$command"
echo "command failed after $timeout attempts: $command"
exit 1
fi
sleep 1
done
}
# macOS specific helper (only defined but used when needed)
ensure_directory() {
[[ "$PLATFORM" != "mac" ]] && return
local dir=$1
if [[ ! -d "$dir" ]]; then
echo "Creating directory: $dir"
mkdir -p "$dir"
fi
}
# Initialize platform detection
init_platform
# Make functions available to subprocesses
export -f prepare_test umount_jfs wait_mount_process_killed compare_md5sum wait_command_success ensure_directory
export PLATFORM META_URL
================================================
FILE: .github/scripts/common/common_win.sh
================================================
#!/bin/bash -e
prepare_win_test()
{
net start redisredis || true
./juicefs.exe umount z: || true
rm -rf C:\jfs\local/myjfs/ || true
rm -rf C:\jfsCache\local/myjfs/ || true
uuid=$(./juicefs.exe status $META_URL | grep UUID | cut -d '"' -f 4) || true
./juicefs.exe destroy --force $META_URL $uuid || true
redis-cli -h 127.0.0.1 -p 6379 -n 1 FLUSHDB
}
compare_md5sum(){
file1=$1
file2=$2
md51=$(md5sum $file1 | awk '{print $1}')
md52=$(md5sum $file2 | awk '{print $1}')
# echo md51 is $md51, md52 is $md52
if [ "$md51" != "$md52" ] ; then
echo "md5 are different: md51 is $md51, md52 is $md52"
exit 1
fi
}
================================================
FILE: .github/scripts/common/run_test.sh
================================================
#!/bin/bash -e
run_one_test()
{
test=$1
test=${test%%(*}
echo -e "\033[0;34mStart Test: $test\033[0m"
START_TIME=$(date +%s)
set +e
( set -e; "${test}" )
EXIT_STATUS=$?
set -e
echo $test exit with $EXIT_STATUS
END_TIME=$(date +%s)
ELAPSED_TIME=$((END_TIME - START_TIME))
if [[ $EXIT_STATUS -eq 0 ]]; then
echo -e "\033[0;34mFinish Test: $test in $ELAPSED_TIME seconds\033[0m"
else
echo -e "\033[0;31mTest Failed: $0 $test in $ELAPSED_TIME seconds\033[0m"
exit 1
fi
}
run_test(){
START_TIME_ALL=$(date +%s)
if [[ ! -z "$@" ]]; then
# run test functions passed by arguments
for test in "$@"; do
if declare -F "$test" > /dev/null; t
gitextract_vo3z05bo/
├── .autocorrectrc
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug-report.md
│ │ ├── enhancement.md
│ │ └── support.md
│ ├── actions/
│ │ ├── build/
│ │ │ └── action.yml
│ │ ├── cancel-outdate-runs/
│ │ │ └── action.yml
│ │ ├── mount-coverage-dir/
│ │ │ └── action.yml
│ │ ├── upload-coverage/
│ │ │ └── action.yml
│ │ └── upload-total-coverage/
│ │ └── action.yml
│ ├── scripts/
│ │ ├── apt_install.sh
│ │ ├── cache.sh
│ │ ├── chaos/
│ │ │ ├── dynamic.yaml
│ │ │ ├── juicefs-csi-driver.Dockerfile
│ │ │ ├── juicefs.Dockerfile
│ │ │ ├── minio.yaml
│ │ │ ├── pvc.yaml
│ │ │ ├── redis.yaml
│ │ │ ├── sc.yaml
│ │ │ └── workflow.yaml
│ │ ├── check_juicefs_log.sh
│ │ ├── cmptree.py
│ │ ├── command/
│ │ │ ├── acl.sh
│ │ │ ├── clone.sh
│ │ │ ├── config.sh
│ │ │ ├── debug.sh
│ │ │ ├── dump_load.sh
│ │ │ ├── dump_load_bench.sh
│ │ │ ├── dump_load_cross_meta.sh
│ │ │ ├── format.sh
│ │ │ ├── fsck.sh
│ │ │ ├── gateway-random.sh
│ │ │ ├── gateway.sh
│ │ │ ├── gc.sh
│ │ │ ├── graceful_upgrade.sh
│ │ │ ├── info.sh
│ │ │ ├── interface.sh
│ │ │ ├── mount.sh
│ │ │ ├── quota.sh
│ │ │ └── random.sh
│ │ ├── command-win/
│ │ │ ├── acl.sh
│ │ │ ├── clone.sh
│ │ │ ├── debug.sh
│ │ │ ├── dump_load.sh
│ │ │ ├── fsck.sh
│ │ │ ├── gateway.sh
│ │ │ ├── gc.sh
│ │ │ ├── profile.sh
│ │ │ └── quota.sh
│ │ ├── common/
│ │ │ ├── common.sh
│ │ │ ├── common_win.sh
│ │ │ └── run_test.sh
│ │ ├── compare_results.sh
│ │ ├── copyFile.js
│ │ ├── fio.sh
│ │ ├── flush_meta.py
│ │ ├── fsrand.py
│ │ ├── hypo/
│ │ │ ├── command.py
│ │ │ ├── command_op.py
│ │ │ ├── command_test.py
│ │ │ ├── common.py
│ │ │ ├── context.py
│ │ │ ├── file.py
│ │ │ ├── file_op.py
│ │ │ ├── file_test.py
│ │ │ ├── fs.py
│ │ │ ├── fs_acl_test.py
│ │ │ ├── fs_op.py
│ │ │ ├── fs_sdk_test.py
│ │ │ ├── fs_test.py
│ │ │ ├── readme.md
│ │ │ ├── s3.py
│ │ │ ├── s3_contant.py
│ │ │ ├── s3_op.py
│ │ │ ├── s3_strategy.py
│ │ │ ├── s3_test.py
│ │ │ ├── stats.py
│ │ │ ├── strategy.py
│ │ │ ├── sync.py
│ │ │ └── sync_test.py
│ │ ├── mutate/
│ │ │ ├── check_coverage.py
│ │ │ ├── check_skip_by_comment.py
│ │ │ ├── how_to_use_mutate_test.md
│ │ │ ├── modify_sdk_pom.py
│ │ │ ├── mutest.sh
│ │ │ ├── mutesting.py
│ │ │ ├── parse_black_list.py
│ │ │ ├── parse_job_total.py
│ │ │ ├── parse_mutate_log.py
│ │ │ ├── parse_test_cases.py
│ │ │ ├── query_report.py
│ │ │ └── save_report.py
│ │ ├── perf/
│ │ │ ├── ai.sh
│ │ │ ├── ai_format_benchmark.py
│ │ │ ├── compare_ai.sh
│ │ │ ├── compare_mdtest_fio.sh
│ │ │ └── mdtest_fio.sh
│ │ ├── prepare_db.sh
│ │ ├── pysdk/
│ │ │ ├── bench.py
│ │ │ └── pysdk_test.py
│ │ ├── random_read_write.py
│ │ ├── save_benchmark.sh
│ │ ├── setup-hdfs.sh
│ │ ├── ssh/
│ │ │ ├── Dockerfile
│ │ │ └── docker-compose.yml
│ │ ├── start_meta_engine.sh
│ │ ├── sync/
│ │ │ ├── sync.sh
│ │ │ ├── sync_cluster.sh
│ │ │ ├── sync_fsrand.sh
│ │ │ └── sync_minio.sh
│ │ ├── test-mac/
│ │ │ ├── mac_commands.sh
│ │ │ └── start_meta_engine.sh
│ │ ├── testVersionCompatible.py
│ │ ├── upload_coverage_report.sh
│ │ ├── utils.py
│ │ └── wins_fs_test.py
│ └── workflows/
│ ├── bash/
│ │ ├── rm_fs
│ │ ├── rm_list.sh
│ │ └── rm_syscalls
│ ├── cache.yml
│ ├── cancel_outdate_runs.yml
│ ├── chaos.yml
│ ├── check-doc.yaml
│ ├── codeql-analysis.yml
│ ├── command-win.yml
│ ├── command.yml
│ ├── command2.yml
│ ├── compile.yml
│ ├── coverage-report.yml
│ ├── dependency-review.yml
│ ├── dockerfile-sftp
│ ├── dump_load.yml
│ ├── dump_load_bench.yml
│ ├── dump_load_cross_meta.yml
│ ├── fsrand.yml
│ ├── fsspec.yml
│ ├── gateway-random.yml
│ ├── gateway.yml
│ ├── integrationtests.yml
│ ├── ltpfs.yml
│ ├── ltpsyscalls.yml
│ ├── mutate-test-sdk.yml
│ ├── mutate-test.yml
│ ├── perf-test.yml
│ ├── permission-check.yaml
│ ├── pjdfstest.yml
│ ├── pysdk.yml
│ ├── random-test.yml
│ ├── release.yml
│ ├── resources/
│ │ ├── core-site.xml
│ │ ├── load-balancer.conf
│ │ ├── sync-options.txt
│ │ ├── tpcds_datagen.scala
│ │ ├── tpcds_run.scala
│ │ ├── vdbench_big_file.conf
│ │ ├── vdbench_long_run.conf
│ │ └── vdbench_small_file.conf
│ ├── rmfiles.yml
│ ├── sdktest.yml
│ ├── sync.yml
│ ├── unit-random-tests.yml
│ ├── unittests.yml
│ ├── vdbench.yml
│ ├── verify.yml
│ ├── version_compatible_hypo.yml
│ ├── wintest.yml
│ └── xattr.yml
├── .gitignore
├── .golangci.yml
├── .goreleaser.yml
├── .markdownlint-cli2.jsonc
├── .pre-commit-config.yaml
├── ADOPTERS.md
├── ADOPTERS_CN.md
├── CODEOWNERS
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── Makefile
├── README.md
├── README_CN.md
├── check-changed.sh
├── cmd/
│ ├── bench.go
│ ├── bench_test.go
│ ├── clone.go
│ ├── compact.go
│ ├── compact_test.go
│ ├── config.go
│ ├── config_test.go
│ ├── debug.go
│ ├── debug_test.go
│ ├── debug_unix.go
│ ├── debug_windows.go
│ ├── destroy.go
│ ├── dump.go
│ ├── dump_test.go
│ ├── flags.go
│ ├── flags_test.go
│ ├── format.go
│ ├── format_test.go
│ ├── fsck.go
│ ├── fsck_test.go
│ ├── gateway.go
│ ├── gateway_noop.go
│ ├── gc.go
│ ├── gc_test.go
│ ├── info.go
│ ├── info_test.go
│ ├── integration_test.go
│ ├── load.go
│ ├── main.go
│ ├── main_test.go
│ ├── mdtest.go
│ ├── mount.go
│ ├── mount_test.go
│ ├── mount_unix.go
│ ├── mount_windows.go
│ ├── objbench.go
│ ├── object.go
│ ├── object_test.go
│ ├── passfd.go
│ ├── printsid.go
│ ├── profile.go
│ ├── quota.go
│ ├── restore.go
│ ├── restore_test.go
│ ├── rmr.go
│ ├── rmr_test.go
│ ├── stats.go
│ ├── status.go
│ ├── status_test.go
│ ├── summary.go
│ ├── sync.go
│ ├── sync_test.go
│ ├── umount.go
│ ├── version.go
│ ├── warmup.go
│ ├── warmup_test.go
│ ├── webdav.go
│ └── webdav_noop.go
├── codecov.yml
├── deploy/
│ └── juicefs-s3-gateway.yaml
├── docs/
│ ├── README.md
│ ├── en/
│ │ ├── administration/
│ │ │ ├── destroy.md
│ │ │ ├── fault_diagnosis_and_analysis.md
│ │ │ ├── metadata/
│ │ │ │ ├── _category_.yml
│ │ │ │ ├── etcd_best_practices.md
│ │ │ │ ├── fdb_best_practices.md
│ │ │ │ ├── mysql_best_practices.md
│ │ │ │ ├── postgresql_best_practices.md
│ │ │ │ ├── redis_best_practices.md
│ │ │ │ └── tikv_best_practices.md
│ │ │ ├── metadata_dump_load.md
│ │ │ ├── monitoring.md
│ │ │ ├── mount_at_boot.md
│ │ │ ├── status_check_and_maintenance.md
│ │ │ ├── sync_accounts_between_multiple_hosts.md
│ │ │ ├── troubleshooting.md
│ │ │ └── upgrade.md
│ │ ├── benchmark/
│ │ │ ├── benchmark.md
│ │ │ ├── fio.md
│ │ │ ├── mdtest.md
│ │ │ ├── metadata_engines_benchmark.md
│ │ │ └── performance_evaluation_guide.md
│ │ ├── community/
│ │ │ ├── _roadmap.md
│ │ │ ├── adopters.md
│ │ │ ├── articles.md
│ │ │ ├── integrations.md
│ │ │ └── usage_tracking.md
│ │ ├── deployment/
│ │ │ ├── _share_via_nfs.md
│ │ │ ├── _share_via_smb.md
│ │ │ ├── automation.md
│ │ │ ├── hadoop_java_sdk.md
│ │ │ ├── how_to_use_on_kubernetes.md
│ │ │ ├── juicefs_on_docker.md
│ │ │ ├── nfs.md
│ │ │ ├── production_deployment_recommendations.md
│ │ │ ├── python_sdk.md
│ │ │ ├── samba.md
│ │ │ └── webdav.md
│ │ ├── development/
│ │ │ ├── contributing_guide.md
│ │ │ └── internals.md
│ │ ├── faq.md
│ │ ├── getting-started/
│ │ │ ├── for_distributed.md
│ │ │ ├── installation.md
│ │ │ └── standalone.md
│ │ ├── grafana_template.json
│ │ ├── guide/
│ │ │ ├── cache.md
│ │ │ ├── clone.md
│ │ │ ├── dir-stats.md
│ │ │ ├── gateway.md
│ │ │ ├── quota.md
│ │ │ └── sync.md
│ │ ├── introduction/
│ │ │ ├── README.md
│ │ │ ├── architecture.md
│ │ │ ├── comparison/
│ │ │ │ ├── _category_.yml
│ │ │ │ ├── juicefs_vs_3fs.md
│ │ │ │ ├── juicefs_vs_alluxio.md
│ │ │ │ ├── juicefs_vs_cephfs.md
│ │ │ │ ├── juicefs_vs_glusterfs.md
│ │ │ │ ├── juicefs_vs_lustre.md
│ │ │ │ ├── juicefs_vs_s3fs.md
│ │ │ │ ├── juicefs_vs_s3ql.md
│ │ │ │ └── juicefs_vs_seaweedfs.md
│ │ │ └── io_processing.md
│ │ ├── reference/
│ │ │ ├── _common_options.mdx
│ │ │ ├── command_reference.mdx
│ │ │ ├── fuse_mount_options.md
│ │ │ ├── how_to_set_up_metadata_engine.md
│ │ │ ├── how_to_set_up_object_storage.md
│ │ │ ├── p8s_metrics.md
│ │ │ ├── posix_compatibility.md
│ │ │ ├── redis-csc.md
│ │ │ └── spec-limits.md
│ │ ├── release_notes.md
│ │ ├── security/
│ │ │ ├── encryption.md
│ │ │ ├── posix_acl.md
│ │ │ └── trash.md
│ │ └── tutorials/
│ │ ├── aliyun.md
│ │ ├── aws.md
│ │ ├── digitalocean.md
│ │ ├── juicefs_on_colab.md
│ │ ├── juicefs_on_k3s.md
│ │ ├── juicefs_on_kubesphere.md
│ │ ├── juicefs_on_rancher.md
│ │ ├── juicefs_on_wsl.md
│ │ ├── qcloud.md
│ │ └── windows.md
│ └── zh_cn/
│ ├── administration/
│ │ ├── destroy.md
│ │ ├── fault_diagnosis_and_analysis.md
│ │ ├── metadata/
│ │ │ ├── _category_.yml
│ │ │ ├── etcd_best_practices.md
│ │ │ ├── fdb_best_practices.md
│ │ │ ├── mysql_best_practices.md
│ │ │ ├── postgresql_best_practices.md
│ │ │ ├── redis_best_practices.md
│ │ │ └── tikv_best_practices.md
│ │ ├── metadata_dump_load.md
│ │ ├── monitoring.md
│ │ ├── mount_at_boot.md
│ │ ├── status_check_and_maintenance.md
│ │ ├── sync_accounts_between_multiple_hosts.md
│ │ ├── troubleshooting.md
│ │ └── upgrade.md
│ ├── benchmark/
│ │ ├── benchmark.md
│ │ ├── fio.md
│ │ ├── mdtest.md
│ │ ├── metadata_engines_benchmark.md
│ │ └── performance_evaluation_guide.md
│ ├── community/
│ │ ├── _roadmap.md
│ │ ├── adopters.md
│ │ ├── articles.md
│ │ ├── integrations.md
│ │ └── usage_tracking.md
│ ├── deployment/
│ │ ├── _share_via_nfs.md
│ │ ├── _share_via_smb.md
│ │ ├── automation.md
│ │ ├── hadoop_java_sdk.md
│ │ ├── how_to_use_on_kubernetes.md
│ │ ├── juicefs_on_docker.md
│ │ ├── nfs.md
│ │ ├── production_deployment_recommendations.md
│ │ ├── python_sdk.md
│ │ ├── samba.md
│ │ └── webdav.md
│ ├── development/
│ │ ├── contributing_guide.md
│ │ └── internals.md
│ ├── faq.md
│ ├── getting-started/
│ │ ├── for_distributed.md
│ │ ├── installation.md
│ │ └── standalone.md
│ ├── guide/
│ │ ├── cache.md
│ │ ├── clone.md
│ │ ├── dir-stats.md
│ │ ├── gateway.md
│ │ ├── quota.md
│ │ └── sync.md
│ ├── introduction/
│ │ ├── README.md
│ │ ├── architecture.md
│ │ ├── comparison/
│ │ │ ├── _category_.yml
│ │ │ ├── juicefs_vs_3fs.md
│ │ │ ├── juicefs_vs_alluxio.md
│ │ │ ├── juicefs_vs_cephfs.md
│ │ │ ├── juicefs_vs_glusterfs.md
│ │ │ ├── juicefs_vs_lustre.md
│ │ │ ├── juicefs_vs_s3fs.md
│ │ │ ├── juicefs_vs_s3ql.md
│ │ │ └── juicefs_vs_seaweedfs.md
│ │ └── io_processing.md
│ ├── reference/
│ │ ├── _common_options.mdx
│ │ ├── command_reference.mdx
│ │ ├── fuse_mount_options.md
│ │ ├── how_to_set_up_metadata_engine.md
│ │ ├── how_to_set_up_object_storage.md
│ │ ├── p8s_metrics.md
│ │ ├── posix_compatibility.md
│ │ └── spec-limits.md
│ ├── release_notes.md
│ ├── security/
│ │ ├── encryption.md
│ │ ├── posix_acl.md
│ │ └── trash.md
│ └── tutorials/
│ ├── aliyun.md
│ ├── aws.md
│ ├── digitalocean.md
│ ├── juicefs_on_colab.md
│ ├── juicefs_on_k3s.md
│ ├── juicefs_on_kubesphere.md
│ ├── juicefs_on_rancher.md
│ ├── juicefs_on_wsl.md
│ ├── qcloud.md
│ └── windows.md
├── go.mod
├── go.sum
├── hack/
│ ├── autocomplete/
│ │ ├── bash_autocomplete
│ │ └── zsh_autocomplete
│ ├── builder/
│ │ ├── Dockerfile
│ │ └── sdk.Dockerfile
│ └── winfsp_headers/
│ ├── fuse.h
│ ├── fuse_common.h
│ ├── fuse_opt.h
│ └── winfsp_fuse.h
├── integration/
│ ├── Makefile
│ ├── ioctl_test.sh
│ └── s3gateway_test.sh
├── main.go
├── package.json
├── pkg/
│ ├── acl/
│ │ ├── acl.go
│ │ ├── cache.go
│ │ └── cache_test.go
│ ├── chunk/
│ │ ├── cache_eviction.go
│ │ ├── cached_store.go
│ │ ├── cached_store_test.go
│ │ ├── chunk.go
│ │ ├── disk_cache.go
│ │ ├── disk_cache_state.go
│ │ ├── disk_cache_state_test.go
│ │ ├── disk_cache_test.go
│ │ ├── mem_cache.go
│ │ ├── metrics.go
│ │ ├── page.go
│ │ ├── page_test.go
│ │ ├── prefetch.go
│ │ ├── prefetch_test.go
│ │ ├── singleflight.go
│ │ ├── singleflight_test.go
│ │ ├── utils_darwin.go
│ │ ├── utils_linux.go
│ │ ├── utils_unix.go
│ │ ├── utils_unix_test.go
│ │ └── utils_windows.go
│ ├── compress/
│ │ ├── compress.go
│ │ └── compress_test.go
│ ├── fs/
│ │ ├── fs.go
│ │ ├── fs_test.go
│ │ ├── http.go
│ │ ├── http_test.go
│ │ └── metrics.go
│ ├── fuse/
│ │ ├── context.go
│ │ ├── device_darwin.go
│ │ ├── device_linux.go
│ │ ├── fuse.go
│ │ ├── fuse_darwin.go
│ │ ├── fuse_linux.go
│ │ ├── fuse_test.go
│ │ ├── gidcache.go
│ │ └── utils.go
│ ├── gateway/
│ │ ├── gateway.go
│ │ └── gateway_test.go
│ ├── meta/
│ │ ├── backup.go
│ │ ├── base.go
│ │ ├── base_test.go
│ │ ├── benchmarks_test.go
│ │ ├── config.go
│ │ ├── config_test.go
│ │ ├── context.go
│ │ ├── dump.go
│ │ ├── info.go
│ │ ├── info_test.go
│ │ ├── interface.go
│ │ ├── interface_test.go
│ │ ├── load_dump_test.go
│ │ ├── lua_scripts.go
│ │ ├── metadata-sub.sample
│ │ ├── metadata.sample
│ │ ├── openfile.go
│ │ ├── pb/
│ │ │ ├── backup.pb.go
│ │ │ └── backup.proto
│ │ ├── quota.go
│ │ ├── random_test.go
│ │ ├── redis.go
│ │ ├── redis_bak.go
│ │ ├── redis_csc.go
│ │ ├── redis_csc_test.go
│ │ ├── redis_lock.go
│ │ ├── slice.go
│ │ ├── sql.go
│ │ ├── sql_bak.go
│ │ ├── sql_lock.go
│ │ ├── sql_mysql.go
│ │ ├── sql_pg.go
│ │ ├── sql_sqlite.go
│ │ ├── sql_test.go
│ │ ├── status.go
│ │ ├── tkv.go
│ │ ├── tkv_badger.go
│ │ ├── tkv_bak.go
│ │ ├── tkv_etcd.go
│ │ ├── tkv_fdb.go
│ │ ├── tkv_fdb_test.go
│ │ ├── tkv_lock.go
│ │ ├── tkv_mem.go
│ │ ├── tkv_prefix.go
│ │ ├── tkv_test.go
│ │ ├── tkv_tikv.go
│ │ ├── utils.go
│ │ ├── utils_darwin.go
│ │ ├── utils_linux.go
│ │ ├── utils_test.go
│ │ └── utils_windows.go
│ ├── metric/
│ │ └── metrics.go
│ ├── object/
│ │ ├── azure.go
│ │ ├── b2.go
│ │ ├── bos.go
│ │ ├── bunny.go
│ │ ├── ceph.go
│ │ ├── checksum.go
│ │ ├── checksum_test.go
│ │ ├── cifs.go
│ │ ├── cos.go
│ │ ├── dragonfly.go
│ │ ├── encrypt.go
│ │ ├── encrypt_test.go
│ │ ├── eos.go
│ │ ├── etcd.go
│ │ ├── file.go
│ │ ├── file_darwin.go
│ │ ├── file_linux.go
│ │ ├── file_unix.go
│ │ ├── file_unix_test.go
│ │ ├── file_windows.go
│ │ ├── filesystem_test.go
│ │ ├── gluster.go
│ │ ├── gluster_test.go
│ │ ├── gs.go
│ │ ├── hdfs.go
│ │ ├── hdfs_kerberos.go
│ │ ├── ibmcos.go
│ │ ├── interface.go
│ │ ├── ks3.go
│ │ ├── mem.go
│ │ ├── minio.go
│ │ ├── nfs.go
│ │ ├── object_storage.go
│ │ ├── object_storage_test.go
│ │ ├── obs.go
│ │ ├── oos.go
│ │ ├── oss.go
│ │ ├── prefix.go
│ │ ├── qingstor.go
│ │ ├── qiniu.go
│ │ ├── redis.go
│ │ ├── response_attrs.go
│ │ ├── response_attrs_test.go
│ │ ├── restful.go
│ │ ├── restful_test.go
│ │ ├── s3.go
│ │ ├── s3_test.go
│ │ ├── scw.go
│ │ ├── sftp.go
│ │ ├── sharding.go
│ │ ├── space.go
│ │ ├── sql.go
│ │ ├── sql_mysql.go
│ │ ├── sql_pg.go
│ │ ├── sql_sqlite.go
│ │ ├── swift.go
│ │ ├── tikv.go
│ │ ├── tos.go
│ │ ├── ufile.go
│ │ ├── wasabi.go
│ │ └── webdav.go
│ ├── sync/
│ │ ├── cluster.go
│ │ ├── cluster_test.go
│ │ ├── config.go
│ │ ├── download.go
│ │ ├── download_test.go
│ │ ├── sync.go
│ │ └── sync_test.go
│ ├── usage/
│ │ ├── usage.go
│ │ └── usage_test.go
│ ├── utils/
│ │ ├── alloc.go
│ │ ├── alloc_test.go
│ │ ├── buffer.go
│ │ ├── buffer_test.go
│ │ ├── clock_test.go
│ │ ├── clock_unix.go
│ │ ├── clock_windows.go
│ │ ├── cond.go
│ │ ├── cond_test.go
│ │ ├── errors.go
│ │ ├── general.go
│ │ ├── humanize.go
│ │ ├── logger.go
│ │ ├── logger_syslog.go
│ │ ├── logger_test.go
│ │ ├── logger_windows.go
│ │ ├── memusage.go
│ │ ├── memusage_test.go
│ │ ├── memusage_windows.go
│ │ ├── proc_title.go
│ │ ├── proc_title_noop.go
│ │ ├── progress.go
│ │ ├── progress_test.go
│ │ ├── rusage.go
│ │ ├── rusage_test.go
│ │ ├── rusage_windows.go
│ │ ├── utils.go
│ │ ├── utils_darwin.go
│ │ ├── utils_linux.go
│ │ ├── utils_test.go
│ │ ├── utils_unix.go
│ │ └── utils_windows.go
│ ├── version/
│ │ ├── .gitattributes
│ │ ├── version.go
│ │ └── version_test.go
│ ├── vfs/
│ │ ├── accesslog.go
│ │ ├── accesslog_test.go
│ │ ├── backup.go
│ │ ├── backup_test.go
│ │ ├── compact.go
│ │ ├── compact_test.go
│ │ ├── fill.go
│ │ ├── fill_test.go
│ │ ├── handle.go
│ │ ├── helpers.go
│ │ ├── helpers_test.go
│ │ ├── internal.go
│ │ ├── reader.go
│ │ ├── vfs.go
│ │ ├── vfs_test.go
│ │ ├── vfs_unix.go
│ │ ├── vfs_windows.go
│ │ └── writer.go
│ ├── win/
│ │ ├── ldap.go
│ │ └── sid.go
│ └── winfsp/
│ ├── log.go
│ └── winfs.go
├── rfcs/
│ └── 1-dir-used-statistics.md
└── sdk/
├── java/
│ ├── .gitignore
│ ├── Makefile
│ ├── conf/
│ │ ├── contract/
│ │ │ └── juicefs.xml
│ │ ├── core-site.xml
│ │ └── log4j.properties
│ ├── kerberos.sh
│ ├── libjfs/
│ │ ├── Makefile
│ │ ├── bridge.go
│ │ ├── bridge_test.go
│ │ ├── callback.c
│ │ ├── guid.go
│ │ ├── guid_unix.go
│ │ ├── guid_windows.go
│ │ ├── kerberos.go
│ │ ├── main.go
│ │ ├── remote_write.go
│ │ └── remote_write_test.go
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── io/
│ │ │ └── juicefs/
│ │ │ ├── FlinkFileSystemFactory.java
│ │ │ ├── JuiceFS.java
│ │ │ ├── JuiceFileSystem.java
│ │ │ ├── JuiceFileSystemImpl.java
│ │ │ ├── KiteDataLoader.java
│ │ │ ├── Main.java
│ │ │ ├── bench/
│ │ │ │ ├── AccumulatingReducer.java
│ │ │ │ ├── IOMapperBase.java
│ │ │ │ ├── NNBench.java
│ │ │ │ └── TestDFSIO.java
│ │ │ ├── exception/
│ │ │ │ └── QuotaExceededException.java
│ │ │ ├── kerberos/
│ │ │ │ ├── AuthCredential.java
│ │ │ │ ├── JuiceFSDelegationTokenIdentifier.java
│ │ │ │ ├── JuiceFSTokenRenewer.java
│ │ │ │ └── KerberosUtil.java
│ │ │ ├── metrics/
│ │ │ │ └── JuiceFSInstrumentation.java
│ │ │ ├── permission/
│ │ │ │ ├── RangerAdminRefresher.java
│ │ │ │ ├── RangerConfig.java
│ │ │ │ ├── RangerJfsAccessRequest.java
│ │ │ │ ├── RangerJfsPlugin.java
│ │ │ │ ├── RangerJfsResource.java
│ │ │ │ ├── RangerPermissionChecker.java
│ │ │ │ ├── RangerPermissionContext.java
│ │ │ │ ├── RangerPluginCfg.java
│ │ │ │ └── RangerRules.java
│ │ │ ├── tools/
│ │ │ │ └── RangerDownloader.java
│ │ │ └── utils/
│ │ │ ├── AclTransformation.java
│ │ │ ├── BgTaskUtil.java
│ │ │ ├── BufferPool.java
│ │ │ ├── CallerContextUtil.java
│ │ │ ├── ConsistentHash.java
│ │ │ ├── FsNodesFetcher.java
│ │ │ ├── FsPermissionExtension.java
│ │ │ ├── NodesFetcher.java
│ │ │ ├── NodesFetcherBuilder.java
│ │ │ ├── PatchUtil.java
│ │ │ ├── PrestoNodesFetcher.java
│ │ │ ├── RedefineClassAgent.java
│ │ │ ├── ReflectionUtil.java
│ │ │ ├── SparkNodesFetcher.java
│ │ │ ├── SparkThriftNodesFetcher.java
│ │ │ └── YarnNodesFetcher.java
│ │ └── resources/
│ │ └── META-INF/
│ │ └── services/
│ │ ├── org.apache.flink.core.fs.FileSystemFactory
│ │ ├── org.apache.hadoop.security.token.TokenIdentifier
│ │ ├── org.apache.hadoop.security.token.TokenRenewer
│ │ └── org.kitesdk.data.spi.Loadable
│ └── test/
│ ├── java/
│ │ └── io/
│ │ └── juicefs/
│ │ ├── JuiceFileSystemBgTaskTest.java
│ │ ├── JuiceFileSystemTest.java
│ │ ├── acl/
│ │ │ └── TestAclCLI.java
│ │ ├── contract/
│ │ │ ├── JuiceFSContract.java
│ │ │ ├── TestAppend.java
│ │ │ ├── TestConcat.java
│ │ │ ├── TestCreate.java
│ │ │ ├── TestDelete.java
│ │ │ ├── TestGetFileStatus.java
│ │ │ ├── TestJuiceFileSystemContract.java
│ │ │ ├── TestMkdir.java
│ │ │ ├── TestOpen.java
│ │ │ ├── TestRename.java
│ │ │ ├── TestSeek.java
│ │ │ └── TestSetTimes.java
│ │ ├── kerberos/
│ │ │ └── KerberosTest.java
│ │ ├── permission/
│ │ │ ├── RangerAdminClientImpl.java
│ │ │ └── RangerPermissionCheckerTest.java
│ │ └── utils/
│ │ ├── BgTaskUtilTest.java
│ │ └── HashTest.java
│ ├── resources/
│ │ ├── hdfs-policies-tag.json
│ │ ├── hdfs-policies.json
│ │ ├── kerberos.cfg
│ │ ├── log4j.properties
│ │ └── testAclCLI.xml
│ └── test-spark.sh
└── python/
├── .gitignore
├── Dockerfile.builder
├── Dockerfile.builder.arm
├── Makefile
├── examples/
│ ├── ffrecord/
│ │ ├── dataloader.py
│ │ ├── dataset.py
│ │ ├── filereader.py
│ │ ├── filereader_dio.py
│ │ ├── main.py
│ │ └── readme.md
│ └── fsspec/
│ ├── main.py
│ └── readme.md
└── juicefs/
├── juicefs/
│ ├── __init__.py
│ ├── juicefs.py
│ └── spec.py
├── setup.py
└── tests/
├── __init__.py
└── test.py
Showing preview only (508K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (6054 symbols across 397 files)
FILE: .github/scripts/cmptree.py
class TreeComparator (line 39) | class TreeComparator(object):
method __init__ (line 40) | def __init__(self, dir1, dir2):
method compare (line 48) | def compare(self, p=""):
method compare_files (line 66) | def compare_files(self, d1, d2, files):
method compare_xattr (line 95) | def compare_xattr(self, f1, f2):
function info (line 105) | def info(s):
function warn (line 107) | def warn(s):
function fail (line 109) | def fail(s, exitcode = 1):
function main (line 112) | def main():
function __entry (line 147) | def __entry():
FILE: .github/scripts/fsrand.py
class Devnull (line 41) | class Devnull(object):
method write (line 42) | def write(self, *args):
class FsRandomizer (line 45) | class FsRandomizer(object):
method __init__ (line 46) | def __init__(self, path, count, seed):
method __stdout (line 56) | def __stdout(self, s):
method __stderr (line 58) | def __stderr(self, s):
method __getdir_recurse (line 60) | def __getdir_recurse(self, path):
method __getdir (line 70) | def __getdir(self):
method __getsubpath (line 75) | def __getsubpath(self, path):
method __gen_unicode_name (line 83) | def __gen_unicode_name(self, lower_limit=1, upper_limit=64):
method __gen_ascii_name (line 101) | def __gen_ascii_name(self, lower_limit=1, upper_limit=64):
method __newname (line 106) | def __newname(self):
method __newsubpath (line 115) | def __newsubpath(self, path):
method __newmode (line 120) | def __newmode(self, mode):
method __random_write (line 122) | def __random_write(self, file):
method __create (line 130) | def __create(self, path):
method __update (line 134) | def __update(self, path):
method randomize (line 138) | def randomize(self):
function info (line 240) | def info(s):
function warn (line 242) | def warn(s):
function fail (line 244) | def fail(s, exitcode = 1):
function main (line 247) | def main():
function __entry (line 274) | def __entry():
FILE: .github/scripts/hypo/command.py
class JuicefsCommandMachine (line 46) | class JuicefsCommandMachine(JuicefsMachine):
method __init__ (line 61) | def __init__(self):
method get_default_rootdir1 (line 64) | def get_default_rootdir1(self):
method get_default_rootdir2 (line 67) | def get_default_rootdir2(self):
method equal (line 70) | def equal(self, result1, result2):
method get_client_version (line 83) | def get_client_version(self, mount):
method should_run (line 87) | def should_run(self, rule):
method info (line 101) | def info(self, entry, raw=True, recuisive=False, strict=True, user='ro...
method rmr (line 110) | def rmr(self, entry, user='root'):
method status (line 118) | def status(self):
method warmup (line 127) | def warmup(self, entry, user='root'):
method gc (line 138) | def gc(self, compact=False, delete=False, user='root'):
method fsck (line 150) | def fsck(self, entry, repair=False, recuisive=False, user='root'):
method clone (line 163) | def clone(self, entry, parent, new_entry_name, preserve=False, user='r...
method dump (line 176) | def dump(self, folder, fast, skip_trash, threads, keep_secret_key, use...
method dump_load_dump (line 192) | def dump_load_dump(self, folder, fast=False, skip_trash=False, threads...
method diff (line 200) | def diff(self, str1:str, str2:str):
method trash_list (line 209) | def trash_list(self, user='root'):
method restore (line 220) | def restore(self, put_back, threads, user='root'):
method compact (line 231) | def compact(self, entry, threads, user='root'):
method config (line 247) | def config(self, capacity, inodes, trash_days, enable_acl, encrypt_sec...
method teardown (line 252) | def teardown(self):
FILE: .github/scripts/hypo/command_op.py
class CommandOperation (line 20) | class CommandOperation:
method __init__ (line 23) | def __init__(self, name, mp, root_dir):
method guess_password (line 30) | def guess_password(self, meta_url):
method get_meta_url (line 38) | def get_meta_url(self, mp):
method run_cmd (line 53) | def run_cmd(self, command:str, stderr=subprocess.STDOUT) -> str:
method seteuid (line 67) | def seteuid(self, user):
method handleException (line 71) | def handleException(self, e, action, path, **kwargs):
method get_raw (line 84) | def get_raw(self, size:str):
method parse_info (line 91) | def parse_info(self, info: str):
method do_info (line 121) | def do_info(self, entry, strict=True, user='root', raw=True, recuisive...
method do_rmr (line 141) | def do_rmr(self, entry, user='root'):
method do_status (line 154) | def do_status(self):
method do_dump (line 166) | def do_dump(self, folder, fast=False, skip_trash=False, threads=1, kee...
method get_dump_cmd (line 182) | def get_dump_cmd(self, meta_url, subdir, fast, skip_trash, keep_secret...
method do_dump_load_dump (line 192) | def do_dump_load_dump(self, folder, fast=False, skip_trash=False, thre...
method clean_dump (line 212) | def clean_dump(self, dump):
method do_warmup (line 232) | def do_warmup(self, entry, user='root'):
method do_gc (line 242) | def do_gc(self, compact:bool, delete:bool, user:str='root'):
method do_clone (line 256) | def do_clone(self, entry, parent, new_entry_name, preserve:bool, user:...
method do_fsck (line 270) | def do_fsck(self, entry, repair=False, recuisive=False, user='root'):
method do_trash_list (line 285) | def do_trash_list(self, user='root'):
method do_restore (line 300) | def do_restore(self, put_back, threads, user='root'):
method do_trash_restore (line 315) | def do_trash_restore(self, index, user='root'):
method do_compact (line 332) | def do_compact(self, entry, threads=5, user='root'):
method do_config (line 342) | def do_config(self, capacity, inodes, trash_days, enable_acl, encrypt_...
FILE: .github/scripts/hypo/command_test.py
class TestCommand (line 4) | class TestCommand(unittest.TestCase):
method test_dump (line 5) | def test_dump(self):
method skip_test_info (line 17) | def skip_test_info(self):
method test_clone (line 24) | def test_clone(self):
method test_config (line 32) | def test_config(self):
method test_clone_4834 (line 38) | def test_clone_4834(self):
FILE: .github/scripts/hypo/common.py
function red (line 10) | def red(s):
function replace (line 13) | def replace(src, old, new):
function run_cmd (line 22) | def run_cmd(command: str) -> str:
function setup_logger (line 40) | def setup_logger(log_file_path, logger_name, log_level='INFO'):
function is_jfs (line 69) | def is_jfs(path):
function get_root (line 74) | def get_root(path):
function get_volume_name (line 87) | def get_volume_name(path):
function get_zones (line 98) | def get_zones(dir):
function get_acl (line 114) | def get_acl(abspath: str):
function support_acl (line 122) | def support_acl(path):
function get_stat_field (line 142) | def get_stat_field(st: os.stat_result):
function create_group (line 153) | def create_group(groupname):
function create_user (line 160) | def create_user(user):
function clean_dir (line 168) | def clean_dir(dir):
function compare_content (line 179) | def compare_content(dir1, dir2):
function compare_stat (line 197) | def compare_stat(dir1, dir2):
function compare_acl (line 212) | def compare_acl(dir1, dir2):
FILE: .github/scripts/hypo/context.py
class Context (line 2) | class Context:
method __init__ (line 3) | def __init__(self, root_dir:str, mp:str) -> None:
FILE: .github/scripts/hypo/file.py
class JuicefsDataMachine (line 24) | class JuicefsDataMachine(RuleBasedStateMachine):
method __init__ (line 44) | def __init__(self):
method equal (line 48) | def equal(self, result1, result2):
method parse_error_message (line 63) | def parse_error_message(self, err):
method should_run (line 73) | def should_run(self, rule):
method init_folders (line 80) | def init_folders(self):
method read (line 92) | def read(self, fd, length):
method write (line 102) | def write(self, fd, content):
method writelines (line 111) | def writelines(self, fd, lines):
method seek (line 121) | def seek(self, fd, offset, whence):
method tell (line 128) | def tell(self, fd):
method close (line 137) | def close(self, fd):
method flush_and_fsync (line 147) | def flush_and_fsync(self, fd):
method fallocate (line 157) | def fallocate(self, fd, offset, length):
method readlines (line 164) | def readlines(self, fd):
method readline (line 171) | def readline(self, fd):
method truncate (line 181) | def truncate(self, fd, size):
method copy_file_range (line 194) | def copy_file_range(self, src, dst, src_offset, dst_offset, length):
method teardown (line 199) | def teardown(self):
FILE: .github/scripts/hypo/file_op.py
class FileOperation (line 30) | class FileOperation:
method __init__ (line 35) | def __init__(self, name, root_dir:str, mount_point=None, use_sdk:bool=...
method run_cmd (line 51) | def run_cmd(self, command:str) -> str:
method get_zones (line 65) | def get_zones(self):
method init_rootdir (line 68) | def init_rootdir(self):
method handleException (line 83) | def handleException(self, e, action, path, **kwargs):
method parse_pysdk_error (line 97) | def parse_pysdk_error(self, err:str):
method get_sdk_path (line 103) | def get_sdk_path(self, abspath):
method do_stat (line 106) | def do_stat(self, entry):
method do_create_file (line 123) | def do_create_file(self, file, content, mode, encoding, errors):
method do_open (line 142) | def do_open(self, file, mode, encoding, errors):
method do_write (line 159) | def do_write(self, fd, file, content):
method do_writelines (line 178) | def do_writelines(self, fd, file, lines):
method do_seek (line 199) | def do_seek(self, fd, file, offset, whence):
method do_tell (line 212) | def do_tell(self, fd, file):
method do_close (line 225) | def do_close(self, fd, file):
method do_flush_and_fsync (line 238) | def do_flush_and_fsync(self, fd, file):
method do_fallocate (line 256) | def do_fallocate(self, fd, file, offset, length):
method do_read (line 275) | def do_read(self, fd, file, length):
method do_readlines (line 299) | def do_readlines(self, fd, file):
method do_readline (line 328) | def do_readline(self, fd, file):
method do_truncate (line 358) | def do_truncate(self, fd, file, size):
method do_copy_file_range (line 379) | def do_copy_file_range(self, src_file, dst_file, src_fd, dst_fd, src_o...
method do_mmap_create (line 393) | def do_mmap_create(self, file, fd):
method do_mmap_read (line 406) | def do_mmap_read(self, file, mm: mmap.mmap, length):
method do_mmap_read_byte (line 420) | def do_mmap_read_byte(self, file, mm:mmap.mmap):
method do_mmap_read_line (line 433) | def do_mmap_read_line(self, file, mm:mmap.mmap):
method do_mmap_write (line 446) | def do_mmap_write(self, file, mm:mmap.mmap, content):
method do_mmap_write_byte (line 459) | def do_mmap_write_byte(self, file, mm: mmap.mmap, byte):
method do_mmap_move (line 472) | def do_mmap_move(self, file, mm: mmap.mmap, dest, src, count):
method do_mmap_resize (line 488) | def do_mmap_resize(self, file, mm: mmap.mmap):
method do_mmap_seek (line 505) | def do_mmap_seek(self, file, mm: mmap.mmap, offset, whence):
method do_mmap_size (line 521) | def do_mmap_size(self, file, mm: mmap.mmap):
method do_mmap_tell (line 534) | def do_mmap_tell(self, file, mm: mmap.mmap):
method do_mmap_flush (line 547) | def do_mmap_flush(self, file, mm: mmap.mmap):
method do_mmap_close (line 560) | def do_mmap_close(self, file, mm: mmap.mmap):
FILE: .github/scripts/hypo/file_test.py
class TestPySdk (line 4) | class TestPySdk(unittest.TestCase):
method test_issue_1522_1 (line 5) | def test_issue_1522_1(self):
method test_issue_1522_2 (line 13) | def test_issue_1522_2(self):
method test_issue_1523 (line 22) | def test_issue_1523(self):
method skip_test_issue_1533 (line 30) | def skip_test_issue_1533(self):
method skip_test_issue_1548 (line 41) | def skip_test_issue_1548(self):
method skip_test_issue_1548_2 (line 51) | def skip_test_issue_1548_2(self):
FILE: .github/scripts/hypo/fs.py
class JuicefsMachine (line 24) | class JuicefsMachine(RuleBasedStateMachine):
method init_folders (line 59) | def init_folders(self):
method create_users (line 64) | def create_users(self, users):
method get_default_rootdir1 (line 69) | def get_default_rootdir1(self):
method get_default_rootdir2 (line 72) | def get_default_rootdir2(self):
method __init__ (line 75) | def __init__(self):
method remove_dangling_files (line 92) | def remove_dangling_files(self):
method equal (line 97) | def equal(self, result1, result2):
method parse_error_message (line 119) | def parse_error_message(self, err):
method seteuid (line 129) | def seteuid(self, user):
method should_run (line 133) | def should_run(self, rule):
method stat (line 144) | def stat(self, entry, user = 'root'):
method lstat (line 154) | def lstat(self, entry, user = 'root'):
method exists (line 164) | def exists(self, entry, user = 'root'):
method open (line 176) | def open(self, file, flags, mode, user='root', umask=0o022):
method open2 (line 186) | def open2(self, file, mode, user='root'):
method write (line 201) | def write(self, file, offset, content, mode, whence, encoding=None, er...
method writelines (line 214) | def writelines(self, file, offset, lines, mode, whence, user='root'):
method fallocate (line 227) | def fallocate(self, file, offset, length, mode, user='root'):
method copy_file_range (line 240) | def copy_file_range(self, src, dst, src_offset, dst_offset, count, user):
method read (line 254) | def read(self, file, mode, offset, length, whence=os.SEEK_CUR, encodin...
method readlines (line 265) | def readlines(self, file, mode, offset, whence=os.SEEK_CUR, user='root'):
method readline (line 276) | def readline(self, file, mode, offset, whence=os.SEEK_CUR, user='root'):
method truncate (line 286) | def truncate(self, file, size, user='root'):
method create_file (line 298) | def create_file(self, parent, file_name, content, mode='xb', buffering...
method listdir (line 310) | def listdir(self, dir, user='root'):
method unlink (line 320) | def unlink(self, file, user='root'):
method rename_file (line 336) | def rename_file(self, entry, parent, new_entry_name, user='root', umas...
method rename_dir (line 352) | def rename_dir(self, entry, parent, new_entry_name, user='root', umask...
method copy_file (line 369) | def copy_file(self, entry, parent, new_entry_name, follow_symlinks, us...
method clone_cp_file (line 386) | def clone_cp_file(self, entry, parent, new_entry_name, preserve, user=...
method clone_cp_dir (line 406) | def clone_cp_dir(self, entry, parent, new_entry_name, preserve, user, ...
method mkdir (line 423) | def mkdir(self, parent, subdir, mode, user='root', umask=0o022):
method rmdir (line 436) | def rmdir(self, dir, user='root'):
method hardlink (line 453) | def hardlink(self, src_file, parent, link_file_name, user='root', umas...
method symlink (line 469) | def symlink(self, src_file, parent, link_file_name, user='root', umask...
method loop_symlink (line 484) | def loop_symlink(self, parent, link_file_name, user='root'):
method readlink (line 497) | def readlink(self, file, user='root'):
method set_xattr (line 510) | def set_xattr(self, file, name, value, flag, user='root'):
method get_xattr (line 523) | def get_xattr(self, xattr, user):
method list_xattr (line 531) | def list_xattr(self, file, user='root'):
method remove_xattr (line 541) | def remove_xattr(self, xattr, user='root'):
method change_groups (line 554) | def change_groups(self, user, group, groups):
method chmod (line 562) | def chmod(self, entry, mode, user='root'):
method get_acl (line 569) | def get_acl(self, entry):
method remove_acl (line 580) | def remove_acl(self, entry: str, option: str, user='root'):
method set_acl (line 604) | def set_acl(self, sudo_user, entry, user, user_perm, group, group_perm...
method utime (line 619) | def utime(self, entry, access_time, modify_time, follow_symlinks, user...
method chown (line 629) | def chown(self, entry, owner, user='root'):
method split_dir (line 639) | def split_dir(self, dir, vdirs):
method merge_dir (line 648) | def merge_dir(self, dir):
method rebalance_dir (line 661) | def rebalance_dir(self, dir, zone1, zone2, is_vdir, pysdk=True):
method rebalance_file (line 674) | def rebalance_file(self, file, zone1, zone2, pysdk=True):
method teardown (line 678) | def teardown(self):
FILE: .github/scripts/hypo/fs_acl_test.py
class TestFsrand2 (line 4) | class TestFsrand2(unittest.TestCase):
method test_acl_913 (line 5) | def test_acl_913(self):
method test_acl_1004 (line 15) | def test_acl_1004(self):
method test_acl_1006 (line 25) | def test_acl_1006(self):
method test_acl_1011 (line 35) | def test_acl_1011(self):
method test_acl_1015 (line 47) | def test_acl_1015(self):
method test_acl_1022 (line 56) | def test_acl_1022(self):
method test_acl_1044 (line 66) | def test_acl_1044(self):
method test_acl_4458 (line 77) | def test_acl_4458(self):
method test_acl_4472 (line 85) | def test_acl_4472(self):
method test_acl_4483 (line 96) | def test_acl_4483(self):
method test_acl_4496 (line 106) | def test_acl_4496(self):
method test_acl_4663 (line 118) | def test_acl_4663(self):
method skip_test_acl_2044 (line 126) | def skip_test_acl_2044(self):
FILE: .github/scripts/hypo/fs_op.py
class FsOperation (line 28) | class FsOperation:
method __init__ (line 32) | def __init__(self, name, root_dir:str, mount_point=None, use_sdk:bool=...
method get_client_for_rebalance (line 52) | def get_client_for_rebalance(self):
method run_cmd (line 62) | def run_cmd(self, command:str) -> str:
method get_zones (line 76) | def get_zones(self):
method init_rootdir (line 79) | def init_rootdir(self):
method seteuid (line 94) | def seteuid(self, user, action=''):
method reset_euid (line 103) | def reset_euid(self, action=''):
method handleException (line 110) | def handleException(self, e, action, path, **kwargs):
method parse_pysdk_error (line 124) | def parse_pysdk_error(self, err:str):
method get_sdk_path (line 130) | def get_sdk_path(self, abspath):
method do_remove_dangling_files (line 133) | def do_remove_dangling_files(self):
method do_check_dangling_files (line 162) | def do_check_dangling_files(self):
method do_stat (line 182) | def do_stat(self, entry, user):
method do_lstat (line 200) | def do_lstat(self, entry, user):
method do_exists (line 217) | def do_exists(self, entry, user):
method do_open (line 234) | def do_open(self, file, flags, mask, mode, user):
method do_open2 (line 256) | def do_open2(self, file, mode, user):
method do_write (line 275) | def do_write(self, file, content, mode:str, encoding, errors, offset, ...
method do_writelines (line 307) | def do_writelines(self, file, lines, mode, offset, whence, user):
method do_fallocate (line 341) | def do_fallocate(self, file, offset, length, mode, user):
method do_copy_file_range (line 364) | def do_copy_file_range(self, src, dst, src_offset, dst_offset, count, ...
method do_read (line 387) | def do_read(self, file, length, mode, offset, whence, user, encoding, ...
method do_readlines (line 429) | def do_readlines(self, file, mode, offset, whence, user):
method do_readline (line 472) | def do_readline(self, file, mode, offset, whence, user):
method do_truncate (line 515) | def do_truncate(self, file, size, user):
method do_create_file (line 546) | def do_create_file(self, parent, file_name, content, mode='xb', user='...
method do_listdir (line 569) | def do_listdir(self, dir, user):
method do_unlink (line 586) | def do_unlink(self, file, user):
method do_rename (line 603) | def do_rename(self, entry, parent, new_entry_name, user, umask):
method do_copy_file (line 627) | def do_copy_file(self, entry, parent, new_entry_name, follow_symlinks,...
method can_clone (line 645) | def can_clone(self, src_dir, dst_dir):
method do_clone_entry (line 653) | def do_clone_entry(self, entry, parent, new_entry_name, preserve, use...
method do_copy_tree (line 682) | def do_copy_tree(self, entry, parent, new_entry_name, symlinks, ignore...
method do_mkdir (line 703) | def do_mkdir(self, parent, subdir, mode, user, umask):
method do_rmdir (line 726) | def do_rmdir(self, dir, user):
method do_hardlink (line 745) | def do_hardlink(self, src_file, parent, link_file_name, user, umask):
method do_symlink (line 769) | def do_symlink(self, src_file, parent, link_file_name, user, umask):
method do_readlink (line 795) | def do_readlink(self, file, user):
method do_loop_symlink (line 811) | def do_loop_symlink(self, parent, link_file_name, user='root'):
method do_set_xattr (line 828) | def do_set_xattr(self, file, name, value, flag, user):
method do_get_xattr (line 846) | def do_get_xattr(self, file, name, user):
method do_list_xattr (line 862) | def do_list_xattr(self, file, user):
method do_remove_xattr (line 889) | def do_remove_xattr(self, file, name, user):
method do_change_groups (line 906) | def do_change_groups(self, user, group, groups):
method do_chmod (line 916) | def do_chmod(self, entry, mode, user):
method do_get_acl (line 933) | def do_get_acl(self, entry: str):
method do_remove_acl (line 943) | def do_remove_acl(self, entry: str, option: str, user: str):
method do_set_acl (line 953) | def do_set_acl(self, sudo_user, entry, user, user_perm, group, group_p...
method do_utime (line 977) | def do_utime(self, entry, access_time, modify_time, follow_symlinks, u...
method do_chown (line 995) | def do_chown(self, entry, owner, user):
method do_split_dir (line 1015) | def do_split_dir(self, dir, vdirs):
method do_merge_dir (line 1029) | def do_merge_dir(self, dir):
method do_rebalance_with_pysdk (line 1044) | def do_rebalance_with_pysdk(self, entry, zone, is_vdir):
method do_rebalance (line 1063) | def do_rebalance(self, entry, zone, is_vdir, pysdk=True):
FILE: .github/scripts/hypo/fs_sdk_test.py
class TestPySdk (line 10) | class TestPySdk(unittest.TestCase):
method test_issue_1331 (line 11) | def test_issue_1331(self):
method test_issue_1339 (line 18) | def test_issue_1339(self):
method test_issue_1349 (line 27) | def test_issue_1349(self):
method test_issue_1359 (line 35) | def test_issue_1359(self):
method test_issue_1361 (line 43) | def test_issue_1361(self):
method test_issue_1362 (line 51) | def test_issue_1362(self):
method test_issue_1364 (line 59) | def test_issue_1364(self):
method test_issue_1365 (line 67) | def test_issue_1365(self):
method test_issue_1369 (line 75) | def test_issue_1369(self):
method test_issue_1369_2 (line 83) | def test_issue_1369_2(self):
method test_issue_1369_3 (line 91) | def test_issue_1369_3(self):
method skip_test_issue_1370 (line 99) | def skip_test_issue_1370(self):
method test_issue_1419 (line 109) | def test_issue_1419(self):
method test_issue_1422 (line 118) | def test_issue_1422(self):
method test_issue_1424 (line 127) | def test_issue_1424(self):
method test_issue_1425 (line 135) | def test_issue_1425(self):
method test_issue_1442 (line 144) | def test_issue_1442(self):
method test_issue_1443 (line 152) | def test_issue_1443(self):
method test_issue_1449 (line 161) | def test_issue_1449(self):
method test_issue_1450 (line 170) | def test_issue_1450(self):
method skip_test_issue_1450_2 (line 178) | def skip_test_issue_1450_2(self):
method test_issue_1457 (line 186) | def test_issue_1457(self):
method skip_test_issue_1465 (line 193) | def skip_test_issue_1465(self):
method test_issue_1481 (line 201) | def test_issue_1481(self):
method test_issue_x (line 209) | def test_issue_x(self):
method test_issue_y (line 218) | def test_issue_y(self):
method test_issue_z (line 225) | def test_issue_z(self):
method test_issue_a (line 233) | def test_issue_a(self):
method test_issue_b (line 240) | def test_issue_b(self):
method test_issue_c (line 247) | def test_issue_c(self):
method test_issue_d (line 253) | def test_issue_d(self):
method test_issue_e (line 265) | def test_issue_e(self):
method test_issue_f (line 272) | def test_issue_f(self):
method test_rename_invalid_arg (line 280) | def test_rename_invalid_arg(self):
method test_rename_to_dir_not_exist (line 292) | def test_rename_to_dir_not_exist(self):
method test_rmdir_check_exist (line 300) | def test_rmdir_check_exist(self):
method test_truncate (line 309) | def test_truncate(self):
method test_unlink (line 320) | def test_unlink(self):
method test_read_utf8 (line 330) | def test_read_utf8(self):
method test_create_isdir (line 338) | def test_create_isdir(self):
method test_rename_file (line 351) | def test_rename_file(self):
FILE: .github/scripts/hypo/fs_test.py
class TestFsrand2 (line 5) | class TestFsrand2(unittest.TestCase):
method test_issue_910 (line 6) | def test_issue_910(self):
method test_issue_914 (line 17) | def test_issue_914(self):
method skip_test_issue_918 (line 26) | def skip_test_issue_918(self):
method test_x (line 36) | def test_x(self):
FILE: .github/scripts/hypo/s3.py
class S3Machine (line 22) | class S3Machine(RuleBasedStateMachine):
method __init__ (line 40) | def __init__(self):
method init_aliases (line 56) | def init_aliases(self):
method init_policies (line 60) | def init_policies(self):
method equal (line 63) | def equal(self, result1, result2):
method info (line 80) | def info(self, alias=ROOT_ALIAS):
method list_buckets (line 87) | def list_buckets(self, alias=ROOT_ALIAS):
method create_bucket (line 97) | def create_bucket(self, bucket_name, alias = ROOT_ALIAS):
method remove_bucket (line 111) | def remove_bucket(self, bucket_name, alias = ROOT_ALIAS):
method set_bucket_policy (line 126) | def set_bucket_policy(self, bucket_name, policy, alias=ROOT_ALIAS):
method get_bucket_policy (line 136) | def get_bucket_policy(self, bucket_name, alias=ROOT_ALIAS):
method list_bucket_policy (line 146) | def list_bucket_policy(self, bucket_name, alias=ROOT_ALIAS, recursive=...
method put_object (line 160) | def put_object(self, bucket_name, object_name, data, use_part_size=Fal...
method get_object (line 179) | def get_object(self, obj:str, offset=0, length=0):
method fput_object (line 192) | def fput_object(self, bucket_name, object_name, alias=ROOT_ALIAS):
method fget_object (line 207) | def fget_object(self, obj:str, file_path, alias = ROOT_ALIAS):
method remove_object (line 220) | def remove_object(self, obj:str, alias=ROOT_ALIAS):
method stat_object (line 236) | def stat_object(self, obj:str, alias=ROOT_ALIAS):
method list_objects (line 252) | def list_objects(self, bucket_name, prefix=None, start_after=None, inc...
method add_user (line 263) | def add_user(self, user_name, secret_key=DEFAULT_SECRET_KEY, alias = R...
method remove_user (line 278) | def remove_user(self, user_name, alias = ROOT_ALIAS):
method enable_user (line 292) | def enable_user(self, user_name, alias=ROOT_ALIAS):
method disable_user (line 302) | def disable_user(self, user_name, alias=ROOT_ALIAS):
method user_info (line 312) | def user_info(self, user_name, alias=ROOT_ALIAS):
method list_users (line 319) | def list_users(self, alias=ROOT_ALIAS):
method list_groups (line 326) | def list_groups(self, alias=ROOT_ALIAS):
method add_group (line 338) | def add_group(self, group_name, members, alias=ROOT_ALIAS):
method group_info (line 351) | def group_info(self, group_name, alias=ROOT_ALIAS):
method remove_group (line 363) | def remove_group(self, group_name, group_members, alias=ROOT_ALIAS):
method disable_group (line 377) | def disable_group(self, group_name, alias=ROOT_ALIAS):
method enable_group (line 387) | def enable_group(self, group_name, alias=ROOT_ALIAS):
method add_policy (line 399) | def add_policy(self, policy_name, policy_document, alias=ROOT_ALIAS):
method remove_policy (line 414) | def remove_policy(self, policy_name, alias=ROOT_ALIAS):
method policy_info (line 430) | def policy_info(self, policy_name, alias=ROOT_ALIAS):
method list_policies (line 437) | def list_policies(self, alias=ROOT_ALIAS):
method set_policy_to_user (line 449) | def set_policy_to_user(self, policy_name, user_name, alias=ROOT_ALIAS):
method set_policy_to_group (line 465) | def set_policy_to_group(self, group_name, policy_name, alias=ROOT_ALIAS):
method unset_policy_from_user (line 480) | def unset_policy_from_user(self, user_policy:str, alias=ROOT_ALIAS):
method unset_policy_from_group (line 497) | def unset_policy_from_group(self, group_policy:str, alias=ROOT_ALIAS):
method set_alias (line 516) | def set_alias(self, alias, user_name, url1=URL1, url2=URL2):
method remove_alias (line 530) | def remove_alias(self, alias):
method teardown (line 540) | def teardown(self):
FILE: .github/scripts/hypo/s3_op.py
class S3Client (line 25) | class S3Client():
method __init__ (line 27) | def __init__(self, prefix, url, url2=None):
method run_cmd (line 34) | def run_cmd(self, command:str, stderr=subprocess.STDOUT) -> str:
method sort_dict (line 48) | def sort_dict(self, obj):
method handleException (line 58) | def handleException(self, e, action, **kwargs):
method do_info (line 77) | def do_info(self, alias):
method remove_all_buckets (line 86) | def remove_all_buckets(self):
method do_list_buckets (line 97) | def do_list_buckets(self, alias):
method do_remove_bucket (line 108) | def do_remove_bucket(self, bucket_name:str, alias):
method do_create_bucket (line 117) | def do_create_bucket(self, bucket_name:str, alias):
method do_set_bucket_policy (line 126) | def do_set_bucket_policy(self, bucket_name:str, policy:str, alias):
method do_get_bucket_policy (line 135) | def do_get_bucket_policy(self, bucket_name:str, alias):
method do_list_bucket_policy (line 144) | def do_list_bucket_policy(self, bucket_name:str, alias, recursive=False):
method do_stat_object (line 156) | def do_stat_object(self, bucket_name:str, object_name:str, alias):
method do_put_object (line 171) | def do_put_object(self, bucket_name:str, object_name:str, data, length...
method do_get_object (line 181) | def do_get_object(self, bucket_name:str, object_name:str, offset=0, le...
method do_fput_object (line 202) | def do_fput_object(self, bucket_name:str, object_name:str, src_path:st...
method do_fget_object (line 211) | def do_fget_object(self, bucket_name:str, object_name:str, file_path:s...
method object_exists (line 220) | def object_exists(self, bucket_name:str, object_name:str, alias):
method do_remove_object (line 227) | def do_remove_object(self, bucket_name:str, object_name:str, alias):
method do_list_objects (line 237) | def do_list_objects(self, bucket_name, prefix, start_after, include_us...
method get_alias (line 248) | def get_alias(self, alias):
method do_add_user (line 251) | def do_add_user(self, access_key, secret_key, alias):
method do_remove_user (line 260) | def do_remove_user(self, access_key, alias):
method do_enable_user (line 269) | def do_enable_user(self, access_key, alias):
method do_disable_user (line 278) | def do_disable_user(self, access_key, alias):
method do_user_info (line 287) | def do_user_info(self, access_key, alias):
method do_list_users (line 296) | def do_list_users(self, alias):
method remove_all_users (line 305) | def remove_all_users(self, alias=ROOT_ALIAS):
method do_list_groups (line 314) | def do_list_groups(self, alias):
method do_add_group (line 323) | def do_add_group(self, group_name, members, alias):
method do_remove_group (line 332) | def do_remove_group(self, group_name, members, alias):
method do_disable_group (line 341) | def do_disable_group(self, group_name, alias):
method do_enable_group (line 350) | def do_enable_group(self, group_name, alias):
method do_group_info (line 359) | def do_group_info(self, group_name, alias):
method remove_all_groups (line 368) | def remove_all_groups(self, alias=ROOT_ALIAS):
method do_add_policy (line 376) | def do_add_policy(self, policy_name, policy_document, alias):
method do_remove_policy (line 390) | def do_remove_policy(self, policy_name, alias):
method do_policy_info (line 399) | def do_policy_info(self, policy_name, alias):
method do_list_policies (line 408) | def do_list_policies(self, alias):
method remove_all_policies (line 418) | def remove_all_policies(self, alias=ROOT_ALIAS):
method do_set_policy_to_user (line 426) | def do_set_policy_to_user(self, policy_name, user_name, alias):
method do_set_policy_to_group (line 435) | def do_set_policy_to_group(self, policy_name, group_name, alias):
method do_unset_policy_from_user (line 444) | def do_unset_policy_from_user(self, policy_name, user_name, alias):
method do_unset_policy_from_group (line 453) | def do_unset_policy_from_group(self, policy_name, group_name, alias):
method do_set_alias (line 462) | def do_set_alias(self, alias, access_key, secret_key, url):
method do_remove_alias (line 472) | def do_remove_alias(self, alias):
method do_list_aliases (line 481) | def do_list_aliases(self):
method remove_all_aliases (line 490) | def remove_all_aliases(self):
FILE: .github/scripts/hypo/s3_test.py
class TestS3 (line 4) | class TestS3(unittest.TestCase):
method test_bucket (line 5) | def test_bucket(self):
method test_user (line 24) | def test_user(self):
method test_group (line 39) | def test_group(self):
method skip_test_issue_4639 (line 57) | def skip_test_issue_4639(self):
method skip_test_issue_4660 (line 68) | def skip_test_issue_4660(self):
method test_issue_4682 (line 78) | def test_issue_4682(self):
FILE: .github/scripts/hypo/stats.py
function singleton (line 1) | def singleton(cls):
class Statistics (line 10) | class Statistics:
method __init__ (line 11) | def __init__(self):
method success (line 14) | def success(self, function_name):
method failure (line 19) | def failure(self, function_name):
method get (line 24) | def get(self):
FILE: .github/scripts/hypo/strategy.py
function valid_dir_name (line 27) | def valid_dir_name():
function utf8_byte_arrays (line 62) | def utf8_byte_arrays(draw, min_size=0, max_size=100):
function utf16_byte_arrays (line 67) | def utf16_byte_arrays(draw, min_size=0, max_size=100):
function ascii_byte_arrays (line 72) | def ascii_byte_arrays(draw, min_size=0, max_size=100):
FILE: .github/scripts/hypo/sync.py
class SyncMachine (line 29) | class SyncMachine(RuleBasedStateMachine):
method init_folders (line 41) | def init_folders(self):
method __init__ (line 50) | def __init__(self):
method equal (line 53) | def equal(self, result1, result2):
method create_file (line 74) | def create_file(self, parent, file_name, content='s', mode='x', user='...
method mkdir (line 89) | def mkdir(self, parent, subdir, mode, user='root', umask=0o022):
method sync (line 100) | def sync(self, options):
method teardown (line 117) | def teardown(self):
FILE: .github/scripts/hypo/sync_test.py
class TestFsrand2 (line 4) | class TestFsrand2(unittest.TestCase):
method test_sync1 (line 6) | def test_sync1(self):
method test_sync2 (line 15) | def test_sync2(self):
method test_sync3 (line 22) | def test_sync3(self):
method test_sync4 (line 29) | def test_sync4(self):
method test_sync5 (line 36) | def test_sync5(self):
method test_sync6 (line 46) | def test_sync6(self):
method test_sync7 (line 53) | def test_sync7(self):
method test_sync8 (line 60) | def test_sync8(self):
method test_sync9 (line 68) | def test_sync9(self):
FILE: .github/scripts/mutate/check_coverage.py
function is_mutation_in_coverage (line 5) | def is_mutation_in_coverage(original_file, changed_file, coverage_file):
function parse_coverage (line 22) | def parse_coverage(file):
FILE: .github/scripts/mutate/check_skip_by_comment.py
function is_mutation_skipped_by_comment (line 6) | def is_mutation_skipped_by_comment(original_file, changed_file):
FILE: .github/scripts/mutate/modify_sdk_pom.py
function get_plugin_str (line 7) | def get_plugin_str(taget_tests, taget_classes, time_constant):
function modify_pom (line 27) | def modify_pom(pom_path, taget_tests, taget_classes, time_constant):
FILE: .github/scripts/mutate/mutesting.py
function do_mutate_test (line 9) | def do_mutate_test(mutation_dir, index, total):
FILE: .github/scripts/mutate/parse_black_list.py
function parse_check_sum (line 7) | def parse_check_sum(test_file_path):
function save_black_list (line 18) | def save_black_list(file_name, check_sum_list):
FILE: .github/scripts/mutate/parse_job_total.py
function parse_test_jobs (line 8) | def parse_test_jobs(test_file_path):
FILE: .github/scripts/mutate/parse_mutate_log.py
function parse_mutate_log (line 7) | def parse_mutate_log(log_file):
FILE: .github/scripts/mutate/parse_test_cases.py
function parse_test_cases (line 7) | def parse_test_cases(test_file_path):
FILE: .github/scripts/mutate/query_report.py
function query_report (line 6) | def query_report(repo, run_id):
FILE: .github/scripts/mutate/save_report.py
function save_report (line 25) | def save_report(job_name, report):
FILE: .github/scripts/perf/ai_format_benchmark.py
class BenchmarkResult (line 58) | class BenchmarkResult:
class AIFormatBenchmark (line 69) | class AIFormatBenchmark:
method __init__ (line 70) | def __init__(self, mount_point: str, results_file: str, version: str):
method clear_cache (line 95) | def clear_cache(self):
method run_benchmark (line 111) | def run_benchmark(self, name: str, func: Callable, file_size: int = None,
method _print_benchmark_result (line 174) | def _print_benchmark_result(self, name: str, stats: BenchmarkResult):
method generate_random_image_bytes (line 194) | def generate_random_image_bytes(self, width=64, height=64, format="JPE...
method generate_lmdb_data_entry (line 202) | def generate_lmdb_data_entry(self, idx, image_size=(64, 64)):
method write_lmdb_data (line 211) | def write_lmdb_data(self, lmdb_path, num_samples, image_size=(64, 64)):
method read_lmdb_data_single_process (line 227) | def read_lmdb_data_single_process(self, lmdb_path):
method lmdb_batch_worker (line 242) | def lmdb_batch_worker(self, lmdb_path, key_batch):
method read_lmdb_data_multi_process (line 258) | def read_lmdb_data_multi_process(self, lmdb_path, num_processes=2):
method benchmark_lmdb (line 289) | def benchmark_lmdb(self):
method benchmark_pytorch_weights (line 350) | def benchmark_pytorch_weights(self):
method benchmark_tensorflow_h5 (line 399) | def benchmark_tensorflow_h5(self):
method benchmark_onnx (line 461) | def benchmark_onnx(self):
method benchmark_huggingface_bin (line 544) | def benchmark_huggingface_bin(self):
method benchmark_tensorflow_checkpoint (line 606) | def benchmark_tensorflow_checkpoint(self):
method benchmark_tfrecord (line 701) | def benchmark_tfrecord(self):
method benchmark_hdf5_dataset (line 808) | def benchmark_hdf5_dataset(self):
method benchmark_parquet (line 871) | def benchmark_parquet(self):
method benchmark_comprehensive (line 936) | def benchmark_comprehensive(self):
method generate_report (line 973) | def generate_report(self):
method save_results (line 1031) | def save_results(self):
method print_detailed_summary (line 1064) | def print_detailed_summary(self):
function main (line 1097) | def main():
FILE: .github/scripts/pysdk/bench.py
function print_stats (line 12) | def print_stats(stats, interval):
function seq_write (line 19) | def seq_write(filename, client: juicefs.Client, protocol, block_size, bu...
function random_write (line 42) | def random_write(filename, client: juicefs.Client, protocol, buffering, ...
function seq_read (line 75) | def seq_read(filename, client: juicefs.Client, protocol, block_size, buf...
function random_read (line 99) | def random_read(filename, client: juicefs.Client, protocol, buffering, b...
function clean_page_cache (line 127) | def clean_page_cache():
FILE: .github/scripts/pysdk/pysdk_test.py
class FileTests (line 19) | class FileTests(unittest.TestCase):
method setUp (line 20) | def setUp(self):
method tearDown (line 25) | def tearDown(self):
method create_file (line 28) | def create_file(self, filename, content=b'content'):
method test_read (line 32) | def test_read(self):
method test_write (line 42) | def test_write(self):
class UtimeTests (line 51) | class UtimeTests(FileTests):
method setUp (line 52) | def setUp(self):
method _test_utime (line 58) | def _test_utime(self, set_time, filename=None):
method test_utime (line 68) | def test_utime(self):
method test_utime_by_times (line 73) | def test_utime_by_times(self):
class MakedirTests (line 77) | class MakedirTests(FileTests):
method test_makedir (line 78) | def test_makedir(self):
class ChownFileTests (line 91) | class ChownFileTests(FileTests):
method test_chown_uid_gid_arguments_must_be_index (line 92) | def test_chown_uid_gid_arguments_must_be_index(self):
method test_chown_with_root (line 101) | def test_chown_with_root(self):
class LinkTests (line 116) | class LinkTests(FileTests):
method setUp (line 117) | def setUp(self):
method are_files_same (line 122) | def are_files_same(self, file1, file2):
method _test_link (line 127) | def _test_link(self, file1, file2):
method test_link (line 136) | def test_link(self):
class SummaryTests (line 140) | class SummaryTests(FileTests):
method setUp (line 144) | def setUp(self):
method test_summary (line 151) | def test_summary(self):
class QuotaTests (line 187) | class QuotaTests(FileTests):
method test_quota (line 188) | def test_quota(self):
function normalize (line 218) | def normalize(d):
class NonLocalSymlinkTests (line 229) | class NonLocalSymlinkTests(FileTests):
method test_directory_link_nonlocal (line 230) | def test_directory_link_nonlocal(self):
class ExtendedAttributeTests (line 236) | class ExtendedAttributeTests(FileTests):
method _check_xattrs_str (line 237) | def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxa...
method _check_xattrs (line 286) | def _check_xattrs(self, *args, **kwargs):
method test_simple (line 293) | def test_simple(self):
method test_fds (line 297) | def test_fds(self):
class BenchTests (line 313) | class BenchTests(FileTests):
method test_seq_write (line 323) | def test_seq_write(self):
method test_random_write (line 338) | def test_random_write(self):
method test_seq_read (line 354) | def test_seq_read(self):
method test_random_read (line 367) | def test_random_read(self):
class ClientParamsTests (line 383) | class ClientParamsTests(FileTests):
method test_readonly_param (line 386) | def test_readonly_param(self):
method test_cache_params (line 395) | def test_cache_params(self):
method test_io_limits (line 420) | def test_io_limits(self):
class CloneTests (line 437) | class CloneTests(FileTests):
method setUp (line 438) | def setUp(self):
method test_basic_clone (line 447) | def test_basic_clone(self):
method test_clone_with_preserve (line 460) | def test_clone_with_preserve(self):
class WarmupTests (line 469) | class WarmupTests(unittest.TestCase):
method setUpClass (line 471) | def setUpClass(self):
method tearDownClass (line 493) | def tearDownClass(self):
method test_basic_warmup (line 498) | def test_basic_warmup(self):
method test_warmup_check (line 515) | def test_warmup_check(self):
method test_warmup_evict (line 522) | def test_warmup_evict(self):
class InfoTests (line 537) | class InfoTests(FileTests):
method test_file_info (line 538) | def test_file_info(self):
FILE: .github/scripts/random_read_write.py
function random_write (line 4) | def random_write(path1, path2, count=1000):
function random_read (line 33) | def random_read(path1, path2):
function read_all (line 43) | def read_all(path1, path2):
FILE: .github/scripts/testVersionCompatible.py
class JuicefsMachine (line 42) | class JuicefsMachine(RuleBasedStateMachine):
method __init__ (line 55) | def __init__(self):
method format (line 92) | def format(self, juicefs, block_size, capacity, inodes, compress, shar...
method config (line 171) | def config(self, juicefs, capacity, inodes, change_bucket, change_aksk...
method status (line 233) | def status(self, juicefs):
method mount (line 284) | def mount(self, juicefs, no_syslog, other_fuse_options, enable_xattr, ...
method info (line 384) | def info(self, juicefs, file_name, data):
method rmr (line 399) | def rmr(self, juicefs, file_name):
method umount (line 425) | def umount(self, juicefs, force):
method destroy (line 438) | def destroy(self, juicefs):
method write_and_read (line 462) | def write_and_read(self, file_name, data):
method write_rand_files (line 473) | def write_rand_files(self, path, seed):
method write_rand_files_and_compare (line 486) | def write_rand_files_and_compare(self):
method dump (line 505) | def dump(self, juicefs):
method load (line 516) | def load(self, juicefs):
method fsck (line 543) | def fsck(self, juicefs):
method bench (line 557) | def bench(self, juicefs, block_size, big_file_size, small_file_size, s...
method warmup (line 580) | def warmup(self, juicefs, threads, background, from_file, directory):
method gc (line 624) | def gc(self, juicefs, compact, delete, threads):
method gateway (line 674) | def gateway(self, juicefs, get_timeout, put_timeout, io_retries, max_u...
method webdav (line 750) | def webdav(self, juicefs, port):
method greater_than_version_formatted (line 764) | def greater_than_version_formatted(self, ver):
method greater_than_version_dumped (line 770) | def greater_than_version_dumped(self, ver):
method greater_than_version_mounted (line 775) | def greater_than_version_mounted(self, ver):
FILE: .github/scripts/utils.py
function flush_meta (line 16) | def flush_meta(meta_url:str):
function create_mysql_db (line 60) | def create_mysql_db(meta_url):
function create_postgres_db (line 75) | def create_postgres_db(meta_url):
function clear_storage (line 82) | def clear_storage(storage, bucket, volume):
function clear_cache (line 118) | def clear_cache():
function is_readonly (line 124) | def is_readonly(filesystem):
function get_upload_delay_seconds (line 131) | def get_upload_delay_seconds(filesystem):
function get_stage_blocks (line 138) | def get_stage_blocks(filesystem):
function write_data (line 148) | def write_data(filesystem, path, data):
function write_block (line 158) | def write_block(filesystem, filepath, bs, count):
function mdtest (line 167) | def mdtest(filesystem, meta_url):
function run_jfs_cmd (line 183) | def run_jfs_cmd( options):
function run_cmd (line 198) | def run_cmd(command):
function is_port_in_use (line 212) | def is_port_in_use(port: int) -> bool:
function get_storage (line 217) | def get_storage(juicefs, meta_url):
FILE: .github/scripts/wins_fs_test.py
class WindowsFSTest (line 11) | class WindowsFSTest(unittest.TestCase):
method setUp (line 12) | def setUp(self):
method tearDown (line 16) | def tearDown(self):
method ensure_clean_dir (line 20) | def ensure_clean_dir(self, path):
method random_string (line 25) | def random_string(self, length=10):
method test_basic_operations (line 28) | def test_basic_operations(self):
method test_rename_case_change (line 44) | def test_rename_case_change(self):
method test_directory_operations (line 57) | def test_directory_operations(self):
method test_concurrent_operations (line 73) | def test_concurrent_operations(self):
method test_special_characters (line 95) | def test_special_characters(self):
method test_large_files (line 111) | def test_large_files(self):
method test_file_attributes (line 128) | def test_file_attributes(self):
method test_symlinks (line 142) | def test_symlinks(self):
method test_long_paths (line 176) | def test_long_paths(self):
FILE: cmd/bench.go
function cmdBench (line 35) | func cmdBench() *cli.Command {
type benchCase (line 101) | type benchCase struct
method writeFiles (line 116) | func (bc *benchCase) writeFiles(index int) {
method readFiles (line 135) | func (bc *benchCase) readFiles(index int) {
method statFiles (line 153) | func (bc *benchCase) statFiles(index int) {
method run (line 163) | func (bc *benchCase) run(test string) float64 {
type benchmark (line 109) | type benchmark struct
method newCase (line 198) | func (bm *benchmark) newCase(name string, fsize, fcount, bsize int) *b...
method colorize (line 216) | func (bm *benchmark) colorize(item string, value, cost float64, prec i...
function newBenchmark (line 187) | func newBenchmark(tmpdir string, blockSize, bigSize, smallSize, smallCou...
function printResult (line 249) | func printResult(result [][]string, leftAlign int, colorful bool) {
function bench (line 308) | func bench(ctx *cli.Context) error {
FILE: cmd/bench_test.go
function TestBench (line 24) | func TestBench(t *testing.T) {
function TestBenchForObject (line 35) | func TestBenchForObject(t *testing.T) {
FILE: cmd/clone.go
function cmdClone (line 32) | func cmdClone() *cli.Command {
function clone (line 65) | func clone(ctx *cli.Context) error {
function findMountpoint (line 160) | func findMountpoint(fpath string) (string, error) {
FILE: cmd/compact.go
function cmdCompact (line 30) | func cmdCompact() *cli.Command {
function compact (line 53) | func compact(ctx *cli.Context) error {
function doCompact (line 90) | func doCompact(inode meta.Ino, path string, coCnt uint16) error {
FILE: cmd/compact_test.go
function createTestFile (line 29) | func createTestFile(path string, size int, partCnt int) error {
type testDir (line 48) | type testDir struct
function initForCompactTest (line 55) | func initForCompactTest(mountDir string, dirs map[string]testDir) {
function TestCompact (line 72) | func TestCompact(t *testing.T) {
FILE: cmd/config.go
function cmdConfig (line 34) | func cmdConfig() *cli.Command {
function configManagementFlags (line 74) | func configManagementFlags() []cli.Flag {
function configFlags (line 99) | func configFlags() []cli.Flag {
function warn (line 113) | func warn(format string, a ...interface{}) {
function userConfirmed (line 117) | func userConfirmed() bool {
function config (line 132) | func config(ctx *cli.Context) error {
FILE: cmd/config_test.go
function getStdout (line 29) | func getStdout(args []string) ([]byte, error) {
function TestConfig (line 45) | func TestConfig(t *testing.T) {
FILE: cmd/debug.go
function cmdDebug (line 46) | func cmdDebug() *cli.Command {
function copyFileOnWindows (line 95) | func copyFileOnWindows(srcPath, destPath string) error {
function copyFile (line 112) | func copyFile(srcPath, destPath string, requireRootPrivileges bool) error {
function getLogPath (line 131) | func getLogPath(cmd string) (string, error) {
function closeFile (line 143) | func closeFile(file *os.File) {
function getPprofPort (line 149) | func getPprofPort(pid, amp string, requireRootPrivileges bool) (int, err...
function getRequest (line 222) | func getRequest(url string, timeout time.Duration) ([]byte, error) {
function checkPort (line 251) | func checkPort(port int, amp string) error {
type metricItem (line 271) | type metricItem struct
function reqAndSaveMetric (line 275) | func reqAndSaveMetric(name string, metric metricItem, outDir string, tim...
function checkAgent (line 298) | func checkAgent(cmd string) bool {
function geneZipFile (line 307) | func geneZipFile(srcPath, destPath string) error {
function collectPprof (line 354) | func collectPprof(ctx *cli.Context, cmd string, pid string, amp string, ...
function collectLog (line 408) | func collectLog(ctx *cli.Context, cmd string, requireRootPrivileges bool...
function collectSysInfo (line 447) | func collectSysInfo(ctx *cli.Context, currDir string) error {
function collectSpecialFile (line 467) | func collectSpecialFile(ctx *cli.Context, amp string, currDir string, re...
function debug (line 505) | func debug(ctx *cli.Context) error {
FILE: cmd/debug_test.go
function TestDebug (line 26) | func TestDebug(t *testing.T) {
FILE: cmd/debug_unix.go
function getCmdMount (line 35) | func getCmdMount(mp string) (uid, pid, cmd string, err error) {
FILE: cmd/debug_windows.go
function getprocessCommandLine (line 35) | func getprocessCommandLine(pid int) (string, error) {
function findMountProcess (line 58) | func findMountProcess(mp string) (int, error) {
function getProcessUserSid (line 126) | func getProcessUserSid(pid int) (string, error) {
function getCmdMount (line 149) | func getCmdMount(mp string) (uid, pid, cmd string, err error) {
FILE: cmd/destroy.go
function cmdDestroy (line 32) | func cmdDestroy() *cli.Command {
function printSessions (line 62) | func printSessions(ss [][3]string) string {
function destroy (line 111) | func destroy(ctx *cli.Context) error {
FILE: cmd/dump.go
function cmdDump (line 32) | func cmdDump() *cli.Command {
function dumpMeta (line 88) | func dumpMeta(m meta.Meta, dst string, threads int, keepSecret, fast, sk...
function dump (line 141) | func dump(ctx *cli.Context) error {
FILE: cmd/dump_test.go
function TestDumpAndLoad (line 27) | func TestDumpAndLoad(t *testing.T) {
FILE: cmd/flags.go
function globalFlags (line 29) | func globalFlags() []cli.Flag {
function addCategory (line 69) | func addCategory(f cli.Flag, cat string) {
function addCategories (line 90) | func addCategories(cat string, flags []cli.Flag) []cli.Flag {
function storageFlags (line 97) | func storageFlags() []cli.Flag {
function getDefaultCacheDir (line 162) | func getDefaultCacheDir() string {
function dataCacheFlags (line 188) | func dataCacheFlags() []cli.Flag {
function metaFlags (line 279) | func metaFlags() []cli.Flag {
function clientFlags (line 338) | func clientFlags(defaultEntryCache float64) []cli.Flag {
function shareInfoFlags (line 347) | func shareInfoFlags() []cli.Flag {
function metaCacheFlags (line 370) | func metaCacheFlags(defaultEntryCache float64) []cli.Flag {
function expandFlags (line 408) | func expandFlags(compoundFlags ...[]cli.Flag) []cli.Flag {
FILE: cmd/flags_test.go
function Test_duration (line 11) | func Test_duration(t *testing.T) {
FILE: cmd/format.go
function cmdFormat (line 50) | func cmdFormat() *cli.Command {
function formatStorageFlags (line 100) | func formatStorageFlags() []cli.Flag {
function formatFlags (line 148) | func formatFlags() []cli.Flag {
function formatManagementFlags (line 180) | func formatManagementFlags() []cli.Flag {
function fixObjectSize (line 214) | func fixObjectSize(s uint64) uint64 {
function createStorage (line 232) | func createStorage(format meta.Format) (object.ObjectStorage, error) {
function randSeq (line 314) | func randSeq(n int) string {
function doTesting (line 323) | func doTesting(store object.ObjectStorage, key string, data []byte) error {
function test (line 362) | func test(store object.ObjectStorage) error {
function loadEncrypt (line 382) | func loadEncrypt(keyPath string) string {
function readKerbConf (line 393) | func readKerbConf(file string) string {
function format (line 404) | func format(c *cli.Context) error {
FILE: cmd/format_test.go
function TestFixObjectSize (line 27) | func TestFixObjectSize(t *testing.T) {
function TestFormat (line 59) | func TestFormat(t *testing.T) {
FILE: cmd/fsck.go
function cmdFsck (line 32) | func cmdFsck() *cli.Command {
function fsck (line 78) | func fsck(ctx *cli.Context) error {
FILE: cmd/fsck_test.go
function TestFsck (line 25) | func TestFsck(t *testing.T) {
function TestFsckRepairDirMode (line 43) | func TestFsckRepairDirMode(t *testing.T) {
FILE: cmd/gateway.go
function cmdGateway (line 46) | func cmdGateway() *cli.Command {
function gateway (line 132) | func gateway(c *cli.Context) error {
function gateway2 (line 224) | func gateway2(ctx *mcli.Context) error {
function initForSvc (line 229) | func initForSvc(c *cli.Context, mp string, svcType, metaUrl, listenAddr ...
FILE: cmd/gateway_noop.go
function cmdGateway (line 27) | func cmdGateway() *cli.Command {
FILE: cmd/gc.go
function cmdGC (line 36) | func cmdGC() *cli.Command {
function gc (line 76) | func gc(ctx *cli.Context) error {
FILE: cmd/gc_test.go
function writeSmallBlocks (line 31) | func writeSmallBlocks(mountDir string) error {
function getFileCount (line 55) | func getFileCount(dir string) int {
function TestGc (line 69) | func TestGc(t *testing.T) {
FILE: cmd/info.go
function cmdInfo (line 35) | func cmdInfo() *cli.Command {
function info (line 75) | func info(ctx *cli.Context) error {
function ltypeToString (line 236) | func ltypeToString(t uint32) string {
function legacyInfo (line 247) | func legacyInfo(d, path string, inode uint64, recursive, raw uint8) {
function legacyPrintChunks (line 295) | func legacyPrintChunks(resp string, raw bool) {
FILE: cmd/info_test.go
function TestInfo (line 29) | func TestInfo(t *testing.T) {
FILE: cmd/integration_test.go
constant gatewayMeta (line 29) | gatewayMeta = "redis://127.0.0.1:6379/14"
constant gatewayVolume (line 30) | gatewayVolume = "gateway-volume"
constant gatewayAddr (line 31) | gatewayAddr = "localhost:9008"
constant webdavMeta (line 32) | webdavMeta = "redis://127.0.0.1:6379/15"
constant webdavVolume (line 33) | webdavVolume = "webdav-volume"
constant webdavAddr (line 34) | webdavAddr = "localhost:9009"
function startGateway (line 36) | func startGateway(t *testing.T) {
function startWebdav (line 56) | func startWebdav(t *testing.T) {
function TestIntegration (line 78) | func TestIntegration(t *testing.T) {
FILE: cmd/load.go
function cmdLoad (line 38) | func cmdLoad() *cli.Command {
type reader (line 88) | type reader struct
method Read (line 93) | func (r *reader) Read(p []byte) (n int, err error) {
method Close (line 97) | func (r *reader) Close() error {
function open (line 107) | func open(src string, key string, algo string) (io.ReadCloser, error) {
function convert (line 157) | func convert(path string, key, algo string) (string, error) {
function load (line 192) | func load(ctx *cli.Context) error {
function statBak (line 262) | func statBak(ctx *cli.Context, path string) error {
function showBakSummary (line 282) | func showBakSummary(ctx *cli.Context, fp *os.File, withOffset bool) error {
function showBakDetail (line 321) | func showBakDetail(ctx *cli.Context, fp *os.File, offset int64) error {
FILE: cmd/main.go
function Main (line 44) | func Main(args []string) error {
function calledViaMount (line 112) | func calledViaMount(args []string) bool {
function handleSysMountArgs (line 123) | func handleSysMountArgs(args []string) ([]string, error) {
function isFlag (line 196) | func isFlag(flags []cli.Flag, option string) (bool, bool) {
function reorderOptions (line 213) | func reorderOptions(app *cli.App, args []string) []string {
function setup (line 272) | func setup(c *cli.Context, n int) {
function setup0 (line 276) | func setup0(c *cli.Context, min, max int) {
function removePassword (line 367) | func removePassword(uris ...string) {
FILE: cmd/main_test.go
function TestArgsOrder (line 27) | func TestArgsOrder(t *testing.T) {
function TestHandleSysMountArgs (line 65) | func TestHandleSysMountArgs(t *testing.T) {
FILE: cmd/mdtest.go
function init (line 42) | func init() {
function createDir (line 50) | func createDir(jfs *fs.FileSystem, root string, d int, width int) error {
function createFile (line 65) | func createFile(jfs *fs.FileSystem, bar *utils.Bar, np int, root string,...
function runTest (line 109) | func runTest(jfs *fs.FileSystem, rootDir string, np, width, depth, files...
function cmdMdtest (line 154) | func cmdMdtest() *cli.Command {
function initForMdtest (line 200) | func initForMdtest(c *cli.Context, mp string, metaUrl string) *fs.FileSy...
function mdtest (line 249) | func mdtest(c *cli.Context) error {
FILE: cmd/mount.go
function cmdMount (line 50) | func cmdMount() *cli.Command {
function exposeMetrics (line 84) | func exposeMetrics(c *cli.Context, registerer prometheus.Registerer, reg...
function wrapRegister (line 139) | func wrapRegister(c *cli.Context, mp, name string) (prometheus.Registere...
function updateFormat (line 167) | func updateFormat(c *cli.Context) func(*meta.Format) {
function relPathToAbs (line 187) | func relPathToAbs(ss []string) []string {
function cacheDirPathToAbs (line 208) | func cacheDirPathToAbs(c *cli.Context) {
function daemonRun (line 234) | func daemonRun(c *cli.Context, addr string, vfsConf *vfs.Config) {
function expandPathForEmbedded (line 248) | func expandPathForEmbedded(addr string) string {
function getVfsConf (line 268) | func getVfsConf(c *cli.Context, metaConf *meta.Config, format *meta.Form...
function registerMetaMsg (line 303) | func registerMetaMsg(m meta.Meta, store chunk.ChunkStore, chunkConf *chu...
function readConfig (line 312) | func readConfig(mp string) ([]byte, error) {
function getMetaConf (line 320) | func getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config {
function getChunkConf (line 356) | func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {
function initBackgroundTasks (line 412) | func initBackgroundTasks(c *cli.Context, vfsConf *vfs.Config, metaConf *...
type storageHolder (line 438) | type storageHolder struct
method Shutdown (line 443) | func (h *storageHolder) Shutdown() {
function NewReloadableStorage (line 447) | func NewReloadableStorage(format *meta.Format, cli meta.Meta, patch func...
function insideContainer (line 479) | func insideContainer() bool {
function getDefaultLogDir (line 508) | func getDefaultLogDir() string {
function mount (line 533) | func mount(c *cli.Context) error {
FILE: cmd/mount_test.go
constant testMeta (line 47) | testMeta = "redis://127.0.0.1:6379/11"
constant testMountPoint (line 48) | testMountPoint = "/tmp/jfs-unit-test"
constant testVolume (line 49) | testVolume = "test"
function Test_exposeMetrics (line 52) | func Test_exposeMetrics(t *testing.T) {
function ResetHttp (line 99) | func ResetHttp() {
function resetTestMeta (line 103) | func resetTestMeta() *redis.Client { // using Redis
function mountTemp (line 112) | func mountTemp(t *testing.T, bucket *string, extraFormatOpts []string, e...
function umountTemp (line 157) | func umountTemp(t *testing.T) {
function TestMount (line 163) | func TestMount(t *testing.T) {
function TestFtruncate (line 172) | func TestFtruncate(t *testing.T) {
function TestUpdateFstab (line 210) | func TestUpdateFstab(t *testing.T) {
function TestUmount (line 241) | func TestUmount(t *testing.T) {
function tryMountTemp (line 254) | func tryMountTemp(t *testing.T, bucket *string, extraFormatOpts []string...
function TestMountVersionMatch (line 307) | func TestMountVersionMatch(t *testing.T) {
function TestParseUIDGID (line 320) | func TestParseUIDGID(t *testing.T) {
FILE: cmd/mount_unix.go
function showThreadStack (line 60) | func showThreadStack(agentAddr string) {
function devMinor (line 78) | func devMinor(dev uint64) uint32 {
function killMountProcess (line 84) | func killMountProcess(pid int, dev uint64, lastActive *int64) {
function loadConfig (line 121) | func loadConfig(path string) (string, *vfs.Config, error) {
function watchdog (line 136) | func watchdog(ctx context.Context, mp string) {
function parseFuseFd (line 196) | func parseFuseFd(mountPoint string) (fd int) {
function checkMountpoint (line 208) | func checkMountpoint(name, mp, logPath string, background bool) {
function checkSvcPort (line 255) | func checkSvcPort(address string) {
function makeDaemonForSvc (line 273) | func makeDaemonForSvc(c *cli.Context, m meta.Meta, metaUrl, listenAddr s...
function getDaemonStage (line 311) | func getDaemonStage() int {
function fuseFlags (line 315) | func fuseFlags() []cli.Flag {
function mountFlags (line 366) | func mountFlags() []cli.Flag {
function disableUpdatedb (line 410) | func disableUpdatedb() {
function getFuserMountVersion (line 457) | func getFuserMountVersion() string {
function setFuseOption (line 467) | func setFuseOption(c *cli.Context, format *meta.Format, vfsConf *vfs.Con...
function genFuseOpt (line 473) | func genFuseOpt(c *cli.Context, name string) string {
function prepareMp (line 497) | func prepareMp(mp string) {
function genFuseOptExt (line 556) | func genFuseOptExt(c *cli.Context, format *meta.Format) (fuseOpt string,...
function shutdownGraceful (line 564) | func shutdownGraceful(mp string) {
function canShutdownGracefully (line 597) | func canShutdownGracefully(mp string, newConf *vfs.Config) bool {
function absPath (line 644) | func absPath(d string) string {
function buildBoolFlagsMap (line 662) | func buildBoolFlagsMap(c *cli.Context) map[string]bool {
function tellFstabOptions (line 682) | func tellFstabOptions(c *cli.Context) string {
function updateFstab (line 715) | func updateFstab(c *cli.Context) error {
function tryToInstallMountExec (line 762) | func tryToInstallMountExec() error {
function fixCacheDirs (line 773) | func fixCacheDirs(c *cli.Context) {
function makeDaemon (line 789) | func makeDaemon(c *cli.Context, conf *vfs.Config) error {
function increaseRlimit (line 826) | func increaseRlimit() {
function installHandler (line 838) | func installHandler(m meta.Meta, mp string, v *vfs.VFS, blob object.Obje...
function launchMount (line 876) | func launchMount(c *cli.Context, mp string, conf *vfs.Config) error {
function getNobodyUIDGID (line 974) | func getNobodyUIDGID() (uint32, uint32) {
function parseUIDGID (line 993) | func parseUIDGID(input string, defaultUid uint32, defaultGid uint32) (ui...
function mountMain (line 1021) | func mountMain(v *vfs.VFS, c *cli.Context) {
FILE: cmd/mount_windows.go
function mountFlags (line 33) | func mountFlags() []cli.Flag {
function makeDaemon (line 133) | func makeDaemon(c *cli.Context, conf *vfs.Config) error {
function makeDaemonForSvc (line 149) | func makeDaemonForSvc(c *cli.Context, m meta.Meta, metaUrl, listenAddr s...
function getDaemonStage (line 154) | func getDaemonStage() int {
function mountMain (line 158) | func mountMain(v *vfs.VFS, c *cli.Context) {
function checkMountpoint (line 176) | func checkMountpoint(name, mp, logPath string, background bool) {}
function prepareMp (line 178) | func prepareMp(mp string) {}
function setFuseOption (line 180) | func setFuseOption(c *cli.Context, format *meta.Format, vfsConf *vfs.Con...
function launchMount (line 182) | func launchMount(c *cli.Context, mp string, conf *vfs.Config) error { re...
function installHandler (line 184) | func installHandler(m meta.Meta, mp string, v *vfs.VFS, blob object.Obje...
function tryToInstallMountExec (line 186) | func tryToInstallMountExec() error { return nil }
function updateFstab (line 188) | func updateFstab(c *cli.Context) error { return nil }
FILE: cmd/objbench.go
function cmdObjbench (line 44) | func cmdObjbench() *cli.Command {
type warning (line 129) | type warning
function objbench (line 134) | func objbench(ctx *cli.Context) error {
function colorize (line 429) | func colorize(item string, value, cost float64, prec int, colorful bool)...
type apiInfo (line 463) | type apiInfo struct
type benchMarkObj (line 471) | type benchMarkObj struct
method run (line 479) | func (bm *benchMarkObj) run(ctx context.Context, api apiInfo) []string {
method put (line 571) | func (bm *benchMarkObj) put(ctx context.Context, key string, startKey ...
method smallPut (line 582) | func (bm *benchMarkObj) smallPut(ctx context.Context, key string, star...
method get (line 633) | func (bm *benchMarkObj) get(ctx context.Context, key string, startKey ...
method smallGet (line 639) | func (bm *benchMarkObj) smallGet(ctx context.Context, key string, star...
method delete (line 645) | func (bm *benchMarkObj) delete(ctx context.Context, key string, startK...
method head (line 649) | func (bm *benchMarkObj) head(ctx context.Context, key string, startKey...
method list (line 654) | func (bm *benchMarkObj) list(ctx context.Context, key string, startKey...
method chown (line 661) | func (bm *benchMarkObj) chown(ctx context.Context, key string, startKe...
method chmod (line 665) | func (bm *benchMarkObj) chmod(ctx context.Context, key string, startKe...
method chtimes (line 669) | func (bm *benchMarkObj) chtimes(ctx context.Context, key string, start...
function getMockData (line 555) | func getMockData(seed []byte, idx int, result *[]byte) {
function getAndCheckN (line 594) | func getAndCheckN(ctx context.Context, blob object.ObjectStorage, key st...
function listAll (line 673) | func listAll(ctx context.Context, s object.ObjectStorage, prefix, marker...
function functionalTesting (line 694) | func functionalTesting(ctx context.Context, blob object.ObjectStorage, r...
FILE: cmd/object.go
function toError (line 50) | func toError(eno syscall.Errno) error {
type juiceFS (line 57) | type juiceFS struct
method String (line 64) | func (j *juiceFS) String() string {
method path (line 68) | func (j *juiceFS) path(key string) string {
method Get (line 101) | func (j *juiceFS) Get(rCtx context.Context, key string, off, limit int...
method Put (line 123) | func (j *juiceFS) Put(rCtx context.Context, key string, in io.Reader, ...
method Delete (line 186) | func (j *juiceFS) Delete(rCtx context.Context, key string, getters ......
method Head (line 220) | func (j *juiceFS) Head(rCtx context.Context, key string) (object.Objec...
method List (line 243) | func (j *juiceFS) List(ctx context.Context, prefix, marker, token, del...
method readDirSorted (line 297) | func (j *juiceFS) readDirSorted(dirname string, followLink bool) ([]*m...
method Chtimes (line 331) | func (j *juiceFS) Chtimes(key string, mtime time.Time) error {
method Chmod (line 356) | func (j *juiceFS) Chmod(key string, mode os.FileMode) error {
method Chown (line 365) | func (j *juiceFS) Chown(key string, owner, group string) error {
method Symlink (line 379) | func (j *juiceFS) Symlink(oldName, newName string) error {
method Readlink (line 391) | func (j *juiceFS) Readlink(name string) (string, error) {
method Shutdown (line 412) | func (j *juiceFS) Shutdown() {
type jFile (line 72) | type jFile struct
method Read (line 77) | func (f *jFile) Read(buf []byte) (int, error) {
method Write (line 92) | func (f *jFile) Write(buf []byte) (int, error) {
method Close (line 97) | func (f *jFile) Close() error {
type jObj (line 199) | type jObj struct
method Key (line 205) | func (o *jObj) Key() string { return o.key }
method Size (line 206) | func (o *jObj) Size() int64 {
method Mtime (line 212) | func (o *jObj) Mtime() time.Time { return o.fi.ModTime() }
method IsDir (line 213) | func (o *jObj) IsDir() bool { return o.fi.IsDir() }
method IsSymlink (line 214) | func (o *jObj) IsSymlink() bool { return o.isSymlink }
method Owner (line 215) | func (o *jObj) Owner() string { return utils.UserName(o.fi.Uid(...
method Group (line 216) | func (o *jObj) Group() string { return utils.GroupName(o.fi.Gid...
method Mode (line 217) | func (o *jObj) Mode() os.FileMode { return o.fi.Mode() }
method StorageClass (line 218) | func (o *jObj) StorageClass() string { return "" }
type mEntry (line 289) | type mEntry struct
function syscallMode (line 341) | func syscallMode(i os.FileMode) (o uint32) {
function getDefaultChunkConf (line 396) | func getDefaultChunkConf(format *meta.Format) *chunk.Config {
function newJFS (line 416) | func newJFS(endpoint, accessKey, secretKey, token string) (object.Object...
function init (line 470) | func init() {
FILE: cmd/object_test.go
function testKeysEqual (line 36) | func testKeysEqual(objs []object.Object, expectedKeys []string) error {
function testFileSystem (line 56) | func testFileSystem(t *testing.T, s object.ObjectStorage) {
function TestJFS (line 169) | func TestJFS(t *testing.T) {
FILE: cmd/passfd.go
function getFd (line 40) | func getFd(via *net.UnixConn, num int) ([]byte, []int, error) {
function putFd (line 85) | func putFd(via *net.UnixConn, msg []byte, fds ...int) error {
function handleFDRequest (line 105) | func handleFDRequest(conn *net.UnixConn) {
function serveFuseFD (line 151) | func serveFuseFD(path string) {
function getFuseFd (line 177) | func getFuseFd(path string) (int, []byte) {
function sendFuseFd (line 202) | func sendFuseFd(path string, msg []byte, fd int) error {
FILE: cmd/printsid.go
function cmdPrintSID (line 11) | func cmdPrintSID() *cli.Command {
function printSID (line 21) | func printSID(ctx *cli.Context) error {
FILE: cmd/profile.go
function cmdProfile (line 35) | func cmdProfile() *cli.Command {
type profiler (line 91) | type profiler struct
method reader (line 183) | func (p *profiler) reader() {
method isWinFuseLog (line 197) | func (p *profiler) isWinFuseLog() bool {
method isValid (line 201) | func (p *profiler) isValid(entry *logEntry) bool {
method counter (line 216) | func (p *profiler) counter() {
method fastCounter (line 255) | func (p *profiler) fastCounter() {
method flush (line 306) | func (p *profiler) flush(timeStamp time.Time, keyStats []keyStat, done...
method flusher (line 329) | func (p *profiler) flusher() {
type stat (line 108) | type stat struct
type keyStat (line 113) | type keyStat struct
type logEntry (line 118) | type logEntry struct
function parseLine (line 126) | func parseLine(line string, winFuseLog bool) *logEntry {
function colorize1 (line 282) | func colorize1(msg string, color int) string {
function printLines (line 286) | func printLines(lines []string, colorful bool) {
function profile (line 371) | func profile(ctx *cli.Context) error {
FILE: cmd/quota.go
function cmdQuota (line 32) | func cmdQuota() *cli.Command {
function quota (line 123) | func quota(c *cli.Context) error {
FILE: cmd/restore.go
function cmdRestore (line 18) | func cmdRestore() *cli.Command {
function restore (line 44) | func restore(ctx *cli.Context) error {
function doRestore (line 65) | func doRestore(m meta.Meta, hour string, putBack bool, threads int) {
FILE: cmd/restore_test.go
function TestRestore (line 11) | func TestRestore(t *testing.T) {
function TestRestorePutBack (line 58) | func TestRestorePutBack(t *testing.T) {
FILE: cmd/rmr.go
function cmdRmr (line 30) | func cmdRmr() *cli.Command {
function openController (line 57) | func openController(dpath string) (*os.File, error) {
function rmr (line 72) | func rmr(ctx *cli.Context) error {
FILE: cmd/rmr_test.go
function TestRmr (line 25) | func TestRmr(t *testing.T) {
FILE: cmd/stats.go
function cmdStats (line 32) | func cmdStats() *cli.Command {
constant BLACK (line 75) | BLACK = 30 + iota
constant RED (line 76) | RED
constant GREEN (line 77) | GREEN
constant YELLOW (line 78) | YELLOW
constant BLUE (line 79) | BLUE
constant MAGENTA (line 80) | MAGENTA
constant CYAN (line 81) | CYAN
constant WHITE (line 82) | WHITE
constant DEFAULT (line 83) | DEFAULT = "00"
constant RESET_SEQ (line 87) | RESET_SEQ = "\033[0m"
constant COLOR_SEQ (line 88) | COLOR_SEQ = "\033[1;"
constant COLOR_DARK_SEQ (line 89) | COLOR_DARK_SEQ = "\033[0;"
constant UNDERLINE_SEQ (line 90) | UNDERLINE_SEQ = "\033[4m"
constant CLEAR_SCREEM (line 91) | CLEAR_SCREEM = "\033[2J\033[1;1H"
constant UNIXTIME_FMT (line 92) | UNIXTIME_FMT = "01-02 15:04:05"
type statsWatcher (line 96) | type statsWatcher struct
method colorize (line 104) | func (w *statsWatcher) colorize(msg string, color int, dark bool, unde...
method buildSchema (line 142) | func (w *statsWatcher) buildSchema(schema string, verbosity uint) {
method formatHeader (line 217) | func (w *statsWatcher) formatHeader() {
method formatU64 (line 247) | func (w *statsWatcher) formatU64(v float64, dark, isByte bool) string {
method formatTime (line 277) | func (w *statsWatcher) formatTime(v float64, dark bool) string {
method formatCPU (line 295) | func (w *statsWatcher) formatCPU(v float64, dark bool) string {
method printDiff (line 312) | func (w *statsWatcher) printDiff(left, right map[string]float64, dark ...
constant metricByte (line 121) | metricByte = 1 << iota
constant metricCount (line 122) | metricCount
constant metricTime (line 123) | metricTime
constant metricCPU (line 124) | metricCPU
constant metricGauge (line 125) | metricGauge
constant metricCounter (line 126) | metricCounter
constant metricHist (line 127) | metricHist
constant metricUnixtime (line 128) | metricUnixtime
type item (line 131) | type item struct
type section (line 137) | type section struct
function padding (line 199) | func padding(name string, width int, char byte) string {
function readStats (line 366) | func readStats(mp string) map[string]float64 {
function stats (line 396) | func stats(ctx *cli.Context) error {
FILE: cmd/status.go
function cmdStatus (line 28) | func cmdStatus() *cli.Command {
function printJson (line 58) | func printJson(v interface{}) {
function status (line 66) | func status(ctx *cli.Context) error {
FILE: cmd/status_test.go
function TestStatus (line 28) | func TestStatus(t *testing.T) {
FILE: cmd/summary.go
function cmdSummary (line 35) | func cmdSummary() *cli.Command {
function summary (line 83) | func summary(ctx *cli.Context) error {
function printCSVResult (line 163) | func printCSVResult(results [][]string) {
function renderTree (line 176) | func renderTree(results *[][]string, tree *meta.TreeSummary, csv bool) {
FILE: cmd/sync.go
function cmdSync (line 40) | func cmdSync() *cli.Command {
function selectionFlags (line 102) | func selectionFlags() []cli.Flag {
function syncActionFlags (line 181) | func syncActionFlags() []cli.Flag {
function syncStorageFlags (line 238) | func syncStorageFlags() []cli.Flag {
function clusterFlags (line 275) | func clusterFlags() []cli.Flag {
function supportHTTPS (line 293) | func supportHTTPS(name, endpoint string) bool {
function isFilePath (line 311) | func isFilePath(uri string) bool {
function extractToken (line 321) | func extractToken(uri string) (string, string) {
function createSyncStorage (line 328) | func createSyncStorage(uri string, conf *sync.Config) (object.ObjectStor...
function isS3PathType (line 452) | func isS3PathType(endpoint string) bool {
function doSync (line 458) | func doSync(c *cli.Context) error {
FILE: cmd/sync_test.go
function TestSync (line 29) | func TestSync(t *testing.T) {
function Test_isS3PathType (line 67) | func Test_isS3PathType(t *testing.T) {
function Test_extractToken (line 96) | func Test_extractToken(t *testing.T) {
FILE: cmd/umount.go
function cmdUmount (line 36) | func cmdUmount() *cli.Command {
function doUmount (line 60) | func doUmount(mp string, force bool) error {
function umount (line 100) | func umount(ctx *cli.Context) error {
function waitWritebackComplete (line 135) | func waitWritebackComplete(stagingDir string) error {
function fileSizeInDir (line 177) | func fileSizeInDir(dir string) (uint64, error) {
function clearLastLine (line 191) | func clearLastLine() {
FILE: cmd/version.go
function cmdVersion (line 25) | func cmdVersion() *cli.Command {
FILE: cmd/warmup.go
function cmdWarmup (line 41) | func cmdWarmup() *cli.Command {
constant batchMax (line 91) | batchMax = 10240
constant maxInterval (line 93) | maxInterval = 300
constant minInterval (line 94) | minInterval = 1
function readControl (line 98) | func readControl(cf *os.File, resp []byte) int {
function readProgress (line 117) | func readProgress(cf *os.File, showProgress func(uint64, uint64)) (data ...
function sendCommand (line 156) | func sendCommand(cf *os.File, action vfs.CacheAction, batch []string, th...
function warmup (line 200) | func warmup(ctx *cli.Context) error {
FILE: cmd/warmup_test.go
function TestWarmup (line 29) | func TestWarmup(t *testing.T) {
FILE: cmd/webdav.go
function cmdWebDav (line 31) | func cmdWebDav() *cli.Command {
function webdav (line 95) | func webdav(c *cli.Context) error {
FILE: cmd/webdav_noop.go
function cmdWebDav (line 28) | func cmdWebDav() *cli.Command {
FILE: hack/winfsp_headers/fuse.h
type fuse (line 36) | struct fuse
type fuse_stat (line 39) | struct fuse_stat
type fuse_dirhandle (line 40) | struct fuse_dirhandle
type fuse_operations (line 44) | struct fuse_operations
type fuse_context (line 124) | struct fuse_context
type fsp_fuse_env (line 137) | struct fsp_fuse_env
type fuse_operations (line 139) | struct fuse_operations
type fsp_fuse_env (line 140) | struct fsp_fuse_env
type fsp_fuse_env (line 142) | struct fsp_fuse_env
type fuse_chan (line 143) | struct fuse_chan
type fuse_args (line 143) | struct fuse_args
type fuse_operations (line 144) | struct fuse_operations
type fsp_fuse_env (line 145) | struct fsp_fuse_env
type fuse (line 146) | struct fuse
type fsp_fuse_env (line 147) | struct fsp_fuse_env
type fuse (line 148) | struct fuse
type fsp_fuse_env (line 149) | struct fsp_fuse_env
type fuse (line 150) | struct fuse
type fsp_fuse_env (line 151) | struct fsp_fuse_env
type fuse (line 152) | struct fuse
type fsp_fuse_env (line 153) | struct fsp_fuse_env
type fuse (line 154) | struct fuse
type fsp_fuse_env (line 155) | struct fsp_fuse_env
type fuse_operations (line 159) | struct fuse_operations
type fuse (line 173) | struct fuse
type fuse_chan (line 173) | struct fuse_chan
type fuse_args (line 173) | struct fuse_args
type fuse_operations (line 174) | struct fuse_operations
type fuse (line 181) | struct fuse
type fuse (line 188) | struct fuse
type fuse (line 195) | struct fuse
type fuse (line 202) | struct fuse
type fuse (line 209) | struct fuse
type fuse_context (line 216) | struct fuse_context
type fuse (line 237) | struct fuse
type fuse_pollhandle (line 245) | struct fuse_pollhandle
type fuse_session (line 252) | struct fuse_session
type fuse (line 252) | struct fuse
type fuse_session (line 254) | struct fuse_session
FILE: hack/winfsp_headers/fuse_common.h
type fuse_file_info (line 82) | struct fuse_file_info
type fuse_conn_info (line 96) | struct fuse_conn_info
type fuse_session (line 108) | struct fuse_session
type fuse_chan (line 109) | struct fuse_chan
type fuse_pollhandle (line 110) | struct fuse_pollhandle
type fuse_bufvec (line 111) | struct fuse_bufvec
type fuse_statfs (line 112) | struct fuse_statfs
type fuse_setattr_x (line 113) | struct fuse_setattr_x
type fsp_fuse_env (line 115) | struct fsp_fuse_env
type fsp_fuse_env (line 116) | struct fsp_fuse_env
type fuse_args (line 117) | struct fuse_args
type fsp_fuse_env (line 118) | struct fsp_fuse_env
type fuse_chan (line 119) | struct fuse_chan
type fsp_fuse_env (line 120) | struct fsp_fuse_env
type fuse_args (line 121) | struct fuse_args
type fsp_fuse_env (line 123) | struct fsp_fuse_env
type fuse_chan (line 134) | struct fuse_chan
type fuse_args (line 134) | struct fuse_args
type fuse_chan (line 141) | struct fuse_chan
type fuse_args (line 148) | struct fuse_args
type fuse_pollhandle (line 156) | struct fuse_pollhandle
type fuse_session (line 168) | struct fuse_session
type fuse_session (line 174) | struct fuse_session
FILE: hack/winfsp_headers/fuse_opt.h
type fuse_opt (line 46) | struct fuse_opt
type fuse_args (line 53) | struct fuse_args
type fuse_args (line 61) | struct fuse_args
type fsp_fuse_env (line 63) | struct fsp_fuse_env
type fuse_args (line 64) | struct fuse_args
type fuse_opt (line 65) | struct fuse_opt
type fsp_fuse_env (line 66) | struct fsp_fuse_env
type fuse_args (line 67) | struct fuse_args
type fsp_fuse_env (line 68) | struct fsp_fuse_env
type fuse_args (line 69) | struct fuse_args
type fsp_fuse_env (line 70) | struct fsp_fuse_env
type fuse_args (line 71) | struct fuse_args
type fsp_fuse_env (line 72) | struct fsp_fuse_env
type fsp_fuse_env (line 74) | struct fsp_fuse_env
type fsp_fuse_env (line 76) | struct fsp_fuse_env
type fuse_opt (line 77) | struct fuse_opt
type fuse_args (line 80) | struct fuse_args
type fuse_opt (line 81) | struct fuse_opt
type fuse_args (line 88) | struct fuse_args
type fuse_args (line 95) | struct fuse_args
type fuse_args (line 102) | struct fuse_args
type fuse_opt (line 123) | struct fuse_opt
FILE: hack/winfsp_headers/winfsp_fuse.h
type fuse_uid_t (line 106) | typedef uint32_t fuse_uid_t;
type fuse_gid_t (line 107) | typedef uint32_t fuse_gid_t;
type fuse_pid_t (line 108) | typedef int32_t fuse_pid_t;
type fuse_dev_t (line 110) | typedef uint32_t fuse_dev_t;
type fuse_ino_t (line 111) | typedef uint64_t fuse_ino_t;
type fuse_mode_t (line 112) | typedef uint32_t fuse_mode_t;
type fuse_nlink_t (line 113) | typedef uint16_t fuse_nlink_t;
type fuse_off_t (line 114) | typedef int64_t fuse_off_t;
type fuse_fsblkcnt_t (line 117) | typedef uint64_t fuse_fsblkcnt_t;
type fuse_fsfilcnt_t (line 118) | typedef uint64_t fuse_fsfilcnt_t;
type fuse_fsblkcnt_t (line 120) | typedef uint32_t fuse_fsblkcnt_t;
type fuse_fsfilcnt_t (line 121) | typedef uint32_t fuse_fsfilcnt_t;
type fuse_blksize_t (line 123) | typedef int32_t fuse_blksize_t;
type fuse_blkcnt_t (line 124) | typedef int64_t fuse_blkcnt_t;
type fuse_utimbuf (line 127) | struct fuse_utimbuf
type fuse_timespec (line 132) | struct fuse_timespec
type fuse_utimbuf (line 138) | struct fuse_utimbuf
type fuse_timespec (line 143) | struct fuse_timespec
type fuse_stat (line 156) | struct fuse_stat
FILE: main.go
function main (line 28) | func main() {
FILE: pkg/acl/acl.go
constant Version (line 26) | Version uint8 = 2
type Entry (line 28) | type Entry struct
type Entries (line 33) | type Entries
method Len (line 35) | func (es *Entries) Len() int { return len(*es) }
method Less (line 36) | func (es *Entries) Less(i, j int) bool { return (*es)[i].Id < (*es)[j]...
method Swap (line 37) | func (es *Entries) Swap(i, j int) { (*es)[i], (*es)[j] = (*es)[j]...
method IsEqual (line 39) | func (es *Entries) IsEqual(other *Entries) bool {
method Encode (line 51) | func (es *Entries) Encode() []byte {
method Decode (line 60) | func (es *Entries) Decode(data []byte) {
type Rule (line 71) | type Rule struct
method String (line 80) | func (r *Rule) String() string {
method Dup (line 85) | func (r *Rule) Dup() *Rule {
method Encode (line 94) | func (r *Rule) Encode() []byte {
method Decode (line 113) | func (r *Rule) Decode(buf []byte) {
method IsEmpty (line 143) | func (r *Rule) IsEmpty() bool {
method IsMinimal (line 149) | func (r *Rule) IsMinimal() bool {
method IsEqual (line 153) | func (r *Rule) IsEqual(other *Rule) bool {
method InheritPerms (line 163) | func (r *Rule) InheritPerms(mode uint16) {
method SetMode (line 175) | func (r *Rule) SetMode(mode uint16) {
method GetMode (line 190) | func (r *Rule) GetMode() uint16 {
method ChildAccessACL (line 198) | func (r *Rule) ChildAccessACL(mode uint16) *Rule {
method Checksum (line 212) | func (r *Rule) Checksum() uint32 {
method CanAccess (line 216) | func (r *Rule) CanAccess(uid uint32, gids []uint32, fUid, fGid uint32,...
function EmptyRule (line 134) | func EmptyRule() *Rule {
constant TypeNone (line 253) | TypeNone = iota
constant TypeAccess (line 254) | TypeAccess
constant TypeDefault (line 255) | TypeDefault
FILE: pkg/acl/cache.go
constant None (line 23) | None = 0
type Cache (line 30) | type Cache interface
function NewCache (line 40) | func NewCache() Cache {
type cache (line 49) | type cache struct
method GetAll (line 56) | func (c *cache) GetAll() map[uint32]*Rule {
method Clear (line 67) | func (c *cache) Clear() {
method GetMissIds (line 76) | func (c *cache) GetMissIds() []uint32 {
method Size (line 94) | func (c *cache) Size() int {
method Get (line 100) | func (c *cache) Get(id uint32) *Rule {
method Put (line 109) | func (c *cache) Put(id uint32, r *Rule) {
method GetId (line 132) | func (c *cache) GetId(r *Rule) uint32 {
FILE: pkg/acl/cache_test.go
function TestCache (line 25) | func TestCache(t *testing.T) {
FILE: pkg/chunk/cache_eviction.go
constant EvictionNone (line 27) | EvictionNone = "none"
constant Eviction2Random (line 28) | Eviction2Random = "2-random"
constant EvictionLRU (line 29) | EvictionLRU = "lru"
constant notInLru (line 32) | notInLru = math.MinInt
type cacheItem (line 34) | type cacheItem struct
type KeyIndex (line 39) | type KeyIndex interface
function NewKeyIndex (line 54) | func NewKeyIndex(config *Config) (KeyIndex, error) {
type noneEviction (line 74) | type noneEviction struct
method name (line 78) | func (p *noneEviction) name() string {
method add (line 82) | func (p *noneEviction) add(key cacheKey, item cacheItem) {
method remove (line 86) | func (p *noneEviction) remove(key cacheKey, staging bool) *cacheItem {
method get (line 98) | func (p *noneEviction) get(key cacheKey) *cacheItem {
method peekAtime (line 107) | func (p *noneEviction) peekAtime(key cacheKey) uint32 {
method len (line 111) | func (p *noneEviction) len() int {
method reset (line 115) | func (p *noneEviction) reset() KeyIndex {
method randomIter (line 121) | func (p *noneEviction) randomIter() func(yield func(key cacheKey, item...
method evictionIter (line 131) | func (p *noneEviction) evictionIter() func(yield func(key cacheKey, it...
type randomEviction (line 136) | type randomEviction struct
method name (line 141) | func (p *randomEviction) name() string {
method reset (line 145) | func (p *randomEviction) reset() KeyIndex {
method evictionIter (line 154) | func (p *randomEviction) evictionIter() func(yield func(key cacheKey, ...
type lruItem (line 185) | type lruItem struct
type atimeHeap (line 191) | type atimeHeap
method Len (line 198) | func (h atimeHeap) Len() int { return len(h) }
method Less (line 200) | func (h atimeHeap) Less(i, j int) bool { // min-heap
method Swap (line 210) | func (h atimeHeap) Swap(i, j int) {
method Push (line 216) | func (h *atimeHeap) Push(x any) {
method Pop (line 222) | func (h *atimeHeap) Pop() any {
type heapItem (line 193) | type heapItem struct
type lruEviction (line 232) | type lruEviction struct
method name (line 237) | func (p *lruEviction) name() string {
method add (line 241) | func (p *lruEviction) add(key cacheKey, item cacheItem) {
method remove (line 260) | func (p *lruEviction) remove(key cacheKey, staging bool) *cacheItem {
method get (line 275) | func (p *lruEviction) get(key cacheKey) *cacheItem {
method peekAtime (line 287) | func (p *lruEviction) peekAtime(key cacheKey) uint32 {
method len (line 294) | func (p *lruEviction) len() int {
method reset (line 298) | func (p *lruEviction) reset() KeyIndex {
method randomIter (line 308) | func (p *lruEviction) randomIter() func(yield func(key cacheKey, item ...
method evictionIter (line 318) | func (p *lruEviction) evictionIter() func(yield func(key cacheKey, ite...
method verifyHeap (line 335) | func (p *lruEviction) verifyHeap() bool {
FILE: pkg/chunk/cached_store.go
constant chunkSize (line 40) | chunkSize = 1 << 26
constant pageSize (line 41) | pageSize = 1 << 16
constant SlowRequest (line 42) | SlowRequest = time.Second * time.Duration(10)
type pendingItem (line 48) | type pendingItem struct
type rSlice (line 56) | type rSlice struct
method blockSize (line 66) | func (s *rSlice) blockSize(indx int) int {
method key (line 74) | func (s *rSlice) key(indx int) string {
method index (line 81) | func (s *rSlice) index(off int) int {
method keys (line 85) | func (s *rSlice) keys() []string {
method ReadAt (line 97) | func (s *rSlice) ReadAt(ctx context.Context, page *Page, off int) (n i...
method delete (line 182) | func (s *rSlice) delete(indx int) error {
method Remove (line 187) | func (s *rSlice) Remove() error {
function sliceForRead (line 62) | func sliceForRead(id uint64, length int, store *cachedStore) *rSlice {
function allocPage (line 213) | func allocPage(sz int) *Page {
function freePage (line 225) | func freePage(p *Page) {
type wSlice (line 238) | type wSlice struct
method SetID (line 257) | func (s *wSlice) SetID(id uint64) {
method SetWriteback (line 261) | func (s *wSlice) SetWriteback(enabled bool) {
method WriteAt (line 265) | func (s *wSlice) WriteAt(p []byte, off int64) (n int, err error) {
method upload (line 398) | func (s *wSlice) upload(indx int) {
method ID (line 468) | func (s *wSlice) ID() uint64 {
method Len (line 472) | func (s *wSlice) Len() int {
method FlushTo (line 476) | func (s *wSlice) FlushTo(offset int) error {
method Finish (line 494) | func (s *wSlice) Finish(length int) error {
method Abort (line 512) | func (s *wSlice) Abort() {
function sliceForWrite (line 248) | func sliceForWrite(id uint64, store *cachedStore) *wSlice {
type Config (line 525) | type Config struct
method SelfCheck (line 559) | func (c *Config) SelfCheck(uuid string) {
method parseHours (line 634) | func (c *Config) parseHours() (start, end int, err error) {
method CacheEnabled (line 659) | func (c *Config) CacheEnabled() bool {
type cachedStore (line 663) | type cachedStore struct
method put (line 310) | func (store *cachedStore) put(key string, p *Page) error {
method delete (line 334) | func (store *cachedStore) delete(key string) error {
method upload (line 354) | func (store *cachedStore) upload(key string, block *Page, s *wSlice) e...
method loadRange (line 703) | func (store *cachedStore) loadRange(ctx context.Context, key string, p...
method load (line 752) | func (store *cachedStore) load(ctx context.Context, key string, page *...
method initMetrics (line 928) | func (store *cachedStore) initMetrics() {
method regMetrics (line 973) | func (store *cachedStore) regMetrics(reg prometheus.Registerer) {
method shouldCache (line 1015) | func (store *cachedStore) shouldCache(size int) bool {
method uploadStagingFile (line 1025) | func (store *cachedStore) uploadStagingFile(key string, stagingPath st...
method addDelayedStaging (line 1085) | func (store *cachedStore) addDelayedStaging(key, stagingPath string, a...
method removePending (line 1108) | func (store *cachedStore) removePending(key string) {
method isPendingValid (line 1114) | func (store *cachedStore) isPendingValid(key string) bool {
method scanDelayedStaging (line 1121) | func (store *cachedStore) scanDelayedStaging() {
method uploader (line 1137) | func (store *cachedStore) uploader() {
method canUpload (line 1143) | func (store *cachedStore) canUpload() bool {
method NewReader (line 1152) | func (store *cachedStore) NewReader(id uint64, length int) Reader {
method NewWriter (line 1156) | func (store *cachedStore) NewWriter(id uint64) Writer {
method Remove (line 1160) | func (store *cachedStore) Remove(id uint64, length int) error {
method FillCache (line 1165) | func (store *cachedStore) FillCache(id uint64, length uint32) error {
method EvictCache (line 1188) | func (store *cachedStore) EvictCache(id uint64, length uint32) error {
method CheckCache (line 1197) | func (store *cachedStore) CheckCache(id uint64, length uint32, handler...
method UsedMemory (line 1211) | func (store *cachedStore) UsedMemory() int64 {
method UpdateLimit (line 1215) | func (store *cachedStore) UpdateLimit(upload, download int64) {
function logRequest (line 693) | func logRequest(typeStr, key, param, reqID string, err error, used time....
function NewCachedStore (line 823) | func NewCachedStore(storage object.ObjectStorage, config Config, reg pro...
function parseObjOrigSize (line 1019) | func parseObjOrigSize(key string) int {
FILE: pkg/chunk/cached_store_test.go
function forgetSlice (line 36) | func forgetSlice(store ChunkStore, sliceId uint64, size int) error {
function testStore (line 45) | func testStore(t *testing.T, store ChunkStore) {
function TestStoreDefault (line 121) | func TestStoreDefault(t *testing.T) {
function TestStoreMemCache (line 134) | func TestStoreMemCache(t *testing.T) {
function TestStoreCompressed (line 147) | func TestStoreCompressed(t *testing.T) {
function TestStoreLimited (line 156) | func TestStoreLimited(t *testing.T) {
function TestStoreFull (line 165) | func TestStoreFull(t *testing.T) {
function TestStoreSmallBuffer (line 173) | func TestStoreSmallBuffer(t *testing.T) {
function TestStoreAsync (line 181) | func TestStoreAsync(t *testing.T) {
function TestForceUpload (line 203) | func TestForceUpload(t *testing.T) {
function TestStoreDelayed (line 254) | func TestStoreDelayed(t *testing.T) {
function TestStoreMultiBuckets (line 272) | func TestStoreMultiBuckets(t *testing.T) {
function TestFillCache (line 280) | func TestFillCache(t *testing.T) {
function BenchmarkCachedRead (line 346) | func BenchmarkCachedRead(b *testing.B) {
function BenchmarkUncachedRead (line 369) | func BenchmarkUncachedRead(b *testing.B) {
type dStore (line 392) | type dStore struct
method Get (line 397) | func (s *dStore) Get(ctx context.Context, key string, off, limit int64...
function TestStoreRetry (line 402) | func TestStoreRetry(t *testing.T) {
FILE: pkg/chunk/chunk.go
type Reader (line 24) | type Reader interface
type Writer (line 28) | type Writer interface
type ChunkStore (line 38) | type ChunkStore interface
FILE: pkg/chunk/disk_cache.go
type cacheKey (line 59) | type cacheKey struct
method String (line 65) | func (k cacheKey) String() string { return fmt.Sprintf("%d_%d_%d", k.i...
type pendingFile (line 67) | type pendingFile struct
type cacheStore (line 73) | type cacheStore struct
method setLimitByFreeRatio (line 171) | func (cache *cacheStore) setLimitByFreeRatio(usage DiskFreeRatio, free...
method lockFilePath (line 194) | func (cache *cacheStore) lockFilePath() string {
method createLockFile (line 198) | func (cache *cacheStore) createLockFile() {
method checkLockFile (line 226) | func (cache *cacheStore) checkLockFile() {
method available (line 240) | func (c *cacheStore) available() bool {
method enabled (line 244) | func (c *cacheStore) enabled() bool {
method full (line 248) | func (c *cacheStore) full() bool {
method checkErr (line 252) | func (cache *cacheStore) checkErr(f func() error) error {
method checkTimeout (line 286) | func (c *cacheStore) checkTimeout() {
method statFile (line 303) | func (c *cacheStore) statFile(path string) error {
method removeFile (line 310) | func (cache *cacheStore) removeFile(path string) error {
method renameFile (line 316) | func (cache *cacheStore) renameFile(oldpath, newpath string) error {
method writeFile (line 322) | func (cache *cacheStore) writeFile(f *os.File, data []byte) error {
method closeFile (line 329) | func (cache *cacheStore) closeFile(f *os.File) error {
method usedMemory (line 335) | func (cache *cacheStore) usedMemory() int64 {
method stats (line 339) | func (cache *cacheStore) stats() (int64, int64) {
method checkFreeSpace (line 345) | func (cache *cacheStore) checkFreeSpace() {
method cleanupExpire (line 366) | func (cache *cacheStore) cleanupExpire() {
method refreshCacheKeys (line 410) | func (cache *cacheStore) refreshCacheKeys() {
method removeStage (line 423) | func (cache *cacheStore) removeStage(key string) error {
method cache (line 436) | func (cache *cacheStore) cache(key string, p *Page, force, dropCache b...
method curFreeRatio (line 483) | func (cache *cacheStore) curFreeRatio() DiskFreeRatio {
method flushPage (line 502) | func (cache *cacheStore) flushPage(path string, data []byte, dropCache...
method createDir (line 557) | func (cache *cacheStore) createDir(dir string) {
method getCacheKey (line 579) | func (cache *cacheStore) getCacheKey(key string) cacheKey {
method getPathFromKey (line 610) | func (cache *cacheStore) getPathFromKey(k cacheKey) string {
method remove (line 618) | func (cache *cacheStore) remove(key string, staging bool) {
method load (line 644) | func (cache *cacheStore) load(key string) (ReadCloser, error) {
method exist (line 675) | func (cache *cacheStore) exist(key string) (bool, error) {
method cachePath (line 704) | func (cache *cacheStore) cachePath(key string) string {
method stagePath (line 708) | func (cache *cacheStore) stagePath(key string) string {
method flush (line 713) | func (cache *cacheStore) flush() {
method add (line 732) | func (cache *cacheStore) add(key string, size int32, atime uint32) {
method stage (line 759) | func (cache *cacheStore) stage(key string, data []byte) (string, error) {
method uploaded (line 787) | func (cache *cacheStore) uploaded(key string, size int) {
method cleanupFull (line 792) | func (cache *cacheStore) cleanupFull() {
method uploadStaging (line 855) | func (cache *cacheStore) uploadStaging() {
method scanCached (line 911) | func (cache *cacheStore) scanCached() {
method scanStaging (line 969) | func (cache *cacheStore) scanStaging() {
function newCacheStore (line 109) | func newCacheStore(m *cacheManagerMetrics, dir string, cacheSize, maxIte...
function getFunctionName (line 282) | func getFunctionName(f interface{}) string {
type DiskFreeRatio (line 475) | type DiskFreeRatio struct
type cacheManager (line 1023) | type cacheManager struct
method getMetrics (line 1136) | func (m *cacheManager) getMetrics() *cacheManagerMetrics {
method cleanup (line 1140) | func (m *cacheManager) cleanup() {
method isEmpty (line 1157) | func (m *cacheManager) isEmpty() bool {
method length (line 1161) | func (m *cacheManager) length() int {
method removeStore (line 1167) | func (m *cacheManager) removeStore(id string) {
method getStore (line 1184) | func (m *cacheManager) getStore(key string) *cacheStore {
method removeStage (line 1197) | func (m *cacheManager) removeStage(key string) error {
method getStoreLegacy (line 1206) | func (m *cacheManager) getStoreLegacy(key string) *cacheStore {
method usedMemory (line 1210) | func (m *cacheManager) usedMemory() int64 {
method stats (line 1220) | func (m *cacheManager) stats() (int64, int64) {
method cache (line 1232) | func (m *cacheManager) cache(key string, p *Page, force, dropCache boo...
method load (line 1245) | func (m *cacheManager) load(key string) (ReadCloser, error) {
method exist (line 1260) | func (m *cacheManager) exist(key string) (string, bool) {
method remove (line 1277) | func (m *cacheManager) remove(key string, staging bool) {
method stage (line 1284) | func (m *cacheManager) stage(key string, data []byte) (string, error) {
method uploaded (line 1292) | func (m *cacheManager) uploaded(key string, size int) {
function legacyKeyHash (line 1031) | func legacyKeyHash(s string) uint32 {
function hasMeta (line 1039) | func hasMeta(path string) bool {
function expandDir (line 1049) | func expandDir(pattern string) []string {
type CacheManager (line 1076) | type CacheManager interface
function newCacheManager (line 1090) | func newCacheManager(config *Config, reg prometheus.Registerer, uploader...
type ReadCloser (line 1239) | type ReadCloser interface
constant CsNone (line 1301) | CsNone = "none"
constant CsFull (line 1302) | CsFull = "full"
constant CsShrink (line 1303) | CsShrink = "shrink"
constant CsExtend (line 1304) | CsExtend = "extend"
constant csBlock (line 1306) | csBlock = 32 << 10
type cacheFile (line 1311) | type cacheFile struct
method ReadAt (line 1354) | func (cf *cacheFile) ReadAt(b []byte, off int64) (n int, err error) {
function checksum (line 1318) | func checksum(data []byte) []byte {
function openCacheFile (line 1332) | func openCacheFile(name string, length int, level string) (*cacheFile, e...
FILE: pkg/chunk/disk_cache_state.go
constant dcUnknown (line 57) | dcUnknown = iota
constant dcNormal (line 58) | dcNormal
constant dcUnstable (line 59) | dcUnstable
constant dcDown (line 60) | dcDown
constant dcUnchanged (line 61) | dcUnchanged
constant eventUnknown (line 65) | eventUnknown = iota
constant eventToNormal (line 66) | eventToNormal
constant eventToUnstable (line 67) | eventToUnstable
constant eventToDown (line 68) | eventToDown
type dcState (line 72) | type dcState interface
type baseDC (line 84) | type baseDC struct
method init (line 106) | func (dc *baseDC) init(cs *cacheStore) {
method stop (line 111) | func (dc *baseDC) stop() {
method onIOErr (line 114) | func (dc *baseDC) onIOErr() {}
method onIOSucc (line 115) | func (dc *baseDC) onIOSucc() {}
method state (line 116) | func (dc *baseDC) state() int { return dcUnknown }
method tick (line 117) | func (dc *baseDC) tick() {}
method checkCacheOp (line 118) | func (dc *baseDC) checkCacheOp() error { return nil }
method beforeCacheOp (line 119) | func (dc *baseDC) beforeCacheOp() {}
method afterCacheOp (line 120) | func (dc *baseDC) afterCacheOp() {}
function newDCState (line 89) | func newDCState(state int, cs *cacheStore) dcState {
type unchangedDC (line 122) | type unchangedDC struct
method state (line 126) | func (dc *unchangedDC) state() int { return dcUnchanged }
type normalDC (line 128) | type normalDC struct
method state (line 133) | func (dc *normalDC) state() int { return dcNormal }
method init (line 135) | func (dc *normalDC) init(cs *cacheStore) {
method tick (line 140) | func (dc *normalDC) tick() {
method onIOErr (line 153) | func (dc *normalDC) onIOErr() {
type unstableDC (line 160) | type unstableDC struct
method state (line 169) | func (dc *unstableDC) state() int { return dcUnstable }
method init (line 171) | func (dc *unstableDC) init(cs *cacheStore) {
method onIOErr (line 176) | func (dc *unstableDC) onIOErr() {
method onIOSucc (line 181) | func (dc *unstableDC) onIOSucc() {
method tick (line 189) | func (dc *unstableDC) tick() {
method probe (line 214) | func (dc *unstableDC) probe() {
method doProbe (line 235) | func (dc *unstableDC) doProbe(key string, page *Page) {
method beforeCacheOp (line 246) | func (dc *unstableDC) beforeCacheOp() { dc.concurrency.Add(1) }
method afterCacheOp (line 247) | func (dc *unstableDC) afterCacheOp() { dc.concurrency.Add(-1) }
method checkCacheOp (line 249) | func (dc *unstableDC) checkCacheOp() error {
function probeCacheKey (line 185) | func probeCacheKey(id, size int) string {
type downDC (line 256) | type downDC struct
method state (line 260) | func (dc *downDC) state() int { return dcDown }
method checkCacheOp (line 261) | func (dc *downDC) checkCacheOp() error { return errCacheDown }
method event (line 263) | func (cache *cacheStore) event(eventType int) {
function getEnvs (line 286) | func getEnvs() {
FILE: pkg/chunk/disk_cache_state_test.go
function setState (line 31) | func setState(s *cacheStore, state int) {
function testDiskCacheState (line 38) | func testDiskCacheState(t *testing.T, cacheNum int) {
function TestDiskCacheState (line 115) | func TestDiskCacheState(t *testing.T) {
FILE: pkg/chunk/disk_cache_test.go
function toFloat64 (line 37) | func toFloat64(c prometheus.Collector) float64 {
function testConf (line 76) | func testConf() Config {
function TestNewCacheStore (line 82) | func TestNewCacheStore(t *testing.T) {
function TestMetrics (line 91) | func TestMetrics(t *testing.T) {
function TestScanCached (line 147) | func TestScanCached(t *testing.T) {
function TestChecksum (line 173) | func TestChecksum(t *testing.T) {
function TestExpand (line 281) | func TestExpand(t *testing.T) {
function BenchmarkLoadCached (line 302) | func BenchmarkLoadCached(b *testing.B) {
function BenchmarkLoadUncached (line 320) | func BenchmarkLoadUncached(b *testing.B) {
function TestCheckPath (line 333) | func TestCheckPath(t *testing.T) {
function shutdownStore (line 369) | func shutdownStore(s *cacheStore) {
function TestCacheManager (line 376) | func TestCacheManager(t *testing.T) {
function TestAtimeNotLost (line 442) | func TestAtimeNotLost(t *testing.T) {
function TestSetlimitByFreeRatio (line 471) | func TestSetlimitByFreeRatio(t *testing.T) {
function TestSetLimitByFreeRatioUnknownInodesKeepExplicitMaxItems (line 493) | func TestSetLimitByFreeRatioUnknownInodesKeepExplicitMaxItems(t *testing...
function TestUnknownInodeStatsShouldNotMarkCacheAsRawFull (line 506) | func TestUnknownInodeStatsShouldNotMarkCacheAsRawFull(t *testing.T) {
function Test2RandomEviction (line 527) | func Test2RandomEviction(t *testing.T) {
function TestLruEviction (line 555) | func TestLruEviction(t *testing.T) {
function TestCooldownAtimeOnWriteFixedOnLoad (line 656) | func TestCooldownAtimeOnWriteFixedOnLoad(t *testing.T) {
FILE: pkg/chunk/mem_cache.go
type memItem (line 28) | type memItem struct
type memcache (line 33) | type memcache struct
method removeStage (line 66) | func (c *memcache) removeStage(key string) error {
method usedMemory (line 70) | func (c *memcache) usedMemory() int64 {
method stats (line 76) | func (c *memcache) stats() (int64, int64) {
method cache (line 82) | func (c *memcache) cache(key string, p *Page, force, dropCache bool) {
method delete (line 107) | func (c *memcache) delete(key string, p *Page) {
method remove (line 114) | func (c *memcache) remove(key string, staging bool) {
method load (line 123) | func (c *memcache) load(key string) (ReadCloser, error) {
method exist (line 133) | func (c *memcache) exist(key string) (string, bool) {
method cleanup (line 147) | func (c *memcache) cleanup() {
method enabled (line 171) | func (c *memcache) enabled() bool {
method full (line 175) | func (c *memcache) full() bool {
method cleanupExpire (line 179) | func (c *memcache) cleanupExpire() {
method stage (line 209) | func (c *memcache) stage(key string, data []byte) (string, error) {
method uploaded (line 212) | func (c *memcache) uploaded(key string, size int) {}
method isEmpty (line 213) | func (c *memcache) isEmpty() bool { return false }
method getMetrics (line 214) | func (c *memcache) getMetrics() *cacheManagerMetrics { return c.metrics }
function newMemStore (line 45) | func newMemStore(config *Config, metrics *cacheManagerMetrics) *memcache {
FILE: pkg/chunk/metrics.go
type cacheManagerMetrics (line 24) | type cacheManagerMetrics struct
method registerMetrics (line 42) | func (c *cacheManagerMetrics) registerMetrics(reg prometheus.Registere...
method initMetrics (line 61) | func (c *cacheManagerMetrics) initMetrics() {
function newCacheManagerMetrics (line 35) | func newCacheManagerMetrics(reg prometheus.Registerer) *cacheManagerMetr...
FILE: pkg/chunk/page.go
type Page (line 33) | type Page struct
method Slice (line 67) | func (p *Page) Slice(off, len int) *Page {
method Acquire (line 75) | func (p *Page) Acquire() {
method Release (line 83) | func (p *Page) Release() {
function NewPage (line 42) | func NewPage(data []byte) *Page {
function NewOffPage (line 46) | func NewOffPage(size int) *Page {
type pageReader (line 99) | type pageReader struct
method Read (line 109) | func (r *pageReader) Read(buf []byte) (int, error) {
method ReadAt (line 115) | func (r *pageReader) ReadAt(buf []byte, off int64) (int, error) {
method Close (line 132) | func (r *pageReader) Close() error {
function NewPageReader (line 104) | func NewPageReader(p *Page) *pageReader {
FILE: pkg/chunk/page_test.go
function TestPage (line 24) | func TestPage(t *testing.T) {
function TestPageReader (line 53) | func TestPageReader(t *testing.T) {
FILE: pkg/chunk/prefetch.go
type prefetcher (line 23) | type prefetcher struct
method do (line 42) | func (p *prefetcher) do() {
method fetch (line 52) | func (p *prefetcher) fetch(key string) {
function newPrefetcher (line 30) | func newPrefetcher(parallel int, fetch func(string)) *prefetcher {
FILE: pkg/chunk/prefetch_test.go
function TestPrefetcher (line 9) | func TestPrefetcher(t *testing.T) {
FILE: pkg/chunk/singleflight.go
type request (line 21) | type request struct
type Controller (line 28) | type Controller struct
method Execute (line 39) | func (con *Controller) Execute(key string, fn func() (*Page, error)) (...
method TryPiggyback (line 67) | func (con *Controller) TryPiggyback(key string) (*Page, error) {
function NewController (line 33) | func NewController() *Controller {
FILE: pkg/chunk/singleflight_test.go
function TestSingleFlight (line 29) | func TestSingleFlight(t *testing.T) {
FILE: pkg/chunk/utils_darwin.go
function getAtime (line 25) | func getAtime(fi os.FileInfo) time.Time {
function dropOSCache (line 33) | func dropOSCache(r ReadCloser) {}
FILE: pkg/chunk/utils_linux.go
function getAtime (line 27) | func getAtime(fi os.FileInfo) time.Time {
function dropOSCache (line 34) | func dropOSCache(r ReadCloser) {
FILE: pkg/chunk/utils_unix.go
function getNlink (line 27) | func getNlink(fi os.FileInfo) int {
function getDiskUsage (line 34) | func getDiskUsage(path string) (uint64, uint64, uint64, uint64) {
function changeMode (line 44) | func changeMode(dir string, st os.FileInfo, mode os.FileMode) {
function inRootVolume (line 51) | func inRootVolume(dir string) bool {
FILE: pkg/chunk/utils_unix_test.go
function TestInRootVolume (line 28) | func TestInRootVolume(t *testing.T) {
FILE: pkg/chunk/utils_windows.go
function getAtime (line 27) | func getAtime(fi os.FileInfo) time.Time {
function dropOSCache (line 36) | func dropOSCache(r ReadCloser) {}
function getNlink (line 38) | func getNlink(fi os.FileInfo) int {
function getDiskUsage (line 42) | func getDiskUsage(path string) (uint64, uint64, uint64, uint64) {
function changeMode (line 52) | func changeMode(dir string, st os.FileInfo, mode os.FileMode) {}
function inRootVolume (line 54) | func inRootVolume(dir string) bool { return false }
FILE: pkg/compress/compress.go
constant ZSTD_LEVEL (line 28) | ZSTD_LEVEL = 1
type Compressor (line 31) | type Compressor interface
function NewCompressor (line 39) | func NewCompressor(algr string) Compressor {
type noOp (line 51) | type noOp struct
method Name (line 53) | func (n noOp) Name() string { return "Noop" }
method CompressBound (line 54) | func (n noOp) CompressBound(l int) int { return l }
method Compress (line 55) | func (n noOp) Compress(dst, src []byte) (int, error) {
method Decompress (line 62) | func (n noOp) Decompress(dst, src []byte) (int, error) {
type ZStandard (line 71) | type ZStandard struct
method Name (line 76) | func (n ZStandard) Name() string { return "Zstd" }
method CompressBound (line 79) | func (n ZStandard) CompressBound(l int) int { return zstd.CompressBoun...
method Compress (line 82) | func (n ZStandard) Compress(dst, src []byte) (int, error) {
method Decompress (line 94) | func (n ZStandard) Decompress(dst, src []byte) (int, error) {
type LZ4 (line 106) | type LZ4 struct
method Name (line 109) | func (l LZ4) Name() string { return "LZ4" }
method CompressBound (line 112) | func (l LZ4) CompressBound(size int) int { return lz4.CompressBound(si...
method Compress (line 115) | func (l LZ4) Compress(dst, src []byte) (int, error) {
method Decompress (line 120) | func (l LZ4) Decompress(dst, src []byte) (int, error) {
FILE: pkg/compress/compress_test.go
function testCompress (line 25) | func testCompress(t *testing.T, c Compressor) {
function TestUncompressed (line 66) | func TestUncompressed(t *testing.T) {
function TestZstd (line 70) | func TestZstd(t *testing.T) {
function TestLZ4 (line 74) | func TestLZ4(t *testing.T) {
function benchmarkDecompress (line 78) | func benchmarkDecompress(b *testing.B, comp Compressor) {
function BenchmarkDecompressZstd (line 107) | func BenchmarkDecompressZstd(b *testing.B) {
function BenchmarkDecompressLZ4 (line 111) | func BenchmarkDecompressLZ4(b *testing.B) {
function BenchmarkDecompressNone (line 115) | func BenchmarkDecompressNone(b *testing.B) {
function benchmarkCompress (line 119) | func benchmarkCompress(b *testing.B, comp Compressor) {
function BenchmarkCompressZstd (line 142) | func BenchmarkCompressZstd(b *testing.B) {
function BenchmarkCompressCLZ4 (line 146) | func BenchmarkCompressCLZ4(b *testing.B) {
function BenchmarkCompressNone (line 149) | func BenchmarkCompressNone(b *testing.B) {
FILE: pkg/fs/fs.go
function IsExist (line 51) | func IsExist(err error) bool {
function IsNotExist (line 55) | func IsNotExist(err error) bool {
function IsNotEmpty (line 59) | func IsNotEmpty(err error) bool {
function errstr (line 63) | func errstr(e error) string {
type FileStat (line 73) | type FileStat struct
method Inode (line 79) | func (fs *FileStat) Inode() Ino { return fs.inode }
method Name (line 80) | func (fs *FileStat) Name() string { return fs.name }
method Size (line 81) | func (fs *FileStat) Size() int64 { return int64(fs.attr.Length) }
method Mode (line 82) | func (fs *FileStat) Mode() os.FileMode {
method ModTime (line 107) | func (fs *FileStat) ModTime() time.Time {
method IsDir (line 110) | func (fs *FileStat) IsDir() bool { return fs.attr.Typ == meta.Typ...
method IsSymlink (line 111) | func (fs *FileStat) IsSymlink() bool { return fs.attr.Typ == meta.Typ...
method Sys (line 112) | func (fs *FileStat) Sys() interface{} { return fs.attr }
method Uid (line 113) | func (fs *FileStat) Uid() int { return int(fs.attr.Uid) }
method Gid (line 114) | func (fs *FileStat) Gid() int { return int(fs.attr.Gid) }
method Atime (line 116) | func (fs *FileStat) Atime() int64 { return fs.attr.Atime*1000 + int64(...
method Mtime (line 117) | func (fs *FileStat) Mtime() int64 { return fs.attr.Mtime*1000 + int64(...
method Attr (line 119) | func (fs *FileStat) Attr() *Attr { return fs.attr }
function AttrToFileInfo (line 121) | func AttrToFileInfo(inode Ino, attr *Attr) *FileStat {
type entryCache (line 125) | type entryCache struct
type attrCache (line 131) | type attrCache struct
type FileSystem (line 136) | type FileSystem struct
method InitMetrics (line 238) | func (fs *FileSystem) InitMetrics(reg prometheus.Registerer) {
method cleanupCache (line 247) | func (fs *FileSystem) cleanupCache() {
method InvalidateEntry (line 282) | func (fs *FileSystem) InvalidateEntry(parent Ino, name string) {
method InvalidateAttr (line 294) | func (fs *FileSystem) InvalidateAttr(ino Ino) {
method log (line 300) | func (fs *FileSystem) log(ctx LogContext, format string, args ...inter...
method flushLog (line 318) | func (fs *FileSystem) flushLog(f *os.File, logBuffer chan string, path...
method Meta (line 372) | func (fs *FileSystem) Meta() meta.Meta {
method StatFS (line 376) | func (fs *FileSystem) StatFS(ctx meta.Context) (totalspace uint64, ava...
method Lopen (line 386) | func (fs *FileSystem) Lopen(ctx meta.Context, path string, flags uint3...
method Open (line 390) | func (fs *FileSystem) Open(ctx meta.Context, path string, flags uint32...
method open (line 394) | func (fs *FileSystem) open(ctx meta.Context, path string, flags uint32...
method Access (line 441) | func (fs *FileSystem) Access(ctx meta.Context, path string, flags int)...
method Stat (line 456) | func (fs *FileSystem) Stat(ctx meta.Context, path string) (fi *FileSta...
method Lstat (line 463) | func (fs *FileSystem) Lstat(ctx meta.Context, path string) (fi *FileSt...
method Mkdir (line 475) | func (fs *FileSystem) Mkdir(ctx meta.Context, p string, mode uint16, u...
method MkdirAll (line 506) | func (fs *FileSystem) MkdirAll(ctx meta.Context, p string, mode uint16...
method MkdirAll0 (line 510) | func (fs *FileSystem) MkdirAll0(ctx meta.Context, p string, mode uint1...
method Unlink (line 524) | func (fs *FileSystem) Unlink(ctx meta.Context, p string) (err syscall....
method Delete (line 531) | func (fs *FileSystem) Delete(ctx meta.Context, p string) (err syscall....
method BatchDeleteEntries (line 538) | func (fs *FileSystem) BatchDeleteEntries(ctx meta.Context, parent stri...
method Delete0 (line 567) | func (fs *FileSystem) Delete0(ctx meta.Context, p string, callByUnlink...
method Rmdir (line 589) | func (fs *FileSystem) Rmdir(ctx meta.Context, p string) (err syscall.E...
method Rmr (line 602) | func (fs *FileSystem) Rmr(ctx meta.Context, p string, skipTrash bool, ...
method Rename (line 635) | func (fs *FileSystem) Rename(ctx meta.Context, oldpath string, newpath...
method Link (line 675) | func (fs *FileSystem) Link(ctx meta.Context, src string, dst string) (...
method Symlink (line 693) | func (fs *FileSystem) Symlink(ctx meta.Context, target string, link st...
method Readlink (line 709) | func (fs *FileSystem) Readlink(ctx meta.Context, link string) (path []...
method Truncate (line 721) | func (fs *FileSystem) Truncate(ctx meta.Context, path string, length u...
method CopyFileRange (line 736) | func (fs *FileSystem) CopyFileRange(ctx meta.Context, src string, soff...
method SetXattr (line 755) | func (fs *FileSystem) SetXattr(ctx meta.Context, p string, name string...
method GetXattr (line 767) | func (fs *FileSystem) GetXattr(ctx meta.Context, p string, name string...
method ListXattr (line 779) | func (fs *FileSystem) ListXattr(ctx meta.Context, p string) (names []b...
method RemoveXattr (line 791) | func (fs *FileSystem) RemoveXattr(ctx meta.Context, p string, name str...
method GetFacl (line 803) | func (fs *FileSystem) GetFacl(ctx meta.Context, p string, acltype uint...
method SetFacl (line 815) | func (fs *FileSystem) SetFacl(ctx meta.Context, p string, acltype uint...
method lookup (line 845) | func (fs *FileSystem) lookup(ctx meta.Context, parent Ino, name string...
method resolve (line 902) | func (fs *FileSystem) resolve(ctx meta.Context, p string, followLastSy...
method doResolve (line 906) | func (fs *FileSystem) doResolve(ctx meta.Context, p string, followLast...
method Create (line 1035) | func (fs *FileSystem) Create(ctx meta.Context, p string, mode uint16, ...
method Flush (line 1078) | func (fs *FileSystem) Flush() error {
method Close (line 1087) | func (fs *FileSystem) Close() error {
method Clone (line 1097) | func (fs *FileSystem) Clone(ctx meta.Context, src, dst string, preserv...
method Warmup (line 1126) | func (fs *FileSystem) Warmup(ctx meta.Context, paths []string, numthre...
method HandleQuota (line 1142) | func (fs *FileSystem) HandleQuota(ctx meta.Context, path string, cmd u...
type File (line 164) | type File struct
method FS (line 1173) | func (f *File) FS() *FileSystem {
method Inode (line 1177) | func (f *File) Inode() Ino {
method Name (line 1181) | func (f *File) Name() string {
method Stat (line 1185) | func (f *File) Stat() (fi os.FileInfo, err error) {
method Chmod (line 1189) | func (f *File) Chmod(ctx meta.Context, mode uint16) (err syscall.Errno) {
method Chown (line 1199) | func (f *File) Chown(ctx meta.Context, uid uint32, gid uint32) (err sy...
method Utime (line 1216) | func (f *File) Utime(ctx meta.Context, atime, mtime int64) (err syscal...
method Utime2 (line 1240) | func (f *File) Utime2(ctx meta.Context, atimeSec, atimeNSec, mtimeSec,...
method Seek (line 1266) | func (f *File) Seek(ctx meta.Context, offset int64, whence int) (int64...
method Read (line 1283) | func (f *File) Read(ctx meta.Context, b []byte) (n int, err error) {
method Pread (line 1295) | func (f *File) Pread(ctx meta.Context, b []byte, offset int64) (n int,...
method pread (line 1306) | func (f *File) pread(ctx meta.Context, b []byte, offset int64) (n int,...
method Write (line 1343) | func (f *File) Write(ctx meta.Context, b []byte) (n int, err syscall.E...
method Pwrite (line 1354) | func (f *File) Pwrite(ctx meta.Context, b []byte, offset int64) (n int...
method pwrite (line 1364) | func (f *File) pwrite(ctx meta.Context, b []byte, offset int64) (n int...
method Truncate (line 1381) | func (f *File) Truncate(ctx meta.Context, length uint64) (err syscall....
method Flush (line 1404) | func (f *File) Flush(ctx meta.Context) (err syscall.Errno) {
method Fsync (line 1418) | func (f *File) Fsync(ctx meta.Context) (err syscall.Errno) {
method Close (line 1432) | func (f *File) Close(ctx meta.Context) (err syscall.Errno) {
method Readdir (line 1456) | func (f *File) Readdir(ctx meta.Context, count int) (fi []os.FileInfo,...
method ReaddirPlus (line 1493) | func (f *File) ReaddirPlus(ctx meta.Context, offset int) (entries []*m...
method Summary (line 1524) | func (f *File) Summary(ctx meta.Context, recursive, strict bool) (s *m...
method GetTreeSummary (line 1535) | func (f *File) GetTreeSummary(ctx meta.Context, depth, entries uint8, ...
method GetQuota (line 1551) | func (f *File) GetQuota(ctx meta.Context) (quota *meta.Quota, err erro...
function NewFileSystem (line 180) | func NewFileSystem(conf *vfs.Config, m meta.Meta, d chunk.ChunkStore, re...
function parentDir (line 471) | func parentDir(p string) string {
function trimDotsForRename (line 615) | func trimDotsForRename(paths []string) (res []string) {
FILE: pkg/fs/fs_test.go
function TestFileStat (line 34) | func TestFileStat(t *testing.T) {
function TestFileSystem (line 71) | func TestFileSystem(t *testing.T) {
function createTestFS (line 275) | func createTestFS(t *testing.T) *FileSystem {
FILE: pkg/fs/http.go
type gzipResponseWriter (line 37) | type gzipResponseWriter struct
method Write (line 42) | func (w gzipResponseWriter) Write(b []byte) (int, error) {
type gzipHandler (line 46) | type gzipHandler struct
method ServeHTTP (line 50) | func (g *gzipHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
function makeGzipHandler (line 62) | func makeGzipHandler(h http.Handler) http.Handler {
function econv (line 73) | func econv(err error) error {
type webdavFS (line 87) | type webdavFS struct
method Mkdir (line 94) | func (hfs *webdavFS) Mkdir(ctx context.Context, name string, perm os.F...
method OpenFile (line 98) | func (hfs *webdavFS) OpenFile(ctx context.Context, name string, flag i...
method RemoveAll (line 127) | func (hfs *webdavFS) RemoveAll(ctx context.Context, name string) error {
method Rename (line 131) | func (hfs *webdavFS) Rename(ctx context.Context, oldName, newName stri...
method Stat (line 135) | func (hfs *webdavFS) Stat(ctx context.Context, name string) (os.FileIn...
type davFile (line 140) | type davFile struct
method DeadProps (line 154) | func (f *davFile) DeadProps() (map[xml.Name]webdav.Property, error) {
method Patch (line 177) | func (f *davFile) Patch(patches []webdav.Proppatch) ([]webdav.Propstat...
method Seek (line 218) | func (f *davFile) Seek(offset int64, whence int) (int64, error) {
method Read (line 223) | func (f *davFile) Read(b []byte) (n int, err error) {
method Write (line 228) | func (f *davFile) Write(buf []byte) (n int, err error) {
method Readdir (line 233) | func (f *davFile) Readdir(count int) (fi []os.FileInfo, err error) {
method Close (line 242) | func (f *davFile) Close() error {
constant webdavDeadProps (line 147) | webdavDeadProps = "webdav-dead-props"
type localProperty (line 149) | type localProperty struct
type WebdavConfig (line 246) | type WebdavConfig struct
type indexHandler (line 258) | type indexHandler struct
method ServeHTTP (line 263) | func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques...
function StartHTTPServer (line 338) | func StartHTTPServer(fs *FileSystem, config WebdavConfig) {
function removeNewLine (line 369) | func removeNewLine(input string) string {
FILE: pkg/fs/http_test.go
function TestWebdav (line 30) | func TestWebdav(t *testing.T) {
FILE: pkg/fuse/context.go
type fuseContext (line 40) | type fuseContext struct
method Uid (line 83) | func (c *fuseContext) Uid() uint32 {
method Gid (line 87) | func (c *fuseContext) Gid() uint32 {
method Gids (line 91) | func (c *fuseContext) Gids() []uint32 {
method Pid (line 98) | func (c *fuseContext) Pid() uint32 {
method Duration (line 102) | func (c *fuseContext) Duration() time.Duration {
method Cancel (line 106) | func (c *fuseContext) Cancel() {
method CheckPermission (line 110) | func (c *fuseContext) CheckPermission() bool {
method Canceled (line 114) | func (c *fuseContext) Canceled() bool {
method WithValue (line 129) | func (c *fuseContext) WithValue(k, v interface{}) meta.Context {
method Err (line 135) | func (c *fuseContext) Err() error {
method newContext (line 58) | func (fs *fileSystem) newContext(cancel <-chan struct{}, header *fuse.In...
function releaseContext (line 79) | func releaseContext(ctx *fuseContext) {
FILE: pkg/fuse/device_darwin.go
function ensureFuseDev (line 16) | func ensureFuseDev() {}
function grantAccess (line 18) | func grantAccess() error {
FILE: pkg/fuse/device_linux.go
function ensureFuseDev (line 30) | func ensureFuseDev() {
function grantAccess (line 41) | func grantAccess() error {
FILE: pkg/fuse/fuse.go
type fileSystem (line 39) | type fileSystem struct
method replyAttr (line 55) | func (fs *fileSystem) replyAttr(ctx *fuseContext, entry *meta.Entry, a...
method replyEntry (line 73) | func (fs *fileSystem) replyEntry(ctx *fuseContext, out *fuse.EntryOut,...
method Lookup (line 85) | func (fs *fileSystem) Lookup(cancel <-chan struct{}, header *fuse.InHe...
method GetAttr (line 100) | func (fs *fileSystem) GetAttr(cancel <-chan struct{}, in *fuse.GetAttr...
method SetAttr (line 115) | func (fs *fileSystem) SetAttr(cancel <-chan struct{}, in *fuse.SetAttr...
method Mknod (line 126) | func (fs *fileSystem) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, ...
method Mkdir (line 136) | func (fs *fileSystem) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, ...
method Unlink (line 146) | func (fs *fileSystem) Unlink(cancel <-chan struct{}, header *fuse.InHe...
method Rmdir (line 153) | func (fs *fileSystem) Rmdir(cancel <-chan struct{}, header *fuse.InHea...
method Rename (line 160) | func (fs *fileSystem) Rename(cancel <-chan struct{}, in *fuse.RenameIn...
method Link (line 167) | func (fs *fileSystem) Link(cancel <-chan struct{}, in *fuse.LinkIn, na...
method Symlink (line 177) | func (fs *fileSystem) Symlink(cancel <-chan struct{}, header *fuse.InH...
method Readlink (line 187) | func (fs *fileSystem) Readlink(cancel <-chan struct{}, header *fuse.In...
method GetXAttr (line 194) | func (fs *fileSystem) GetXAttr(cancel <-chan struct{}, header *fuse.In...
method ListXAttr (line 205) | func (fs *fileSystem) ListXAttr(cancel <-chan struct{}, header *fuse.I...
method SetXAttr (line 216) | func (fs *fileSystem) SetXAttr(cancel <-chan struct{}, in *fuse.SetXAt...
method RemoveXAttr (line 223) | func (fs *fileSystem) RemoveXAttr(cancel <-chan struct{}, header *fuse...
method Create (line 230) | func (fs *fileSystem) Create(cancel <-chan struct{}, in *fuse.CreateIn...
method Open (line 241) | func (fs *fileSystem) Open(cancel <-chan struct{}, in *fuse.OpenIn, ou...
method Read (line 263) | func (fs *fileSystem) Read(cancel <-chan struct{}, in *fuse.ReadIn, bu...
method Release (line 273) | func (fs *fileSystem) Release(cancel <-chan struct{}, in *fuse.Release...
method Write (line 279) | func (fs *fileSystem) Write(cancel <-chan struct{}, in *fuse.WriteIn, ...
method Flush (line 289) | func (fs *fileSystem) Flush(cancel <-chan struct{}, in *fuse.FlushIn) ...
method Fsync (line 296) | func (fs *fileSystem) Fsync(cancel <-chan struct{}, in *fuse.FsyncIn) ...
method Fallocate (line 303) | func (fs *fileSystem) Fallocate(cancel <-chan struct{}, in *fuse.Fallo...
method CopyFileRange (line 310) | func (fs *fileSystem) CopyFileRange(cancel <-chan struct{}, in *fuse.C...
method GetLk (line 325) | func (fs *fileSystem) GetLk(cancel <-chan struct{}, in *fuse.LkIn, out...
method SetLk (line 336) | func (fs *fileSystem) SetLk(cancel <-chan struct{}, in *fuse.LkIn) (co...
method SetLkw (line 340) | func (fs *fileSystem) SetLkw(cancel <-chan struct{}, in *fuse.LkIn) (c...
method setLk (line 344) | func (fs *fileSystem) setLk(cancel <-chan struct{}, in *fuse.LkIn, blo...
method Flock (line 355) | func (fs *fileSystem) Flock(cancel <-chan struct{}, in *fuse.LkIn, blo...
method OpenDir (line 362) | func (fs *fileSystem) OpenDir(cancel <-chan struct{}, in *fuse.OpenIn,...
method ReadDir (line 374) | func (fs *fileSystem) ReadDir(cancel <-chan struct{}, in *fuse.ReadIn,...
method ReadDirPlus (line 391) | func (fs *fileSystem) ReadDirPlus(cancel <-chan struct{}, in *fuse.Rea...
method ReleaseDir (line 418) | func (fs *fileSystem) ReleaseDir(in *fuse.ReleaseIn) {
method StatFs (line 424) | func (fs *fileSystem) StatFs(cancel <-chan struct{}, in *fuse.InHeader...
method Ioctl (line 445) | func (fs *fileSystem) Ioctl(cancel <-chan struct{}, in *fuse.IoctlIn, ...
function newFileSystem (line 45) | func newFileSystem(conf *vfs.Config, v *vfs.VFS) *fileSystem {
type setTimeout (line 53) | type setTimeout
function Serve (line 453) | func Serve(v *vfs.VFS, options string, xattrs, ioctl bool) error {
function GenFuseOpt (line 544) | func GenFuseOpt(conf *vfs.Config, options string, mt int, noxattr, noacl...
function Shutdown (line 586) | func Shutdown() bool {
FILE: pkg/fuse/fuse_darwin.go
function getCreateUmask (line 23) | func getCreateUmask(mask uint32, defMask uint16) uint16 {
function getUmask (line 30) | func getUmask(mask uint32, defMask uint16, isDir bool) uint16 {
function setBlksize (line 40) | func setBlksize(out *fuse.Attr, size uint32) {
FILE: pkg/fuse/fuse_linux.go
function getCreateUmask (line 23) | func getCreateUmask(mask uint32, defMask uint16) uint16 {
function getUmask (line 30) | func getUmask(mask uint32, defMask uint16, isDir bool) uint16 {
function setBlksize (line 37) | func setBlksize(out *fuse.Attr, size uint32) {
FILE: pkg/fuse/fuse_test.go
function format (line 46) | func format(url string) {
function mount (line 62) | func mount(url, mp string) {
function umount (line 122) | func umount(mp string, force bool) {
function waitMountpoint (line 144) | func waitMountpoint(mp string) chan error {
function setUp (line 160) | func setUp(metaUrl, mp string) error {
function cleanup (line 166) | func cleanup(mp string) {
function StatFS (line 181) | func StatFS(t *testing.T, mp string) {
function Xattrs (line 197) | func Xattrs(t *testing.T, mp string) {
function Flock (line 224) | func Flock(t *testing.T, mp string) {
function PosixLock (line 240) | func PosixLock(t *testing.T, mp string) {
function TestFUSE (line 274) | func TestFUSE(t *testing.T) {
FILE: pkg/fuse/gidcache.go
type cItem (line 30) | type cItem struct
type gidCache (line 35) | type gidCache struct
method cleanup (line 50) | func (g *gidCache) cleanup() {
method get (line 99) | func (g *gidCache) get(pid, gid uint32) []uint32 {
function newGidCache (line 41) | func newGidCache(cacheto time.Duration) *gidCache {
function findProcessGroups (line 64) | func findProcessGroups(pid, gid uint32) []uint32 {
FILE: pkg/fuse/utils.go
function attrToStat (line 25) | func attrToStat(inode Ino, attr *Attr, out *fuse.Attr) {
FILE: pkg/gateway/gateway.go
constant sep (line 54) | sep = "/"
constant metaBucket (line 55) | metaBucket = ".sys"
constant subDirPrefix (line 56) | subDirPrefix = 3
type Config (line 62) | type Config struct
function NewJFSGateway (line 73) | func NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) ...
type jfsObjects (line 80) | type jfsObjects struct
method PutObjectMetadata (line 88) | func (n *jfsObjects) PutObjectMetadata(ctx context.Context, s string, ...
method NSScanner (line 92) | func (n *jfsObjects) NSScanner(ctx context.Context, bf *minio.BloomFil...
method IsCompressionSupported (line 96) | func (n *jfsObjects) IsCompressionSupported() bool {
method IsEncryptionSupported (line 100) | func (n *jfsObjects) IsEncryptionSupported() bool {
method IsReady (line 105) | func (n *jfsObjects) IsReady(_ context.Context) bool {
method Shutdown (line 109) | func (n *jfsObjects) Shutdown(ctx context.Context) error {
method StorageInfo (line 113) | func (n *jfsObjects) StorageInfo(ctx context.Context) (info minio.Stor...
method isValidBucketName (line 172) | func (n *jfsObjects) isValidBucketName(bucket string) error {
method path (line 185) | func (n *jfsObjects) path(p ...string) string {
method tpath (line 192) | func (n *jfsObjects) tpath(p ...string) string {
method upath (line 196) | func (n *jfsObjects) upath(bucket, uploadID string) string {
method ppath (line 200) | func (n *jfsObjects) ppath(bucket, uploadID, part string) string {
method ppathFlat (line 204) | func (n *jfsObjects) ppathFlat(bucket, uploadID, part string) string {...
method DeleteBucket (line 208) | func (n *jfsObjects) DeleteBucket(ctx context.Context, bucket string, ...
method MakeBucketWithLocation (line 223) | func (n *jfsObjects) MakeBucketWithLocation(ctx context.Context, bucke...
method GetBucketInfo (line 242) | func (n *jfsObjects) GetBucketInfo(ctx context.Context, bucket string)...
method ListBuckets (line 264) | func (n *jfsObjects) ListBuckets(ctx context.Context) (buckets []minio...
method isLeafDir (line 306) | func (n *jfsObjects) isLeafDir(bucket, leafPath string) bool {
method isLeaf (line 310) | func (n *jfsObjects) isLeaf(bucket, leafPath string) bool {
method listDirFactory (line 314) | func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
method checkBucket (line 359) | func (n *jfsObjects) checkBucket(ctx context.Context, bucket string) e...
method ListObjects (line 373) | func (n *jfsObjects) ListObjects(ctx context.Context, bucket, prefix, ...
method ListObjectsV2 (line 445) | func (n *jfsObjects) ListObjectsV2(ctx context.Context, bucket, prefix...
method setFileAtime (line 468) | func (n *jfsObjects) setFileAtime(p string, atime int64) {
method DeleteObject (line 479) | func (n *jfsObjects) DeleteObject(ctx context.Context, bucket, object ...
method delObj (line 489) | func (n *jfsObjects) delObj(bucket string, object string) error {
method DeleteObjects (line 514) | func (n *jfsObjects) DeleteObjects(ctx context.Context, bucket string,...
method GetObjectNInfo (line 567) | func (n *jfsObjects) GetObjectNInfo(ctx context.Context, bucket, objec...
method CopyObject (line 588) | func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObj...
method GetObjectInfo (line 696) | func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object...
method mkdirAll (line 762) | func (n *jfsObjects) mkdirAll(ctx context.Context, p string) error {
method putObject (line 785) | func (n *jfsObjects) putObject(ctx context.Context, bucket, object str...
method PutObject (line 849) | func (n *jfsObjects) PutObject(ctx context.Context, bucket string, obj...
method NewMultipartUpload (line 914) | func (n *jfsObjects) NewMultipartUpload(ctx context.Context, bucket st...
method getObjMeta (line 957) | func (n *jfsObjects) getObjMeta(p string) (objMeta map[string]string, ...
method setObjMeta (line 974) | func (n *jfsObjects) setObjMeta(p string, metadata map[string]string) ...
method ListMultipartUploads (line 1003) | func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket ...
method checkUploadIDExists (line 1104) | func (n *jfsObjects) checkUploadIDExists(ctx context.Context, bucket, ...
method ListObjectParts (line 1112) | func (n *jfsObjects) ListObjectParts(ctx context.Context, bucket, obje...
method CopyObjectPart (line 1155) | func (n *jfsObjects) CopyObjectPart(ctx context.Context, srcBucket, sr...
method PutObjectPart (line 1167) | func (n *jfsObjects) PutObjectPart(ctx context.Context, bucket, object...
method GetMultipartInfo (line 1189) | func (n *jfsObjects) GetMultipartInfo(ctx context.Context, bucket, obj...
method CompleteMultipartUpload (line 1199) | func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, buck...
method AbortMultipartUpload (line 1325) | func (n *jfsObjects) AbortMultipartUpload(ctx context.Context, bucket,...
method cleanup (line 1333) | func (n *jfsObjects) cleanup() {
method cleanupDir (line 1354) | func (n *jfsObjects) cleanupDir(dir string) bool {
method NewNSLock (line 1474) | func (n *jfsObjects) NewNSLock(bucket string, objects ...string) minio...
method BackendInfo (line 1507) | func (n *jfsObjects) BackendInfo() madmin.BackendInfo {
method LocalStorageInfo (line 1511) | func (n *jfsObjects) LocalStorageInfo(ctx context.Context) (minio.Stor...
method ListObjectVersions (line 1515) | func (n *jfsObjects) ListObjectVersions(ctx context.Context, bucket, p...
method getObjectInfoNoFSLock (line 1524) | func (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket...
method Walk (line 1544) | func (n *jfsObjects) Walk(ctx context.Context, bucket, prefix string, ...
method SetBucketPolicy (line 1548) | func (n *jfsObjects) SetBucketPolicy(ctx context.Context, bucket strin...
method GetBucketPolicy (line 1564) | func (n *jfsObjects) GetBucketPolicy(ctx context.Context, bucket strin...
method DeleteBucketPolicy (line 1575) | func (n *jfsObjects) DeleteBucketPolicy(ctx context.Context, bucket st...
method SetDriveCounts (line 1584) | func (n *jfsObjects) SetDriveCounts() []int {
method HealFormat (line 1588) | func (n *jfsObjects) HealFormat(ctx context.Context, dryRun bool) (mad...
method HealBucket (line 1592) | func (n *jfsObjects) HealBucket(ctx context.Context, bucket string, op...
method HealObject (line 1596) | func (n *jfsObjects) HealObject(ctx context.Context, bucket, object, v...
method HealObjects (line 1600) | func (n *jfsObjects) HealObjects(ctx context.Context, bucket, prefix s...
method GetMetrics (line 1604) | func (n *jfsObjects) GetMetrics(ctx context.Context) (*minio.BackendMe...
method Health (line 1608) | func (n *jfsObjects) Health(ctx context.Context, opts minio.HealthOpti...
method ReadHealth (line 1617) | func (n *jfsObjects) ReadHealth(ctx context.Context) bool {
method PutObjectTags (line 1622) | func (n *jfsObjects) PutObjectTags(ctx context.Context, bucket, object...
method GetObjectTags (line 1632) | func (n *jfsObjects) GetObjectTags(ctx context.Context, bucket, object...
method DeleteObjectTags (line 1644) | func (n *jfsObjects) DeleteObjectTags(ctx context.Context, bucket, obj...
method IsNotificationSupported (line 1654) | func (n *jfsObjects) IsNotificationSupported() bool {
method IsListenSupported (line 1658) | func (n *jfsObjects) IsListenSupported() bool {
method IsTaggingSupported (line 1662) | func (n *jfsObjects) IsTaggingSupported() bool {
function jfsToObjectErr (line 119) | func jfsToObjectErr(ctx context.Context, err error, params ...string) er...
function isReservedOrInvalidBucket (line 257) | func isReservedOrInvalidBucket(bucketEntry string, strict bool) bool {
type fReader (line 559) | type fReader struct
method Read (line 563) | func (f *fReader) Read(b []byte) (int, error) {
constant uploadKeyName (line 941) | uploadKeyName = "s3-object"
constant s3Etag (line 942) | s3Etag = "s3-etag"
constant s3Tags (line 945) | s3Tags = "s3-tags"
constant s3Meta (line 948) | s3Meta = "s3-meta"
constant amzMeta (line 949) | amzMeta = "x-amz-meta-"
type jfsFLock (line 1385) | type jfsFLock struct
method GetLock (line 1393) | func (j *jfsFLock) GetLock(ctx context.Context, timeout *minio.Dynamic...
method getFlockWithTimeOut (line 1397) | func (j *jfsFLock) getFlockWithTimeOut(ctx context.Context, ltype uint...
method Unlock (line 1450) | func (j *jfsFLock) Unlock() {
method GetRLock (line 1460) | func (j *jfsFLock) GetRLock(ctx context.Context, timeout *minio.Dynami...
method RUnlock (line 1464) | func (j *jfsFLock) RUnlock() {
FILE: pkg/gateway/gateway_test.go
function TestGatewayLock (line 34) | func TestGatewayLock(t *testing.T) {
FILE: pkg/meta/backup.go
constant BakMagic (line 35) | BakMagic = 0x747083
constant BakVersion (line 36) | BakVersion = 1
constant BakEOS (line 37) | BakEOS = BakMagic
constant segTypeUnknown (line 41) | segTypeUnknown = iota
constant segTypeFormat (line 42) | segTypeFormat
constant segTypeCounter (line 43) | segTypeCounter
constant segTypeNode (line 44) | segTypeNode
constant segTypeEdge (line 45) | segTypeEdge
constant segTypeChunk (line 46) | segTypeChunk
constant segTypeSliceRef (line 47) | segTypeSliceRef
constant segTypeSymlink (line 48) | segTypeSymlink
constant segTypeSustained (line 49) | segTypeSustained
constant segTypeDelFile (line 50) | segTypeDelFile
constant segTypeXattr (line 51) | segTypeXattr
constant segTypeAcl (line 52) | segTypeAcl
constant segTypeStat (line 53) | segTypeStat
constant segTypeQuota (line 54) | segTypeQuota
constant segTypeParent (line 55) | segTypeParent
constant segTypeMax (line 56) | segTypeMax
function getMessageFromType (line 78) | func getMessageFromType(typ int) (proto.Message, error) {
function createMessageByName (line 91) | func createMessageByName(name protoreflect.FullName) (proto.Message, err...
type BakFormat (line 100) | type BakFormat struct
method writeSegment (line 117) | func (f *BakFormat) writeSegment(w io.Writer, seg *BakSegment) error {
method ReadSegment (line 140) | func (f *BakFormat) ReadSegment(r io.Reader) (*BakSegment, error) {
method writeFooter (line 148) | func (f *BakFormat) writeFooter(w io.Writer) error {
method writeEOS (line 155) | func (f *BakFormat) writeEOS(w io.Writer) error {
method ReadFooter (line 162) | func (f *BakFormat) ReadFooter(r io.ReadSeeker) (*BakFooter, error) { ...
function newBakFormat (line 105) | func newBakFormat() *BakFormat {
type BakFooter (line 174) | type BakFooter struct
method Marshal (line 179) | func (h *BakFooter) Marshal(w io.Writer) error {
method Unmarshal (line 196) | func (h *BakFooter) Unmarshal(r io.ReadSeeker) error {
type BakSegment (line 219) | type BakSegment struct
method Name (line 225) | func (s *BakSegment) Name() string {
method String (line 232) | func (s *BakSegment) String() string {
method num (line 281) | func (s *BakSegment) num() uint64 {
method Marshal (line 319) | func (s *BakSegment) Marshal(w io.Writer) (int, error) {
method Unmarshal (line 343) | func (s *BakSegment) Unmarshal(r io.Reader) error {
function newBakSegment (line 242) | func newBakSegment(val proto.Message) *BakSegment {
type DumpOption (line 372) | type DumpOption struct
method check (line 378) | func (opt *DumpOption) check() *DumpOption {
method dumpFormat (line 388) | func (m *baseMeta) dumpFormat(ctx Context, opt *DumpOption, ch chan<- *d...
type dumpedResult (line 401) | type dumpedResult struct
function dumpResult (line 406) | func dumpResult(ctx context.Context, ch chan<- *dumpedResult, res *dumpe...
type LoadOption (line 415) | type LoadOption struct
method check (line 420) | func (opt *LoadOption) check() {
type txSessionKey (line 428) | type txSessionKey struct
type txMaxRetryKey (line 429) | type txMaxRetryKey struct
FILE: pkg/meta/base.go
constant inodeBatch (line 48) | inodeBatch = 1 << 10
constant sliceIdBatch (line 49) | sliceIdBatch = 4 << 10
constant nlocks (line 50) | nlocks = 1024
constant maxSymCacheNum (line 51) | maxSymCacheNum = int32(10000)
constant unknownUsage (line 52) | unknownUsage = -1
function checkInodeName (line 66) | func checkInodeName(name string) syscall.Errno {
type engine (line 73) | type engine interface
type trashSliceScan (line 165) | type trashSliceScan
type pendingSliceScan (line 166) | type pendingSliceScan
type trashFileScan (line 167) | type trashFileScan
type pendingFileScan (line 168) | type pendingFileScan
type fsStat (line 172) | type fsStat struct
type cchunk (line 180) | type cchunk struct
type symlinkCache (line 186) | type symlinkCache struct
method Store (line 230) | func (symCache *symlinkCache) Store(inode Ino, path []byte) {
method clean (line 236) | func (symCache *symlinkCache) clean(ctx Context, wg *sync.WaitGroup) {
method doClean (line 250) | func (symCache *symlinkCache) doClean() {
type ugQuotaDelta (line 193) | type ugQuotaDelta struct
type ugQuotaDeltas (line 200) | type ugQuotaDeltas
method add (line 202) | func (ds ugQuotaDeltas) add(delta *ugQuotaDelta) {
type batchCloneResult (line 212) | type batchCloneResult struct
function ugKey (line 219) | func ugKey(uid, gid uint32) uint64 {
function newSymlinkCache (line 223) | func newSymlinkCache(cap int32) *symlinkCache {
type baseMeta (line 265) | type baseMeta struct
method InitSharedMetrics (line 531) | func (m *baseMeta) InitSharedMetrics(reg prometheus.Registerer) {
method InitMetrics (line 592) | func (m *baseMeta) InitMetrics(reg prometheus.Registerer) {
method timeit (line 603) | func (m *baseMeta) timeit(method string, start time.Time) {
method getBase (line 610) | func (m *baseMeta) getBase() *baseMeta {
method checkRoot (line 614) | func (m *baseMeta) checkRoot(inode Ino) Ino {
method txLock (line 625) | func (r *baseMeta) txLock(idx uint) {
method txUnlock (line 629) | func (r *baseMeta) txUnlock(idx uint) {
method txBatchLock (line 633) | func (r *baseMeta) txBatchLock(inodes ...Ino) func() {
method OnMsg (line 663) | func (r *baseMeta) OnMsg(mtype uint32, cb MsgCallback) {
method newMsg (line 669) | func (r *baseMeta) newMsg(mid uint32, args ...interface{}) error {
method Load (line 679) | func (m *baseMeta) Load(checkVersion bool) (*Format, error) {
method newSessionInfo (line 702) | func (m *baseMeta) newSessionInfo() []byte {
method NewSession (line 733) | func (m *baseMeta) NewSession(record bool) error {
method startDeleteSliceTasks (line 789) | func (m *baseMeta) startDeleteSliceTasks() {
method stopDeleteSliceTasks (line 817) | func (m *baseMeta) stopDeleteSliceTasks() {
method expireTime (line 829) | func (m *baseMeta) expireTime() int64 {
method OnReload (line 837) | func (m *baseMeta) OnReload(fn func(f *Format)) {
method refresh (line 845) | func (m *baseMeta) refresh(ctx Context) {
method CleanStaleSessions (line 912) | func (m *baseMeta) CleanStaleSessions(ctx Context) {
method CloseSession (line 931) | func (m *baseMeta) CloseSession() error {
method FlushSession (line 947) | func (m *baseMeta) FlushSession() {
method Init (line 957) | func (m *baseMeta) Init(format *Format, force bool) error {
method cleanupDeletedFiles (line 961) | func (m *baseMeta) cleanupDeletedFiles(ctx Context) {
method cleanupSlices (line 997) | func (m *baseMeta) cleanupSlices(ctx Context) {
method StatFS (line 1028) | func (m *baseMeta) StatFS(ctx Context, ino Ino, totalspace, availspace...
method statRootFs (line 1074) | func (m *baseMeta) statRootFs(ctx Context, totalspace, availspace, ius...
method resolveCase (line 1142) | func (m *baseMeta) resolveCase(ctx Context, parent Ino, name string) *...
method Lookup (line 1154) | func (m *baseMeta) Lookup(ctx Context, parent Ino, name string, inode ...
method parseAttr (line 1233) | func (m *baseMeta) parseAttr(buf []byte, attr *Attr) {
method marshal (line 1237) | func (m *baseMeta) marshal(attr *Attr) []byte {
method encodeDelayedSlice (line 1241) | func (m *baseMeta) encodeDelayedSlice(id uint64, size uint32) []byte {
method decodeDelayedSlices (line 1248) | func (m *baseMeta) decodeDelayedSlices(buf []byte, ss *[]Slice) {
method Resolve (line 1281) | func (r *baseMeta) Resolve(ctx Context, parent Ino, path string, inode...
method Access (line 1285) | func (m *baseMeta) Access(ctx Context, inode Ino, mmask uint8, attr *A...
method GetAttr (line 1324) | func (m *baseMeta) GetAttr(ctx Context, inode Ino, attr *Attr) syscall...
method SetAttr (line 1366) | func (m *baseMeta) SetAttr(ctx Context, inode Ino, set uint16, sugidcl...
method nextInode (line 1401) | func (m *baseMeta) nextInode() (Ino, error) {
method prefetchInodes (line 1433) | func (m *baseMeta) prefetchInodes() {
method allocateInodes (line 1447) | func (m *baseMeta) allocateInodes() (freeID, error) {
method Mknod (line 1455) | func (m *baseMeta) Mknod(ctx Context, parent Ino, name string, _type u...
method Create (line 1520) | func (m *baseMeta) Create(ctx Context, parent Ino, name string, mode u...
method Mkdir (line 1534) | func (m *baseMeta) Mkdir(ctx Context, parent Ino, name string, mode ui...
method Symlink (line 1544) | func (m *baseMeta) Symlink(ctx Context, parent Ino, name string, path ...
method Link (line 1557) | func (m *baseMeta) Link(ctx Context, inode, parent Ino, name string, a...
method ReadLink (line 1605) | func (m *baseMeta) ReadLink(ctx Context, inode Ino, path *[]byte) sysc...
method Unlink (line 1649) | func (m *baseMeta) Unlink(ctx Context, parent Ino, name string, skipCh...
method Rmdir (line 1674) | func (m *baseMeta) Rmdir(ctx Context, parent Ino, name string, skipChe...
method BatchUnlink (line 1706) | func (m *baseMeta) BatchUnlink(ctx Context, parent Ino, entries []*Ent...
method BatchClone (line 1724) | func (m *baseMeta) BatchClone(ctx Context, srcParent Ino, dstParent In...
method Rename (line 1744) | func (m *baseMeta) Rename(ctx Context, parentSrc Ino, nameSrc string, ...
method touchAtime (line 1855) | func (m *baseMeta) touchAtime(ctx Context, inode Ino, attr *Attr) {
method Open (line 1879) | func (m *baseMeta) Open(ctx Context, inode Ino, flags uint32, attr *At...
method InvalidateChunkCache (line 1931) | func (m *baseMeta) InvalidateChunkCache(ctx Context, inode Ino, indx u...
method Read (line 1936) | func (m *baseMeta) Read(ctx Context, inode Ino, indx uint32, slices *[...
method NewSlice (line 1981) | func (m *baseMeta) NewSlice(ctx Context, id *uint64) syscall.Errno {
method Close (line 1997) | func (m *baseMeta) Close(ctx Context, inode Ino) syscall.Errno {
method Write (line 2012) | func (m *baseMeta) Write(ctx Context, inode Ino, indx uint32, off uint...
method Truncate (line 2038) | func (m *baseMeta) Truncate(ctx Context, inode Ino, flags uint8, lengt...
method Fallocate (line 2058) | func (m *baseMeta) Fallocate(ctx Context, inode Ino, mode uint8, off u...
method Readdir (line 2094) | func (m *baseMeta) Readdir(ctx Context, inode Ino, plus uint8, entries...
method SetXattr (line 2135) | func (m *baseMeta) SetXattr(ctx Context, inode Ino, name string, value...
method RemoveXattr (line 2152) | func (m *baseMeta) RemoveXattr(ctx Context, inode Ino, name string) sy...
method GetParents (line 2164) | func (m *baseMeta) GetParents(ctx Context, inode Ino) map[Ino]int {
method GetPaths (line 2180) | func (m *baseMeta) GetPaths(ctx Context, inode Ino) []string {
method countDirNlink (line 2264) | func (m *baseMeta) countDirNlink(ctx Context, inode Ino) (uint32, sysc...
method walk (line 2280) | func (m *baseMeta) walk(ctx Context, inode Ino, p string, attr *Attr, ...
method Check (line 2305) | func (m *baseMeta) Check(ctx Context, fpath string, opt *CheckOpt) err...
method Chroot (line 2528) | func (m *baseMeta) Chroot(ctx Context, subdir string) syscall.Errno {
method chroot (line 2554) | func (m *baseMeta) chroot(inode Ino) {
method resolve (line 2558) | func (m *baseMeta) resolve(ctx Context, dpath string, inode *Ino, crea...
method getFormat (line 2584) | func (m *baseMeta) getFormat() *Format {
method GetFormat (line 2590) | func (m *baseMeta) GetFormat() Format {
method CompactAll (line 2594) | func (m *baseMeta) CompactAll(ctx Context, threads int, bar *utils.Bar...
method compactChunk (line 2619) | func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bo...
method Compact (line 2720) | func (m *baseMeta) Compact(ctx Context, inode Ino, concurrency int, pr...
method fileDeleted (line 2771) | func (m *baseMeta) fileDeleted(opened, force bool, inode Ino, length u...
method tryDeleteFileData (line 2781) | func (m *baseMeta) tryDeleteFileData(inode Ino, length uint64, force b...
method deleteSlice_ (line 2797) | func (m *baseMeta) deleteSlice_(id uint64, size uint32) {
method deleteSlice (line 2807) | func (m *baseMeta) deleteSlice(id uint64, size uint32) {
method toTrash (line 2824) | func (m *baseMeta) toTrash(parent Ino) bool {
method checkTrash (line 2831) | func (m *baseMeta) checkTrash(parent Ino, trash *Ino) syscall.Errno {
method trashEntry (line 2865) | func (m *baseMeta) trashEntry(parent, inode Ino, name string) string {
method cleanupTrash (line 2874) | func (m *baseMeta) cleanupTrash(ctx Context) {
method CleanupDetachedNodesBefore (line 2932) | func (m *baseMeta) CleanupDetachedNodesBefore(ctx Context, edge time.T...
method CleanupTrashBefore (line 2944) | func (m *baseMeta) CleanupTrashBefore(ctx Context, edge time.Time, inc...
method scanTrashEntry (line 3017) | func (m *baseMeta) scanTrashEntry(ctx Context, scan func(inode Ino, si...
method scanTrashFiles (line 3039) | func (m *baseMeta) scanTrashFiles(ctx Context, scan trashFileScan) err...
method doCleanupTrash (line 3075) | func (m *baseMeta) doCleanupTrash(ctx Context, days int, force bool, s...
method cleanupDelayedSlices (line 3083) | func (m *baseMeta) cleanupDelayedSlices(ctx Context, days int, count *...
method ScanDeletedObject (line 3101) | func (m *baseMeta) ScanDeletedObject(ctx Context, tss trashSliceScan, ...
method Clone (line 3160) | func (m *baseMeta) Clone(ctx Context, srcParentIno, srcIno, parent Ino...
method cloneEntry (line 3227) | func (m *baseMeta) cloneEntry(ctx Context, srcIno Ino, parent Ino, nam...
method mergeAttr (line 3386) | func (m *baseMeta) mergeAttr(ctx Context, inode Ino, set uint16, cur, ...
method CheckSetAttr (line 3482) | func (m *baseMeta) CheckSetAttr(ctx Context, inode Ino, set uint16, at...
method getFaclFromCache (line 3494) | func (m *baseMeta) getFaclFromCache(ctx Context, ino Ino, aclType uint...
method saveACL (line 3541) | func (m *baseMeta) saveACL(rule *aclAPI.Rule, aclMaxId *uint32) uint32 {
method SetFacl (line 3554) | func (m *baseMeta) SetFacl(ctx Context, ino Ino, aclType uint8, rule *...
method GetFacl (line 3572) | func (m *baseMeta) GetFacl(ctx Context, ino Ino, aclType uint8, rule *...
method StoreToken (line 3588) | func (m *baseMeta) StoreToken(ctx Context, token []byte) (id uint32, s...
method UpdateToken (line 3593) | func (m *baseMeta) UpdateToken(ctx Context, id uint32, token []byte) s...
method LoadToken (line 3598) | func (m *baseMeta) LoadToken(ctx Context, id uint32) (token []byte, st...
method DeleteTokens (line 3603) | func (m *baseMeta) DeleteTokens(ctx Context, ids []uint32) syscall.Err...
method ListTokens (line 3608) | func (m *baseMeta) ListTokens(ctx Context) (tokens map[uint32][]byte, ...
method NewDirHandler (line 3630) | func (m *baseMeta) NewDirHandler(ctx Context, inode Ino, plus bool, in...
method DumpMetaV2 (line 3806) | func (m *baseMeta) DumpMetaV2(ctx Context, w io.Writer, opt *DumpOptio...
method LoadMetaV2 (line 3854) | func (m *baseMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOptio...
function newBaseMeta (line 352) | func newBaseMeta(addr string, conf *Config) *baseMeta {
constant bgJobSucc (line 784) | bgJobSucc = "success"
constant bgJobFail (line 785) | bgJobFail = "failed"
constant bgJobCanceled (line 786) | bgJobCanceled = "canceled"
constant UmountCode (line 843) | UmountCode = 11
method reset (line 1212) | func (attr *Attr) reset() {
function clearSUGID (line 1257) | func clearSUGID(ctx Context, cur *Attr, set *Attr) {
type metaWalkFunc (line 2278) | type metaWalkFunc
function setAttrACLId (line 3511) | func setAttrACLId(attr *Attr, aclType uint8, id uint32) {
function getAttrACLId (line 3520) | func getAttrACLId(attr *Attr, aclType uint8) uint32 {
function setXAttrACL (line 3530) | func setXAttrACL(xattrs *[]byte, accessACL, defaultACL uint32) {
function inGroup (line 3613) | func inGroup(ctx Context, gid uint32) bool {
type DirHandler (line 3622) | type DirHandler interface
type dirBatch (line 3681) | type dirBatch struct
method contain (line 3689) | func (b *dirBatch) contain(offset int) bool {
method predecessor (line 3696) | func (b *dirBatch) predecessor(offset int) bool {
type dirFetcher (line 3700) | type dirFetcher
type dirHandler (line 3702) | type dirHandler struct
method fetch (line 3713) | func (h *dirHandler) fetch(ctx Context, offset int) (*dirBatch, error) {
method List (line 3736) | func (h *dirHandler) List(ctx Context, offset int) ([]*Entry, syscall....
method Delete (line 3763) | func (h *dirHandler) Delete(name string) {
method Insert (line 3782) | func (h *dirHandler) Insert(inode Ino, name string, attr *Attr) {
method Read (line 3795) | func (h *dirHandler) Read(offset int) {
method Close (line 3799) | func (h *dirHandler) Close() {
FILE: pkg/meta/base_test.go
function testConfig (line 48) | func testConfig() *Config {
function testFormat (line 54) | func testFormat() *Format {
function TestRedisClient (line 58) | func TestRedisClient(t *testing.T) {
function TestKeyDB (line 66) | func TestKeyDB(t *testing.T) { // skip mutate
function TestRedisCluster (line 111) | func TestRedisCluster(t *testing.T) { // skip mutate
function testMeta (line 122) | func testMeta(t *testing.T, m Meta) {
function testAccess (line 166) | func testAccess(t *testing.T, m Meta) {
function testACL (line 225) | func testACL(t *testing.T, m Meta) {
function testKerberosToken (line 405) | func testKerberosToken(t *testing.T, m Meta) {
function testMetaClient (line 495) | func testMetaClient(t *testing.T, m Meta) {
function testStickyBit (line 1070) | func testStickyBit(t *testing.T, m Meta) {
function testListLocks (line 1136) | func testListLocks(t *testing.T, m Meta) {
function testLocks (line 1210) | func testLocks(t *testing.T, m Meta) {
function testResolve (line 1358) | func testResolve(t *testing.T, m Meta) {
function testRemove (line 1404) | func testRemove(t *testing.T, m Meta) {
function testCaseIncensi (line 1445) | func testCaseIncensi(t *testing.T, m Meta) {
function testCaseIncensiRename (line 1488) | func testCaseIncensiRename(t *testing.T, m Meta) {
function testCaseIncensiHardlinkRename (line 1563) | func testCaseIncensiHardlinkRename(t *testing.T, m Meta) {
type compactor (line 1644) | type compactor interface
function testCompaction (line 1648) | func testCompaction(t *testing.T, m Meta, trash bool) {
function testConcurrentWrite (line 1854) | func testConcurrentWrite(t *testing.T, m Meta) {
function testTruncateAndDelete (line 1922) | func testTruncateAndDelete(t *testing.T, m Meta) {
function testCopyFileRange (line 1986) | func testCopyFileRange(t *testing.T, m Meta) {
function testCloseSession (line 2054) | func testCloseSession(t *testing.T, m Meta) {
function testTrash (line 2115) | func testTrash(t *testing.T, m Meta) {
function testParents (line 2360) | func testParents(t *testing.T, m Meta) {
function testOpenCache (line 2430) | func testOpenCache(t *testing.T, m Meta) {
function testReadOnly (line 2462) | func testReadOnly(t *testing.T, m Meta) {
function testConcurrentDir (line 2489) | func testConcurrentDir(t *testing.T, m Meta) {
function testAttrFlags (line 2560) | func testAttrFlags(t *testing.T, m Meta) {
function setAttr (line 2717) | func setAttr(t *testing.T, m Meta, inode Ino, attr *Attr) {
function testCheckAndRepair (line 2758) | func testCheckAndRepair(t *testing.T, m Meta) {
function testDirStat (line 2883) | func testDirStat(t *testing.T, m Meta) {
function testBatchClone (line 2988) | func testBatchClone(t *testing.T, m Meta) {
function testClone (line 3215) | func testClone(t *testing.T, m Meta) {
function checkEntryTree (line 3497) | func checkEntryTree(t *testing.T, m Meta, srcIno, dstIno Ino, walkFunc f...
function checkEntry (line 3523) | func checkEntry(t *testing.T, m Meta, srcEntry, dstEntry *Entry, dstPare...
function testQuota (line 3571) | func testQuota(t *testing.T, m Meta) {
function testAtime (line 3726) | func testAtime(t *testing.T, m Meta) {
function TestQuotaEdgeCases (line 3855) | func TestQuotaEdgeCases(t *testing.T) {
function TestCheckQuotaFileOwner (line 3915) | func TestCheckQuotaFileOwner(t *testing.T) {
function TestSymlinkCache (line 3960) | func TestSymlinkCache(t *testing.T) {
function TestTxBatchLock (line 3985) | func TestTxBatchLock(t *testing.T) {
function testCheckQuotaFileOwnerSimple (line 4045) | func testCheckQuotaFileOwnerSimple(t *testing.T, m Meta) {
function testQuotaEdgeCases (line 4095) | func testQuotaEdgeCases(t *testing.T, m Meta) {
function testQuotaEdgeCasesComplex (line 4138) | func testQuotaEdgeCasesComplex(t *testing.T, m Meta) {
function testCheckQuotaFileOwner (line 4215) | func testCheckQuotaFileOwner(t *testing.T, m Meta) {
type testContext (line 4488) | type testContext struct
method Uid (line 4494) | func (c *testContext) Uid() uint32 { return c.u...
method Gid (line 4495) | func (c *testContext) Gid() uint32 { return c.g...
method Gids (line 4496) | func (c *testContext) Gids() []uint32 { return []u...
method Pid (line 4497) | func (c *testContext) Pid() uint32 { return 0 }
method WithValue (line 4498) | func (c *testContext) WithValue(k, v interface{}) Context { return c }
method Cancel (line 4499) | func (c *testContext) Cancel() {}
method Canceled (line 4500) | func (c *testContext) Canceled() bool { return fal...
method CheckPermission (line 4501) | func (c *testContext) CheckPermission() bool { return true }
function cleanupQuotaTest (line 4503) | func cleanupQuotaTest(ctx Context, m Meta, parent Ino, uid, gid uint32) {
function testBasicQuotaOperations (line 4532) | func testBasicQuotaOperations(t *testing.T, m Meta, ctx Context, uid, gi...
function testQuotaFileOperations (line 4582) | func testQuotaFileOperations(t *testing.T, m Meta, ctx Context, parent I...
function testQuotaErrorCases (line 4652) | func testQuotaErrorCases(t *testing.T, m Meta, ctx Context, uid, gid uin...
function testQuotaConcurrentOperations (line 4684) | func testQuotaConcurrentOperations(t *testing.T, m Meta, ctx Context) {
function testQuotaMixedTypes (line 4715) | func testQuotaMixedTypes(t *testing.T, m Meta, ctx Context, uid, gid uin...
function testQuotaUsageStatistics (line 4745) | func testQuotaUsageStatistics(t *testing.T, m Meta, ctx Context, parent ...
function testUserGroupQuota (line 4806) | func testUserGroupQuota(t *testing.T, m Meta) {
function testHardlinkQuota (line 4866) | func testHardlinkQuota(t *testing.T, m Meta, ctx Context, parent Ino, ui...
function testBatchUnlinkWithUserGroupQuota (line 5037) | func testBatchUnlinkWithUserGroupQuota(t *testing.T, m Meta, ctx Context...
FILE: pkg/meta/benchmarks_test.go
constant redisAddr (line 30) | redisAddr = "redis://127.0.0.1/1"
constant sqlAddr (line 31) | sqlAddr = "sqlite3://juicefs.db"
constant tkvAddr (line 34) | tkvAddr = "badger://test_db"
function init (line 38) | func init() {
function encodeSlices (line 43) | func encodeSlices(size int) []string {
function encodeSlicesAsBuf (line 58) | func encodeSlicesAsBuf(nSlices uint32) []byte {
function BenchmarkReadSlices (line 70) | func BenchmarkReadSlices(b *testing.B) {
function BenchmarkReadSliceBuf (line 94) | func BenchmarkReadSliceBuf(b *testing.B) {
function prepareParent (line 118) | func prepareParent(m Meta, name string, inode *Ino) error {
function benchMkdir (line 129) | func benchMkdir(b *testing.B, m Meta) {
function benchMvdir (line 143) | func benchMvdir(b *testing.B, m Meta) { // rename dir
function benchRmdir (line 160) | func benchRmdir(b *testing.B, m Meta) {
function benchResolve (line 179) | func benchResolve(b *testing.B, m Meta) {
function benchReaddir (line 203) | func benchReaddir(b *testing.B, m Meta, n int) {
function benchMknod (line 227) | func benchMknod(b *testing.B, m Meta) {
function benchCreate (line 241) | func benchCreate(b *testing.B, m Meta) {
function benchRename (line 255) | func benchRename(b *testing.B, m Meta) {
function benchUnlink (line 272) | func benchUnlink(b *testing.B, m Meta) {
function benchLookup (line 291) | func benchLookup(b *testing.B, m Meta) {
function benchGetAttr (line 310) | func benchGetAttr(b *testing.B, m Meta) {
function benchSetAttr (line 328) | func benchSetAttr(b *testing.B, m Meta) {
function benchAccess (line 347) | func benchAccess(b *testing.B, m Meta) { // contains a Getattr
function benchSetXattr (line 365) | func benchSetXattr(b *testing.B, m Meta) {
function benchGetXattr (line 384) | func benchGetXattr(b *testing.B, m Meta) {
function benchRemoveXattr (line 405) | func benchRemoveXattr(b *testing.B, m Meta) {
function benchListXattr (line 427) | func benchListXattr(b *testing.B, m Meta, n int) {
function benchLink (line 450) | func benchLink(b *testing.B, m Meta) {
function benchSymlink (line 468) | func benchSymlink(b *testing.B, m Meta) {
function benchNewChunk (line 510) | func benchNewChunk(b *testing.B, m Meta) {
function benchWrite (line 520) | func benchWrite(b *testing.B, m Meta) {
function benchRead (line 550) | func benchRead(b *testing.B, m Meta, n int) {
function benchmarkDir (line 579) | func benchmarkDir(b *testing.B, m Meta) { // mkdir, rename dir, rmdir, r...
function benchmarkFile (line 589) | func benchmarkFile(b *testing.B, m Meta) {
function benchmarkXattr (line 600) | func benchmarkXattr(b *testing.B, m Meta) {
function benchmarkLink (line 608) | func benchmarkLink(b *testing.B, m Meta) {
function benchmarkData (line 615) | func benchmarkData(b *testing.B, m Meta) {
function benchmarkAll (line 624) | func benchmarkAll(b *testing.B, m Meta) {
function BenchmarkRedis (line 634) | func BenchmarkRedis(b *testing.B) {
function BenchmarkSQL (line 639) | func BenchmarkSQL(b *testing.B) {
function BenchmarkTKV (line 644) | func BenchmarkTKV(b *testing.B) {
FILE: pkg/meta/config.go
type Config (line 38) | type Config struct
method SelfCheck (line 63) | func (c *Config) SelfCheck() {
function DefaultConf (line 59) | func DefaultConf() *Config {
type Format (line 77) | type Format struct
method update (line 111) | func (f *Format) update(old *Format, force bool) error {
method RemoveSecret (line 147) | func (f *Format) RemoveSecret() {
method String (line 159) | func (f *Format) String() string {
method CheckVersion (line 166) | func (f *Format) CheckVersion() error {
method CheckCliVersion (line 175) | func (f *Format) CheckCliVersion(ver *version.Semver) error {
method Encrypt (line 229) | func (f *Format) Encrypt() error {
method Decrypt (line 259) | func (f *Format) Decrypt() error {
function newCipher (line 203) | func newCipher(algo string, key string) (cipher.AEAD, error) {
FILE: pkg/meta/config_test.go
function TestRemoveSecret (line 27) | func TestRemoveSecret(t *testing.T) {
function TestEncrypt (line 43) | func TestEncrypt(t *testing.T) {
function TestFormat_Update_KeyConflict (line 71) | func TestFormat_Update_KeyConflict(t *testing.T) {
FILE: pkg/meta/context.go
type CtxKey (line 24) | type CtxKey
type Context (line 26) | type Context interface
function Background (line 38) | func Background() Context {
type wrapContext (line 42) | type wrapContext struct
method Uid (line 50) | func (c *wrapContext) Uid() uint32 {
method Gid (line 54) | func (c *wrapContext) Gid() uint32 {
method Gids (line 58) | func (c *wrapContext) Gids() []uint32 {
method Pid (line 62) | func (c *wrapContext) Pid() uint32 {
method Cancel (line 66) | func (c *wrapContext) Cancel() {
method Canceled (line 72) | func (c *wrapContext) Canceled() bool {
method WithValue (line 76) | func (c *wrapContext) WithValue(k, v interface{}) Context {
method CheckPermission (line 82) | func (c *wrapContext) CheckPermission() bool {
function NewContext (line 86) | func NewContext(pid, uid uint32, gids []uint32) Context {
function WrapContext (line 90) | func WrapContext(ctx context.Context) Context {
function WrapWithCancel (line 94) | func WrapWithCancel(ctx context.Context, pid, uid uint32, gids []uint32)...
function WrapWithTimeout (line 99) | func WrapWithTimeout(ctx Context, timeout time.Duration) Context {
function WrapWithoutCancel (line 104) | func WrapWithoutCancel(ctx context.Context, pid, uid uint32, gids []uint...
function containsGid (line 108) | func containsGid(ctx Context, gid uint32) bool {
FILE: pkg/meta/dump.go
constant jsonIndent (line 34) | jsonIndent = " "
constant jsonWriteSize (line 35) | jsonWriteSize = 64 << 10
type DumpedCounters (line 38) | type DumpedCounters struct
type DumpedDelFile (line 48) | type DumpedDelFile struct
type DumpedSustained (line 54) | type DumpedSustained struct
type DumpedAttr (line 59) | type DumpedAttr struct
type DumpedSlice (line 78) | type DumpedSlice struct
type DumpedChunk (line 87) | type DumpedChunk struct
type DumpedXattr (line 92) | type DumpedXattr struct
type DumpedQuota (line 97) | type DumpedQuota struct
type DumpedACLEntry (line 104) | type DumpedACLEntry struct
type DumpedACL (line 109) | type DumpedACL struct
type DumpedEntry (line 118) | type DumpedEntry struct
method writeJSON (line 227) | func (de *DumpedEntry) writeJSON(bw *bufio.Writer, depth int) error {
method writeJsonWithOutEntry (line 288) | func (de *DumpedEntry) writeJsonWithOutEntry(bw *bufio.Writer, depth i...
type wrapEntryPool (line 130) | type wrapEntryPool struct
method Get (line 134) | func (p *wrapEntryPool) Get() *DumpedEntry {
method Put (line 138) | func (p *wrapEntryPool) Put(de *DumpedEntry) {
function escape (line 165) | func escape(original string) string {
function parseHex (line 194) | func parseHex(c byte) (byte, error) {
function unescape (line 204) | func unescape(s string) []byte {
type DumpedMeta (line 327) | type DumpedMeta struct
method validate (line 337) | func (dm *DumpedMeta) validate() error {
method writeJsonWithOutTree (line 344) | func (dm *DumpedMeta) writeJsonWithOutTree(w io.Writer) (*bufio.Writer...
method loadDumpedQuotas (line 359) | func (m *baseMeta) loadDumpedQuotas(ctx Context, quotas map[Ino]*DumpedQ...
function dumpAttr (line 369) | func dumpAttr(a *Attr, d *DumpedAttr) {
function loadAttr (line 393) | func loadAttr(d *DumpedAttr) *Attr {
type chunkKey (line 412) | type chunkKey struct
function loadEntries (line 417) | func loadEntries(r io.Reader, load func(*DumpedEntry), addChunk func(*ch...
function decodeEntry (line 478) | func decodeEntry(dec *json.Decoder, parent Ino, cs *DumpedCounters, pare...
function dumpACL (line 613) | func dumpACL(rule *aclAPI.Rule) *DumpedACL {
function dumpACLEntries (line 627) | func dumpACLEntries(entries aclAPI.Entries) []DumpedACLEntry {
function loadACL (line 639) | func loadACL(dumped *DumpedACL) *aclAPI.Rule {
function loadACLEntries (line 653) | func loadACLEntries(dumpedEnts []DumpedACLEntry) aclAPI.Entries {
FILE: pkg/meta/info.go
type redisVersion (line 25) | type redisVersion struct
method olderThan (line 47) | func (ver redisVersion) olderThan(v2 redisVersion) bool {
method String (line 57) | func (ver redisVersion) String() string {
function parseRedisVersion (line 32) | func parseRedisVersion(v string) (ver redisVersion, err error) {
type redisInfo (line 61) | type redisInfo struct
function checkRedisInfo (line 68) | func checkRedisInfo(rawInfo string) (info redisInfo, err error) {
FILE: pkg/meta/info_test.go
function TestOlderThan (line 21) | func TestOlderThan(t *testing.T) {
function TestParseRedisVersion (line 43) | func TestParseRedisVersion(t *testing.T) {
function TestParseRedisInfo (line 67) | func TestParseRedisInfo(t *testing.T) {
FILE: pkg/meta/interface.go
constant MaxVersion (line 37) | MaxVersion = 1
constant ChunkBits (line 39) | ChunkBits = 26
constant ChunkSize (line 41) | ChunkSize = 1 << ChunkBits
constant DeleteSlice (line 43) | DeleteSlice = 1000
constant CompactChunk (line 45) | CompactChunk = 1001
constant Rmr (line 47) | Rmr = 1002
constant LegacyInfo (line 49) | LegacyInfo = 1003
constant FillCache (line 51) | FillCache = 1004
constant InfoV2 (line 53) | InfoV2 = 1005
constant Clone (line 55) | Clone = 1006
constant OpSummary (line 57) | OpSummary = 1007
constant CompactPath (line 59) | CompactPath = 1008
constant TypeFile (line 63) | TypeFile = 1
constant TypeDirectory (line 64) | TypeDirectory = 2
constant TypeSymlink (line 65) | TypeSymlink = 3
constant TypeFIFO (line 66) | TypeFIFO = 4
constant TypeBlockDev (line 67) | TypeBlockDev = 5
constant TypeCharDev (line 68) | TypeCharDev = 6
constant TypeSocket (line 69) | TypeSocket = 7
constant RenameNoReplace (line 73) | RenameNoReplace = 1 << iota
constant RenameExchange (line 74) | RenameExchange
constant RenameWhiteout (line 75) | RenameWhiteout
constant _renameReserved1 (line 76) | _renameReserved1
constant _renameReserved2 (line 77) | _renameReserved2
constant RenameRestore (line 78) | RenameRestore
constant SetAttrMode (line 83) | SetAttrMode = 1 << iota
constant SetAttrUID (line 84) | SetAttrUID
constant SetAttrGID (line 85) | SetAttrGID
constant SetAttrSize (line 86) | SetAttrSize
constant SetAttrAtime (line 87) | SetAttrAtime
constant SetAttrMtime (line 88) | SetAttrMtime
constant SetAttrCtime (line 89) | SetAttrCtime
constant SetAttrAtimeNow (line 90) | SetAttrAtimeNow
constant SetAttrMtimeNow (line 91) | SetAttrMtimeNow
constant SetAttrCtimeNow (line 92) | SetAttrCtimeNow
constant SetAttrFlag (line 93) | SetAttrFlag = 1 << 15
constant FlagImmutable (line 97) | FlagImmutable = 1 << iota
constant FlagAppend (line 98) | FlagAppend
constant FlagWindowsHidden (line 99) | FlagWindowsHidden
constant FlagWindowsSystem (line 100) | FlagWindowsSystem
constant FlagWindowsArchive (line 101) | FlagWindowsArchive
constant FlagSkipTrash (line 102) | FlagSkipTrash
constant QuotaSet (line 106) | QuotaSet uint8 = iota
constant QuotaGet (line 107) | QuotaGet
constant QuotaDel (line 108) | QuotaDel
constant QuotaList (line 109) | QuotaList
constant QuotaCheck (line 110) | QuotaCheck
constant MaxName (line 113) | MaxName = 255
constant MaxSymlink (line 114) | MaxSymlink = 4096
type Ino (line 116) | type Ino
method String (line 123) | func (i Ino) String() string {
method IsValid (line 127) | func (i Ino) IsValid() bool {
method IsTrash (line 131) | func (i Ino) IsTrash() bool {
method IsNormal (line 135) | func (i Ino) IsNormal() bool {
constant RootInode (line 118) | RootInode Ino = 1
constant TrashInode (line 119) | TrashInode Ino = 0x7FFFFFFF10000000
constant RmrDefaultThreads (line 121) | RmrDefaultThreads = 50
type internalNode (line 141) | type internalNode struct
constant CPROGRESS (line 147) | CPROGRESS = 0xFE
constant CDATA (line 148) | CDATA = 0xFF
type MsgCallback (line 151) | type MsgCallback
type Attr (line 154) | type Attr struct
method Marshal (line 178) | func (attr *Attr) Marshal() []byte {
method Unmarshal (line 206) | func (attr *Attr) Unmarshal(buf []byte) {
method SMode (line 301) | func (a *Attr) SMode() uint32 {
function typeToStatType (line 237) | func typeToStatType(_type uint8) uint32 {
function typeToString (line 258) | func typeToString(_type uint8) string {
function typeFromString (line 279) | func typeFromString(s string) uint8 {
type Entry (line 306) | type Entry struct
type Slice (line 314) | type Slice struct
type Summary (line 323) | type Summary struct
type TreeSummary (line 330) | type TreeSummary struct
type SessionInfo (line 340) | type SessionInfo struct
type Flock (line 349) | type Flock struct
type Plock (line 355) | type Plock struct
type Session (line 362) | type Session struct
type Meta (line 372) | type Meta interface
type CheckOpt (line 544) | type CheckOpt struct
type CleanupTrashStats (line 553) | type CleanupTrashStats struct
type Creator (line 557) | type Creator
function Register (line 561) | func Register(name string, register Creator) {
function injectPasswordIntoURI (line 565) | func injectPasswordIntoURI(uri, password string) (string, error) {
function readPasswordFromFile (line 584) | func readPasswordFromFile(filePath string) (string, error) {
function setPasswordFromEnv (line 592) | func setPasswordFromEnv(uri string) (string, error) {
function NewClient (line 612) | func NewClient(uri string, conf *Config) Meta {
FILE: pkg/meta/interface_test.go
function Test_injectPasswordIntoURI (line 25) | func Test_injectPasswordIntoURI(t *testing.T) {
function Test_setPasswordFromEnv (line 124) | func Test_setPasswordFromEnv(t *testing.T) {
function Test_readPasswordFromFile (line 214) | func Test_readPasswordFromFile(t *testing.T) {
FILE: pkg/meta/load_dump_test.go
constant sampleFile (line 36) | sampleFile = "metadata.sample"
constant subSampleFile (line 37) | subSampleFile = "metadata-sub.sample"
function TestEscape (line 39) | func TestEscape(t *testing.T) {
function Utf8ToGbk (line 76) | func Utf8ToGbk(s []byte) ([]byte, error) {
function GbkToUtf8 (line 85) | func GbkToUtf8(s []byte) ([]byte, error) {
function checkMeta (line 94) | func checkMeta(t *testing.T, m Meta) {
function testLoadSub (line 269) | func testLoadSub(t *testing.T, uri, fname string) {
function testDump (line 297) | func testDump(t *testing.T, m Meta, root Ino, expect, result string) {
function testLoadDump (line 323) | func testLoadDump(t *testing.T, name, addr string) {
function TestLoadDump (line 339) | func TestLoadDump(t *testing.T) { //skip mutate
function testDumpV2 (line 346) | func testDumpV2(t *testing.T, m Meta, result string, opt *DumpOption) {
function testLoad (line 364) | func testLoad(t *testing.T, uri, fname string, v2 bool) Meta {
function testLoadDumpV2 (line 387) | func testLoadDumpV2(t *testing.T, name, addr1, addr2 string) {
function testLoadOtherEngine (line 403) | func testLoadOtherEngine(t *testing.T, src, dst, dstAddr string) {
function TestLoadDumpV2 (line 410) | func TestLoadDumpV2(t *testing.T) {
function TestLoadDumpSlow (line 436) | func TestLoadDumpSlow(t *testing.T) { //skip mutate
function TestLoadDump_MemKV (line 447) | func TestLoadDump_MemKV(t *testing.T) {
function testSecretAndTrash (line 468) | func testSecretAndTrash(t *testing.T, addr, addr2 string) {
FILE: pkg/meta/lua_scripts.go
constant scriptLookup (line 20) | scriptLookup = `
constant scriptResolve (line 33) | scriptResolve = `
FILE: pkg/meta/openfile.go
constant invalidateAllChunks (line 9) | invalidateAllChunks = 0xFFFFFFFF
constant invalidateAttrOnly (line 10) | invalidateAttrOnly = 0xFFFFFFFE
type openFile (line 19) | type openFile struct
method invalidateChunk (line 28) | func (o *openFile) invalidateChunk() {
method release (line 35) | func (o *openFile) release() {
type openfiles (line 44) | type openfiles struct
method cleanup (line 61) | func (o *openfiles) cleanup() {
method OpenCheck (line 108) | func (o *openfiles) OpenCheck(ino Ino, attr *Attr) bool {
method Open (line 122) | func (o *openfiles) Open(ino Ino, attr *Attr) {
method Close (line 143) | func (o *openfiles) Close(ino Ino) bool {
method Check (line 154) | func (o *openfiles) Check(ino Ino, attr *Attr) bool {
method Update (line 168) | func (o *openfiles) Update(ino Ino, attr *Attr) bool {
method IsOpen (line 188) | func (o *openfiles) IsOpen(ino Ino) bool {
method ReadChunk (line 195) | func (o *openfiles) ReadChunk(ino Ino, indx uint32) ([]Slice, bool) {
method CacheChunk (line 210) | func (o *openfiles) CacheChunk(ino Ino, indx uint32, cs []Slice) {
method InvalidateChunk (line 227) | func (o *openfiles) InvalidateChunk(ino Ino, indx uint32) {
method find (line 243) | func (o *openfiles) find(ino Ino) *openFile {
function newOpenFiles (line 51) | func newOpenFiles(expire time.Duration, limit uint64) *openfiles {
FILE: pkg/meta/pb/backup.pb.go
constant _ (line 18) | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
constant _ (line 20) | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
type Format (line 23) | type Format struct
method Reset (line 31) | func (x *Format) Reset() {
method String (line 38) | func (x *Format) String() string {
method ProtoMessage (line 42) | func (*Format) ProtoMessage() {}
method ProtoReflect (line 44) | func (x *Format) ProtoReflect() protoreflect.Message {
method Descriptor (line 57) | func (*Format) Descriptor() ([]byte, []int) {
method GetData (line 61) | func (x *Format) GetData() []byte {
type Counter (line 68) | type Counter struct
method Reset (line 77) | func (x *Counter) Reset() {
method String (line 84) | func (x *Counter) String() string {
method ProtoMessage (line 88) | func (*Counter) ProtoMessage() {}
method ProtoReflect (line 90) | func (x *Counter) ProtoReflect() protoreflect.Message {
method Descriptor (line 103) | func (*Counter) Descriptor() ([]byte, []int) {
method GetKey (line 107) | func (x *Counter) GetKey() string {
method GetValue (line 114) | func (x *Counter) GetValue() int64 {
type Sustained (line 121) | type Sustained struct
method Reset (line 130) | func (x *Sustained) Reset() {
method String (line 137) | func (x *Sustained) String() string {
method ProtoMessage (line 141) | func (*Sustained) ProtoMessage() {}
method ProtoReflect (line 143) | func (x *Sustained) ProtoReflect() protoreflect.Message {
method Descriptor (line 156) | func (*Sustained) Descriptor() ([]byte, []int) {
method GetSid (line 160) | func (x *Sustained) GetSid() uint64 {
method GetInodes (line 167) | func (x *Sustained) GetInodes() []uint64 {
type DelFile (line 174) | type DelFile struct
method Reset (line 184) | func (x *DelFile) Reset() {
method String (line 191) | func (x *DelFile) String() string {
method ProtoMessage (line 195) | func (*DelFile) ProtoMessage() {}
method ProtoReflect (line 197) | func (x *DelFile) ProtoReflect() protoreflect.Message {
method Descriptor (line 210) | func (*DelFile) Descriptor() ([]byte, []int) {
method GetInode (line 214) | func (x *DelFile) GetInode() uint64 {
method GetLength (line 221) | func (x *DelFile) GetLength() uint64 {
method GetExpire (line 228) | func (x *DelFile) GetExpire() int64 {
type SliceRef (line 235) | type SliceRef struct
method Reset (line 245) | func (x *SliceRef) Reset() {
method String (line 252) | func (x *SliceRef) String() string {
method ProtoMessage (line 256) | func (*SliceRef) ProtoMessage() {}
method ProtoReflect (line 258) | func (x *SliceRef) ProtoReflect() protoreflect.Message {
method Descriptor (line 271) | func (*SliceRef) Descriptor() ([]byte, []int) {
method GetId (line 275) | func (x *SliceRef) GetId() uint64 {
method GetSize (line 282) | func (x *SliceRef) GetSize() uint32 {
method GetRefs (line 289) | func (x *SliceRef) GetRefs() int64 {
type Acl (line 296) | type Acl struct
method Reset (line 305) | func (x *Acl) Reset() {
method String (line 312) | func (x *Acl) String() string {
method ProtoMessage (line 316) | func (*Acl) ProtoMessage() {}
method ProtoReflect (line 318) | func (x *Acl) ProtoReflect() protoreflect.Message {
method Descriptor (line 331) | func (*Acl) Descriptor() ([]byte, []int) {
method GetId (line 335) | func (x *Acl) GetId() uint32 {
method GetData (line 342) | func (x *Acl) GetData() []byte {
type Xattr (line 349) | type Xattr struct
method Reset (line 359) | func (x *Xattr) Reset() {
method String (line 366) | func (x *Xattr) String() string {
method ProtoMessage (line 370) | func (*Xattr) ProtoMessage() {}
method ProtoReflect (line 372) | func (x *Xattr) ProtoReflect() protoreflect.Message {
method Descriptor (line 385) | func (*Xattr) Descriptor() ([]byte, []int) {
method GetInode (line 389) | func (x *Xattr) GetInode() uint64 {
method GetName (line 396) | func (x *Xattr) GetName() string {
method GetValue (line 403) | func (x *Xattr) GetValue() []byte {
type Quota (line 410) | type Quota struct
method Reset (line 422) | func (x *Quota) Reset() {
method String (line 429) | func (x *Quota) String() string {
method ProtoMessage (line 433) | func (*Quota) ProtoMessage() {}
method ProtoReflect (line 435) | func (x *Quota) ProtoReflect() protoreflect.Message {
method Descriptor (line 448) | func (*Quota) Descriptor() ([]byte, []int) {
method GetInode (line 452) | func (x *Quota) GetInode() uint64 {
method GetMaxSpace (line 459) | func (x *Quota) GetMaxSpace() int64 {
method GetMaxInodes (line 466) | func (x *Quota) GetMaxInodes() int64 {
method GetUsedSpace (line 473) | func (x *Quota) GetUsedSpace() int64 {
method GetUsedInodes (line 480) | func (x *Quota) GetUsedInodes() int64 {
type Stat (line 487) | type Stat struct
method Reset (line 498) | func (x *Stat) Reset() {
method String (line 505) | func (x *Stat) String() string {
method ProtoMessage (line 509) | func (*Stat) ProtoMessage() {}
method ProtoReflect (line 511) | func (x *Stat) ProtoReflect() protoreflect.Message {
method Descriptor (line 524) | func (*Stat) Descriptor() ([]byte, []int) {
method GetInode (line 528) | func (x *Stat) GetInode() uint64 {
method GetDataLength (line 535) | func (x *Stat) GetDataLength() int64 {
method GetUsedSpace (line 542) | func (x *Stat) GetUsedSpace() int64 {
method GetUsedInodes (line 549) | func (x *Stat) GetUsedInodes() int64 {
type Node (line 556) | type Node struct
method Reset (line 565) | func (x *Node) Reset() {
method String (line 572) | func (x *Node) String() string {
method ProtoMessage (line 576) | func (*Node) ProtoMessage() {}
method ProtoReflect (line 578) | func (x *Node) ProtoReflect() protoreflect.Message {
method Descriptor (line 591) | func (*Node) Descriptor() ([]byte, []int) {
method GetInode (line 595) | func (x *Node) GetInode() uint64 {
method GetData (line 602) | func (x *Node) GetData() []byte {
type Edge (line 609) | type Edge struct
method Reset (line 620) | func (x *Edge) Reset() {
method String (line 627) | func (x *Edge) String() string {
method ProtoMessage (line 631) | func (*Edge) ProtoMessage() {}
method ProtoReflect (line 633) | func (x *Edge) ProtoReflect() protoreflect.Message {
method Descriptor (line 646) | func (*Edge) Descriptor() ([]byte, []int) {
method GetParent (line 650) | func (x *Edge) GetParent() uint64 {
method GetInode (line 657) | func (x *Edge) GetInode() uint64 {
method GetName (line 664) | func (x *Edge) GetName() []byte {
method GetType (line 671) | func (x *Edge) GetType() uint32 {
type Parent (line 679) | type Parent struct
method Reset (line 689) | func (x *Parent) Reset() {
method String (line 696) | func (x *Parent) String() string {
method ProtoMessage (line 700) | func (*Parent) ProtoMessage() {}
method ProtoReflect (line 702) | func (x *Parent) ProtoReflect() protoreflect.Message {
method Descriptor (line 715) | func (*Parent) Descriptor() ([]byte, []int) {
method GetInode (line 719) | func (x *Parent) GetInode() uint64 {
method GetParent (line 726) | func (x *Parent) GetParent() uint64 {
method GetCnt (line 733) | func (x *Parent) GetCnt() int64 {
type Chunk (line 740) | type Chunk struct
method Reset (line 750) | func (x *Chunk) Reset() {
method String (line 757) | func (x *Chunk) String() string {
method ProtoMessage (line 761) | func (*Chunk) ProtoMessage() {}
method ProtoReflect (line 763) | func (x *Chunk) ProtoReflect() protoreflect.Message {
method Descriptor (line 776) | func (*Chunk) Descriptor() ([]byte, []int) {
method GetInode (line 780) | func (x *Chunk) GetInode() uint64 {
method GetIndex (line 787) | func (x *Chunk) GetIndex() uint32 {
method GetSlices (line 794) | func (x *Chunk) GetSlices() []byte {
type Symlink (line 801) | type Symlink struct
method Reset (line 810) | func (x *Symlink) Reset() {
method String (line 817) | func (x *Symlink) String() string {
method ProtoMessage (line 821) | func (*Symlink) ProtoMessage() {}
method ProtoReflect (line 823) | func (x *Symlink) ProtoReflect() protoreflect.Message {
method Descriptor (line 836) | func (*Symlink) Descriptor() ([]byte, []int) {
method GetInode (line 840) | func (x *Symlink) GetInode() uint64 {
method GetTarget (line 847) | func (x *Symlink) GetTarget() []byte {
type Batch (line 854) | type Batch struct
method Reset (line 874) | func (x *Batch) Reset() {
method String (line 881) | func (x *Batch) String() string {
method ProtoMessage (line 885) | func (*Batch) ProtoMessage() {}
method ProtoReflect (line 887) | func (x *Batch) ProtoReflect() protoreflect.Message {
method Descriptor (line 900) | func (*Batch) Descriptor() ([]byte, []int) {
method GetNodes (line 904) | func (x *Batch) GetNodes() []*Node {
method GetEdges (line 911) | func (x *Batch) GetEdges() []*Edge {
method GetChunks (line 918) | func (x *Batch) GetChunks() []*Chunk {
method GetSliceRefs (line 925) | func (x *Batch) GetSliceRefs() []*SliceRef {
method GetXattrs (line 932) | func (x *Batch) GetXattrs() []*Xattr {
method GetParents (line 939) | func (x *Batch) GetParents() []*Parent {
method GetSymlinks (line 946) | func (x *Batch) GetSymlinks() []*Symlink {
method GetSustained (line 953) | func (x *Batch) GetSustained() []*Sustained {
method GetDelfiles (line 960) | func (x *Batch) GetDelfiles() []*DelFile {
method GetDirstats (line 967) | func (x *Batch) GetDirstats() []*Stat {
method GetQuotas (line 974) | func (x *Batch) GetQuotas() []*Quota {
method GetAcls (line 981) | func (x *Batch) GetAcls() []*Acl {
method GetCounters (line 988) | func (x *Batch) GetCounters() []*Counter {
type Footer (line 995) | type Footer struct
method Reset (line 1005) | func (x *Footer) Reset() {
method String (line 1012) | func (x *Footer) String() string {
method ProtoMessage (line 1016) | func (*Footer) ProtoMessage() {}
method ProtoReflect (line 1018) | func (x *Footer) ProtoReflect() protoreflect.Message {
method Descriptor (line 1031) | func (*Footer) Descriptor() ([]byte, []int) {
method GetMagic (line 1035) | func (x *Footer) GetMagic() uint32 {
method GetVersion (line 1042) | func (x *Footer) GetVersion() uint32 {
method GetInfos (line 1049) | func (x *Footer) GetInfos() map[string]*Footer_SegInfo {
type Footer_SegInfo (line 1056) | type Footer_SegInfo struct
method Reset (line 1065) | func (x *Footer_SegInfo) Reset() {
method String (line 1072) | func (x *Footer_SegInfo) String() string {
method ProtoMessage (line 1076) | func (*Footer_SegInfo) ProtoMessage() {}
method ProtoReflect (line 1078) | func (x *Footer_SegInfo) ProtoReflect() protoreflect.Message {
method Descriptor (line 1091) | func (*Footer_SegInfo) Descriptor() ([]byte, []int) {
method GetOffset (line 1095) | func (x *Footer_SegInfo) GetOffset() []uint64 {
method GetNum (line 1102) | func (x *Footer_SegInfo) GetNum() uint64 {
function file_pkg_meta_pb_backup_proto_rawDescGZIP (line 1232) | func file_pkg_meta_pb_backup_proto_rawDescGZIP() []byte {
function init (line 1283) | func init() { file_pkg_meta_pb_backup_proto_init() }
function file_pkg_meta_pb_backup_proto_init (line 1284) | func file_pkg_meta_pb_backup_proto_init() {
FILE: pkg/meta/quota.go
type dirStat (line 33) | type dirStat struct
constant DirQuotaType (line 40) | DirQuotaType = iota
constant UserQuotaType (line 41) | UserQuotaType
constant GroupQuotaType (line 42) | GroupQuotaType
type Quota (line 45) | type Quota struct
method check (line 58) | func (q *Quota) check(space, inodes int64) bool {
method update (line 74) | func (q *Quota) update(space, inodes int64) {
method snap (line 79) | func (q *Quota) snap() Quota {
method sanitize (line 91) | func (q *Quota) sanitize() {
type iQuota (line 51) | type iQuota struct
method para
Condensed preview — 755 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (6,629K chars).
[
{
"path": ".autocorrectrc",
"chars": 810,
"preview": "rules:\n # Default rules: https://github.com/huacnlee/autocorrect/raw/main/autocorrect/.autocorrectrc.default\n spellche"
},
{
"path": ".github/ISSUE_TEMPLATE/bug-report.md",
"chars": 879,
"preview": "---\nname: Bug Report\nabout: Report a bug encountered while operating JuiceFS\nlabels: kind/bug\n---\n\n<!--\nPlease use this "
},
{
"path": ".github/ISSUE_TEMPLATE/enhancement.md",
"chars": 247,
"preview": "---\nname: Enhancement Request\nabout: Suggest an enhancement to the JuiceFS project\nlabels: kind/feature\n---\n\n<!-- Please"
},
{
"path": ".github/ISSUE_TEMPLATE/support.md",
"chars": 410,
"preview": "---\nname: Support Request\nabout: Support request or question relating to JuiceFS\nlabels: kind/question\n---\n\n<!--\nSTOP --"
},
{
"path": ".github/actions/build/action.yml",
"chars": 1755,
"preview": "name: 'Build Action'\ndescription: 'Build action'\ninputs:\n target:\n description: 'build target: juicefs, juicefs.fdb "
},
{
"path": ".github/actions/cancel-outdate-runs/action.yml",
"chars": 2595,
"preview": "name: 'Cancel Outdate Runs'\ndescription: 'Cancel Outdate Runs'\ninputs:\n head_sha:\n description: 'head_sha triggers t"
},
{
"path": ".github/actions/mount-coverage-dir/action.yml",
"chars": 2807,
"preview": "name: 'mount_coverage_dir'\ndescription: 'mount coverage directory'\ninputs:\n mount_point:\n description: 'mount point'"
},
{
"path": ".github/actions/upload-coverage/action.yml",
"chars": 1453,
"preview": "name: 'upload_coverage_report'\ndescription: 'upload coverage report of one job'\ninputs:\n UPLOAD_TOKEN:\n description:"
},
{
"path": ".github/actions/upload-total-coverage/action.yml",
"chars": 1720,
"preview": "name: 'upload_total_coverage_report'\ndescription: 'upload total coverage report of all jobs in workflow'\ninputs:\n UPLOA"
},
{
"path": ".github/scripts/apt_install.sh",
"chars": 1578,
"preview": "#!/bin/bash\n\nset -e\n\n# Set the maximum number of retries\nMAX_RETRIES=3\n\n# Define a function to run a command and check t"
},
{
"path": ".github/scripts/cache.sh",
"chars": 25286,
"preview": "#!/bin/bash -e\ndpkg -s redis-server || .github/scripts/apt_install.sh redis-tools redis-server\ndpkg -s fio || .github/s"
},
{
"path": ".github/scripts/chaos/dynamic.yaml",
"chars": 1355,
"preview": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: dynamic-ce\n labels:\n juicefs-app-type: dynamic-ce\nspec:\n rep"
},
{
"path": ".github/scripts/chaos/juicefs-csi-driver.Dockerfile",
"chars": 875,
"preview": "FROM golang:1.20-buster as builder\n\nARG GOPROXY\n# refs/remotes/pull/3056/merge\nARG GITHUB_REF\n# 4ac69613b5919142d87f21a6"
},
{
"path": ".github/scripts/chaos/juicefs.Dockerfile",
"chars": 223,
"preview": "FROM juicedata/mount:nightly\nCOPY ./juicefs /usr/local/bin/juicefs\n# RUN apt-get update && apt-get install -y musl-tools"
},
{
"path": ".github/scripts/chaos/minio.yaml",
"chars": 1176,
"preview": "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: minio-server\n namespace: kube-system\n labels:\n app: minio-s"
},
{
"path": ".github/scripts/chaos/pvc.yaml",
"chars": 187,
"preview": "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: dynamic-ce\nspec:\n accessModes:\n - ReadWriteMany\n resourc"
},
{
"path": ".github/scripts/chaos/redis.yaml",
"chars": 943,
"preview": "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: redis-server\n namespace: kube-system\n labels:\n app: redis-s"
},
{
"path": ".github/scripts/chaos/sc.yaml",
"chars": 997,
"preview": "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: dynamic-ce\nparameters:\n csi.storage.k8s.io/node-publ"
},
{
"path": ".github/scripts/chaos/workflow.yaml",
"chars": 5207,
"preview": "apiVersion: chaos-mesh.org/v1alpha1\nkind: Workflow\nmetadata:\n name: juicefs-workflow\nspec:\n entry: the-entry\n templat"
},
{
"path": ".github/scripts/check_juicefs_log.sh",
"chars": 240,
"preview": "#!/bin/bash -e\nfor log_file in /var/log/juicefs.log $HOME/.juicefs/juicefs.log; do\n if [ -f $log_file ]; then\n "
},
{
"path": ".github/scripts/cmptree.py",
"chars": 6421,
"preview": "#!/usr/bin/env python\n\n# Copyright (c) 2015, Bill Zissimopoulos. All rights reserved.\n#\n# Redistribution and use in so"
},
{
"path": ".github/scripts/command/acl.sh",
"chars": 1616,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/command/clone.sh",
"chars": 5290,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/command/config.sh",
"chars": 2764,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/command/debug.sh",
"chars": 1293,
"preview": "#!/bin/bash -e\n\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_m"
},
{
"path": ".github/scripts/command/dump_load.sh",
"chars": 10396,
"preview": "#!/bin/bash -ex\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_m"
},
{
"path": ".github/scripts/command/dump_load_bench.sh",
"chars": 2547,
"preview": "#!/bin/bash -ex\n\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\n[[ -z \"$START_META\" ]] && STA"
},
{
"path": ".github/scripts/command/dump_load_cross_meta.sh",
"chars": 8023,
"preview": "#!/bin/bash -ex\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META1\" ]] && META1=sqlite3\n[[ -z \"$META2\" ]] && META2=r"
},
{
"path": ".github/scripts/command/format.sh",
"chars": 12533,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/command/fsck.sh",
"chars": 3550,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/command/gateway-random.sh",
"chars": 2155,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\n[[ -z \"$SUBDIR\" ]] && SUBDIR=fa"
},
{
"path": ".github/scripts/command/gateway.sh",
"chars": 8754,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=redis\nsource .github/scripts/start_meta"
},
{
"path": ".github/scripts/command/gc.sh",
"chars": 1776,
"preview": "#!/bin/bash -e\n\npython3 -c \"import xattr\" || pip install xattr \ndpkg -s redis-tools || .github/scripts/apt_install.sh re"
},
{
"path": ".github/scripts/command/graceful_upgrade.sh",
"chars": 10453,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/command/info.sh",
"chars": 571,
"preview": "#!/bin/bash -e\n\nsudo dpkg -s redis-tools || sudo .github/scripts/apt_install.sh redis-tools\nsource .github/scripts/commo"
},
{
"path": ".github/scripts/command/interface.sh",
"chars": 2005,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/command/mount.sh",
"chars": 7974,
"preview": "#!/bin/bash -e\n\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_m"
},
{
"path": ".github/scripts/command/quota.sh",
"chars": 22960,
"preview": "#!/bin/bash -e\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMET"
},
{
"path": ".github/scripts/command/random.sh",
"chars": 1241,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n[[ -z \"$MAX_EXAMPLE\" ]] && MAX_EXAMPLE=100\n[[ -z \"$STEP_COUNT\" ]]"
},
{
"path": ".github/scripts/command-win/acl.sh",
"chars": 832,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\nt"
},
{
"path": ".github/scripts/command-win/clone.sh",
"chars": 1542,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n"
},
{
"path": ".github/scripts/command-win/debug.sh",
"chars": 1330,
"preview": "#!/bin/bash -e\n\nsource .github/scripts/common/common_win.sh\n[[ -z \"$META_URL\" ]] && META=redis://127.0.0.1:6379/1\n\n\nchec"
},
{
"path": ".github/scripts/command-win/dump_load.sh",
"chars": 3247,
"preview": "#!/bin/bash -ex\nsource .github/scripts/common/common_win.sh\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n"
},
{
"path": ".github/scripts/command-win/fsck.sh",
"chars": 711,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n"
},
{
"path": ".github/scripts/command-win/gateway.sh",
"chars": 8125,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n\n"
},
{
"path": ".github/scripts/command-win/gc.sh",
"chars": 1085,
"preview": "#!/bin/bash -e\n\nsource .github/scripts/common/common_win.sh\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n\n"
},
{
"path": ".github/scripts/command-win/profile.sh",
"chars": 591,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n"
},
{
"path": ".github/scripts/command-win/quota.sh",
"chars": 5868,
"preview": "#!/bin/bash -e\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\nHEARTBEAT_INTERVAL=3\nHEARTBEAT_SLEEP=3\nDIR_QU"
},
{
"path": ".github/scripts/common/common.sh",
"chars": 4926,
"preview": "#!/bin/bash -e\n\n# Common variables and initialization\ninit_platform() {\n case \"$(uname -s)\" in\n Darwin*) PL"
},
{
"path": ".github/scripts/common/common_win.sh",
"chars": 684,
"preview": "#!/bin/bash -e\nprepare_win_test()\n{\n net start redisredis || true\n ./juicefs.exe umount z: || true\n rm -rf C"
},
{
"path": ".github/scripts/common/run_test.sh",
"chars": 1596,
"preview": "#!/bin/bash -e\nrun_one_test()\n{\n test=$1\n test=${test%%(*}\n echo -e \"\\033[0;34mStart Test: $test\\033[0m\"\n ST"
},
{
"path": ".github/scripts/compare_results.sh",
"chars": 4937,
"preview": "#!/bin/bash\nset -e\n\nCURRENT_RESULTS=$1\nOLD_RESULTS=$2\n\nextract_metrics() {\n awk '{\n op_description=$1; \n "
},
{
"path": ".github/scripts/copyFile.js",
"chars": 517,
"preview": "const fs = require('fs');\nconst path = require('path');\nconst crypto = require('crypto');\n\nif (process.argv.length !== 4"
},
{
"path": ".github/scripts/fio.sh",
"chars": 5767,
"preview": "#/bin/bash -e \nget_fio_job_options(){\n fio_job_name=$1\n case \"$fio_job_name\" in\n \"big-file-sequential-read\""
},
{
"path": ".github/scripts/flush_meta.py",
"chars": 239,
"preview": "import argparse\nimport os\nfrom posixpath import expanduser\nfrom utils import *\n\nif __name__ == \"__main__\":\n p = argpa"
},
{
"path": ".github/scripts/fsrand.py",
"chars": 11357,
"preview": "#!/usr/bin/env python\n\n# Copyright (c) 2015, Bill Zissimopoulos. All rights reserved.\n#\n# Redistribution and use in so"
},
{
"path": ".github/scripts/hypo/command.py",
"chars": 12906,
"preview": "from difflib import Differ\nimport json\nimport os\nimport re\nimport subprocess\ntry: \n __import__('jsondiff')\nexcept Imp"
},
{
"path": ".github/scripts/hypo/command_op.py",
"chars": 15500,
"preview": "import json\nimport os\nimport pwd\nimport re\nimport shlex\nimport subprocess\ntry: \n __import__('xattr')\nexcept ImportErr"
},
{
"path": ".github/scripts/hypo/command_test.py",
"chars": 2189,
"preview": "import unittest\nfrom command import JuicefsCommandMachine\n\nclass TestCommand(unittest.TestCase):\n def test_dump(self)"
},
{
"path": ".github/scripts/hypo/common.py",
"chars": 8180,
"preview": "\nimport grp\nimport json\nimport logging\nimport os\nimport pwd\nimport subprocess\nimport sys\nimport stat\ndef red(s):\n ret"
},
{
"path": ".github/scripts/hypo/context.py",
"chars": 159,
"preview": "\nclass Context:\n def __init__(self, root_dir:str, mp:str) -> None:\n self.root_dir = root_dir\n self.mp ="
},
{
"path": ".github/scripts/hypo/file.py",
"chars": 11077,
"preview": "import os\nimport pwd\nimport re\nimport subprocess\nimport json\nimport common\nfrom common import red\ntry:\n __import__(\"h"
},
{
"path": ".github/scripts/hypo/file_op.py",
"chars": 23109,
"preview": "import hashlib\nimport io\nimport mmap\nimport os\nimport pwd\nimport re\nimport shutil\nimport stat\nimport subprocess\n\ntry: \n "
},
{
"path": ".github/scripts/hypo/file_test.py",
"chars": 2054,
"preview": "import unittest\nfrom file import JuicefsDataMachine\n\nclass TestPySdk(unittest.TestCase):\n def test_issue_1522_1(self)"
},
{
"path": ".github/scripts/hypo/fs.py",
"chars": 36173,
"preview": "import os\nimport pwd\nimport re\nimport subprocess\nimport json\nimport common\nfrom common import red\ntry:\n __import__(\"h"
},
{
"path": ".github/scripts/hypo/fs_acl_test.py",
"chars": 12351,
"preview": "import unittest\nfrom fs import JuicefsMachine\n\nclass TestFsrand2(unittest.TestCase):\n def test_acl_913(self):\n "
},
{
"path": ".github/scripts/hypo/fs_op.py",
"chars": 49715,
"preview": "import io\nimport os\nimport pwd\nimport re\nimport shutil\nimport stat\nimport subprocess\n\ntry: \n __import__('xattr')\nexce"
},
{
"path": ".github/scripts/hypo/fs_sdk_test.py",
"chars": 19445,
"preview": "import unittest\nimport subprocess\ntry: \n __import__('xattr')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"i"
},
{
"path": ".github/scripts/hypo/fs_test.py",
"chars": 1920,
"preview": "import os\nimport unittest\nfrom fs import JuicefsMachine\n\nclass TestFsrand2(unittest.TestCase):\n def test_issue_910(se"
},
{
"path": ".github/scripts/hypo/readme.md",
"chars": 434,
"preview": "1. format juicefs with trash day of 0. \n ./juicefs format sqlite3://test.db myjfs\n2. mount juicefs with xatrr enable.\n"
},
{
"path": ".github/scripts/hypo/s3.py",
"chars": 26735,
"preview": "import json\nimport os\nfrom string import ascii_lowercase\nimport subprocess\ntry:\n __import__(\"hypothesis\")\nexcept Impo"
},
{
"path": ".github/scripts/hypo/s3_contant.py",
"chars": 231,
"preview": "ROOT_ALIAS = 'admin'\nROOT_ACCESS_KEY = 'minioadmin'\nROOT_SECRET_KEY = 'minioadmin'\nDEFAULT_ACCESS_KEY = ROOT_ACCESS_KEY\n"
},
{
"path": ".github/scripts/hypo/s3_op.py",
"chars": 24275,
"preview": "import hashlib\nimport json\nimport os\nimport re\nimport subprocess\ntry: \n __import__('xattr')\nexcept ImportError:\n s"
},
{
"path": ".github/scripts/hypo/s3_strategy.py",
"chars": 1817,
"preview": "from hypothesis import strategies as st\nfrom string import ascii_lowercase\n\nMAX_OBJECT_SIZE=10*1024*1024\n# https://min.i"
},
{
"path": ".github/scripts/hypo/s3_test.py",
"chars": 3220,
"preview": "import unittest\nfrom s3 import S3Machine\nfrom s3_contant import *\nclass TestS3(unittest.TestCase):\n def test_bucket(s"
},
{
"path": ".github/scripts/hypo/stats.py",
"chars": 742,
"preview": "def singleton(cls):\n instances = {}\n def get_instance(*args, **kwargs):\n if cls not in instances:\n "
},
{
"path": ".github/scripts/hypo/strategy.py",
"chars": 4150,
"preview": "import subprocess\ntry:\n __import__(\"xattr\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"]"
},
{
"path": ".github/scripts/hypo/sync.py",
"chars": 6000,
"preview": "import os\nimport subprocess\nimport json\nimport common\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subproce"
},
{
"path": ".github/scripts/hypo/sync_test.py",
"chars": 3167,
"preview": "import unittest\nfrom sync import SyncMachine\n\nclass TestFsrand2(unittest.TestCase):\n\n def test_sync1(self):\n s"
},
{
"path": ".github/scripts/mutate/check_coverage.py",
"chars": 1779,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\n\ndef is_mutation_in_coverage(original_file, changed_file, cove"
},
{
"path": ".github/scripts/mutate/check_skip_by_comment.py",
"chars": 761,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\n\n\ndef is_mutation_skipped_by_comment(original_file, changed_fi"
},
{
"path": ".github/scripts/mutate/how_to_use_mutate_test.md",
"chars": 3145,
"preview": "# what is mutatation testing?\nMutation testing (or Mutation analysis or Program mutation) is used to design new software"
},
{
"path": ".github/scripts/mutate/modify_sdk_pom.py",
"chars": 1380,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef get_plugin_str(taget_tests, taget_classes, tim"
},
{
"path": ".github/scripts/mutate/mutest.sh",
"chars": 2983,
"preview": "#!/bin/bash\n\n# This exec script implements\n# - the replacement of the original file with the mutation,\n# - the execution"
},
{
"path": ".github/scripts/mutate/mutesting.py",
"chars": 2591,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport glob\nimport json\nimport os\nimport sys\nfrom tkinter import Tcl\n\nde"
},
{
"path": ".github/scripts/mutate/parse_black_list.py",
"chars": 864,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef parse_check_sum(test_file_path):\n check_sum"
},
{
"path": ".github/scripts/mutate/parse_job_total.py",
"chars": 601,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\nimport sys\n\n\ndef parse_test_jobs(test_file_path):\n "
},
{
"path": ".github/scripts/mutate/parse_mutate_log.py",
"chars": 1188,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef parse_mutate_log(log_file):\n mutants = {}\n "
},
{
"path": ".github/scripts/mutate/parse_test_cases.py",
"chars": 890,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef parse_test_cases(test_file_path):\n test_cas"
},
{
"path": ".github/scripts/mutate/query_report.py",
"chars": 1173,
"preview": "\nimport os\nimport sys\nimport MySQLdb\n\ndef query_report(repo, run_id):\n passowrd = os.environ['MYSQL_PASSWORD']\n db"
},
{
"path": ".github/scripts/mutate/save_report.py",
"chars": 2278,
"preview": "\nimport json\nimport os\nfrom sys import argv\nimport MySQLdb\nfrom datetime import datetime\nimport argparse\n# CREATE DATABA"
},
{
"path": ".github/scripts/perf/ai.sh",
"chars": 1093,
"preview": "#!/bin/bash\n# ai_format_benchmark.sh\nset -e\n\nMNT_POINT=$1\nRESULTS_FILE=$2\nVERSION=$3\n\n# Create Python virtual environmen"
},
{
"path": ".github/scripts/perf/ai_format_benchmark.py",
"chars": 46071,
"preview": "#!/usr/bin/env python3\n\"\"\"\nAI Training Format Performance Benchmark Script - Fixed Version\nComprehensive performance tes"
},
{
"path": ".github/scripts/perf/compare_ai.sh",
"chars": 3259,
"preview": "#!/bin/bash\n# fixed_compare.sh\n\ncurrent_file=\"$1\"\nold_file=\"$2\"\nTOLERANCE=${TOLERANCE:-0.3}\nEXIT_ON_REGRESSION=${EXIT_ON"
},
{
"path": ".github/scripts/perf/compare_mdtest_fio.sh",
"chars": 14261,
"preview": "#!/bin/bash\nset -e\n\nCURRENT_RESULTS=$1\nOLD_RESULTS=$2\nFILTER_OPS=(\"File read\" \"File stat\" \"File removal\" \"Tree removal\" "
},
{
"path": ".github/scripts/perf/mdtest_fio.sh",
"chars": 9067,
"preview": "#!/bin/bash\nset -e\n\nMNT_POINT=$1\nRESULTS_FILE=$2\nVERSION=$3\nMETA_URL=$4\n\nif [[ -z \"$META_URL\" ]]; then\n echo \"ERROR: "
},
{
"path": ".github/scripts/prepare_db.sh",
"chars": 5143,
"preview": "#!/bin/bash -e\nsource .github/scripts/start_meta_engine.sh\n[ -z \"$TEST\" ] && echo \"TEST is not set\" && exit 1\n\n# check p"
},
{
"path": ".github/scripts/pysdk/bench.py",
"chars": 7280,
"preview": "import os\nimport random\nimport sys\nimport time\nimport argparse\nimport threading\nimport hashlib\n\nsys.path.append('.')\nfro"
},
{
"path": ".github/scripts/pysdk/pysdk_test.py",
"chars": 19693,
"preview": "import errno\nimport fractions\nimport unittest\nimport os\nimport pwd\nfrom os.path import dirname\nimport sys\nimport time\nsy"
},
{
"path": ".github/scripts/random_read_write.py",
"chars": 2081,
"preview": "import random\nimport os\n\ndef random_write(path1, path2, count=1000):\n if not os.path.exists(path1):\n os.system"
},
{
"path": ".github/scripts/save_benchmark.sh",
"chars": 2445,
"preview": "#/bin/bash -e\n\nmount_jfs(){\n mkdir -p /root/.juicefs\n wget -q s.juicefs.com/static/Linux/mount -O /root/.juicefs/j"
},
{
"path": ".github/scripts/setup-hdfs.sh",
"chars": 2729,
"preview": "#!/bin/bash\n\n# JuiceFS, Copyright 2021 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": ".github/scripts/ssh/Dockerfile",
"chars": 434,
"preview": "FROM ubuntu:latest\nRUN apt update && apt install openssh-server sudo -y\nRUN groupadd juicedata && useradd -ms /bin/bash"
},
{
"path": ".github/scripts/ssh/docker-compose.yml",
"chars": 437,
"preview": "version: '2'\nservices:\n worker1:\n image: juicedata/ssh\n container_name: worker1\n restart: unless-stopped\n n"
},
{
"path": ".github/scripts/start_meta_engine.sh",
"chars": 11033,
"preview": "#!/bin/bash -e\nREDIS_CSC_QUERY=\"client-cache=true&client-cache-size=500&client-cache-expire=60s&client-cache-preload=100"
},
{
"path": ".github/scripts/sync/sync.sh",
"chars": 9544,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$ENCRYPT\" ]] && ENCRYPT=false\n[[ -z \"$META\" ]] && META=sq"
},
{
"path": ".github/scripts/sync/sync_cluster.sh",
"chars": 15571,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n[[ -z \"$CI\" ]] && CI=false\n[[ -z \"$META\" ]] && META=redis\n[[ -z \""
},
{
"path": ".github/scripts/sync/sync_fsrand.sh",
"chars": 8124,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_met"
},
{
"path": ".github/scripts/sync/sync_minio.sh",
"chars": 6766,
"preview": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_me"
},
{
"path": ".github/scripts/test-mac/mac_commands.sh",
"chars": 14044,
"preview": "#!/bin/bash -e\n\nsource .github/scripts/common/common.sh\nsource .github/scripts/test-mac/start_meta_engine.sh\n\n\n[[ -z \"$M"
},
{
"path": ".github/scripts/test-mac/start_meta_engine.sh",
"chars": 3151,
"preview": "#!/bin/bash -e\n\nREDIS_CSC_QUERY=\"client-cache=true&client-cache-size=500&client-cache-expire=60s&client-cache-preload=10"
},
{
"path": ".github/scripts/testVersionCompatible.py",
"chars": 39877,
"preview": "import subprocess\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hy"
},
{
"path": ".github/scripts/upload_coverage_report.sh",
"chars": 778,
"preview": "#!/bin/bash\n\n# 参数检查\nif [ \"$#\" -ne 3 ]; then\n echo \"Usage: $0 <coverage_file> <upload_path> <token>\"\n exit 1\nfi\n\nCOVERA"
},
{
"path": ".github/scripts/utils.py",
"chars": 9536,
"preview": "import subprocess\ntry:\n __import__(\"minio\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"minio\"]"
},
{
"path": ".github/scripts/wins_fs_test.py",
"chars": 6619,
"preview": "import os\nimport sys\nimport time\nimport shutil\nimport random\nimport string\nimport threading\nimport unittest\nfrom pathlib"
},
{
"path": ".github/workflows/bash/rm_fs",
"chars": 3759,
"preview": "gf01 growfiles -W gf01 -b -e 1 -u -i 0 -L 20 -w -C 1 -l -I r -T 10 -f glseek20 -S 2 -d $TMPDIR\ngf02 growfiles -W gf02 -b"
},
{
"path": ".github/workflows/bash/rm_list.sh",
"chars": 169,
"preview": "#!/bin/bash\nLIST=`cat $1`\n\nfor LINE in $LIST; do\n # should remove empty line and comment line\n sed -i -e \"\\!^$"
},
{
"path": ".github/workflows/bash/rm_syscalls",
"chars": 7507,
"preview": "alarm02 alarm02\nalarm03 alarm03\nalarm05 alarm05\nalarm06 alarm06\nalarm07 alarm07\nbind01 bind01\nbind02 bind02\nbind03 bind0"
},
{
"path": ".github/workflows/cache.yml",
"chars": 2359,
"preview": "name: \"cache\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/cache.yml'\n - '"
},
{
"path": ".github/workflows/cancel_outdate_runs.yml",
"chars": 2472,
"preview": "name: cancel_outdate_runs\non:\n pull_request:\n branches:\n - main\n - release**\n\njobs:\n cancel-outdate-runs:"
},
{
"path": ".github/workflows/chaos.yml",
"chars": 11249,
"preview": "name: \"chaos-test\"\n\non:\n push:\n branches:\n - 'release-**'\n - 'main'\n paths:\n - '**/chaos.yml'\n pu"
},
{
"path": ".github/workflows/check-doc.yaml",
"chars": 1156,
"preview": "name: Check document\n\non:\n push:\n branches: [main]\n paths:\n - 'README*.md'\n - 'docs/**'\n - 'packag"
},
{
"path": ".github/workflows/codeql-analysis.yml",
"chars": 3147,
"preview": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# Y"
},
{
"path": ".github/workflows/command-win.yml",
"chars": 8619,
"preview": "name: \"command-win\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/command-win.ym"
},
{
"path": ".github/workflows/command.yml",
"chars": 7694,
"preview": "name: \"command-test\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '.github/scripts/"
},
{
"path": ".github/workflows/command2.yml",
"chars": 5100,
"preview": "name: \"command-random-test\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '.github/s"
},
{
"path": ".github/workflows/compile.yml",
"chars": 5372,
"preview": "name: \"compile\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/compile.yml'\n pull_request:\n"
},
{
"path": ".github/workflows/coverage-report.yml",
"chars": 4979,
"preview": "name: \"coverage-report\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/coverage-report"
},
{
"path": ".github/workflows/dependency-review.yml",
"chars": 933,
"preview": "# Dependency Review Action\n#\n# This Action will scan dependency manifest files that change as part of a Pull Request, su"
},
{
"path": ".github/workflows/dockerfile-sftp",
"chars": 413,
"preview": "FROM debian:stable-slim\nRUN apt-get clean\nRUN apt-get update\n\nRUN apt-get install openssh-server -y\n\nRUN mkdir /run/sshd"
},
{
"path": ".github/workflows/dump_load.yml",
"chars": 5339,
"preview": "name: \"dump_load\"\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/start_meta_engine"
},
{
"path": ".github/workflows/dump_load_bench.yml",
"chars": 5976,
"preview": "name: \"dump_load_bench\"\non:\n push:\n branches:\n - \"main\"\n - \"release-**\"\n paths:\n - \"**/dump_load_b"
},
{
"path": ".github/workflows/dump_load_cross_meta.yml",
"chars": 4879,
"preview": "name: \"dump_load_cross_meta\"\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/start_"
},
{
"path": ".github/workflows/fsrand.yml",
"chars": 5621,
"preview": "name: \"fsrand\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/fsrand.yml'\n - '**/fs.py'\n "
},
{
"path": ".github/workflows/fsspec.yml",
"chars": 3441,
"preview": "name: \"fsspec\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/fsspec.yml'\n pull_reque"
},
{
"path": ".github/workflows/gateway-random.yml",
"chars": 4698,
"preview": "name: \"gateway-random\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/gateway-ran"
},
{
"path": ".github/workflows/gateway.yml",
"chars": 7251,
"preview": "name: \"gateway-test\"\n\non:\n push:\n branches: \n - release-**\n paths-ignore:\n - 'docs/**'\n - '**.md'\n"
},
{
"path": ".github/workflows/integrationtests.yml",
"chars": 6113,
"preview": "name: \"integrationtests\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**.c'\n -"
},
{
"path": ".github/workflows/ltpfs.yml",
"chars": 2953,
"preview": "name: \"ltpfs\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/ltpfs.yml'\n pull_re"
},
{
"path": ".github/workflows/ltpsyscalls.yml",
"chars": 4990,
"preview": "name: \"ltp-syscalls\"\n\non:\n push:\n branches:\n - 'release-**'\n paths-ignore:\n - 'docs/**'\n pull_request:"
},
{
"path": ".github/workflows/mutate-test-sdk.yml",
"chars": 3416,
"preview": "name: mutate-test-sdk\non:\n pull_request:\n branches:\n - 'main'\n paths:\n - '**/JuiceFileSystemTest.java'\n"
},
{
"path": ".github/workflows/mutate-test.yml",
"chars": 10706,
"preview": "name: mutate-test\non:\n pull_request:\n branches:\n - 'main'\n paths:\n - '**/*_test.go'\n\n workflow_dispatc"
},
{
"path": ".github/workflows/perf-test.yml",
"chars": 8520,
"preview": "name: \"JuiceFS mdtest Performance Comparison\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n "
},
{
"path": ".github/workflows/permission-check.yaml",
"chars": 4300,
"preview": "name: \"permission-check\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths-ignore:\n - '.auto"
},
{
"path": ".github/workflows/pjdfstest.yml",
"chars": 5637,
"preview": "name: \"pjdfstest\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths-ignore:\n - '.autocorrect"
},
{
"path": ".github/workflows/pysdk.yml",
"chars": 8482,
"preview": "name: \"pysdk\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/hypo/fs_op.py'\n - '**/hypo/f"
},
{
"path": ".github/workflows/random-test.yml",
"chars": 7210,
"preview": "name: \"random-test\"\non:\n pull_request:\n types: [opened, synchronize, reopened, ready_for_review]\n branches:\n "
},
{
"path": ".github/workflows/release.yml",
"chars": 2312,
"preview": "name: release\n\non:\n push:\n tags:\n - v*\n\njobs:\n releaser:\n runs-on: ubuntu-22.04\n steps:\n - name: Cl"
},
{
"path": ".github/workflows/resources/core-site.xml",
"chars": 763,
"preview": "<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n<configuration>\n <property>\n "
},
{
"path": ".github/workflows/resources/load-balancer.conf",
"chars": 438,
"preview": " upstream backend {\n server 127.0.0.1:9000;\n server 127.0.0.1:9001;\n }\n\n # This server accepts all traff"
},
{
"path": ".github/workflows/resources/sync-options.txt",
"chars": 1599,
"preview": "--dirs --include .* , -r --include .* , enable "
},
{
"path": ".github/workflows/resources/tpcds_datagen.scala",
"chars": 3985,
"preview": "// Copyright 2015 Databricks\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": ".github/workflows/resources/tpcds_run.scala",
"chars": 2911,
"preview": "// Copyright 2015 Databricks\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": ".github/workflows/resources/vdbench_big_file.conf",
"chars": 429,
"preview": "data_errors=1\nfsd=fsd1,anchor=/tmp/vdbench/vdbench-big,depth=1,width=1,files=4,size=1g,openflags=o_direct\n\nfwd=fwd1,fsd="
},
{
"path": ".github/workflows/resources/vdbench_long_run.conf",
"chars": 291,
"preview": "data_errors=1\nfsd=fsd1,anchor=/tmp/jfs,depth=1,width=2,files=2,sizes=(10m,0),shared=yes,openflags=o_direct\nfwd=fwd1,fsd="
},
{
"path": ".github/workflows/resources/vdbench_small_file.conf",
"chars": 571,
"preview": "data_errors=1\nfsd=fsd1,anchor=/tmp/vdbench/vdbench-small,depth=3,width=10,files=10,size=128k,openflags=o_direct\n\nfwd=fwd"
},
{
"path": ".github/workflows/rmfiles.yml",
"chars": 5382,
"preview": "name: \"rmr-test\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/rmfiles.yml'\n pu"
},
{
"path": ".github/workflows/sdktest.yml",
"chars": 3513,
"preview": "name: \"sdktest\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths-ignore:\n - '.autocorrectrc'"
},
{
"path": ".github/workflows/sync.yml",
"chars": 5529,
"preview": "name: \"sync\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**.go'\n - 'Makefile"
},
{
"path": ".github/workflows/unit-random-tests.yml",
"chars": 3768,
"preview": "name: \"unit-random-tests\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - 'pkg/meta/ran"
},
{
"path": ".github/workflows/unittests.yml",
"chars": 4770,
"preview": "name: \"unittests\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths-ignore:\n - '.autocorrectr"
},
{
"path": ".github/workflows/vdbench.yml",
"chars": 5904,
"preview": "name: \"vdbench\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**/vdbench.yml'\n pull"
},
{
"path": ".github/workflows/verify.yml",
"chars": 2751,
"preview": "name: verify\n\non:\n push:\n branches:\n - main\n - \"release-**\"\n paths-ignore:\n - \".autocorrectrc\"\n "
},
{
"path": ".github/workflows/version_compatible_hypo.yml",
"chars": 5556,
"preview": "name: \"version-compatible-test-hypo\"\n\non:\n push:\n branches: \n - main\n paths:\n - '**/testVersionCompatib"
},
{
"path": ".github/workflows/wintest.yml",
"chars": 6885,
"preview": "name: \"wintest\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/wintest.yml'\n "
},
{
"path": ".github/workflows/xattr.yml",
"chars": 4981,
"preview": "name: \"xattr\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**.go'\n - '**.c'\n "
},
{
"path": ".gitignore",
"chars": 359,
"preview": "*.o\n*.sw[po]\nltmain.sh\n*.orig\n*.rej\n.deps\n.dirstamp\njfs\n*.rdb\n.release-env\n*.so\nlibjfs.h\ndocs/node_modules\ncmd/cmd\n.hypo"
},
{
"path": ".golangci.yml",
"chars": 34,
"preview": "run:\n timeout: 5m\n tests: false\n"
},
{
"path": ".goreleaser.yml",
"chars": 2592,
"preview": "project_name: juicefs\nenv:\n - GO111MODULE=on\n - CGO_ENABLED=1\n - REVISIONDATE={{ .Env.REVISIONDATE }}\nbefore:\n hooks"
},
{
"path": ".markdownlint-cli2.jsonc",
"chars": 3547,
"preview": "{\n \"customRules\": [\n \"markdownlint-rule-enhanced-proper-names/src/enhanced-proper-names.js\",\n \"markdownlint-rule-"
},
{
"path": ".pre-commit-config.yaml",
"chars": 325,
"preview": "repos:\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v2.3.0\n hooks:\n - id: check-yaml\n "
},
{
"path": "ADOPTERS.md",
"chars": 124,
"preview": "# JuiceFS Adopters\n\nPlease visit [JuiceFS Official Documentation](https://juicefs.com/docs/community/adopters) for detai"
},
{
"path": "ADOPTERS_CN.md",
"chars": 87,
"preview": "# JuiceFS 使用者\n\n请访问 [JuiceFS 官方文档](https://juicefs.com/docs/zh/community/adopters)了解详情。\n"
},
{
"path": "CODEOWNERS",
"chars": 19,
"preview": "/docs/ @CaitinChen\n"
},
{
"path": "CODE_OF_CONDUCT.md",
"chars": 5483,
"preview": "\n# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make particip"
},
{
"path": "CONTRIBUTING.md",
"chars": 2551,
"preview": "# Contributing to JuiceFS\n\n## Guidelines\n\n- Before starting work on a feature or bug fix, please search GitHub or reach "
},
{
"path": "LICENSE",
"chars": 11358,
"preview": "\n Apache License\n Version 2.0, January 2004\n "
},
{
"path": "Makefile",
"chars": 5724,
"preview": "export GO111MODULE=on\n\nall: juicefs\n\nREVISION := $(shell git rev-parse --short HEAD 2>/dev/null)\nREVISIONDATE := $(shell"
},
{
"path": "README.md",
"chars": 14599,
"preview": "<p align=\"center\"><a href=\"https://github.com/juicedata/juicefs\"><img alt=\"JuiceFS Logo\" src=\"docs/en/images/juicefs-log"
},
{
"path": "README_CN.md",
"chars": 8870,
"preview": "<p align=\"center\"><a href=\"https://github.com/juicedata/juicefs\"><img alt=\"JuiceFS Logo\" src=\"docs/zh_cn/images/juicefs-"
},
{
"path": "check-changed.sh",
"chars": 420,
"preview": "#!/bin/bash\n\nset -e\n\nif [ x\"${TRAVIS_COMMIT_RANGE}\" == x ] ; then\n CHANGED_FILES=`git diff --name-only HEAD~1`\nelse\n C"
},
{
"path": "cmd/bench.go",
"chars": 13772,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/bench_test.go",
"chars": 1125,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/clone.go",
"chars": 4780,
"preview": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/compact.go",
"chars": 3060,
"preview": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/compact_test.go",
"chars": 2619,
"preview": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/config.go",
"chars": 12016,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/config_test.go",
"chars": 3610,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/debug.go",
"chars": 17155,
"preview": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/debug_test.go",
"chars": 1587,
"preview": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/debug_unix.go",
"chars": 2855,
"preview": "//go:build !windows\r\n// +build !windows\r\n\r\n/*\r\n * JuiceFS, Copyright 2025 Juicedata, Inc.\r\n *\r\n * Licensed under the Apa"
},
{
"path": "cmd/debug_windows.go",
"chars": 4776,
"preview": "package cmd\r\n\r\n/*\r\n * JuiceFS, Copyright 2025 Juicedata, Inc.\r\n *\r\n * Licensed under the Apache License, Version 2.0 (th"
},
{
"path": "cmd/destroy.go",
"chars": 5672,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/dump.go",
"chars": 4523,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/dump_test.go",
"chars": 2224,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/flags.go",
"chars": 10677,
"preview": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/flags_test.go",
"chars": 951,
"preview": "package cmd\n\nimport (\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert"
},
{
"path": "cmd/format.go",
"chars": 17321,
"preview": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/format_test.go",
"chars": 2447,
"preview": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
},
{
"path": "cmd/fsck.go",
"chars": 6816,
"preview": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * y"
}
]
// ... and 555 more files (download for full content)
About this extraction
This page contains the full source code of the juicedata/juicefs GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 755 files (5.9 MB), approximately 1.6M tokens, and a symbol index with 6054 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.