[ { "path": ".autocorrectrc", "content": "rules:\n # Default rules: https://github.com/huacnlee/autocorrect/raw/main/autocorrect/.autocorrectrc.default\n spellcheck: 1\ntextRules:\n # Config some special rule for some texts\n # For example, if we wants to let \"Hello你好\" just warning, and \"Hi你好\" to ignore\n # \"Hello你好\": 2\n # \"Hi你好\": 0\nfileTypes:\n # Config the files associations, you config is higher priority than default.\n # \"rb\": ruby\n # \"Rakefile\": ruby\n # \"*.js\": javascript\n # \".mdx\": markdown\nspellcheck:\n words:\n # Please do not add a general English word (eg. apple, python) here.\n # Users can add their special words to their .autocorrectrc file by their need.\n - Digital Ocean = DigitalOcean\n - JucieFS = JuiceFS\n - JueicFS = JuiceFS\n - JuiecFS = JuiceFS\n - filesystem = file system\n - mountpoint = mount point\n" }, { "path": ".github/ISSUE_TEMPLATE/bug-report.md", "content": "---\nname: Bug Report\nabout: Report a bug encountered while operating JuiceFS\nlabels: kind/bug\n---\n\n\n\n**What happened**:\n\n**What you expected to happen**:\n\n**How to reproduce it (as minimally and precisely as possible)**:\n\n**Anything else we need to know?**\n\n**Environment**:\n- JuiceFS version (use `juicefs --version`) or Hadoop Java SDK version:\n- Cloud provider or hardware configuration running JuiceFS:\n- OS (e.g `cat /etc/os-release`):\n- Kernel (e.g. `uname -a`):\n- Object storage (cloud provider and region, or self maintained):\n- Metadata engine info (version, cloud provider managed or self maintained):\n- Network connectivity (JuiceFS to metadata engine, JuiceFS to object storage):\n- Others:\n" }, { "path": ".github/ISSUE_TEMPLATE/enhancement.md", "content": "---\nname: Enhancement Request\nabout: Suggest an enhancement to the JuiceFS project\nlabels: kind/feature\n---\n\n\n\n**What would you like to be added**:\n\n**Why is this needed**:\n" }, { "path": ".github/ISSUE_TEMPLATE/support.md", "content": "---\nname: Support Request\nabout: Support request or question relating to JuiceFS\nlabels: kind/question\n---\n\n\n" }, { "path": ".github/actions/build/action.yml", "content": "name: 'Build Action'\ndescription: 'Build action'\ninputs:\n target:\n description: 'build target: juicefs, juicefs.fdb etc'\n required: true\n default: 'juicefs'\n beta:\n description: 'beta version for the following test'\n required: false\nruns:\n using: \"composite\"\n steps:\n - uses: actions/setup-go@v3\n with:\n go-version: 'oldstable'\n cache: true\n\n - name: Change go version for root user\n shell: bash\n run: |\n go_path=`which go`\n echo $go_path\n root_go_path=`sudo which go`\n echo $root_go_path\n sudo rm -f $root_go_path\n sudo ln -s $go_path $root_go_path\n go version\n sudo go version\n\n - name: Install tools\n shell: bash\n run: |\n if [ \"${{inputs.target}}\" == \"juicefs.fdb\" ]; then\n wget -q https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb\n sudo dpkg -i foundationdb-clients_6.3.23-1_amd64.deb\n elif [ \"${{inputs.target}}\" == \"juicefs.gluster\" ]; then\n sudo .github/scripts/apt_install.sh uuid-dev libglusterfs-dev\n fi\n\n - name: Build linux target\n shell: bash\n run: |\n if [[ -n \"${{ inputs.beta }}\" ]]; then\n echo \"use beta version of juicefs: ${{inputs.beta}}\"\n wget -q https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/juicefs_beta/${{inputs.beta}} -O juicefs\n chmod +x juicefs\n ./juicefs version\n else\n echo \"start to build ${{inputs.target}}\"\n make ${{inputs.target}}.cover\n [ \"${{inputs.target}}\" != \"juicefs\" ] && mv ${{inputs.target}} juicefs\n ./juicefs version\n echo \"build ${{inputs.target}} succeed\"\n fi" }, { "path": ".github/actions/cancel-outdate-runs/action.yml", "content": "name: 'Cancel Outdate Runs'\ndescription: 'Cancel Outdate Runs'\ninputs:\n head_sha:\n description: 'head_sha triggers the workflow runs'\n required: true\n type: string\n per_page:\n description: 'Page size of runs to cancel'\n required: true\n type: number\n default: 5\n page:\n description: 'Page number of runs to cancel'\n required: true\n type: number\n default: 1\n github_token:\n description: 'GITHUB_TOKEN'\n required: true\n type: string\n\nruns:\n using: \"composite\"\n steps:\n - name: display parameters\n shell: bash\n run: |\n echo \"head_sha is ${{inputs.head_sha}}\"\n echo \"per_page is ${{inputs.per_page}}\"\n echo \"page is ${{inputs.page}}\"\n \n - uses: octokit/request-action@v2.x\n id: get_active_workflows\n with:\n route: GET /repos/${{github.repository}}/actions/runs?status=in_progress&event=pull_request&per_page=${{inputs.per_page}}&page=${{inputs.page}}&head_sha=${{inputs.head_sha}}\n env:\n GITHUB_TOKEN: ${{inputs.github_token}}\n\n - name: display active workflows\n shell: bash\n env:\n data: ${{ steps.get_active_workflows.outputs.data }}\n run: |\n echo \"$data\" | jq '.workflow_runs | map({id, head_sha, pull_request_number:.pull_requests[0].number})'\n \n - name: Extract workflow ids\n shell: bash\n id: extract_workflow_ids\n env:\n data: ${{ steps.get_active_workflows.outputs.data }}\n run: |\n echo pull_request_number is ${{ github.event.pull_request.number }}\n echo head_sha is ${{ github.event.pull_request.head.sha }}\n workflow_ids=$(echo \"$data\" | \\\n jq '.workflow_runs | map({id, head_sha, pull_request_number:.pull_requests[0].number})' | \\\n jq 'map(select( .pull_request_number == ${{ github.event.pull_request.number }} and .head_sha != \"${{ github.event.pull_request.head.sha }}\")) | map(.id)' | \\\n jq 'join(\",\")')\n echo workflow_ids is $workflow_ids\n echo 'WORKFLOW_IDS='$(echo $workflow_ids | tr -d '\"') >> $GITHUB_ENV\n \n - name: Cancel active workflows\n shell: bash\n run: |\n for i in ${WORKFLOW_IDS//,/ }\n do\n echo \"Cancelling workflow with id: $i\"\n # use curl here as I have no idea how to use a github action in a loop\n curl \\\n -X POST \\\n -H \"Accept: application/vnd.github+json\" \\\n -H \"Authorization: Bearer ${{inputs.github_token}}\" \\\n https://api.github.com/repos/${{ github.repository }}/actions/runs/$i/cancel\n done\n\n\n \n" }, { "path": ".github/actions/mount-coverage-dir/action.yml", "content": "name: 'mount_coverage_dir'\ndescription: 'mount coverage directory'\ninputs:\n mount_point:\n description: 'mount point'\n required: true\n type: string\n subdir:\n description: 'subdir'\n required: false\n type: string\n token:\n description: 'token of jfs'\n required: true\n type: string\n access_key:\n description: 'access key of object storage service'\n required: true\n type: string\n secret_key:\n description: 'secret key of object storage service'\n required: true\n type: string\n\nruns:\n using: \"composite\"\n steps:\n - name: set subdir\n shell: bash\n env:\n GH_TOKEN: ${{ github.token }}\n run: |\n jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs)\n job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name==\"${{ runner.name }}\") | select(.status==\"in_progress\") | .id')\n echo Job ID is: ${job_id}\n\n if [ \"${{ github.event_name }}\" == \"pull_request\" ]; then\n branch=${GITHUB_BASE_REF} # 目标分支\n elif [ \"${{ github.event_name }}\" == \"push\" ]; then\n branch=${GITHUB_REF#refs/heads/} # 当前分支\n else\n branch=${GITHUB_REF#refs/heads/} # 对于 schedule 和 workflow_dispatch\n fi\n echo input.subdir is ${{inputs.subdir}}\n if [ -n \"${{inputs.subdir}}\" ]; then\n subdir=${{inputs.subdir}}\n elif [[ \"${{github.event_name}}\" == \"schedule\" ]]; then\n subdir=juicefs/schedule/$(date +\"%Y%m%d\")/${{github.workflow}}\n elif [[ \"${{github.job}}\" == \"success-all-test\" ]]; then \n subdir=juicefs/pr/$branch/${{github.workflow}}/${{github.run_id}}\n else\n subdir=juicefs/pr/$branch/${{github.workflow}}/${{github.run_id}}/${job_id} \n fi\n echo \"subdir=$subdir\"\n echo \"subdir=$subdir\" >> $GITHUB_ENV\n\n - name: mount coverage dir\n shell: bash\n run: |\n sudo mkdir -p /root/.juicefs\n if ! sudo test -f /root/.juicefs/jfsmount; then\n sudo wget -q s.juicefs.com/static/Linux/mount -O /root/.juicefs/jfsmount \n sudo chmod +x /root/.juicefs/jfsmount\n fi\n sudo curl -s -L https://juicefs.com/static/juicefs -o /usr/local/bin/juicefs && sudo chmod +x /usr/local/bin/juicefs\n if [[ -n \"${{inputs.access_key}}\" && -n \"${{inputs.secret_key}}\" && -n \"${{inputs.token}}\" ]]; then\n sudo juicefs auth ci-coverage --access-key ${{ inputs.access_key }} --secret-key ${{ inputs.secret_key }} --token ${{inputs.token}} --encrypt-keys\n sudo juicefs mount ci-coverage --subdir ${subdir} ${{inputs.mount_point}} --allow-other\n else\n echo \"no jfs secrets provided, use local dir instead of jfs\"\n mkdir -p ${{inputs.mount_point}}\n fi\n \n" }, { "path": ".github/actions/upload-coverage/action.yml", "content": "name: 'upload_coverage_report'\ndescription: 'upload coverage report of one job'\ninputs:\n UPLOAD_TOKEN:\n description: 'upload token'\n required: true\n type: string\n\nruns:\n using: \"composite\"\n steps:\n - name: umount juicefs\n shell: bash\n run: |\n sudo umount /tmp/jfs || true\n sleep 3s\n\n - name: get job id\n shell: bash\n env:\n GH_TOKEN: ${{ github.token }}\n run: |\n jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs)\n job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name==\"${{ runner.name }}\") | select(.status==\"in_progress\") | .id')\n echo Job ID is: ${job_id}\n echo \"job_id=$job_id\" >> $GITHUB_ENV\n\n - name: generate mount coverage report\n shell: bash\n run: |\n echo \"generate coverage percentage report\"\n sudo go tool covdata percent -i=cover/ | sudo tee cover/percent.txt\n echo \"generate coverage text report\"\n sudo go tool covdata textfmt -i=cover/ -o cover/cover.txt\n echo \"generate coverage html report\"\n sudo go tool cover -html=cover/cover.txt -o cover/cover.html\n [[ -z \"${{inputs.UPLOAD_TOKEN}}\" ]] && echo \"no upload token, skip upload\" && exit 0 || true\n .github/scripts/upload_coverage_report.sh cover/cover.html juicefs_${{github.workflow}}_${{github.run_id}}_${job_id}.html ${{inputs.UPLOAD_TOKEN}}\n " }, { "path": ".github/actions/upload-total-coverage/action.yml", "content": "name: 'upload_total_coverage_report'\ndescription: 'upload total coverage report of all jobs in workflow'\ninputs:\n UPLOAD_TOKEN:\n description: 'upload token'\n required: true\n type: string\n\nruns:\n using: \"composite\"\n steps:\n - name: generate total coverage report\n shell: bash\n run: |\n echo \"current dir is $(pwd)\"\n if [[ \"${{github.event_name}}\" == \"schedule\" ]]; then\n coverdirs=\"cover,\"\n else\n for dir in $(find cover -mindepth 1 -maxdepth 1 -type d -exec basename {} \\;); do\n coverdirs+=\"cover/$dir/,\"\n done\n fi\n coverdirs=${coverdirs%,}\n echo coverdirs is $coverdirs\n [[ -z \"$coverdirs\" ]] && echo -e \"\\e[31m no coverage dir found\\e[0m\" && exit 0\n sudo go tool covdata percent -i=$coverdirs | sudo tee cover/cover.percent\n echo \"generated coverage percent report:\" $(realpath cover/cover.percent)\n sudo go tool covdata textfmt -o cover/cover.txt -i=$coverdirs \n echo \"generated coverage report in text format:\" $(realpath cover/cover.txt)\n sudo go tool cover -html=cover/cover.txt -o cover/cover.html\n echo \"generated coverage report in html format:\" $(realpath cover/cover.html)\n ls -l cover/cover*\n\n - name: upload coverage report\n shell: bash\n run: |\n [[ -z \"${{inputs.UPLOAD_TOKEN}}\" ]] && echo -e \"\\e[31m no upload token, skip upload \\e[0m\" && exit 0 || true\n if [[ -f cover/cover.html ]]; then\n .github/scripts/upload_coverage_report.sh cover/cover.html ${{github.workflow}}_${{github.run_id}}.html ${{inputs.UPLOAD_TOKEN}}\n else\n echo -e \"\\e[31m no coverage report found\\e[0m\" && exit 0\n fi" }, { "path": ".github/scripts/apt_install.sh", "content": "#!/bin/bash\n\nset -e\n\n# Set the maximum number of retries\nMAX_RETRIES=3\n\n# Define a function to run a command and check the return code\n# The function takes two arguments: the command to run and a description of the command\nfunction run_command() {\n local cmd=$1\n local retries=0\n local retry_cmd=\"$cmd\"\n while true; do\n # Run the command and capture the return code\n $retry_cmd 2>&1 | tee /tmp/install.log || true\n local ret=$?\n # If the command succeeded, break out of the loop\n if [[ $ret -eq 0 ]]; then\n break\n fi\n # If the command failed and we have retries left, print a warning and retry\n if [[ $retries -lt $MAX_RETRIES ]]; then\n retries=$((retries + 1))\n echo \"WARNING: $cmd failed with return code $ret. Retrying ($retries/$MAX_RETRIES)...\"\n # If the error message indicates missing packages, retry with --fix-missing\n if [[ $cmd == \"apt-get update\"* ]] && grep -q 'Failed to fetch' /tmp/install.log; then\n retry_cmd=\"apt-get update -y --fix-missing\"\n elif [[ $cmd == \"apt-get install\"* ]] && grep -q 'Unable to fetch some archives' /tmp/install.log; then\n retry_cmd=\"apt-get install -y --fix-missing $package_name\"\n fi\n else\n # If we've exhausted all retries, exit with an error\n echo \"ERROR: $cmd failed with return code $ret after $MAX_RETRIES retries.\"\n exit 1\n fi\n done\n}\n\n# Run apt-get update and check the return code\nrun_command \"apt-get update -y\" \npackage_name=$@\n# Run apt-get install and check the return code\nrun_command \"apt-get install -y $package_name\"\n" }, { "path": ".github/scripts/cache.sh", "content": "#!/bin/bash -e\ndpkg -s redis-server || .github/scripts/apt_install.sh redis-tools redis-server\ndpkg -s fio || .github/scripts/apt_install.sh fio\nsource .github/scripts/common/common.sh\nsource .github/scripts/start_meta_engine.sh\n[[ -z \"$META\" ]] && META=sqlite3\nstart_meta_engine $META minio\nMETA_URL=$(get_meta_url $META)\nif [[ \"$META\" == \"sqlite3\" ]]; then\n META_URL=\"sqlite3:///tmp/test.db\"\nfi\n\ntest_warmup_in_background(){\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount $META_URL /tmp/jfs -d\n dd if=/dev/zero of=/tmp/jfs/test bs=1M count=1024\n ./juicefs warmup /tmp/jfs/test --evict\n ./juicefs warmup /tmp/jfs/test --background\n wait_warmup_finish /tmp/jfs/test 100\n ./juicefs warmup /tmp/jfs/test --background --evict \n wait_warmup_finish /tmp/jfs/test 0\n}\n\ntest_batch_warmup(){\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount $META_URL /tmp/jfs -d\n rm -f file.list\n file_count=11000\n time seq 1 $file_count | xargs -P 8 -I {} sh -c 'echo {} > /tmp/jfs/test_{}; echo /tmp/jfs/test_{} >> file.list'\n # time for i in $(seq 1 $file_count); do echo $i > /tmp/jfs/test_$i; echo /tmp/jfs/test_$i >> file.list; done\n ./juicefs warmup -f file.list 2>&1 | tee warmup.log\n files=$(get_cache_file_count)\n [[ $files -ne $file_count ]] && echo \"warmup failed, expect $file_count files, actual $files\" && exit 1 || true\n ./juicefs warmup -f file.list --check 2>&1 | tee warmup.log\n files=$(get_cache_file_count)\n [[ $files -ne $file_count ]] && echo \"warmup failed, expect $file_count files, actual $files\" && exit 1 || true\n grep \"(100.0%)\" warmup.log || (echo \"warmup failed, expect 100.0% warmup\" && exit 1)\n ./juicefs warmup -f file.list --evict 2>&1 | tee warmup.log \n files=$(get_cache_file_count)\n [[ $files -ne $file_count ]] && echo \"warmup evict failed, expect $file_count files, actual $files\" && exit 1 || true\n ./juicefs warmup -f file.list --check 2>&1 | tee warmup.log\n files=$(get_cache_file_count)\n [[ $files -ne $file_count ]] && echo \"warmup evict failed, expect $file_count files, actual $files\" && exit 1 || true\n grep \"(0.0%)\" warmup.log || (echo \"warmup failed, expect 0.0% warmup\" && exit 1)\n\n ./juicefs warmup /tmp/jfs/test* 2>&1 | tee warmup.log\n files=$(get_cache_file_count)\n [[ $files -ne $file_count ]] && echo \"warmup failed, expect $file_count files, actual $files\" && exit 1 || true\n}\n\ntest_kernel_writeback_cache(){\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount $META_URL /tmp/jfs -d -o writeback_cache\n mkdir /tmp/jfs/fio\n runtime=15\n cat /tmp/jfs/.stats | grep fuse | grep 'juicefs_fuse_written_size_bytes_sum\\|juicefs_fuse_ops_total_write'\n fio --name=seq_write_test --rw=write --bs=10 --size=4M --numjobs=8 --nrfiles=1 --runtime=$runtime --time_based --group_reporting --directory=/tmp/jfs/fio | tee fio.log\n cat /tmp/jfs/.stats | grep fuse | grep 'juicefs_fuse_written_size_bytes_sum\\|juicefs_fuse_ops_total_write'\n bytes=$(cat /tmp/jfs/.stats | grep juicefs_fuse_written_size_bytes_sum | awk '{print $2}')\n ops=$(cat /tmp/jfs/.stats | grep juicefs_fuse_ops_total_write | awk '{print $2}')\n [[ $((bytes/ops)) -lt 10240 ]] && echo \"writeback_cache may not enabled\" && exit 1 || true\n}\n\ntest_o_tmpfile(){\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount $META_URL /tmp/jfs -d -o writeback_cache\n TEST_DIR=\"/tmp/jfs/tmp\"\n mkdir -p \"$TEST_DIR\"\n\n cat > /tmp/test_otmp.c << 'EOF'\n#define _GNU_SOURCE\n#include \n#include \n#include \n#include \n#include \nint main() {\n int fd = openat(AT_FDCWD, \"/tmp/jfs/tmp\", O_RDWR|O_EXCL|O_CLOEXEC|O_TMPFILE, 0600);\n if (fd < 0) {\n perror(\"openat\");\n return 1;\n }\n puts(\"openat ok\");\n if (write(fd, \"x\", 1) < 0) perror(\"write\");\n if (close(fd) < 0) {\n printf(\"close: %s\\n\", strerror(errno));\n return 1;\n }\n puts(\"close ok\");\n return 0;\n}\nEOF\n gcc -o /tmp/test_otmp /tmp/test_otmp.c\n /tmp/test_otmp\n result=$?\n if [ $result -ne 0 ]; then\n echo \"TEST FAILED: close fail\"\n exit 1\n else\n echo \"TEST PASSED\"\n fi\n}\n\ntest_cache_items(){\n do_test_cache_items 2-random\n}\n\ntest_cache_items_lru(){\n do_test_cache_items lru\n}\n\ndo_test_cache_items(){\n cache_eviction=$1\n prepare_test\n ./juicefs format $META_URL myjfs\n cache_items=500\n ./juicefs mount $META_URL /tmp/jfs -d --cache-items $cache_items --cache-eviction $cache_eviction\n seq 1 $((cache_items*2)) | xargs -P 8 -I {} sh -c 'echo {} > /tmp/jfs/test_{};'\n ./juicefs warmup /tmp/jfs/\n ./juicefs warmup /tmp/jfs/ --check 2>&1 | tee warmup.log\n ratio=$(get_warmup_ratio)\n [[ $ratio -lt 55 ]] || (echo \"ratio should less than 55%\" && exit 1)\n}\n\ntest_evict_on_writeback(){\n prepare_test\n ./juicefs format $META_URL myjfs --compress zstd\n ./juicefs mount $META_URL /tmp/jfs -d --writeback --upload-delay 3s\n dd if=/dev/urandom of=/tmp/test bs=1M count=200\n cp /tmp/test /tmp/jfs/test\n sleep 3\n stageBlocks=$(grep \"juicefs_staging_blocks\" /tmp/jfs/.stats | awk '{print $2}')\n [[ $stageBlocks -eq 0 ]] && echo \"stage blocks should not be 0\" && exit 1 || true\n ./juicefs warmup /tmp/jfs/test --evict\n wait_stage_uploaded\n compare_md5sum /tmp/test /tmp/jfs/test\n}\n\ntest_remount_on_writeback(){\n prepare_test\n ./juicefs format $META_URL myjfs --compress lz4\n ./juicefs mount $META_URL /tmp/jfs -d --writeback --upload-delay 3s\n dd if=/dev/urandom of=/tmp/test bs=1M count=200\n cp /tmp/test /tmp/jfs/test\n umount_jfs /tmp/jfs $META_URL\n ./juicefs mount $META_URL /tmp/jfs -d --writeback\n sleep 3\n stage_size=$(du -shm $(get_rawstaging_dir) | awk '{print $1}')\n [[ $stage_size -gt 2 ]] && echo \"stage size should not great than 2M\" && exit 1 || true\n ./juicefs warmup /tmp/jfs/test --evict\n compare_md5sum /tmp/test /tmp/jfs/test\n}\ntest_memory_cache_none(){\n do_test_memory_cache none\n}\n\ntest_memory_cache_2_random(){\n do_test_memory_cache 2-random\n}\n\ntest_memory_cache_lru_fallback(){\n prepare_test\n ./juicefs format $META_URL myjfs --compress lz4\n ./juicefs mount $META_URL /tmp/jfs -d --cache-dir memory --cache-size 100M --cache-eviction lru\n eviction=$(get_cache_eviction)\n [[ \"$eviction\" == \"2-random\" ]] || (echo \"memory cache should fallback to 2-random, actual is $eviction\" && exit 1)\n\n dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n ratio=$(get_warmup_ratio)\n [[ \"$ratio\" -gt 40 && \"$ratio\" -lt 60 ]] || (echo \"ratio($ratio) should between 40% and 60% after lru fallback\" && exit 1)\n}\n\ntest_cache_eviction_invalid_fallback(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount $META_URL /tmp/jfs -d --cache-size 100M --cache-eviction invalid-policy\n eviction=$(get_cache_eviction)\n [[ \"$eviction\" == \"2-random\" ]] || (echo \"invalid cache-eviction should fallback to 2-random, actual is $eviction\" && exit 1)\n}\n\ndo_test_memory_cache(){\n cache_eviction=$1\n prepare_test\n ./juicefs format $META_URL myjfs --compress lz4\n ./juicefs mount $META_URL /tmp/jfs -d --cache-dir memory --cache-size 100M --cache-eviction $cache_eviction\n dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n ratio=$(get_warmup_ratio)\n if [[ $cache_eviction == \"2-random\" ]]; then\n [[ \"$ratio\" -gt 40 && \"$ratio\" -lt 60 ]] || (echo \"ratio($ratio) should between 40% and 60%\" && exit 1)\n elif [[ $cache_eviction == \"none\" ]]; then\n [[ \"$ratio\" -gt 40 && \"$ratio\" -lt 60 ]] || (echo \"ratio($ratio) should between 40% and 60%\" && exit 1)\n fi\n ./juicefs warmup /tmp/jfs/test --evict\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n ratio=$(get_warmup_ratio)\n [[ \"$ratio\" = 0 ]] || (echo \"ratio($ratio) should less than 0\" && exit 1)\n}\n\ntest_cache_expired(){\n do_test_cache_expired /var/jfsCache/myjfs 2-random\n}\n\ntest_cache_expired_memory(){\n do_test_cache_expired memory 2-random\n}\n\ntest_cache_expired_lru(){\n do_test_cache_expired /var/jfsCache/myjfs lru\n}\n\ndo_test_cache_expired(){\n cache_dir=$1\n cache_eviction=$2\n [[ -z $cache_eviction ]] && cache_eviction=2-random\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount $META_URL /tmp/jfs -d --cache-dir $cache_dir --cache-expire 3s --cache-eviction $cache_eviction\n dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200\n for i in $(seq 1 1100); do\n dd if=/dev/zero of=/tmp/jfs/test$i bs=32k count=1 status=none\n done\n ./juicefs warmup /tmp/jfs/ 2>&1 | tee warmup.log\n sleep 15\n ./juicefs warmup /tmp/jfs/ --check 2>&1 | tee warmup.log\n grep \"(0.0%)\" warmup.log || (echo \"cache should expired\" && exit 1)\n}\n\ntest_cache_large_write(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount $META_URL /tmp/jfs -d -v\n dd if=/dev/zero of=/tmp/jfs/test bs=1M count=200\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n ratio=$(get_warmup_ratio)\n [[ \"$ratio\" = 0 ]] || (echo \"ratio($ratio) should less than 0\" && exit 1)\n ./juicefs mount $META_URL /tmp/jfs -d --cache-large-write \n dd if=/dev/zero of=/tmp/jfs/test1 bs=1M count=200\n ./juicefs warmup /tmp/jfs/test1 --check 2>&1 | tee warmup.log\n # TODO: should check the ratio\n check_warmup_log 90\n}\n\ntest_cache_mode(){\n prepare_test\n ./juicefs format $META_URL myjfs\n cache_mode=$(printf \"%03o\" $((RANDOM % 512)))\n echo \"cache mode is $cache_mode\"\n ./juicefs mount $META_URL /tmp/jfs -d --cache-mode $cache_mode --writeback --upload-delay 3s\n dd if=/dev/zero of=/tmp/jfs/test bs=1M count=32\n ./juicefs warmup /tmp/jfs/test\n find $(get_raw_dir) -type f ! -perm $cache_mode -exec echo \"perm of {} is incorrect\" \\; -exec false {} +\n find $(get_rawstaging_dir) -type f ! -perm $cache_mode -exec echo \"perm of {} is incorrect\" \\; -exec false {} +\n sleep 5s \n find $(get_raw_dir) -type f ! -perm $cache_mode -exec echo \"perm of {} is incorrect\" \\; -exec false {} +\n find $(get_rawstaging_dir) -type f ! -perm $cache_mode -exec echo \"perm of {} is incorrect\" \\; -exec false {} +\n}\n\ntest_cache_compressed(){\n prepare_test\n ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test \\\n --access-key minioadmin --secret-key minioadmin --compress lz4 --hash-prefix\n ./juicefs mount $META_URL /tmp/jfs -d \n dd if=/dev/urandom of=/tmp/test bs=1M count=200\n cp /tmp/test /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test --evict\n ./juicefs warmup /tmp/jfs/test\n docker stop minio\n compare_md5sum /tmp/test /tmp/jfs/test\n docker start minio\n}\n\ntest_cache_checksum_none(){\n do_test_cache_checksum none\n}\n\ntest_cache_checksum_full(){\n do_test_cache_checksum full\n}\n\ntest_cache_checksum_shrink(){\n do_test_cache_checksum shrink\n}\n\ntest_cache_checksum_extend(){\n do_test_cache_checksum extend\n}\n\ndo_test_cache_checksum(){\n checksum_level=$1\n prepare_test\n ./juicefs format $META_URL myjfs --compress lz4\n ./juicefs mount $META_URL /tmp/jfs -d --verify-cache-checksum $checksum_level\n mkdir -p /tmp/jfs/rand-rw\n fio --name=seq_rw --rw=readwrite --bsrange=1k-4k --size=80M --numjobs=4 --runtime=5 --time_based --group_reporting --filename=/tmp/jfs/req-rw\n fio --name=rand_rw --rw=randrw --bsrange=1k-4k --size=80M --numjobs=4 --runtime=5 --time_based --group_reporting --directory=/tmp/jfs/rand-rw --nrfiles=1000 --filesize=4k\n}\n\ntest_disk_full_2_random(){\n do_test_disk_full 2-random\n}\n\ntest_disk_full_lru(){\n do_test_disk_full lru\n}\n\ntest_disk_full_none(){\n do_test_disk_full none\n}\n\ndo_test_disk_full(){\n cache_eviction=$1\n prepare_test\n mount_jfsCache1 1G\n ./juicefs format $META_URL myjfs \n ./juicefs mount $META_URL /tmp/jfs -d --cache-dir /var/jfsCache1 --cache-eviction $cache_eviction --free-space-ratio 0.2\n dd if=/dev/zero of=/tmp/test bs=1M count=1200\n cp /tmp/test /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test\n sleep 3 # wait to free space\n df -h /var/jfsCache1\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n used_percent=$(df /var/jfsCache1 | tail -1 | awk '{print $5}' | tr -d %)\n echo \"used percent is $used_percent\"\n if [[ $cache_eviction == \"2-random\" || $cache_eviction == \"lru\" ]]; then \n [[ $used_percent -gt 80 ]] && echo \"used percent($used_percent) should not more than 80%\" && exit 1 || true\n elif [[ $cache_eviction == \"none\" ]]; then\n # cache will not evict even reach the free-space-ratio.\n [[ $used_percent -lt 80 ]] && echo \"used percent($used_percent) should not less than 80%\" && exit 1 || true\n fi\n}\n\ntest_lru_hotset_prefer_recent(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount $META_URL /tmp/jfs -d --cache-size 128M --cache-items 40 --cache-eviction lru\n\n mkdir -p /tmp/jfs/lru\n for i in $(seq 1 80); do\n dd if=/dev/zero of=/tmp/jfs/lru/f_$i bs=64k count=1 status=none\n done\n\n for i in $(seq 1 40); do\n ./juicefs warmup /tmp/jfs/lru/f_$i > /dev/null\n done\n\n sleep 2\n for i in $(seq 1 10); do\n cat /tmp/jfs/lru/f_$i > /dev/null\n done\n\n sleep 2\n for i in $(seq 41 70); do\n ./juicefs warmup /tmp/jfs/lru/f_$i > /dev/null\n done\n\n rm -f hot.list cold.list\n for i in $(seq 1 10); do\n echo /tmp/jfs/lru/f_$i >> hot.list\n done\n for i in $(seq 11 40); do\n echo /tmp/jfs/lru/f_$i >> cold.list\n done\n\n ./juicefs warmup -f hot.list --check 2>&1 | tee warmup.log\n hot_ratio=$(get_warmup_ratio)\n [[ \"$hot_ratio\" -eq 100 ]] || (echo \"hot set ratio($hot_ratio) should be 100% for lru\" && exit 1)\n\n ./juicefs warmup -f cold.list --check 2>&1 | tee warmup.log\n cold_ratio=$(get_warmup_ratio)\n [[ \"$cold_ratio\" -lt 20 ]] || (echo \"cold set ratio($cold_ratio) should be less than 20% for lru\" && exit 1)\n}\n\ntest_inode_full(){\n prepare_test\n mount_jfsCache1 100G 1000\n ./juicefs format $META_URL myjfs\n ./juicefs mount $META_URL /tmp/jfs -d --cache-dir /var/jfsCache1 --free-space-ratio 0.2\n seq 1 1000 | xargs -P 8 -I {} sh -c 'echo {} > /tmp/jfs/test_{};'\n ./juicefs warmup /tmp/jfs/\n ./juicefs warmup /tmp/jfs/ --check 2>&1 | tee warmup.log\n sleep 3\n used_percent=$(df -i /var/jfsCache1 | tail -1 | awk '{print $5}' | tr -d %)\n [[ $used_percent -gt 85 ]] && echo \"used percent($used_percent) should less than 85%\" && exit 1 || true\n}\n\ntest_disk_full_with_writeback(){\n prepare_test\n mount_jfsCache1 1G\n ./juicefs format $META_URL myjfs --compress zstd\n ./juicefs mount $META_URL /tmp/jfs -d --cache-dir /var/jfsCache1 --writeback --free-space-ratio 0.2 --upload-delay 5s\n dd if=/dev/urandom of=/tmp/test bs=1M count=1400\n cp /tmp/test /tmp/jfs/test\n wait_stage_uploaded\n sleep 3\n used_percent=$(df /var/jfsCache1 | tail -1 | awk '{print $5}' | tr -d %)\n [[ $used_percent -gt 80 ]] && echo \"used percent($used_percent) should less than 80%\" && exit 1 || true\n echo 3 > /proc/sys/vm/drop_caches\n ./juicefs warmup /tmp/jfs/test --evict\n compare_md5sum /tmp/test /tmp/jfs/test\n}\n\ntest_disk_failover()\n{\n prepare_test\n mount_jfsCache1\n rm -rf /var/log/juicefs.log\n rm -rf /var/jfsCache2 /var/jfsCache3\n ./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin\n JFS_MAX_DURATION_TO_DOWN=10s JFS_MAX_IO_DURATION=3s ./juicefs mount $META_URL /tmp/jfs -d \\\n --cache-dir=/var/jfsCache1:/var/jfsCache2:/var/jfsCache3 --io-retries 1 \n dd if=/dev/urandom of=/tmp/test bs=1M count=1024\n cp /tmp/test /tmp/jfs/test\n /etc/init.d/redis-server stop\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log\n check_warmup_log 50\n wait_disk_down 60\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log\n check_warmup_log 98\n check_cache_distribute 1024 /var/jfsCache2 /var/jfsCache3\n echo stop minio && docker stop minio\n compare_md5sum /tmp/test /tmp/jfs/test\n docker start minio && sleep 3\n}\n\ntest_disk_failover_lru()\n{\n prepare_test\n mount_jfsCache1\n rm -rf /var/log/juicefs.log\n rm -rf /var/jfsCache2 /var/jfsCache3\n ./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin\n JFS_MAX_DURATION_TO_DOWN=10s JFS_MAX_IO_DURATION=3s ./juicefs mount $META_URL /tmp/jfs -d \\\n --cache-dir=/var/jfsCache1:/var/jfsCache2:/var/jfsCache3 --io-retries 1 --cache-eviction lru\n dd if=/dev/urandom of=/tmp/test bs=1M count=1024\n cp /tmp/test /tmp/jfs/test\n /etc/init.d/redis-server stop\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log\n check_warmup_log 50\n wait_disk_down 60\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup --check /tmp/jfs 2>&1 | tee warmup.log\n check_warmup_log 98\n check_cache_distribute 1024 /var/jfsCache2 /var/jfsCache3\n echo stop minio && docker stop minio\n compare_md5sum /tmp/test /tmp/jfs/test\n docker start minio && sleep 3\n}\n\ntest_manual_delete_cache_data_lru()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin\n ./juicefs mount $META_URL /tmp/jfs -d --cache-eviction lru --cache-size 1G --cache-scan-interval -1\n\n dd if=/dev/urandom of=/tmp/test bs=1M count=256\n cp /tmp/test /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n check_warmup_log 95\n\n raw_dir=$(get_raw_dir)\n find \"$raw_dir\" -type f | head -n 200 | xargs rm -f\n sync\n echo 3 > /proc/sys/vm/drop_caches || true\n\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n ratio=$(get_warmup_ratio)\n [[ \"$ratio\" -lt 90 ]] || (echo \"after manually deleting cache data, warmup ratio($ratio) should be less than 90%\" && exit 1)\n\n compare_md5sum /tmp/test /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n check_warmup_log 95\n}\n\ntest_disk_failure_on_writeback()\n{\n prepare_test\n mount_jfsCache1\n rm -rf /var/log/juicefs.log\n rm -rf /var/jfsCache2 /var/jfsCache3\n mkdir -p /var/jfsCache2 /var/jfsCache3\n ./juicefs format $META_URL myjfs --trash-days 0 --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin\n JFS_MAX_DURATION_TO_DOWN=5s JFS_MAX_IO_DURATION=3s ./juicefs mount $META_URL /tmp/jfs -d \\\n --cache-dir=/var/jfsCache? --io-retries 1 --writeback -v\n dd if=/dev/urandom of=/tmp/test bs=1M count=1024\n cp /tmp/test /tmp/jfs/test\n dd if=/dev/urandom of=/tmp/jfs/test2 bs=1M count=10\n /etc/init.d/redis-server stop\n ./juicefs warmup /tmp/jfs/test2 &\n sleep 15\n grep -q \"state change from unstable to down\" /var/log/juicefs.log && echo \"disk should not down\" && exit 1 || true\n /etc/init.d/redis-server start\n ./juicefs warmup /tmp/jfs/test\n ./juicefs warmup /tmp/jfs/test --check 2>&1 | tee warmup.log\n # TODO: the ratio should be 100%\n check_warmup_log 60\n check_cache_distribute 1024 /var/jfsCache1 /var/jfsCache2 /var/jfsCache3\n compare_md5sum /tmp/test /tmp/jfs/test\n}\n\nprepare_test()\n{\n df -h /\n umount_jfs /tmp/jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs || true\n rm -rf /var/jfsCache/myjfs || true\n [[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc\n mc alias set myminio http://localhost:9000 minioadmin minioadmin\n mc rm --force --recursive myminio/test || true\n}\n\nwait_warmup_finish(){\n path=$1\n expected_ratio=$2\n timeout=30\n for i in $(seq 1 $timeout); do\n ./juicefs warmup $path --check 2>&1 |tee warmup.log\n ratio=$(get_warmup_ratio)\n if [[ \"$ratio\" == \"$expected_ratio\" ]]; then\n echo \"warmup finished after $i seconds, ratio is $ratio, expected ratio is $expected_ratio\"\n break\n else\n echo \"wait warmup finish $i\"\n sleep 1\n fi\n if [[ $i -eq $timeout ]]; then\n echo \"wait warmup finish timeout after $timeout seconds\" && exit 1\n fi\n done\n}\n\nwait_stage_uploaded()\n{\n echo \"wait stage upload\"\n for i in {1..30}; do\n stageBlocks=$(grep \"juicefs_staging_blocks\" /tmp/jfs/.stats | awk '{print $2}')\n if [[ \"$stageBlocks\" -eq 0 ]]; then\n echo \"stageBlocks is now 0\"\n break\n fi\n echo \"wait stage upload $i\" && sleep 1\n done\n if [[ \"$stageBlocks\" -ne 0 ]]; then\n echo \"stage blocks have not uploaded: $stageBlocks\" && exit 1\n fi\n}\n\nmount_jfsCache1(){\n capacity=$1\n [[ -z $capacity ]] && capacity=100G\n inodes=$2\n [[ -z $inodes ]] && inodes=10000000\n /etc/init.d/redis-server start\n timeout 30s bash -c 'until nc -zv localhost 6379; do sleep 1; done'\n umount -l /var/jfsCache1 || true\n rm -rf /var/jfsCache1\n redis-cli flushall\n rm -rf /var/jfs/test\n ./juicefs format \"redis://localhost/1?read-timeout=3&write-timeout=1&max-retry-backoff=3\" test --trash-days 0 --capacity $capacity --inodes $inodes\n ./juicefs mount redis://localhost/1 /var/jfsCache1 -d --log /tmp/juicefs.log\n # trap \"echo umount /var/jfsCache1 && umount -l /var/jfsCache1\" EXIT\n}\n\nget_cache_dir(){\n grep CacheDir /tmp/jfs/.config | awk -F'\"' '{print $4}'\n}\n\nget_cache_eviction(){\n grep CacheEviction /tmp/jfs/.config | awk -F'\"' '{print $4}'\n}\n\nget_raw_dir(){\n echo $(get_cache_dir)/raw/\n}\n\nget_rawstaging_dir(){\n echo $(get_cache_dir)/rawstaging/\n}\n\ncheck_evict_log(){\n ratio=$(get_warmup_ratio)\n if [[ \"$ratio\" -gt 0 ]]; then\n echo \"cache ratio($ratio) should be 0 after evict\"\n exit 1\n fi\n}\n\ncheck_warmup_log(){\n expected_ratio=$1\n ratio=$(get_warmup_ratio)\n if [[ \"$ratio\" -lt \"$expected_ratio\" ]]; then\n echo \"cache ratio($ratio) should be more than expected_ratio($expected_ratio) after warmup\"\n exit 1\n fi\n}\n\nget_cache_file_count(){\n sed -n 's/.* \\([0-9]\\+\\) files.*/\\1/p' warmup.log\n}\n\nget_cache_file_size(){\n sed -n 's/.* \\([0-9]*\\) MiB of.*/\\1/p' warmup.log\n}\n\nget_warmup_ratio(){\n sed -n 's/.*(\\([0-9]*\\.[0-9]*%\\)).*/\\1/p' warmup.log | sed 's/%//' | awk '{print int($1)}'\n}\n\n\ncheck_cache_distribute() {\n max_total_size=$(echo \"$1 * 1024\" | bc | awk '{printf \"%.0f\", $1}')\n echo check_cache_distribute, max_total_size is $max_total_size\n shift\n total_weight=0\n declare -A weights\n declare -A sizes\n # Parse directory names and weights\n for arg in \"$@\"; do\n dir=$(echo \"$arg\" | awk -F: '{print $1}')\n weight=$(echo \"$arg\" | awk -F: '{print $2}')\n if [[ -z $weight ]]; then\n weight=1\n fi\n weights[\"$dir\"]=$weight\n total_weight=$((total_weight + weight))\n done\n \n # Calculate total size and sizes of each directory\n for dir in \"${!weights[@]}\"; do\n echo dir is $dir\n du -sh \"$dir\" || true\n size=$(du -s \"$dir\" | awk '{print $1}')\n echo size is $size\n sizes[\"$dir\"]=$size\n done\n \n # Check if total size exceeds max limit\n total_size=0\n for dir in \"${!sizes[@]}\"; do\n size=${sizes[\"$dir\"]}\n total_size=$((total_size + size))\n done\n echo \"total size is $total_size, max_total_size is $max_total_size\"\n if [[ $total_size -gt $((max_total_size + max_total_size/10)) ]]; then\n echo \"Total size of directories exceeds max limit\"\n return 1\n fi\n \n # Check if each directory is evenly distributed based on its weight\n for dir in \"${!sizes[@]}\"; do\n size=${sizes[\"$dir\"]}\n weight=${weights[\"$dir\"]}\n avg_size=$((total_size * weight / total_weight))\n min_size=$((avg_size * 5 / 10))\n max_size=$((avg_size * 20 / 10))\n \n if [[ $size -lt $min_size || $size -gt $max_size ]]; then\n echo \"$dir is not evenly distributed, size: $size, weight: $weight, ave_size: $avg_size, min_size: $min_size, max_size: $max_size\"\n exit 1\n else\n echo \"$dir is evenly distributed\"\n fi\n done\n}\n\nwait_disk_down()\n{\n timeout=$1\n for i in $(seq 1 $timeout); do\n if grep -q \"state change from unstable to down\" /var/log/juicefs.log; then\n echo \"state changed from unstable to down after $i seconds\"\n return\n else\n echo \"\\rWait for state change to down, $i\"\n sleep 1\n count=$((count+1))\n fi\n done\n echo \"Wait for state change to down timeout after $timeout seconds\" && exit 1\n} \n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/chaos/dynamic.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: dynamic-ce\n labels:\n juicefs-app-type: dynamic-ce\nspec:\n replicas: 1\n selector:\n matchLabels:\n juicefs-app-type: dynamic-ce\n template:\n metadata:\n labels:\n juicefs-app-type: dynamic-ce\n spec:\n containers:\n - name: vdbench\n image: zwwhdlsdocker/vdbench:latest\n imagePullPolicy: IfNotPresent\n volumeMounts:\n - mountPath: /data\n name: data\n - mountPath: /vdbench/config\n name: vdbench-cfg\n - mountPath: /vdbench/output\n name: output\n command: [\"sh\", \"-c\", \"./vdbench -f /vdbench/config/vdbench.vdb -v\"]\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: dynamic-ce\n - name: output\n hostPath:\n path: /root/vdbench/output\n - name: vdbench-cfg\n configMap:\n name: dynamic-ce\n items:\n - key: \"vdbench.vdb\"\n path: \"vdbench.vdb\"\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: dynamic-ce\ndata:\n vdbench.vdb: |\n messagescan=no\n fsd=fsd1,anchor=/data,depth=1,width=1,files=20000,size=4k,openflags=o_direct\n fwd=fwd1,fsd=fsd1,operation=write,xfersize=4k,fileio=random,fileselect=random,threads=1\n rd=rd1,fwd=fwd1,fwdrate=max,format=yes,elapsed=60,interval=2\n" }, { "path": ".github/scripts/chaos/juicefs-csi-driver.Dockerfile", "content": "FROM golang:1.20-buster as builder\n\nARG GOPROXY\n# refs/remotes/pull/3056/merge\nARG GITHUB_REF\n# 4ac69613b5919142d87f21a64ca744ae537192d6\nARG GITHUB_SHA\nARG JUICEFS_REPO_URL=https://github.com/juicedata/juicefs\n\nWORKDIR /workspace\nENV GOPROXY=${GOPROXY:-https://proxy.golang.org}\nENV STATIC=1\n\nRUN apt-get update && apt-get install -y musl-tools upx-ucl && \\\n cd /workspace && git clone --depth=1 $JUICEFS_REPO_URL && \\\n cd juicefs && git fetch --no-tags --prune origin +$GITHUB_SHA:$GITHUB_REF && \\\n git checkout $GITHUB_REF && \\\n make juicefs\n\nFROM juicedata/juicefs-csi-driver:nightly\n\nWORKDIR /app\nCOPY --from=builder /workspace/juicefs/juicefs /usr/local/bin/\n\nRUN ls -l /usr/local/bin/juicefs\n\nRUN /usr/local/bin/juicefs --version\nRUN echo GITHUB_REF is $GITHUB_REF\nRUN echo GITHUB_SHA is $GITHUB_SHA\n\n# ENTRYPOINT [\"/tini\", \"--\", \"/bin/juicefs-csi-driver\"]\n" }, { "path": ".github/scripts/chaos/juicefs.Dockerfile", "content": "FROM juicedata/mount:nightly\nCOPY ./juicefs /usr/local/bin/juicefs\n# RUN apt-get update && apt-get install -y musl-tools upx-ucl && STATIC=1 make\n# RUN cp -f juicefs /usr/local/bin/juicefs\nRUN /usr/local/bin/juicefs version" }, { "path": ".github/scripts/chaos/minio.yaml", "content": "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: minio-server\n namespace: kube-system\n labels:\n app: minio-server\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: minio-server\n serviceName: minio\n template:\n metadata:\n labels:\n app: minio-server\n spec:\n containers:\n - name: minio\n image: minio/minio\n resources:\n limits:\n memory: \"500Mi\"\n cpu: \"500m\"\n limits:\n memory: \"100Mi\"\n cpu: \"100m\"\n env:\n - name: MINIO_ROOT_USER\n value: minioadmin\n - name: MINIO_ROOT_PASSWORD\n value: minioadmin\n args:\n - server\n - /data\n volumeMounts:\n - mountPath: /data\n name: minio-data\n ports:\n - containerPort: 9000\n name: sever\n volumes:\n - name: minio-data\n hostPath:\n path: /data/minio-data\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: minio\n namespace: kube-system\nspec:\n type: NodePort\n selector:\n app: minio-server\n ports:\n - protocol: TCP\n port: 9000\n targetPort: 9000\n nodePort: 31275\n name: server" }, { "path": ".github/scripts/chaos/pvc.yaml", "content": "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: dynamic-ce\nspec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n storage: 5Pi\n storageClassName: dynamic-ce" }, { "path": ".github/scripts/chaos/redis.yaml", "content": "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: redis-server\n namespace: kube-system\n labels:\n app: redis-server\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: redis-server\n serviceName: redis\n template:\n metadata:\n labels:\n app: redis-server\n spec:\n containers:\n - name: redis\n image: redis\n volumeMounts:\n - mountPath: /data\n name: redis-data\n resources:\n limits:\n memory: \"500Mi\"\n cpu: \"500m\"\n limits:\n memory: \"100Mi\"\n cpu: \"100m\"\n ports:\n - containerPort: 6379\n volumes:\n - name: redis-data\n hostPath:\n path: /data/redis\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: redis\n namespace: kube-system\nspec:\n type: NodePort\n selector:\n app: redis-server\n ports:\n - protocol: TCP\n port: 6379\n targetPort: 6379\n nodePort: 31274\n" }, { "path": ".github/scripts/chaos/sc.yaml", "content": "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: dynamic-ce\nparameters:\n csi.storage.k8s.io/node-publish-secret-name: dynamic-ce\n csi.storage.k8s.io/node-publish-secret-namespace: kube-system\n csi.storage.k8s.io/provisioner-secret-name: dynamic-ce\n csi.storage.k8s.io/provisioner-secret-namespace: kube-system\n juicefs/mount-cpu-limit: 5000m\n juicefs/mount-memory-limit: 1Gi\n juicefs/mount-cpu-request: 100m\n juicefs/mount-memory-request: 500Mi\n juicefs/mount-image: juicedata/mount:ci\n#mountOptions:\n# - cache-dir=/var/foo:/var/foo1:/var/foo2\nprovisioner: csi.juicefs.com\nreclaimPolicy: Delete\nvolumeBindingMode: Immediate\n---\napiVersion: v1\nstringData:\n access-key: minioadmin\n bucket: http://minio.kube-system:9000/minio/dynamic-ce\n name: dynamic-ce\n metaurl: redis://redis.kube-system:6379/0\n secret-key: minioadmin\n storage: minio\n format-options: trash-days=0,block-size=4096\nkind: Secret\nmetadata:\n name: dynamic-ce\n namespace: kube-system\ntype: Opaque\n" }, { "path": ".github/scripts/chaos/workflow.yaml", "content": "apiVersion: chaos-mesh.org/v1alpha1\nkind: Workflow\nmetadata:\n name: juicefs-workflow\nspec:\n entry: the-entry\n templates:\n - name: the-entry\n templateType: Parallel\n children:\n # - minio-delay\n # - minio-io\n # - minio-memory\n # - minio-cpu\n # - minio-bandwidth\n # - redis-bandwidth\n # - redis-io\n # - redis-delay\n # - redis-memory\n # - redis-cpu\n # - juicefs-bandwidth\n # - juicefs-memory\n # - juicefs-cpu\n # - juicefs-delay\n # minio 带宽\n - name: minio-bandwidth\n templateType: NetworkChaos\n deadline: 20s\n networkChaos:\n action: bandwidth\n mode: all\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: minio-server\n bandwidth:\n rate: '500bps'\n limit: 100\n buffer: 10000\n # minio 网络延迟\n - name: minio-delay\n templateType: NetworkChaos\n networkChaos:\n action: delay\n mode: all\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: minio-server\n delay:\n latency: '500ms'\n correlation: '50'\n jitter: '500ms'\n # minio 磁盘读写延迟\n - name: minio-io\n templateType: IOChaos\n ioChaos:\n action: latency\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: minio-server\n volumePath: /data\n delay: '50ms'\n # minio 内存压力\n - name: minio-memory\n templateType: StressChaos\n stressChaos:\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: minio-server\n stressors:\n memory:\n workers: 4\n size: '128MB'\n # minio cpu 压力\n - name: minio-cpu\n templateType: StressChaos\n stressChaos:\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: minio-server\n stressors:\n cpu:\n workers: 4\n load: 100\n # redis 带宽\n - name: redis-bandwidth\n templateType: NetworkChaos\n networkChaos:\n action: bandwidth\n mode: all\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: redis-server\n bandwidth:\n rate: '200mbps'\n limit: 100\n buffer: 10000\n - name: redis-delay\n templateType: NetworkChaos\n networkChaos:\n action: delay\n mode: all\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: redis-server\n delay:\n latency: '100ms'\n correlation: '50'\n jitter: '500ms'\n # redis 磁盘读写延迟\n - name: redis-io\n templateType: IOChaos\n ioChaos:\n action: latency\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: redis-server\n volumePath: /redis\n delay: '1s'\n # redis 内存压力\n - name: redis-memory\n templateType: StressChaos\n stressChaos:\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: redis-server\n stressors:\n memory:\n workers: 4\n size: '2GB'\n # redis cpu 压力\n - name: redis-cpu\n templateType: StressChaos\n stressChaos:\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app: redis-server\n stressors:\n cpu:\n workers: 4\n load: 100\n # 客户端带宽\n - name: juicefs-bandwidth\n templateType: NetworkChaos\n deadline: 20s\n networkChaos:\n action: bandwidth\n mode: all\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app.kubernetes.io/name: juicefs-mount\n bandwidth:\n rate: '100bps'\n limit: 100\n buffer: 10000\n - name: juicefs-delay\n templateType: NetworkChaos\n networkChaos:\n action: delay\n mode: all\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app.kubernetes.io/name: juicefs-mount\n delay:\n latency: '100ms'\n correlation: '50'\n jitter: '500ms'\n # 客户端内存压力\n - name: juicefs-memory\n templateType: StressChaos\n stressChaos:\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app.kubernetes.io/name: juicefs-mount\n stressors:\n memory:\n workers: 4\n size: '1GB'\n # 客户端cpu压力\n - name: juicefs-cpu\n templateType: StressChaos\n stressChaos:\n mode: one\n selector:\n namespaces:\n - kube-system\n labelSelectors:\n app.kubernetes.io/name: juicefs-mount\n stressors:\n cpu:\n workers: 4\n load: 100\n" }, { "path": ".github/scripts/check_juicefs_log.sh", "content": "#!/bin/bash -e\nfor log_file in /var/log/juicefs.log $HOME/.juicefs/juicefs.log; do\n if [ -f $log_file ]; then\n break\n fi\ndone\necho \"tail -1000 $log_file\"\ntail -1000 $log_file\ngrep -i \"\\|panic\" $log_file && exit 1 || true" }, { "path": ".github/scripts/cmptree.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2015, Bill Zissimopoulos. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met:\n#\n# 1. Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n#\n# 2. Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n#\n# 3. Neither the name of the copyright holder nor the names of its\n# contributors may be used to endorse or promote products derived from\n# this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS\n# IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED\n# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A\n# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\nimport subprocess\ntry:\n __import__(\"xattr\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\nimport filecmp, os\nimport xattr\n\nclass TreeComparator(object):\n def __init__(self, dir1, dir2):\n self.dir1 = dir1\n self.dir2 = dir2\n self.left_only = []\n self.right_only = []\n self.common_funny = []\n self.funny_files = []\n self.diff_files = []\n def compare(self, p=\"\"):\n d1 = os.path.join(self.dir1, p)\n d2 = os.path.join(self.dir2, p)\n print(f'compare {d1} with {d2}')\n dcmp = filecmp.dircmp(d1, d2, ignore=[])\n self.left_only.extend(os.path.join(p, n) for n in dcmp.left_only)\n self.right_only.extend(os.path.join(p, n) for n in dcmp.right_only)\n self.common_funny.extend(os.path.join(p, n) for n in dcmp.common_funny)\n self.funny_files.extend(os.path.join(p, n) for n in dcmp.funny_files)\n #(match, mismatch, errors) = filecmp.cmpfiles(d1, d2, dcmp.common_files, shallow=False)\n #self.diff_files.extend(os.path.join(p, n) for n in mismatch)\n #self.funny_files.extend(os.path.join(p, n) for n in errors)\n (match, mismatch, errors) = self.compare_files(d1, d2, dcmp.common_files)\n self.diff_files.extend(os.path.join(p, n) for n in mismatch)\n self.funny_files.extend(os.path.join(p, n) for n in errors)\n for d in dcmp.common_dirs:\n self.compare(os.path.join(p, d))\n\n def compare_files(self, d1, d2, files):\n match = []\n mismatch = []\n errors = []\n for f in files:\n f1 = os.path.join(d1, f)\n f2 = os.path.join(d2, f)\n try:\n s1 = os.stat(f1)\n s2 = os.stat(f2) \n for attr in ['st_mode', 'st_nlink', 'st_uid', 'st_gid', 'st_size']:\n if getattr(s1, attr) != getattr(s2, attr):\n print(f'{attr} mismatch with {f1}:{getattr(s1, attr)} and {f2}:{getattr(s2, attr)}')\n mismatch.append(f)\n continue\n if not filecmp.cmp(f1, f2):\n print(f'content mismatch with {f1} and {f2}')\n mismatch.append(f)\n continue\n if not self.compare_xattr(f1, f2):\n print(f'xattr mismatch with {f1} and {f2}')\n mismatch.append(f)\n continue\n match.append(f)\n except:\n print(f'error: {f}')\n errors.append(f)\n return match, mismatch, errors\n\n def compare_xattr(self, f1, f2):\n for attr in xattr.listxattr(f1):\n a1 = xattr.getxattr(f1, attr)\n a2 = xattr.getxattr(f2, attr)\n if a1 != a2:\n return False\n return True\n\nif \"__main__\" == __name__:\n import argparse, sys\n def info(s):\n print (\"%s: %s\" % (os.path.basename(sys.argv[0]), s))\n def warn(s):\n print (\"%s: %s\" % (os.path.basename(sys.argv[0]), s))\n def fail(s, exitcode = 1):\n warn(s)\n sys.exit(exitcode)\n def main():\n p = argparse.ArgumentParser()\n p.add_argument(\"-q\", \"--quiet\", action=\"store_true\")\n p.add_argument(\"dir1\")\n p.add_argument(\"dir2\")\n args = p.parse_args(sys.argv[1:])\n print('start compare tree')\n tcmp = TreeComparator(args.dir1, args.dir2)\n tcmp.compare()\n res = len(tcmp.left_only) + len(tcmp.right_only) + \\\n len(tcmp.funny_files) + len(tcmp.diff_files)\n # res = len(tcmp.left_only) + len(tcmp.right_only) + \\\n # len(tcmp.common_funny) + len(tcmp.funny_files) + len(tcmp.diff_files)\n if not args.quiet:\n if tcmp.left_only:\n print (\"Left only:\")\n for n in tcmp.left_only:\n print( \" %s\" % n)\n if tcmp.right_only:\n print (\"Right only:\")\n for n in tcmp.right_only:\n print( \" %s\" % n)\n if tcmp.funny_files:\n print (\"Funny files:\")\n for n in tcmp.funny_files:\n print( \" %s\" % n)\n # if tcmp.common_funny:\n # print (\"Differing stats:\")\n # for n in tcmp.common_funny:\n # print (\" %s\" % n)\n if tcmp.diff_files:\n print (\"Differing files:\")\n for n in tcmp.diff_files:\n print (\" %s\" % n)\n sys.exit(int(0 < res))\n def __entry():\n try:\n main()\n except EnvironmentError as ex:\n fail(ex)\n except KeyboardInterrupt:\n fail(\"interrupted\", 130)\n __entry()" }, { "path": ".github/scripts/command/acl.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\nprepare_test()\n{\n umount_jfs /tmp/jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs || true\n rm -rf /var/jfsCache/myjfs || true\n}\n\ntest_acl_with_kernel_check()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --enable-acl --trash-days 0\n ./juicefs mount -d $META_URL /tmp/jfs\n python3 .github/scripts/hypo/fs_acl_test.py \n}\n\ntest_acl_with_user_space_check()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --enable-acl --trash-days 0\n ./juicefs mount -d $META_URL /tmp/jfs --non-default-permission\n python3 .github/scripts/hypo/fs_acl_test.py \n}\n\ntest_modify_acl_config()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount -d $META_URL /tmp/jfs\n touch /tmp/jfs/test\n setfacl -m u:root:rw /tmp/jfs/test && echo \"setfacl should failed\" && exit 1\n ./juicefs config $META_URL --enable-acl=true\n ./juicefs mount -d $META_URL /tmp/jfs\n setfacl -m u:root:rw /tmp/jfs/test\n ./juicefs config $META_URL --enable-acl\n umount_jfs /tmp/jfs $META_URL\n ./juicefs mount -d $META_URL /tmp/jfs\n setfacl -m u:root:rw /tmp/jfs/test\n ./juicefs config $META_URL --enable-acl=false && echo \"should not disable acl\" && exit 1 || true \n ./juicefs config $META_URL | grep EnableACL | grep \"true\" || (echo \"EnableACL should be true\" && exit 1) \n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command/clone.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\ntest_clone_preserve_with_file()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n id -u juicefs && sudo userdel juicefs\n sudo useradd -u 1101 juicefs\n sudo -u juicefs touch /jfs/test\n for mode in 777 755 644; do\n sudo -u juicefs chmod $mode /jfs/test\n check_guid_after_clone true\n check_guid_after_clone false\n done\n}\n\ntest_clone_preserve_with_dir()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n id -u juicefs && sudo userdel juicefs\n sudo useradd -u 1101 juicefs\n sudo -u juicefs mkdir /jfs/test\n for mode in 777 755 644; do\n sudo -u juicefs chmod $mode /jfs/test\n check_guid_after_clone true\n check_guid_after_clone false\n done\n}\n\ntest_clone_with_jfs_source()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n [[ ! -d /jfs/juicefs ]] && git clone https://github.com/juicedata/juicefs.git /jfs/juicefs --depth 1\n do_clone true\n do_clone false\n}\n\nskip_test_clone_with_fsrand()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n seed=$(date +%s)\n python3 .github/scripts/fsrand.py -a -c 2000 -s $seed /jfs/juicefs\n do_clone true\n do_clone false \n}\n\ndo_clone()\n{\n is_preserve=$1\n rm -rf /jfs/juicefs1\n rm -rf /jfs/juicefs2\n [[ \"$is_preserve\" == \"true\" ]] && preserve=\"--preserve\" || preserve=\"\"\n cp -r /jfs/juicefs /jfs/juicefs1 $preserve\n ./juicefs clone /jfs/juicefs /jfs/juicefs2 $preserve\n diff -ur /jfs/juicefs1 /jfs/juicefs2 --no-dereference\n cd /jfs/juicefs1/ && find . -printf \"%m\\t%u\\t%g\\t%p\\n\" | sort -k4 >/tmp/log1 && cd -\n cd /jfs/juicefs2/ && find . -printf \"%m\\t%u\\t%g\\t%p\\n\" | sort -k4 >/tmp/log2 && cd -\n diff -u /tmp/log1 /tmp/log2\n}\n\ntest_clone_with_big_file()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n dd if=/dev/urandom of=/tmp/test bs=1M count=1000\n cp /tmp/test /jfs/test\n ./juicefs clone /jfs/test /jfs/test1\n rm /jfs/test -rf\n diff /tmp/test /jfs/test1\n}\ntest_clone_with_big_file2()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n dd if=/dev/urandom of=/tmp/test bs=1M count=1000\n echo \"a\" | tee -a /tmp/test\n cp /tmp/test /jfs/test\n ./juicefs clone /jfs/test /jfs/test1\n rm /jfs/test -rf\n diff /tmp/test /jfs/test1\n}\n\ntest_clone_with_random_write(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n PATH1=/tmp/test PATH2=/jfs/test python3 .github/scripts/random_read_write.py \n ./juicefs clone /jfs/test /jfs/test1\n rm /jfs/test -rf\n diff /tmp/test /jfs/test1\n}\n\ntest_clone_with_sparse_file()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n fallocate -l 1.0001g /jfs/test\n ./juicefs clone /jfs/test /jfs/test1\n diff /jfs/test /jfs/test1\n}\n\ntest_clone_with_sparse_file2()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n fallocate -l 1.1T /jfs/test\n ./juicefs clone /jfs/test /jfs/test1\n}\n\ntest_clone_with_small_files(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs/test\n for i in $(seq 1 2000); do\n echo $i > /jfs/test/$i\n done\n ./juicefs clone /jfs/test /jfs/test1\n diff -ur /jfs/test1 /jfs/test1\n}\n\nskip_test_clone_with_mdtest1()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n ./juicefs mdtest $META_URL /test --depth 2 --dirs 10 --files 10 --threads 100 --write 8192\n ./juicefs clone /jfs/test /jfs/test1\n ./juicefs rmr /jfs/test\n ./juicefs rmr /jfs/test1\n}\n\nskip_test_clone_with_mdtest2()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n ./juicefs mdtest $META_URL /test --depth 1 --dirs 1 --files 1000 --threads 100 --write 8192\n ./juicefs clone /jfs/test /jfs/test1\n ./juicefs rmr /jfs/test\n ./juicefs rmr /jfs/test1\n}\n\ncheck_guid_after_clone(){\n is_preserve=$1\n echo \"check_guid_after_clone, is_preserve: $is_preserve\"\n [[ \"$is_preserve\" == \"true\" ]] && preserve=\"--preserve\" || preserve=\"\"\n rm /jfs/test1 -rf\n sleep 3\n ls /jfs/test1 && echo \"test1 should not exist\" && exit 1 || echo \"/jfs/test1 not exist\" \n rm /jfs/test2 -rf\n ./juicefs clone /jfs/test /jfs/test1 $preserve\n cp /jfs/test /jfs/test2 -rf $preserve\n uid1=$(stat -c %u /jfs/test1)\n gid1=$(stat -c %g /jfs/test1)\n mode1=$(stat -c %a /jfs/test1)\n uid2=$(stat -c %u /jfs/test2)\n gid2=$(stat -c %g /jfs/test2)\n mode2=$(stat -c %a /jfs/test2)\n\n if [[ \"$uid1\" != \"$uid2\" ]] || [[ \"$gid1\" != \"$gid2\" ]] || [[ \"$mode1\" != \"$mode2\" ]]; then\n echo >&2 \": clone does not same as cp: uid1: $uid1, uid2: $uid2, gid1: $gid1, gid2: $gid2, mode1: $mode1, mode2: $mode2\"\n exit 1\n fi\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command/config.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META minio\nMETA_URL=$(get_meta_url $META)\n# version lower than 1.3.0 does not support parameter max_open_conns\nif [[ $META_URL == *\"?max_open_conns=\"* ]]; then\n META_URL=${META_URL%%\\?*}\nfi\nLEGACY_META_URL=$META_URL\nif [[ \"$META\" == \"redis\" ]]; then\n LEGACY_META_URL=${META_URL%%\\?*}\nfi\n[ ! -x mc ] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x mc\n\ndownload_juicefs_client(){\n version=$1\n wget -q https://github.com/juicedata/juicefs/releases/download/v$version/juicefs-$version-linux-amd64.tar.gz\n tar -xzf juicefs-$version-linux-amd64.tar.gz -C /tmp/\n sudo cp /tmp/juicefs juicefs-$version\n ./juicefs-$version version\n}\n\ntest_config_min_client_version()\n{\n prepare_test\n download_juicefs_client 1.0.0\n ./juicefs format $META_URL myjfs\n ./juicefs-1.0.0 mount $LEGACY_META_URL /jfs -d && exit 1 || true\n ./juicefs config $META_URL --min-client-version 1.0.1\n ./juicefs-1.0.0 mount $LEGACY_META_URL /jfs -d && exit 1 || true\n ./juicefs config $META_URL --min-client-version 1.0.0\n ./juicefs-1.0.0 mount $LEGACY_META_URL /jfs -d\n}\n\ntest_config_max_client_version()\n{\n prepare_test\n current_version=$(./juicefs version | awk '{print $3}')\n download_juicefs_client 1.0.0\n ./juicefs-1.0.0 format $LEGACY_META_URL myjfs\n ./juicefs-1.0.0 config $LEGACY_META_URL --max-client-version 1.0.1\n ./juicefs mount $META_URL /jfs -d && exit 1 || true\n ./juicefs config $META_URL --max-client-version $current_version\n ./juicefs mount $META_URL /jfs -d\n}\n\ntest_config_secret_key(){\n # # Consider command as failed when any component of the pipe fails:\n # https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash\n prepare_test\n set -o pipefail\n ./mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin\n ./mc admin user add minio juicedata juicedata\n ./mc admin policy attach minio consoleAdmin --user juicedata\n ./juicefs format --storage minio --bucket http://localhost:9000/jfs-test --access-key juicedata --secret-key juicedata $META_URL myjfs\n ./juicefs mount $META_URL /jfs -d --io-retries 1 --no-usage-report --heartbeat 3\n\n ./mc admin user remove minio juicedata\n ./mc admin user add minio juicedata1 juicedata1\n ./mc admin policy attach minio consoleAdmin --user juicedata1\n ./juicefs config $META_URL --access-key juicedata1 --secret-key juicedata1\n sleep 6\n echo abc | tee /jfs/abc.txt && echo \"write success\"\n cat /jfs/abc.txt | grep abc && echo \"read success\"\n}\n \n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command/debug.sh", "content": "#!/bin/bash -e\n\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\ncheck_debug_file(){\n files=(\"system-info.log\" \"juicefs.log\" \"config.txt\" \"stats.txt\" \"stats.5s.txt\" \"pprof\")\n debug_dir=\"debug\"\n if [ ! -d \"$debug_dir\" ]; then\n echo \"error:no debug dir\"\n exit 1\n fi\n all_files_exist=true\n for file in \"${files[@]}\"; do\n exist=`find \"$debug_dir\" -name $file | wc -l`\n if [ \"$exist\" == 0 ]; then\n echo \"no $file\"\n all_files_exist=false\n fi\n done\n if [ \"$all_files_exist\" = true ]; then\n echo \"pass\"\n else\n exit 1\n fi\n}\n\ntest_debug_juicefs(){\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs\n dd if=/dev/urandom of=/jfs/bigfile bs=1M count=128\n ./juicefs debug /jfs/\n check_debug_file\n ./juicefs rmr /jfs/bigfile\n}\n\ntest_debug_abnormal_juicefs(){\n rm -rf debug | true\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs\n dd if=/dev/urandom of=/jfs/bigfile bs=1M count=128\n killall -9 redis-server | true\n ./juicefs debug /jfs/\n# check_debug_file\n ./juicefs rmr /jfs/bigfile\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/dump_load.sh", "content": "#!/bin/bash -ex\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\nMETA_URL2=$(get_meta_url2 $META)\n[[ -z \"$SEED\" ]] && SEED=$(date +%s)\nHEARTBEAT_INTERVAL=2\nDIR_QUOTA_FLUSH_INTERVAL=4\n# [[ -z \"$SEED\" ]] && SEED=1711594639\n[[ -z \"$BINARY\" ]] && BINARY=false\n[[ -z \"$FAST\" ]] && FAST=false\n\ntrap \"echo random seed is $SEED\" EXIT\n\nif ! docker ps | grep -q minio; then\n docker run -d -p 9000:9000 --name minio \\\n -e \"MINIO_ACCESS_KEY=minioadmin\" \\\n -e \"MINIO_SECRET_KEY=minioadmin\" \\\n -v /tmp/data:/data \\\n -v /tmp/config:/root/.minio \\\n minio/minio server /data\nfi\n[[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc\nsleep 3s\nmc alias set myminio http://localhost:9000 minioadmin minioadmin\npython3 -c \"import xattr\" || sudo pip install xattr\n\ntest_dump_load_sustained_file(){\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount -d $META_URL /jfs\n file_count=100\n for i in $(seq 1 $file_count); do\n touch /jfs/file$i\n exec {fd}<>/jfs/file$i\n echo fd is $fd\n fds[$i]=$fd\n rm /jfs/file$i\n done\n ./juicefs dump $META_URL dump.json $(get_dump_option)\n for i in $(seq 1 $file_count); do\n fd=${fds[$i]}\n exec {fd}>&-\n done\n if [[ \"$BINARY\" == \"true\" ]]; then\n sustained=$(./juicefs load dump.json --binary --stat | grep sustained | awk -F\"|\" '{print $2}')\n else\n sustained=$(jq '.Sustained[].inodes | length' dump.json)\n fi\n echo \"sustained file count: $sustained\"\n # TODO： uncomment this line \n # [[ \"$sustained\" -eq \"$file_count\" ]] || (echo \"sustained file count($sustained) should be $file_count\" && exit 1)\n umount_jfs /jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n ./juicefs load $META_URL dump.json $(get_load_option)\n ./juicefs mount -d $META_URL /jfs \n}\n\ntest_dump_load_with_copy_file_range(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n rm -rf /tmp/test\n dd if=/dev/zero of=/tmp/test bs=1M count=1024\n cp /tmp/test /jfs/test\n node .github/scripts/copyFile.js /jfs/test /jfs/test1\n ./juicefs dump $META_URL dump.json $(get_dump_option)\n umount_jfs /jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n ./juicefs load $META_URL dump.json $(get_load_option)\n ./juicefs mount -d $META_URL /jfs\n compare_md5sum /tmp/test /jfs/test1\n}\n\ntest_dump_load_with_quota(){\n prepare_test\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n ./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1\n ./juicefs dump --log-level error $META_URL $(get_dump_option) > dump.json\n umount_jfs /jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n ./juicefs load $META_URL dump.json $(get_load_option)\n ./juicefs mount $META_URL /jfs -d --heartbeat $HEARTBEAT_INTERVAL\n ./juicefs quota get $META_URL --path /d\n dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/d/test1 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n}\n\ntest_dump_load_with_iflag(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --enable-ioctl\n echo \"hello\" > /jfs/hello.txt\n chattr +i /jfs/hello.txt\n ./juicefs dump $META_URL dump.json $(get_dump_option)\n umount_jfs /jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n ./juicefs load $META_URL dump.json $(get_load_option)\n ./juicefs mount -d $META_URL /jfs --enable-ioctl\n echo \"hello\" > /jfs/hello.txt && echo \"write should fail\" && exit 1 || true\n chattr -i /jfs/hello.txt\n echo \"world\" > /jfs/hello.txt\n cat /jfs/hello.txt | grep world\n}\n\ntest_dump_load_with_keep_secret_key()\n{\n option=$@\n prepare_test\n ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin\n ./juicefs dump --keep-secret-key $META_URL dump.json $(get_dump_option)\n python3 .github/scripts/flush_meta.py $META_URL\n ./juicefs load $META_URL dump.json $(get_load_option)\n ./juicefs mount -d $META_URL /jfs\n echo \"hello\" > /jfs/hello.txt\n cat /jfs/hello.txt | grep hello\n\n umount_jfs /jfs $META_URL\n ./juicefs dump $META_URL dump.json $(get_dump_option)\n python3 .github/scripts/flush_meta.py $META_URL\n ./juicefs load $META_URL dump.json $(get_load_option)\n ./juicefs mount -d $META_URL /jfs && echo \"mount should fail\" && exit 1 || true\n ./juicefs config --secret-key minioadmin $META_URL\n ./juicefs mount -d $META_URL /jfs\n echo \"hello\" > /jfs/hello.txt\n cat /jfs/hello.txt | grep hello\n}\n\ntest_load_encrypted_meta_backup()\n{\n prepare_test\n [[ ! -f my-priv-key.pem ]] && openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048\n export JFS_RSA_PASSPHRASE=12345678\n ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem\n ./juicefs mount -d $META_URL /jfs\n SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=50 STEP_COUNT=50 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true\n umount /jfs\n SKIP_BACKUP_META_CHECK=true ./juicefs mount -d --backup-meta 10s $META_URL /jfs\n sleep 10s\n backup_file=$(ls -l /var/jfs/myjfs/meta/ |tail -1 | awk '{print $NF}')\n backup_path=/var/jfs/myjfs/meta/$backup_file\n ls -l $backup_path\n\n ./juicefs load sqlite3://test2.db $backup_path --encrypt-rsa-key my-priv-key.pem --encrypt-algo aes256gcm-rsa\n ./juicefs mount -d sqlite3://test2.db /jfs2\n diff -ur /jfs/test /jfs2/test --no-dereference\n umount_jfs /jfs2 sqlite3://test2.db\n rm test2.db -rf\n}\n\ntest_dump_load_with_random_test()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --enable-acl\n ./juicefs mount -d $META_URL /jfs \n ./random-test runOp -baseDir /jfs/test -files 500000 -ops 5000000 -threads 50 -dirSize 100 -duration 30s -createOp 30,uniform -deleteOp 5,end --linkOp 10,uniform --symlinkOp 20,uniform --setXattrOp 10,uniform --truncateOp 10,uniform \n ./juicefs dump $META_URL dump.json $(get_dump_option)\n create_database $META_URL2\n ./juicefs load $META_URL2 dump.json $(get_load_option)\n ./juicefs dump $META_URL2 dump2.json $(get_dump_option)\n ./juicefs mount -d $META_URL2 /jfs2\n diff -ur /jfs/test /jfs2/test --no-dereference\n diff -ur /jfs/.trash /jfs2/.trash --no-dereference\n # compare_stat_acl_xattr /jfs/test /jfs2/test\n umount_jfs /jfs2 $META_URL2\n ./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --yes $META_URL2 $UUID\n}\n\ntest_dump_load_with_fsrand()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0 --enable-acl\n ./juicefs mount -d $META_URL /jfs --enable-xattr\n rm -rf /tmp/test\n SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true \n ./juicefs dump $META_URL dump.json $(get_dump_option)\n create_database $META_URL2\n ./juicefs load $META_URL2 dump.json $(get_load_option)\n ./juicefs dump $META_URL2 dump2.json $(get_dump_option)\n # if [[ \"$BINARY\" == \"false\" ]]; then\n # compare_dump_json\n # fi\n ./juicefs mount -d $META_URL2 /jfs2\n diff -ur /jfs/test /jfs2/test --no-dereference\n compare_stat_acl_xattr /jfs/test /jfs2/test\n umount_jfs /jfs2 $META_URL2\n ./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --yes $META_URL2 $UUID\n}\n\ncompare_dump_json(){\n cp dump.json dump.json.bak\n cp dump2.json dump2.json.bak\n sed -i '/usedSpace/d' dump*.json.bak\n sed -i '/usedInodes/d' dump*.json.bak\n sed -i '/nextInodes/d' dump*.json.bak\n sed -i '/nextChunk/d' dump*.json.bak\n sed -i '/nextTrash/d' dump*.json.bak\n sed -i '/nextSession/d' dump*.json.bak\n sed -i 's/\"inode\":[0-9]\\+/\"inode\":0/g' dump*.json.bak\n diff -ur dump.json.bak dump2.json.bak\n}\n\ncompare_stat_acl_xattr(){\n dir1=$1\n dir2=$2\n files1=($(find \"$dir1\" -type f -o -type d -exec stat -c \"%n\" {} + | sort))\n files2=($(find \"$dir2\" -type f -o -type d -exec stat -c \"%n\" {} + | sort))\n [[ ${#files1[@]} -ne ${#files2[@]} ]] && echo \"compare_stat_acl: number of files differs\" && exit 1\n for i in \"${!files1[@]}\"; do\n stat1=$(stat -c \"%F %a %s %h %U %G\" \"${files1[$i]}\")\n stat2=$(stat -c \"%F %a %s %h %U %G\" \"${files2[$i]}\")\n acl1=$(getfacl -p \"${files1[$i]}\" | tail -n +2)\n acl2=$(getfacl -p \"${files2[$i]}\" | tail -n +2)\n xattr1=$(getfattr -d -m . -e hex \"${files1[$i]}\" 2>/dev/null | tail -n +2 | sort)\n xattr2=$(getfattr -d -m . -e hex \"${files2[$i]}\" 2>/dev/null | tail -n +2 | sort)\n [[ \"$stat1\" != \"$stat2\" ]] && echo \"compare_stat_acl: stat for ${files1[$i]} and ${files2[$i]} differs\" && echo $stat1 && echo $stat2 && exit 1\n [[ \"$acl1\" != \"$acl2\" ]] && echo \"compare_stat_acl: ACLs for ${files1[$i]} and ${files2[$i]} differs\" && echo $acl1 && echo $acl2 && exit 1\n [[ \"$xattr1\" != \"$xattr2\" ]] && echo \"compare_stat_acl: xattrs for ${files1[$i]} and ${files2[$i]} differs\" && echo $xattr1 && echo $xattr2 && exit 1\n\n done\n echo \"compare_stat_acl: ACLs and stats are the same\"\n}\n\nget_dump_option(){\n if [[ \"$BINARY\" == \"true\" ]]; then \n option=\"--binary\"\n elif [[ \"$FAST\" == \"true\" ]]; then\n option=\"--fast\"\n else\n option=\"\"\n fi\n echo $option\n}\n\nget_load_option(){\n if [[ \"$BINARY\" == \"true\" ]]; then \n option=\"--binary\"\n else\n option=\"\"\n fi\n echo $option\n}\n\nprepare_test(){\n umount_jfs /jfs $META_URL\n umount_jfs /jfs2 sqlite3://test2.db\n python3 .github/scripts/flush_meta.py $META_URL\n rm test2.db -rf \n rm -rf /var/jfs/myjfs || true\n mc rm --force --recursive myminio/test || true\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/dump_load_bench.sh", "content": "#!/bin/bash -ex\n\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\n[[ -z \"$START_META\" ]] && START_META=true\nsource .github/scripts/start_meta_engine.sh\nMETA_URL=$(get_meta_url $META)\nMETA_URL2=$(get_meta_url2 $META)\nFILE_COUNT_IN_BIGDIR=100000\n\nprepare_test_data(){\n umount_jfs /tmp/jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs || true\n create_database $META_URL\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /tmp/jfs\n threads=10\n ./juicefs mdtest $META_URL /bigdir --depth=1 --dirs=0 --files=$((FILE_COUNT_IN_BIGDIR/threads)) --threads=$threads --write=8192\n ./juicefs mdtest $META_URL /smalldir --depth=3 --dirs=10 --files=10 --threads=10 --write=8192\n}\n\nif [[ \"$START_META\" == \"true\" ]]; then \n start_meta_engine $META\n prepare_test_data\nfi\n\ntest_dump_load(){\n do_dump_load dump.json\n}\n\ntest_dump_load_fast(){\n do_dump_load dump.json.gz --fast\n}\n\ntest_dump_load_in_binary(){\n do_dump_load dump.bin --binary\n}\n\ndo_dump_load(){\n dump_file=$1\n shift\n options=$@\n ./juicefs dump $META_URL $dump_file $options --threads=50\n # python3 .github/scripts/flush_meta.py $META_URL2\n create_database $META_URL2\n if [[ \"$options\" == *\"--binary\"* ]]; then\n ./juicefs load $META_URL2 $dump_file $options\n else\n ./juicefs load $META_URL2 $dump_file\n fi\n \n ./juicefs mount $META_URL2 /tmp/jfs2 -d\n df -i /tmp/jfs /tmp/jfs2\n iused1=$(df -i /tmp/jfs | tail -1 | awk '{print $3}')\n iused2=$(df -i /tmp/jfs2 | tail -1 | awk '{print $3}')\n [[ \"$iused1\" == \"$iused2\" ]] || (echo \": iused error: $iused1 $iused2\" && exit 1)\n ./juicefs summary /tmp/jfs/ --csv\n ./juicefs summary /tmp/jfs2/ --csv\n summary1=$(./juicefs summary /tmp/jfs/ --csv | head -n +2 | tail -n 1)\n summary2=$(./juicefs summary /tmp/jfs2/ --csv | head -n +2 | tail -n 1)\n [[ \"$summary1\" == \"$summary2\" ]] || (echo \": summary error: $summary1 $summary2\" && exit 1)\n \n file_count=$(ls -l /tmp/jfs2/bigdir/test-dir.0-0/mdtest_tree.0/ | wc -l)\n file_count=$((file_count-1))\n if [[ \"$file_count\" -ne \"$FILE_COUNT_IN_BIGDIR\" ]]; then \n echo \": file_count error: $file_count\"\n exit 1\n fi\n\n ./juicefs rmr /tmp/jfs2/smalldir\n ls /tmp/jfs2/smalldir && echo \": ls should fail\" && exit 1 || true\n umount_jfs /tmp/jfs2 $META_URL2\n ./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --yes $META_URL2 $UUID\n}\n\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n " }, { "path": ".github/scripts/command/dump_load_cross_meta.sh", "content": "#!/bin/bash -ex\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META1\" ]] && META1=sqlite3\n[[ -z \"$META2\" ]] && META2=redis\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META1\nstart_meta_engine $META2\nMETA_URL1=$(get_meta_url $META1)\nMETA_URL2=$(get_meta_url $META2)\n[[ -z \"$SEED\" ]] && SEED=$(date +%s)\n\n# [[ -z \"$SEED\" ]] && SEED=1711594639\n[[ -z \"$BINARY\" ]] && BINARY=false\n[[ -z \"$FAST\" ]] && FAST=false\n\ntrap \"echo random seed is $SEED\" EXIT\n\nif ! docker ps | grep -q minio; then\n docker run -d -p 9000:9000 --name minio \\\n -e \"MINIO_ACCESS_KEY=minioadmin\" \\\n -e \"MINIO_SECRET_KEY=minioadmin\" \\\n -v /tmp/data:/data \\\n -v /tmp/config:/root/.minio \\\n minio/minio server /data\nfi\n[[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc\nsleep 3s\nmc alias set myminio http://localhost:9000 minioadmin minioadmin\n[[ ! -x random-test ]] && wget -q https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/random-test/random-test -O random-test && chmod +x random-test\npython3 -c \"import xattr\" || sudo pip install xattr\n\ntest_dump_load_with_rmr()\n{\n # ref: https://github.com/juicedata/juicefs/pull/6188\n prepare_test\n ./juicefs format $META_URL1 myjfs --trash-days 0 --enable-acl\n ./juicefs mount -d $META_URL1 /jfs --enable-xattr\n dd if=/dev/urandom of=/jfs/file1 bs=1M count=1024\n ./juicefs dump $META_URL1 dump1.json\n ./juicefs dump $META_URL1 dump1 $(get_dump_option)\n create_database $META_URL2\n ./juicefs load $META_URL2 dump1 $(get_load_option)\n ./juicefs dump $META_URL2 dump2.json\n compare_dump_json dump1.json dump2.json\n ./juicefs mount -d $META_URL2 /jfs2 --no-bgjob\n ./juicefs rmr --skip-trash /jfs2/file1\n JFS_GC_SKIPPEDTIME=1 ./juicefs gc $META_URL2 2>&1| tee gc.log\n count=$(sed -n 's/.*\\([0-9]\\+\\) leaked.*/\\1/p' gc.log)\n [[ \"$count\" -ne 0 ]] && echo \"Expected 0 leaked file, but got $count\" && exit 1 || true\n}\n\nskip_test_dump_load_with_fsrand()\n{\n # unskip the test after fix: https://github.com/juicedata/juicefs/issues/6230\n prepare_test\n ./juicefs format $META_URL1 myjfs --trash-days 0 --enable-acl\n ./juicefs mount -d $META_URL1 /jfs --enable-xattr\n rm -rf /tmp/test\n SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true \n for i in {1..60}; do \n JFS_GC_SKIPPEDTIME=1 ./juicefs gc -v $META_URL1 2>&1| tee gc.log\n count=$(sed -n 's/.*\\([0-9]\\+\\) leaked.*/\\1/p' gc.log)\n if [[ \"$count\" -eq 0 ]]; then \n echo \"Expected 0 leaked file after rmr /jfs2/test, got $count\"\n break\n else\n echo \"Expected 0 leaked file after rmr /jfs2/test, got $count, retrying...\"\n sleep 1s\n fi\n [[ $i -eq 60 ]] && echo \"Expected 0 leaked file after rmr /jfs2/test, but got $count\" && exit 1 || true\n done\n ./juicefs dump $META_URL1 dump1.json\n ./juicefs dump $META_URL1 dump1 $(get_dump_option)\n create_database $META_URL2\n ./juicefs load $META_URL2 dump1 $(get_load_option)\n ./juicefs dump $META_URL2 dump2.json $(get_dump_option)\n # compare_dump_json\n ./juicefs mount -d $META_URL2 /jfs2 --no-bgjob\n diff -ur /jfs/test /jfs2/test --no-dereference\n compare_stat_acl_xattr /jfs/test /jfs2/test\n ./juicefs rmr --skip-trash /jfs2/test\n for i in {1..60}; do \n JFS_GC_SKIPPEDTIME=1 ./juicefs gc -v $META_URL2 2>&1| tee gc.log\n count=$(sed -n 's/.*\\([0-9]\\+\\) leaked.*/\\1/p' gc.log)\n if [[ \"$count\" -eq 0 ]]; then \n echo \"Expected 0 leaked file after rmr /jfs2/test, got $count\"\n break\n else\n echo \"Expected 0 leaked file after rmr /jfs2/test, got $count, retrying...\"\n sleep 1s\n fi\n [[ $i -eq 60 ]] && echo \"Expected 0 leaked file after rmr /jfs2/test, but got $count\" && exit 1 || true\n done\n}\n\nskip_test_dump_load_with_random_test()\n{\n # unskip the test after fix: https://github.com/juicedata/juicefs/issues/6230\n prepare_test\n ./juicefs format $META_URL1 myjfs --trash-days 0 --enable-acl\n ./juicefs mount -d $META_URL1 /jfs --enable-xattr\n ./random-test runOp --baseDir /jfs/test --logDir random-test-log --withData --writeSize 1,10240 \\\n --duration 30s --files 10000000 --ops 100000000 --threads 200 --dirSize 100 \\\n --mkdirOp 10,uniform -createOp 10,uniform -readOp 1,uniform -lsOp 1,uniform -deleteOp 0.01,uniform -rmrOp 0.01,end -renameOp 1,uniform -linkOp 3,uniform \n ./juicefs clone /jfs/test /jfs/test_clone\n ./juicefs dump $META_URL1 dump1.json\n ./juicefs dump $META_URL1 dump1 $(get_dump_option)\n create_database $META_URL2\n ./juicefs load $META_URL2 dump1 $(get_load_option)\n ./juicefs dump $META_URL2 dump2.json $(get_dump_option)\n ./juicefs mount -d $META_URL2 /jfs2 --no-bgjob\n diff -ur /jfs/test /jfs2/test --no-dereference\n diff -ur /jfs/test_clone /jfs2/test_clone --no-dereference\n ./juicefs clone /jfs2/test /jfs2/test_clone2\n for dir in /jfs2/test_clone /jfs2/test /jfs2/test_clone2; do\n ./juicefs rmr --skip-trash $dir\n JFS_GC_SKIPPEDTIME=1 ./juicefs gc -v $META_URL2 2>&1| tee gc.log\n count=$(sed -n 's/.*\\([0-9]\\+\\) leaked.*/\\1/p' gc.log)\n [[ \"$count\" -ne 0 ]] && echo \"Expected 0 leaked file after rmr $dir, but got $count\" && exit 1 || true\n done\n}\n\ncompare_dump_json(){\n cat dump1.json\n cat dump2.json\n cp dump1.json dump1.json.bak\n cp dump2.json dump2.json.bak\n sed -i '/usedSpace/d' dump*.json.bak\n sed -i '/usedInodes/d' dump*.json.bak\n sed -i '/nextInodes/d' dump*.json.bak\n sed -i '/nextChunk/d' dump*.json.bak\n sed -i '/nextTrash/d' dump*.json.bak\n sed -i '/nextSession/d' dump*.json.bak\n sed -i 's/\"inode\":[0-9]\\+/\"inode\":0/g' dump*.json.bak\n diff -ur dump1.json.bak dump2.json.bak\n echo \"compare_dump_json: dump json files are the same\"\n}\n\ncompare_stat_acl_xattr(){\n dir1=$1\n dir2=$2\n files1=($(find \"$dir1\" -type f -o -type d -exec stat -c \"%n\" {} + | sort))\n files2=($(find \"$dir2\" -type f -o -type d -exec stat -c \"%n\" {} + | sort))\n [[ ${#files1[@]} -ne ${#files2[@]} ]] && echo \"compare_stat_acl: number of files differs\" && exit 1\n for i in \"${!files1[@]}\"; do\n stat1=$(stat -c \"%F %a %s %h %U %G\" \"${files1[$i]}\")\n stat2=$(stat -c \"%F %a %s %h %U %G\" \"${files2[$i]}\")\n acl1=$(getfacl -p \"${files1[$i]}\" | tail -n +2)\n acl2=$(getfacl -p \"${files2[$i]}\" | tail -n +2)\n xattr1=$(getfattr -d -m . -e hex \"${files1[$i]}\" 2>/dev/null | tail -n +2 | sort)\n xattr2=$(getfattr -d -m . -e hex \"${files2[$i]}\" 2>/dev/null | tail -n +2 | sort)\n [[ \"$stat1\" != \"$stat2\" ]] && echo \"compare_stat_acl: stat for ${files1[$i]} and ${files2[$i]} differs\" && echo $stat1 && echo $stat2 && exit 1\n [[ \"$acl1\" != \"$acl2\" ]] && echo \"compare_stat_acl: ACLs for ${files1[$i]} and ${files2[$i]} differs\" && echo $acl1 && echo $acl2 && exit 1\n [[ \"$xattr1\" != \"$xattr2\" ]] && echo \"compare_stat_acl: xattrs for ${files1[$i]} and ${files2[$i]} differs\" && echo $xattr1 && echo $xattr2 && exit 1\n\n done\n echo \"compare_stat_acl: ACLs and stats are the same\"\n}\n\nget_dump_option(){\n if [[ \"$BINARY\" == \"true\" ]]; then \n option=\"--binary\"\n elif [[ \"$FAST\" == \"true\" ]]; then\n option=\"--fast\"\n else\n option=\"\"\n fi\n echo $option\n}\n\nget_load_option(){\n if [[ \"$BINARY\" == \"true\" ]]; then \n option=\"--binary\"\n else\n option=\"\"\n fi\n echo $option\n}\n\nprepare_test(){\n umount_jfs /jfs $META_URL1\n umount_jfs /jfs2 $META_URL2\n python3 .github/scripts/flush_meta.py $META_URL1\n python3 .github/scripts/flush_meta.py $META_URL2\n rm -rf /var/jfs/myjfs || true\n mc rm --force --recursive myminio/test || true\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/format.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\nSMB_CONTAINER_NAME=\"juicefs-ci-smb\"\nSMB_USER=\"juicefs\"\nSMB_PASSWORD=\"juicefs\"\nSMB_SHARE=\"share\"\n\ncleanup_smb_container()\n{\n docker rm -f \"$SMB_CONTAINER_NAME\" >/dev/null 2>&1 || true\n rm -rf /tmp/${SMB_CONTAINER_NAME}-data >/dev/null 2>&1 || true\n}\n\nstart_smb_container()\n{\n cleanup_smb_container\n mkdir -p /tmp/${SMB_CONTAINER_NAME}-data\n chmod 0777 /tmp/${SMB_CONTAINER_NAME}-data\n if [[ \"$(uname)\" == \"Darwin\" ]]; then\n docker run -d --name \"$SMB_CONTAINER_NAME\" -p 1445:445 \\\n -v /tmp/${SMB_CONTAINER_NAME}-data:/mount \\\n dperson/samba \\\n -u \"$SMB_USER;$SMB_PASSWORD\" \\\n -s \"$SMB_SHARE;/mount;yes;no;no;$SMB_USER\" >/dev/null\n wait_tcp_ready 127.0.0.1 1445 40\n SMB_ENDPOINT=\"127.0.0.1:1445/${SMB_SHARE}\"\n export SMB_ENDPOINT\n return\n fi\n\n docker run -d --name \"$SMB_CONTAINER_NAME\" \\\n -v /tmp/${SMB_CONTAINER_NAME}-data:/mount \\\n dperson/samba \\\n -u \"$SMB_USER;$SMB_PASSWORD\" \\\n -s \"$SMB_SHARE;/mount;yes;no;no;$SMB_USER\" >/dev/null\n\n local container_ip\n container_ip=$(docker container inspect \"$SMB_CONTAINER_NAME\" --format '{{ .NetworkSettings.IPAddress }}')\n wait_tcp_ready \"$container_ip\" 445 40\n SMB_ENDPOINT=\"${container_ip}/${SMB_SHARE}\"\n export SMB_ENDPOINT\n}\n\nassert_objbench_result()\n{\n local log_file=$1\n local test_name=$2\n local expected=$3\n if ! grep -E \"${test_name}.*${expected}\" \"$log_file\" >/dev/null; then\n echo \"objbench assertion failed: test=${test_name}, expected=${expected}\"\n echo \"--- objbench log ---\"\n cat \"$log_file\"\n exit 1\n fi\n}\n\nkill_gateway_by_port()\n{\n local port=$1\n lsof -t -i :$port | xargs -r kill -9 >/dev/null 2>&1 || true\n}\n\nwait_tcp_ready()\n{\n local host=$1\n local port=$2\n local timeout=${3:-30}\n for _ in $(seq 1 \"$timeout\"); do\n if (echo > /dev/tcp/${host}/${port}) >/dev/null 2>&1; then\n return\n fi\n sleep 1\n done\n echo \"tcp ${host}:${port} is not ready in ${timeout} seconds\"\n exit 1\n}\n\nensure_mc_binary()\n{\n if [[ -x ./mc ]]; then\n return\n fi\n local os_arch\n local cpu_arch\n cpu_arch=$(uname -m)\n if [[ \"$(uname)\" == \"Darwin\" ]]; then\n if [[ \"$cpu_arch\" == \"arm64\" ]]; then\n os_arch=\"darwin-arm64\"\n else\n os_arch=\"darwin-amd64\"\n fi\n else\n if [[ \"$cpu_arch\" == \"aarch64\" || \"$cpu_arch\" == \"arm64\" ]]; then\n os_arch=\"linux-arm64\"\n else\n os_arch=\"linux-amd64\"\n fi\n fi\n wget -q \"https://dl.min.io/client/mc/release/${os_arch}/mc\" -O ./mc\n chmod +x ./mc\n}\n\ngenerate_sha_manifest()\n{\n local root_dir=$1\n local output_file=$2\n rm -f \"$output_file\"\n if [[ \"$(uname)\" == \"Darwin\" ]]; then\n while IFS= read -r rel; do\n sum=$(shasum -a 256 \"$root_dir/$rel\" | awk '{print $1}')\n echo \"$sum $rel\" >> \"$output_file\"\n done < <(cd \"$root_dir\" && find . -type f | sort | sed 's#^\\./##')\n else\n while IFS= read -r rel; do\n sum=$(sha256sum \"$root_dir/$rel\" | awk '{print $1}')\n echo \"$sum $rel\" >> \"$output_file\"\n done < <(cd \"$root_dir\" && find . -type f | sort | sed 's#^\\./##')\n fi\n}\n\nprepare_sync_source_tree()\n{\n local src_dir=$1\n mkdir -p \"$src_dir/dir1/dir2\"\n echo \"hello-juicefs\" > \"$src_dir/plain.txt\"\n echo \"with space\" > \"$src_dir/dir1/file with space.txt\"\n echo \"cifs-中文文件\" > \"$src_dir/dir1/中文文件.txt\"\n : > \"$src_dir/empty.file\"\n dd if=/dev/urandom of=\"$src_dir/dir1/dir2/binary.bin\" bs=1M count=4 >/dev/null 2>&1\n}\n\nskip_test_mount_process_exit_on_format()\n{\n prepare_test\n echo \"round $i\"\n ./juicefs format $META_URL volume-$i\n ./juicefs mount -d $META_URL /tmp/myjfs$i_$j --no-usage-report\n cd /tmp/myjfs$i_$j\n bash -c 'for k in {1..300}; do echo abc>$k; sleep 0.2; done' || true & \n cd -\n sleep 3\n uuid=$(./juicefs status $META_URL | grep UUID | cut -d '\"' -f 4) \n ./juicefs destroy --force $META_URL $uuid\n ./juicefs format $META_URL new-volume-$i \n sleep 15 \n ps -ef | grep juicefs\n # TODO: fix the bug and remove the following line\n # SEE https://github.com/juicedata/juicefs/issues/4534\n pidof juicefs && exit 1\n uuid=$(./juicefs status $META_URL | grep UUID | cut -d '\"' -f 4) \n ./juicefs destroy --force $META_URL $uuid\n}\n\ntest_format_sftp_object()\n{\n docker run -d --name sftp -p 2222:22 juicedata/ci-sftp\n prepare_test\n CONTAINER_IP=$(docker container inspect sftp --format '{{ .NetworkSettings.IPAddress }}')\n echo \"round $i\"\n ./juicefs format $META_URL volume-$i --storage sftp \\\n --bucket $CONTAINER_IP:myjfs/ \\\n --access-key testUser1 \\\n --secret-key password\n ./juicefs mount -d $META_URL /tmp/jfs --no-usage-report --cache-size 0\n cd /tmp/jfs\n bash -c 'for k in {1..100}; do echo abc>$k; sleep 0.1; done' || true &\n bg_pid=$!\n cd -\n sleep 1\n docker stop sftp\n sleep 10\n docker start sftp\n sleep 2\n wait $bg_pid\n echo \"Checking JuiceFS read/write\"\n echo abc > /tmp/jfs/101\n for k in {1..100}; do\n if [[ $(cat /tmp/jfs/$k) != \"abc\" ]]; then\n echo \"ERROR: File $k corrupted after SFTP restart!\"\n exit 1\n fi\n done\n uuid=$(./juicefs status $META_URL | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --force $META_URL $uuid\n ./juicefs format $META_URL new-volume-$i\n}\n\ntest_format_cifs_objbench_matrix()\n{\n prepare_test\n start_smb_container\n local log_raw=/tmp/objbench-cifs-raw.log\n local log_plain=/tmp/objbench-cifs.log\n ./juicefs objbench --storage cifs \\\n --access-key \"$SMB_USER\" \\\n --secret-key \"$SMB_PASSWORD\" \\\n --threads 2 \\\n --small-objects 5 \\\n --small-object-size 4K \\\n --block-size 1M \\\n --big-object-size 8M \\\n \"$SMB_ENDPOINT\" 2>&1 | tee \"$log_raw\"\n\n sed -E 's/\\x1B\\[[0-9;]*[mK]//g' \"$log_raw\" > \"$log_plain\"\n\n assert_objbench_result \"$log_plain\" \"create a bucket\" \"pass\"\n assert_objbench_result \"$log_plain\" \"put an object\" \"pass\"\n assert_objbench_result \"$log_plain\" \"get an object\" \"pass\"\n assert_objbench_result \"$log_plain\" \"get non-exist\" \"pass\"\n assert_objbench_result \"$log_plain\" \"get partial object\" \"pass\"\n assert_objbench_result \"$log_plain\" \"head an object\" \"pass\"\n assert_objbench_result \"$log_plain\" \"delete an object\" \"pass\"\n assert_objbench_result \"$log_plain\" \"delete non-exist\" \"pass\"\n assert_objbench_result \"$log_plain\" \"list objects\" \"pass\"\n assert_objbench_result \"$log_plain\" \"special key\" \"put encode file failed\"\n assert_objbench_result \"$log_plain\" \"put a big object\" \"pass\"\n assert_objbench_result \"$log_plain\" \"put an empty object\" \"pass\"\n assert_objbench_result \"$log_plain\" \"multipart upload\" \"not support\"\n assert_objbench_result \"$log_plain\" \"change owner/group\" \"failed to chown object\"\n assert_objbench_result \"$log_plain\" \"change permission\" \"expect mode 777 but got\"\n assert_objbench_result \"$log_plain\" \"change mtime\" \"pass\"\n\n cleanup_smb_container\n}\n\ntest_format_smb_object_alias()\n{\n prepare_test\n start_smb_container\n local volume_name=\"smb-alias-$RANDOM\"\n local mount_point=\"/tmp/jfs-smb-$RANDOM\"\n ./juicefs format $META_URL \"$volume_name\" --storage smb \\\n --bucket \"$SMB_ENDPOINT\" \\\n --access-key \"$SMB_USER\" \\\n --secret-key \"$SMB_PASSWORD\"\n\n mkdir -p \"$mount_point\"\n ./juicefs mount -d $META_URL \"$mount_point\" --no-usage-report --cache-size 0\n\n echo \"smb-alias-ok\" > \"$mount_point/smb-alias.txt\"\n read_content=$(cat \"$mount_point/smb-alias.txt\")\n [[ \"$read_content\" != \"smb-alias-ok\" ]] && echo \"smb alias read/write check failed\" && exit 1\n\n ./juicefs umount \"$mount_point\" || true\n rm -rf \"$mount_point\"\n\n uuid=$(./juicefs status $META_URL | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --force $META_URL $uuid\n cleanup_smb_container\n}\n\ntest_format_cifs_sync_consistency()\n{\n prepare_test\n start_smb_container\n local volume_name=\"cifs-sync-$RANDOM\"\n local mount_point=\"/tmp/jfs-cifs-sync-$RANDOM\"\n local mount_data_dir\n local src_dir=\"/tmp/cifs-sync-src-$RANDOM\"\n local dst_dir=\"/tmp/cifs-sync-dst-$RANDOM\"\n local src_manifest=\"/tmp/cifs-sync-src-$RANDOM.sha256\"\n local dst_manifest=\"/tmp/cifs-sync-dst-$RANDOM.sha256\"\n\n ./juicefs format $META_URL \"$volume_name\" --storage cifs \\\n --bucket \"$SMB_ENDPOINT\" \\\n --access-key \"$SMB_USER\" \\\n --secret-key \"$SMB_PASSWORD\"\n\n mkdir -p \"$mount_point\"\n ./juicefs mount -d $META_URL \"$mount_point\" --no-usage-report --cache-size 0\n mount_data_dir=\"$mount_point/sync-data\"\n mkdir -p \"$mount_data_dir\"\n\n rm -rf \"$src_dir\" \"$dst_dir\"\n mkdir -p \"$src_dir\" \"$dst_dir\"\n prepare_sync_source_tree \"$src_dir\"\n\n ./juicefs sync \"$src_dir/\" \"$mount_data_dir/\" --threads 8 --dirs\n ./juicefs sync \"$mount_data_dir/\" \"$dst_dir/\" --threads 8 --dirs\n\n generate_sha_manifest \"$src_dir\" \"$src_manifest\"\n generate_sha_manifest \"$dst_dir\" \"$dst_manifest\"\n diff \"$src_manifest\" \"$dst_manifest\"\n\n src_count=$(find \"$src_dir\" -type f | wc -l | tr -d ' ')\n dst_count=$(find \"$dst_dir\" -type f | wc -l | tr -d ' ')\n [[ \"$src_count\" != \"$dst_count\" ]] && echo \"sync file count mismatch: $src_count vs $dst_count\" && exit 1\n\n ./juicefs umount \"$mount_point\" || true\n rm -rf \"$mount_point\" \"$src_dir\" \"$dst_dir\"\n\n uuid=$(./juicefs status $META_URL | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --force $META_URL $uuid\n cleanup_smb_container\n}\n\ntest_format_cifs_object_recovery()\n{\n prepare_test\n start_smb_container\n local volume_name=\"cifs-recovery-$RANDOM\"\n local mount_point=\"/tmp/jfs-cifs-recovery-$RANDOM\"\n\n ./juicefs format $META_URL \"$volume_name\" --storage cifs \\\n --bucket \"$SMB_ENDPOINT\" \\\n --access-key \"$SMB_USER\" \\\n --secret-key \"$SMB_PASSWORD\"\n\n mkdir -p \"$mount_point\"\n ./juicefs mount -d $META_URL \"$mount_point\" --no-usage-report --cache-size 0\n\n for k in {1..20}; do\n echo \"before-restart-$k\" > \"$mount_point/before-$k.txt\"\n done\n\n docker stop \"$SMB_CONTAINER_NAME\"\n sleep 8\n docker start \"$SMB_CONTAINER_NAME\"\n container_ip=$(docker container inspect \"$SMB_CONTAINER_NAME\" --format '{{ .NetworkSettings.IPAddress }}')\n wait_tcp_ready \"$container_ip\" 445 40\n sleep 3\n\n for k in {1..20}; do\n content=$(cat \"$mount_point/before-$k.txt\")\n [[ \"$content\" != \"before-restart-$k\" ]] && echo \"file check failed after restart: before-$k.txt\" && exit 1\n done\n echo \"after-restart\" > \"$mount_point/after-restart.txt\"\n [[ \"$(cat \"$mount_point/after-restart.txt\")\" != \"after-restart\" ]] && echo \"write/read failed after cifs restart\" && exit 1\n\n ./juicefs umount \"$mount_point\" || true\n rm -rf \"$mount_point\"\n\n uuid=$(./juicefs status $META_URL | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --force $META_URL $uuid\n cleanup_smb_container\n}\n\ntest_format_cifs_gateway_read_write()\n{\n prepare_test\n start_smb_container\n ensure_mc_binary\n local volume_name=\"cifs-gateway-$RANDOM\"\n local gateway_port=9015\n\n ./juicefs format $META_URL \"$volume_name\" --storage cifs \\\n --bucket \"$SMB_ENDPOINT\" \\\n --access-key \"$SMB_USER\" \\\n --secret-key \"$SMB_PASSWORD\"\n\n kill_gateway_by_port $gateway_port\n export MINIO_ROOT_USER=admin\n export MINIO_ROOT_PASSWORD=admin123\n ./juicefs gateway $META_URL 127.0.0.1:${gateway_port} --multi-buckets --keep-etag --object-tag -background\n wait_tcp_ready 127.0.0.1 $gateway_port 30\n\n ./mc alias set cifsgw http://127.0.0.1:${gateway_port} admin admin123 --api S3v4\n ./mc mb cifsgw/test-cifs-gw\n echo \"gateway-cifs-ok\" > /tmp/cifs-gateway-file.txt\n ./mc cp /tmp/cifs-gateway-file.txt cifsgw/test-cifs-gw/cifs-gateway-file.txt\n ./mc cat cifsgw/test-cifs-gw/cifs-gateway-file.txt | grep \"gateway-cifs-ok\"\n\n ./mc rm cifsgw/test-cifs-gw/cifs-gateway-file.txt\n ./mc rb cifsgw/test-cifs-gw --force\n kill_gateway_by_port $gateway_port\n\n uuid=$(./juicefs status $META_URL | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --force $META_URL $uuid\n cleanup_smb_container\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command/fsck.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\ntest_fix_nlink(){\n if [[ \"$META\" == \"sqlite3\" ]]; then\n do_fix_nlink_sqlite3\n fi\n}\ndo_fix_nlink_sqlite3(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs/a\n mkdir /jfs/a/b\n touch /jfs/a/c\n sleep 4s # to wait dir stat update\n ./juicefs fsck $META_URL --path / -r\n sqlite3 test.db \"update jfs_node set nlink=100 where inode=2\"\n sqlite3 test.db \"select nlink from jfs_node where inode=2\"\n ./juicefs fsck $META_URL --path / -r && exit 1 || true\n ./juicefs fsck $META_URL --path / -r --repair\n ./juicefs fsck $META_URL --path / -r\n}\n\ntest_sync_dir_stat()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n ./juicefs mdtest $META_URL /d --depth 15 --dirs 2 --files 100 --threads 10 & \n pid=$!\n sleep 15s\n kill -9 $pid\n ./juicefs info -r /jfs/d\n ./juicefs info -r /jfs/d --strict \n ./juicefs fsck $META_URL --path /d --sync-dir-stat --repair -r\n ./juicefs info -r /jfs/d | tee info1.log\n ./juicefs info -r /jfs/d --strict | tee info2.log\n diff info1.log info2.log\n rm info*.log\n ./juicefs fsck $META_URL --path / --sync-dir-stat --repair -r\n ./juicefs info -r /jfs | tee info1.log\n ./juicefs info -r /jfs --strict | tee info2.log\n diff info1.log info2.log\n}\n\ntest_fsck_with_random_test()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n ./random-test runOp -baseDir /jfs/test -files 500000 -ops 5000000 -threads 50 -dirSize 100 -duration 30s -createOp 30,uniform -deleteOp 5,end --linkOp 10,uniform --symlinkOp 20,uniform --setXattrOp 10,uniform --truncateOp 10,uniform \n ./juicefs fsck $META_URL --path /test --sync-dir-stat --repair -r\n ./juicefs info -r /jfs | tee info1.log\n ./juicefs info -r /jfs --strict | tee info2.log\n diff info1.log info2.log || true\n}\n\ntest_fsck_delete_object()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n echo \"test\" > /jfs/test.txt\n sleep 1\n object=$(./juicefs info /jfs/test.txt | grep chunks | awk '{print $4}')\n rm /var/jfs/$object\n ./juicefs fsck $META_URL 2>&1 | tee fsck.log\n grep -q \"1 objects are lost\" fsck.log || exit 1\n rm fsck.log\n # ./juicefs fsck $META_URL --path / --sync-dir-stat --repair -r 2>&1 | tee fsck.log\n # grep -q \"1 objects are lost\" fsck.log || exit 1\n # rm fsck.log\n ./juicefs rmr /jfs/test.txt --skip-trash\n ./juicefs fsck $META_URL || { echo \"files is deleted, fsck should success\"; exit 1; }\n}\n\ntest_sync_dir_df()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n ./juicefs mdtest $META_URL /d --depth 15 --dirs 2 --files 100 --threads 10 & \n pid=$!\n sleep 60s\n kill -9 $pid\n ./juicefs info -r /jfs/d --strict\n #df -h /jfs的Used和\n df -h /jfs\n ./juicefs fsck $META_URL --path /d --sync-dir-stat --repair -r\n ./juicefs info -r /jfs/d | tee info1.log\n ./juicefs info -r /jfs/d --strict | tee info2.log\n diff info1.log info2.log\n rm info*.log\n ./juicefs fsck $META_URL --path / --sync-dir-stat --repair -r\n ./juicefs info -r /jfs | tee info1.log\n ./juicefs info -r /jfs --strict | tee info2.log\n diff info1.log info2.log\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/gateway-random.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\n[[ -z \"$SUBDIR\" ]] && SUBDIR=false\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n[[ ! -x /usr/local/bin/mc ]] && wget -q https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc\n# docker ps -aq --filter \"status=exited\" --filter \"name=minio_old\" | xargs -r docker rm -v\nif ! docker ps --filter \"name=minio_old$\" | grep minio_old; then\n echo start minio_old\n docker run -d -p 9000:9000 --name minio_old -e \"MINIO_ACCESS_KEY=minioadmin\" -e \"MINIO_SECRET_KEY=minioadmin\" minio/minio:RELEASE.2021-04-22T15-44-28Z server /tmp/minio_old\n while ! curl -s http://localhost:9000/minio/health/live > /dev/null; do\n echo \"Waiting for MinIO to be ready...\"\n sleep 1\n done\n echo \"MinIO is ready.\"\nfi\n\ntimeout 30 bash -c 'counter=0; until lsof -i:9000; do echo -ne \"wait port ready in $counter\\r\" && ((counter++)) && sleep 1; done'\n\n[[ -n $CI ]] && trap 'kill_gateway 9005;' EXIT\nkill_gateway() {\n port=$1\n lsof -i:$port || true\n lsof -t -i :$port | xargs -r kill -9 || true\n}\n\nprepare_test()\n{\n umount_jfs /tmp/jfs $META_URL\n kill_gateway 9005\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs || true\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount -d $META_URL /tmp/jfs\n if [ \"$SUBDIR\" = true ]; then\n echo \"start gateway with subdir\"\n mkdir /tmp/jfs/subdir\n MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ./juicefs gateway \\\n $META_URL localhost:9005 --multi-buckets --keep-etag -d --subdir /subdir\n else\n MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ./juicefs gateway \\\n $META_URL localhost:9005 --multi-buckets --keep-etag -d\n fi\n}\n\ntest_run_example()\n{\n prepare_test\n python3 .github/scripts/hypo/s3_test.py\n}\n\ntest_run_all()\n{\n prepare_test\n python3 .github/scripts/hypo/s3.py\n}\n\n\n\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command/gateway.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=redis\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\nwget https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O mc\nchmod +x mc\nexport MINIO_ROOT_USER=admin\nexport MINIO_ROOT_PASSWORD=admin123\nexport MINIO_REFRESH_IAM_INTERVAL=3s\n\nprepare_test()\n{\n umount_jfs /tmp/jfs $META_URL\n kill_gateway 9001\n kill_gateway 9002\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs || true\n rm -rf /var/jfsCache/myjfs || true\n}\n\nkill_gateway() {\n port=$1\n lsof -i:$port || true\n lsof -t -i :$port | xargs -r kill -9 || true\n}\n\ntrap 'kill_gateway 9001; kill_gateway 9002; kill_gateway 9003' EXIT\n\nstart_two_gateway()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount -d $META_URL /tmp/jfs\n export MINIO_ROOT_USER=admin\n export MINIO_ROOT_PASSWORD=admin123\n ./juicefs gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag -background\n sleep 1\n ./juicefs gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag -background \n sleep 2\n ./mc alias set gateway1 http://127.0.0.1:9001 admin admin123\n ./mc alias set gateway2 http://127.0.0.1:9002 admin admin123\n}\n\ntest_user_management()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n sleep 5\n user=$(./mc admin user list gateway2 | grep user1) || true\n if [ -z \"$user\" ]\n then\n echo \"user synchronization error\"\n exit 1\n fi\n ./mc mb gateway1/test1\n ./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n if ./mc cp mc gateway1_user1/test1/file1\n then\n echo \"By default, the user has no read and write permission\"\n exit 1\n fi\n ./mc admin policy set gateway1 readwrite user=user1\n if ./mc cp mc gateway1_user1/test1/file1\n then \n echo \"readwrite policy can read and write objects\" \n else\n echo \"set readwrite policy fail\"\n exit 1\n fi\n ./mc cp gateway2/test1/file1 .\n compare_md5sum file1 mc \n ./mc admin user disable gateway1 user1\n ./mc admin user remove gateway2 user1\n sleep 5\n user=$(./mc admin user list gateway1 | grep user1) || true\n if [ ! -z \"$user\" ]\n then\n echo \"remove user user1 fail\"\n echo $user\n exit 1\n fi\n}\n\ntest_group_management()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n ./mc admin user add gateway1 user2 admin123\n ./mc admin user add gateway1 user3 admin123\n ./mc admin group add gateway1 testcents user1 user2 user3\n result=$(./mc admin group info gateway1 testcents | grep Members |awk '{print $2}') || true\n if [ \"$result\" != \"user1,user2,user3\" ]\n then\n echo \"error,result is '$result'\"\n exit 1\n fi\n ./mc admin policy set gateway1 readwrite group=testcents\n sleep 5\n ./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n ./mc mb gateway1/test1\n if ./mc cp mc gateway1_user1/test1/file1\n then\n echo \"readwrite policy can read write\"\n else\n echo \"the readwrite group has no read and write permission\"\n exit 1\n fi\n ./mc admin policy set gateway1 readonly group=testcents\n sleep 5\n if ./mc cp mc gateway1_user1/test1/file1\n then\n echo \"readonly group policy can not write\"\n exit 1\n else\n echo \"the readonly group has no write permission\"\n fi\n\n ./mc admin group remove gateway1 testcents user1 user2 user3 \n ./mc admin group remove gateway1 testcents\n}\n\ntest_mult_gateways_set_group()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n ./mc admin user add gateway1 user2 admin123\n ./mc admin user add gateway1 user3 admin123\n ./mc admin group add gateway1 testcents user1 user2 user3\n ./mc admin group disable gateway2 testcents\n sleep 5\n result=$(./mc admin group info gateway2 testcents | grep Members |awk '{print $2}') || true\n if [ \"$result\" != \"user1,user2,user3\" ]\n then\n echo \"error,result is '$result'\"\n exit 1\n fi\n ./mc admin group enable gateway1 testcents\n ./mc admin user add gateway1 user4 admin123\n ./mc admin group add gateway1 testcents user4\n sleep 1\n ./mc admin group disable gateway2 testcents\n sleep 5\n result=$(./mc admin group info gateway2 testcents | grep Members |awk '{print $2}') || true\n if [ \"$result\" != \"user1,user2,user3,user4\" ]\n then\n echo \"error,result is '$result'\"\n exit 1\n fi\n}\n\ntest_user_svcacct_add()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n ./mc admin policy set gateway1 consoleAdmin user=user1\n ./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n ./mc admin user svcacct add gateway1_user1 user1 --access-key 12345678 --secret-key 12345678\n ./mc admin user svcacct info gateway1_user1 12345678\n ./mc admin user svcacct set gateway1_user1 12345678 --secret-key 123456789\n ./mc alias set svcacct1 http://127.0.0.1:9001 12345678 123456789\n ./mc mb svcacct1/test1\n if ./mc cp mc svcacct1/test1/file1\n then\n echo \"svcacct user consoleAdmin policy can read write\"\n else\n echo \"the svcacct user has no read and write permission\"\n exit 1\n fi\n ./mc admin user svcacct disable gateway1_user1 12345678\n ./mc admin user svcacct rm gateway1_user1 12345678\n}\n\ntest_user_admin_svcacct_add()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n ./mc admin policy set gateway1 readwrite user=user1\n ./mc admin user svcacct add gateway1 user1 --access-key 12345678 --secret-key 12345678\n ./mc admin user svcacct info gateway1 12345678\n ./mc admin user svcacct set gateway1 12345678 --secret-key 12345678910\n ./mc alias set svcacct1 http://127.0.0.1:9001 12345678 12345678910\n ./mc mb svcacct1/test1\n if ./mc cp mc svcacct1/test1/file1\n then\n echo \"amdin user can do svcacct \"\n else\n echo \"the svcacct user has no read and write permission\"\n exit 1\n fi\n ./mc admin user svcacct disable gateway1 12345678\n ./mc admin user svcacct rm gateway1 12345678\n}\n\ntest_user_sts()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n ./mc admin policy set gateway1 consoleAdmin user=user1\n ./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n git clone https://github.com/juicedata/minio.git -b gateway-1.1\n ./mc mb gateway1_user1/test1\n ./mc cp mc gateway1_user1/test1/mc\n cd minio\n go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1 -d\n go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1\n cd -\n ./mc admin user remove gateway1 user1 \n}\n\n\nskip_test_change_credentials()\n{\n prepare_test\n start_two_gateway\n ./mc mb gateway1/test1\n ./mc cp mc gateway1/test1/file1\n lsof -i :9001 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true\n lsof -i :9002 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true\n export MINIO_ROOT_USER=newadmin\n export MINIO_ROOT_PASSWORD=newadmin123\n export MINIO_ROOT_USER_OLD=admin\n export MINIO_ROOT_PASSWORD_OLD=admin123\n ./juicefs gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag -background\n ./juicefs gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag -background\n sleep 5\n ./mc alias set gateway1 http://127.0.0.1:9001 newadmin newadmin123\n ./mc alias set gateway2 http://127.0.0.1:9002 newadmin newadmin123\n ./mc cp gateway1/test1/file1 file1\n ./mc cp gateway2/test1/file1 file2\n compare_md5sum file1 mc\n compare_md5sum file2 mc \n}\n\n\ntest_ro_gateway()\n{ \n prepare_test\n start_two_gateway\n ./juicefs gateway $META_URL 127.0.0.1:9003 --read-only --multi-buckets --keep-etag --object-tag -background \n ./mc alias set gateway3 http://127.0.0.1:9003 admin admin123 \n ./mc mb gateway1/test1\n ./mc cp mc gateway1/test1/file1\n ./mc admin user add gateway1 user1 admin123\n sleep 4\n user=$(./mc admin user list gateway3 | grep user1) || true\n [[ -z \"$user\" ]] && echo \"user synchronization error\" && exit 1 || true\n ./mc mb gateway3/test3 && echo \"By default, the ro has no write permission for creating buckets\" && exit 1 || true\n ./mc cp mc gateway3/test1/file1 && echo \"By default, the ro has no write permission for copying files\" && exit 1 || true\n ./mc cp gateway3/test1/file1 .\n diff mc file1\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command/gc.sh", "content": "#!/bin/bash -e\n\npython3 -c \"import xattr\" || pip install xattr \ndpkg -s redis-tools || .github/scripts/apt_install.sh redis-tools\ndpkg -s fio || .github/scripts/apt_install.sh fio\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\ntest_delay_delete_slice_after_compaction(){\n if [[ \"$META\" != redis* ]]; then\n echo \"this test only runs for redis meta engine\"\n return\n fi\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 1\n ./juicefs mount -d $META_URL /jfs --no-usage-report\n fio --name=abc --rw=randwrite --refill_buffers --size=500M --bs=256k --directory=/jfs\n redis-cli save\n # don't skip files when gc compact\n export JFS_SKIPPED_TIME=1\n ./juicefs gc --compact --delete $META_URL\n killall -9 redis-server\n sleep 3\n ./juicefs fsck $META_URL\n}\n\ntest_gc_trash_slices(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n PATH1=/tmp/test PATH2=/jfs/test python3 .github/scripts/random_read_write.py \n ./juicefs status --more $META_URL\n ./juicefs config $META_URL --trash-days 0 --yes\n ./juicefs gc $META_URL \n ./juicefs gc $META_URL --delete\n ./juicefs status --more $META_URL\n}\n\ntest_gc_trash_files(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n python3 .github/scripts/fsrand.py -c 1000 /jfs/fsrand\n rm -rf /jfs/fsrand\n ./juicefs status --more $META_URL\n ./juicefs config $META_URL --trash-days 0 --yes\n ./juicefs gc $META_URL \n ./juicefs gc $META_URL --delete\n ./juicefs status --more $META_URL\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/graceful_upgrade.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\nLEGACY_META_URL=$META_URL\nif [[ \"$META\" == \"redis\" ]]; then\n LEGACY_META_URL=${META_URL%%\\?*}\nfi\necho meta_url is $META_URL\n\ndpkg -s fio >/dev/null 2>&1 || .github/scripts/apt_install.sh fio\ndpkg -s attr >/dev/null 2>&1 || .github/scripts/apt_install.sh attr\n\nif [[ ! -x \"./juicefs-1.1\" ]]; then \n wget -q https://github.com/juicedata/juicefs/releases/download/v1.1.0/juicefs-1.1.0-linux-amd64.tar.gz\n rm /tmp/juicefs -rf && mkdir -p /tmp/juicefs\n tar -xzvf juicefs-1.1.0-linux-amd64.tar.gz -C /tmp/juicefs\n mv /tmp/juicefs/juicefs juicefs-1.1 && chmod +x juicefs-1.1 \n rm /tmp/juicefs -rf && rm juicefs-1.1.0-linux-amd64.tar.gz\n ./juicefs-1.1 version | grep \"version 1.1\"\nfi\n[[ ! -f my-priv-key.pem ]] && openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048\n\n\ntest_kill_mount_process()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount $META_URL /tmp/jfs -d\n wait_process_started 1\n force_kill_child_process\n sleep 3\n wait_process_started 2\n kill_parent_process\n wait_command_success \"ps -ef | grep \"mount\" | grep \"/tmp/jfs\" | grep -v grep | wc -l\" 0\n ./juicefs mount $META_URL /tmp/jfs -d\n kill_child_process\n wait_command_success \"ps -ef | grep \"mount\" | grep \"/tmp/jfs\" | grep -v grep | wc -l\" 0\n ./juicefs mount $META_URL /tmp/jfs -d\n ./juicefs umount /tmp/jfs\n wait_command_success \"ps -ef | grep \"mount\" | grep \"/tmp/jfs\" | grep -v grep | wc -l\" 0\n}\n\nskip_test_update_with_flock(){\n prepare_test \n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /tmp/jfs\n ps -ef | grep mount\n cat /tmp/jfs/.config | grep -i sid\n echo abc | tee /tmp/jfs/test\n sleep 1s\n flock -x /tmp/jfs/test -c cat & \n sleep 1s\n flock -s /tmp/jfs/test -c \"echo abc\" > flock.log 2>&1 &\n sleep 1s\n exit 1\n ./juicefs mount -d $META_URL /tmp/jfs\n ps -ef | grep mount\n cat /tmp/jfs/.config | grep -i sid\n cat flock.log\n count=$(ps -ef | grep flock | grep -v grep | wc -l)\n [[ $count -ne 2 ]] && echo \"flock process should be 2, count=$count\" && exit 1 || true \n}\n\ntest_update_non_fuse_option(){\n prepare_test\n JFS_RSA_PASSPHRASE=12345678 ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem\n JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs\n echo abc | tee /tmp/jfs/test\n JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs --read-only\n echo abc | tee /tmp/jfs/test && (echo \"should not write read-only file system\" && exit 1) || true\n JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs \n echo abc | tee /tmp/jfs/test\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 2 ]] && echo \"mount process count should be 2, count=$count\" && exit 1 || true\n umount /tmp/jfs\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 0 ]] && echo \"mount process count should be 0, count=$count\" && exit 1 || true\n}\n\ntest_update_on_failure(){\n prepare_test\n JFS_RSA_PASSPHRASE=12345678 ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem\n JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL /tmp/jfs\n echo abc | tee /tmp/jfs/test\n JFS_RSA_PASSPHRASE=abc123xx ./juicefs mount -d $META_URL /tmp/jfs || true\n echo abc | tee /tmp/jfs/test\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 2 ]] && echo \"mount process count should be 2, count=$count\" && exit 1 || true\n umount /tmp/jfs\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 0 ]] && echo \"mount process count should be 0, count=$count\" && exit 1 || true\n}\n#TODO: fio test failed on database locked.\ntest_update_on_fio(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /tmp/jfs --buffer-size 300\n fio -name=fio -filename=/tmp/jfs/testfile -direct=1 -iodepth 16 -ioengine=libaio \\\n -rw=randwrite -bs=4k -size=100M -numjobs=4 -runtime=30 -group_reporting >fio.log 2>&1 &\n fio_pid=$!\n trap \"kill -9 $fio_pid > /dev/null || true\" EXIT\n for i in {1..5}; do\n echo \"update buffer-size to $((i+300))\"\n ./juicefs mount -d $META_URL /tmp/jfs --buffer-size $((i+300))\n wait_command_success \"ps -ef | grep juicefs | grep mount | grep \\\"buffer-size $((i+300))\\\" | wc -l\" 2\n echo abc | tee /tmp/jfs/test\n done\n kill -9 $fio_pid > /dev/null 2>&1 || true\n # umount_jfs /tmp/jfs $META_URL\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 2 ]] && echo \"mount process count should be 2, count=$count\" && exit 1 || true\n}\n\ntest_update_fuse_option(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /tmp/jfs --enable-xattr\n setfattr -n user.test -v \"juicedata\" /tmp/jfs\n getfattr -n user.test /tmp/jfs | grep juicedata\n ./juicefs mount -d $META_URL /tmp/jfs\n getfattr -n user.test /tmp/jfs && exit 1 || true\n ./juicefs mount -d $META_URL /tmp/jfs --enable-xattr\n getfattr -n user.test /tmp/jfs | grep juicedata\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 4 ]] && echo \"mount process count should be 4, count=$count\" && exit 1 || true\n umount /tmp/jfs\n getfattr -n user.test /tmp/jfs && exit 1 || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 2 ]] && echo \"mount process count should be 2, count=$count\" && exit 1 || true\n umount /tmp/jfs\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 0 ]] && echo \"mount process count should be 0, count=$count\" && exit 1 || true\n}\n\ntest_update_from_old_version(){\n prepare_test\n ./juicefs-1.1 format $LEGACY_META_URL myjfs\n ./juicefs-1.1 mount -d $LEGACY_META_URL /tmp/jfs\n echo hello |tee /tmp/jfs/test\n ./juicefs mount -d $META_URL /tmp/jfs\n count=$(ps -ef | grep juicefs | grep mount | wc -l)\n [[ $count -ne 3 ]] && echo \"mount process count should be 3\" && exit 1 || true\n version=$(./juicefs version | awk '{print $3,$4,$5}')\n grep Version /tmp/jfs/.config | grep $version\n grep \"hello\" /tmp/jfs/test\n echo world | tee /tmp/jfs/test \n ./juicefs umount /tmp/jfs\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 1 ]] && echo \"mount process count should be 1\" && exit 1 || true\n ./juicefs umount /tmp/jfs\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l)\n [[ $count -ne 0 ]] && echo \"mount process count should be 0\" && exit 1 || true\n}\n\ntest_update_on_fstab(){\n prepare_test\n ./juicefs format $META_URL myjfs\n umount_jfs /tmp/jfs $META_URL\n rm /sbin/mount.juicefs -rf \n ./juicefs mount --update-fstab $META_URL /tmp/jfs -d \\\n -o debug,allow_other,writeback_cache \\\n --max-uploads 20 --prefetch 3 --upload-limit 3 \\\n --download-limit 100 --get-timeout 60 --put-timeout 60\n grep /tmp/jfs /etc/fstab\n ls /sbin/mount.juicefs -l\n umount /tmp/jfs\n for i in {1..5}; do\n mount /tmp/jfs\n wait_command_success \"ps -ef | grep juicefs | grep /tmp/jfs | grep -v grep | wc -l\" 2\n # cat /tmp/jfs/.config\n done\n}\n\nprepare_test(){\n umount_jfs /tmp/jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs || true\n}\n\nkill_child_process()\n{\n echo \"kill_child_process\"\n child_pid=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | awk '$3 != 1 {print $2}')\n kill $child_pid\n}\n\nforce_kill_child_process()\n{\n echo \"force_kill_child_process\"\n child_pid=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | awk '$3 != 1 {print $2}')\n kill -9 $child_pid\n}\n\n\nkill_parent_process()\n{\n echo \"kill_parent_process\"\n parent_pid=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | awk '$3 == 1 {print $2}')\n kill $parent_pid\n}\n\nwait_process_started()\n{ \n echo \"wait_process_to_start $1\"\n wait_seconds=15\n for i in $(seq 1 $wait_seconds); do\n if check_process_is_alive ; then\n echo \"mount process is started\"\n break\n fi\n if [ $i -eq $wait_seconds ]; then\n ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep \n echo \"mount process is not started after $wait_seconds\"\n exit 1\n fi\n echo \"wait process to start\" && sleep 1\n done\n}\n\ncheck_process_is_alive()\n{ \n echo >&2 \"check_process_is_alive $1\"\n count=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | wc -l)\n if [ $count -ne 2 ]; then\n ps -ef | grep \"juicefs\" | grep -v \"grep\"\n echo >&2 \"mount process is not equal 2\"\n return 1\n fi\n child_count=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | awk '$3 != 1 {print $2}' | wc -l)\n if [[ $child_count -ne 1 ]]; then\n ps -ef | grep \"juicefs\" | grep -v \"grep\"\n echo >&2 \"mount child process is not equal 1\"\n return 1\n fi\n parent_count=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | awk '$3 == 1 {print $2}' | wc -l)\n if [ $parent_count -ne 1 ]; then\n ps -ef | grep \"juicefs\" | grep -v \"grep\"\n echo >&2 \"mount parent process is not equal 1\"\n return 1\n fi\n ppid1=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | awk '$3 == 1 {print $2}')\n ppid2=$(ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v grep | awk '$3 != 1 {print $3}')\n if [ $ppid1 -ne $ppid2 ]; then\n ps -ef | grep \"juicefs\" | grep \"mount\" | grep -v \"grep\"\n echo >&2 \"mount parent process is not equal child process's ppid\"\n return 1\n fi\n}\n\n\nsource .github/scripts/common/run_test.sh && run_test $@" }, { "path": ".github/scripts/command/info.sh", "content": "#!/bin/bash -e\n\nsudo dpkg -s redis-tools || sudo .github/scripts/apt_install.sh redis-tools\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\ntest_info_big_file(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n dd if=/dev/zero of=/jfs/bigfile bs=1M count=4096\n ./juicefs info /jfs/bigfile\n ./juicefs rmr /jfs/bigfile\n df -h /jfs\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/interface.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\ntest_list_large_dir()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n local files_count=100000\n if [[ \"$META_URL\" == redis://* ]]; then\n files_count=1300000\n fi\n ./juicefs mdtest $META_URL /test --depth 0 --dirs 1 --files $files_count --threads 1\n du /jfs/test & du_pid=$!\n sleep 2\n kill -INT $du_pid || true\n wait $du_pid || true\n if ! [ -d \"/jfs/test\" ]; then\n echo >&2 \": directory /jfs/test is not accessible after ls interruption\"\n exit 1\n fi\n}\n\ntest_deep_nested_dirs() {\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n dir=\"/juicefs1/test\"\n for i in $(seq 1 100); do\n dir=\"$dir/dir$i\"\n mkdir -p \"$dir\"\n echo \"content$i\" > \"$dir/file$i\"\n done\n max_jobs=10\n for i in $(seq 1 50); do\n nested_dir=\"/juicefs1/test\"\n for j in $(seq 1 $i); do\n nested_dir=\"$nested_dir/dir$j\"\n done\n ls \"$nested_dir\" > /dev/null 2>&1 &\n if (( $(jobs -p | wc -l) >= max_jobs )); then\n wait -n\n fi\n done\n wait\n file_count=$(find /juicefs1/test -type f | wc -l)\n if [[ $file_count -ne 100 ]]; then\n echo \"File number error： $file_count\"\n return 1\n fi\n for i in $(seq 1 100); do\n nested_dir=\"/juicefs1/test\"\n for j in $(seq 1 $i); do\n nested_dir=\"$nested_dir/dir$j\"\n done\n expected_content=\"content$i\"\n actual_content=$(cat \"$nested_dir/file$i\" 2>/dev/null)\n if [[ \"$actual_content\" != \"$expected_content\" ]]; then\n echo \"expect: '$expected_content'，actual: '$actual_content'\"\n return 1\n fi\n done\n return 0\n}\n\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command/mount.sh", "content": "#!/bin/bash -e\n\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\ntest_sort_dir(){\n prepare_test\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs --sort-dir\n \n for i in {1..1000}; do\n touch \"/jfs/file_$i\"\n done\n mkdir -p /jfs/subdir\n for i in {1..1000}; do\n touch \"/jfs/subdir/file_$i\"\n done \n ls -lh /jfs > /tmp/sorted_no_u\n ls -U -lh /jfs > /tmp/sorted_with_u\n diff /tmp/sorted_no_u /tmp/sorted_with_u\n \n ls -lh /jfs/subdir > /tmp/subdir_sorted_no_u\n ls -U -lh /jfs/subdir > /tmp/subdir_sorted_with_u\n diff /tmp/subdir_sorted_no_u /tmp/subdir_sorted_with_u \n rm -f /tmp/sorted_*\n rm -f /tmp/subdir_sorted_*\n}\n\nmeasure_lookup_time() {\n local start_time end_time elapsed\n start_time=$(date +%s.%N)\n for file in \"${FILE_LIST[@]}\"; do\n if [[ -e \"$file\" ]]; then\n echo \"Error: $file exists!\" >&2\n exit 1\n fi\n done\n end_time=$(date +%s.%N)\n elapsed=$(echo \"$end_time - $start_time\" | bc)\n echo \"$elapsed\"\n}\n\ntest_negative_dir(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --negative-entry-cache 5\n TEST_DIR=\"/jfs/test_dir_$$\"\n mkdir -p \"${TEST_DIR}\"\n\n FILE_LIST=()\n for i in {1..1000}; do\n FILE_LIST+=(\"${TEST_DIR}/nonexistent_file_$(printf \"%04d\" $i)\")\n done\n echo -e \"\\n=== First lookup (uncached) ===\"\n time1=$(measure_lookup_time)\n echo \"Time taken: ${time1} seconds\"\n echo -e \"\\n=== Second lookup (cached) ===\"\n time2=$(measure_lookup_time)\n echo \"Time taken: ${time2} seconds\"\n echo -e \"\\n=== Waiting for cache to expire... ===\"\n sleep 6 \n echo -e \"\\n=== Third lookup (after cache expiry) ===\"\n time3=$(measure_lookup_time)\n echo \"Time taken: ${time3} seconds\"\n echo -e \"\\n=== Test Result ===\"\n if (( $(echo \"$time1 > 2 * $time2\" | bc -l) )) && \\\n (( $(echo \"$time3 > 2 * $time2\" | bc -l) )) && \\\n (( $(echo \"$time1 - $time3 < 0.5\" | bc -l) )); then\n echo \"PASS: Caching behavior matches expectations:\"\n else\n echo \"FAIL: Caching behavior does NOT match expectations:\"\n echo \"Expected: First ≈ Third > 2 x Second\"\n exit 1\n fi\n rm -rf \"${TEST_DIR}\"\n echo -e \"\\nTest directory removed: ${TEST_DIR}\"\n}\n\ntest_redis_client_cache()\n{\n if [[ \"$META\" != \"redis\" ]]; then\n echo \"Skip redis client cache test for META=$META\"\n return 0\n fi\n\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs2 || true\n ./juicefs mount -d $META_URL /jfs2\n\n mkdir -p /jfs/redis_csc\n for i in {1..100}; do\n echo \"v$i\" > \"/jfs/redis_csc/file_$i\"\n done\n\n wait_command_success \"ls /jfs2/redis_csc | wc -l\" \"100\" 30\n echo \"cache-sync\" > /jfs/redis_csc/shared_file\n wait_command_success \"cat /jfs2/redis_csc/shared_file\" \"cache-sync\" 30\n\n ./juicefs umount /jfs2 || umount -l /jfs2 || true\n}\n\ntest_check_storage(){\n start_meta_engine $META minio\n prepare_test\n sleep 2\n ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test \\\n --access-key minioadmin --secret-key minioadmin --compress lz4 --hash-prefix\n docker stop minio\n ./juicefs mount $META_URL /tmp/jfs --check-storage || echo \"PASS: Mount failed as expected when storage is not accessible\"\n docker start minio\n sleep 2\n ./juicefs mount $META_URL /tmp/jfs -d\n ./juicefs umount /tmp/jfs\n docker stop minio && docker rm minio\n}\n\ntest_capabilities()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --enable-xattr --enable-cap\n cp /bin/ls /jfs/test_ls\n cp /bin/ping /jfs/test_ping\n chmod +x /jfs/test_ls /jfs/test_ping\n setcap \"cap_net_raw+ep\" /jfs/test_ping\n setcap \"cap_dac_override+ep\" /jfs/test_ls\n sleep 1\n getcap /jfs/test_ping | grep -E \"cap_net_raw[+=]ep\" || {\n echo \"FAIL: capability not set correctly on test_ping\"\n exit 1\n }\n getcap /jfs/test_ls | grep -E \"cap_dac_override[+=]ep\" || {\n echo \"FAIL: capability not set correctly on test_ls\"\n exit 1\n }\n capsh --print | grep \"Current:\" || {\n echo \"FAIL: cannot get current capabilities\"\n exit 1\n }\n setcap -r /jfs/test_ping\n setcap -r /jfs/test_ls\n getcap /jfs/test_ping | grep -E \"cap_net_raw[+=]ep\" && {\n echo \"FAIL: capability not removed from test_ping\"\n exit 1\n }\n getcap /jfs/test_ls | grep -E \"cap_dac_override[+=]ep\" && {\n echo \"FAIL: capability not removed from test_ls\"\n exit 1\n }\n rm -f /jfs/test_ls /jfs/test_ping\n echo \"PASS: Capabilities test completed successfully\"\n}\n\ntest_all_squash()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --all-squash 1101:1101\n mkdir -p /jfs/test_dir\n touch /jfs/test_dir/test_file\n uid1=$(stat -c %u /jfs/test_dir)\n gid1=$(stat -c %g /jfs/test_dir)\n uid2=$(stat -c %u /jfs/test_dir/test_file)\n gid2=$(stat -c %g /jfs/test_dir/test_file)\n if [[ \"$uid1\" != \"1101\" ]] || [[ \"$gid1\" != \"1101\" ]] || [[ \"$uid2\" != \"1101\" ]] || [[ \"$gid2\" != \"1101\" ]]; then\n echo >&2 \": uid/gid does not same as squash: uid1: $uid1, uid2: $uid2, gid1: $gid1, gid2: $gid2\"\n exit 1\n fi\n}\n\ntest_umask()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --umask 0027\n\n mkdir -p /jfs/test_dir\n dir_perms=$(stat -c %a /jfs/test_dir)\n if [[ \"$dir_perms\" != \"750\" ]]; then\n echo >&2 \": Directory permissions incorrect. Expected: 750, Got: $dir_perms\"\n exit 1\n fi\n touch /jfs/test_file\n file_perms=$(stat -c %a /jfs/test_file)\n if [[ \"$file_perms\" != \"640\" ]]; then\n echo >&2 \": File permissions incorrect. Expected: 640, Got: $file_perms\"\n exit 1\n fi\n touch /jfs/test_dir/nested_file\n nested_perms=$(stat -c %a /jfs/test_dir/nested_file)\n if [[ \"$nested_perms\" != \"640\" ]]; then\n echo >&2 \": Nested file permissions incorrect. Expected: 640, Got: $nested_perms\"\n exit 1\n fi\n echo \"PASS: Umask test completed successfully\"\n}\n\ntest_close_to_open1()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs2 || true\n ./juicefs mount -d $META_URL /jfs2\n file1=\"/jfs/testfile.tmp\"\n file2=\"/jfs2/testfile.tmp\"\n rm $file1 || true\n openssl rand -base64 -out $file1 512000\n sleep 3\n ls -ls $file2\n echo \"#########################\"\n echo \"hello\" > $file1\n hex_file2=$(cat $file2 | hexdump -C)\n echo \"#########################\"\n hex_file2_2=$(cat $file2 | hexdump -C)\n hex_file1=$(cat $file1 | hexdump -C)\n [[ \"$hex_file2\" != \"$hex_file1\" ]] && echo \"Content of $hex_file2 and $hex_file1 do not match\" && exit 1 || true\n [[ \"$hex_file2_2\" != \"$hex_file1\" ]] && echo \"Content of $hex_file2_2 and $hex_file1 do not match\" && exit 1 || true\n}\n\ntest_colse_to_open2()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs2 || true\n ./juicefs mount -d $META_URL /jfs2\n file1=\"/jfs/testfile.tmp\"\n file2=\"/jfs2/testfile.tmp\"\n rm $file1 || true\n python3 -c \"\nfor i in range(1, 101):\n with open('$file1', 'a') as f:\n f.write(f'{i}\\\\n')\n with open('$file2', 'a') as f:\n f.write(f'{i}\\\\n')\n\"\n line_count1=$(cat $file1 | wc -l)\n line_count2=$(cat $file2 | wc -l)\n [[ $line_count1 -ne 200 ]] && cat $file1 && echo \"Error: $file1 should have 200 lines but has $line_count1\" && exit 1 || true\n [[ $line_count2 -ne 200 ]] && cat $file2 && echo \"Error: $file2 should have 200 lines but has $line_count2\" && exit 1 || true\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/quota.sh", "content": "#!/bin/bash -e\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\nHEARTBEAT_INTERVAL=3\nHEARTBEAT_SLEEP=3\nDIR_QUOTA_FLUSH_INTERVAL=4\nVOLUME_QUOTA_FLUSH_INTERVAL=2\nsource .github/scripts/common/common.sh\n\ntest_total_capacity()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --capacity 1\n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL --debug\n dd if=/dev/zero of=/jfs/test1 bs=1G count=1\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/test1 2>error.log && echo \"echo should fail on out of space\" && exit 1 || true\n grep \"No space left on device\" error.log\n ./juicefs config $META_URL --capacity 2\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/jfs/test2 bs=1G count=1\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/test2 2>error.log && echo \"echo should fail on out of space\" && exit 1 || true\n grep \"No space left on device\" error.log\n\n rm /jfs/test1 -rf\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/test3 2>error.log && echo \"echo should fail on out of space\" && exit 1 || true\n\n ./juicefs rmr /jfs/.trash\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/test3 \n\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n ln /jfs/test2 /jfs/test4\n ln /jfs/test2 /jfs/test5\n}\n\ntest_total_inodes(){\n prepare_test\n ./juicefs format $META_URL myjfs --inodes 1000\n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n set +x\n for i in {1..1000}; do\n echo $i | tee /jfs/test$i > /dev/null\n done\n set -x\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee /jfs/test1001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n grep \"No space left on device\" error.log\n ./juicefs config $META_URL --inodes 2000\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n set +x\n for i in {1001..2000}; do\n echo $i | tee /jfs/test$i > /dev/null || (df -i /jfs && ls /jfs/ -l | wc -l && exit 1)\n done\n set -x\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee /jfs/test2001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n}\n\ntest_nested_dir(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n file_count=1000\n mkdir -p /jfs/d1/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}\n dir_count=$(find /jfs/d1 -type d | wc -l)\n echo \"dir_count: $dir_count\"\n ./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count-1))\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n for i in $(seq 1 $file_count); do\n subdir=$(find /jfs/d1/ -type d | shuf -n 1)\n echo \"touch $subdir/test$i\" && touch $subdir/test$i\n done\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n subdir=$(find /jfs/d1/ -type d | shuf -n 1)\n touch $subdir/test 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n\n ./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count))\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n subdir=$(find /jfs/d1/ -type d | shuf -n 1)\n touch $subdir/test\n}\n\ntest_remove_and_restore(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n ./juicefs quota set $META_URL --path /d --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n echo a | tee -a /jfs/d/test1 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n\n echo \"remove test1\" && rm /jfs/d/test1 -rf\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"0%\" ]] && echo \"used should be 0%\" && exit 1 || true\n\n trash_dir=$(ls /jfs/.trash)\n ./juicefs restore $META_URL $trash_dir --put-back\n ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n echo a | tee -a /jfs/d/test1 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n\n echo \"remove test1\" && rm /jfs/d/test1 -rf\n dd if=/dev/zero of=/jfs/d/test2 bs=1M count=1\n trash_dir=$(ls /jfs/.trash)\n ./juicefs restore $META_URL $trash_dir --put-back 2>&1 | tee restore.log\n grep \"disk quota exceeded\" restore.log || (echo \"check restore log failed\" && exit 1)\n}\n\ntest_dir_capacity(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n ./juicefs quota set $META_URL --path /d --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota get $META_URL --path /d\n used=$(./juicefs quota get $META_URL --path /d 2>&1 | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n echo a | tee -a /jfs/d/test1 2>error.log && echo \"echo should fail on out of space\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n\n ./juicefs quota set $META_URL --path /d --capacity 2\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/jfs/d/test2 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/d/test2 2>error.log && echo \"echo should fail on out of space\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm -rf /jfs/d/test1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n used=$(./juicefs quota get $META_URL --path /d 2>&1 | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"50%\" ]] && echo \"used should be 50%\" && exit 1 || true\n dd if=/dev/zero of=/jfs/d/test3 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota check $META_URL --path /d --strict\n}\n\ntest_dir_inodes(){\n prepare_test\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n ./juicefs quota set $META_URL --path /d --inodes 1000\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n set +x\n for i in {1..1000}; do\n echo $i > /jfs/d/test$i > /dev/null\n done\n set -x\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee /jfs/d/test1001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm -rf error.log\n ./juicefs quota set $META_URL --path /d --inodes 2000\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n set +x\n for i in {1001..2000}; do\n echo $i | tee /jfs/d/test$i > /dev/null\n done\n set -x\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee /jfs/d/test2001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm /jfs/d/test1 -rf\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee /jfs/d/test2001\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota check $META_URL --path /d --strict\n}\n\ntest_sub_dir(){\n prepare_test\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n ./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n umount_jfs /jfs $META_URL\n ./juicefs mount -d $META_URL --subdir /d /jfs --heartbeat 2\n size=$(df -h /jfs | grep \"JuiceFS\" | awk '{print $2}')\n [[ $size != \"1.0G\" ]] && echo \"size should be 1.0G\" && exit 1 || true\n inodes=$(df -ih /jfs | grep \"JuiceFS\" | awk '{print $2}')\n [[ $inodes != \"1000\" ]] && echo \"inodes should be 1000\" && exit 1 || true\n dd if=/dev/zero of=/jfs/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/test1 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm /jfs/test1 -rf\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n set +x\n for i in {1..1000}; do\n echo $i | tee /jfs/test$i > /dev/null\n done\n set -x\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo $i | tee /jfs/test1001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n ./juicefs quota check $META_URL --path /d --strict\n}\n\ntest_dump_load(){\n prepare_test\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n ./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n ./juicefs dump --log-level error $META_URL --fast > dump.json\n umount_jfs /jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n ./juicefs load $META_URL dump.json\n ./juicefs mount $META_URL /jfs -d --heartbeat 5\n dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee -a /jfs/d/test1 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm /jfs/d/test1 -rf\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n set +x\n for i in {1..1000}; do\n echo $i | tee /jfs/d/test$i > /dev/null\n done\n set -x\n sleep 3s\n echo a | tee /jfs/d/test1001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n ./juicefs quota check $META_URL --path /d --strict\n}\n\ntest_hard_link(){\n prepare_test\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n dd if=/dev/zero of=/jfs/file bs=1G count=1\n ./juicefs quota set $META_URL --path /d --capacity 2\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ln /jfs/file /jfs/d/test2\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ln /jfs/file /jfs/d/test3 2>error.log && echo \"hard link should fail on out of space\" && exit 1 || true\n grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota check $META_URL --path /d --strict\n}\n\ntest_check_and_repair_quota(){\n prepare_test\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /jfs/d\n ./juicefs quota set $META_URL --path /d --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1\n pid=$(ps -ef | grep \"juicefs mount\" | grep -v grep | awk '{print $2}')\n kill -9 $pid\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n # ./juicefs quota check $META_URL --path /d --strict && echo \"quota check should fail\" && exit 1 || true\n ./juicefs quota check $META_URL --path /d --strict --repair\n ./juicefs quota check $META_URL --path /d --strict\n}\n\nwait_until()\n{ \n key=$1\n value=$2\n echo \"wait until $key becomes $value\"\n wait_seconds=15\n for i in $(seq 1 $wait_seconds); do\n if [ \"$key\" == \"ifree\" ]; then\n expect_value=$(df -ih /jfs | grep JuiceFS | awk '{print $4}')\n elif [ \"$key\" == \"avail_size\" ]; then\n expect_value=$(df h /jfs | grep JuiceFS | awk '{print $4}')\n fi\n if [ \"$expect_value\" == \"$value\" ]; then\n echo \"$key becomes $value\" && return 0\n fi\n echo \"wait until $key becomes $value\" && sleep 1s\n done\n echo \"wait until $key becomes $value failed after $wait_seconds seconds\" && exit 1\n}\n\nprepare_ug_quota_test()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs config $META_URL --user-group-quota\n ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n}\n\nresolve_test_users()\n{\n if [[ -n \"$TEST_USER_1\" ]] && [[ -n \"$TEST_USER_2\" ]]; then\n return 0\n fi\n\n TEST_USER_1=\"\"\n TEST_USER_2=\"\"\n\n for candidate in nobody daemon bin; do\n if id \"$candidate\" >/dev/null 2>&1; then\n candidate_uid=$(id -u \"$candidate\")\n candidate_gid=$(id -g \"$candidate\")\n if [[ \"$candidate_uid\" == \"0\" ]] || [[ \"$candidate_gid\" == \"0\" ]]; then\n continue\n fi\n if [[ -z \"$TEST_USER_1\" ]]; then\n TEST_USER_1=\"$candidate\"\n TEST_UID_1=$candidate_uid\n TEST_GID_1=$candidate_gid\n elif [[ \"$candidate_uid\" != \"$TEST_UID_1\" ]]; then\n TEST_USER_2=\"$candidate\"\n TEST_UID_2=$candidate_uid\n TEST_GID_2=$candidate_gid\n break\n fi\n fi\n done\n create_temp_user()\n {\n idx=$1\n if ! command -v useradd >/dev/null 2>&1; then\n return 1\n fi\n name=\"jfs-quota-test-${idx}-${RANDOM}\"\n if ! useradd -M -s /usr/sbin/nologin \"$name\" >/dev/null 2>&1; then\n return 1\n fi\n uid=$(id -u \"$name\" 2>/dev/null || echo 0)\n gid=$(id -g \"$name\" 2>/dev/null || echo 0)\n if [[ \"$uid\" == \"0\" ]] || [[ \"$gid\" == \"0\" ]]; then\n userdel -f \"$name\" >/dev/null 2>&1 || true\n return 1\n fi\n echo \"$name:$uid:$gid\"\n return 0\n }\n\n if [[ -z \"$TEST_USER_1\" ]] || [[ -z \"$TEST_USER_2\" ]]; then\n if [[ \"$(id -u)\" != \"0\" ]]; then\n echo \"cannot find two non-root users for user/group quota tests\"\n return 1\n fi\n for i in 1 2 3 4; do\n info=$(create_temp_user \"$i\") || continue\n name=$(echo \"$info\" | cut -d: -f1)\n uid=$(echo \"$info\" | cut -d: -f2)\n gid=$(echo \"$info\" | cut -d: -f3)\n if [[ -z \"$TEST_USER_1\" ]]; then\n TEST_USER_1=\"$name\"\n TEST_UID_1=$uid\n TEST_GID_1=$gid\n elif [[ -z \"$TEST_USER_2\" ]] && [[ \"$uid\" != \"$TEST_UID_1\" ]]; then\n TEST_USER_2=\"$name\"\n TEST_UID_2=$uid\n TEST_GID_2=$gid\n break\n fi\n done\n fi\n\n if [[ -z \"$TEST_USER_1\" ]] || [[ -z \"$TEST_USER_2\" ]]; then\n echo \"cannot find two non-root users for user/group quota tests\"\n return 1\n fi\n\n echo \"test users: $TEST_USER_1($TEST_UID_1:$TEST_GID_1), $TEST_USER_2($TEST_UID_2:$TEST_GID_2)\"\n}\n\nrun_as_user_cmd()\n{\n user=$1\n shift\n cmd=\"$*\"\n\n if [[ \"$(id -un)\" == \"$user\" ]]; then\n bash -c \"$cmd\"\n return $?\n fi\n\n if command -v sudo >/dev/null 2>&1; then\n sudo -n -u \"$user\" bash -c \"$cmd\" && return 0 || true\n fi\n\n if command -v runuser >/dev/null 2>&1; then\n runuser -u \"$user\" -- bash -c \"$cmd\" && return 0 || true\n fi\n\n if command -v su >/dev/null 2>&1; then\n su -s /bin/bash \"$user\" -c \"$cmd\" && return 0 || true\n fi\n\n echo \"cannot run command as user $user\"\n return 1\n}\n\nset_quota_by_username()\n{\n username=$1\n capacity=$2\n inodes=$3\n uid=$(id -u \"$username\")\n ./juicefs quota set $META_URL --uid \"$uid\" --capacity \"$capacity\" --inodes \"$inodes\"\n}\n\ntest_user_group_quota_set_get_list_delete(){\n prepare_ug_quota_test\n resolve_test_users || return 0\n\n ./juicefs quota set $META_URL --uid \"$TEST_UID_1\" --capacity 1 --inodes 20\n ./juicefs quota set $META_URL --gid \"$TEST_GID_1\" --capacity 1 --inodes 20\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n ./juicefs quota get $META_URL --uid \"$TEST_UID_1\" 2>&1 | tee uid_quota.log\n grep \"UID:$TEST_UID_1\" uid_quota.log || (echo \"uid quota should exist\" && exit 1)\n\n ./juicefs quota get $META_URL --gid \"$TEST_GID_1\" 2>&1 | tee gid_quota.log\n grep \"GID:$TEST_GID_1\" gid_quota.log || (echo \"gid quota should exist\" && exit 1)\n\n ./juicefs quota list $META_URL 2>&1 | tee quota_list.log\n grep \"UID:$TEST_UID_1\" quota_list.log || (echo \"uid quota should be listed\" && exit 1)\n grep \"GID:$TEST_GID_1\" quota_list.log || (echo \"gid quota should be listed\" && exit 1)\n\n ./juicefs quota delete $META_URL --uid \"$TEST_UID_1\"\n ./juicefs quota delete $META_URL --gid \"$TEST_GID_1\"\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n ./juicefs quota list $META_URL 2>&1 | tee quota_list_after_delete.log\n grep \"UID:$TEST_UID_1\" quota_list_after_delete.log && echo \"uid quota should be deleted\" && exit 1 || true\n grep \"GID:$TEST_GID_1\" quota_list_after_delete.log && echo \"gid quota should be deleted\" && exit 1 || true\n}\n\ntest_uid_quota_check_on_write_and_truncate(){\n prepare_ug_quota_test\n resolve_test_users || return 0\n\n mkdir -p /jfs/uidq\n chmod 777 /jfs/uidq\n\n ./juicefs quota set $META_URL --uid \"$TEST_UID_2\" --capacity 1 --inodes 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n\n run_as_user_cmd \"$TEST_USER_2\" \"touch /jfs/uidq/inode1\"\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n run_as_user_cmd \"$TEST_USER_2\" \"touch /jfs/uidq/inode2\" 2>error.log && echo \"second inode should fail for uid quota\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"uid inode quota check failed\" && exit 1)\n\n ./juicefs quota set $META_URL --uid \"$TEST_UID_2\" --capacity 1 --inodes 10\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n rm -f /jfs/uidq/inode1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n\n run_as_user_cmd \"$TEST_USER_2\" \"truncate -s 900M /jfs/uidq/space1\"\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n run_as_user_cmd \"$TEST_USER_2\" \"truncate -s 1100M /jfs/uidq/space1\" 2>error.log && echo \"truncate should fail for uid capacity quota\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"uid capacity quota check failed\" && exit 1)\n}\n\ntest_gid_quota_check_on_write(){\n prepare_ug_quota_test\n resolve_test_users || return 0\n\n mkdir -p /jfs/gidq\n chmod 777 /jfs/gidq\n\n ./juicefs quota set $META_URL --gid \"$TEST_GID_2\" --inodes 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n\n run_as_user_cmd \"$TEST_USER_2\" \"touch /jfs/gidq/file1\"\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n run_as_user_cmd \"$TEST_USER_2\" \"touch /jfs/gidq/file2\" 2>error.log && echo \"second inode should fail for gid quota\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"gid inode quota check failed\" && exit 1)\n}\n\ntest_chown_transfer_user_group_quota(){\n prepare_ug_quota_test\n resolve_test_users || return 0\n\n mkdir -p /jfs/chownq\n chmod 777 /jfs/chownq\n\n ./juicefs quota set $META_URL --uid \"$TEST_UID_1\" --inodes 1\n ./juicefs quota set $META_URL --uid \"$TEST_UID_2\" --inodes 1\n ./juicefs quota set $META_URL --gid \"$TEST_GID_1\" --inodes 1\n ./juicefs quota set $META_URL --gid \"$TEST_GID_2\" --inodes 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n\n run_as_user_cmd \"$TEST_USER_1\" \"touch /jfs/chownq/src_file\"\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n run_as_user_cmd \"$TEST_USER_1\" \"touch /jfs/chownq/src_file2\" 2>error.log && echo \"user1 should exceed inode quota before chown\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"user1 pre-chown quota check failed\" && exit 1)\n\n chown \"$TEST_UID_2:$TEST_GID_2\" /jfs/chownq/src_file\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n run_as_user_cmd \"$TEST_USER_1\" \"touch /jfs/chownq/src_file2\"\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n run_as_user_cmd \"$TEST_USER_2\" \"touch /jfs/chownq/dst_file\" 2>error.log && echo \"user2 should exceed inode quota after chown transfer\" && exit 1 || true\n grep -i \"Disk quota exceeded\" error.log || (echo \"user2 post-chown quota check failed\" && exit 1)\n}\n\ntest_set_quota_by_username(){\n prepare_ug_quota_test\n resolve_test_users || return 0\n\n set_quota_by_username \"$TEST_USER_2\" 1 10\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n uid=$(id -u \"$TEST_USER_2\")\n ./juicefs quota get $META_URL --uid \"$uid\" 2>&1 | tee username_quota.log\n grep \"UID:$uid\" username_quota.log || (echo \"quota set by username should be visible in uid quota\" && exit 1)\n\n ./juicefs quota list $META_URL 2>&1 | tee username_quota_list.log\n grep \"UID:$uid\" username_quota_list.log || (echo \"quota set by username should be listed\" && exit 1)\n}\n\ntest_quota_list_uid_filter_regression(){\n prepare_ug_quota_test\n resolve_test_users || return 0\n\n ./juicefs quota set $META_URL --uid \"$TEST_UID_1\" --capacity 1 --inodes 3\n ./juicefs quota set $META_URL --uid \"$TEST_UID_2\" --capacity 1 --inodes 7\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n\n ./juicefs quota list $META_URL --uid \"$TEST_UID_1\" 2>&1 | tee uid_filter_1.log\n grep \"UID:$TEST_UID_1\" uid_filter_1.log || (echo \"uid filter should show requested uid quota\" && exit 1)\n grep \"UID:$TEST_UID_2\" uid_filter_1.log && echo \"uid filter should not include other uid quota\" && exit 1 || true\n uid_rows=$(grep -c \"UID:\" uid_filter_1.log || true)\n [[ \"$uid_rows\" -ne 1 ]] && echo \"uid filter should only return one UID row\" && exit 1 || true\n inodes_value=$(grep \"UID:$TEST_UID_1\" uid_filter_1.log | head -n1 | awk -F'|' '{gsub(/[[:space:]]/,\"\",$6); print $6}')\n [[ \"$inodes_value\" != \"3\" ]] && echo \"uid filter should return uid1 inodes=3\" && exit 1 || true\n\n ./juicefs quota list $META_URL --uid \"$TEST_UID_2\" 2>&1 | tee uid_filter_2.log\n grep \"UID:$TEST_UID_2\" uid_filter_2.log || (echo \"uid filter should show requested uid quota\" && exit 1)\n grep \"UID:$TEST_UID_1\" uid_filter_2.log && echo \"uid filter should not include other uid quota\" && exit 1 || true\n uid_rows=$(grep -c \"UID:\" uid_filter_2.log || true)\n [[ \"$uid_rows\" -ne 1 ]] && echo \"uid filter should only return one UID row\" && exit 1 || true\n inodes_value=$(grep \"UID:$TEST_UID_2\" uid_filter_2.log | head -n1 | awk -F'|' '{gsub(/[[:space:]]/,\"\",$6); print $6}')\n [[ \"$inodes_value\" != \"7\" ]] && echo \"uid filter should return uid2 inodes=7\" && exit 1 || true\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command/random.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n[[ -z \"$MAX_EXAMPLE\" ]] && MAX_EXAMPLE=100\n[[ -z \"$STEP_COUNT\" ]] && STEP_COUNT=50\n\n[[ -z \"$META1\" ]] && META1=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META1\nMETA_URL1=$(get_meta_url $META1)\n\n[[ -z \"$META2\" ]] && META2=redis\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META2\nMETA_URL2=$(get_meta_url $META2)\n\nprepare_test()\n{\n meta_url=$1\n mp=$2\n volume=$3\n shift 3\n options=$@\n umount_jfs $mp $meta_url\n python3 .github/scripts/flush_meta.py $meta_url\n rm -rf /var/jfs/$volume || true\n rm -rf /var/jfsCache/$volume || true\n ./juicefs format $meta_url $volume $options\n ./juicefs mount -d $meta_url $mp\n}\n\ntest_run_examples()\n{\n prepare_test $META_URL1 /tmp/jfs1 myjfs1 --enable-acl --trash-days 0\n prepare_test $META_URL2 /tmp/jfs2 myjfs2 --enable-acl --trash-days 0\n python3 .github/scripts/hypo/command_test.py\n}\n\ntest_run_all()\n{\n prepare_test $META_URL1 /tmp/jfs1 myjfs1\n prepare_test $META_URL2 /tmp/jfs2 myjfs2\n CHECK_NLINK=false MAX_EXAMPLE=$MAX_EXAMPLE STEP_COUNT=$STEP_COUNT python3 .github/scripts/hypo/command.py\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command-win/acl.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\ntest_modify_acl_config()\n{\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs --trash-days 0\n ./juicefs.exe mount -d $META_URL z:\n touch z:test\n cmd.exe /c \"icacls z:\\test /grant Everyone:(R,W)\" && echo \"setfacl should failed\" && exit 1\n ./juicefs.exe config $META_URL --enable-acl=true\n ./juicefs.exe umount z:\n ./juicefs.exe mount -d $META_URL z:\n cmd.exe /c \"icacls z:\\test /grant Everyone:(R,W)\"\n ./juicefs.exe config $META_URL --enable-acl=false && echo \"should not disable acl\" && exit 1 || true \n ./juicefs.exe config $META_URL | grep EnableACL | grep \"true\" || (echo \"EnableACL should be true\" && exit 1) \n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command-win/clone.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\ntest_clone_with_jfs_source()\n{\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z:\n ls /z\n [[ ! -d /z/juicefs ]] && git clone https://github.com/juicedata/juicefs.git /z/juicefs --depth 1\n ls /z/juicefs\n do_clone true\n echo \"test clone without --preserve\"\n# do_clone false\n}\n\ndo_clone()\n{\n is_preserve=$1\n cmd.exe /c \"taskkill /F /IM git.exe 2>nul || ver>nul\"\n cmd.exe /c \"rmdir /s /q z:\\juicefs1 2>nul || ver>nul\"\n cmd.exe /c \"rmdir /s /q z:\\juicefs2 2>nul || ver>nul\"\n sleep 1\n \n [[ \"$is_preserve\" == \"true\" ]] && preserve=\"--preserve\" || preserve=\"\"\n cp -r /z/juicefs /z/juicefs1 $preserve\n ./juicefs.exe clone /z/juicefs /z/juicefs2 $preserve\n diff -ur /z/juicefs1 /z/juicefs2 --no-dereference\n # CURRENT_DIR=$(pwd)\n # cmd.exe /c \"dir /s /b /a z:\\juicefs1\" > \"${CURRENT_DIR}/log1\"\n # cmd.exe /c \"dir /s /b /a z:\\juicefs2\" > \"${CURRENT_DIR}/log2\"\n # diff -u \"${CURRENT_DIR}/log1\" \"${CURRENT_DIR}/log2\"\n # rm -f \"${CURRENT_DIR}/log1\" \"${CURRENT_DIR}/log2\"\n}\n\ntest_clone_with_small_files(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z:\n mkdir /z/test\n for i in $(seq 1 2000); do\n echo $i > /z/test/$i\n done\n ./juicefs.exe clone /z/test /z/test1\n diff -ur /z/test1 /z/test1\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command-win/debug.sh", "content": "#!/bin/bash -e\n\nsource .github/scripts/common/common_win.sh\n[[ -z \"$META_URL\" ]] && META=redis://127.0.0.1:6379/1\n\n\ncheck_debug_file(){\n files=(\"system-info.log\" \"juicefs.log\" \"config.txt\" \"stats.txt\" \"stats.5s.txt\")\n debug_dir=\"debug\"\n if [ ! -d \"$debug_dir\" ]; then\n echo \"error:no debug dir\"\n exit 1\n fi\n all_files_exist=true\n for file in \"${files[@]}\"; do\n exist=`find \"$debug_dir\" -name $file | wc -l`\n if [ \"$exist\" == 0 ]; then\n echo \"no $file\"\n all_files_exist=false\n fi\n done\n if [ \"$all_files_exist\" = true ]; then\n echo \"pass\"\n else\n exit 1\n fi\n}\n\ntest_debug_juicefs(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs \n ./juicefs.exe mount -d $META_URL z:\n dd if=/dev/urandom of=/z/bigfile bs=1M count=1024\n ./juicefs.exe debug z:\n check_debug_file\n find debug -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'\n ./juicefs.exe rmr /z/bigfile\n}\n\ntest_debug_abnormal_juicefs(){\n rm -rf debug | true\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs \n ./juicefs.exe mount -d $META_URL z:\n dd if=/dev/urandom of=/z/bigfile bs=1M count=1024\n killall -9 redis-server | true\n ./juicefs.exe debug z:\n check_debug_file\n ./juicefs.exe rmr /z/bigfile\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@" }, { "path": ".github/scripts/command-win/dump_load.sh", "content": "#!/bin/bash -ex\nsource .github/scripts/common/common_win.sh\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n[[ -z \"$SEED\" ]] && SEED=$(date +%s)\nHEARTBEAT_INTERVAL=2\nDIR_QUOTA_FLUSH_INTERVAL=4\n[[ -z \"$BINARY\" ]] && BINARY=false\n[[ -z \"$FAST\" ]] && FAST=false\n\ntrap \"echo random seed is $SEED\" EXIT\n\ntest_dump_load_sustained_file(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs --trash-days 0\n ./juicefs.exe mount -d $META_URL z:\n file_count=100\n for i in $(seq 1 $file_count); do\n touch /z/file$i\n exec {fd}<>/z/file$i\n echo fd is $fd\n fds[$i]=$fd\n rm /z/file$i\n done\n ./juicefs.exe dump $META_URL dump.json $(get_dump_option)\n for i in $(seq 1 $file_count); do\n fd=${fds[$i]}\n exec {fd}>&-\n done\n if [[ \"$BINARY\" == \"true\" ]]; then\n sustained=$(./juicefs.exe load dump.json --binary --stat | grep sustained | awk -F\"|\" '{print $2}')\n else\n sustained=$(jq '.Sustained[].inodes | length' dump.json)\n fi\n echo \"sustained file count: $sustained\"\n ./juicefs.exe umount z:\n prepare_win_test\n ./juicefs.exe load $META_URL dump.json $(get_load_option)\n ./juicefs.exe mount -d $META_URL z:\n}\n\ntest_dump_load_with_copy_file_range(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z:\n rm -rf /tmp/test\n dd if=/dev/zero of=/tmp/test bs=1M count=1024\n cp /tmp/test /z/test\n node .github/scripts/copyFile.js /z/test /z/test1\n ./juicefs.exe dump $META_URL dump.json $(get_dump_option)\n ./juicefs.exe umount z:\n redis-cli -h 127.0.0.1 -p 6379 -n 1 FLUSHDB\n ./juicefs.exe load $META_URL dump.json $(get_load_option)\n ./juicefs.exe mount -d $META_URL z:\n compare_md5sum /tmp/test /z/test1\n}\n\ntest_dump_load_with_quota(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs \n ./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /z/d\n ./juicefs.exe quota set $META_URL --path //d --inodes 1000 --capacity 1\n ./juicefs.exe dump --log-level error $META_URL $(get_dump_option) > dump.json\n ./juicefs.exe umount z:\n redis-cli -h 127.0.0.1 -p 6379 -n 1 FLUSHDB\n ./juicefs.exe load $META_URL dump.json $(get_load_option)\n ./juicefs.exe mount $META_URL z: -d --heartbeat $HEARTBEAT_INTERVAL\n ./juicefs.exe quota get $META_URL --path //d\n dd if=/dev/zero of=/z/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n dd if=/dev/zero of=/z/d/test2 bs=1G count=1 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n}\n\nget_dump_option(){\n if [[ \"$BINARY\" == \"true\" ]]; then \n option=\"--binary\"\n elif [[ \"$FAST\" == \"true\" ]]; then\n option=\"--fast\"\n else\n option=\"\"\n fi\n echo $option\n}\n\nget_load_option(){\n if [[ \"$BINARY\" == \"true\" ]]; then \n option=\"--binary\"\n else\n option=\"\"\n fi\n echo $option\n}\n\nprepare_test(){\n umount_jfs /jfs $META_URL\n umount_jfs /jfs2 sqlite3://test2.db\n python3 .github/scripts/flush_meta.py $META_URL\n rm test2.db -rf \n rm -rf /var/jfs/myjfs || true\n mc rm --force --recursive myminio/test || true\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command-win/fsck.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n\ntest_sync_dir_stat()\n{\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z:\n ./juicefs.exe mdtest $META_URL //d --depth 15 --dirs 2 --files 100 --threads 10 & \n pid=$!\n sleep 15s\n kill -9 $pid\n ./juicefs.exe info -r /z/d\n ./juicefs.exe info -r /z/d --strict \n ./juicefs.exe fsck $META_URL --path //d --sync-dir-stat --repair -r\n ./juicefs.exe info -r /z/d | tee info1.log\n ./juicefs.exe info -r /z/d --strict | tee info2.log\n diff info1.log info2.log\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command-win/gateway.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n\nwget https://dl.min.io/client/mc/release/windows-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O mc.exe\nchmod +x mc.exe\nexport MINIO_ROOT_USER=admin\nexport MINIO_ROOT_PASSWORD=admin123\nexport MINIO_REFRESH_IAM_INTERVAL=3s\n\nprepare_test()\n{\n kill_gateway 9001 || true\n kill_gateway 9002 || true\n prepare_win_test\n}\n\nkill_gateway() {\n port=$1\n for pid in $(netstat -ano | findstr \":$port\" | findstr \"LISTENING\" | awk '{print $5}'); do\n taskkill //F //PID $pid\n done\n}\n\ntrap 'kill_gateway 9001; kill_gateway 9002' EXIT\n\nstart_two_gateway()\n{\n prepare_test\n ./juicefs.exe format $META_URL myjfs --trash-days 0\n ./juicefs.exe mount -d $META_URL z:\n export MINIO_ROOT_USER=admin\n export MINIO_ROOT_PASSWORD=admin123\n nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag --log=gateway1.log &\n sleep 1\n nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag --log=gateway2.log &\n sleep 2\n ./mc.exe alias set gateway1 http://127.0.0.1:9001 admin admin123\n ./mc.exe alias set gateway2 http://127.0.0.1:9002 admin admin123\n}\n\ntest_user_management()\n{\n prepare_test\n start_two_gateway\n ./mc.exe admin user add gateway1 user1 admin123\n sleep 5\n user=$(./mc.exe admin user list gateway2 | grep user1) || true\n if [ -z \"$user\" ]\n then\n echo \"user synchronization error\"\n exit 1\n fi\n ./mc.exe mb gateway1/test1\n ./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n if ./mc.exe cp mc.exe gateway1_user1/test1/file1\n then\n echo \"By default, the user has no read and write permission\"\n exit 1\n fi\n ./mc.exe admin policy set gateway1 readwrite user=user1\n if ./mc.exe cp mc.exe gateway1_user1/test1/file1\n then \n echo \"readwrite policy can read and write objects\" \n else\n echo \"set readwrite policy fail\"\n exit 1\n fi\n ./mc.exe cp gateway2/test1/file1 .\n compare_md5sum file1 mc.exe\n ./mc.exe admin user disable gateway1 user1\n ./mc.exe admin user remove gateway2 user1\n sleep 5\n user=$(./mc.exe admin user list gateway1 | grep user1) || true\n if [ ! -z \"$user\" ]\n then\n echo \"remove user user1 fail\"\n echo $user\n exit 1\n fi\n}\n\ntest_group_management()\n{\n prepare_test\n start_two_gateway\n ./mc.exe admin user add gateway1 user1 admin123\n ./mc.exe admin user add gateway1 user2 admin123\n ./mc.exe admin user add gateway1 user3 admin123\n ./mc.exe admin group add gateway1 testcents user1 user2 user3\n result=$(./mc.exe admin group info gateway1 testcents | grep Members |awk '{print $2}') || true\n if [ \"$result\" != \"user1,user2,user3\" ]\n then\n echo \"error,result is '$result'\"\n exit 1\n fi\n ./mc.exe admin policy set gateway1 readwrite group=testcents\n sleep 5\n ./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n ./mc.exe mb gateway1/test1\n if ./mc.exe cp mc.exe gateway1_user1/test1/file1\n then\n echo \"readwrite policy can read write\"\n else\n echo \"the readwrite group has no read and write permission\"\n exit 1\n fi\n ./mc.exe admin policy set gateway1 readonly group=testcents\n sleep 5\n if ./mc.exe cp mc.exe gateway1_user1/test1/file1\n then\n echo \"readonly group policy can not write\"\n exit 1\n else\n echo \"the readonly group has no write permission\"\n fi\n\n ./mc.exe admin group remove gateway1 testcents user1 user2 user3 \n ./mc.exe admin group remove gateway1 testcents\n}\n\ntest_mult_gateways_set_group()\n{\n prepare_test\n start_two_gateway\n ./mc.exe admin user add gateway1 user1 admin123\n ./mc.exe admin user add gateway1 user2 admin123\n ./mc.exe admin user add gateway1 user3 admin123\n ./mc.exe admin group add gateway1 testcents user1 user2 user3\n ./mc.exe admin group disable gateway2 testcents\n sleep 5\n result=$(./mc.exe admin group info gateway2 testcents | grep Members |awk '{print $2}') || true\n if [ \"$result\" != \"user1,user2,user3\" ]\n then\n echo \"error,result is '$result'\"\n exit 1\n fi\n ./mc.exe admin group enable gateway1 testcents\n ./mc.exe admin user add gateway1 user4 admin123\n ./mc.exe admin group add gateway1 testcents user4\n sleep 1\n ./mc.exe admin group disable gateway2 testcents\n sleep 5\n result=$(./mc.exe admin group info gateway2 testcents | grep Members |awk '{print $2}') || true\n if [ \"$result\" != \"user1,user2,user3,user4\" ]\n then\n echo \"error,result is '$result'\"\n exit 1\n fi\n}\n\ntest_user_svcacct_add()\n{\n prepare_test\n start_two_gateway\n ./mc.exe admin user add gateway1 user1 admin123\n ./mc.exe admin policy set gateway1 consoleAdmin user=user1\n ./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n ./mc.exe admin user svcacct add gateway1_user1 user1 --access-key 12345678 --secret-key 12345678\n ./mc.exe admin user svcacct info gateway1_user1 12345678\n ./mc.exe admin user svcacct set gateway1_user1 12345678 --secret-key 123456789\n ./mc.exe alias set svcacct1 http://127.0.0.1:9001 12345678 123456789\n ./mc.exe mb svcacct1/test1\n if ./mc.exe cp mc.exe svcacct1/test1/file1\n then\n echo \"svcacct user consoleAdmin policy can read write\"\n else\n echo \"the svcacct user has no read and write permission\"\n exit 1\n fi\n ./mc.exe admin user svcacct disable gateway1_user1 12345678\n ./mc.exe admin user svcacct rm gateway1_user1 12345678\n}\n\ntest_user_admin_svcacct_add()\n{\n prepare_test\n start_two_gateway\n ./mc.exe admin user add gateway1 user1 admin123\n ./mc.exe admin policy set gateway1 readwrite user=user1\n ./mc.exe admin user svcacct add gateway1 user1 --access-key 12345678 --secret-key 12345678\n ./mc.exe admin user svcacct info gateway1 12345678\n ./mc.exe admin user svcacct set gateway1 12345678 --secret-key 12345678910\n ./mc.exe alias set svcacct1 http://127.0.0.1:9001 12345678 12345678910\n ./mc.exe mb svcacct1/test1\n if ./mc.exe cp mc.exe svcacct1/test1/file1\n then\n echo \"amdin user can do svcacct \"\n else\n echo \"the svcacct user has no read and write permission\"\n exit 1\n fi\n ./mc.exe admin user svcacct disable gateway1 12345678\n ./mc.exe admin user svcacct rm gateway1 12345678\n}\n\ntest_user_sts()\n{\n prepare_test\n start_two_gateway\n ./mc.exe admin user add gateway1 user1 admin123\n ./mc.exe admin policy set gateway1 consoleAdmin user=user1\n ./mc.exe alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n git clone https://github.com/juicedata/minio.git -b gateway-1.1\n ./mc.exe mb gateway1_user1/test1\n ./mc.exe cp mc.exe gateway1_user1/test1/mc\n cd minio\n go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1 -d\n go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1\n cd -\n ./mc.exe admin user remove gateway1 user1 \n}\n\n\ntest_change_credentials()\n{\n prepare_test\n start_two_gateway\n ./mc.exe mb gateway1/test1\n ./mc.exe cp mc.exe gateway1/test1/file1\n kill_gateway 9001 || true\n kill_gateway 9002 || true\n export MINIO_ROOT_USER=newadmin\n export MINIO_ROOT_PASSWORD=newadmin123\n export MINIO_ROOT_USER_OLD=admin\n export MINIO_ROOT_PASSWORD_OLD=admin123\n nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag --log=gateway1.log &\n nohup ./juicefs.exe gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag --log=gateway2.log &\n sleep 5\n ./mc.exe alias set gateway1 http://127.0.0.1:9001 newadmin newadmin123\n ./mc.exe alias set gateway2 http://127.0.0.1:9002 newadmin newadmin123\n ./mc.exe cp gateway1/test1/file1 file1\n ./mc.exe cp gateway2/test1/file1 file2\n compare_md5sum file1 mc.exe\n compare_md5sum file2 mc.exe \n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n\n" }, { "path": ".github/scripts/command-win/gc.sh", "content": "#!/bin/bash -e\n\nsource .github/scripts/common/common_win.sh\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n\ntest_delay_delete_slice_after_compaction(){\n if [[ \"$META_URL\" != redis* ]]; then\n echo \"this test only runs for redis meta engine\"\n return\n fi\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs --trash-days 1\n ./juicefs.exe mount -d $META_URL z: --no-usage-report\n redis-cli save\n # don't skip files when gc compact\n export JFS_SKIPPED_TIME=1\n ./juicefs.exe gc --compact --delete $META_URL\n ./juicefs.exe fsck $META_URL\n}\n\ntest_gc_trash_slices(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z: --no-usage-report\n PATH1=test PATH2=z:\\\\test python3 .github/scripts/random_read_write.py \n ./juicefs.exe status --more $META_URL\n ./juicefs.exe config $META_URL --trash-days 0 --yes\n ./juicefs.exe gc $META_URL \n ./juicefs.exe gc $META_URL --delete\n ./juicefs.exe status --more $META_URL\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command-win/profile.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common_win.sh\n\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\n\ntest_profile()\n{\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z:\n ./juicefs.exe mdtest $META_URL //d --depth 3 --dirs 3 --files 10 --threads 5 \n timeout 5s ./juicefs profile /z/.accesslog || EXIT_CODE=$?\n if [ \"$EXIT_CODE\" = \"124\" ]; then\n echo \"juicefs profile success\"\n else\n echo \"juicefs profile failed\"\n exit 1\n fi\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/command-win/quota.sh", "content": "#!/bin/bash -e\n\n[[ -z \"$META_URL\" ]] && META_URL=redis://127.0.0.1:6379/1\n\nHEARTBEAT_INTERVAL=3\nHEARTBEAT_SLEEP=3\nDIR_QUOTA_FLUSH_INTERVAL=4\nVOLUME_QUOTA_FLUSH_INTERVAL=2\nsource .github/scripts/common/common_win.sh\n\ntest_total_capacity()\n{\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs --capacity 1\n ./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL --debug\n dd if=/dev/zero of=/z/test1 bs=1G count=1\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n dd if=/dev/zero of=/z/test2 bs=1G count=1 && echo \"dd should fail on out of space\" && exit 1 || true\n rm /z/test1 -rf\n ./juicefs.exe rmr /z/.trash\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n dd if=/dev/zero of=/z/test2 bs=104857600 count=1\n}\n\ntest_total_inodes(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs --inodes 1000\n ./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL\n set +x\n for i in {1..1000}; do\n echo $i | tee /z/test$i > /dev/null\n done\n set -x\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee /z/test1001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n # grep \"No space left on device\" error.log\n ./juicefs.exe config $META_URL --inodes 2000\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n set +x\n for i in {1001..2000}; do\n echo $i | tee /z/test$i > /dev/null || (df -i /z && ls /z/ -l | wc -l && exit 1)\n done\n set -x\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee /z/test2001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n}\n\ntest_remove_and_restore(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /z/d\n ./juicefs.exe quota set $META_URL --path //d --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/z/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs.exe quota get $META_URL --path //d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n dd if=/dev/zero of=/z/d/test2 bs=1G count=1 && echo \"write should fail on out of space\" && exit 1 || true\n echo \"remove test1\" && rm /z/d/test* -rf\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs.exe quota get $META_URL --path //d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"0%\" ]] && echo \"used should be 0%\" && exit 1 || true\n\n trash_dir=$(ls /z/.trash)\n ./juicefs.exe restore $META_URL $trash_dir --put-back\n ./juicefs.exe quota get $META_URL --path //d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/z/d/test2 bs=1G count=1 && echo \"write should fail on out of space\" && exit 1 || true\n echo \"remove test1\" && rm /z/d/test1 -rf\n}\n\ntest_dir_capacity(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs\n ./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /z/d\n ./juicefs.exe quota set $META_URL --path //d --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/z/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs.exe quota get $META_URL --path //d\n used=$(./juicefs.exe quota get $META_URL --path //d 2>&1 | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n dd if=/dev/zero of=/z/d/test2 bs=1G count=1 && echo \"echo should fail on out of space\" && exit 1 || true\n\n ./juicefs.exe quota set $META_URL --path //d --capacity 2\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=/z/d/test2 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n dd if=/dev/zero of=/z/d/test3 bs=1G count=1 && echo \"echo should fail on out of space\" && exit 1 || true\n rm -rf /z/d/test1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n used=$(./juicefs.exe quota get $META_URL --path //d 2>&1 | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"50%\" ]] && echo \"used should be 50%\" && exit 1 || true\n dd if=/dev/zero of=/z/d/test3 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs.exe quota check $META_URL --path //d --strict\n}\n\ntest_dir_inodes(){\n prepare_win_test\n ./juicefs.exe format $META_URL myjfs \n ./juicefs.exe mount -d $META_URL z: --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p /z/d\n ./juicefs.exe quota set $META_URL --path //d --inodes 1000\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n set +x\n for i in {1..1000}; do\n echo $i > /z/d/test$i > /dev/null\n done\n set -x\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee /jfs/d/test1001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n #grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm -rf error.log\n ./juicefs.exe quota set $META_URL --path //d --inodes 2000\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n set +x\n for i in {1001..2000}; do\n echo $i | tee /z/d/test$i > /dev/null\n done\n set -x\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee /z/d/test2001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n #grep \"Disk quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm /z/d/test1 -rf\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee /z/d/test2001\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs.exe quota check $META_URL --path //d --strict\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/common/common.sh", "content": "#!/bin/bash -e\n\n# Common variables and initialization\ninit_platform() {\n case \"$(uname -s)\" in\n Darwin*) PLATFORM=\"mac\";;\n Linux*) PLATFORM=\"linux\";;\n *) PLATFORM=\"unknown\"\n esac\n\n # Install jq if missing\n if ! command -v jq &> /dev/null; then\n case \"$PLATFORM\" in\n mac) brew install jq;;\n linux) .github/scripts/apt_install.sh jq;;\n *) echo \"Unsupported platform\"; exit 1\n esac\n fi\n}\n\n# Platform-agnostic functions with internal branching\nprepare_test() {\n case \"$PLATFORM\" in\n mac)\n ./juicefs umount ~/jfs || true\n umount_jfs ~/jfs \"$META_URL\"\n sleep 1\n python3 .github/scripts/flush_meta.py \"$META_URL\"\n rm -rf ~/.juicefs/local/myjfs/ || true\n rm -rf ~/.juicefs/cache || true\n ;;\n linux)\n umount_jfs /jfs \"$META_URL\"\n python3 .github/scripts/flush_meta.py \"$META_URL\"\n rm -rf /var/jfs/myjfs || true\n rm -rf /var/jfsCache/myjfs || true\n ;;\n esac\n}\n\numount_jfs() {\n local mp=$1\n local meta_url=$2\n [[ -z \"$mp\" ]] && echo \"mount point is empty\" && exit 1\n [[ -z \"$meta_url\" ]] && echo \"meta url is empty\" && exit 1\n \n echo \"umount_jfs $mp $meta_url\"\n [[ ! -f \"$mp/.config\" ]] && return\n \n ls -l \"$mp/.config\"\n local status_log=\"status.log\"\n ./juicefs status --log-level error \"$meta_url\" 2>/dev/null | tee \"$status_log\"\n \n local pids\n pids=$(jq --arg mp \"$mp\" '.Sessions[] | select(.MountPoint == $mp) | .ProcessID' \"$status_log\")\n [[ -z \"$pids\" ]] && cat \"$status_log\" && echo \"pid is empty\" && return\n \n echo \"umount is $mp, pids are $pids\"\n \n for pid in $pids; do\n case \"$PLATFORM\" in\n mac)\n if mount | grep -q \"$mp\"; then\n diskutil unmount \"$mp\" || umount \"$mp\"\n fi\n ;;\n linux)\n umount -l \"$mp\"\n ;;\n esac\n done\n \n for pid in $pids; do\n wait_mount_process_killed \"$pid\" 60\n done\n}\n\nwait_mount_process_killed() {\n local pid=$1\n local wait_seconds=$2\n [[ -z \"$pid\" ]] && echo \"pid is empty\" && exit 1\n [[ -z \"$wait_seconds\" ]] && echo \"wait_seconds is empty\" && exit 1\n \n echo \"waiting for mount process $pid to exit within $wait_seconds seconds\"\n for i in $(seq 1 \"$wait_seconds\"); do\n case \"$PLATFORM\" in\n mac)\n if ! ps -p \"$pid\" > /dev/null; then\n echo \"mount process is killed\"\n break\n fi\n ;;\n linux)\n count=$(ps -ef | grep \"juicefs mount\" | awk '{print $2}' | grep \"^$pid$\" | wc -l)\n if [ \"$count\" -eq 0 ]; then\n echo \"mount process is killed\"\n break\n fi\n ;;\n esac\n \n if [ \"$i\" -eq \"$wait_seconds\" ]; then\n case \"$PLATFORM\" in\n mac) ps -p \"$pid\";;\n linux) ps -ef | grep \"juicefs mount\" | grep -v \"grep\";;\n esac\n echo \": mount process is not killed after $wait_seconds\"\n exit 1\n fi\n sleep 1\n done\n}\n\ncompare_md5sum() {\n local file1=$1\n local file2=$2\n \n case \"$PLATFORM\" in\n mac)\n md51=$(md5 -q \"$file1\")\n md52=$(md5 -q \"$file2\")\n ;;\n linux)\n md51=$(md5sum \"$file1\" | awk '{print $1}')\n md52=$(md5sum \"$file2\" | awk '{print $1}')\n ;;\n esac\n \n if [ \"$md51\" != \"$md52\" ]; then\n echo \"md5 are different: $file1 ($md51) vs $file2 ($md52)\"\n exit 1\n fi\n}\n\nwait_command_success() {\n local command=$1\n local expected=$2\n local timeout=${3:-30}\n \n echo \"waiting for command success: cmd='$command', expected='$expected', timeout=$timeout\"\n for i in $(seq 1 \"$timeout\"); do\n result=$(eval \"$command\" 2>/dev/null | tr -d ' ')\n echo \"attempt $i: result=$result\"\n \n if [[ \"$result\" == \"$expected\" ]]; then\n echo \"command succeeded\"\n return 0\n fi\n \n if [ \"$i\" -eq \"$timeout\" ]; then\n eval \"$command\"\n echo \"command failed after $timeout attempts: $command\"\n exit 1\n fi\n sleep 1\n done\n}\n\n# macOS specific helper (only defined but used when needed)\nensure_directory() {\n [[ \"$PLATFORM\" != \"mac\" ]] && return\n local dir=$1\n if [[ ! -d \"$dir\" ]]; then\n echo \"Creating directory: $dir\"\n mkdir -p \"$dir\"\n fi\n}\n\n# Initialize platform detection\ninit_platform\n\n# Make functions available to subprocesses\nexport -f prepare_test umount_jfs wait_mount_process_killed compare_md5sum wait_command_success ensure_directory\nexport PLATFORM META_URL" }, { "path": ".github/scripts/common/common_win.sh", "content": "#!/bin/bash -e\nprepare_win_test()\n{\n net start redisredis || true\n ./juicefs.exe umount z: || true\n rm -rf C:\\jfs\\local/myjfs/ || true\n rm -rf C:\\jfsCache\\local/myjfs/ || true\n uuid=$(./juicefs.exe status $META_URL | grep UUID | cut -d '\"' -f 4) || true\n ./juicefs.exe destroy --force $META_URL $uuid || true\n redis-cli -h 127.0.0.1 -p 6379 -n 1 FLUSHDB\n}\n\ncompare_md5sum(){\n file1=$1\n file2=$2\n md51=$(md5sum $file1 | awk '{print $1}')\n md52=$(md5sum $file2 | awk '{print $1}')\n # echo md51 is $md51, md52 is $md52\n if [ \"$md51\" != \"$md52\" ] ; then\n echo \"md5 are different: md51 is $md51, md52 is $md52\"\n exit 1\n fi\n}" }, { "path": ".github/scripts/common/run_test.sh", "content": "#!/bin/bash -e\nrun_one_test()\n{\n test=$1\n test=${test%%(*}\n echo -e \"\\033[0;34mStart Test: $test\\033[0m\"\n START_TIME=$(date +%s) \n set +e \n ( set -e; \"${test}\" )\n EXIT_STATUS=$?\n set -e\n echo $test exit with $EXIT_STATUS\n END_TIME=$(date +%s)\n ELAPSED_TIME=$((END_TIME - START_TIME))\n if [[ $EXIT_STATUS -eq 0 ]]; then\n echo -e \"\\033[0;34mFinish Test: $test in $ELAPSED_TIME seconds\\033[0m\"\n else\n echo -e \"\\033[0;31mTest Failed: $0 $test in $ELAPSED_TIME seconds\\033[0m\"\n exit 1\n fi\n}\n\nrun_test(){\n START_TIME_ALL=$(date +%s) \n if [[ ! -z \"$@\" ]]; then\n # run test functions passed by arguments\n for test in \"$@\"; do\n if declare -F \"$test\" > /dev/null; then\n run_one_test $test\n else\n echo -e \"\\033[0;31mTest $test was not found in $0\\033[0m\"\n exit 1\n fi\n done\n else\n # Find and run all test functions\n if [[ \"$(uname)\" == \"Darwin\" ]]; then\n tests=$(grep -E '^[[:space:]]*test_[[:alnum:]_]+[[:space:]]*\\([[:space:]]*\\)' \"$0\")\n else\n tests=$(grep -oP '^\\s*test_\\w+\\s*\\(\\s*\\)' \"$0\")\n fi\n if [[ -z \"$tests\" ]]; then\n echo -e \"\\033[0;31mNo test function found in $0\\033[0m\"\n else\n for test in ${tests}; do\n run_one_test $test\n done\n fi\n fi\n END_TIME_ALL=$(date +%s)\n ELAPSED_TIME_ALL=$((END_TIME_ALL - START_TIME_ALL))\n echo -e \"\\033[0;34mAll tests passed in $ELAPSED_TIME_ALL seconds\\033[0m\"\n}" }, { "path": ".github/scripts/compare_results.sh", "content": "#!/bin/bash\nset -e\n\nCURRENT_RESULTS=$1\nOLD_RESULTS=$2\n\nextract_metrics() {\n awk '{\n op_description=$1; \n op_type=$2;\n for(i=3;i<=NF;i++) if($i == \":\") break;\n max=$(i+1); min=$(i+2); mean=$(i+3); stddev=$(i+4);\n print op_description, op_type, max, min, mean, stddev\n }' <<< \"$1\"\n}\n\ncompare_with_tolerance() {\n local current=$1\n local old=$2\n\n tolerance=$(echo \"$old * 0.1\" | bc -l)\n lower_bound=$(echo \"$old - $tolerance\" | bc -l)\n upper_bound=$(echo \"$old + $tolerance\" | bc -l)\n\n if (( $(echo \"$current <= $upper_bound && $current >= $lower_bound\" | bc -l) )); then\n echo \"same\"\n elif (( $(echo \"$current > $old\" | bc -l) )); then\n echo \"better\"\n else\n echo \"worse\"\n fi\n}\n\ncompare_scenario() {\n local scenario=$1\n local current_file=\"${CURRENT_RESULTS}.${scenario}.summary\"\n local old_file=\"${OLD_RESULTS}.${scenario}.summary\"\n\n echo \"\"\n echo \"====================================================================\"\n echo \"Detailed Comparison for $scenario (with 10% tolerance)\"\n echo \"====================================================================\"\n printf \"%-30s %-12s %-12s %-12s %-12s %-12s\\n\" \"Operation\" \"Current Max\" \"Old Max\" \"Diff\" \"Status\" \"Variance\"\n echo \"--------------------------------------------------------------------\"\n\n while IFS= read -r current_line && IFS= read -r old_line <&3; do\n if [ -z \"$current_line\" ] || [ -z \"$old_line\" ]; then\n continue\n fi\n\n current_metrics=($(extract_metrics \"$current_line\"))\n old_metrics=($(extract_metrics \"$old_line\"))\n\n current_op=\"${current_metrics[0]} ${current_metrics[1]}\"\n old_op=\"${old_metrics[0]} ${old_metrics[1]}\"\n\n if [ \"$current_op\" != \"$old_op\" ]; then\n echo \"Warning: Operation mismatch ('$current_op' vs '$old_op'), skipping...\"\n continue\n fi\n\n current_max=${current_metrics[2]}\n old_max=${old_metrics[2]}\n\n if [[ \"$current_max\" =~ ^[0-9.]+$ ]] && [[ \"$old_max\" =~ ^[0-9.]+$ ]]; then\n diff=$(echo \"$current_max - $old_max\" | bc -l)\n variance=$(echo \"scale=2; ($current_max - $old_max)*100/$old_max\" | bc -l)\n\n comparison=$(compare_with_tolerance $current_max $old_max)\n\n case $comparison in\n \"worse\") status=\"❌ Worse\" ;;\n \"better\") status=\"✅ Better\" ;;\n \"same\") status=\"⚖️ Same\" ;;\n *) status=\"⚠️ Unknown\" ;;\n esac\n\n printf \"%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s%%\\n\" \\\n \"$current_op\" \"$current_max\" \"$old_max\" \"$diff\" \"$status\" \"$variance\"\n else\n printf \"%-30s %-12s %-12s %-12s %-12s %-12s\\n\" \\\n \"$current_op\" \"N/A\" \"N/A\" \"N/A\" \"⚠️ Invalid\" \"N/A\"\n fi\n done < \"$current_file\" 3< \"$old_file\"\n}\n\ncompare_scenario \"scenario1\"\ncompare_scenario \"scenario2\"\n\n# Check if any scenario has \"worse\" results\ncheck_regression() {\n local scenario=$1\n local current_file=\"${CURRENT_RESULTS}.${scenario}.summary\"\n local old_file=\"${OLD_RESULTS}.${scenario}.summary\"\n local regression_detected=0\n\n while IFS= read -r current_line && IFS= read -r old_line <&3; do\n # Skip empty lines\n if [ -z \"$current_line\" ] || [ -z \"$old_line\" ]; then\n continue\n fi\n\n current_metrics=($(extract_metrics \"$current_line\"))\n old_metrics=($(extract_metrics \"$old_line\"))\n\n current_op=\"${current_metrics[0]} ${current_metrics[1]}\"\n old_op=\"${old_metrics[0]} ${old_metrics[1]}\"\n\n if [ \"$current_op\" != \"$old_op\" ]; then\n continue\n fi\n\n current_max=${current_metrics[2]}\n old_max=${old_metrics[2]}\n\n if [[ \"$current_max\" =~ ^[0-9.]+$ ]] && [[ \"$old_max\" =~ ^[0-9.]+$ ]]; then\n comparison=$(compare_with_tolerance $current_max $old_max)\n if [ \"$comparison\" == \"worse\" ]; then\n variance=$(echo \"scale=2; ($current_max - $old_max)*100/$old_max\" | bc -l)\n echo \"Regression detected in $scenario for $current_op: Current $current_max vs Old $old_max (Variance: ${variance}%)\"\n regression_detected=1\n fi\n fi\n done < \"$current_file\" 3< \"$old_file\"\n\n return $regression_detected\n}\n\necho \"\"\necho \"====================================================================\"\necho \"Regression Check Summary (with 10% tolerance)\"\necho \"====================================================================\"\n\nregression_found=0\nif ! check_regression \"scenario1\"; then\n regression_found=1\nfi\nif ! check_regression \"scenario2\"; then\n regression_found=1\nfi\n\nif [ $regression_found -eq 1 ]; then\n echo \"\"\n echo \"ERROR: Performance regression detected compared to old version!\"\n exit 1\nelse\n echo \"\"\n echo \"SUCCESS: No performance regression detected.\"\n exit 0\nfi" }, { "path": ".github/scripts/copyFile.js", "content": "const fs = require('fs');\nconst path = require('path');\nconst crypto = require('crypto');\n\nif (process.argv.length !== 4) {\n console.error('Usage: node copyFile.js ');\n process.exit(1);\n}\n\nconst sourceFile = path.resolve(process.argv[2]);\nconst destinationFile = path.resolve(process.argv[3]);\n\nfs.copyFile(sourceFile, destinationFile, async (err) => {\n if (err) {\n console.error('Error copying file:', err);\n process.exit(1);\n }\n console.log('File copied successfully.');\n});" }, { "path": ".github/scripts/fio.sh", "content": "#/bin/bash -e \nget_fio_job_options(){\n fio_job_name=$1\n case \"$fio_job_name\" in\n \"big-file-sequential-read\") fio_job=\"big-file-sequential-read: --rw=read --refill_buffers --bs=256k --size=1G\"\n ;;\n \"big-file-sequential-write\") fio_job=\"big-file-sequential-write: --rw=write --refill_buffers --bs=256k --size=1G\"\n ;;\n \"big-file-multi-read-1\") fio_job=\"big-file-multi-read-1: --rw=read --refill_buffers --bs=256k --size=1G --numjobs=1\"\n ;;\n \"big-file-multi-read-4\") fio_job=\"big-file-multi-read-4: --rw=read --refill_buffers --bs=256k --size=1G --numjobs=4\"\n ;;\n \"big-file-multi-read-16\") fio_job=\"big-file-multi-read-16: --rw=read --refill_buffers --bs=256k --size=1G --numjobs=16\"\n ;;\n \"big-file-multi-write-1\") fio_job=\"big-file-multi-write-1: --rw=write --refill_buffers --bs=256k --size=1G --numjobs=1\"\n ;;\n \"big-file-multi-write-4\") fio_job=\"big-file-multi-write-4: --rw=write --refill_buffers --bs=256k --size=1G --numjobs=4\"\n ;;\n \"big-file-multi-write-16\") fio_job=\"big-file-multi-write-16: --rw=write --refill_buffers --bs=256k --size=1G --numjobs=16\"\n ;;\n \"big-file-rand-read-4k\") fio_job=\"big-file-rand-read-4k: --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=4k\"\n ;;\n \"big-file-rand-read-256k\") fio_job=\"big-file-rand-read-256k: --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=256k\"\n ;;\n \"big-file-random-write-16k\") fio_job=\"big-file-random-write-16k: --rw=randwrite --refill_buffers --size=1G --bs=16k\"\n ;;\n \"big-file-random-write-256k\") fio_job=\"big-file-random-write-256k: --rw=randwrite --refill_buffers --size=1G --bs=256k\"\n ;;\n \"small-file-seq-read-4k\") fio_job=\"small-file-seq-read-4k: --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 :--cache-size=0\"\n ;;\n \"small-file-seq-read-256k\") fio_job=\"small-file-seq-read-256k: --rw=read --file_service_type=sequential --bs=256k --filesize=256k --nrfiles=10000 :--cache-size=0\"\n ;;\n \"small-file-seq-write-4k\") fio_job=\"small-file-seq-write-4k: --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 :--writeback\"\n ;;\n \"small-file-seq-write-256k\") fio_job=\"small-file-seq-write-256k: --rw=write --file_service_type=sequential --bs=256k --filesize=256k --nrfiles=10000 :--writeback\"\n ;;\n \"small-file-multi-read-1\") fio_job=\"small-file-multi-read-1: --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=1\"\n ;;\n \"small-file-multi-read-4\") fio_job=\"small-file-multi-read-4: --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=4\"\n ;;\n \"small-file-multi-read-16\") fio_job=\"small-file-multi-read-16: --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=16\"\n ;;\n \"small-file-multi-write-1\") fio_job=\"small-file-multi-write-1: --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=1\"\n ;;\n \"small-file-multi-write-4\") fio_job=\"small-file-multi-write-4: --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=4\"\n ;;\n \"small-file-multi-write-16\") fio_job=\"small-file-multi-write-16: --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=16\"\n ;;\n esac\n echo $fio_job\n}\nparse_bandwidth(){\n echo \"parse bandwidth\" >&2\n cat fio.log 1>&2\n bw_str=$(tail -1 fio.log | awk '{print $2}' | awk -F '=' '{print $2}' )\n echo bw_str is $bw_str >&2\n bw=$(echo $bw_str | sed 's/.iB.*//g') \n if [[ $bw_str == *KiB* ]]; then\n bw=$(echo \"scale=2; $bw/1024.0\" | bc -l)\n elif [[ $bw_str == *GiB* ]]; then\n bw=$(echo \"scale=2; $bw*1024.0\" | bc -l)\n fi\n echo bw is $bw >&2\n echo $bw \n}\n \nfio_test()\n{\n meta_url=$1\n fio_job_name=$2\n echo \"Fio Benchmark\"\n fio_job_options=$(get_fio_job_options $fio_job_name)\n echo fio_job_options is $fio_job_options\n name=$(echo $fio_job_options | awk -F: '{print $1}' | xargs)\n fio_arg=$(echo $fio_job_options | awk -F: '{print $2}' | xargs)\n mount_arg=$(echo $fio_job_options | awk -F: '{print $3}' | xargs)\n ./juicefs format --trash-days 0 --storage minio --bucket http://localhost:9000/fio --access-key minioadmin --secret-key minioadmin $meta_url fio\n ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report $mount_arg\n if [[ \"$name\" =~ ^big-file-rand-read.* ]]; then\n block_size=$(echo $name | awk -F- '{print $NF}' | xargs)\n echo block_size is $block_size\n fio --name=big-file-rand-read-preload --directory=/tmp/jfs --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=$block_size --pre_read=1\n sudo sync \n sudo bash -c \"echo 3 > /proc/sys/vm/drop_caches\"\n fi\n echo \"start fio\"\n fio --name=$name --directory=/tmp/jfs $fio_arg | tee \"fio.log\"\n echo \"finish fio\"\n ./juicefs umount -f /tmp/jfs\n uuid=$(./juicefs status $meta_url | grep UUID | cut -d '\"' -f 4)\n if [ -n \"$uuid\" ]; then\n sudo ./juicefs destroy --yes $meta_url $uuid\n fi\n}\nmeta_url=$1\nname=$2\nfio_test $meta_url $name\nbandwidth=$(parse_bandwidth)\necho bandwidth is $bandwidth\n[[ -z \"$bandwidth\" ]] && echo \"bandwidth is empty\" && exit 1\nmeta=$(echo $meta_url | awk -F: '{print $1}')\necho meta is $meta\n[[ -z \"$meta\" ]] && echo \"meta is empty\" && exit 1\n.github/scripts/save_benchmark.sh --name $name --result $bandwidth --meta $meta --storage minio" }, { "path": ".github/scripts/flush_meta.py", "content": "import argparse\nimport os\nfrom posixpath import expanduser\nfrom utils import *\n\nif __name__ == \"__main__\":\n p = argparse.ArgumentParser()\n p.add_argument(\"meta_url\")\n args = p.parse_args(sys.argv[1:])\n flush_meta(args.meta_url)" }, { "path": ".github/scripts/fsrand.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2015, Bill Zissimopoulos. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met:\n#\n# 1. Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n#\n# 2. Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n#\n# 3. Neither the name of the copyright holder nor the names of its\n# contributors may be used to endorse or promote products derived from\n# this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS\n# IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED\n# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A\n# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\nimport subprocess\ntry:\n __import__(\"xattr\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\nimport os, random\nimport platform\nimport unicodedata\nfrom xmlrpc.client import boolean\nimport xattr \nclass Devnull(object):\n def write(self, *args):\n pass\ndevnull = Devnull()\nclass FsRandomizer(object):\n def __init__(self, path, count, seed):\n self.stdout = devnull\n self.stderr = devnull\n self.verbose = 0\n self.maxofs = 192*1024\n self.maxlen = 64*1024\n self.path = os.path.realpath(path)\n self.count = count\n self.random = random.Random(seed)\n self.dictionary = None\n def __stdout(self, s):\n self.stdout.write(str(s) + \"\\n\")\n def __stderr(self, s):\n self.stderr.write(str(s) + \"\\n\")\n def __getdir_recurse(self, path):\n try:\n n = self.random.choice(sorted(os.listdir(path)))\n except:\n return path\n p = os.path.join(path, n)\n if os.path.isdir(p):\n return self.__getdir_recurse(p)\n else:\n return path\n def __getdir(self):\n path = self.__getdir_recurse(self.path)\n parts = path[len(self.path):].split(os.sep)\n parts = parts[0:self.random.randint(1, len(parts))]\n return os.path.join(self.path, *parts)\n def __getsubpath(self, path):\n try:\n # print(\"\\t\".join(sorted(os.listdir(path))))\n n = self.random.choice(sorted(os.listdir(path)))\n except:\n return path\n return os.path.join(path, n)\n\n def __gen_unicode_name(self, lower_limit=1, upper_limit=64):\n unicodes = ''.join(\n chr(char)\n for char in range(1000)\n # use the unicode categories that don't include control codes\n # if unicodedata.category(chr(char))[0] in ('LMNPSZ') and chr(char) != '/'\n if unicodedata.category(chr(char))[0] in ('LMNPSZ') and chr(char) != '/'\n )\n assert('/' not in unicodes)\n rand_length = self.random.randint(lower_limit, upper_limit)\n # generate it\n utf_string = ''.join([self.random.choice(unicodes) for i in range(rand_length)])\n if utf_string == '.' or utf_string == '..':\n utf_string = 'ABC'\n assert('/' not in utf_string)\n # print(''.join([unicodedata.category(c) for c in utf_string]))\n return utf_string\n\n def __gen_ascii_name(self, lower_limit=1, upper_limit=64):\n l = self.random.randint(lower_limit, upper_limit)\n n = [self.random.choice(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\") for i in range(l)]\n return \"\".join(n)\n\n def __newname(self):\n if self.dictionary:\n return self.random.choice(self.dictionary)\n else:\n if self.ascii:\n return self.__gen_ascii_name()\n else:\n return self.__gen_unicode_name()\n\n def __newsubpath(self, path):\n while True:\n p = os.path.join(path, self.__newname())\n if not os.path.lexists(p):\n return p\n def __newmode(self, mode):\n return mode | self.random.randint(0, 0o077)\n def __random_write(self, file):\n o = self.random.randint(0, self.maxofs)\n l = self.random.randint(0, self.maxlen)\n # b = bytearray(self.random.getrandbits(8) for _ in range(l))\n # b = self.random.randbytes(l)\n b = bytes('abc', \"utf-8\")\n file.seek(o)\n file.write(b)\n def __create(self, path):\n assert not os.path.exists(path)\n with open(path, \"wb\") as f:\n self.__random_write(f)\n def __update(self, path):\n assert os.path.exists(path)\n with open(path, \"r+b\") as f:\n self.__random_write(f)\n def randomize(self):\n for i in range(self.count):\n op = self.random.choice(\"CCRUUSL\")\n if op == \"C\":\n path = self.__newsubpath(self.__getdir())\n if self.verbose:\n self.__stderr(\"%s, CREATE %s\" % (str(i), path))\n if self.random.randint(0, 1):\n self.__create(path)\n os.chmod(path, self.__newmode(0o0600))\n else:\n os.mkdir(path)\n os.chmod(path, self.__newmode(0o0700))\n elif op == \"S\":\n src = self.__getsubpath(self.__getdir())\n if not os.path.exists(src):\n continue\n if os.path.isdir(src):\n continue\n dest = self.__newsubpath(self.__getdir())\n assert(not os.path.exists(dest))\n if self.verbose:\n self.__stderr(\"%s, CREATE SYMLINK from %s to %s\" % (str(i), src, dest))\n try:\n os.symlink(src, dest)\n except: \n print(\"\".join([str(ord(c)) for c in src]))\n print(\"\".join([str(ord(c)) for c in dest]))\n raise Exception(\"OS error: {0}\".format(err))\n elif op == \"L\":\n src = self.__getsubpath(self.__getdir())\n if not os.path.exists(src):\n continue\n if os.path.isdir(src):\n continue\n dest = self.__newsubpath(self.__getdir())\n assert(not os.path.exists(dest))\n if self.verbose:\n self.__stderr(\"%s, CREATE LINK from %s to %s\" % (str(i), src, dest))\n try:\n os.link(src, dest)\n except OSError as err :\n print(\"\".join([str(ord(c)) for c in src]))\n print(\"\".join([str(ord(c)) for c in dest]))\n print(\"OS error: {0}\".format(err))\n raise Exception(\"OS error: {0}\".format(err))\n elif op == \"R\":\n path = self.__getsubpath(self.__getdir())\n if os.path.realpath(path) == self.path:\n continue\n if self.verbose:\n self.__stderr(\"%s, REMOVE %s\" % (str(i), path))\n if not os.path.isdir(path):\n os.unlink(path)\n else:\n try:\n os.rmdir(path)\n except:\n pass\n\n elif op == \"X\":\n path = self.__getsubpath(self.__getdir())\n if not os.path.exists(path):\n continue\n if self.verbose:\n self.__stderr(\"%s, SETXATTR %s\" % (str(i), path))\n key = self.__gen_unicode_name()\n value = self.__gen_unicode_name()\n if platform.system() == 'Linux':\n os.system(f'setfattr -n {key} -v {value} {path}')\n else:\n xattr.setxattr(path, key, bytes(value, \"utf-8\"))\n \n value_set = xattr.getxattr(path, key)\n assert( bytes(value, 'utf-8') == value_set)\n\n elif op == \"U\":\n path = self.__getsubpath(self.__getdir())\n if os.path.realpath(path) == self.path:\n continue\n if not os.path.exists(path):\n continue\n if self.verbose:\n self.__stderr(\"%s, UPDATE %s\" % (str(i), path))\n u = self.random.randint(0, 2)\n if u == 0:\n if not os.path.isdir(path):\n os.chmod(path, self.__newmode(0o0600))\n else:\n os.chmod(path, self.__newmode(0o0700))\n elif u == 1:\n if not os.path.isdir(path):\n self.__update(path)\n else:\n if not os.path.isdir(path):\n self.__update(path)\n os.chmod(path, self.__newmode(0o0600))\n else:\n os.chmod(path, self.__newmode(0o0700))\n \nif \"__main__\" == __name__:\n import argparse, sys, time\n def info(s):\n print (\"%s: %s\" % (os.path.basename(sys.argv[0]), s))\n def warn(s):\n print (\"%s: %s\" % (os.path.basename(sys.argv[0]), s))\n def fail(s, exitcode = 1):\n warn(s)\n sys.exit(exitcode)\n def main():\n p = argparse.ArgumentParser()\n p.add_argument(\"-v\", \"--verbose\", action=\"count\", default=0)\n p.add_argument(\"-c\", \"--count\", type=int, default=100)\n p.add_argument(\"-s\", \"--seed\", type=int, default=0)\n p.add_argument(\"-a\", \"--ascii\", action=\"count\", default=0)\n p.add_argument(\"-d\", \"--dictionary\")\n p.add_argument(\"path\")\n args = p.parse_args(sys.argv[1:])\n if args.seed == 0:\n args.seed = int(time.time())\n if not os.path.isdir(args.path):\n os.mkdir(args.path)\n # fail(\"path must exist and be a directory\")\n if args.dictionary:\n with open(args.dictionary) as f:\n args.dictionary = [l.strip() for l in f]\n info(\"count=%s seed=%s \" % (args.count, args.seed))\n os.umask(0)\n fsrand = FsRandomizer(args.path, args.count, args.seed)\n fsrand.dictionary = args.dictionary\n fsrand.stdout = sys.stdout\n fsrand.stderr = sys.stderr\n fsrand.verbose = args.verbose\n fsrand.ascii = args.ascii\n fsrand.randomize()\n info(\"create files succeed\")\n def __entry():\n try:\n main()\n except EnvironmentError as ex:\n fail(ex)\n except KeyboardInterrupt:\n fail(\"interrupted\", 130)\n __entry()" }, { "path": ".github/scripts/hypo/command.py", "content": "from difflib import Differ\nimport json\nimport os\nimport re\nimport subprocess\ntry: \n __import__('jsondiff')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"jsondiff\"])\nfrom jsondiff import diff\ntry: \n __import__('psutil')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"psutil\"])\nimport psutil\ntry: \n __import__('fallocate')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"fallocate\"])\ntry: \n __import__('xattr')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hypothesis\"])\nfrom hypothesis import HealthCheck, assume, strategies as st, settings, Verbosity\nfrom hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple\nfrom hypothesis import Phase, seed\nfrom hypothesis.database import DirectoryBasedExampleDatabase\nimport random\nfrom common import run_cmd\nfrom strategy import *\nfrom fs_op import FsOperation\nfrom command_op import CommandOperation\nfrom fs import JuicefsMachine\nimport common\n\nSEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))\n\nSUDO_USERS = ['root', 'user1']\nst_sudo_user = st.sampled_from(SUDO_USERS)\n\n@seed(SEED)\nclass JuicefsCommandMachine(JuicefsMachine):\n Files = Bundle('files')\n Folders = Bundle('folders')\n Entries = Files | Folders\n MP1 = '/tmp/jfs1'\n MP2 = '/tmp/jfs2'\n ROOT_DIR1=os.path.join(MP1, 'fsrand')\n ROOT_DIR2=os.path.join(MP2, 'fsrand')\n EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', 'config']\n # EXCLUDE_RULES = []\n INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr', 'dump']\n cmd1 = CommandOperation('cmd1', MP1, ROOT_DIR1)\n cmd2 = CommandOperation('cmd2', MP2, ROOT_DIR2)\n fsop1 = FsOperation('fs1', ROOT_DIR1)\n fsop2 = FsOperation('fs2', ROOT_DIR2)\n def __init__(self):\n super().__init__()\n \n def get_default_rootdir1(self):\n return os.path.join(self.MP1, 'fsrand')\n \n def get_default_rootdir2(self):\n return os.path.join(self.MP2, 'fsrand')\n\n def equal(self, result1, result2):\n if type(result1) != type(result2):\n return False\n if isinstance(result1, Exception):\n if 'panic:' in str(result1) or 'panic:' in str(result2):\n return False\n result1 = str(result1)\n result2 = str(result2)\n result1 = common.replace(result1, self.MP1, '***')\n result2 = common.replace(result2, self.MP2, '***')\n # print(f'result1 is {result1}\\nresult2 is {result2}')\n return result1 == result2\n\n def get_client_version(self, mount):\n output = run_cmd(f'{mount} version')\n return output.split()[2]\n\n def should_run(self, rule):\n if len(self.EXCLUDE_RULES) > 0:\n return rule not in self.EXCLUDE_RULES\n else:\n return rule in self.INCLUDE_RULES\n\n @rule(\n entry = Entries.filter(lambda x: x != multiple()),\n raw = st.just(True),\n recuisive = st.booleans(),\n strict = st.just(True),\n user = st_sudo_user\n )\n @precondition(lambda self: self.should_run('info'))\n def info(self, entry, raw=True, recuisive=False, strict=True, user='root'):\n result1 = self.cmd1.do_info(entry=entry, user=user, strict=strict, raw=raw, recuisive=recuisive) \n result2 = self.cmd2.do_info(entry=entry, user=user, strict=strict, raw=raw, recuisive=recuisive)\n assert self.equal(result1, result2), f'\\033[31minfo:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(entry = Entries.filter(lambda x: x != multiple()),\n user = st_sudo_user\n )\n @precondition(lambda self: self.should_run('rmr'))\n def rmr(self, entry, user='root'):\n assume(entry != '')\n result1 = self.cmd1.do_rmr(entry=entry, user=user)\n result2 = self.cmd2.do_rmr(entry=entry, user=user)\n assert self.equal(result1, result2), f'\\033[31mrmr:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule()\n @precondition(lambda self: self.should_run('status'))\n def status(self):\n result1 = self.cmd1.do_status()\n result2 = self.cmd2.do_status()\n assert result1 == result2, f'\\033[31mresult1 is {result1}\\nresult2 is {result2}, {diff(result1, result2)}\\033[0m'\n\n @rule(entry = Entries.filter(lambda x: x != multiple()),\n user = st_sudo_user\n )\n @precondition(lambda self: self.should_run('warmup'))\n def warmup(self, entry, user='root'):\n result1 = self.cmd1.do_warmup(entry=entry, user=user)\n result2 = self.cmd2.do_warmup(entry=entry, user=user)\n assert self.equal(result1, result2), f'\\033[31mwarmup:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n compact = st.booleans(),\n delete = st.booleans(),\n user = st.just('root'),\n )\n @precondition(lambda self: self.should_run('gc'))\n def gc(self, compact=False, delete=False, user='root'):\n result1 = self.cmd1.do_gc(compact=compact, delete=delete, user=user)\n result2 = self.cmd2.do_gc(compact=compact, delete=delete, user=user)\n assert self.equal(result1, result2), f'\\033[31mgc:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n entry = Entries.filter(lambda x: x != multiple()),\n repair = st.booleans(),\n recuisive = st.booleans(),\n user = st_sudo_user, \n )\n @precondition(lambda self: self.should_run('fsck'))\n def fsck(self, entry, repair=False, recuisive=False, user='root'):\n result1 = self.cmd1.do_fsck(entry=entry, repair=repair, recuisive=recuisive, user=user)\n result2 = self.cmd2.do_fsck(entry=entry, repair=repair, recuisive=recuisive, user=user)\n assert self.equal(result1, result2), f'\\033[31mfsck:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n entry = Entries.filter(lambda x: x != multiple()),\n parent = Folders.filter(lambda x: x != multiple()),\n new_entry_name = st_file_name,\n user = st_sudo_user,\n preserve = st.booleans()\n )\n @precondition(lambda self: self.should_run('clone'))\n def clone(self, entry, parent, new_entry_name, preserve=False, user='root'):\n result1 = self.cmd1.do_clone(entry=entry, parent=parent, new_entry_name=new_entry_name, preserve=preserve, user=user)\n result2 = self.cmd2.do_clone(entry=entry, parent=parent, new_entry_name=new_entry_name, preserve=preserve, user=user)\n assert self.equal(result1, result2), f'\\033[31mclone:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(folder = Folders.filter(lambda x: x != multiple()),\n fast = st.booleans(),\n skip_trash = st.booleans(),\n threads = st.integers(min_value=1, max_value=10),\n keep_secret_key = st.booleans(),\n user = st.just('root')\n )\n @precondition(lambda self: self.should_run('dump'))\n def dump(self, folder, fast, skip_trash, threads, keep_secret_key, user='root'):\n result1 = self.cmd1.do_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)\n result2 = self.cmd2.do_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)\n d=''\n if isinstance(result1, str) and isinstance(result2, str):\n d=self.diff(result1, result2)\n assert self.equal(result1, result2), f'\\033[31mdump:\\nresult1 is {result1}\\nresult2 is {result2}\\ndiff is {d}\\033[0m'\n\n @rule(folder = st.just(''),\n fast = st.booleans(),\n skip_trash = st.booleans(),\n threads = st.integers(min_value=1, max_value=10),\n keep_secret_key = st.booleans(),\n user = st.just('root')\n )\n @precondition(lambda self: self.should_run('dump_load_dump'))\n def dump_load_dump(self, folder, fast=False, skip_trash=False, threads=10, keep_secret_key=False, user='root'):\n result1 = self.cmd1.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)\n result2 = self.cmd2.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)\n d=''\n if isinstance(result1, str) and isinstance(result2, str):\n d=self.diff(result1, result2)\n assert self.equal(result1, result2), f'\\033[31mdump:\\nresult1 is {result1}\\nresult2 is {result2}\\ndiff is {d}\\033[0m'\n\n def diff(self, str1:str, str2:str):\n differ = Differ()\n diff = differ.compare(str1.splitlines(), str2.splitlines())\n return '\\n'.join([line for line in diff])\n \n @rule(\n user = st_sudo_user\n )\n @precondition(lambda self: self.should_run('trash_list') and False)\n def trash_list(self, user='root'):\n result1 = self.cmd1.do_trash_list(user=user)\n result2 = self.cmd2.do_trash_list(user=user)\n assert self.equal(result1, result2), f'\\033[31mtrash_list:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n put_back = st.booleans(),\n threads = st.integers(min_value=1, max_value=10),\n user=st_sudo_user\n )\n @precondition(lambda self: self.should_run('restore') and False)\n def restore(self, put_back, threads, user='root'):\n result1 = self.cmd1.do_restore(put_back=put_back, threads=threads, user=user)\n result2 = self.cmd2.do_restore(put_back=put_back, threads=threads, user=user)\n assert self.equal(result1, result2), f'\\033[31mrestore:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n entry = Entries.filter(lambda x: x != multiple()),\n threads = st.integers(min_value=1, max_value=10),\n user = st_sudo_user\n )\n @precondition(lambda self: self.should_run('compact'))\n def compact(self, entry, threads, user='root'):\n result1 = self.cmd1.do_compact(entry=entry, threads=threads, user=user)\n result2 = self.cmd2.do_compact(entry=entry, threads=threads, user=user)\n assert self.equal(result1, result2), f'\\033[31mcompact:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n capacity = st.integers(min_value=1, max_value=2),\n inodes = st.one_of(st.just(0), st.integers(min_value=50, max_value=100)),\n trash_days = st.integers(min_value=0, max_value=1),\n enable_acl = st.booleans(),\n encrypt_secret = st.booleans(),\n force = st.booleans(),\n yes = st.just(True),\n user = st_sudo_user\n )\n @precondition(lambda self: self.should_run('config'))\n def config(self, capacity, inodes, trash_days, enable_acl, encrypt_secret, force, yes, user='root'):\n result1 = self.cmd1.do_config(capacity=capacity, inodes=inodes, trash_days=trash_days, enable_acl=enable_acl, encrypt_secret=encrypt_secret, force=force, yes=yes, user=user)\n result2 = self.cmd2.do_config(capacity=capacity, inodes=inodes, trash_days=trash_days, enable_acl=enable_acl, encrypt_secret=encrypt_secret, force=force, yes=yes, user=user)\n assert self.equal(result1, result2), f'\\033[31mconfig:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n def teardown(self):\n pass\n\nif __name__ == '__main__':\n MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))\n STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))\n ci_db = DirectoryBasedExampleDatabase(\".hypothesis/examples\") \n settings.register_profile(\"dev\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])\n settings.register_profile(\"schedule\", max_examples=500, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=200, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target], \n database=ci_db)\n settings.register_profile(\"pull_request\", max_examples=100, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=50, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target], \n database=ci_db)\n \n if os.environ.get('CI'):\n event_name = os.environ.get('GITHUB_EVENT_NAME')\n if event_name == 'schedule':\n profile = 'schedule'\n else:\n profile = 'pull_request'\n else:\n profile = os.environ.get('PROFILE', 'dev')\n print(f'profile is {profile}')\n settings.load_profile(profile)\n \n juicefs_machine = JuicefsCommandMachine.TestCase()\n juicefs_machine.runTest()\n print(json.dumps(FsOperation.stats.get(), sort_keys=True, indent=4))\n \n " }, { "path": ".github/scripts/hypo/command_op.py", "content": "import json\nimport os\nimport pwd\nimport re\nimport shlex\nimport subprocess\ntry: \n __import__('xattr')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\ntry: \n __import__('psutil')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"psutil\"])\nimport psutil\nfrom stats import Statistics\nimport common\n\n\nclass CommandOperation:\n JFS_CONTROL_FILES=['.accesslog', '.config', '.stats']\n stats = Statistics()\n def __init__(self, name, mp, root_dir):\n self.logger = common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO'))\n self.name = name\n self.mp = mp\n self.root_dir = root_dir\n self.meta_url = self.get_meta_url(mp)\n \n def guess_password(self, meta_url):\n if '****' not in meta_url:\n return meta_url\n if meta_url.startswith('postgres://'):\n return meta_url.replace('****', 'postgres')\n else:\n return meta_url.replace('****', 'root')\n\n def get_meta_url(self, mp):\n with open(os.path.join(mp, '.config')) as f:\n config = json.loads(f.read())\n pid = config['Pid']\n process = psutil.Process(pid)\n cmdline = process.cmdline()\n for item in cmdline:\n if ' ' in item:\n for subitem in item.split(' '):\n if '://' in subitem:\n return self.guess_password(subitem)\n elif '://' in item:\n return self.guess_password(item)\n raise Exception(f'get_meta_url: {cmdline} does not contain meta url')\n \n def run_cmd(self, command:str, stderr=subprocess.STDOUT) -> str:\n self.logger.info(f'run_cmd: {command}')\n if '|' in command or '>' in command or '&' in command:\n ret=os.system(command)\n if ret == 0:\n return ret\n else: \n raise Exception(f\"run command {command} failed with {ret}\")\n try:\n output = subprocess.run(command.split(), check=True, stdout=subprocess.PIPE, stderr=stderr)\n except subprocess.CalledProcessError as e:\n raise e\n return output.stdout.decode()\n\n def seteuid(self, user):\n os.seteuid(pwd.getpwnam(user).pw_uid)\n os.setegid(pwd.getpwnam(user).pw_gid)\n \n def handleException(self, e, action, path, **kwargs):\n if isinstance(e, subprocess.CalledProcessError):\n err = e.output.decode()\n else:\n err = str(e)\n err = '\\n'.join([elem.split(':')[-1].split(':')[-1] for elem in err.split('\\n')])\n err = re.sub(r'\\[\\w+\\.go:\\d+\\]', '', err)\n if err.find('setfacl') != -1 and err.find('\\n') != -1:\n err = '\\n'.join(sorted(err.split('\\n')))\n self.stats.failure(action)\n self.logger.info(f'{action} {path} {kwargs} failed: {err}')\n return Exception(err)\n\n def get_raw(self, size:str):\n # get bytes count from '4.00 KiB (4096 Bytes)' or '3 Bytes'\n if size.find('(') > -1:\n return size.split('(')[1].split(' ')[0]\n else:\n return size.split(' ')[0]\n\n def parse_info(self, info: str):\n li = info.split('\\n')\n if \"GOCOVERDIR\" in li[0]:\n li = li[1:]\n filename = li[0].split(':')[0].strip()\n # assert li[0].strip().startswith('inode:'), f'parse_info: {li[0]} should start with inode:'\n # inode = li[0].split(':')[1].strip()\n assert li[2].strip().startswith('files:'), f'parse_info: {li[2]} should start with files:'\n files = li[2].split(':')[1].strip() \n assert li[3].strip().startswith('dirs:'), f'parse_info: {li[3]} should start with dirs:' \n dirs = li[3].split(':')[1].strip()\n assert li[4].strip().startswith('length:'), f'parse_info: {li[4]} should start with length:'\n length = li[4].split(':')[1].strip()\n length = self.get_raw(length)\n assert li[5].strip().startswith('size:'), f'parse_info: {li[5]} should start with size:'\n size = li[5].split(':')[1].strip()\n size = self.get_raw(size)\n assert li[6].strip().startswith('path'), f'parse_info: {li[6]} should start with path:'\n paths = []\n if li[6].strip().startswith('path:'):\n paths.append(li[6].split(':')[1].strip())\n elif li[6].strip().startswith('paths:'):\n for i in range(7, len(li)):\n if li[i].strip().startswith('/'):\n paths.append(li[i].strip())\n else:\n break\n paths = ','.join(sorted(paths))\n return filename, files, dirs, length, size, paths\n\n def do_info(self, entry, strict=True, user='root', raw=True, recuisive=False):\n abs_path = os.path.join(self.root_dir, entry)\n try:\n cmd = f'sudo -u {user} ./juicefs info --log-level error {abs_path}'\n if raw:\n cmd += ' --raw'\n if recuisive:\n cmd += ' --recursive'\n if strict:\n cmd += ' --strict'\n result = self.run_cmd(cmd)\n if ':' in result or \"permission denied\" in result:\n return self.handleException(Exception(result), 'do_info', abs_path)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_info', abs_path)\n result = self.parse_info(result)\n self.stats.success('do_info')\n self.logger.info(f'do_info {abs_path} succeed')\n return result \n \n def do_rmr(self, entry, user='root'):\n abspath = os.path.join(self.root_dir, entry)\n try:\n result = self.run_cmd(f'sudo -u {user} ./juicefs rmr --log-level error {abspath}')\n if ':' in result:\n return self.handleException(Exception(result), 'do_rmr', abspath)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_rmr', abspath)\n assert not os.path.exists(abspath), f'do_rmr: {abspath} should not exist'\n self.stats.success('do_rmr')\n self.logger.info(f'do_rmr {abspath} succeed')\n return True\n \n def do_status(self):\n try:\n result = self.run_cmd(f'./juicefs status {self.meta_url} --log-level error', stderr=subprocess.DEVNULL)\n result = json.loads(result)['Setting']\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_status', '')\n self.stats.success('do_status')\n self.logger.info(f'do_status succeed')\n return result['Storage'], result['Bucket'], result['BlockSize'], result['Compression'], \\\n result['EncryptAlgo'], result['TrashDays'], result['MetaVersion'], \\\n result['MinClientVersion'], result['DirStats'], result['EnableACL']\n \n def do_dump(self, folder, fast=False, skip_trash=False, threads=1, keep_secret_key=False, user='root'):\n abspath = os.path.join(self.root_dir, folder)\n subdir = os.path.relpath(abspath, self.mp)\n try:\n # compact before dump to avoid slice difference\n self.do_compact(folder)\n cmd=self.get_dump_cmd(self.meta_url, subdir, fast, skip_trash, keep_secret_key, threads, user)\n result = self.run_cmd(cmd, stderr=subprocess.DEVNULL)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_dump', abspath)\n self.stats.success('do_dump')\n self.logger.info(f'do_dump {abspath} succeed')\n # with open(f'dump_{self.name}.json', 'w') as f:\n # f.write(self.clean_dump(result))\n return self.clean_dump(result)\n\n def get_dump_cmd(self, meta_url, subdir, fast, skip_trash, keep_secret_key, threads, user='root'):\n cmd = f'sudo -u {user} ./juicefs dump --log-level error {meta_url} '\n cmd += f' --subdir /{subdir}' if subdir != '' else ''\n cmd += f' --fast' if fast else ''\n cmd += f' --skip-trash' if skip_trash else ''\n cmd += f' --keep-secret-key' if keep_secret_key else ''\n cmd += f' --threads {threads}'\n cmd += f' --log-level error'\n return cmd\n\n def do_dump_load_dump(self, folder, fast=False, skip_trash=False, threads=1, keep_secret_key=False, user='root'):\n abspath = os.path.join(self.root_dir, folder)\n subdir = os.path.relpath(abspath, self.mp)\n try:\n print(f'meta_url is {self.meta_url}')\n cmd = self.get_dump_cmd(self.meta_url, subdir, fast, skip_trash, keep_secret_key, threads, user)\n result = self.run_cmd(cmd, stderr=subprocess.DEVNULL)\n with open('dump.json', 'w') as f:\n f.write(result)\n if os.path.exists('load.db'):\n os.remove('load.db')\n self.run_cmd(f'sudo -u {user} ./juicefs load sqlite3://load.db dump.json')\n cmd = self.get_dump_cmd('sqlite3://load.db', '', fast, skip_trash, keep_secret_key, threads, user)\n result = self.run_cmd(cmd, stderr=subprocess.DEVNULL)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_dump', abspath)\n self.stats.success('do_dump')\n self.logger.info(f'do_dump {abspath} succeed')\n return self.clean_dump(result)\n\n def clean_dump(self, dump):\n lines = dump.split('\\n')\n new_lines = []\n exclude_keys = ['Name', 'UUID', 'usedSpace', 'usedInodes', 'nextInodes', 'nextChunk', 'nextTrash', 'nextSession']\n reset_keys = ['id', 'inode', 'atimensec', 'mtimensec', 'ctimensec', 'atime', 'ctime', 'mtime']\n for line in lines:\n should_delete = False\n for key in exclude_keys:\n if f'\"{key}\"' in line:\n should_delete = True\n break\n if should_delete:\n continue\n for key in reset_keys:\n if f'\"{key}\"' in line:\n pattern = rf'\"{key}\":(\\d+)'\n line = re.sub(pattern, f'\"{key}\":0', line)\n new_lines.append(line)\n return '\\n'.join(new_lines)\n\n def do_warmup(self, entry, user='root'):\n abspath = os.path.join(self.root_dir, entry)\n try:\n self.run_cmd(f'sudo -u {user} ./juicefs warmup --log-level error {abspath}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_warmup', abspath)\n self.stats.success('do_warmup')\n self.logger.info(f'do_warmup {abspath} succeed')\n return True\n\n def do_gc(self, compact:bool, delete:bool, user:str='root'):\n try:\n cmd = f'sudo -u {user} ./juicefs gc --log-level error {self.meta_url}'\n if compact:\n cmd += ' --compact'\n if delete:\n cmd += ' --delete'\n self.run_cmd(cmd)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_gc', '')\n self.stats.success('do_gc')\n self.logger.info(f'do_gc succeed')\n return True\n \n def do_clone(self, entry, parent, new_entry_name, preserve:bool, user:str='root'):\n abspath = os.path.join(self.root_dir, entry)\n dest_abspath = os.path.join(self.root_dir, parent, new_entry_name)\n try:\n cmd = f'sudo -u {user} ./juicefs clone --log-level error {abspath} {dest_abspath}'\n if preserve:\n cmd += ' --preserve'\n self.run_cmd(cmd)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_clone', '')\n self.stats.success('do_clone')\n self.logger.info(f'do_clone succeed')\n return True \n \n def do_fsck(self, entry, repair=False, recuisive=False, user='root'):\n abspath = os.path.join(self.root_dir, entry)\n try:\n cmd = f'sudo -u {user} ./juicefs fsck --log-level error {self.meta_url} --path {abspath}'\n if repair:\n cmd += ' --repair'\n if recuisive:\n cmd += ' --recursive'\n self.run_cmd(cmd, stderr=subprocess.DEVNULL)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_fsck', '')\n self.stats.success('do_fsck')\n self.logger.info(f'do_fsck succeed')\n return True\n \n def do_trash_list(self, user='root'):\n abspath = os.path.join(self.mp, '.trash')\n try:\n self.seteuid(user)\n li = os.listdir(abspath) \n li = sorted(li)\n except Exception as e:\n return self.handleException(e, 'do_trash_list', abspath, user=user)\n finally:\n os.seteuid(0)\n os.setegid(0)\n self.stats.success('do_trash_list')\n self.logger.info(f'do_trash_list succeed')\n return tuple(li)\n \n def do_restore(self, put_back, threads, user='root'):\n abspath = os.path.join(self.mp, '.trash')\n try:\n li = os.listdir(abspath)\n for trash_dir in li:\n cmd = f'sudo -u {user} ./juicefs restore {trash_dir} --threads {threads}'\n if put_back:\n cmd += ' --put-back'\n self.run_cmd(cmd)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_restore', abspath, user=user)\n self.stats.success('do_restore')\n self.logger.info(f'do_restore succeed')\n return True\n\n def do_trash_restore(self, index, user='root'):\n trash_list = self.do_trash_list()\n if len(trash_list) == 0:\n return ''\n index = index % len(trash_list)\n trash_file:str = trash_list[index]\n abspath = os.path.join(self.mp, '.trash', shlex.quote(trash_file))\n try:\n self.run_cmd(f'sudo -u {user} mv {abspath} {self.mp}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_trash_restore', abspath, user=user)\n restored_path = os.path.join(self.mp, '/'.join(trash_file.split('|')[1:]))\n restored_path = os.path.relpath(restored_path, self.root_dir)\n self.stats.success('do_trash_restore')\n self.logger.info(f'do_trash_restore succeed')\n return restored_path\n \n def do_compact(self, entry, threads=5, user='root'):\n path = os.path.join(self.root_dir, entry)\n try:\n self.run_cmd(f'sudo -u {user} ./juicefs compact --log-level error {path} --threads {threads}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_compact', path, user=user)\n self.stats.success('do_compact')\n self.logger.info(f'do_compact succeed')\n return True\n \n def do_config(self, capacity, inodes, trash_days, enable_acl, encrypt_secret, force, yes, user):\n try:\n cmd = f'sudo -u {user} ./juicefs config {self.meta_url} --capacity {capacity} --inodes {inodes} --trash-days {trash_days} --enable-acl {enable_acl} --encrypt-secret {encrypt_secret}'\n if force:\n cmd += ' --force'\n if yes:\n cmd += ' --yes'\n self.run_cmd(cmd)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_config', '')\n self.stats.success('do_config')\n self.logger.info(f'do_config succeed')\n return True" }, { "path": ".github/scripts/hypo/command_test.py", "content": "import unittest\nfrom command import JuicefsCommandMachine\n\nclass TestCommand(unittest.TestCase):\n def test_dump(self):\n state = JuicefsCommandMachine()\n folders_0 = state.init_folders()\n files_0 = state.create_file(content='', file_name='aazz', mode='w', parent=folders_0, umask=312, user='root')\n value = ''.join([chr(i) for i in range(256)])\n value = value.encode('latin-1')\n value = b'\\x2580q\\x2589'\n value = b'M\\x25DB'\n state.set_xattr(file=files_0, flag=1, name='\\x9d', user='root', value=value)\n state.dump_load_dump(folders_0)\n state.teardown()\n\n def skip_test_info(self):\n state = JuicefsCommandMachine()\n folders_0 = state.init_folders()\n files_2 = state.create_file(content='0', file_name='mvvd', mode='a', parent=folders_0, umask=293, user='root')\n state.info(entry=folders_0, raw=True, recuisive=True, user='user1')\n state.teardown()\n\n def test_clone(self):\n state = JuicefsCommandMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content='\\x9bcR\\xba', file_name='ygbl', mode='x', parent=v1, umask=466, user='root')\n state.chmod(entry=v1, mode=715, user='root')\n state.clone(entry=v2, new_entry_name='drqj', parent=v1, preserve=False, user='user1')\n state.teardown()\n\n def test_config(self):\n state = JuicefsCommandMachine()\n folders_0 = state.init_folders()\n state.config(capacity=1, enable_acl=True, encrypt_secret=True, force=False, inodes=81, trash_days=0, user='root', yes=True)\n state.teardown()\n\n def test_clone_4834(self):\n #SEE https://github.com/juicedata/juicefs/issues/4834\n state = JuicefsCommandMachine()\n folders_0 = state.init_folders()\n state.chmod(entry=folders_0, mode=2427, user='root')\n folders_1 = state.mkdir(mode=2931, parent=folders_0, subdir='vhjp', umask=369, user='root')\n state.chmod(entry=folders_1, mode=1263, user='root')\n state.clone(entry=folders_1, new_entry_name='tbim', parent=folders_0, preserve=False, user='user1')\n state.teardown()\n\nif __name__ == '__main__':\n unittest.main()" }, { "path": ".github/scripts/hypo/common.py", "content": "\nimport grp\nimport json\nimport logging\nimport os\nimport pwd\nimport subprocess\nimport sys\nimport stat\ndef red(s):\n return f'\\033[31m{s}\\033[0m'\n\ndef replace(src, old, new):\n if isinstance(src, str):\n return src.replace(old, new)\n elif isinstance(src, list) or isinstance(src, tuple):\n return [replace(x, old, new) for x in src]\n elif isinstance(src, dict):\n return {k: replace(v, old, new) for k, v in src.items()}\n else:\n return src\ndef run_cmd(command: str) -> str:\n print('run_cmd:'+command)\n if '|' in command or '>' in command:\n ret=os.system(command)\n if ret == 0:\n return ret\n else: \n raise Exception(f\"run command {command} failed with {ret}\")\n try:\n output = subprocess.run(command.split(), check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as e:\n # print(f': subprocess run error: {e.output.decode()}')\n raise e\n # print(output.stdout.decode())\n # print('run_cmd succeed')\n return output.stdout.decode()\n\n\ndef setup_logger(log_file_path, logger_name, log_level='INFO'):\n if log_level == 'DEBUG':\n log_level = logging.DEBUG\n elif log_level == 'INFO':\n log_level = logging.INFO\n elif log_level == 'WARNING':\n log_level = logging.WARNING\n elif log_level == 'ERROR':\n log_level = logging.ERROR\n # Create a logger object\n assert os.path.exists(os.path.dirname(log_file_path)), red(f'setup_logger: {log_file_path} should exist')\n print(f'setup_logger {log_file_path}')\n logger = logging.getLogger(logger_name)\n logger.setLevel(logging.DEBUG)\n # Create a file handler for the logger\n file_handler = logging.FileHandler(log_file_path)\n file_handler.setLevel(logging.DEBUG)\n # Create a stream handler for the logger\n stream_handler = logging.StreamHandler()\n stream_handler.setLevel(log_level)\n # Create a formatter for the log messages\n formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n file_handler.setFormatter(formatter)\n stream_handler.setFormatter(formatter)\n # Add the file and stream handlers to the logger\n logger.addHandler(file_handler)\n logger.addHandler(stream_handler)\n return logger\n\ndef is_jfs(path):\n root = get_root(path)\n file = os.path.join(root, '.jfsconfig')\n return os.path.isfile( file )\n\ndef get_root(path):\n path = os.path.abspath(path)\n d = path if os.path.isdir(path) else os.path.dirname(path)\n while d != '/':\n try:\n st = os.stat(d)\n if st.st_ino == 1:\n return d\n except:\n pass\n d = os.path.dirname(d)\n return d\n\ndef get_volume_name(path):\n root = get_root(path)\n file = os.path.join(root, '.config')\n if os.path.isfile(file):\n with open(file, 'r') as f:\n config = json.load(f)\n try :\n return config['Meta']['Volume']\n except KeyError:\n return config['Format']['Name']\n\ndef get_zones(dir):\n zones = []\n root = get_root(dir)\n for i in range(0, 8):\n try:\n zone = os.path.join(root, f'.jfszone{i}')\n os.stat(zone)\n zones.append(f'.jfszone{i}')\n except Exception as e:\n # print(f'zone {zone} not exist, {str(e)}')\n pass\n if len(zones) > 0:\n return zones\n else:\n return [''] \n \ndef get_acl(abspath: str):\n s = run_cmd(f'getfacl {abspath}')\n lines = s.split('\\n')\n # s = s.replace(\"# file: \", \"# file: /\")\n lines = [line for line in lines if not line.startswith(\"# file: \")]\n s = '\\n'.join(lines)\n return s\n\ndef support_acl(path):\n root = get_root(path)\n file = os.path.join(root, '.config')\n if os.path.isfile(file):\n with open(file, 'r') as f:\n config = json.load(f)\n if config['Meta'].get('Args', '').find('--enable-acl') != -1:\n return True\n elif config['Format'].get('EnableACL', False):\n return True\n else:\n return False\n else:\n mount_point = subprocess.check_output([\"df\", root]).decode(\"utf-8\").splitlines()[-1].split()[0]\n mount_options = subprocess.check_output([\"sudo\", \"tune2fs\", \"-l\", mount_point]).decode(\"utf-8\")\n if \"acl\" not in mount_options:\n return False\n else:\n return True\n\ndef get_stat_field(st: os.stat_result):\n if stat.S_ISREG(st.st_mode):\n return st.st_gid, st.st_uid, st.st_size, oct(st.st_mode), st.st_nlink\n elif stat.S_ISDIR(st.st_mode):\n return st.st_gid, st.st_uid, oct(st.st_mode)\n elif stat.S_ISLNK(st.st_mode):\n return st.st_gid, st.st_uid, oct(st.st_mode)\n else:\n return ()\n \n \ndef create_group(groupname):\n try:\n grp.getgrnam(groupname)\n except KeyError:\n subprocess.run(['groupadd', groupname], check=True)\n print(f\"create Group {groupname}\")\n\ndef create_user(user):\n try:\n pwd.getpwnam(user)\n subprocess.run(['usermod', '-g', user, '-G', '', user], check=True)\n except KeyError:\n subprocess.run(['useradd', '-g', user, '-G', '', user], check=True)\n print(f\"create User {user} with group {user}\")\n\ndef clean_dir(dir):\n try:\n subprocess.check_call(f'rm -rf {dir}'.split())\n assert not os.path.exists(dir), red(f'clean_dir: {dir} should not exist')\n subprocess.check_call(f'mkdir -p {dir}'.split())\n assert os.path.isdir(dir), red(f'clean_dir: {dir} should be dir')\n except subprocess.CalledProcessError as e:\n print(f'clean_dir {dir} failed:{e}, {e.returncode}, {e.output}')\n sys.exit(1)\n\n\ndef compare_content(dir1, dir2):\n os.system('find /tmp/fsrand -type l ! -exec test -e {} \\; -print > broken_symlink.log ')\n exclude_files = []\n with open('broken_symlink.log', 'r') as f:\n lines = f.readlines()\n for line in lines:\n filename = os.path.basename(line.strip())\n exclude_files.append(filename)\n exclude_options = [f'--exclude=\"{item}\"' for item in exclude_files ]\n exclude_options = ' '.join(exclude_options)\n diff_command = f'diff -ur --no-dereference {dir1} {dir2} {exclude_options} 2>&1 |tee diff.log'\n print(diff_command)\n os.system(diff_command)\n with open('diff.log', 'r') as f:\n lines = f.readlines()\n filtered_lines = [line for line in lines if \"recursive directory loop\" not in line]\n assert len(filtered_lines) == 0, red(f'found diff: \\n' + '\\n'.join(filtered_lines))\n\ndef compare_stat(dir1, dir2):\n for root, dirs, files in os.walk(dir1):\n for file in files:\n path1 = os.path.join(root, file)\n path2 = os.path.join(dir2, os.path.relpath(path1, dir1))\n stat1 = get_stat_field(os.stat(path1))\n stat2 = get_stat_field(os.stat(path2))\n assert stat1 == stat2, red(f\"{path1}: {stat1} and {path2}: {stat2} have different stats\")\n for dir in dirs:\n path1 = os.path.join(root, dir)\n path2 = os.path.join(dir2, os.path.relpath(path1, dir1))\n stat1 = get_stat_field(os.stat(path1))\n stat2 = get_stat_field(os.stat(path2))\n assert stat1 == stat2, red(f\"{path1}: {stat1} and {path2}: {stat2} have different stats\")\n\ndef compare_acl(dir1, dir2):\n for root, dirs, files in os.walk(dir1):\n for file in files:\n path1 = os.path.join(root, file)\n path2 = os.path.join(dir2, os.path.relpath(path1, dir1))\n if os.path.exists(path2):\n acl1 = get_acl(path1)\n acl2 = get_acl(path2)\n assert acl1 == acl2, red(f\"{path1}: {acl1} and {path2}: {acl2} have different acl\")\n for dir in dirs:\n path1 = os.path.join(root, dir)\n path2 = os.path.join(dir2, os.path.relpath(path1, dir1))\n if os.path.exists(path2):\n acl1 = get_acl(path1)\n acl2 = get_acl(path2)\n assert acl1 == acl2, red(f\"{path1}: {acl1} and {path2}: {acl2} have different acl\")\n" }, { "path": ".github/scripts/hypo/context.py", "content": "\nclass Context:\n def __init__(self, root_dir:str, mp:str) -> None:\n self.root_dir = root_dir\n self.mp = mp\n self.meta_url = ''\n " }, { "path": ".github/scripts/hypo/file.py", "content": "import os\nimport pwd\nimport re\nimport subprocess\nimport json\nimport common\nfrom common import red\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hypothesis\"])\nfrom hypothesis import assume, strategies as st, settings, Verbosity\nfrom hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes\nfrom hypothesis import Phase, seed\nfrom hypothesis.database import DirectoryBasedExampleDatabase\nfrom strategy import *\nfrom file_op import FileOperation\nimport random\nimport time\n\nSEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))\n\n@seed(SEED)\nclass JuicefsDataMachine(RuleBasedStateMachine):\n FILE_NAME = 'a'\n fds = Bundle('fd')\n mms = Bundle('mm')\n use_sdk = os.environ.get('USE_SDK', 'false').lower() == 'true'\n meta_url = os.environ.get('META_URL')\n INCLUDE_RULES = []\n EXCLUDE_RULES = ['seek']\n if os.environ.get('EXCLUDE_RULES'):\n EXCLUDE_RULES = os.environ.get('EXCLUDE_RULES').split(',')\n # EXCLUDE_RULES = ['readline', 'readlines', 'truncate', 'seek', 'flush']\n ROOT_DIR1=os.environ.get('ROOT_DIR1', '/tmp/fsrand')\n ROOT_DIR2=os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand')\n if use_sdk:\n fsop1 = FileOperation(name='fs1', root_dir=ROOT_DIR1, use_sdk=use_sdk, is_jfs=False, volume_name=None)\n fsop2 = FileOperation(name='fs2', root_dir=ROOT_DIR2, use_sdk=use_sdk, is_jfs=True, volume_name='test-volume', meta_url=meta_url)\n else:\n fsop1 = FileOperation(name='fs1', root_dir=ROOT_DIR1)\n fsop2 = FileOperation(name='fs2', root_dir=ROOT_DIR2)\n\n def __init__(self):\n super(JuicefsDataMachine, self).__init__()\n print(f'__init__')\n\n def equal(self, result1, result2):\n if type(result1) != type(result2):\n return False\n if isinstance(result1, Exception):\n if 'panic:' in str(result1) or 'panic:' in str(result2):\n return False\n result1 = str(result1)\n result2 = str(result2)\n if self.use_sdk:\n result1 = self.parse_error_message(result1)\n result2 = self.parse_error_message(result2)\n result1 = common.replace(result1, self.fsop1.root_dir, '***')\n result2 = common.replace(result2, self.fsop2.root_dir, '***')\n return result1 == result2\n\n def parse_error_message(self, err):\n # extract \"[Errno 22] Invalid argument\" from the following error message\n # [Errno 22] Invalid argument: '/tmp/fsrand/' -> '/tmp/fsrand/izsn/rfnn'\n # [Errno 22] Invalid argument: (b'/fsrand', b'/fsrand/izsn/rfnn', c_uint(0))\n match = re.search(r\"\\[Errno \\d+\\] [^:]+\", err)\n if match:\n return match.group(0)\n else:\n return err\n\n def should_run(self, rule):\n if len(self.EXCLUDE_RULES) > 0:\n return rule not in self.EXCLUDE_RULES\n else:\n return rule in self.INCLUDE_RULES\n\n @initialize(target = fds)\n def init_folders(self):\n self.fsop1.init_rootdir()\n self.fsop2.init_rootdir()\n f1, _ = self.fsop1.do_open(file=self.FILE_NAME, mode='w+', encoding='utf8', errors='strict')\n f2, _ = self.fsop2.do_open(file=self.FILE_NAME, mode='w+', encoding='utf8', errors='strict')\n assert f1 is not None and f2 is not None, red(f'init_folders:\\nf1 is {f1}\\nf2 is {f2}')\n return (self.FILE_NAME, f1, f2)\n\n \n @rule( fd = fds.filter(lambda x: x != multiple()), \n length = st.integers(min_value=0, max_value=MAX_FILE_SIZE))\n @precondition(lambda self: self.should_run('read'))\n def read(self, fd, length):\n result1 = self.fsop1.do_read(fd=fd[1], file=fd[0], length=length)\n result2 = self.fsop2.do_read(fd=fd[2], file=fd[0], length=length)\n assert self.equal(result1, result2), red(f'read:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(\n fd = fds.filter(lambda x: x != multiple()), \n content = st_content,\n )\n @precondition(lambda self: self.should_run('write'))\n def write(self, fd, content):\n result1 = self.fsop1.do_write(fd=fd[1], file=fd[0], content=content)\n result2 = self.fsop2.do_write(fd=fd[2], file=fd[0], content=content)\n assert self.equal(result1, result2), red(f'write:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(fd = fds.filter(lambda x: x != multiple()), \n lines = st_lines,\n )\n @precondition(lambda self: self.should_run('writelines'))\n def writelines(self, fd, lines):\n result1 = self.fsop1.do_writelines(fd=fd[1], file=fd[0], lines=lines)\n result2 = self.fsop2.do_writelines(fd=fd[2], file=fd[0], lines=lines)\n assert self.equal(result1, result2), red(f'write:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule(fd = fds.filter(lambda x: x != multiple()), \n offset = st_offset, \n whence = st_whence\n )\n @precondition(lambda self: self.should_run('seek'))\n def seek(self, fd, offset, whence):\n result1 = self.fsop1.do_seek(fd=fd[1], file=fd[0], offset=offset, whence=whence)\n result2 = self.fsop2.do_seek(fd=fd[2], file=fd[0], offset=offset, whence=whence)\n assert self.equal(result1, result2), red(f'seek:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(fd = fds.filter(lambda x: x != multiple())) \n @precondition(lambda self: self.should_run('tell'))\n def tell(self, fd):\n result1 = self.fsop1.do_tell(fd=fd[1], file=fd[0])\n result2 = self.fsop2.do_tell(fd=fd[2], file=fd[0])\n assert self.equal(result1, result2), red(f'tell:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule(\n target = fds, \n fd = consumes(fds).filter(lambda x: x != multiple()))\n @precondition(lambda self: self.should_run('close'))\n def close(self, fd):\n result1 = self.fsop1.do_close(fd=fd[1], file=fd[0])\n result2 = self.fsop2.do_close(fd=fd[2], file=fd[0])\n assert self.equal(result1, result2), red(f'close:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return fd\n else:\n return multiple()\n @rule(fd = fds.filter(lambda x: x != multiple()))\n @precondition(lambda self: self.should_run('flush_and_fsync'))\n def flush_and_fsync(self, fd):\n result1 = self.fsop1.do_flush_and_fsync(fd=fd[1], file=fd[0])\n result2 = self.fsop2.do_flush_and_fsync(fd=fd[2], file=fd[0])\n assert self.equal(result1, result2), red(f'flush:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(fd = fds.filter(lambda x: x != multiple()),\n offset = st_offset,\n length = st_fallocate_length,\n )\n @precondition(lambda self: self.should_run('fallocate') and not self.use_sdk)\n def fallocate(self, fd, offset, length):\n result1 = self.fsop1.do_fallocate(fd=fd[1], file=fd[0], offset=offset, length=length)\n result2 = self.fsop2.do_fallocate(fd=fd[2], file=fd[0], offset=offset, length=length)\n assert self.equal(result1, result2), red(f'fallocate:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule( fd = fds.filter(lambda x: x != multiple()))\n @precondition(lambda self: self.should_run('readlines'))\n def readlines(self, fd):\n result1 = self.fsop1.do_readlines(fd=fd[1], file=fd[0])\n result2 = self.fsop2.do_readlines(fd=fd[2], file=fd[0])\n assert self.equal(result1, result2), red(f'readlines:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule( fd = fds.filter(lambda x: x != multiple()))\n @precondition(lambda self: self.should_run('readline'))\n def readline(self, fd):\n result1 = self.fsop1.do_readline(fd=fd[1], file=fd[0])\n result2 = self.fsop2.do_readline(fd=fd[2], file=fd[0])\n assert self.equal(result1, result2), red(f'readline:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n\n @rule(fd=fds.filter(lambda x: x != multiple()), \n size=st_truncate_length, \n )\n @precondition(lambda self: self.should_run('truncate'))\n def truncate(self, fd, size):\n result1 = self.fsop1.do_truncate(fd=fd[1], file=fd[0], size=size)\n result2 = self.fsop2.do_truncate(fd=fd[2], file=fd[0], size=size)\n assert self.equal(result1, result2), red(f'truncate:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(\n src=fds.filter(lambda x: x != multiple()),\n dst=fds.filter(lambda x: x != multiple()),\n src_offset = st_offset,\n dst_offset = st_offset,\n length = st_length,\n )\n @precondition(lambda self: self.should_run('copy_file_range') and not self.use_sdk)\n def copy_file_range(self, src, dst, src_offset, dst_offset, length):\n result1 = self.fsop1.do_copy_file_range(src_file=src[0], dst_file=dst[0], src_fd=src[1], dst_fd=dst[1], src_offset=src_offset, dst_offset=dst_offset, length=length)\n result2 = self.fsop2.do_copy_file_range(src_file=src[0], dst_file=dst[0], src_fd=src[2], dst_fd=dst[2], src_offset=src_offset, dst_offset=dst_offset, length=length)\n assert self.equal(result1, result2), red(f'copy_file_range:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n def teardown(self):\n pass\n \nif __name__ == '__main__':\n MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))\n STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))\n ci_db = DirectoryBasedExampleDatabase(\".hypothesis/examples\") \n settings.register_profile(\"dev\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])\n settings.register_profile(\"schedule\", max_examples=1000, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=200, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target], \n database=ci_db)\n settings.register_profile(\"pull_request\", max_examples=100, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=50, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target], \n database=ci_db)\n settings.register_profile(\"generate\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \\\n phases=[Phase.generate, Phase.target])\n \n if os.environ.get('CI'):\n event_name = os.environ.get('GITHUB_EVENT_NAME')\n if event_name == 'schedule':\n profile = 'schedule'\n else:\n profile = 'pull_request'\n else:\n profile = os.environ.get('PROFILE', 'dev')\n print(f'profile is {profile}')\n settings.load_profile(profile)\n juicefs_machine = JuicefsDataMachine.TestCase()\n juicefs_machine.runTest()\n print(json.dumps(FileOperation.stats.get(), sort_keys=True, indent=4))" }, { "path": ".github/scripts/hypo/file_op.py", "content": "import hashlib\nimport io\nimport mmap\nimport os\nimport pwd\nimport re\nimport shutil\nimport stat\nimport subprocess\n\ntry: \n __import__('xattr')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\nimport xattr\nfrom common import get_acl, get_root, red\nfrom typing import Dict\ntry: \n __import__('fallocate')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"fallocate\"])\nimport fallocate\nfrom stats import Statistics\nimport common\nfrom os.path import dirname\nimport sys\nsys.path.append('.')\nfrom sdk.python.juicefs.juicefs import juicefs\n\nclass FileOperation:\n JFS_CONTROL_FILES=['.accesslog', '.config', '.stats']\n stats = Statistics()\n Files = {}\n \n def __init__(self, name, root_dir:str, mount_point=None, use_sdk:bool=False, is_jfs=False, volume_name=None, meta_url=None):\n self.logger =common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO'))\n self.root_dir = root_dir.rstrip('/')\n self.use_sdk = use_sdk\n self.is_jfs = is_jfs\n if mount_point:\n self.mount_point = mount_point\n else:\n self.mount_point = common.get_root(self.root_dir)\n self.client = None\n if use_sdk and self.is_jfs:\n if meta_url:\n self.client = juicefs.Client(volume_name, meta=meta_url, access_log=\"/tmp/jfs.log\")\n else:\n self.client = juicefs.Client(volume_name, conf_dir='deploy/docker', access_log=\"/tmp/jfs.log\")\n\n def run_cmd(self, command:str) -> str:\n self.logger.info(f'run_cmd: {command}')\n if '|' in command or '>' in command or '&' in command:\n ret=os.system(command)\n if ret == 0:\n return ret\n else: \n raise Exception(f\"run command {command} failed with {ret}\")\n try:\n output = subprocess.run(command.split(), check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as e:\n raise e\n return output.stdout.decode()\n\n def get_zones(self):\n return common.get_zones(self.root_dir)\n\n def init_rootdir(self):\n if self.client:\n self.logger.debug(f'init_rootdir {self.root_dir} with use_sdk={self.use_sdk}')\n sdk_root_dir = self.get_sdk_path(self.root_dir)\n if self.client.exists(sdk_root_dir):\n self.client.rmr(sdk_root_dir)\n assert not self.client.exists(sdk_root_dir), red(f'{self.root_dir} should not exist')\n self.client.makedirs(sdk_root_dir)\n assert self.client.exists(sdk_root_dir), red(f'{self.root_dir} should exist')\n else:\n if not os.path.exists(self.root_dir):\n os.makedirs(self.root_dir)\n if os.environ.get('PROFILE', 'dev') != 'generate':\n common.clean_dir(self.root_dir)\n \n def handleException(self, e, action, path, **kwargs):\n if isinstance(e, subprocess.CalledProcessError):\n err = e.output.decode()\n else:\n err = type(e).__name__ + \":\" + str(e)\n err = '\\n'.join([elem.split(':')[-1].split(':')[-1] for elem in err.split('\\n')])\n err = re.sub(r'\\[\\w+\\.go:\\d+\\]', '', err)\n if err.find('setfacl') != -1 and err.find('\\n') != -1:\n err = '\\n'.join(sorted(err.split('\\n')))\n err = self.parse_pysdk_error(err)\n self.stats.failure(action)\n self.logger.info(f'{action} {path} {kwargs} failed: {err}')\n return Exception(err)\n \n def parse_pysdk_error(self, err:str):\n # error message : call jfs_rename failed: [Errno 22] Invalid argument: (b'/fsrand', b'/fsrand/izsn/rfnn', c_uint(0))\n if not err.startswith(\"call jfs_\"):\n return err\n return re.sub(r'call jfs_\\w+ failed: ', '', err)\n \n def get_sdk_path(self, abspath):\n return '/'+os.path.relpath(abspath, self.mount_point)\n\n def do_stat(self, entry):\n self.logger.debug(f'do_stat {self.root_dir} {entry}')\n abspath = os.path.join(self.root_dir, entry)\n try:\n if self.client:\n st = self.client.stat(self.get_sdk_path(abspath))\n else:\n st = os.stat(abspath)\n except Exception as e :\n return self.handleException(e, 'do_stat', abspath, entry=entry)\n finally:\n pass\n self.stats.success('do_stat')\n self.logger.info(f'do_stat {abspath} with succeed')\n self.logger.debug(f'do_stat st is {st}')\n return common.get_stat_field(st)\n \n def do_create_file(self, file, content, mode, encoding, errors):\n self.logger.debug(f'do_create_file {self.root_dir} {file} {mode}')\n abspath = os.path.join(self.root_dir, file)\n f = None\n try:\n if self.client:\n f = self.client.open(self.get_sdk_path(abspath), mode=mode, encoding=encoding, errors=errors) \n else:\n f = open(abspath, mode=mode, encoding=encoding, errors=errors)\n f.write(content)\n f.flush()\n except Exception as e :\n return f, self.handleException(e, 'do_create_file', abspath, mode=mode)\n finally:\n pass\n self.stats.success('do_create_file')\n self.logger.info(f'do_create_file {abspath} {mode} succeed')\n return f, 'succeed'\n\n def do_open(self, file, mode, encoding, errors):\n self.logger.debug(f'do_open {self.root_dir} {file} {mode}')\n abspath = os.path.join(self.root_dir, file)\n f = None\n try:\n if self.client:\n f = self.client.open(self.get_sdk_path(abspath), mode=mode, encoding=encoding, errors=errors) \n else:\n f = open(abspath, mode=mode, encoding=encoding, errors=errors)\n except Exception as e :\n return f, self.handleException(e, 'do_open', abspath, mode=mode)\n finally:\n pass\n self.stats.success('do_open')\n self.logger.info(f'do_open {abspath} {mode} succeed')\n return f, 'succeed'\n\n def do_write(self, fd, file, content):\n self.logger.debug(f'do_write {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n fd.write(content)\n else:\n fd.write(content)\n except (io.UnsupportedOperation) as e:\n e = Exception(f'io.UnsupportedOperation: write')\n return self.handleException(e, 'do_write', abspath)\n except Exception as e :\n return self.handleException(e, 'do_write', abspath)\n finally:\n pass\n self.stats.success('do_write')\n self.logger.info(f'do_write {abspath} succeed')\n return 'succeed'\n\n def do_writelines(self, fd, file, lines):\n self.logger.debug(f'do_writelines {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n fd.writelines(lines)\n else:\n fd.writelines(lines)\n except (TypeError,io.UnsupportedOperation) as e:\n self.logger.debug(f'writelines: {str(e)}')\n e = Exception(f'writelines')\n return self.handleException(e, 'do_writelines', abspath)\n except Exception as e :\n return self.handleException(e, 'do_writelines', abspath)\n finally:\n pass\n self.stats.success('do_writelines')\n self.logger.info(f'do_writelines {abspath} succeed')\n return 'succeed'\n\n\n def do_seek(self, fd, file, offset, whence):\n self.logger.debug(f'do_seek {self.root_dir} file={file} offset={offset} whence={whence}')\n abspath = os.path.join(self.root_dir, file)\n try:\n pos = fd.seek(offset, whence)\n except Exception as e :\n return self.handleException(e, 'do_seek', abspath, offset=offset, whence=whence)\n finally:\n pass\n self.stats.success('do_seek')\n self.logger.info(f'do_seek {abspath} offset={offset} whence={whence} succeed, pos={pos}')\n return pos\n\n def do_tell(self, fd, file):\n self.logger.debug(f'do_tell {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n offset = fd.tell()\n except Exception as e :\n return self.handleException(e, 'do_tell', abspath)\n finally:\n pass\n self.stats.success('do_tell')\n self.logger.info(f'do_tell {abspath} succeed, offset={offset}')\n return offset\n \n def do_close(self, fd, file):\n self.logger.debug(f'do_close {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n fd.close()\n except Exception as e :\n return self.handleException(e, 'do_close', abspath)\n finally:\n pass\n self.stats.success('do_close')\n self.logger.info(f'do_close {abspath} succeed')\n return self.do_stat(file)\n \n def do_flush_and_fsync(self, fd, file):\n self.logger.debug(f'do_flush {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n fd.flush()\n fd.fsync()\n else:\n fd.flush()\n os.fsync(fd.fileno())\n except Exception as e :\n return self.handleException(e, 'do_flush', abspath)\n finally:\n pass\n self.stats.success('do_flush')\n self.logger.info(f'do_flush {abspath} succeed')\n return self.do_stat(file)\n\n def do_fallocate(self, fd, file, offset, length):\n self.logger.debug(f'do_fallocate {self.root_dir} {file} {offset} {length}')\n abspath = os.path.join(self.root_dir, file)\n try:\n file_size = os.stat(abspath).st_size\n if file_size == 0:\n offset = 0\n else:\n offset = offset % file_size\n fallocate.fallocate(fd.fileno(), offset, length)\n except Exception as e :\n return self.handleException(e, 'do_fallocate', abspath, offset=offset, length=length)\n finally:\n pass\n self.stats.success('do_fallocate')\n self.logger.info(f'do_fallocate {abspath} offset={offset} length={length} succeed')\n return self.do_stat(file)\n \n\n def do_read(self, fd, file, length):\n self.logger.debug(f'do_read {self.root_dir} {file} {length}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n result = fd.read(length) \n else:\n result = fd.read(length)\n if isinstance(result, str):\n result = result.replace('\\r', '\\n') # SEE: https://github.com/juicedata/jfs/issues/1472\n result = result.encode()\n self.logger.debug(f'do_read result is {result}')\n result = hashlib.md5(result).hexdigest()\n except io.UnsupportedOperation as e:\n e = Exception(f'io.UnsupportedOperation: read')\n return self.handleException(e, 'do_read', abspath, length=length)\n except Exception as e :\n return self.handleException(e, 'do_read', abspath, length=length)\n finally:\n pass\n self.stats.success('do_read')\n self.logger.info(f'do_read {abspath} length={length} succeed')\n return (result, )\n\n def do_readlines(self, fd, file):\n self.logger.debug(f'do_readlines {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n result = ''.join(fd.readlines())\n else:\n result = ''.join(fd.readlines())\n if isinstance(result, str):\n result = result.replace('\\r', '\\n') # SEE: https://github.com/juicedata/jfs/issues/1472\n result = result.encode()\n self.logger.debug(f'do_readlines result is {result}')\n result = hashlib.md5(result).hexdigest()\n except UnicodeDecodeError as e:\n # SEE: https://github.com/juicedata/jfs/issues/1450#issuecomment-2213518638\n self.logger.debug(f'UnicodeDecodeError: {e.encoding} {e.object} {e.start} {e.end} {e.reason}')\n e = UnicodeDecodeError(e.encoding, e.object, 0, 0, e.reason)\n return self.handleException(e, 'do_readlines', abspath)\n except io.UnsupportedOperation as e:\n e = Exception(f'io.UnsupportedOperation: readlines')\n return self.handleException(e, 'do_readlines', abspath)\n except Exception as e :\n return self.handleException(e, 'do_readlines', abspath)\n finally:\n pass\n self.stats.success('do_readlines')\n self.logger.info(f'do_readlines {abspath} succeed')\n return (result, )\n\n def do_readline(self, fd, file):\n self.logger.debug(f'do_readline {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n result = fd.readline()\n else:\n result = fd.readline()\n if isinstance(result, str):\n result = result.replace('\\r', '\\n') # SEE: https://github.com/juicedata/jfs/issues/1472\n result = result.encode()\n self.logger.debug(f'do_readline result is {result}')\n result = hashlib.md5(result).hexdigest()\n except UnicodeDecodeError as e:\n # SEE: https://github.com/juicedata/jfs/issues/1450#issuecomment-2213518638\n self.logger.debug(f'UnicodeDecodeError: {e.encoding} {e.object} {e.start} {e.end} {e.reason}')\n e = UnicodeDecodeError(e.encoding, e.object, 0, 0, e.reason)\n return self.handleException(e, 'do_readline', abspath)\n except io.UnsupportedOperation as e:\n e = Exception(f'io.UnsupportedOperation: readline')\n return self.handleException(e, 'do_readline', abspath)\n except Exception as e :\n return self.handleException(e, 'do_readline', abspath)\n finally:\n pass\n\n self.stats.success('do_readline')\n self.logger.info(f'do_readline {abspath} succeed')\n return (result, )\n\n def do_truncate(self, fd, file, size):\n self.logger.debug(f'do_truncate {self.root_dir} {file} {size}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n fd.flush()\n fd.truncate(size)\n st = self.client.stat(self.get_sdk_path(abspath))\n else:\n fd.flush()\n os.ftruncate(fd.fileno(), size)\n st = os.stat(abspath)\n except Exception as e :\n return self.handleException(e, 'do_truncate', abspath, size=size)\n finally:\n pass\n assert st.st_size == size, red(f'do_truncate: {abspath} size should be {size} but {st.st_size}')\n self.stats.success('do_truncate')\n self.logger.info(f'do_truncate {abspath} size={size} succeed')\n return 'succeed'\n\n def do_copy_file_range(self, src_file, dst_file, src_fd, dst_fd, src_offset, dst_offset, length):\n self.logger.debug(f'do_copy_file_range from {self.root_dir}/{src_file} to {self.root_dir}/{dst_file} {src_offset} {dst_offset} {length}')\n src_abspath = os.path.join(self.root_dir, src_file)\n dst_abspath = os.path.join(self.root_dir, dst_file)\n try:\n os.copy_file_range(src_fd, dst_fd, length, src_offset, dst_offset)\n except Exception as e :\n return self.handleException(e, 'do_copy_file_range', src_abspath, src_offset=src_offset, dst_offset=dst_offset, length=length)\n finally:\n pass\n self.stats.success('do_copy_file_range')\n self.logger.info(f'do_copy_file_range {src_abspath} to {dst_abspath} {src_offset} {dst_offset} {length} succeed')\n return os.stat(dst_abspath).st_size\n\n def do_mmap_create(self, file, fd):\n self.logger.debug(f'do_mmap_create {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n mm = mmap.mmap(fd.fileno(), 0)\n except Exception as e :\n return None, self.handleException(e, 'do_mmap_create', abspath)\n finally:\n pass\n self.stats.success('do_mmap_create')\n self.logger.info(f'do_mmap_create {abspath} succeed')\n return mm, len(mm)\n\n def do_mmap_read(self, file, mm: mmap.mmap, length):\n self.logger.debug(f'do_mmap_read {self.root_dir} {file} {length}')\n abspath = os.path.join(self.root_dir, file)\n try:\n length = length % mm.size()\n result = mm.read(length)\n except Exception as e :\n return self.handleException(e, 'do_mmap_read', abspath, length=length)\n finally:\n pass\n self.stats.success('do_mmap_read')\n self.logger.info(f'do_mmap_read {abspath} {length} succeed')\n return result\n\n def do_mmap_read_byte(self, file, mm:mmap.mmap):\n self.logger.debug(f'do_mmap_read_byte {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n result = mm.read_byte()\n except Exception as e :\n return self.handleException(e, 'do_mmap_read_byte', abspath)\n finally:\n pass\n self.stats.success('do_mmap_read_byte')\n self.logger.info(f'do_mmap_read_byte {abspath} succeed')\n return result\n \n def do_mmap_read_line(self, file, mm:mmap.mmap):\n self.logger.debug(f'do_mmap_read_line {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n result = mm.readline()\n except Exception as e :\n return self.handleException(e, 'do_mmap_read_line', abspath)\n finally:\n pass\n self.stats.success('do_mmap_read_line')\n self.logger.info(f'do_mmap_read_line {abspath} succeed')\n return result\n\n def do_mmap_write(self, file, mm:mmap.mmap, content):\n self.logger.debug(f'do_mmap_write {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n mm.write(content)\n except Exception as e :\n return self.handleException(e, 'do_mmap_write', abspath)\n finally:\n pass\n self.stats.success('do_mmap_write')\n self.logger.info(f'do_mmap_write {abspath} succeed')\n return mm.size(), mm.tell()\n\n def do_mmap_write_byte(self, file, mm: mmap.mmap, byte):\n self.logger.debug(f'do_mmap_write_byte {self.root_dir}')\n abspath = os.path.join(self.root_dir, file)\n try:\n mm.write_byte(byte)\n except Exception as e :\n return self.handleException(e, 'do_mmap_write_byte', abspath)\n finally:\n pass\n self.stats.success('do_mmap_write_byte')\n self.logger.info(f'do_mmap_write_byte {abspath} succeed')\n return 'succeed'\n\n def do_mmap_move(self, file, mm: mmap.mmap, dest, src, count):\n self.logger.debug(f'do_mmap_move {self.root_dir} {file} {dest} {src} {count}')\n abspath = os.path.join(self.root_dir, file)\n try:\n dest = dest % mm.size()\n src = src % mm.size()\n count = count % mm.size()\n mm.move(dest, src, count)\n except Exception as e :\n return self.handleException(e, 'do_mmap_move', abspath, dest=dest, src=src, count=count)\n finally:\n pass\n self.stats.success('do_mmap_move')\n self.logger.info(f'do_mmap_move {abspath} {dest} {src} {count} succeed')\n return mm.size(), mm.tell()\n\n def do_mmap_resize(self, file, mm: mmap.mmap):\n self.logger.debug(f'do_mmap_resize {self.root_dir}')\n abspath = os.path.join(self.root_dir, file)\n try:\n if self.client:\n newsize = self.client.stat(self.get_sdk_path(abspath)).st_size\n else:\n newsize = os.stat(abspath).st_size\n mm.resize(newsize)\n except Exception as e :\n return self.handleException(e, 'do_mmap_resize', self.root_dir)\n finally:\n pass\n self.stats.success('do_mmap_resize')\n self.logger.info(f'do_mmap_resize succeed')\n return mm.size()\n\n def do_mmap_seek(self, file, mm: mmap.mmap, offset, whence):\n self.logger.debug(f'do_mmap_seek {self.root_dir} {file} {offset} {whence}')\n abspath = os.path.join(self.root_dir, file)\n try:\n assert mm.size() != 0, red(f'do_mmap_seek size should not be 0')\n offset = offset % mm.size()\n mm.seek(offset, whence)\n pos = mm.tell()\n except Exception as e :\n return self.handleException(e, 'do_mmap_seek', abspath, offset=offset, whence=whence)\n finally:\n pass\n self.stats.success('do_mmap_seek')\n self.logger.info(f'do_mmap_seek {abspath} {offset} {whence} succeed')\n return pos\n \n def do_mmap_size(self, file, mm: mmap.mmap):\n self.logger.debug(f'do_mmap_size {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n size = mm.size()\n except Exception as e :\n return self.handleException(e, 'do_mmap_size', abspath)\n finally:\n pass\n self.stats.success('do_mmap_size')\n self.logger.info(f'do_mmap_size {abspath} succeed')\n return size\n\n def do_mmap_tell(self, file, mm: mmap.mmap):\n self.logger.debug(f'do_mmap_tell {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n pos = mm.tell()\n except Exception as e :\n return self.handleException(e, 'do_mmap_tell', abspath)\n finally:\n pass\n self.stats.success('do_mmap_tell')\n self.logger.info(f'do_mmap_tell {abspath} succeed')\n return pos\n\n def do_mmap_flush(self, file, mm: mmap.mmap):\n self.logger.debug(f'do_mmap_flush {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n mm.flush()\n except Exception as e :\n return self.handleException(e, 'do_mmap_flush', abspath)\n finally:\n pass\n self.stats.success('do_mmap_flush')\n self.logger.info(f'do_mmap_flush {abspath} succeed')\n return 'succeed'\n \n def do_mmap_close(self, file, mm: mmap.mmap):\n self.logger.debug(f'do_mmap_close {self.root_dir} {file}')\n abspath = os.path.join(self.root_dir, file)\n try:\n mm.close()\n except Exception as e :\n return self.handleException(e, 'do_mmap_close', abspath)\n finally:\n pass\n self.stats.success('do_mmap_close')\n self.logger.info(f'do_mmap_close {abspath} succeed')\n return 'succeed'" }, { "path": ".github/scripts/hypo/file_test.py", "content": "import unittest\nfrom file import JuicefsDataMachine\n\nclass TestPySdk(unittest.TestCase):\n def test_issue_1522_1(self):\n # SEE https://github.com/juicedata/jfs/issues/1522\n state = JuicefsDataMachine()\n v1 = state.init_folders()\n state.write(fd=v1, content='abc')\n state.seek(fd=v1, offset=0, whence=1)\n state.teardown()\n\n def test_issue_1522_2(self):\n # SEE https://github.com/juicedata/jfs/issues/1522\n state = JuicefsDataMachine()\n v1 = state.init_folders()\n state.seek(fd=v1, offset=1, whence=0)\n state.write(fd=v1, content='')\n state.seek(fd=v1, offset=0, whence=2)\n state.teardown()\n\n def test_issue_1523(self):\n # SEE https://github.com/juicedata/jfs/issues/1523\n state = JuicefsDataMachine()\n v1 = state.init_folders()\n state.truncate(fd=v1, size=1)\n state.readline(fd=v1)\n state.teardown()\n\n def skip_test_issue_1533(self):\n # SEE https://github.com/juicedata/jfs/issues/1533\n state = JuicefsDataMachine()\n v1 = state.init_folders()\n state.write(fd=v1, content='ab')\n state.seek(fd=v1, offset=0, whence=0)\n state.read(fd=v1, length=1)\n state.write(content='', fd=v1)\n state.read(fd=v1, length=1)\n state.teardown()\n\n def skip_test_issue_1548(self):\n # SEE https://github.com/juicedata/jfs/issues/1548\n state = JuicefsDataMachine()\n fd_0 = state.init_folders()\n state.write(fd=fd_0, content='a')\n state.seek(fd=fd_0, offset=0, whence=0)\n state.write(fd=fd_0, content='b')\n state.read(fd=fd_0, length=1)\n state.teardown()\n\n def skip_test_issue_1548_2(self):\n # SEE https://github.com/juicedata/jfs/issues/1548\n state = JuicefsDataMachine()\n fd_0 = state.init_folders()\n state.truncate(fd=fd_0, size=3)\n state.write(content='a', fd=fd_0)\n state.readline(fd=fd_0)\n state.teardown()\n\nif __name__ == '__main__':\n unittest.main()\n\n\n" }, { "path": ".github/scripts/hypo/fs.py", "content": "import os\nimport pwd\nimport re\nimport subprocess\nimport json\nimport common\nfrom common import red\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hypothesis\"])\nfrom hypothesis import assume, strategies as st, settings, Verbosity\nfrom hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes\nfrom hypothesis import Phase, seed\nfrom hypothesis.database import DirectoryBasedExampleDatabase\nfrom strategy import *\nfrom fs_op import FsOperation\nimport random\nimport time\n\nSEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))\n\n@seed(SEED)\nclass JuicefsMachine(RuleBasedStateMachine):\n Files = Bundle('files')\n Folders = Bundle('folders')\n Entries = Files | Folders\n EntryWithACL = Bundle('entry_with_acl')\n Xattrs = Bundle('xattrs')\n start = time.time()\n use_sdk = os.environ.get('USE_SDK', 'false').lower() == 'true'\n meta_url = os.environ.get('META_URL')\n SUDO_USERS = ['root']\n if use_sdk:\n SUDO_USERS = ['root']\n if os.uname().sysname == 'Darwin':\n USERS=['root']\n GROUPS = ['root']\n else:\n USERS=['root', 'user1', 'user2','user3']\n GROUPS = USERS+['group1', 'group2', 'group3', 'group4']\n group_created = False\n INCLUDE_RULES = []\n if os.getenv('EXCLUDE_RULES'):\n EXCLUDE_RULES = os.getenv('EXCLUDE_RULES').split(',')\n else:\n EXCLUDE_RULES = ['readlines', 'readline']\n # EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', 'clone_cp_file', 'clone_cp_dir', 'loop_symlink', 'hardlink', 'rename_dir', 'chown']\n ROOT_DIR1=os.environ.get('ROOT_DIR1', '/tmp/fsrand')\n ROOT_DIR2=os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand')\n if use_sdk:\n fsop1 = FsOperation(name='fs1', root_dir=ROOT_DIR1, use_sdk=use_sdk, is_jfs=False, volume_name=None)\n fsop2 = FsOperation(name='fs2', root_dir=ROOT_DIR2, mount_point='/tmp/jfs', use_sdk=use_sdk, is_jfs=True, volume_name='test-volume', meta_url=meta_url)\n else:\n fsop1 = FsOperation(name='fs1', root_dir=ROOT_DIR1, is_jfs=common.is_jfs(ROOT_DIR1))\n fsop2 = FsOperation(name='fs2', root_dir=ROOT_DIR2, is_jfs=common.is_jfs(ROOT_DIR2))\n check_dangling = os.environ.get('CHECK_DANGLING', 'false').lower() == 'true'\n @initialize(target=Folders)\n def init_folders(self):\n self.fsop1.init_rootdir()\n self.fsop2.init_rootdir()\n return ''\n \n def create_users(self, users):\n for user in users:\n if user != 'root':\n common.create_user(user)\n\n def get_default_rootdir1(self):\n return '/tmp/fsrand'\n \n def get_default_rootdir2(self):\n return '/tmp/jfs/fsrand'\n\n def __init__(self):\n super(JuicefsMachine, self).__init__()\n print(f'__init__')\n MAX_RUNTIME=int(os.environ.get('MAX_RUNTIME', '36000'))\n duration = time.time() - self.start\n print(f'duration is {duration}')\n if duration > MAX_RUNTIME:\n raise Exception(f'run out of time: {duration}')\n \n if not self.group_created:\n for group in self.GROUPS:\n if group != 'root':\n common.create_group(group)\n self.group_created = True\n self.create_users(self.USERS)\n self.remove_dangling_files()\n\n def remove_dangling_files(self):\n if self.check_dangling:\n self.fsop1.do_remove_dangling_files()\n self.fsop2.do_remove_dangling_files()\n\n def equal(self, result1, result2):\n if os.getenv('PROFILE', 'dev') == 'generate':\n return True\n if type(result1) != type(result2):\n return False\n # TODO: ignore the diff temp, we should check the difference of result1 and result2 in the future.\n # Ref: https://github.com/juicedata/juicefs/issues/5982\n ignore_diff_errors = os.environ.get('IGNORE_DIFF_ERRORS', 'false').lower() == 'true'\n if ignore_diff_errors and isinstance(result1, Exception) and isinstance(result2, Exception):\n return True\n if isinstance(result1, Exception):\n if 'panic:' in str(result1) or 'panic:' in str(result2):\n return False\n result1 = str(result1)\n result2 = str(result2)\n if self.use_sdk:\n result1 = self.parse_error_message(result1)\n result2 = self.parse_error_message(result2)\n result1 = common.replace(result1, self.fsop1.root_dir, '***')\n result2 = common.replace(result2, self.fsop2.root_dir, '***')\n return result1 == result2\n\n def parse_error_message(self, err):\n # extract \"[Errno 22] Invalid argument\" from the following error message\n # [Errno 22] Invalid argument: '/tmp/fsrand/' -> '/tmp/fsrand/izsn/rfnn'\n # [Errno 22] Invalid argument: (b'/fsrand', b'/fsrand/izsn/rfnn', c_uint(0))\n match = re.search(r\"\\[Errno \\d+\\] [^:]+\", err)\n if match:\n return match.group(0)\n else:\n return err\n\n def seteuid(self, user):\n os.seteuid(pwd.getpwnam(user).pw_uid)\n # os.setegid(pwd.getpwnam(user).pw_gid)\n\n def should_run(self, rule):\n if len(self.EXCLUDE_RULES) > 0:\n return rule not in self.EXCLUDE_RULES\n else:\n return rule in self.INCLUDE_RULES\n\n @rule(\n entry = Entries,\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('stat'))\n def stat(self, entry, user = 'root'):\n result1 = self.fsop1.do_stat(entry=entry, user=user)\n result2 = self.fsop2.do_stat(entry=entry, user=user)\n assert self.equal(result1, result2), red(f'stat:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule(\n entry = Entries,\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('lstat'))\n def lstat(self, entry, user = 'root'):\n result1 = self.fsop1.do_lstat(entry=entry, user=user)\n result2 = self.fsop2.do_lstat(entry=entry, user=user)\n assert self.equal(result1, result2), red(f'lstat:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(\n entry = Entries,\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('exists'))\n def exists(self, entry, user = 'root'):\n result1 = self.fsop1.do_exists(entry=entry, user=user)\n result2 = self.fsop2.do_exists(entry=entry, user=user)\n assert result1 == result2, red(f'exists:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(file = Files.filter(lambda x: x != multiple()), \n flags = st_open_flags, \n umask = st_umask,\n mode = st_entry_mode,\n user = st.sampled_from(SUDO_USERS), \n )\n @precondition(lambda self: self.should_run('open') and not self.use_sdk)\n def open(self, file, flags, mode, user='root', umask=0o022):\n result1 = self.fsop1.do_open(file, flags, umask, mode, user)\n result2 = self.fsop2.do_open(file, flags, umask, mode, user)\n assert self.equal(result1, result2), red(f'open:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule(file = Files.filter(lambda x: x != multiple()), \n mode = st_open_mode, \n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('open') and not self.use_sdk)\n def open2(self, file, mode, user='root'):\n result1 = self.fsop1.do_open2(file=file, mode=mode, user=user)\n result2 = self.fsop2.do_open2(file=file, mode=mode, user=user)\n assert self.equal(result1, result2), red(f'open:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule(file = Files.filter(lambda x: x != multiple()), \n offset = st_offset, \n content = st_content,\n mode = st_open_mode,\n encoding = st_open_encoding, \n errors = st_open_errors,\n whence = st_whence,\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('write'))\n def write(self, file, offset, content, mode, whence, encoding=None, errors=None, user='root'):\n result1 = self.fsop1.do_write(file=file, offset=offset, content=content, mode=mode, encoding=encoding, errors=errors, whence=whence, user=user)\n result2 = self.fsop2.do_write(file=file, offset=offset, content=content, mode=mode, encoding=encoding, errors=errors, whence=whence, user=user)\n assert self.equal(result1, result2), red(f'write:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n # TODO: fix hardcode mode\n @rule(file = Files.filter(lambda x: x != multiple()), \n offset = st_offset, \n lines = st_lines,\n mode = st_open_mode,\n whence = st_whence,\n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('writelines'))\n def writelines(self, file, offset, lines, mode, whence, user='root'):\n result1 = self.fsop1.do_writelines(file=file, offset=offset, lines=lines, mode=mode, whence=whence, user=user)\n result2 = self.fsop2.do_writelines(file=file, offset=offset, lines=lines, mode=mode, whence=whence, user=user)\n assert self.equal(result1, result2), red(f'write:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n\n @rule(file = Files.filter(lambda x: x != multiple()),\n offset = st.integers(min_value=0, max_value=MAX_FILE_SIZE),\n length = st.integers(min_value=0, max_value=MAX_FALLOCATE_LENGTH),\n mode = st.just(0), \n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('fallocate') and not self.use_sdk)\n def fallocate(self, file, offset, length, mode, user='root'):\n result1 = self.fsop1.do_fallocate(file, offset, length, mode, user)\n result2 = self.fsop2.do_fallocate(file, offset, length, mode, user)\n assert self.equal(result1, result2), red(f'fallocate:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(src = Files.filter(lambda x: x != multiple()),\n dst = Files.filter(lambda x: x != multiple()),\n src_offset = st_offset,\n dst_offset = st_offset,\n count = st_length,\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('copy_file_range') and not self.use_sdk)\n def copy_file_range(self, src, dst, src_offset, dst_offset, count, user):\n result1 = self.fsop1.do_copy_file_range(src=src, dst=dst, src_offset=src_offset, dst_offset=dst_offset, count=count, user=user)\n result2 = self.fsop2.do_copy_file_range(src=src, dst=dst, src_offset=src_offset, dst_offset=dst_offset, count=count, user=user)\n assert self.equal(result1, result2), red(f'copy_file_range:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule( file = Files.filter(lambda x: x != multiple()), \n mode = st_open_mode,\n encoding = st_open_encoding,\n errors = st_open_errors,\n offset = st_offset, \n length = st.integers(min_value=0, max_value=MAX_FILE_SIZE), \n whence = st_whence,\n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('read'))\n def read(self, file, mode, offset, length, whence=os.SEEK_CUR, encoding=None, errors=None, user='root'):\n result1 = self.fsop1.do_read(file=file, mode=mode, length=length, offset=offset, whence=whence, user=user, encoding=encoding, errors=errors)\n result2 = self.fsop2.do_read(file=file, mode=mode, length=length, offset=offset, whence=whence, user=user, encoding=encoding, errors=errors)\n assert self.equal(result1, result2), red(f'read:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule( file = Files.filter(lambda x: x != multiple()), \n mode = st_open_mode,\n offset = st_offset, \n whence = st_whence,\n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('readlines'))\n def readlines(self, file, mode, offset, whence=os.SEEK_CUR, user='root'):\n result1 = self.fsop1.do_readlines(file=file, mode=mode, offset=offset, whence=whence, user=user)\n result2 = self.fsop2.do_readlines(file=file, mode=mode, offset=offset, whence=whence, user=user)\n assert self.equal(result1, result2), red(f'readlines:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule( file = Files.filter(lambda x: x != multiple()), \n mode = st_open_mode,\n offset = st_offset, \n whence = st_whence,\n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('readline'))\n def readline(self, file, mode, offset, whence=os.SEEK_CUR, user='root'):\n result1 = self.fsop1.do_readline(file=file, mode=mode, offset=offset, whence=whence, user=user)\n result2 = self.fsop2.do_readline(file=file, mode=mode, offset=offset, whence=whence, user=user)\n assert self.equal(result1, result2), red(f'readline:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n\n @rule(file=Files.filter(lambda x: x != multiple()), \n size=st.integers(min_value=0, max_value=MAX_TRUNCATE_LENGTH), \n user=st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('truncate'))\n def truncate(self, file, size, user='root'):\n result1 = self.fsop1.do_truncate(file=file, size=size, user=user)\n result2 = self.fsop2.do_truncate(file=file, size=size, user=user)\n assert self.equal(result1, result2), red(f'truncate:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule(target=Files, \n parent = Folders.filter(lambda x: x != multiple()), \n file_name = st_file_name, \n content = st_content,\n user = st.sampled_from(SUDO_USERS), \n umask = st_umask)\n @precondition(lambda self: self.should_run('create_file'))\n def create_file(self, parent, file_name, content, mode='xb', buffering=-1, user='root', umask=0o022):\n result1 = self.fsop1.do_create_file(parent=parent, file_name=file_name, mode=mode, buffering=buffering, content=content, user=user, umask=umask)\n result2 = self.fsop2.do_create_file(parent=parent, file_name=file_name, mode=mode, buffering=buffering, content=content, user=user, umask=umask)\n assert self.equal(result1, result2), red(f'create_file:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, file_name)\n\n @rule(dir = Folders.filter(lambda x: x != multiple()), \n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('listdir'))\n def listdir(self, dir, user='root'):\n result1 = self.fsop1.do_listdir(dir=dir, user=user)\n result2 = self.fsop2.do_listdir(dir=dir, user=user)\n assert self.equal(result1, result2), red(f'listdir:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(\n target = Files,\n file = consumes(Files).filter(lambda x: x != multiple()),\n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('unlink'))\n def unlink(self, file, user='root'):\n result1 = self.fsop1.do_unlink(file=file, user=user)\n result2 = self.fsop2.do_unlink(file=file, user=user)\n assert self.equal(result1, result2), red(f'unlink:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return file\n else:\n return multiple()\n \n @rule( target=Files, \n entry = consumes(Files).filter(lambda x: x != multiple()),\n parent = Folders, \n new_entry_name = st_file_name, \n user = st.sampled_from(SUDO_USERS), \n umask = st_umask)\n @precondition(lambda self: self.should_run('rename_file'))\n def rename_file(self, entry, parent, new_entry_name, user='root', umask=0o022):\n result1 = self.fsop1.do_rename(entry=entry, parent=parent, new_entry_name=new_entry_name, user=user, umask=umask)\n result2 = self.fsop2.do_rename(entry=entry, parent=parent, new_entry_name=new_entry_name, user=user, umask=umask)\n assert self.equal(result1, result2), red(f'rename_file:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return entry\n else:\n return os.path.join(parent, new_entry_name)\n \n @rule( target=Folders, \n entry = consumes(Folders).filter(lambda x: x != multiple()), \n parent = Folders, \n new_entry_name = valid_dir_name(),\n user = st.sampled_from(SUDO_USERS),\n umask = st_umask)\n @precondition(lambda self: self.should_run('rename_dir'))\n def rename_dir(self, entry, parent, new_entry_name, user='root', umask=0o022):\n result1 = self.fsop1.do_rename(entry=entry, parent=parent, new_entry_name=new_entry_name, user=user, umask=umask)\n result2 = self.fsop2.do_rename(entry=entry, parent=parent, new_entry_name=new_entry_name, user=user, umask=umask)\n assert self.equal(result1, result2), red(f'rename_dir:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return entry\n else:\n return os.path.join(parent, new_entry_name)\n \n\n @rule( target=Files, entry = Files.filter(lambda x: x != multiple()),\n parent = Folders.filter(lambda x: x != multiple()),\n new_entry_name = st_file_name, \n follow_symlinks = st.booleans(),\n user = st.sampled_from(SUDO_USERS), \n umask = st_umask )\n @precondition(lambda self: self.should_run('copy_file') and not self.use_sdk)\n def copy_file(self, entry, parent, new_entry_name, follow_symlinks, user='root', umask=0o022):\n result1 = self.fsop1.do_copy_file(entry, parent, new_entry_name, follow_symlinks, user, umask)\n result2 = self.fsop2.do_copy_file(entry, parent, new_entry_name, follow_symlinks, user, umask)\n assert self.equal(result1, result2), red(f'copy_file:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, new_entry_name)\n \n @rule( target=Files, entry = Files.filter(lambda x: x != multiple()),\n parent = Folders.filter(lambda x: x != multiple()),\n new_entry_name = st_file_name, \n preserve = st.just(False),\n user = st.sampled_from(SUDO_USERS), \n umask = st_umask )\n @precondition(lambda self: self.should_run('clone_cp_file') \\\n and (self.fsop1.singlezone or self.fsop2.singlezone))\n def clone_cp_file(self, entry, parent, new_entry_name, preserve, user='root', umask=0o022):\n result1 = self.fsop1.do_clone_entry(entry, parent, new_entry_name, preserve, user, umask)\n result2 = self.fsop2.do_clone_entry(entry, parent, new_entry_name, preserve, user, umask)\n assert type(result1) == type(result2), red(f'clone_cp_file:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n assert result1 == result2, red(f'clone_cp_file:\\nresult1 is {result1}\\nresult2 is {result2}')\n return os.path.join(parent, new_entry_name)\n \n @rule( target=Folders, \n entry = Folders.filter(lambda x: x != multiple()),\n parent = Folders.filter(lambda x: x != multiple()),\n new_entry_name = valid_dir_name(), \n preserve = st.just(False),\n user = st.sampled_from(SUDO_USERS), \n umask = st_umask,\n )\n @precondition(lambda self: self.should_run('clone_cp_dir') \\\n and (self.fsop1.singlezone or self.fsop2.singlezone))\n def clone_cp_dir(self, entry, parent, new_entry_name, preserve, user, umask):\n result1 = self.fsop1.do_clone_entry(entry, parent, new_entry_name, preserve, user, umask)\n result2 = self.fsop2.do_clone_entry(entry, parent, new_entry_name, preserve, user, umask)\n assert self.equal(result1, result2), red(f'clone_cp_dir:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n assert result1 == result2, red(f'clone_cp_dir:\\nresult1 is {result1}\\nresult2 is {result2}')\n return os.path.join(parent, new_entry_name)\n\n @rule( target = Folders, \n parent = Folders.filter(lambda x: x != multiple()),\n subdir = valid_dir_name(),\n mode = st_entry_mode,\n user = st.sampled_from(SUDO_USERS), \n umask = st_umask)\n @precondition(lambda self: self.should_run('mkdir'))\n def mkdir(self, parent, subdir, mode, user='root', umask=0o022):\n result1 = self.fsop1.do_mkdir(parent=parent, subdir=subdir, mode=mode, user=user, umask=umask)\n result2 = self.fsop2.do_mkdir(parent=parent, subdir=subdir, mode=mode, user=user, umask=umask)\n assert self.equal(result1, result2), red(f'mkdir:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, subdir)\n\n @rule( target = Folders,\n dir = consumes(Folders).filter(lambda x: x != multiple()),\n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('rmdir'))\n def rmdir(self, dir, user='root'):\n assume(dir != '')\n result1 = self.fsop1.do_rmdir(dir=dir, user=user)\n result2 = self.fsop2.do_rmdir(dir=dir, user=user)\n assert self.equal(result1, result2), red(f'rmdir:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return dir\n else:\n return multiple()\n\n @rule(target = Files, \n src_file = Files.filter(lambda x: x != multiple()), \n parent = Folders.filter(lambda x: x != multiple()), \n link_file_name = st_file_name, \n user = st.sampled_from(SUDO_USERS), \n umask = st_umask)\n @precondition(lambda self: self.should_run('hardlink'))\n def hardlink(self, src_file, parent, link_file_name, user='root', umask=0o022):\n result1 = self.fsop1.do_hardlink(src_file=src_file, parent=parent, link_file_name=link_file_name, user=user, umask=umask)\n result2 = self.fsop2.do_hardlink(src_file=src_file, parent=parent, link_file_name=link_file_name, user=user, umask=umask)\n assert self.equal(result1, result2), red(f'hardlink:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, link_file_name)\n \n @rule(target = Files , \n src_file = Files.filter(lambda x: x != multiple()), \n parent = Folders.filter(lambda x: x != multiple()),\n link_file_name = st_file_name, \n user = st.sampled_from(SUDO_USERS), \n umask = st_umask)\n @precondition(lambda self: self.should_run('symlink'))\n def symlink(self, src_file, parent, link_file_name, user='root', umask=0o022):\n result1 = self.fsop1.do_symlink(src_file=src_file, parent=parent, link_file_name=link_file_name, user=user, umask=umask)\n result2 = self.fsop2.do_symlink(src_file=src_file, parent=parent, link_file_name=link_file_name, user=user, umask=umask)\n assert self.equal(result1, result2), red(f'symlink:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, link_file_name)\n\n @rule(target = Files , \n parent = Folders.filter(lambda x: x != multiple()),\n link_file_name = st_file_name, \n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('loop_symlink'))\n def loop_symlink(self, parent, link_file_name, user='root'):\n result1 = self.fsop1.do_loop_symlink(parent=parent, link_file_name=link_file_name, user=user)\n result2 = self.fsop2.do_loop_symlink(parent=parent, link_file_name=link_file_name, user=user)\n assert self.equal(result1, result2), red(f'loop_symlink:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, link_file_name)\n\n @rule(file = Files.filter(lambda x: x != multiple()),\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('readlink'))\n def readlink(self, file, user='root'):\n result1 = self.fsop1.do_readlink(file=file, user=user)\n result2 = self.fsop2.do_readlink(file=file, user=user)\n assert self.equal(result1, result2), red(f'read_link:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(target=Xattrs, \n file = Files.filter(lambda x: x != multiple()), \n name = st_xattr_name,\n value = st_xattr_value, \n flag = st_xattr_flag,\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('set_xattr'))\n def set_xattr(self, file, name, value, flag, user='root'):\n result1 = self.fsop1.do_set_xattr(file=file, name=name, value=value, flag=flag, user=user)\n result2 = self.fsop2.do_set_xattr(file=file, name=name, value=value, flag=flag, user=user)\n assert self.equal(result1, result2), red(f'set_xattr:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return (file, name)\n\n @rule(xattr = Xattrs.filter(lambda x: x != multiple()),\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('get_xattr'))\n def get_xattr(self, xattr, user):\n result1 = self.fsop1.do_get_xattr(file=xattr[0], name=xattr[1], user=user)\n result2 = self.fsop2.do_get_xattr(file=xattr[0], name=xattr[1], user=user)\n assert self.equal(result1, result2), red(f'get_xattr:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(file=Files.filter(lambda x: x != multiple()), \n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('list_xattr'))\n def list_xattr(self, file, user='root'):\n result1 = self.fsop1.do_list_xattr(file=file, user=user)\n result2 = self.fsop2.do_list_xattr(file=file, user=user)\n assert self.equal(result1, result2), red(f'list_xattr:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(\n target = Xattrs,\n xattr = consumes(Xattrs).filter(lambda x: x != multiple()), \n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('remove_xattr'))\n def remove_xattr(self, xattr, user='root'):\n result1 = self.fsop1.do_remove_xattr(file=xattr[0], name=xattr[1], user=user)\n result2 = self.fsop2.do_remove_xattr(file=xattr[0], name=xattr[1], user=user)\n assert self.equal(result1, result2), red(f'remove_xattr:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return xattr\n else:\n return multiple()\n \n @rule(user = st.sampled_from(USERS).filter(lambda x: x != 'root'), \n group = st.sampled_from(GROUPS),\n groups = st.lists(st.sampled_from(GROUPS), unique=True))\n @precondition(lambda self: self.should_run('change_groups') and not self.use_sdk)\n def change_groups(self, user, group, groups):\n self.fsop1.do_change_groups(user, group, groups)\n self.fsop2.do_change_groups(user, group, groups)\n\n @rule(entry = Entries.filter(lambda x: x != multiple()), \n mode = st_entry_mode, \n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('chmod'))\n def chmod(self, entry, mode, user='root'):\n result1 = self.fsop1.do_chmod(entry=entry, mode=mode, user=user)\n result2 = self.fsop2.do_chmod(entry=entry, mode=mode, user=user)\n assert self.equal(result1, result2), red(f'chmod:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(entry = Entries.filter(lambda x: x != multiple()))\n @precondition(lambda self: self.should_run('get_acl') and not self.use_sdk)\n def get_acl(self, entry):\n result1 = self.fsop1.do_get_acl(entry)\n result2 = self.fsop2.do_get_acl(entry)\n assert self.equal(result1, result2), red(f'get_acl:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n \n @rule(entry = EntryWithACL.filter(lambda x: x != multiple()), \n option = st.sampled_from(['--remove-all', '--remove-default']),\n user = st.sampled_from(SUDO_USERS)\n )\n @precondition(lambda self: self.should_run('remove_acl') and not self.use_sdk)\n def remove_acl(self, entry: str, option: str, user='root'):\n result1 = self.fsop1.do_remove_acl(entry, option, user)\n result2 = self.fsop2.do_remove_acl(entry, option, user)\n assert self.equal(result1, result2), red(f'remove_acl:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n @rule(\n target=EntryWithACL,\n sudo_user = st.sampled_from(SUDO_USERS),\n entry = Entries.filter(lambda x: x != multiple()), \n user=st.sampled_from(USERS+['']),\n user_perm = st.sets(st.sampled_from(['r', 'w', 'x'])),\n group=st.sampled_from(GROUPS+['']),\n group_perm = st.sets(st.sampled_from(['r', 'w', 'x'])),\n other_perm = st.sets(st.sampled_from(['r', 'w', 'x'])),\n set_mask = st.booleans(),\n mask = st.sets(st.sampled_from(['r', 'w', 'x'])),\n default = st.booleans(),\n recursive = st.booleans(),\n recalc_mask = st.booleans(),\n not_recalc_mask = st.booleans(),\n logical = st.booleans(),\n physical = st.booleans(),\n )\n @precondition(lambda self: self.should_run('set_acl') and not self.use_sdk)\n def set_acl(self, sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical):\n result1 = self.fsop1.do_set_acl(sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical)\n result2 = self.fsop2.do_set_acl(sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical)\n assert self.equal(result1, result2), red(f'set_acl:\\nresult1 is {result1}\\nresult2 is {result2}')\n if isinstance(result1, Exception):\n return multiple()\n else:\n return entry\n\n @rule(entry = Entries.filter(lambda x: x != multiple()),\n access_time=st_time, \n modify_time=st_time, \n follow_symlinks=st.booleans(), \n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('utime') and False)\n def utime(self, entry, access_time, modify_time, follow_symlinks, user='root'):\n result1 = self.fsop1.do_utime(entry=entry, access_time=access_time, modify_time=modify_time, follow_symlinks=follow_symlinks, user=user)\n result2 = self.fsop2.do_utime(entry=entry, access_time=access_time, modify_time=modify_time, follow_symlinks=follow_symlinks, user=user)\n assert self.equal(result1, result2), red(f'utime:\\nresult1 is {result1}\\nresult2 is {result2}')\n\n\n @rule(entry = Entries.filter(lambda x: x != multiple()), \n owner= st.sampled_from(USERS), \n user = st.sampled_from(SUDO_USERS))\n @precondition(lambda self: self.should_run('chown'))\n def chown(self, entry, owner, user='root'):\n result1 = self.fsop1.do_chown(entry=entry, owner=owner, user=user)\n result2 = self.fsop2.do_chown(entry=entry, owner=owner, user=user)\n assert self.equal(result1, result2), red(f'chown:\\nresult1 is {result1}\\nresult2 is {result2}')\n \n @rule( dir =Folders, vdirs = st.integers(min_value=2, max_value=31) )\n @precondition(lambda self: self.should_run('split_dir') \\\n and (self.fsop1.is_jfs or self.fsop2.is_jfs) \\\n and not self.use_sdk\n )\n def split_dir(self, dir, vdirs):\n self.fsop1.do_split_dir(dir, vdirs)\n self.fsop2.do_split_dir(dir, vdirs)\n\n @rule(dir = Folders)\n @precondition(lambda self: self.should_run('merge_dir') \\\n and (self.fsop1.is_jfs or self.fsop2.is_jfs) \\\n and not self.use_sdk\n )\n def merge_dir(self, dir):\n self.fsop1.do_merge_dir(dir)\n self.fsop2.do_merge_dir(dir)\n \n @rule(dir = Folders,\n zone1=st.sampled_from(common.get_zones(ROOT_DIR1)),\n zone2=st.sampled_from(common.get_zones(ROOT_DIR2)),\n is_vdir=st.booleans())\n @precondition(lambda self: self.should_run('rebalance_dir') \\\n and (self.fsop1.is_jfs or self.fsop2.is_jfs) \\\n and not self.use_sdk \\\n and os.getenv('PROFILE', 'dev') != 'generate'\n )\n def rebalance_dir(self, dir, zone1, zone2, is_vdir, pysdk=True):\n self.fsop1.do_rebalance(entry=dir, zone=zone1, is_vdir=is_vdir, pysdk=pysdk)\n self.fsop2.do_rebalance(entry=dir, zone=zone2, is_vdir=is_vdir, pysdk=pysdk)\n\n @rule(file = Files, \n zone1=st.sampled_from(common.get_zones(ROOT_DIR1)),\n zone2=st.sampled_from(common.get_zones(ROOT_DIR2)),\n )\n @precondition(lambda self: self.should_run('rebalance_file') \\\n and (self.fsop1.is_jfs or self.fsop2.is_jfs) \\\n and not self.use_sdk \\\n and os.getenv('PROFILE', 'dev') != 'generate'\n )\n def rebalance_file(self, file, zone1, zone2, pysdk=True):\n self.fsop1.do_rebalance(entry=file, zone=zone1, is_vdir=False, pysdk=pysdk)\n self.fsop2.do_rebalance(entry=file, zone=zone2, is_vdir=False, pysdk=pysdk)\n\n def teardown(self):\n if self.check_dangling:\n self.fsop1.do_check_dangling_files()\n self.fsop2.do_check_dangling_files()\n\nif __name__ == '__main__':\n MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))\n STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))\n ci_db = DirectoryBasedExampleDatabase(\".hypothesis/examples\") \n settings.register_profile(\"dev\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])\n settings.register_profile(\"schedule\", max_examples=1000, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=200, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain], \n database=ci_db)\n settings.register_profile(\"pull_request\", max_examples=100, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=50, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain], \n database=ci_db)\n settings.register_profile(\"generate\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \\\n phases=[Phase.generate, Phase.target])\n \n if os.environ.get('CI'):\n event_name = os.environ.get('GITHUB_EVENT_NAME')\n if event_name == 'schedule':\n profile = 'schedule'\n else:\n profile = 'pull_request'\n else:\n profile = os.environ.get('PROFILE', 'dev')\n print(f'profile is {profile}')\n settings.load_profile(profile)\n juicefs_machine = JuicefsMachine.TestCase()\n juicefs_machine.runTest()\n print(json.dumps(FsOperation.stats.get(), sort_keys=True, indent=4))" }, { "path": ".github/scripts/hypo/fs_acl_test.py", "content": "import unittest\nfrom fs import JuicefsMachine\n\nclass TestFsrand2(unittest.TestCase):\n def test_acl_913(self):\n # See: https://github.com/juicedata/jfs/issues/913\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='aaaa', mode='w', parent=v1, user='root')\n v3 = state.set_acl(default=False, entry=v1, group='root', group_perm=set(), logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='user1', user_perm=set())\n state.chmod(entry=v1, mode=4, user='root')\n state.set_acl(default=False, entry=v1, group='root', group_perm=set(), logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=True, set_mask=False, sudo_user='user1', user='root', user_perm=set())\n state.teardown()\n\n def test_acl_1004(self):\n # SEE https://github.com/juicedata/jfs/issues/1004\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.listdir(dir=v1, user='root')\n state.change_groups(group='root', groups=[], user='user1')\n v2 = state.set_acl(default=False, entry=v1, group='root', group_perm={'r'}, logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm=set())\n state.listdir(dir=v1, user='user1')\n state.teardown()\n\n def test_acl_1006(self):\n # SEE https://github.com/juicedata/jfs/issues/1006\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.create_file(content=b'', file_name='aaaa', mode='w', parent=v1, umask=0, user='root')\n state.set_acl(default=False, entry=v1, group='root', group_perm={'r'}, logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm=set())\n state.set_acl(default=False, entry=v1, group='user1', group_perm={'r'}, logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm=set())\n state.set_acl(default=False, entry=v1, group='root', group_perm={'r'}, logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=True, set_mask=False, sudo_user='user1', user='root', user_perm=set())\n state.teardown()\n\n def test_acl_1011(self):\n # SEE https://github.com/juicedata/jfs/issues/1011\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.chmod(entry=v1, mode=0, user='root')\n state.split_dir(dir=v1, vdirs=2)\n state.change_groups(group='root', groups=[], user='user1')\n v2 = state.set_acl(default=False, entry=v1, group='root', group_perm={'r'}, logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm=set())\n v3 = state.create_file(content=b'', file_name='aaaa', mode='w', parent=v1, umask=0, user='root')\n state.listdir(dir=v1, user='user1')\n state.teardown()\n\n def test_acl_1015(self):\n # SEE: https://github.com/juicedata/jfs/issues/1015\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='aaaa', mode='w', parent=v1, umask=0, user='root')\n state.set_acl(default=False, entry=v1, group='root', group_perm={'r'}, logical=False, mask=set(), not_recalc_mask=False, other_perm={'r', 'w', 'x'}, physical=False, recalc_mask=False, recursive=True, set_mask=True, sudo_user='root', user='user1', user_perm={'r', 'w', 'x'})\n state.set_acl(default=False, entry=v1, group='root', group_perm={'r'}, logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=True, set_mask=False, sudo_user='user1', user='root', user_perm=set())\n state.teardown()\n\n def test_acl_1022(self):\n # SEE https://github.com/juicedata/jfs/issues/1022\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.create_file(content=b'\\xda\\x07', file_name='lbca', mode='w', parent=v1, umask=103, user='root')\n state.set_acl(default=False, entry=v1, group='user1', group_perm={'r', 'w'}, logical=False, mask={'r', 'w', 'x'}, not_recalc_mask=True, other_perm=set(), physical=True, recalc_mask=True, recursive=True, set_mask=True, sudo_user='root', user='root', user_perm={'r', 'w', 'x'})\n state.chmod(entry=v1, mode=0o4004, user='root')\n state.set_acl(default=True, entry=v1, group='group4', group_perm={'x'}, logical=False, mask={'w', 'x'}, not_recalc_mask=False, other_perm=set(), physical=True, recalc_mask=False, recursive=True, set_mask=True, sudo_user='user1', user='user2', user_perm=set())\n state.teardown()\n\n def test_acl_1044(self):\n # SEE: https://github.com/juicedata/jfs/issues/1044\n state = JuicefsMachine()\n v1 = state.init_folders()\n v3 = state.create_file(content=b'', file_name='aaca', mode='wb', parent=v1, umask=0, user='root')\n v4 = state.set_xattr(file=v3, flag=2, name='user.0', user='root', value=b\"abc\")\n v5 = state.set_acl(default=False, entry=v3, group='root', group_perm={'r'}, logical=False, mask={'r'}, not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm={'r'})\n state.remove_acl(entry=v3, option='--remove-all', user='root')\n state.list_xattr(file=v3, user='root')\n state.teardown()\n\n def test_acl_4458(self):\n # SEE: https://github.com/juicedata/juicefs/issues/4458\n state = JuicefsMachine()\n v1 = state.init_folders()\n v3 = state.set_acl(default=True, entry=v1, group='root', group_perm=set(), logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=True, recursive=True, set_mask=True, sudo_user='root', user='user1', user_perm={v1, 'r', 'w', 'x'})\n state.create_file(content=b'', file_name='afds', mode='w', parent=v1, umask=295, user='root')\n state.teardown()\n\n def test_acl_4472(self):\n # SEE: https://github.com/juicedata/juicefs/issues/4472\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='stsn', mode='xb', parent=v1, umask=464, user='root')\n v3 = state.set_acl(default=True, entry=v1, group='group4', group_perm={'x'}, logical=False, mask={'w'}, not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=True, recursive=True, set_mask=True, sudo_user='root', user='root', user_perm={'r'})\n v8 = state.create_file(content=b'', file_name='qpyt', mode='wb', parent=v1, umask=233, user='root')\n v9 = state.copy_file(entry=v2, follow_symlinks=False, new_entry_name='knmh', parent=v1, umask=23, user='root')\n state.open(file=v8, flags=[512], mode=2579, umask=34, user='root')\n state.teardown()\n\n def test_acl_4483(self):\n # SEE https://github.com/juicedata/juicefs/issues/4483\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.set_acl(default=True, entry=v1, group='root', group_perm={'r'}, logical=True, mask={'r'}, not_recalc_mask=False, other_perm={'r', 'x'}, physical=False, recalc_mask=False, recursive=True, set_mask=True, sudo_user='user1', user='user2', user_perm={'r', 'w', 'x'})\n v4 = state.create_file(content=b'\\xe65', file_name='abha', mode='ab', parent=v1, umask=3, user='root')\n v5 = state.set_acl(default=False, entry=v4, group='user3', group_perm={'x'}, logical=False, mask={'x'}, not_recalc_mask=True, other_perm=set(), physical=True, recalc_mask=True, recursive=False, set_mask=False, sudo_user='root', user='user1', user_perm=set())\n state.list_xattr(file=v4, user='root')\n state.teardown()\n\n def test_acl_4496(self):\n # SEE https://github.com/juicedata/juicefs/issues/4496\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.chmod(entry=v1, mode=3291, user='root')\n state.remove_acl(entry=v1, option='--remove-default', user='user1')\n v40 = state.mkdir(mode=1122, parent=v1, subdir='uopt', umask=367, user='root')\n state.chown(entry=v40, owner='user1', user='root')\n state.change_groups(group='group4', groups=['group2'], user='user1')\n state.set_acl(default=False, entry=v40, group='group2', group_perm={'r', 'w', 'x'}, logical=True, mask={'r', 'w', 'x'}, not_recalc_mask=True, other_perm={'x'}, physical=False, recalc_mask=True, recursive=False, set_mask=False, sudo_user='user1', user=v1, user_perm=set())\n state.teardown()\n\n def test_acl_4663(self):\n #SEE https://github.com/juicedata/juicefs/issues/4663\n state = JuicefsMachine()\n v1 = state.init_folders()\n v3 = state.set_acl(default=True, entry=v1, group=v1, group_perm=set(), logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm={'r'})\n state.mkdir(mode=0, parent=v1, subdir='aaaa', umask=0, user='root')\n state.teardown()\n\n def skip_test_acl_2044(self):\n #SEE https://github.com/juicedata/jfs/issues/2044\n for i in range(5):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n files_0 = state.create_file(content=b'$\\xca<', file_name='f', parent=folders_0, umask=18, user='root')\n files_1 = state.rename_file(entry=files_0, new_entry_name='yedw', parent=folders_0, umask=18, user='root')\n state.set_acl(default=True, entry=files_1, group='user2', group_perm=set(), logical=False, mask=set(), not_recalc_mask=False, other_perm=set(), physical=False, recalc_mask=True, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm=set())\n state.open(file=files_1, flags=[0, 64, 2, 512, 4096, 1, 1052672, 1024, 128], mode=231, umask=18, user='root')\n state.rebalance_file(file=files_1, zone1=folders_0, zone2='.jfszone1')\n files_2 = state.hardlink(link_file_name='a', parent=folders_0, src_file=files_1, umask=18, user='root')\n folders_1 = state.mkdir(mode=76, parent=folders_0, subdir='j', umask=18, user='root')\n files_3 = state.hardlink(link_file_name='v', parent=folders_0, src_file=files_1, umask=18, user='root')\n files_4 = state.rename_file(entry=files_1, new_entry_name='ypzn', parent=folders_0, umask=18, user='root')\n files_5 = state.copy_file(entry=files_2, follow_symlinks=True, new_entry_name='iydv', parent=folders_1, umask=18, user='root')\n state.open(file=files_2, flags=[4096, 128], mode=250, umask=18, user='root')\n entry_with_acl_0 = state.set_acl(default=False, entry=files_4, group='group1', group_perm=set(), logical=False, mask=set(), not_recalc_mask=False, other_perm={'x'}, physical=True, recalc_mask=True, recursive=False, set_mask=True, sudo_user='root', user='user1', user_perm=set())\n state.unlink(file=files_2, user='root')\n state.fallocate(file=files_4, length=66667, mode=0, offset=6713, user='root')\n state.open(file=files_5, flags=[64], mode=441, umask=18, user='root')\n state.set_acl(default=True, entry=files_4, group='group3', group_perm={'x'}, logical=False, mask={'r', 'w', 'x'}, not_recalc_mask=False, other_perm={'r', 'w', 'x'}, physical=True, recalc_mask=True, recursive=False, set_mask=True, sudo_user='root', user='user3', user_perm=set())\n state.remove_acl(entry=entry_with_acl_0, option='--remove-all', user='root')\n files_7 = state.rename_file(entry=files_3, new_entry_name='fgq', parent=folders_0, umask=18, user='root')\n state.chmod(entry=files_7, mode=433, user='root')\n state.remove_acl(entry=entry_with_acl_0, option='--remove-default', user='root')\n state.teardown()\n\nif __name__ == '__main__':\n unittest.main()" }, { "path": ".github/scripts/hypo/fs_op.py", "content": "import io\nimport os\nimport pwd\nimport re\nimport shutil\nimport stat\nimport subprocess\n\ntry: \n __import__('xattr')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\nimport xattr\nfrom common import get_acl, get_root, red\nfrom typing import Dict\ntry: \n __import__('fallocate')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"fallocate\"])\nimport fallocate\nfrom stats import Statistics\nimport common\nfrom os.path import dirname\nimport sys\nsys.path.append('.')\nfrom sdk.python.juicefs.juicefs import juicefs\n\nclass FsOperation:\n JFS_CONTROL_FILES=['.accesslog', '.config', '.stats']\n stats = Statistics()\n \n def __init__(self, name, root_dir:str, mount_point=None, use_sdk:bool=False, is_jfs=False, volume_name=None, meta_url=None):\n self.logger =common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO'))\n self.root_dir = root_dir.rstrip('/')\n self.use_sdk = use_sdk\n self.is_jfs = is_jfs\n self.singlezone = False\n if is_jfs:\n self.singlezone = len(common.get_zones(root_dir)) == 1\n if mount_point:\n self.mount_point = mount_point\n else:\n self.mount_point = common.get_root(self.root_dir)\n self.client = None\n if use_sdk and self.is_jfs:\n if meta_url:\n self.client = juicefs.Client(volume_name, meta_url, access_log=\"/tmp/jfs.log\")\n else:\n self.client = juicefs.Client(volume_name, conf_dir='deploy/docker', access_log=\"/tmp/jfs.log\")\n self.client2 = None\n\n def get_client_for_rebalance(self):\n if self.client2 == None:\n self.client2 = juicefs.Client(common.get_volume_name(self.root_dir), \n conf_dir='deploy/docker', \n access_log=\"/tmp/rebalance.log\", \n attr_cache=\"0s\",\n entry_cache=\"0s\", \n dir_entry_cache=\"0s\",)\n return self.client2\n\n def run_cmd(self, command:str) -> str:\n self.logger.info(f'run_cmd: {command}')\n if '|' in command or '>' in command or '&' in command:\n ret=os.system(command)\n if ret == 0:\n return ret\n else: \n raise Exception(f\"run command {command} failed with {ret}\")\n try:\n output = subprocess.run(command.split(), check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as e:\n raise e\n return output.stdout.decode()\n\n def get_zones(self):\n return common.get_zones(self.root_dir)\n\n def init_rootdir(self):\n if self.client:\n self.logger.debug(f'init_rootdir {self.root_dir} with use_sdk={self.use_sdk}')\n sdk_root_dir = self.get_sdk_path(self.root_dir)\n if self.client.exists(sdk_root_dir):\n self.client.rmr(sdk_root_dir)\n assert not self.client.exists(sdk_root_dir), red(f'{self.root_dir} should not exist')\n self.client.makedirs(sdk_root_dir)\n assert self.client.exists(sdk_root_dir), red(f'{self.root_dir} should exist')\n else:\n if not os.path.exists(self.root_dir):\n os.makedirs(self.root_dir)\n if os.environ.get('PROFILE', 'dev') != 'generate':\n common.clean_dir(self.root_dir)\n\n def seteuid(self, user, action=''):\n if self.client:\n return\n uid = pwd.getpwnam(user).pw_uid\n gid = pwd.getpwnam(user).pw_gid\n os.setegid(gid)\n os.seteuid(uid)\n self.logger.debug(f'{action} seteuid uid={uid} gid={gid} succeed')\n\n def reset_euid(self, action=''):\n if self.client:\n return\n os.setegid(0) \n os.seteuid(0)\n self.logger.debug(f'{action} reset euid and egid succeed')\n \n def handleException(self, e, action, path, **kwargs):\n if isinstance(e, subprocess.CalledProcessError):\n err = e.output.decode()\n else:\n err = type(e).__name__ + \":\" + str(e)\n err = '\\n'.join([elem.split(':')[-1].split(':')[-1] for elem in err.split('\\n')])\n err = re.sub(r'\\[\\w+\\.go:\\d+\\]', '', err)\n if err.find('setfacl') != -1 and err.find('\\n') != -1:\n err = '\\n'.join(sorted(err.split('\\n')))\n err = self.parse_pysdk_error(err)\n self.stats.failure(action)\n self.logger.info(f'{action} {path} {kwargs} failed: {err}')\n return Exception(err)\n \n def parse_pysdk_error(self, err:str):\n # error message : call jfs_rename failed: [Errno 22] Invalid argument: (b'/fsrand', b'/fsrand/izsn/rfnn', c_uint(0))\n if not err.startswith(\"call jfs_\"):\n return err\n return re.sub(r'call jfs_\\w+ failed: ', '', err)\n \n def get_sdk_path(self, abspath):\n return '/'+os.path.relpath(abspath, self.mount_point)\n\n def do_remove_dangling_files(self):\n if not self.is_jfs or self.use_sdk:\n self.logger.debug(f'do_remove_dangling_files {self.mount_point} skip')\n return\n self.logger.debug(f'do_remove_dangling_files {self.mount_point}')\n zones = common.get_zones(self.mount_point)\n for zone in zones:\n zone_dir = os.path.join(self.mount_point, zone)\n entries = os.listdir(zone_dir)\n for entry in entries:\n if 'dangling' in entry:\n abspath = os.path.join(zone_dir, entry)\n if os.path.isdir(abspath):\n shutil.rmtree(abspath)\n elif os.path.isfile(abspath):\n os.unlink(abspath)\n backup_dir = os.path.join(zone_dir, '.backup')\n if os.path.exists(backup_dir):\n entries = os.listdir(backup_dir)\n for entry in entries:\n if 'dangling' in entry:\n abspath = os.path.join(backup_dir, entry)\n if os.path.isdir(abspath):\n shutil.rmtree(abspath)\n elif os.path.isfile(abspath):\n os.unlink(abspath)\n\n self.logger.info(f'do_remove_dangling_files {self.mount_point} succeed')\n\n def do_check_dangling_files(self):\n if not self.is_jfs or self.use_sdk:\n self.logger.debug(f'do_check_dangling_files {self.mount_point} skip')\n return\n self.logger.debug(f'do_check_dangling_files {self.mount_point}')\n zones = common.get_zones(self.mount_point)\n for zone in zones:\n zone_dir = os.path.join(self.mount_point, zone)\n entries = os.listdir(zone_dir)\n for entry in entries:\n if 'dangling' in entry:\n assert False, red(f'{entry} should not exist in {zone_dir}')\n backup_dir = os.path.join(zone_dir, '.backup')\n if os.path.exists(backup_dir):\n entries = os.listdir(backup_dir)\n for entry in entries:\n if 'dangling' in entry:\n assert False, red(f'{entry} should not exist in {backup_dir}')\n self.logger.info(f'do_check_dangling_files {self.mount_point} succeed')\n\n def do_stat(self, entry, user):\n self.logger.debug(f'do_stat {self.root_dir} {entry}')\n abspath = os.path.join(self.root_dir, entry)\n try:\n self.seteuid(user, action='do_stat')\n if self.client:\n st = self.client.stat(self.get_sdk_path(abspath))\n else:\n st = os.stat(abspath)\n except Exception as e :\n return self.handleException(e, 'do_stat', abspath, entry=entry, user=user)\n finally:\n self.reset_euid(action='do_stat')\n self.stats.success('do_stat')\n self.logger.info(f'do_stat {abspath} with user={user} succeed')\n self.logger.debug(f'do_stat st is {st}')\n return common.get_stat_field(st)\n \n def do_lstat(self, entry, user):\n self.logger.debug(f'do_lstat {self.root_dir} {entry}')\n abspath = os.path.join(self.root_dir, entry)\n try:\n self.seteuid(user)\n if self.client:\n st = self.client.lstat(self.get_sdk_path(abspath))\n else:\n st = os.lstat(abspath)\n except Exception as e :\n return self.handleException(e, 'do_lstat', abspath, entry=entry, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_lstat')\n self.logger.info(f'do_lstat {abspath} with user={user} succeed')\n return common.get_stat_field(st)\n\n def do_exists(self, entry, user):\n self.logger.debug(f'do_exists {self.root_dir} {entry}')\n abspath = os.path.join(self.root_dir, entry)\n try:\n self.seteuid(user)\n if self.client:\n exists = self.client.exists(self.get_sdk_path(abspath))\n else:\n exists = os.path.exists(abspath)\n except Exception as e :\n return self.handleException(e, 'do_exists', abspath, entry=entry, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_exists')\n self.logger.info(f'do_exists {abspath} with user={user} succeed')\n return exists\n\n def do_open(self, file, flags, mask, mode, user):\n self.logger.debug(f'do_open {self.root_dir} {file} {flags} {mode} {user}')\n abspath = os.path.join(self.root_dir, file)\n flag = 0\n fd = -1\n for f in flags:\n flag |= f\n try:\n old_mask = os.umask(mask)\n self.seteuid(user)\n fd = os.open(abspath, flags=flag, mode=mode)\n except Exception as e :\n return self.handleException(e, 'do_open', abspath, flags=flags, mode=mode, user=user)\n finally:\n self.reset_euid()\n os.umask(old_mask)\n if fd > 0:\n os.close(fd)\n self.stats.success('do_open')\n self.logger.info(f'do_open {abspath} {flags} {mode} succeed')\n return self.do_stat(file, user)\n \n def do_open2(self, file, mode, user):\n self.logger.debug(f'do_open2 {self.root_dir} {file} {mode} {user}')\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n with self.client.open(self.get_sdk_path(abspath), mode) as f:\n pass\n else:\n with open(abspath, mode) as f:\n pass\n except Exception as e :\n return self.handleException(e, 'do_open2', abspath, mode=mode, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_open2')\n self.logger.info(f'do_open2 {abspath} {mode} succeed')\n return self.do_stat(file, user)\n\n def do_write(self, file, content, mode:str, encoding, errors, offset, whence, user):\n self.logger.debug(f'do_write {self.root_dir} {file} {offset}')\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n size = self.client.stat(self.get_sdk_path(abspath)).st_size\n else:\n size = os.stat(abspath).st_size\n if size == 0:\n offset = 0\n else:\n offset = offset % size\n if self.client:\n with self.client.open(self.get_sdk_path(abspath), mode, encoding=encoding, errors=errors) as f:\n f.seek(offset, whence)\n count=f.write(content)\n else:\n with open(abspath, mode, encoding=encoding, errors=errors) as f:\n f.seek(offset, whence)\n count=f.write(content)\n except (io.UnsupportedOperation) as e:\n e = Exception(f'io.UnsupportedOperation: write')\n return self.handleException(e, 'do_write', abspath, offset=offset, whence=whence, mode=mode, user=user)\n except Exception as e :\n return self.handleException(e, 'do_write', abspath, offset=offset, whence=whence, mode=mode, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_write')\n self.logger.info(f'do_write {abspath} offset={offset} whence={whence} mode={mode} user={user} succeed')\n return count, self.do_stat(file, user)\n \n def do_writelines(self, file, lines, mode, offset, whence, user):\n self.logger.debug(f'do_writelines {self.root_dir} {file} {offset}')\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n size = self.client.stat(self.get_sdk_path(abspath)).st_size\n else:\n size = os.stat(abspath).st_size\n if size == 0:\n offset = 0\n else:\n offset = offset % size\n if self.client:\n with self.client.open(self.get_sdk_path(abspath), mode) as f:\n f.seek(offset, whence)\n f.writelines(lines)\n else:\n with open(abspath, mode) as f:\n # f.seek(offset, whence)\n f.seek(offset, whence)\n f.writelines(lines)\n except (TypeError,io.UnsupportedOperation) as e:\n self.logger.debug(f'writelines: {str(e)}')\n e = Exception(f'writelines')\n return self.handleException(e, 'do_writelines', abspath, offset=offset, whence=whence, mode=mode, user=user)\n except Exception as e :\n return self.handleException(e, 'do_writelines', abspath, offset=offset, whence=whence, mode=mode, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_writelines')\n self.logger.info(f'do_writelines {abspath} offset={offset} whence={whence} mode={mode} user={user} succeed')\n return self.do_stat(file, user)\n\n def do_fallocate(self, file, offset, length, mode, user):\n self.logger.debug(f'do_fallocate {self.root_dir} {file} {offset} {length} {mode} {user}')\n abspath = os.path.join(self.root_dir, file)\n fd = -1\n try:\n self.seteuid(user)\n file_size = os.stat(abspath).st_size\n if file_size == 0:\n offset = 0\n else:\n offset = offset % file_size\n fd = os.open(abspath, os.O_RDWR)\n fallocate.fallocate(fd, offset, length, mode)\n except Exception as e :\n return self.handleException(e, 'do_fallocate', abspath, offset=offset, length=length, mode=mode, user=user)\n finally:\n if fd > 0:\n os.close(fd)\n self.reset_euid()\n self.stats.success('do_fallocate')\n self.logger.info(f'do_fallocate {abspath} offset={offset} length={length} mode={mode} user={user} succeed')\n return self.do_stat(file, user)\n\n def do_copy_file_range(self, src, dst, src_offset, dst_offset, count, user):\n self.logger.debug(f'do_copy_file_range {self.root_dir} {src} {dst} {src_offset} {dst_offset} {count} {user}')\n src_abspath = os.path.join(self.root_dir, src)\n dst_abspath = os.path.join(self.root_dir, dst)\n src_fd = -1\n dst_fd = -1\n try:\n self.seteuid(user)\n src_fd = os.open(src_abspath, os.O_RDONLY)\n dst_fd = os.open(dst_abspath, os.O_WRONLY)\n os.copy_file_range(src_fd, dst_fd, count, src_offset, dst_offset)\n except Exception as e :\n return self.handleException(e, 'do_copy_file_range', src_abspath, dst_abspath=dst_abspath, src_offset=src_offset, dst_offset=dst_offset, count=count, user=user)\n finally:\n if src_fd > 0:\n os.close(src_fd)\n if dst_fd > 0:\n os.close(dst_fd)\n self.reset_euid()\n self.stats.success('do_copy_file_range')\n self.logger.info(f'do_copy_file_range {src_abspath} {dst_abspath} src_offset={src_offset} dst_offset={dst_offset} count={count} user={user} succeed')\n return self.do_stat(dst, user)\n\n def do_read(self, file, length, mode, offset, whence, user, encoding, errors):\n self.logger.debug(f'do_read {self.root_dir} {file} {mode} {length} {offset} {whence}')\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n size = self.client.stat(self.get_sdk_path(abspath)).st_size\n else:\n size = os.stat(abspath).st_size\n if size == 0:\n offset = 0\n else:\n offset = offset % size\n if self.client:\n with self.client.open(self.get_sdk_path(abspath), mode, encoding=encoding, errors=errors) as f:\n f.seek(offset, whence)\n result = f.read(length) \n else:\n with open(abspath, mode, encoding=encoding, errors=errors) as f: \n # f.seek(offset, whence)\n f.seek(offset, whence)\n result = f.read(length)\n if isinstance(result, str):\n result = result.replace('\\r', '\\n') # SEE: https://github.com/juicedata/jfs/issues/1472\n result = result.encode()\n # result = binascii.hexlify(result)\n except UnicodeDecodeError as e:\n # SEE: https://github.com/juicedata/jfs/issues/1450#issuecomment-2213518638\n self.logger.debug(f'UnicodeDecodeError: {e.encoding} {e.object} {e.start} {e.end} {e.reason}')\n e = UnicodeDecodeError(e.encoding, e.object, 0, 0, e.reason)\n return self.handleException(e, 'do_read', abspath, offset=offset, length=length, whence=whence, user=user)\n except io.UnsupportedOperation as e:\n e = Exception(f'io.UnsupportedOperation: read')\n return self.handleException(e, 'do_read', abspath, offset=offset, length=length, whence=whence, user=user)\n except Exception as e :\n return self.handleException(e, 'do_read', abspath, offset=offset, length=length, whence=whence, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_read')\n self.logger.info(f'do_read {abspath} mode={mode} length={length} offset={offset} whence={whence} user={user} succeed')\n return (result, )\n\n def do_readlines(self, file, mode, offset, whence, user):\n self.logger.debug(f'do_readlines {self.root_dir} {file} {mode} {offset} {whence}')\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n size = self.client.stat(self.get_sdk_path(abspath)).st_size\n else:\n size = os.stat(abspath).st_size\n if size == 0:\n offset = 0\n else:\n offset = offset % size\n self.logger.debug(f'do_readlines offset={offset} size={size}')\n if self.client:\n with self.client.open(self.get_sdk_path(abspath), mode) as f:\n f.seek(offset, whence)\n result = ''.join(f.readlines())\n else:\n with open(abspath, mode) as f:\n # f.seek(offset, whence)\n f.seek(offset, whence)\n result = ''.join(f.readlines())\n if isinstance(result, str):\n result = result.replace('\\r', '\\n') # SEE: https://github.com/juicedata/jfs/issues/1472\n result = result.encode()\n # result = binascii.hexlify(result)\n except UnicodeDecodeError as e:\n # SEE: https://github.com/juicedata/jfs/issues/1450#issuecomment-2213518638\n self.logger.debug(f'UnicodeDecodeError: {e.encoding} {e.object} {e.start} {e.end} {e.reason}')\n e = UnicodeDecodeError(e.encoding, e.object, 0, 0, e.reason)\n return self.handleException(e, 'do_readlines', abspath, offset=offset, whence=whence, user=user)\n except io.UnsupportedOperation as e:\n e = Exception(f'io.UnsupportedOperation: readlines')\n return self.handleException(e, 'do_readlines', abspath, offset=offset, whence=whence, user=user)\n except Exception as e :\n return self.handleException(e, 'do_readlines', abspath, offset=offset, whence=whence, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_readlines')\n self.logger.info(f'do_readlines {abspath} mode={mode} offset={offset} whence={whence} user={user} succeed')\n return (result, )\n\n def do_readline(self, file, mode, offset, whence, user):\n self.logger.debug(f'do_readline {self.root_dir} {file} {mode} {offset} {whence}')\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n size = self.client.stat(self.get_sdk_path(abspath)).st_size\n else:\n size = os.stat(abspath).st_size\n if size == 0:\n offset = 0\n else:\n offset = offset % size\n if self.client:\n with self.client.open(self.get_sdk_path(abspath), mode) as f:\n f.seek(offset, whence)\n result = f.readline()\n else:\n with open(abspath, mode) as f:\n # f.seek(offset, whence)\n f.seek(offset, whence)\n result = f.readline()\n if isinstance(result, str):\n result = result.replace('\\r', '\\n') # SEE: https://github.com/juicedata/jfs/issues/1472\n result = result.encode()\n # result = binascii.hexlify(result)\n except UnicodeDecodeError as e:\n # SEE: https://github.com/juicedata/jfs/issues/1450#issuecomment-2213518638\n self.logger.debug(f'UnicodeDecodeError: {e.encoding} {e.object} {e.start} {e.end} {e.reason}')\n e = UnicodeDecodeError(e.encoding, e.object, 0, 0, e.reason)\n return self.handleException(e, 'do_readline', abspath, offset=offset, whence=whence, user=user)\n except io.UnsupportedOperation as e:\n e = Exception(f'io.UnsupportedOperation: readline')\n return self.handleException(e, 'do_readline', abspath, offset=offset, whence=whence, user=user)\n except Exception as e :\n return self.handleException(e, 'do_readline', abspath, offset=offset, whence=whence, user=user)\n finally:\n self.reset_euid()\n\n self.stats.success('do_readline')\n self.logger.info(f'do_readline {abspath} mode={mode} offset={offset} whence={whence} user={user} succeed')\n return (result, )\n\n def do_truncate(self, file, size, user):\n self.logger.debug(f'do_truncate {self.root_dir} {file} {size}')\n abspath = os.path.join(self.root_dir, file)\n fd = -1\n try:\n self.seteuid(user, action='do_truncate')\n if self.client:\n st = self.client.stat(self.get_sdk_path(abspath))\n else:\n st = os.stat(abspath)\n if st.st_size == 0:\n size = 0\n else:\n size = size % st.st_size\n if self.client:\n self.client.truncate(self.get_sdk_path(abspath), size)\n st = self.client.stat(self.get_sdk_path(abspath))\n else:\n os.truncate(abspath, size)\n st = os.stat(abspath)\n except Exception as e :\n return self.handleException(e, 'do_truncate', abspath, size=size, user=user)\n finally:\n if fd > 0:\n os.close(fd)\n self.reset_euid()\n assert st.st_size == size, red(f'do_truncate: {abspath} size should be {size} but {st.st_size}')\n self.stats.success('do_truncate')\n self.logger.info(f'do_truncate {abspath} size={size} user={user} succeed')\n return self.do_stat(file, user)\n\n def do_create_file(self, parent, file_name, content, mode='xb', user='root', umask=0o022, buffering=-1):\n relpath = os.path.join(parent, file_name)\n abspath = os.path.join(self.root_dir, relpath)\n try:\n old_umask = os.umask(umask)\n self.seteuid(user, action='do_create_file')\n if self.client:\n with self.client.open(self.get_sdk_path(abspath), mode, buffering=buffering) as f:\n f.write(content)\n count=f.write(content)\n else:\n with open(abspath, mode, buffering=buffering) as f:\n f.write(content)\n count=f.write(content)\n except Exception as e :\n return self.handleException(e, 'do_create_file', abspath, mode=mode, user=user)\n finally:\n self.reset_euid(action='do_create_file')\n os.umask(old_umask)\n self.stats.success('do_create_file')\n self.logger.info(f'do_create_file {abspath} with mode {mode} succeed')\n return count, self.do_stat(relpath, user)\n \n def do_listdir(self, dir, user):\n abspath = os.path.join(self.root_dir, dir)\n try:\n self.seteuid(user)\n if self.client:\n li = self.client.listdir(self.get_sdk_path(abspath))\n else:\n li = os.listdir(abspath) \n li = sorted(list(filter(lambda x: x not in self.JFS_CONTROL_FILES, li)))\n except Exception as e:\n return self.handleException(e, 'do_listdir', abspath, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_listdir')\n self.logger.info(f'do_listdir {abspath} with user={user} succeed')\n return tuple(li)\n\n def do_unlink(self, file, user):\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n self.client.unlink(self.get_sdk_path(abspath))\n else:\n os.unlink(abspath)\n except Exception as e:\n return self.handleException(e, 'do_unlink', abspath, user=user)\n finally:\n self.reset_euid()\n assert not os.path.exists(abspath), red(f'do_unlink: {abspath} should not exist')\n self.stats.success('do_unlink')\n self.logger.info(f'do_unlink {abspath} with user={user} succeed')\n return True \n\n def do_rename(self, entry, parent, new_entry_name, user, umask):\n abspath = os.path.join(self.root_dir, entry)\n new_relpath = os.path.join(parent, new_entry_name)\n new_abspath = os.path.join(self.root_dir, new_relpath)\n try:\n self.seteuid(user)\n old_umask = os.umask(umask)\n if self.client:\n path = self.get_sdk_path(abspath)\n new_path = self.get_sdk_path(new_abspath)\n self.client.rename(path, new_path)\n else:\n os.rename(abspath, new_abspath)\n except Exception as e:\n return self.handleException(e, 'do_rename', abspath, new_abspath=new_abspath, user=user)\n finally:\n self.reset_euid()\n os.umask(old_umask)\n if not self.use_sdk:\n assert os.path.lexists(new_abspath), red(f'do_rename: {new_abspath} should exist')\n self.stats.success('do_rename')\n self.logger.info(f'do_rename {abspath} {new_abspath} with user={user} succeed')\n return self.do_stat(new_relpath, user)\n\n def do_copy_file(self, entry, parent, new_entry_name, follow_symlinks, user, umask):\n abspath = os.path.join(self.root_dir, entry)\n new_relpath = os.path.join(parent, new_entry_name)\n new_abspath = os.path.join(self.root_dir, new_relpath)\n try:\n old_umask = os.umask(umask)\n self.seteuid(user)\n shutil.copy(abspath, new_abspath, follow_symlinks=follow_symlinks)\n except Exception as e:\n return self.handleException(e, 'do_copy_file', abspath, new_abspath=new_abspath, user=user, follow_symlinks=follow_symlinks, umask=umask)\n finally:\n self.reset_euid()\n os.umask(old_umask)\n assert os.path.lexists(new_abspath), red(f'do_copy_file: {new_abspath} should exist')\n self.stats.success('do_copy_file')\n self.logger.info(f'do_copy_file {abspath} {new_abspath} with follow_symlinks={follow_symlinks} user={user} umask={umask} succeed')\n return self.do_stat(new_relpath, user)\n\n def can_clone(self, src_dir, dst_dir):\n if os.path.commonpath([src_dir]) == os.path.commonpath([src_dir, dst_dir]) or \\\n os.path.commonpath([dst_dir]) == os.path.commonpath([src_dir, dst_dir]):\n return False\n if os.path.exists(dst_dir):\n return False\n return True\n \n def do_clone_entry(self, entry, parent, new_entry_name, preserve, user='root', umask=0o022, mount='./juicefs'):\n root_dir = self.root_dir\n abspath = os.path.join(root_dir, entry)\n new_relpath = os.path.join(parent, new_entry_name)\n new_abspath = os.path.join(root_dir, new_relpath)\n if not self.can_clone(abspath, new_abspath):\n return self.handleException(Exception(f'can not clone {abspath} to {new_abspath}'), 'do_clone_entry', abspath, new_abspath=new_abspath, user=user)\n try:\n old_umask = os.umask(umask)\n if self.is_jfs:\n if preserve:\n self.run_cmd(f'sudo -u {user} {mount} clone {abspath} {new_abspath} --preserve')\n else:\n self.run_cmd(f'sudo -u {user} {mount} clone {abspath} {new_abspath}')\n else:\n if preserve:\n self.run_cmd(f'sudo -u {user} cp -r {abspath} {new_abspath} -L --preserve=all')\n else:\n self.run_cmd(f'sudo -u {user} cp -r {abspath} {new_abspath} -L')\n except subprocess.CalledProcessError as e:\n self.logger.error(f'run command failed: {e.output.decode()}')\n return self.handleException(Exception(f'do_clone_entry failed'), 'do_clone_entry', abspath, new_abspath=new_abspath, user=user)\n finally:\n os.umask(old_umask)\n assert os.path.lexists(new_abspath), red(f'do_clone_entry: {new_abspath} should exist')\n self.stats.success('do_clone_entry')\n self.logger.info(f'do_clone_entry {abspath} {new_abspath} succeed')\n return self.do_stat(new_relpath, user)\n \n def do_copy_tree(self, entry, parent, new_entry_name, symlinks, ignore_dangling_symlinks, dir_exist_ok, user, umask):\n abspath = os.path.join(self.root_dir, entry)\n new_relpath = os.path.join(parent, new_entry_name)\n new_abspath = os.path.join(self.root_dir, new_relpath)\n try:\n old_mask = os.umask(umask)\n self.seteuid(user)\n shutil.copytree(abspath, new_abspath, \\\n symlinks=symlinks, \\\n ignore_dangling_symlinks=ignore_dangling_symlinks, \\\n dirs_exist_ok=dir_exist_ok)\n except Exception as e:\n return self.handleException(e, 'do_copy_tree', abspath, new_abspath=new_abspath, user=user)\n finally:\n self.reset_euid()\n os.umask(old_mask)\n assert os.path.lexists(new_abspath), red(f'do_copy_tree: {new_abspath} should exist')\n self.stats.success('do_copy_tree')\n self.logger.info(f'do_copy_tree {abspath} {new_abspath} succeed')\n return self.do_stat(new_relpath, user)\n\n def do_mkdir(self, parent, subdir, mode, user, umask):\n relpath = os.path.join(parent, subdir)\n abspath = os.path.join(self.root_dir, relpath)\n try:\n self.seteuid(user)\n old_mask = os.umask(umask)\n if self.client:\n sdk_path = self.get_sdk_path(abspath)\n self.client.mkdir(sdk_path, mode)\n st = self.client.stat(sdk_path)\n else:\n os.mkdir(abspath, mode)\n st = os.stat(abspath)\n except Exception as e:\n return self.handleException(e, 'do_mkdir', abspath, mode=mode, user=user)\n finally:\n self.reset_euid()\n os.umask(old_mask)\n assert stat.S_ISDIR(st.st_mode), red(f'do_mkdir: {abspath} should be dir')\n self.stats.success('do_mkdir')\n self.logger.info(f'do_mkdir {abspath} with mode={oct(mode)} user={user} succeed')\n return self.do_stat(entry=relpath, user=user)\n \n def do_rmdir(self, dir, user):\n abspath = os.path.join(self.root_dir, dir)\n try:\n self.seteuid(user)\n if self.client:\n self.client.rmdir(self.get_sdk_path(abspath))\n exist = self.client.exists(self.get_sdk_path(abspath))\n else:\n os.rmdir(abspath)\n exist = os.path.exists(abspath)\n except Exception as e:\n return self.handleException(e, 'do_rmdir', abspath, user=user)\n finally:\n self.reset_euid()\n assert not exist, red(f'do_rmdir: {abspath} should not exist')\n self.stats.success('do_rmdir')\n self.logger.info(f'do_rmdir {abspath} with user={user} succeed')\n return True\n\n def do_hardlink(self, src_file, parent, link_file_name, user, umask):\n src_abs_path = os.path.join(self.root_dir, src_file)\n link_rel_path = os.path.join(parent, link_file_name)\n link_abs_path = os.path.join(self.root_dir, link_rel_path)\n try:\n self.seteuid(user)\n old_mask = os.umask(umask)\n if self.client:\n path = self.get_sdk_path(src_abs_path)\n link_path = self.get_sdk_path(link_abs_path)\n self.client.link(path, link_path)\n else:\n os.link(src_abs_path, link_abs_path)\n except Exception as e:\n return self.handleException(e, 'do_hardlink', src_abs_path, link_abs_path=link_abs_path, user=user)\n finally:\n self.reset_euid()\n os.umask(old_mask)\n # time.sleep(0.005)\n # assert st.st_nlink > 1, red(f'do_hardlink: nlink({st.st_nlink}) of {link_abs_path} should greater than 1')\n self.stats.success('do_hardlink')\n self.logger.info(f'do_hardlink {src_abs_path} {link_abs_path} with user={user} umask={oct(umask)} succeed')\n return self.do_stat(link_rel_path, user)\n\n def do_symlink(self, src_file, parent, link_file_name, user, umask):\n src_abs_path = os.path.join(self.root_dir, src_file)\n link_rel_path = os.path.join(parent, link_file_name)\n link_abs_path = os.path.join(self.root_dir, link_rel_path)\n relative_path = os.path.relpath(src_abs_path, os.path.dirname(link_abs_path))\n try:\n self.seteuid(user)\n old_mask = os.umask(umask)\n if self.client:\n path = self.get_sdk_path(src_abs_path)\n link_path = self.get_sdk_path(link_abs_path)\n self.client.symlink(path, link_path)\n st = self.client.lstat(link_path)\n else:\n os.symlink(relative_path, link_abs_path)\n st = os.lstat(link_abs_path)\n except Exception as e:\n return self.handleException(e, 'do_symlink', src_abs_path, link_abs_path=link_abs_path, user=user)\n finally:\n self.reset_euid()\n os.umask(old_mask)\n assert stat.S_ISLNK(st.st_mode), red(f'do_symlink: {link_abs_path} should be link')\n self.stats.success('do_symlink')\n self.logger.info(f'do_symlink {src_abs_path} {link_abs_path} with user={user} umask={oct(umask)} succeed')\n return self.do_stat(link_rel_path, user)\n \n def do_readlink(self, file, user):\n link_abs_path = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n dest = self.client.readlink(self.get_sdk_path(link_abs_path))\n else:\n dest = os.readlink(link_abs_path)\n except Exception as e:\n return self.handleException(e, 'do_read_link', link_abs_path, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_read_link')\n self.logger.info(f'do_read_link {link_abs_path} with user={user} succeed')\n return (dest,)\n\n def do_loop_symlink(self, parent, link_file_name, user='root'):\n link_abs_path = os.path.join(self.root_dir, parent, link_file_name)\n try:\n self.seteuid(user)\n if self.client:\n sdk_path = self.get_sdk_path(link_abs_path)\n self.client.symlink(sdk_path, sdk_path)\n else:\n os.symlink(link_file_name, link_abs_path)\n except Exception as e:\n return self.handleException(e, 'do_loop_symlink', link_abs_path)\n finally:\n self.reset_euid()\n self.stats.success('do_loop_symlink')\n self.logger.info(f'do_loop_symlink {link_abs_path} succeed')\n return True\n \n def do_set_xattr(self, file, name, value, flag, user):\n xattr_map = {0:0, xattr.XATTR_CREATE: juicefs.XATTR_CREATE, xattr.XATTR_REPLACE: juicefs.XATTR_REPLACE}\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n flag = xattr_map[flag]\n self.client.setxattr(self.get_sdk_path(abspath), name, value, flag)\n else:\n xattr.setxattr(abspath, name, value, flag)\n except Exception as e:\n return self.handleException(e, 'do_set_xattr', abspath, name=name, value=value, flag=flag, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_set_xattr')\n self.logger.info(f\"do_set_xattr {abspath} with name={name} value={value} flag={flag} user={user} succeed\")\n return 'succeed'\n\n def do_get_xattr(self, file, name, user):\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n value = self.client.getxattr(self.get_sdk_path(abspath), name)\n else:\n value = xattr.getxattr(abspath, name)\n except Exception as e:\n return self.handleException(e, 'do_get_xattr', abspath, name=name, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_get_xattr')\n self.logger.info(f\"do_get_xattr {abspath} with name={name} user={user} succeed\")\n return (value,)\n\n def do_list_xattr(self, file, user):\n abspath = os.path.join(self.root_dir, file)\n xattr_list = []\n try:\n self.seteuid(user) \n if self.client:\n path = self.get_sdk_path(abspath)\n xattrs = self.client.listxattr(path)\n else:\n xattrs = xattr.listxattr(abspath)\n xattr_list = []\n for attr in xattrs:\n if self.client:\n path = self.get_sdk_path(abspath)\n value = self.client.getxattr(path, attr)\n else:\n value = xattr.getxattr(abspath, attr)\n xattr_list.append((attr, value))\n xattr_list.sort() # Sort the list based on xattr names\n except Exception as e:\n return self.handleException(e, 'do_list_xattr', abspath, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_list_xattr')\n self.logger.info(f\"do_list_xattr {abspath} with user={user} succeed\")\n return xattr_list\n\n def do_remove_xattr(self, file, name, user):\n abspath = os.path.join(self.root_dir, file)\n try:\n self.seteuid(user)\n if self.client:\n self.client.removexattr(self.get_sdk_path(abspath), name)\n else:\n xattr.removexattr(abspath, name)\n # self.run_cmd(f'sudo -u {user} setfattr -x {name} {abspath}', root_dir)\n except Exception as e:\n return self.handleException(e, 'do_remove_xattr', abspath, name=name, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_remove_xattr')\n self.logger.info(f\"do_remove_xattr {abspath} name={name} user={user} succeed\")\n return 'succeed'\n \n def do_change_groups(self, user, group, groups):\n try:\n subprocess.run(['usermod', '-g', group, '-G', \",\".join(groups), user], check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as e:\n self.stats.failure('do_change_groups')\n self.logger.info(f\"do_change_groups {user} {group} {groups} failed: {e.output.decode()}\")\n return\n self.stats.success('do_change_groups')\n self.logger.info(f\"do_change_groups {user} {group} {groups} succeed\")\n\n def do_chmod(self, entry, mode, user):\n abspath = os.path.join(self.root_dir, entry)\n try:\n self.seteuid(user)\n if self.client:\n self.client.chmod(self.get_sdk_path(abspath), mode)\n else:\n os.chmod(abspath, mode)\n # self.run_cmd(f'sudo -u {user} chmod {mode} {abspath}', root_dir)\n except Exception as e:\n return self.handleException(e, 'do_chmod', abspath, mode=mode, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_chmod')\n self.logger.info(f\"do_chmod {abspath} mode={oct(mode)} user={user} succeed\")\n return self.do_stat(entry, user)\n\n def do_get_acl(self, entry: str):\n abspath = os.path.join(self.root_dir, entry)\n try:\n acl = get_acl(abspath)\n except Exception as e:\n return self.handleException(e, 'do_get_acl', abspath)\n self.stats.success('do_get_acl')\n self.logger.info(f\"do_get_acl {abspath} succeed\")\n return acl\n\n def do_remove_acl(self, entry: str, option: str, user: str):\n abspath = os.path.join(self.root_dir, entry)\n try:\n self.run_cmd(f'sudo -u {user} setfacl {option} {abspath} ')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_remove_acl', abspath, option=option,user=user)\n self.stats.success('do_remove_acl')\n self.logger.info(f\"do_remove_acl {abspath} with {option} succeed\")\n return get_acl(abspath)\n \n def do_set_acl(self, sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical):\n abspath = os.path.join(self.root_dir, entry)\n user_perm = ''.join(user_perm) == '' and '-' or ''.join(user_perm)\n group_perm = ''.join(group_perm) == '' and '-' or ''.join(group_perm)\n other_perm = ''.join(other_perm) == '' and '-' or ''.join(other_perm)\n mask = ''.join(mask) == '' and '-' or ''.join(mask)\n default = default and '-d' or ''\n recursive = recursive and '-R' or ''\n recalc_mask = recalc_mask and '--mask' or ''\n not_recalc_mask = not_recalc_mask and '--no-mask' or ''\n logical = (recursive and logical) and '-L' or ''\n physical = (recursive and physical) and '-P' or ''\n try:\n text = f'u:{user}:{user_perm},g:{group}:{group_perm},o::{other_perm}'\n if set_mask:\n text += f',m::{mask}'\n self.run_cmd(f'sudo -u {sudo_user} setfacl {default} {recursive} {recalc_mask} {not_recalc_mask} {logical} {physical} -m {text} {abspath}')\n acl = get_acl(abspath)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_set_acl', abspath, user_perm=user_perm, group_perm=group_perm, other_perm=other_perm)\n self.stats.success('do_set_acl')\n self.logger.info(f\"do_set_acl {abspath} with {text} succeed\")\n return (acl,)\n\n def do_utime(self, entry, access_time, modify_time, follow_symlinks, user):\n abspath = os.path.join(self.root_dir, entry)\n try:\n self.seteuid(user)\n if self.client:\n self.client.utime(self.get_sdk_path(abspath), (access_time, modify_time))\n else:\n os.utime(abspath, (access_time, modify_time), follow_symlinks=follow_symlinks)\n # self.run_cmd(f'sudo -u {user} touch -a -t {access_time} {abspath}', root_dir)\n # self.run_cmd(f'sudo -u {user} touch -m -t {modify_time} {abspath}', root_dir)\n except Exception as e:\n return self.handleException(e, 'do_utime', abspath, access_time=access_time, modify_time=modify_time, follow_symlinks=follow_symlinks, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_utime')\n self.logger.info(f\"do_utime {abspath} with access_time={access_time} modify_time={modify_time} follow_symlinks={follow_symlinks} user={user} succeed\")\n return self.do_stat(entry, user)\n\n def do_chown(self, entry, owner, user):\n abspath = os.path.join(self.root_dir, entry)\n info = pwd.getpwnam(owner)\n uid = info.pw_uid\n gid = info.pw_gid\n try:\n self.seteuid(user)\n if self.client:\n self.client.chown(self.get_sdk_path(abspath), uid, gid)\n else:\n os.chown(abspath, uid, gid)\n # self.run_cmd(f'sudo -u {user} chown {owner} {abspath}', root_dir)\n except Exception as e:\n return self.handleException(e, 'do_chown', abspath, owner=owner, user=user)\n finally:\n self.reset_euid()\n self.stats.success('do_chown')\n self.logger.info(f\"do_chown {abspath} with owner={owner} user={user} succeed\")\n return self.do_stat(entry, user)\n\n def do_split_dir(self, dir, vdirs):\n relpath = os.path.join(dir, f'.jfs_split#{vdirs}')\n abspath = os.path.join(self.root_dir, relpath)\n if not self.is_jfs:\n return \n try:\n subprocess.check_call(['touch', abspath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n except Exception as e:\n self.stats.failure('do_split_dir')\n self.logger.info(f\"do_split_dir {abspath} {vdirs} failed: {str(e)}\")\n return\n self.stats.success('do_split_dir')\n self.logger.info(f\"do_split_dir {abspath} {vdirs} succeed\")\n\n def do_merge_dir(self, dir):\n relpath = os.path.join(dir, f'.jfs_split#1')\n abspath = os.path.join(self.root_dir, relpath)\n if not self.is_jfs:\n return \n try:\n subprocess.run(['touch', abspath], check=True, capture_output=True, text=True)\n except subprocess.CalledProcessError as e:\n error = f'{e.cmd} exit with {e.returncode}, {e.stderr}'.strip()\n self.stats.failure('do_merge_dir')\n self.logger.info(f\"do_merge_dir {abspath} failed: {error}\")\n return\n self.stats.success('do_merge_dir')\n self.logger.info(f\"do_merge_dir {abspath} succeed\")\n\n def do_rebalance_with_pysdk(self, entry, zone, is_vdir):\n if zone == '':\n # print(f'{self.root_dir} is not multizoned, skip rebalance')\n return\n abspath = os.path.join(self.root_dir, entry)\n vdir_relpath = os.path.join(entry, '.jfs#1')\n vdir_abspath = os.path.join(self.root_dir, vdir_relpath)\n if is_vdir and os.path.isfile( vdir_abspath ):\n abspath = vdir_abspath\n try :\n dest = os.path.join(get_root(abspath), zone, os.path.basename(abspath.rstrip('/')))\n os.rename(abspath, dest)\n except Exception as e:\n self.stats.failure('do_rebalance')\n self.logger.info(f\"do_rebalance {abspath} {dest} failed: {str(e)}\")\n return\n self.stats.success('do_rebalance')\n self.logger.info(f\"do_rebalance {abspath} {dest} succeed\")\n\n def do_rebalance(self, entry, zone, is_vdir, pysdk=True):\n if zone == '':\n # print(f'{self.root_dir} is not multizoned, skip rebalance')\n return\n abspath = os.path.join(self.root_dir, entry)\n vdir_relpath = os.path.join(entry, '.jfs#1')\n vdir_abspath = os.path.join(self.root_dir, vdir_relpath)\n if is_vdir and os.path.isfile( vdir_abspath ):\n abspath = vdir_abspath\n try :\n dest = os.path.join(get_root(abspath), zone, os.path.basename(abspath.rstrip('/')))\n if pysdk:\n client = self.get_client_for_rebalance()\n if client.exists(self.get_sdk_path(abspath)):\n client.rename(self.get_sdk_path(abspath), self.get_sdk_path(dest))\n else:\n if os.path.exists(abspath):\n os.rename(abspath, dest)\n except OSError as e:\n self.stats.failure('do_rebalance')\n self.logger.info(f\"do_rebalance {abspath} {dest} failed: {str(e)}\")\n self.stats.success('do_rebalance')\n self.logger.info(f\"do_rebalance {abspath} {dest} succeed\")" }, { "path": ".github/scripts/hypo/fs_sdk_test.py", "content": "import unittest\nimport subprocess\ntry: \n __import__('xattr')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\nimport xattr\nfrom fs import JuicefsMachine\n\nclass TestPySdk(unittest.TestCase):\n def test_issue_1331(self):\n # SEE: https://github.com/juicedata/jfs/issues/1331\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.mkdir(mode=0o2164, parent=v1, subdir='ouyz', umask=0o022, user='root')\n state.teardown()\n\n def test_issue_1339(self):\n # SEE: https://github.com/juicedata/jfs/issues/1339\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.exists(entry=v1, user='root')\n v2 = state.loop_symlink(link_file_name='kydl', parent=v1)\n state.exists(entry=v2, user='root')\n state.teardown()\n \n def test_issue_1349(self):\n # SEE: https://github.com/juicedata/jfs/issues/1349\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.loop_symlink(link_file_name='pmjl', parent=v1)\n state.set_xattr(file=v2, flag=xattr.XATTR_CREATE, name='user.abc', user='root', value=b'def')\n state.teardown()\n\n def test_issue_1359(self):\n # SEE: https://github.com/juicedata/jfs/issues/1359\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.merge_dir(dir=v1)\n state.create_file(buffering=0, content=b'\\x16\\x0cu\\x01\\x01\\x01\\x01\\x01\\x01', file_name='bbbb', mode='ab', parent=v1, umask=18, user='root')\n state.teardown()\n\n def test_issue_1361(self):\n # SEE: https://github.com/juicedata/jfs/issues/1361\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(buffering=1, content=b'', file_name='abbc', mode='ab', parent=v1, umask=18, user='root')\n state.hardlink(src_file=v2, link_file_name='aaaa', parent=v1, umask=18, user='root')\n state.teardown()\n\n def test_issue_1362(self):\n #SEE: https://github.com/juicedata/jfs/issues/1362\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(buffering=-1, content=b'abc', file_name='iazj', mode='ab', parent=v1, umask=18, user='root')\n state.read(file=v2, length=4949, mode='w+', offset=1, user='root', whence=2)\n state.teardown()\n\n def test_issue_1364(self):\n # SEE: https://github.com/juicedata/jfs/issues/1364\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(buffering=10, content=b'', file_name='abag', mode='wb', parent=v1, umask=18, user='root')\n state.set_xattr(file=v2, flag=xattr.XATTR_REPLACE, name='user.abc', user='root', value=b'def')\n state.teardown()\n\n def test_issue_1365(self):\n # SEE: https://github.com/juicedata/jfs/issues/1365\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.mkdir(mode=15, parent=v1, subdir='coue', umask=18, user='root')\n state.rmdir(dir=v2, user='root')\n state.teardown()\n \n def test_issue_1369(self):\n # SEE: https://github.com/juicedata/jfs/issues/1369\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(buffering=-1, content=b'', file_name='aaaa', mode='wb', parent=v1, umask=18, user='root')\n state.write(content=b'abcd', file=v2, mode='rb', offset=0, user='root', whence=0)\n state.teardown()\n\n def test_issue_1369_2(self):\n # SEE: https://github.com/juicedata/jfs/issues/1369\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(buffering=-1, content=b'\\x00', file_name='aaaa', mode='wb', parent=v1, umask=18, user='root')\n state.write(content=b'', file=v2, mode='xb', offset=0, user='root', whence=0)\n state.teardown()\n\n def test_issue_1369_3(self):\n # SEE: https://github.com/juicedata/jfs/issues/1369\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'a', file_name='aaaa', mode='wb', parent=v1, umask=18, user='root')\n state.write(content=b'b', file=v2, mode='ab', offset=0, user='root', whence=0)\n state.teardown()\n\n def skip_test_issue_1370(self):\n # SEE: https://github.com/juicedata/jfs/issues/1370\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(buffering=-1, content=b'', file_name='aaab', mode='wb', parent=v1, umask=18, user='root')\n v3 = state.symlink(src_file=v2, link_file_name='aaaa', parent=v1, umask=18, user='root')\n state.unlink(file=v2, user='root')\n state.open2(file=v3, mode='w+', user='root')\n state.teardown()\n\n def test_issue_1419(self):\n # SEE: https://github.com/juicedata/jfs/issues/1419\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.lstat(entry=v1, user='root')\n v2 = state.create_file(content=b'^\\x85\\n\\xa1;1*ek\\xc8', file_name='d', parent=v1, umask=18, user='root')\n state.readline(file=v2, mode='a+', offset=9070, user='root', whence=0)\n state.teardown()\n\n def test_issue_1422(self):\n # SEE: https://github.com/juicedata/jfs/issues/1422\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'a', file_name='p', parent=v1, umask=18, user='root')\n v3 = state.symlink(src_file=v2, link_file_name='ab', parent=v1, umask=18, user='root')\n state.hardlink(src_file=v3, link_file_name='a', parent=v1, umask=18, user='root')\n state.teardown()\n\n def test_issue_1424(self):\n # SEE: https://github.com/juicedata/jfs/issues/1424\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content='²', mode='a', file_name='w', parent=v1, umask=18, user='root')\n state.readline(file=v2, mode='r', offset=1708, user='root', whence=0)\n state.teardown()\n\n def test_issue_1425(self):\n # SEE: https://github.com/juicedata/jfs/issues/1425\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'a', file_name='a', parent=v1, umask=18, user='root')\n v3 = state.mkdir(mode=0, parent=v1, subdir='b', umask=18, user='root')\n state.rename_dir(entry=v3, new_entry_name=v2, parent=v1, umask=18, user='root')\n state.teardown()\n\n def test_issue_1442(self):\n # SEE: https://github.com/juicedata/jfs/issues/1442\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'aqdranzfk', file_name='fz', parent=v1, umask=18, user='root')\n state.set_xattr(file=v2, flag=0, name='user.0', user='root', value=b'\\x01\\x01\\x00\\x01')\n state.teardown()\n\n def test_issue_1443(self):\n # SEE: https://github.com/juicedata/jfs/issues/1443\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'bcb', file_name='bcba', parent=v1, umask=18, user='root')\n v3 = state.hardlink(src_file=v2, link_file_name='a', parent=v1, umask=18, user='root')\n state.rename_file(entry=v2, new_entry_name=v3, parent=v1, umask=18, user='root')\n state.teardown()\n\n def test_issue_1449(self):\n # SEE: https://github.com/juicedata/jfs/issues/1449\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'\\x1d\\x00', file_name='b', parent=v1, umask=18, user='root')\n state.listdir(dir=v1, user='root')\n state.readline(file=v2, mode='r', offset=0, user='root', whence=0)\n state.teardown()\n\n def test_issue_1450(self):\n # SEE: https://github.com/juicedata/jfs/issues/1450\n state = JuicefsMachine()\n v1 = state.init_folders()\n v6 = state.create_file(content=b'\\xa5\\x08\\xee', file_name='mzeg', parent=v1, umask=18, user='root')\n state.readline(file=v6, mode='r+', offset=1, user='root', whence=0)\n state.teardown()\n\n def skip_test_issue_1450_2(self):\n # SEE: https://github.com/juicedata/jfs/issues/1450 \n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'10', file_name='v', parent=v1, umask=18, user='root')\n state.read(encoding='utf-16', errors='strict', file=v2, length=2, mode='r', offset=0, user='root', whence=0)\n state.teardown()\n\n def test_issue_1457(self):\n # SEE: https://github.com/juicedata/jfs/issues/1457\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content='\\ufeff', mode='x', file_name='v', parent=v1, umask=18, user='root')\n state.teardown()\n \n def skip_test_issue_1465(self):\n # SEE: https://github.com/juicedata/jfs/issues/1465\n state = JuicefsMachine()\n v1 = state.init_folders()\n v38 = state.create_file(content=b'\\x05{\\xf3\\x9bg\\x93\\x00\\ry0', file_name='kfhg', parent=v1, umask=18, user='root')\n state.readline(file=v38, mode='rb', offset=7694, user='root', whence=1)\n state.teardown()\n\n def test_issue_1481(self):\n # SEE: https://github.com/juicedata/jfs/issues/1481\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='b', parent=v1, umask=18, user='root')\n state.set_xattr(file=v2, flag=0, name='user.\\uda5d', user='root', value=b'!')\n state.teardown()\n\n def test_issue_x(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'\\x035\\x03\\x02\\x00', file_name='a', parent=v1, umask=18, user='root')\n state.lstat(entry=v1, user='root')\n v3 = state.create_file(content=b'10', file_name='v', parent=v1, umask=18, user='root')\n state.write(content=b'\\x01\\x01', encoding='utf-8', errors='ignore', file=v2, mode='r', offset=258, user='root', whence=1)\n state.teardown()\n\n def test_issue_y(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='a', parent=v1, umask=18, user='root')\n state.readlines(file=v2, mode='a', offset=0, user='root', whence=0)\n state.teardown()\n\n def test_issue_z(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'1}26B', file_name='gv', parent=v1, umask=18, user='root')\n # state.write(content='\\x04', encoding='utf-8', errors='ignore', file=v2, mode='a', offset=4900, user='root', whence=1)\n state.writelines(file=v2, lines=['hp', 'uwq'], mode='a+b', offset=160, user='root', whence=2)\n state.teardown()\n\n def test_issue_a(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='a', parent=v1, umask=18, user='root')\n state.writelines(file=v2, lines=[''], mode='rb', offset=9841, user='root', whence=1)\n state.teardown()\n\n def test_issue_b(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n v7 = state.create_file(content=b'', file_name='vkg', parent=v1, umask=18, user='root')\n state.write(content='í', encoding='ascii', errors='strict', file=v7, mode='r', offset=5117, user='root', whence=1)\n state.teardown()\n\n def test_issue_c(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.create_file(content='\\udfc5', mode='xb', file_name='a', parent=v1, umask=18, user='root')\n state.teardown()\n\n def test_issue_d(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n state.exists(entry=v1, user='root')\n v3 = state.create_file(content=b'\\x10\\x1ata\\xd6', file_name='x', parent=v1, umask=18, user='root')\n # v15 = state.create_file(content=b'', file_name='nbln', parent=v1, umask=18, user='root')\n # v20 = state.create_file(content=b'\\x82\\xd7\\xc0\\xff\\xac\\x94\\xe5\\x8f\\x03\\x10', file_name='exc', parent=v1, umask=18, user='root')\n # state.write(content=b'', encoding='utf-8', errors='backslashreplace', file=v20, mode='w+', offset=3658, user='root', whence=1)\n # state.write(content=b'7q\\x0b\\xe4\\x9f\\xb4b', encoding='latin-1', errors='namereplace', file=v15, mode='r+b', offset=6691, user='root', whence=2)\n state.write(content='È\\U000c3fe7𧶤÷\\x89\\x00𭊦cç¤Ìk', encoding='latin-1', errors='strict', file=v3, mode='r', offset=10240, user='root', whence=0)\n state.teardown()\n\n def test_issue_e(self):\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'abc\\r', file_name='a', parent=v1, umask=18, user='root')\n state.read(file=v2, mode='r', offset=0, user='root', whence=0, length=4)\n state.teardown()\n\n def test_issue_f(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n files_0 = state.create_file(content=b'', file_name='b', parent=folders_0, umask=18, user='root')\n state.chown(entry=folders_0, owner='user1', user='root')\n state.create_file(content=b'', file_name='a', parent=folders_0, umask=18, user='root')\n state.teardown()\n\n def test_rename_invalid_arg(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n files_0 = state.loop_symlink(link_file_name='aa', parent=folders_0, user='root')\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir='a', umask=18, user='root')\n folders_2 = state.mkdir(mode=0, parent=folders_1, subdir='b', umask=18, user='root')\n state.exists(entry=files_0, user='root')\n state.rename_dir(entry=folders_1, new_entry_name=folders_1, parent=folders_0, umask=18, user='root')\n state.rename_dir(entry=folders_0, new_entry_name=folders_1, parent=folders_1, umask=18, user='root')\n state.rename_dir(entry=folders_2, new_entry_name=folders_1, parent=folders_0, umask=18, user='root')\n state.teardown()\n\n def test_rename_to_dir_not_exist(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n files_0 = state.create_file(content=b'', file_name='aa', parent=folders_0, umask=18, user='root')\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir='a', umask=18, user='root')\n state.rename_dir(entry=folders_0, new_entry_name='a/a', parent=folders_1, umask=18, user='root')\n state.teardown()\n\n def test_rmdir_check_exist(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir='a', umask=18, user='root')\n files_0 = state.loop_symlink(link_file_name=folders_1, parent=folders_1, user='root')\n state.exists(entry=files_0, user='root')\n state.rmdir(dir=folders_1, user='root')\n state.teardown()\n\n def test_truncate(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n files_0 = state.loop_symlink(link_file_name='a', parent=folders_0, user='root')\n files_1 = state.rename_file(entry=files_0, new_entry_name='b', parent=folders_0, umask=18, user='root')\n state.listdir(dir=folders_0, user='root')\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir=files_0, umask=18, user='root')\n state.listdir(dir=files_0, user='root')\n state.truncate(file=files_0, size=0, user='root')\n state.teardown()\n\n def test_unlink(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n state.listdir(dir=folders_0, user='root')\n files_0 = state.loop_symlink(link_file_name='a', parent=folders_0, user='root')\n files_1 = state.rename_file(entry=files_0, new_entry_name='b', parent=folders_0, umask=18, user='root')\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir=files_0, umask=18, user='root')\n state.unlink(file=files_0, user='root')\n state.teardown()\n\n def test_read_utf8(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n files_0 = state.create_file(content=b'', file_name='aa', parent=folders_0, umask=18, user='root')\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir='a', umask=18, user='root')\n state.rename_dir(entry=folders_0, new_entry_name='a/a', parent=folders_1, umask=18, user='root')\n state.teardown()\n\n def test_create_isdir(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n state.listdir(dir=folders_0, user='root')\n files_0 = state.loop_symlink(link_file_name='a', parent=folders_0, user='root')\n state.chmod(entry=files_0, mode=0, user='root')\n files_1 = state.rename_file(entry=files_0, new_entry_name='b', parent=folders_0, umask=18, user='root')\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir=files_0, umask=18, user='root')\n state.truncate(file=files_0, size=0, user='root')\n state.readlink(file=files_0, user='root')\n state.writelines(file=files_0, lines=['dua', 'hbixuhv'], mode='wb', offset=8344, user='root', whence=1)\n state.teardown()\n\n def test_rename_file(self):\n state = JuicefsMachine()\n folders_0 = state.init_folders()\n files_0 = state.create_file(content=b'', file_name='q', parent=folders_0, umask=18, user='root')\n state.list_xattr(file=files_0, user='root')\n files_1 = state.create_file(content=b'', file_name='ezmt', parent=folders_0, umask=18, user='root')\n files_2 = state.create_file(content=b'', file_name='a', parent=folders_0, umask=18, user='root')\n state.list_xattr(file=files_2, user='root')\n files_3 = state.rename_file(entry=files_2, new_entry_name='buln', parent=folders_0, umask=18, user='root')\n state.list_xattr(file=files_3, user='root')\n state.list_xattr(file=files_3, user='root')\n files_4 = state.create_file(content=b'', file_name='sj', parent=folders_0, umask=18, user='root')\n state.list_xattr(file=files_4, user='root')\n files_5 = state.create_file(content=b'', file_name=files_2, parent=folders_0, umask=18, user='root')\n state.list_xattr(file=files_2, user='root')\n files_6 = state.create_file(content=b'', file_name='alwr', parent=folders_0, umask=18, user='root')\n files_7 = state.create_file(content=b'', file_name='m', parent=folders_0, umask=18, user='root')\n files_8 = state.symlink(link_file_name='rd', parent=folders_0, src_file=files_7, umask=18, user='root')\n folders_1 = state.mkdir(mode=0, parent=folders_0, subdir='n', umask=18, user='root')\n state.list_xattr(file=files_8, user='root')\n state.list_xattr(file=files_8, user='root')\n folders_2 = state.mkdir(mode=0, parent=folders_1, subdir=files_2, umask=18, user='root')\n files_9 = state.symlink(link_file_name='g', parent=folders_2, src_file=files_8, umask=18, user='root')\n files_10 = state.symlink(link_file_name=files_2, parent=folders_2, src_file=files_9, umask=18, user='root')\n files_11 = state.rename_file(entry=files_3, new_entry_name=files_2, parent=folders_2, umask=18, user='root')\n state.rename_file(entry=files_9, new_entry_name=files_2, parent=folders_0, umask=18, user='root')\n state.teardown()\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": ".github/scripts/hypo/fs_test.py", "content": "import os\nimport unittest\nfrom fs import JuicefsMachine\n\nclass TestFsrand2(unittest.TestCase):\n def test_issue_910(self):\n # See: https://github.com/juicedata/jfs/issues/910\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='aaaa', mode='wb', parent=v1, user='root')\n state.chmod(entry=v1, mode=32, user='root')\n state.listdir(dir=v1, user='root')\n state.change_groups(group='root', groups=['root'], user='user1')\n state.listdir(dir=v1, user='user1')\n state.teardown()\n\n def test_issue_914(self):\n # See: https://github.com/juicedata/jfs/issues/914\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'yl\\xff{', file_name='tadj', mode='xb', parent=v1, user='root')\n state.fallocate(file=v2, length=22911, mode=0, offset=7849, user='root')\n state.copy_file(entry=v2, follow_symlinks=True, new_entry_name='npyn', parent=v1, user='root')\n state.teardown()\n\n def skip_test_issue_918(self):\n # See: https://github.com/juicedata/jfs/issues/918\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='lcka', mode='wb', parent=v1, user='root')\n v3 = state.clone_cp_file(entry=v2, new_entry_name='bbbb', parent=v1, preserve=True, user='root')\n state.chmod(entry=v3, mode=258, user='root')\n v5 = state.clone_cp_file(entry=v3, new_entry_name='mbbb', parent=v1, preserve=True, user='root')\n state.teardown()\n\n def test_x(self):\n # See: https://github.com/juicedata/jfs/issues/918\n state = JuicefsMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='lcka', mode='wb', parent=v1, user='root')\n state.teardown()\n\nif __name__ == '__main__':\n unittest.main()" }, { "path": ".github/scripts/hypo/readme.md", "content": "1. format juicefs with trash day of 0. \n ./juicefs format sqlite3://test.db myjfs\n2. mount juicefs with xatrr enable.\n ./juicefs mount sqlite3://test.db /tmp/jfs --enable-xattr\n3. run the test.\n python3 .github/scripts/hypo/fs.py\n4. run the test with custom examples and step count to reach deep bugs.\n MAX_EXAMPLE=1000 STEP_COUNT=500 .github/scripts/hypo/fs.py\n5. you can modify EXCLUDE_RULES to skip running some operations." }, { "path": ".github/scripts/hypo/s3.py", "content": "import json\nimport os\nfrom string import ascii_lowercase\nimport subprocess\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hypothesis\"])\nfrom hypothesis import assume, settings, Verbosity\nfrom hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes\nfrom hypothesis import Phase, seed\nfrom hypothesis import strategies as st\nfrom hypothesis.database import DirectoryBasedExampleDatabase\nimport random\nfrom s3_op import S3Client\nfrom s3_strategy import *\nfrom s3_contant import *\nimport common\n\nSEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))\n@seed(SEED)\nclass S3Machine(RuleBasedStateMachine):\n aliases = Bundle('aliases')\n buckets = Bundle('buckets')\n objects = Bundle('objects')\n users = Bundle('users')\n groups = Bundle('groups')\n policies = Bundle('policies')\n user_policies = Bundle('user_policy')\n group_policies = Bundle('group_policy')\n PREFIX1 = 'minio'\n PREFIX2 = 'juice'\n URL1 = 'localhost:9000'\n URL2 = 'localhost:9005'\n URL3 = 'localhost:9006'\n client1 = S3Client(prefix=PREFIX1, url=URL1)\n client2 = S3Client(prefix=PREFIX2, url=URL2, url2=URL3)\n EXCLUDE_RULES = []\n\n def __init__(self):\n super().__init__()\n self.client1.remove_all_aliases()\n self.client2.remove_all_aliases()\n self.client1.do_set_alias(ROOT_ALIAS, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, self.URL1)\n self.client2.do_set_alias(ROOT_ALIAS, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, self.URL2)\n self.client1.remove_all_buckets()\n self.client2.remove_all_buckets()\n self.client1.remove_all_users()\n self.client2.remove_all_users()\n self.client1.remove_all_groups()\n self.client2.remove_all_groups()\n self.client1.remove_all_policies()\n self.client2.remove_all_policies()\n\n @initialize(target=aliases)\n def init_aliases(self):\n return ROOT_ALIAS\n\n @initialize(target=policies)\n def init_policies(self):\n return multiple(*BUILD_IN_POLICIES)\n\n def equal(self, result1, result2):\n if os.getenv('PROFILE', 'dev') == 'generate':\n return True\n if type(result1) != type(result2):\n return False\n if isinstance(result1, Exception):\n result1 = str(result1)\n result2 = str(result2)\n result1 = common.replace(result1, self.PREFIX1, '***')\n result1 = common.replace(result1, self.URL1, '***')\n result2 = common.replace(result2, self.PREFIX2, '***')\n result2 = common.replace(result2, self.URL2, '***')\n # print(f'result1 is {result1}\\nresult2 is {result2}')\n return result1 == result2\n\n @rule(alias = aliases)\n @precondition(lambda self: False)\n def info(self, alias=ROOT_ALIAS):\n result1 = self.client1.do_info(alias)\n result2 = self.client2.do_info(alias)\n assert self.equal(result1, result2), f'\\033[31minfo:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(alias = aliases)\n @precondition(lambda self: 'list_buckets' not in self.EXCLUDE_RULES)\n def list_buckets(self, alias=ROOT_ALIAS):\n result1 = self.client1.do_list_buckets(alias)\n result2 = self.client2.do_list_buckets(alias)\n assert self.equal(result1, result2), f'\\033[31mdo_list_buckets:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target = buckets,\n alias = aliases,\n bucket_name = st_bucket_name)\n @precondition(lambda self: 'create_bucket' not in self.EXCLUDE_RULES)\n def create_bucket(self, bucket_name, alias = ROOT_ALIAS):\n result1 = self.client1.do_create_bucket(bucket_name, alias)\n result2 = self.client2.do_create_bucket(bucket_name, alias)\n assert self.equal(result1, result2), f'\\033[31mcreate_bucket:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return bucket_name\n @rule(\n target = buckets, \n bucket_name = consumes(buckets),\n alias = aliases\n )\n @precondition(lambda self: 'remove_bucket' not in self.EXCLUDE_RULES)\n def remove_bucket(self, bucket_name, alias = ROOT_ALIAS):\n result1 = self.client1.do_remove_bucket(bucket_name, alias)\n result2 = self.client2.do_remove_bucket(bucket_name, alias)\n assert self.equal(result1, result2), f'\\033[31mremove_bucket:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return bucket_name\n else:\n return multiple()\n\n @rule(\n alias = aliases,\n bucket_name = buckets.filter(lambda x: x != multiple()),\n policy = st.sampled_from(['public', 'download', 'upload', 'none'])\n )\n @precondition(lambda self: 'set_bucket_policy' not in self.EXCLUDE_RULES)\n def set_bucket_policy(self, bucket_name, policy, alias=ROOT_ALIAS):\n result1 = self.client1.do_set_bucket_policy(bucket_name, policy, alias)\n result2 = self.client2.do_set_bucket_policy(bucket_name, policy, alias)\n assert self.equal(result1, result2), f'\\033[31mset_bucket_policy:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n bucket_name = buckets,\n alias = aliases\n )\n @precondition(lambda self: 'get_bucket_policy' not in self.EXCLUDE_RULES)\n def get_bucket_policy(self, bucket_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_get_bucket_policy(bucket_name, alias)\n result2 = self.client2.do_get_bucket_policy(bucket_name, alias)\n assert self.equal(result1, result2), f'\\033[31mget_bucket_policy:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n bucket_name = buckets,\n alias = aliases, \n recursive = st.booleans()\n )\n def list_bucket_policy(self, bucket_name, alias=ROOT_ALIAS, recursive=False):\n result1 = self.client1.do_list_bucket_policy(bucket_name, alias, recursive)\n result2 = self.client2.do_list_bucket_policy(bucket_name, alias, recursive)\n assert self.equal(result1, result2), f'\\033[31mlist_bucket_policy:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target=objects,\n bucket_name = buckets,\n object_name = st_object_name, \n data = st_content,\n use_part_size = st.booleans(),\n part_size = st_part_size\n )\n @precondition(lambda self: 'put_object' not in self.EXCLUDE_RULES)\n def put_object(self, bucket_name, object_name, data, use_part_size=False, part_size=5*1024*1024):\n if use_part_size:\n result1 = self.client1.do_put_object(bucket_name, object_name, data, -1, part_size=part_size)\n result2 = self.client2.do_put_object(bucket_name, object_name, data, -1, part_size=part_size)\n else:\n result1 = self.client1.do_put_object(bucket_name, object_name, data, len(data))\n result2 = self.client2.do_put_object(bucket_name, object_name, data, len(data))\n assert self.equal(result1, result2), f'\\033[31mput_object:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return f'{bucket_name}:{object_name}'\n\n @rule(\n obj = objects,\n offset = st_offset, \n length = st_length\n )\n @precondition(lambda self: 'get_object' not in self.EXCLUDE_RULES)\n def get_object(self, obj:str, offset=0, length=0):\n bucket_name = obj.split(':')[0]\n object_name = obj.split(':')[1]\n result1 = self.client1.do_get_object(bucket_name, object_name, offset=offset, length=length)\n result2 = self.client2.do_get_object(bucket_name, object_name, offset=offset, length=length)\n assert self.equal(result1, result2), f'\\033[31mget_object:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target=objects,\n alias = aliases,\n bucket_name = buckets,\n object_name = st_object_name)\n @precondition(lambda self: 'fput_object' not in self.EXCLUDE_RULES)\n def fput_object(self, bucket_name, object_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_fput_object(bucket_name, object_name, 'README.md', alias)\n result2 = self.client2.do_fput_object(bucket_name, object_name, 'README.md', alias)\n assert self.equal(result1, result2), f'\\033[31mfput_object:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return f'{bucket_name}:{object_name}'\n\n @rule(\n obj = objects,\n alias = aliases,\n file_path = st.just('/tmp/file')\n )\n @precondition(lambda self: 'fget_object' not in self.EXCLUDE_RULES)\n def fget_object(self, obj:str, file_path, alias = ROOT_ALIAS):\n bucket_name = obj.split(':')[0]\n object_name = obj.split(':')[1]\n result1 = self.client1.do_fget_object(bucket_name, object_name, file_path, alias)\n result2 = self.client2.do_fget_object(bucket_name, object_name, file_path, alias)\n assert self.equal(result1, result2), f'\\033[31mfget_object:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target = objects,\n alias = aliases,\n obj = consumes(objects)\n )\n @precondition(lambda self: 'remove_object' not in self.EXCLUDE_RULES)\n def remove_object(self, obj:str, alias=ROOT_ALIAS):\n bucket_name = obj.split(':')[0]\n object_name = obj.split(':')[1]\n result1 = self.client1.do_remove_object(bucket_name, object_name, alias)\n result2 = self.client2.do_remove_object(bucket_name, object_name, alias)\n assert self.equal(result1, result2), f'\\033[31mremove_object:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return obj\n else:\n return multiple()\n \n @rule(\n obj = objects, \n alias = aliases\n )\n @precondition(lambda self: 'stat_object' not in self.EXCLUDE_RULES)\n def stat_object(self, obj:str, alias=ROOT_ALIAS):\n bucket_name = obj.split(':')[0]\n object_name = obj.split(':')[1]\n result1 = self.client1.do_stat_object(bucket_name, object_name, alias)\n result2 = self.client2.do_stat_object(bucket_name, object_name, alias)\n assert self.equal(result1, result2), f'\\033[31mstat_object:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n bucket_name = buckets,\n prefix = st.just(None), # st.one_of(st_object_prefix, st.just(None)),\n start_after = st.one_of(st_object_name, st.just(None)),\n include_user_meta = st.booleans(),\n include_version = st.just(False),\n use_url_encoding_type = st.booleans(),\n recursive=st.booleans())\n @precondition(lambda self: 'list_objects' not in self.EXCLUDE_RULES)\n def list_objects(self, bucket_name, prefix=None, start_after=None, include_user_meta=False, include_version=False, use_url_encoding_type=True, recursive=False):\n result1 = self.client1.do_list_objects(bucket_name=bucket_name, prefix=prefix, start_after=start_after, include_user_meta=include_user_meta, include_version=include_version, use_url_encoding_type=use_url_encoding_type, recursive=recursive)\n result2 = self.client2.do_list_objects(bucket_name=bucket_name, prefix=prefix, start_after=start_after, include_user_meta=include_user_meta, include_version=include_version, use_url_encoding_type=use_url_encoding_type, recursive=recursive)\n assert self.equal(result1, result2), f'\\033[31mlist_objects:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target = users,\n alias = aliases,\n user_name = st_user_name, \n )\n @precondition(lambda self: 'add_user' not in self.EXCLUDE_RULES)\n def add_user(self, user_name, secret_key=DEFAULT_SECRET_KEY, alias = ROOT_ALIAS):\n result1 = self.client1.do_add_user(user_name, secret_key, alias)\n result2 = self.client2.do_add_user(user_name, secret_key, alias)\n assert self.equal(result1, result2), f'\\033[31madd_user:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return user_name\n \n @rule(\n target = users,\n alias = aliases,\n user_name = consumes(users).filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'remove_user' not in self.EXCLUDE_RULES)\n def remove_user(self, user_name, alias = ROOT_ALIAS):\n result1 = self.client1.do_remove_user(user_name, alias)\n result2 = self.client2.do_remove_user(user_name, alias)\n assert self.equal(result1, result2), f'\\033[31mremove_user:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return user_name\n else:\n return multiple()\n\n @rule(\n alias = aliases,\n user_name = users.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'enable_user' not in self.EXCLUDE_RULES)\n def enable_user(self, user_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_enable_user(user_name, alias)\n result2 = self.client2.do_enable_user(user_name, alias)\n assert self.equal(result1, result2), f'\\033[31menable_user:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n alias = aliases,\n user_name = users.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'disable_user' not in self.EXCLUDE_RULES)\n def disable_user(self, user_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_disable_user(user_name, alias)\n result2 = self.client2.do_disable_user(user_name, alias)\n assert self.equal(result1, result2), f'\\033[31mdisable_user:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n alias = aliases,\n user_name = users.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'user_info' not in self.EXCLUDE_RULES)\n def user_info(self, user_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_user_info(user_name, alias)\n result2 = self.client2.do_user_info(user_name, alias)\n assert self.equal(result1, result2), f'\\033[31muser_info:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(alias = aliases)\n @precondition(lambda self: 'list_users' not in self.EXCLUDE_RULES)\n def list_users(self, alias=ROOT_ALIAS):\n result1 = self.client1.do_list_users(alias)\n result2 = self.client2.do_list_users(alias)\n assert self.equal(result1, result2), f'\\033[31mlist_users:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(alias = aliases)\n @precondition(lambda self: 'list_groups' not in self.EXCLUDE_RULES)\n def list_groups(self, alias=ROOT_ALIAS):\n result1 = self.client1.do_list_groups(alias)\n result2 = self.client2.do_list_groups(alias)\n assert self.equal(result1, result2), f'\\033[31mlist_groups:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target = groups, \n alias = aliases,\n group_name=st_group_name, \n members = st.lists(users, min_size=1, max_size=3)\n )\n @precondition(lambda self: 'add_group' not in self.EXCLUDE_RULES)\n def add_group(self, group_name, members, alias=ROOT_ALIAS):\n result1 = self.client1.do_add_group(group_name, members, alias)\n result2 = self.client2.do_add_group(group_name, members, alias)\n assert self.equal(result1, result2), f'\\033[31madd_group:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return group_name\n \n @rule(\n group_name = groups, \n alias = aliases)\n @precondition(lambda self: 'group_info' not in self.EXCLUDE_RULES)\n def group_info(self, group_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_group_info(group_name, alias)\n result2 = self.client2.do_group_info(group_name, alias)\n assert self.equal(result1, result2), f'\\033[31mgroup_info:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n \n @rule(\n target = groups,\n alias = aliases,\n group_name=consumes(groups).filter(lambda x: x != multiple()),\n group_members = st_group_members\n )\n @precondition(lambda self: 'remove_group' not in self.EXCLUDE_RULES)\n def remove_group(self, group_name, group_members, alias=ROOT_ALIAS):\n result1 = self.client1.do_remove_group(group_name, group_members, alias)\n result2 = self.client2.do_remove_group(group_name, group_members, alias)\n assert self.equal(result1, result2), f'\\033[31mremove_group:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return group_name\n else:\n return multiple()\n \n @rule(\n alias = aliases,\n group_name=groups.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'disable_group' not in self.EXCLUDE_RULES)\n def disable_group(self, group_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_disable_group(group_name, alias)\n result2 = self.client2.do_disable_group(group_name, alias)\n assert self.equal(result1, result2), f'\\033[31mdisable_group:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n alias = aliases,\n group_name=groups.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'enable_group' not in self.EXCLUDE_RULES)\n def enable_group(self, group_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_enable_group(group_name, alias)\n result2 = self.client2.do_enable_group(group_name, alias)\n assert self.equal(result1, result2), f'\\033[31menable_group:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target = policies,\n alias = st.just(ROOT_ALIAS),\n policy_name = st_policy_name,\n policy_document = st_policy\n )\n @precondition(lambda self: 'add_policy' not in self.EXCLUDE_RULES)\n def add_policy(self, policy_name, policy_document, alias=ROOT_ALIAS):\n result1 = self.client1.do_add_policy(policy_name, policy_document, alias)\n result2 = self.client2.do_add_policy(policy_name, policy_document, alias)\n assert self.equal(result1, result2), f'\\033[31madd_policy:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return policy_name\n \n @rule(\n target = policies,\n alias = st.just(ROOT_ALIAS),\n policy_name = consumes(policies).filter(lambda x: x != multiple()).filter(lambda x: x not in BUILD_IN_POLICIES)\n )\n @precondition(lambda self: 'remove_policy' not in self.EXCLUDE_RULES)\n def remove_policy(self, policy_name, alias=ROOT_ALIAS):\n assume(policy_name not in BUILD_IN_POLICIES)\n assert policy_name not in BUILD_IN_POLICIES, f'policy_name {policy_name} is in BUILD_IN_POLICIES'\n result1 = self.client1.do_remove_policy(policy_name, alias)\n result2 = self.client2.do_remove_policy(policy_name, alias)\n assert self.equal(result1, result2), f'\\033[31mremove_policy:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return policy_name\n else:\n return multiple()\n\n @rule(\n alias = st.just(ROOT_ALIAS),\n policy_name = policies.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'policy_info' not in self.EXCLUDE_RULES)\n def policy_info(self, policy_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_policy_info(policy_name, alias)\n result2 = self.client2.do_policy_info(policy_name, alias)\n assert self.equal(result1, result2), f'\\033[31mpolicy_info:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n \n @rule(alias = st.just(ROOT_ALIAS))\n @precondition(lambda self: 'list_policies' not in self.EXCLUDE_RULES)\n def list_policies(self, alias=ROOT_ALIAS):\n result1 = self.client1.do_list_policies(alias)\n result2 = self.client2.do_list_policies(alias)\n assert self.equal(result1, result2), f'\\033[31mlist_policies:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n\n @rule(\n target = user_policies,\n alias = st.just(ROOT_ALIAS),\n user_name = users.filter(lambda x: x != multiple()),\n policy_name = policies.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'set_policy_to_user' not in self.EXCLUDE_RULES)\n def set_policy_to_user(self, policy_name, user_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_set_policy_to_user(policy_name, user_name, alias)\n result2 = self.client2.do_set_policy_to_user(policy_name, user_name, alias)\n assert self.equal(result1, result2), f'\\033[31mset_policy_to_user:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return f'{user_name}:{policy_name}'\n\n @rule(\n target = group_policies, \n alias = st.just(ROOT_ALIAS),\n group_name = groups.filter(lambda x: x != multiple()),\n policy_name = policies.filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'set_policy_to_group' not in self.EXCLUDE_RULES)\n def set_policy_to_group(self, group_name, policy_name, alias=ROOT_ALIAS):\n result1 = self.client1.do_set_policy_to_group(policy_name, group_name, alias)\n result2 = self.client2.do_set_policy_to_group(policy_name, group_name, alias)\n assert self.equal(result1, result2), f'\\033[31mset_policy_to_group:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return f'{group_name}:{policy_name}'\n \n @rule(\n target = user_policies,\n alias = st.just(ROOT_ALIAS),\n user_policy = consumes(user_policies).filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'unset_policy_from_user' not in self.EXCLUDE_RULES)\n def unset_policy_from_user(self, user_policy:str, alias=ROOT_ALIAS):\n user_name = user_policy.split(':')[0]\n policy_name = user_policy.split(':')[1]\n result1 = self.client1.do_unset_policy_from_user(policy_name, user_name, alias)\n result2 = self.client2.do_unset_policy_from_user(policy_name, user_name, alias)\n assert self.equal(result1, result2), f'\\033[31munset_policy_from_user:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return user_policy\n else:\n return multiple()\n \n @rule(\n target = group_policies,\n alias = st.just(ROOT_ALIAS),\n group_policy = consumes(group_policies).filter(lambda x: x != multiple())\n )\n @precondition(lambda self: 'unset_policy_from_group' not in self.EXCLUDE_RULES)\n def unset_policy_from_group(self, group_policy:str, alias=ROOT_ALIAS):\n group_name = group_policy.split(':')[0]\n policy_name = group_policy.split(':')[1]\n result1 = self.client1.do_unset_policy_from_group(policy_name, group_name, alias)\n result2 = self.client2.do_unset_policy_from_group(policy_name, group_name, alias)\n assert self.equal(result1, result2), f'\\033[31munset_policy_from_group:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return group_policy\n else:\n return multiple()\n\n @rule(\n target=aliases, \n alias = st_alias_name,\n user_name = st_user_name,\n url1=st.just(URL1),\n url2=st.sampled_from([URL2])\n )\n @precondition(lambda self: 'set_alias' not in self.EXCLUDE_RULES)\n def set_alias(self, alias, user_name, url1=URL1, url2=URL2):\n result1 = self.client1.do_set_alias(alias, user_name, DEFAULT_SECRET_KEY, url1)\n result2 = self.client2.do_set_alias(alias, user_name, DEFAULT_SECRET_KEY, url2)\n assert self.equal(result1, result2), f'\\033[31mset_alias:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return alias\n\n @rule(\n target = aliases,\n alias = consumes(aliases)\n )\n @precondition(lambda self: 'remove_alias' not in self.EXCLUDE_RULES)\n def remove_alias(self, alias):\n assume(alias != ROOT_ALIAS)\n result1 = self.client1.do_remove_alias(alias)\n result2 = self.client2.do_remove_alias(alias)\n assert self.equal(result1, result2), f'\\033[31mremove_alias:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return alias\n else:\n return multiple()\n \n def teardown(self):\n pass\n\nif __name__ == '__main__':\n MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))\n STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))\n ci_db = DirectoryBasedExampleDatabase(\".hypothesis/examples\") \n settings.register_profile(\"dev\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target])\n settings.register_profile(\"schedule\", max_examples=500, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=200, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target], \n database=ci_db)\n settings.register_profile(\"pull_request\", max_examples=100, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=30, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target], \n database=ci_db)\n if os.environ.get('CI'):\n event_name = os.environ.get('GITHUB_EVENT_NAME')\n if event_name == 'schedule' or event_name == 'workflow_dispatch':\n profile = 'schedule'\n else:\n profile = 'pull_request'\n else:\n profile = os.environ.get('PROFILE', 'dev')\n print(f'profile is {profile}')\n settings.load_profile(profile)\n \n s3machine = S3Machine.TestCase()\n s3machine.runTest()\n print(json.dumps(S3Client.stats.get(), sort_keys=True, indent=4))\n \n " }, { "path": ".github/scripts/hypo/s3_contant.py", "content": "ROOT_ALIAS = 'admin'\nROOT_ACCESS_KEY = 'minioadmin'\nROOT_SECRET_KEY = 'minioadmin'\nDEFAULT_ACCESS_KEY = ROOT_ACCESS_KEY\nDEFAULT_SECRET_KEY = ROOT_SECRET_KEY\nBUILD_IN_POLICIES = ('consoleAdmin', 'readonly', 'readwrite', 'writeonly')" }, { "path": ".github/scripts/hypo/s3_op.py", "content": "import hashlib\nimport json\nimport os\nimport re\nimport subprocess\ntry: \n __import__('xattr')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\ntry: \n __import__('minio')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"minio\"])\ntry: \n __import__('fallocate')\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"fallocate\"])\nfrom stats import Statistics\nfrom minio.error import S3Error\nimport common\nfrom minio import Minio\nimport io\nfrom s3_contant import *\n\nclass S3Client():\n stats = Statistics()\n def __init__(self, prefix, url, url2=None):\n self.prefix = prefix\n self.url = url\n self.url2 = url2\n log_level=os.environ.get('LOG_LEVEL', 'INFO')\n self.logger = common.setup_logger(f'./{prefix}.log', f'{prefix}', log_level)\n\n def run_cmd(self, command:str, stderr=subprocess.STDOUT) -> str:\n self.logger.info(f'run_cmd: {command}')\n if '|' in command or '>' in command or '&' in command:\n ret=os.system(command)\n if ret == 0:\n return ret\n else: \n raise Exception(f\"run command {command} failed with {ret}\")\n try:\n output = subprocess.run(command.split(), check=True, stdout=subprocess.PIPE, stderr=stderr)\n except subprocess.CalledProcessError as e:\n raise e\n return output.stdout.decode()\n \n def sort_dict(self, obj):\n if isinstance(obj, dict):\n return {k: self.sort_dict(v) for k, v in obj.items()}\n elif isinstance(obj, list) and all(isinstance(elem, (int, float, str)) for elem in obj):\n return sorted(obj)\n elif isinstance(obj, list) and all(isinstance(elem, dict) for elem in obj):\n return [self.sort_dict(elem) for elem in obj]\n else:\n return obj\n \n def handleException(self, e, action, **kwargs):\n self.stats.failure(action)\n if isinstance(e, S3Error):\n self.logger.info(f'{action} {kwargs} failed: {e}')\n return Exception(f'code:{e.code} message:{e.message}')\n elif isinstance(e, subprocess.CalledProcessError):\n self.logger.info(f'{action} {kwargs} failed: {e.output.decode()}')\n try:\n output = json.loads(e.output.decode())\n message = output.get('error', {}).get('message', 'error message not found')\n return Exception(f'returncode:{e.returncode} {message}')\n except ValueError as ve:\n output = e.output.decode()\n output = re.sub(r'\\b\\d+\\.\\d+\\b|\\b\\d+\\b', '***', output)\n return Exception(f'returncode:{e.returncode} output:{output}')\n else:\n self.logger.info(f'{action} {kwargs} failed: {e}')\n return e\n \n def do_info(self, alias):\n try:\n self.run_cmd(f'mc admin info {self.get_alias(alias)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_info')\n self.stats.success('do_info')\n self.logger.info(f'do_info succeed')\n return True\n\n def remove_all_buckets(self):\n client=Minio(self.url,access_key=ROOT_ACCESS_KEY,secret_key=ROOT_SECRET_KEY,secure=False)\n buckets = client.list_buckets()\n for bucket in buckets:\n bucket_name = bucket.name\n objects = client.list_objects(bucket_name, recursive=True)\n for obj in objects:\n client.remove_object(bucket_name, obj.object_name)\n client.remove_bucket(bucket_name)\n print(f\"Bucket '{bucket_name}' removed successfully.\")\n \n def do_list_buckets(self, alias):\n try:\n result = self.run_cmd(f'mc ls {self.get_alias(alias)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_list_buckets')\n self.stats.success('do_list_buckets')\n self.logger.info(f'do_list_buckets succeed')\n result = [item.split()[-1][:-1] for item in result.split(\"\\n\") if item.strip()]\n # print(result)\n return sorted(result)\n \n def do_remove_bucket(self, bucket_name:str, alias):\n try:\n self.run_cmd(f'mc rb {self.get_alias(alias)}/{bucket_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_remove_bucket', bucket_name=bucket_name, alias=alias)\n self.stats.success('do_remove_bucket')\n self.logger.info(f'do_remove_bucket {alias} {bucket_name} succeed')\n return True\n\n def do_create_bucket(self, bucket_name:str, alias):\n try:\n self.run_cmd(f'mc mb {self.get_alias(alias)}/{bucket_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_create_bucket', bucket_name=bucket_name)\n self.stats.success('do_create_bucket')\n self.logger.info(f'do_create_bucket {bucket_name} succeed')\n return True\n\n def do_set_bucket_policy(self, bucket_name:str, policy:str, alias):\n try:\n self.run_cmd(f'mc policy set {policy} {self.get_alias(alias)}/{bucket_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_set_bucket_policy', bucket_name=bucket_name, policy=policy)\n self.stats.success('do_set_bucket_policy')\n self.logger.info(f'do_set_bucket_policy {bucket_name} {policy} succeed')\n return True\n \n def do_get_bucket_policy(self, bucket_name:str, alias):\n try:\n result = self.run_cmd(f'mc policy get {self.get_alias(alias)}/{bucket_name} --json')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_get_bucket_policy', bucket_name=bucket_name)\n self.stats.success('do_get_bucket_policy')\n self.logger.info(f'do_get_bucket_policy {bucket_name} succeed')\n return self.sort_dict(json.loads(result))\n\n def do_list_bucket_policy(self, bucket_name:str, alias, recursive=False):\n try:\n cmd = f'mc policy list {self.get_alias(alias)}/{bucket_name}'\n if recursive:\n cmd += ' --recursive'\n result = self.run_cmd(cmd)\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_list_bucket_policy', bucket_name=bucket_name)\n self.stats.success('do_list_bucket_policy')\n self.logger.info(f'do_list_bucket_policy {bucket_name} succeed')\n return sorted(result.split(\"\\n\"))\n\n def do_stat_object(self, bucket_name:str, object_name:str, alias):\n try:\n result = self.run_cmd(f'mc stat {self.get_alias(alias)}/{bucket_name}/{object_name} ')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_stat_object', bucket_name=bucket_name, object_name=object_name)\n stat = {}\n for line in result.split('\\n'):\n if line.strip() and ':' in line:\n key, value = line.split(':', 1)\n stat[key.strip()] = value.strip()\n self.stats.success('do_stat_object')\n self.logger.info(f'do_stat_object {bucket_name} {object_name} succeed')\n # print(stat)\n return stat['Name'], stat['Size'], stat['ETag'], stat['Type']\n\n def do_put_object(self, bucket_name:str, object_name:str, data, length, content_type='application/octet-stream', part_size=5*1024*1024):\n client=Minio(self.url,access_key=ROOT_ACCESS_KEY,secret_key=ROOT_SECRET_KEY,secure=False)\n try:\n client.put_object(bucket_name, object_name, io.BytesIO(data), length=length, content_type=content_type, part_size=part_size)\n except S3Error as e:\n return self.handleException(e, 'do_put_object', bucket_name=bucket_name, object_name=object_name, length=length, part_size=part_size)\n self.stats.success('do_put_object')\n self.logger.info(f'do_put_object {bucket_name} {object_name} succeed')\n return self.do_stat_object(bucket_name, object_name, alias=ROOT_ALIAS)\n\n def do_get_object(self, bucket_name:str, object_name:str, offset=0, length=0):\n client=Minio(self.url,access_key=ROOT_ACCESS_KEY,secret_key=ROOT_SECRET_KEY,secure=False)\n try:\n stat = client.stat_object(bucket_name, object_name)\n if stat.size == 0:\n offset = 0\n else:\n offset = offset % stat.size\n if length > stat.size - offset:\n length = stat.size - offset\n response = client.get_object(bucket_name, object_name, offset=offset, length=length)\n md5_hash = hashlib.md5()\n for data in response.stream(32*1024):\n md5_hash.update(data)\n md5_hex = md5_hash.hexdigest()\n except S3Error as e:\n return self.handleException(e, 'do_get_object', bucket_name=bucket_name, object_name=object_name, offset=offset, length=length)\n self.stats.success('do_get_object')\n self.logger.info(f'do_get_object {bucket_name} {object_name} succeed')\n return md5_hex\n\n def do_fput_object(self, bucket_name:str, object_name:str, src_path:str, alias):\n try:\n self.run_cmd(f'mc cp {src_path} {self.get_alias(alias)}/{bucket_name}/{object_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_fput_object', bucket_name=bucket_name, object_name=object_name, src_path=src_path)\n self.stats.success('do_fput_object')\n self.logger.info(f'do_fput_object {bucket_name} {object_name} {src_path} succeed')\n return self.do_stat_object(bucket_name, object_name, alias=ROOT_ALIAS)\n \n def do_fget_object(self, bucket_name:str, object_name:str, file_path:str, alias):\n try:\n self.run_cmd(f'mc cp {self.get_alias(alias)}/{bucket_name}/{object_name} {file_path}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_fget_object', bucket_name=bucket_name, object_name=object_name, file_path=file_path)\n self.stats.success('do_fget_object')\n self.logger.info(f'do_fget_object {bucket_name} {object_name} {file_path} succeed')\n return os.stat(file_path).st_size\n\n def object_exists(self, bucket_name:str, object_name:str, alias):\n try:\n self.run_cmd(f'mc stat {self.get_alias(alias)}/{bucket_name}/{object_name}')\n except subprocess.CalledProcessError as e:\n return False\n return True\n\n def do_remove_object(self, bucket_name:str, object_name:str, alias):\n try:\n self.run_cmd(f'mc rm {self.get_alias(alias)}/{bucket_name}/{object_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_remove_object', bucket_name=bucket_name, object_name=object_name)\n assert not self.object_exists(bucket_name, object_name, ROOT_ALIAS)\n self.stats.success('do_remove_object')\n self.logger.info(f'do_remove_object {bucket_name} {object_name} succeed')\n return True\n \n def do_list_objects(self, bucket_name, prefix, start_after, include_user_meta, include_version, use_url_encoding_type, recursive):\n client=Minio(self.url,access_key=ROOT_ACCESS_KEY,secret_key=ROOT_SECRET_KEY,secure=False)\n try:\n objects = client.list_objects(bucket_name, prefix=prefix, start_after=start_after, include_user_meta=include_user_meta, include_version=include_version, use_url_encoding_type=use_url_encoding_type, recursive=recursive)\n except S3Error as e:\n return self.handleException(e, 'do_list_objects', bucket_name=bucket_name, prefix=prefix, start_after=start_after, include_user_meta=include_user_meta, include_version=include_version, use_url_encoding_type=use_url_encoding_type, recursive=recursive)\n self.stats.success('do_list_objects')\n self.logger.info(f'do_list_objects {bucket_name} {prefix} {start_after} {include_user_meta} {include_version} {use_url_encoding_type} {recursive} succeed')\n result = '\\n'.join([f'{obj.object_name} {obj.size} {obj.etag}' for obj in objects])\n return result\n \n def get_alias(self, alias):\n return self.prefix + '_' + alias\n\n def do_add_user(self, access_key, secret_key, alias):\n try:\n self.run_cmd(f'mc admin user add {self.get_alias(alias)} {access_key} {secret_key}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_add_user', access_key=access_key, secret_key=secret_key)\n self.stats.success('do_add_user')\n self.logger.info(f'do_add_user {access_key} succeed')\n return True\n \n def do_remove_user(self, access_key, alias):\n try:\n self.run_cmd(f'mc admin user remove {self.get_alias(alias)} {access_key}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_remove_user', access_key=access_key)\n self.stats.success('do_remove_user')\n self.logger.info(f'do_remove_user {access_key} succeed')\n return True\n\n def do_enable_user(self, access_key, alias):\n try:\n self.run_cmd(f'mc admin user enable {self.get_alias(alias)} {access_key}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_enable_user', access_key=access_key)\n self.stats.success('do_enable_user')\n self.logger.info(f'do_enable_user {access_key} succeed')\n return True\n \n def do_disable_user(self, access_key, alias):\n try:\n self.run_cmd(f'mc admin user disable {self.get_alias(alias)} {access_key}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_disable_user', access_key=access_key)\n self.stats.success('do_disable_user')\n self.logger.info(f'do_disable_user {access_key} succeed')\n return True\n \n def do_user_info(self, access_key, alias):\n try:\n self.run_cmd(f'mc admin user info {self.get_alias(alias)} {access_key}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_user_info', access_key=access_key)\n self.stats.success('do_user_info')\n self.logger.info(f'do_user_info {access_key} succeed')\n return True\n \n def do_list_users(self, alias):\n try:\n result = self.run_cmd(f'mc admin user list {self.get_alias(alias)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_list_users')\n self.stats.success('do_list_users')\n self.logger.info(f'do_list_users succeed')\n return sorted(result.split(\"\\n\"))\n\n def remove_all_users(self, alias=ROOT_ALIAS):\n lines = self.run_cmd(f'mc admin user list {self.get_alias(alias)}').split(\"\\n\")\n for line in lines:\n if not line.strip():\n continue\n user = line.split()[1]\n self.run_cmd(f'mc admin user remove {self.get_alias(alias)} {user}')\n print(f\"User '{user}' removed successfully.\")\n\n def do_list_groups(self, alias):\n try:\n result = self.run_cmd(f'mc admin group list {self.get_alias(alias)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_list_groups')\n self.stats.success('do_list_groups')\n self.logger.info(f'do_list_groups succeed')\n return sorted(result.split(\"\\n\"))\n\n def do_add_group(self, group_name, members, alias):\n try:\n self.run_cmd(f'mc admin group add {self.get_alias(alias)} {group_name} {\" \".join(members)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_add_group', group_name=group_name, members=members)\n self.stats.success('do_add_group')\n self.logger.info(f'do_add_group {group_name} {members} succeed')\n return self.do_group_info(group_name, alias)\n\n def do_remove_group(self, group_name, members, alias):\n try:\n self.run_cmd(f'mc admin group remove {self.get_alias(alias)} {group_name} {\" \".join(members)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_remove_group', group_name=group_name)\n self.stats.success('do_remove_group')\n self.logger.info(f'do_remove_group {group_name} succeed')\n return True\n\n def do_disable_group(self, group_name, alias):\n try:\n self.run_cmd(f'mc admin group disable {self.get_alias(alias)} {group_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_disable_group', group_name=group_name)\n self.stats.success('do_disable_group')\n self.logger.info(f'do_disable_group {group_name} succeed')\n return True\n \n def do_enable_group(self, group_name, alias):\n try:\n self.run_cmd(f'mc admin group enable {self.get_alias(alias)} {group_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_enable_group', group_name=group_name)\n self.stats.success('do_enable_group')\n self.logger.info(f'do_enable_group {group_name} succeed')\n return True\n\n def do_group_info(self, group_name, alias):\n try:\n self.run_cmd(f'mc admin group info {self.get_alias(alias)} {group_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_group_info', group_name=group_name)\n self.stats.success('do_group_info')\n self.logger.info(f'do_group_info {group_name} succeed')\n return True\n \n def remove_all_groups(self, alias=ROOT_ALIAS):\n groups = self.run_cmd(f'mc admin group list {self.get_alias(alias)}').split(\"\\n\")\n for group in groups:\n if not group.strip():\n continue\n self.run_cmd(f'mc admin group remove {self.get_alias(alias)} {group}')\n print(f\"Group '{group}' removed successfully.\")\n \n def do_add_policy(self, policy_name, policy_document, alias):\n policy = json.dumps(policy_document)\n print(policy)\n policy_path = 'policy.json'\n with open(policy_path, 'w') as f:\n f.write(policy)\n try:\n self.run_cmd(f'mc admin policy add {self.get_alias(alias)} {policy_name} {policy_path}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_add_policy', policy_name=policy_name)\n self.stats.success('do_add_policy')\n self.logger.info(f'do_add_policy {policy_name} succeed')\n return True\n \n def do_remove_policy(self, policy_name, alias):\n try:\n self.run_cmd(f'mc admin policy remove {self.get_alias(alias)} {policy_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_remove_policy', policy_name=policy_name)\n self.stats.success('do_remove_policy')\n self.logger.info(f'do_remove_policy {policy_name} succeed')\n return True\n \n def do_policy_info(self, policy_name, alias):\n try:\n result = self.run_cmd(f'mc admin policy info {self.get_alias(alias)} {policy_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_policy_info', policy_name=policy_name)\n self.stats.success('do_policy_info')\n self.logger.info(f'do_policy_info {policy_name} succeed')\n return self.sort_dict(json.loads(result))\n \n def do_list_policies(self, alias):\n try:\n result = self.run_cmd(f'mc admin policy list {self.get_alias(alias)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_list_policies')\n self.stats.success('do_list_policies')\n self.logger.info(f'do_list_policies succeed')\n result = [item.strip() for item in result.split(\"\\n\") if item.strip()!='diagnostics' and item.strip()!='']\n return sorted(result)\n \n def remove_all_policies(self, alias=ROOT_ALIAS):\n policies = self.do_list_policies(alias)\n for policy in policies:\n if policy in BUILD_IN_POLICIES:\n continue\n self.run_cmd(f'mc admin policy remove {self.get_alias(alias)} {policy}')\n print(f\"Policy '{policy}' removed successfully.\")\n\n def do_set_policy_to_user(self, policy_name, user_name, alias):\n try:\n self.run_cmd(f'mc admin policy set {self.get_alias(alias)} {policy_name} user={user_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_set_policy_to_user', policy_name=policy_name, user_name=user_name)\n self.stats.success('do_set_policy_to_user')\n self.logger.info(f'do_set_policy_to_user {policy_name} {user_name} succeed')\n return True\n\n def do_set_policy_to_group(self, policy_name, group_name, alias):\n try:\n self.run_cmd(f'mc admin policy set {self.get_alias(alias)} {policy_name} group={group_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_set_policy_to_group', policy_name=policy_name, group_name=group_name)\n self.stats.success('do_set_policy_to_group')\n self.logger.info(f'do_set_policy_to_group {policy_name} {group_name} succeed')\n return True\n \n def do_unset_policy_from_user(self, policy_name, user_name, alias):\n try:\n self.run_cmd(f'mc admin policy unset {self.get_alias(alias)} {policy_name} user={user_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_unset_policy_from_user', policy_name=policy_name, user_name=user_name)\n self.stats.success('do_unset_policy_from_user')\n self.logger.info(f'do_unset_policy_from_user {policy_name} {user_name} succeed')\n return True\n \n def do_unset_policy_from_group(self, policy_name, group_name, alias):\n try:\n self.run_cmd(f'mc admin policy unset {self.get_alias(alias)} {policy_name} group={group_name}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_unset_policy_from_group', policy_name=policy_name, group_name=group_name)\n self.stats.success('do_unset_policy_from_group')\n self.logger.info(f'do_unset_policy_from_group {policy_name} {group_name} succeed')\n return True\n \n def do_set_alias(self, alias, access_key, secret_key, url):\n alias_name = self.get_alias(alias)\n try:\n self.run_cmd(f'mc alias set {alias_name} http://{url} {access_key} {secret_key}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_set_alias', alias=alias, url=url, access_key=access_key, secret_key=secret_key)\n self.stats.success('do_set_alias')\n self.logger.info(f'do_set_alias {alias} {url} {access_key} {secret_key} succeed')\n return True\n \n def do_remove_alias(self, alias):\n try:\n self.run_cmd(f'mc alias remove {self.get_alias(alias)}')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_remove_alias', alias=alias)\n self.stats.success('do_remove_alias')\n self.logger.info(f'do_remove_alias {alias} succeed')\n return True\n \n def do_list_aliases(self):\n try:\n result = self.run_cmd(f'mc alias list')\n except subprocess.CalledProcessError as e:\n return self.handleException(e, 'do_list_aliases')\n self.stats.success('do_list_aliases')\n self.logger.info(f'do_list_aliases succeed')\n return sorted([line.strip() for line in result.split(\"\\n\") if line.strip() and ':' not in line])\n \n def remove_all_aliases(self):\n aliases = self.do_list_aliases()\n for alias in aliases:\n if alias.startswith(self.prefix+'_'):\n self.run_cmd(f'mc alias remove {alias}')\n print(f\"Alias '{alias}' removed successfully.\")" }, { "path": ".github/scripts/hypo/s3_strategy.py", "content": "from hypothesis import strategies as st\nfrom string import ascii_lowercase\n\nMAX_OBJECT_SIZE=10*1024*1024\n# https://min.io/docs/minio/linux/administration/identity-access-management/policy-based-access-control.html#minio-policy-actions\nS3_ACTION_LIST = [\"s3:*\", \"s3:DeleteObject\", \"s3:GetObject\",\"s3:ListBucket\",\"s3:PutObject\", \"s3:PutObjectTagging\", \"s3:GetObjectTagging\", \"s3:DeleteObjectTagging\"]\nst_user_name = st.sampled_from(['user1', 'user2', 'user3'])\nst_group_name = st.sampled_from(['group1', 'group2', 'group3'])\nst_group_members = st.lists(st_user_name, max_size=3, unique=True)\nst_secret_key = st.text(alphabet=ascii_lowercase, min_size=8, max_size=8)\nst_alias_name = st.text(alphabet=ascii_lowercase, min_size=4, max_size=4)\nst_bucket_name = st.text(alphabet=ascii_lowercase, min_size=4, max_size=4)\nst_object_name = st.text(alphabet=ascii_lowercase, min_size=4, max_size=4)\nst_object_prefix = st.text(alphabet=ascii_lowercase, min_size=1, max_size=1)\nst_content = st.binary(min_size=0, max_size=MAX_OBJECT_SIZE)\nst_part_size = st.sampled_from([5*1024*1024, 8*1024*1024])\nst_offset = st.integers(min_value=0, max_value=MAX_OBJECT_SIZE)\nst_length = st.integers(min_value=0, max_value=MAX_OBJECT_SIZE)\nst_policy_name = st.text(alphabet=ascii_lowercase, min_size=4, max_size=4)\nst_policy = st.fixed_dictionaries({\n \"Version\": st.just(\"2012-10-17\"),\n \"Statement\": st.lists(\n st.fixed_dictionaries({\n \"Effect\": st.sampled_from([\"Allow\", \"Deny\"]),\n \"Principal\": st.fixed_dictionaries({\"AWS\": st.just(\"*\")}),\n \"Action\": st.lists(\n st.sampled_from(S3_ACTION_LIST),\n min_size=1, max_size=3,\n unique=True\n ),\n \"Resource\": st.just(\"arn:aws:s3:::*\"),\n }),\n min_size=1, max_size=3\n )\n})\n" }, { "path": ".github/scripts/hypo/s3_test.py", "content": "import unittest\nfrom s3 import S3Machine\nfrom s3_contant import *\nclass TestS3(unittest.TestCase):\n def test_bucket(self):\n state = S3Machine()\n state.set_alias('alias1', DEFAULT_ACCESS_KEY)\n state.create_bucket('bucket1')\n state.create_bucket('bucket2')\n state.fput_object('bucket1', 'object1', alias='alias1')\n state.fput_object('bucket1', 'object2', alias='alias1')\n state.fput_object('bucket2', 'object1', alias='alias1')\n state.fput_object('bucket2', 'object2', alias='alias1')\n state.list_buckets()\n state.list_objects('bucket1')\n state.list_objects('bucket2')\n state.list_objects('bucket1', prefix='obj')\n state.remove_object('bucket1:object1')\n state.remove_object('bucket1:object2')\n state.remove_bucket('bucket1')\n state.remove_bucket('bucket2')\n state.teardown()\n\n def test_user(self):\n state = S3Machine()\n state.create_bucket('bucket1')\n state.add_user('user1')\n state.add_user('user2')\n state.list_users()\n state.remove_user('user1')\n state.list_users()\n state.disable_user('user2')\n state.enable_user('user2')\n state.list_users()\n state.remove_user('user2')\n state.list_users()\n state.teardown()\n \n def test_group(self):\n state = S3Machine()\n state.create_bucket('bucket1')\n state.add_user('user1')\n state.add_user('user2')\n state.add_user('user3')\n state.add_group('group1', ['user1', 'user2'])\n state.add_group('group2', ['user2', 'user3'])\n state.list_groups()\n state.disable_group('group2')\n state.remove_group('group1', ['user1'])\n state.remove_group('group1', ['user2'])\n state.remove_group('group1', [])\n state.list_groups()\n state.enable_group('group2')\n state.list_groups()\n state.teardown()\n\n def skip_test_issue_4639(self):\n # SEE https://github.com/juicedata/juicefs/issues/4639\n state = S3Machine()\n v1 = state.init_aliases()\n v2, v3, v4, v5 = state.init_policies()\n state.remove_policy(alias=v1, policy_name=v3)\n state.list_groups(alias=v1)\n state.remove_policy(alias=v1, policy_name=v2)\n state.policy_info(alias=v1, policy_name=v5)\n state.teardown()\n\n def skip_test_issue_4660(self):\n #SEE https://github.com/juicedata/juicefs/issues/4660\n state = S3Machine()\n v1 = state.init_aliases()\n v2, v3, v4, v5 = state.init_policies()\n v8 = state.add_user(alias=v1, user_name='user1')\n state.disable_user(alias=v1, user_name=v8)\n state.set_alias(alias='pjzm', url1='localhost:9000', url2='localhost:9006', user_name=v8)\n state.teardown()\n\n def test_issue_4682(self):\n # SEE https://github.com/juicedata/juicefs/issues/4682\n state = S3Machine()\n v1 = state.init_aliases()\n v2, v3, v4, v5 = state.init_policies()\n v6 = state.create_bucket(alias=v1, bucket_name='nzpy')\n state.get_bucket_policy(alias=v1, bucket_name=v6)\n state.teardown()\n\nif __name__ == '__main__':\n unittest.main()" }, { "path": ".github/scripts/hypo/stats.py", "content": "def singleton(cls):\n instances = {}\n def get_instance(*args, **kwargs):\n if cls not in instances:\n instances[cls] = cls(*args, **kwargs)\n return instances[cls]\n return get_instance\n\n@singleton\nclass Statistics:\n def __init__(self):\n self.stats = {}\n\n def success(self, function_name):\n if function_name not in self.stats:\n self.stats[function_name] = {'success': 0, 'failure': 0}\n self.stats[function_name]['success'] += 1\n\n def failure(self, function_name):\n if function_name not in self.stats:\n self.stats[function_name] = {'success': 0, 'failure': 0}\n self.stats[function_name]['failure'] += 1\n\n def get(self):\n return self.stats" }, { "path": ".github/scripts/hypo/strategy.py", "content": "import subprocess\ntry:\n __import__(\"xattr\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"xattr\"])\nimport xattr\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hypothesis\"])\nfrom hypothesis import strategies as st\nfrom string import ascii_lowercase\nimport time\nimport os\nMIN_DIR_NAME=1\nMAX_DIR_NAME=8\nMIN_FILE_NAME=1\nMAX_FILE_NAME=4\nMAX_XATTR_NAME=255+10\nMAX_XATTR_VALUE=65535+100\nMAX_FILE_SIZE=1024*10\nMAX_TRUNCATE_LENGTH=1024*128\nMAX_FALLOCATE_LENGTH=1024*128\nst_file_name = st.text(alphabet=ascii_lowercase, min_size=MIN_FILE_NAME, max_size=MAX_FILE_NAME)\ndir_alphabet = ascii_lowercase + './'\n# st_dir_name = st.text(alphabet=dir_alphabet, min_size=MIN_DIR_NAME, max_size=MAX_DIR_NAME)\ndef valid_dir_name():\n name_part = st.text(alphabet=dir_alphabet, min_size=MIN_DIR_NAME, max_size=MAX_DIR_NAME)\n def is_valid(s:str):\n if s.startswith('/'):\n return False\n if '.' in s and (not s.endswith('/.') or not s.endswith('/..')):\n return False\n return True\n return name_part.filter(is_valid)\n# st_entry_name = st.text(min_size=MIN_FILE_NAME, max_size=MAX_FILE_NAME)\n#TODO: remove filter when bugfix https://github.com/juicedata/jfs/issues/776\n#TODO: use characters instead of ascii_lowercase\nst_xattr_name = st.text(alphabet=ascii_lowercase, min_size=1, max_size=MAX_XATTR_NAME).filter(lambda x: '\\x00' not in x).map(lambda s: \"user.\" + s)\nst_xattr_value = st.binary(min_size=1, max_size=MAX_XATTR_VALUE)\nst_xattr_flag = st.sampled_from([0, xattr.XATTR_CREATE, xattr.XATTR_REPLACE])\n# st_umask = st.integers(min_value=0o000, max_value=0o777)\nst_umask = st.just(0o022)\nst_entry_mode = st.integers(min_value=0o000, max_value=0o0777)\n\n# TODO: remove alphabet=ascii_lowercase, \nst_lines = st.lists(st.text(alphabet=ascii_lowercase, min_size=0, max_size=10), min_size=1, max_size=10)\n# TODO: remove filter a\nst_open_mode = st.sampled_from([ 'x', 'a', 'r', 'w', 'a+', 'r+', 'w+', 'xb', 'ab', 'rb', 'wb', 'a+b', 'r+b', 'w+b'])\nst_open_errors = st.sampled_from(['strict', 'ignore', 'replace', 'backslashreplace', 'namereplace'])\nst_open_flags = st.lists(st.sampled_from([os.O_RDONLY, os.O_WRONLY, os.O_RDWR, os.O_APPEND, os.O_CREAT, os.O_EXCL, os.O_TRUNC, os.O_SYNC, os.O_DSYNC]), unique=True, min_size=1)\n# TODO: add 0 to buffering when bugfix: https://github.com/juicedata/jfs/issues/1359\nst_buffering = st.sampled_from([-1, 1, 10, 1024])\nst_time = st.integers(min_value=0, max_value=int(time.time()))\nst_offset = st.integers(min_value=0, max_value=MAX_FILE_SIZE)\nst_length = st.integers(min_value=0, max_value=MAX_FILE_SIZE)\nst_truncate_length = st.integers(min_value=0, max_value=MAX_TRUNCATE_LENGTH)\nst_fallocate_length = st.integers(min_value=0, max_value=MAX_FALLOCATE_LENGTH)\nst_whence = st.sampled_from([os.SEEK_SET, os.SEEK_CUR, os.SEEK_END])\n\n@st.composite\ndef utf8_byte_arrays(draw, min_size=0, max_size=100):\n text = draw(st.text(min_size=min_size, max_size=max_size))\n return text.encode('utf-8')\n\n@st.composite\ndef utf16_byte_arrays(draw, min_size=0, max_size=100):\n text = draw(st.text(min_size=min_size, max_size=max_size))\n return text.encode('utf-16')\n\n@st.composite\ndef ascii_byte_arrays(draw, min_size=0, max_size=100):\n text = draw(st.text(alphabet=st.characters(blacklist_categories=['Cs', 'Cc', 'Co', 'Cn'], max_codepoint=127), min_size=min_size, max_size=max_size))\n return text.encode('ascii')\nst_binary = st.binary(min_size=0, max_size=MAX_FILE_SIZE)\nst_ascii_lowercase = st.text(alphabet=ascii_lowercase, min_size=0, max_size=MAX_FILE_SIZE) # | st.binary(min_size=0, max_size=MAX_FILE_SIZE)\nst_unicode = st.text(alphabet=st.characters(max_codepoint=0x10FFFF), min_size=0, max_size=MAX_FILE_SIZE)\n\n# st_content = st.one_of(utf8_byte_arrays(), utf16_byte_arrays(), ascii_byte_arrays(), st_binary, st_unicode, st_ascii_lowercase)\nst_content = st.one_of(utf8_byte_arrays(), ascii_byte_arrays(), st_binary, st_unicode, st_ascii_lowercase)\n# st_open_encoding = st.sampled_from(['utf-8', 'utf-16', 'utf-32', 'ascii', 'latin-1'])\nst_open_encoding = st.sampled_from(['utf-8', 'ascii', 'latin-1'])\n" }, { "path": ".github/scripts/hypo/sync.py", "content": "import os\nimport subprocess\nimport json\nimport common\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hypothesis\"])\nfrom hypothesis import assume, strategies as st, settings, Verbosity\nfrom hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes, invariant\nfrom hypothesis import Phase, seed\nfrom strategy import *\nfrom fs_op import FsOperation\nimport random\n\nst_entry_name = st.text(alphabet='abc*?', min_size=1, max_size=4)\nst_patterns = st.lists(st.sampled_from(['a','?','/','*']), min_size=1, max_size=10)\\\n .map(''.join).filter(lambda s: s.find('***') == -1 or (s.count('***') == 1 and s.endswith('/***')))\n\nst_option = st.fixed_dictionaries({\n \"option\": st.just(\"--include\") | st.just(\"--exclude\"),\n \"pattern\": st_patterns\n})\n\nst_options = st.lists(st_option, min_size=1, max_size=10)\n\nSEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))\n@seed(SEED)\nclass SyncMachine(RuleBasedStateMachine):\n Files = Bundle('files')\n Folders = Bundle('folders')\n ROOT_DIR1 = '/tmp/sync_src'\n ROOT_DIR2 = '/tmp/sync_src2'\n DEST_RSYNC = '/tmp/rsync'\n DEST_JUICESYNC = '/tmp/juicesync'\n \n fsop1 = FsOperation('fs1', ROOT_DIR1)\n fsop2 = FsOperation('fs2', ROOT_DIR2)\n \n @initialize(target=Folders)\n def init_folders(self):\n if not os.path.exists(self.ROOT_DIR1):\n os.makedirs(self.ROOT_DIR1)\n if not os.path.exists(self.ROOT_DIR2):\n os.makedirs(self.ROOT_DIR2)\n common.clean_dir(self.ROOT_DIR1)\n common.clean_dir(self.ROOT_DIR2)\n return ''\n \n def __init__(self):\n super(SyncMachine, self).__init__()\n \n def equal(self, result1, result2):\n if type(result1) != type(result2):\n return False\n if isinstance(result1, Exception):\n r1 = str(result1).replace(self.ROOT_DIR1, '')\n r2 = str(result2).replace(self.ROOT_DIR2, '')\n return r1 == r2\n elif isinstance(result1, tuple):\n return result1 == result2\n elif isinstance(result1, str):\n r1 = str(result1).replace(self.ROOT_DIR1, '')\n r2 = str(result2).replace(self.ROOT_DIR2, '')\n return r1 == r2\n else:\n return result1 == result2\n\n @rule(target=Files, \n parent = Folders.filter(lambda x: x != multiple()), \n file_name = st_entry_name, \n umask = st_umask, \n )\n def create_file(self, parent, file_name, content='s', mode='x', user='root', umask=0o022):\n result1 = self.fsop1.do_create_file(parent=parent, file_name=file_name, mode=mode, content=content, user=user, umask=umask)\n result2 = self.fsop2.do_create_file(parent=parent, file_name=file_name, mode=mode, content=content, user=user, umask=umask)\n assert self.equal(result1, result2), f'\\033[31mcreate_file:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, file_name)\n \n @rule( target = Folders, \n parent = Folders.filter(lambda x: x != multiple()),\n subdir = st_entry_name,\n mode = st_entry_mode,\n umask = st_umask, \n )\n def mkdir(self, parent, subdir, mode, user='root', umask=0o022):\n result1 = self.fsop1.do_mkdir(parent, subdir, mode, user, umask)\n result2 = self.fsop2.do_mkdir(parent, subdir, mode, user, umask)\n assert self.equal(result1, result2), f'\\033[31mmkdir:\\nresult1 is {result1}\\nresult2 is {result2}\\033[0m'\n if isinstance(result1, Exception):\n return multiple()\n else:\n return os.path.join(parent, subdir)\n\n @rule(options = st_options\n )\n def sync(self, options):\n subprocess.check_call(['rm', '-rf', self.DEST_RSYNC])\n subprocess.check_call(['rm', '-rf', self.DEST_JUICESYNC])\n options_run = ' '.join([f'{item[\"option\"]} {item[\"pattern\"]}' for item in options])\n options_display = ' '.join([f'{item[\"option\"]} \"{item[\"pattern\"]}\"' for item in options])\n print(f'rsync -r -vvv {self.ROOT_DIR1}/ {self.DEST_RSYNC}/ {options_display}')\n subprocess.check_call(f'rsync -r -vvv {self.ROOT_DIR1}/ {self.DEST_RSYNC}/ {options_run}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n print(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options_display}')\n subprocess.check_call(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options_run}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n try:\n subprocess.check_call(['diff', '-r', self.DEST_RSYNC, self.DEST_JUICESYNC])\n except subprocess.CalledProcessError as e:\n print(f'\\033[31m{e}\\033[0m')\n raise e\n self.fsop1.stats.success('do_sync')\n self.fsop2.stats.success('do_sync')\n\n def teardown(self):\n pass\n\nif __name__ == '__main__':\n MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '1000'))\n STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))\n settings.register_profile(\"dev\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, \n print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])\n settings.register_profile(\"ci\", max_examples=MAX_EXAMPLE, verbosity=Verbosity.normal, \n print_blob=False, stateful_step_count=STEP_COUNT, deadline=None, \\\n report_multiple_bugs=False, \n phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])\n profile = os.environ.get('PROFILE', 'dev')\n settings.load_profile(profile)\n juicefs_machine = SyncMachine.TestCase()\n juicefs_machine.runTest()\n print(json.dumps(FsOperation.stats.get(), sort_keys=True, indent=4))\n" }, { "path": ".github/scripts/hypo/sync_test.py", "content": "import unittest\nfrom sync import SyncMachine\n\nclass TestFsrand2(unittest.TestCase):\n\n def test_sync1(self):\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.mkdir(mode=0, parent=v1, subdir='a', umask=0)\n v3 = state.create_file(content=b'', file_name=v2, mode='w', parent=v2, umask=0)\n state.sync(options=[{'option': '--include', 'pattern': 'aa/***'},\n {'option': '--exclude', 'pattern': 'a?**'}])\n state.teardown()\n\n def test_sync2(self):\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)\n state.sync(options=[{'option': '--exclude', 'pattern': '**/***'}])\n state.teardown()\n\n def test_sync3(self):\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)\n state.sync(options=[{'option': '--exclude', 'pattern': '/***'}])\n state.teardown()\n\n def test_sync4(self):\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)\n state.sync(options=[{'option': '--exclude', 'pattern': '*/***'}])\n state.teardown()\n\n def test_sync5(self):\n state = SyncMachine()\n v1 = state.init_folders()\n state.sync(options=[{'option': '--include', 'pattern': 'a'}])\n v2 = state.mkdir(mode=0, parent=v1, subdir='a', umask=0)\n v3 = state.create_file(content=b'', file_name=v2, mode='w', parent=v2, umask=0)\n state.sync(options=[{'option': '--include', 'pattern': 'aa'},\n {'option': '--exclude', 'pattern': 'a?**'}])\n state.teardown()\n\n def test_sync6(self):\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)\n state.sync(options=[{'option': '--exclude', 'pattern': '**a'}])\n state.teardown()\n\n def test_sync7(self):\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.create_file(content=b'', file_name='aa', mode='w', parent=v1, umask=0)\n state.sync(options=[{'option': '--exclude', 'pattern': 'aa**a'}])\n state.teardown()\n \n def test_sync8(self):\n # SEE: https://github.com/juicedata/juicefs/issues/4471\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.mkdir(mode=8, parent=v1, subdir='a', umask=0)\n state.sync(options=[{'option': '--exclude', 'pattern': 'a/**/a'}])\n state.teardown()\n\n def test_sync9(self):\n # SEE: https://github.com/juicedata/juicefs/issues/4471\n state = SyncMachine()\n v1 = state.init_folders()\n v2 = state.mkdir(mode=8, parent=v1, subdir='aa', umask=0) \n v3 = state.create_file(content=b'', file_name='a', mode='w', parent=v2, umask=0)\n state.sync(options=[{'option': '--include', 'pattern': '**aa**'},\n {'option': '--exclude', 'pattern': 'a'}])\n state.teardown()\n\nif __name__ == '__main__':\n unittest.main()" }, { "path": ".github/scripts/mutate/check_coverage.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\n\ndef is_mutation_in_coverage(original_file, changed_file, coverage_file):\n\tcoverage = parse_coverage(coverage_file)\n\t# print(coverage)\n\toriginal = open(original_file, 'r').readlines()\n\tchanged = open(changed_file, 'r').readlines()\n\tfor i in range( min(len(original), len(changed)) ):\n\t\tif original[i] != changed[i]:\n\t\t\t# print(f'line {i+1} is different')\n\t\t\tif (i+1) not in coverage:\n\t\t\t\t# print(f'line {i+1} is not in coverage')\n\t\t\t\treturn False\n\t\t\telse:\n\t\t\t\t# print(f'line {i+1} is in coverage')\n\t\t\t\treturn True\n\treturn True\n\n\ndef parse_coverage(file):\n\tcov = set()\n\twith open(file, 'r') as f:\n\t\tlines = f.readlines()\n\t\tfor line in lines[1:]:\n\t\t\tname = line.split(':')[0]\n\t\t\tcount = int(line.split(' ')[2])\n\t\t\tif count > 0:\n\t\t\t\tstart_line = int(line.split(':')[1].split(' ')[0].split(',')[0].split('.')[0])\n\t\t\t\tend_line = int(line.split(':')[1].split(' ')[0].split(',')[1].split('.')[0])\n\t\t\t\tfor i in range(start_line, end_line+1):\n\t\t\t\t\tcov.add(i)\n\treturn cov\n\t\nif __name__ == '__main__':\n\t# MUTATE_ORIGINAL=../cmd/meta/xattr.go MUTATE_CHANGED=../cmd/meta/xattr_copy.go COVERAGE_FILE=xattr-cov.out python3 check_coverage.py\n\t# MUTATE_ORIGINAL=cmd/meta/xattr.go MUTATE_CHANGED=/var/folders/jz/mvf43cj13sl4l17z1yy8m92h0000gn/T/go-mutesting-3937777628/xattr.go.4 COVERAGE_FILE=cmd/meta/xattr-cov.out python3 scripts/check_coverage.py\n\toriginal_file = os.environ['MUTATE_ORIGINAL']\n\tchanged_file = os.environ['MUTATE_CHANGED']\n\tcoverage_file = os.environ['COVERAGE_FILE']\n\t# print(f'MUTATE_ORIGINAL={original_file} MUTATE_CHANGED={changed_file} COVERAGE_FILE={coverage_file} python3 ../../scripts/check_coverage.py')\n\tr = is_mutation_in_coverage(original_file, changed_file, coverage_file)\n\tif r:\n\t\texit(0)\n\telse:\n\t\texit(3)" }, { "path": ".github/scripts/mutate/check_skip_by_comment.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\n\n\ndef is_mutation_skipped_by_comment(original_file, changed_file):\n original = open(original_file, 'r').readlines()\n changed = open(changed_file, 'r').readlines()\n for i in range( min(len(original), len(changed)) ):\n if original[i] != changed[i]:\n # print(f'line {i+1} is different')\n if 'skip mutate' in original[i]:\n print(f'line {i+1} is skipped by comment')\n return True\n return False\n\n\nif __name__ == '__main__':\n original_file = os.environ['MUTATE_ORIGINAL']\n changed_file = os.environ['MUTATE_CHANGED']\n if is_mutation_skipped_by_comment(original_file, changed_file):\n exit(1)\n else:\n exit(0)\n\n " }, { "path": ".github/scripts/mutate/how_to_use_mutate_test.md", "content": "# what is mutatation testing?\nMutation testing (or Mutation analysis or Program mutation) is used to design new software tests and evaluate the quality of existing software tests. Mutation testing involves modifying a program in small ways. Each mutated version is called a mutant and tests detect and reject mutants by causing the behavior of the original version to differ from the mutant. This is called killing the mutant. Test suites are measured by the percentage of mutants that they kill. New tests can be designed to kill additional mutants.\n\n# what is the difference between mutants?\nthere are several kind of mutants:\n1. killed mutants: the mutants which is killed by the unit test. which is identified by \"tests passed -> FAIL\" in the log\n2. failed or escaped mutants: the mutants which pass the unit test. which is identified by \"tests failed -> PASS\" in the log\n3. skipped mutants: the mutants may skipped because of 1. out of coverage code. 2. in the black list, 3. skipped by comment. \n4. other exception cases.\n# how to checkout the failed mutants?\n1. open the github action workflow page.\n2. click \"run mutate test\" step.\n3. search \"tests passed \" keyword, all the \"tests passed -> FAIL\" mutants are failed.\nyou can try here: https://github.com/juicedata/juicefs/actions/runs/3565436367/jobs/5990603552\n# how to fix failed mutants?\n1. open the github action workflow page.\n2. click \"run mutate test\" step.\n3. search \"tests passed \" keyword, all the \"tests passed -> FAIL\" mutants are failed.\n3. find which line is changed by mutation.\n4. copy the changed line to .go source file\n5. run all the tests in corresponding go test file, all the tests should passed.\n6. you should add test case to make the test failed, which kill this mutant.\n# how to add a mutation to black list?\n1. find the checksum from the github action log, like FAIL \"/tmp/go-mutesting-1324412688/pkg/chunk/prefetch.go.0\" with checksum bb9e9497f17e191adf89b5a2ef6764eb\n2. add a line //checksum: bb9e9497f17e191adf89b5a2ef6764eb in the go test file.\nFor example:\n//checksum 9cb13bb28aa7918edaf4f0f4ca92eea5\n//checksum 05debda2840d31bac0ab5c20c5510591\nfunc TestMin(t *testing.T) {\n\tassertEqual(t, Min(1, 2), 1)\n\tassertEqual(t, Min(-1, -2), -2)\n\tassertEqual(t, Min(0, 0), 0)\n}\n\n# how to skip mutate a specific line?\nAdd \"//skip mutate\" to the end of the line you don't want to mutate in the source file.\nFor example:\n\tif err != nil { //skip mutate\n\t\treturn \"\", fmt.Errorf(\"failed to execute command `lsb_release`: %s\", err)\n\t}\n\n# how to skip a specific test case?\nif you don't want to run a specific test case, you can add \"//skip mutate\" after the test case function.\nFor example:\nfunc TestRandomWrite(t *testing.T) {//skip mutate\n\t...\n}\n\n# how to customize mutate test job in parallel?\nif the mutants of the target source file is more than 200, we will use 4 github jobs to run it. otherwise we will use 1 job to run.\nyou can customize it in your test file with adding \"//mutate_test_job_number: number\", eg: //mutate_test_job_number: 8\n\n# how to disable muate test for a specific go file?\nadd //mutate:disable in the *_test.go file to disable the mutate test." }, { "path": ".github/scripts/mutate/modify_sdk_pom.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef get_plugin_str(taget_tests, taget_classes, time_constant):\n s = \"\"\" \n\t\t\t\torg.pitest\n\t\t\t\tpitest-maven\n\t\t\t\t1.9.11\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t{taget_classes}\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t{taget_tests}\n\t\t\t\t\t\n\t\t\t\t\t{time_constant}\n\t\t\t\t\n\t\t\t \"\"\"\n s = s.replace('{taget_classes}', taget_classes)\n s = s.replace('{taget_tests}', taget_tests)\n s = s.replace('{time_constant}', time_constant)\n return s\n\ndef modify_pom(pom_path, taget_tests, taget_classes, time_constant):\n new_lines = []\n with open(pom_path, 'r') as f:\n for line in f.readlines():\n if line.strip() == '':\n new_lines.append(get_plugin_str(taget_tests, taget_classes, time_constant)+'\\n')\n new_lines.append(line)\n with open(pom_path, 'w') as f:\n f.writelines(new_lines)\n\nif __name__ == '__main__':\n pom_path = os.environ['POM_XML_PATH']\n taget_tests = os.environ['TARGET_TESTS']\n taget_classes = os.environ['TARGET_CLASSES']\n time_constant = os.environ['TIME_CONSTANT']\n modify_pom(pom_path, taget_tests, taget_classes, time_constant)\n " }, { "path": ".github/scripts/mutate/mutest.sh", "content": "#!/bin/bash\n\n# This exec script implements\n# - the replacement of the original file with the mutation,\n# - the execution of all tests originating from the package of the mutated file,\n# - and the reporting if the mutation was killed.\n\nif [ -z ${MUTATE_CHANGED+x} ]; then echo \"MUTATE_CHANGED is not set\"; exit 1; fi\nif [ -z ${MUTATE_ORIGINAL+x} ]; then echo \"MUTATE_ORIGINAL is not set\"; exit 1; fi\nif [ -z ${MUTATE_PACKAGE+x} ]; then echo \"MUTATE_PACKAGE is not set\"; exit 1; fi\nif [ -z ${COVERAGE_FILE+x} ]; then echo \"COVERAGE_FILE is not set\"; exit 1; fi\nif [ -z ${TEST_FILE_NAME+x} ]; then echo \"TEST_FILE_NAME is not set\"; exit 1; fi\nif [ -z ${PACKAGE_PATH+x} ]; then echo \"PACKAGE_PATH is not set\"; exit 1; fi\n\nfunction clean_up {\n\tif [ -f $MUTATE_ORIGINAL.tmp ];\n\tthen\n\t\tmv $MUTATE_ORIGINAL.tmp $MUTATE_ORIGINAL\n\tfi\n}\n\nfunction sig_handler {\n\tclean_up\n\n\texit $GOMUTESTING_RESULT\n}\ntrap sig_handler SIGHUP SIGINT SIGTERM\n\nexport MUTATE_TIMEOUT=${MUTATE_TIMEOUT:-10}\n\nif [ -n \"$TEST_RECURSIVE\" ]; then\n\tTEST_RECURSIVE=\"/...\"\nfi\n\nexport GOMUTESTING_DIFF=$(diff -u $MUTATE_ORIGINAL $MUTATE_CHANGED)\nif [ -z \"$GOMUTESTING_DIFF\" ]; then\n\techo \"mutate file is the same as original file\", $MUTATE_CHANGED\n\texit 100\nfi\n\npython3 .github/scripts/mutate/check_coverage.py\n\nif [ $? -ne 0 ]; then\n\techo \"mutate is out of code coverage\", $MUTATE_CHANGED\n\texit 101\nfi\n\npython3 .github/scripts/mutate/check_skip_by_comment.py\nif [ $? -ne 0 ]; then\n\techo \"mutate is skipped by comment\", $MUTATE_CHANGED\n\texit 102\nfi\n\ntest_cases=$(python3 .github/scripts/mutate/parse_test_cases.py)\nif [ $? -ne 0 ]; then\n\techo \"no test cases in test file \", $TEST_FILE_NAME\n\texit 103\nfi\n\nmv $MUTATE_ORIGINAL $MUTATE_ORIGINAL.tmp\ncp $MUTATE_CHANGED $MUTATE_ORIGINAL\necho \"------------------------------------------------------------------------\"\necho \"Start unit test with: $MUTATE_CHANGED\"\ngo test ./$PACKAGE_PATH/... -run \"$test_cases\" -v -cover -count=1 -timeout=5m \n# GOMUTESTING_TEST=$(go test -timeout $(printf '%ds' $MUTATE_TIMEOUT) $MUTATE_PACKAGE$TEST_RECURSIVE 2>&1)\nexport GOMUTESTING_RESULT=$?\n\n\nif [ \"$MUTATE_DEBUG\" = true ] ; then\n\techo \"$GOMUTESTING_TEST\"\nfi\n\nclean_up\n\ncase $GOMUTESTING_RESULT in\n0) # tests passed -> FAIL\n\techo \"$GOMUTESTING_DIFF\"\n\techo \"tests passed -> FAIL\"\n\texit 1\n\t;;\n1) # tests failed -> PASS\n\techo \"$GOMUTESTING_DIFF\"\n\techo \"tests failed -> PASS\"\n\texit 0\n\t;;\n2) # did not compile -> SKIP\n\tif [ \"$MUTATE_VERBOSE\" = true ] ; then\n\t\techo \"Mutation did not compile\"\n\tfi\n\n\tif [ \"$MUTATE_DEBUG\" = true ] ; then\n\t\techo \"$GOMUTESTING_DIFF\"\n\tfi\n\techo \"did not compile -> SKIP\"\n\texit 2\n\t;;\n3) # mutation is out of coverage -> SKIP\n\techo \"mutation is out of coverage -> SKIP\"\n\techo \"$GOMUTESTING_DIFF\"\n\n\texit $GOMUTESTING_RESULT\n\t;;\n4) # check coverage failed -> SKIP\n\techo \"check coverage failed -> SKIP\"\n\techo \"$GOMUTESTING_DIFF\"\n\n\texit $GOMUTESTING_RESULT\n\t;;\n\n*) # Unkown exit code -> SKIP\n\techo \"Unknown exit code\"\n\techo \"$GOMUTESTING_DIFF\"\n\n\texit $GOMUTESTING_RESULT\n\t;;\nesac" }, { "path": ".github/scripts/mutate/mutesting.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport glob\nimport json\nimport os\nimport sys\nfrom tkinter import Tcl\n\ndef do_mutate_test(mutation_dir, index, total):\n print(f'mutation dir is {mutation_dir}, inde is {index}, total is {total}', file=sys.stderr)\n # os.system(f'ls -l {mutation_dir}')\n list_of_files = Tcl().call('lsort', '-dict', glob.glob(mutation_dir + '/*.go.*') )\n if len(list_of_files) > 0 and 'original' in list_of_files[-1]:\n list_of_files = list_of_files[:-1]\n # print('\\n'.join(list_of_files), file=sys.stderr)\n stats = {'passed':0, 'failed':0, 'compile_error':0, 'out_of_coverage':0, 'skip_by_comment':0, 'others':0, 'total':0}\n count = int(len(list_of_files)/total) + 1\n start = index*count\n end = start + count\n print(f'count:{count}, start:{start}, end:{end}', file=sys.stderr)\n if end > len(list_of_files):\n end = len(list_of_files)\n for changed_file in list_of_files[start:end]:\n # timestamp_str = time.strftime( '%m/%d/%Y :: %H:%M:%S',\n # time.gmtime(os.path.getmtime(changed_file))) \n # print(timestamp_str, ' -->', changed_file) \n os.environ['MUTATE_CHANGED'] = changed_file\n ret = os.system('.github/scripts/mutate/mutest.sh') >> 8\n if ret == 0:\n stats['passed'] += 1\n elif ret == 1:\n stats['failed'] += 1\n elif ret == 2:\n stats['compile_error'] += 1\n elif ret == 101:\n stats['out_of_coverage'] += 1\n elif ret == 102:\n stats['skip_by_comment'] += 1\n else:\n stats['others'] += 1\n stats['total'] += 1\n if stats['passed'] + stats['failed'] == 0:\n stats['score'] = 1.0\n else:\n stats['score'] = stats['passed'] / (stats['passed'] + stats['failed'])\n return stats\n\nif __name__ == '__main__':\n os.environ['MUTATE_PACKAGE'] = ''\n mutation_dir = os.path.join(os.environ['MUTATION_DIR'], os.environ['PACKAGE_PATH'])\n print(f'mutation dir is {mutation_dir}', file=sys.stderr)\n original_file = os.environ['MUTATE_ORIGINAL']\n print(f'original file is {original_file}', file=sys.stderr)\n if not os.environ['JOB_INDEX']:\n index = 0\n else:\n index = int(os.environ['JOB_INDEX'])-1\n total = int(os.environ['JOB_TOTAL'])\n stats = do_mutate_test(mutation_dir, index, total)\n print(stats)\n stat_result_file = os.environ['STAT_RESULT_FILE']\n print(f'stat result file is {stat_result_file}', file=sys.stderr)\n with open(stat_result_file, \"w\") as f:\n json.dump(stats, f)" }, { "path": ".github/scripts/mutate/parse_black_list.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef parse_check_sum(test_file_path):\n check_sum_list = []\n with open(test_file_path) as f:\n lines = f.readlines()\n for line in lines:\n # //checksum 5b1ca0cfedd786d9df136a0e042df23a\n group = re.match('//checksum\\s+(.{32})$', line.strip())\n if group:\n check_sum_list.append(group.group(1))\n return check_sum_list\n\ndef save_black_list(file_name, check_sum_list):\n with open(file_name, 'w') as f:\n f.write('\\n'.join(check_sum_list))\n\nif __name__ == '__main__':\n test_file_path = os.environ['TEST_FILE_NAME']\n if not test_file_path:\n print('test file name is empty')\n exit(1)\n black_list_file = os.environ['BLACK_LIST_FILE']\n save_black_list(black_list_file, parse_check_sum(test_file_path))" }, { "path": ".github/scripts/mutate/parse_job_total.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\nimport sys\n\n\ndef parse_test_jobs(test_file_path):\n with open(test_file_path) as f:\n lines = f.readlines()\n for line in lines:\n g = re.search('^//mutate_test_job_number:\\s*(.+)', line.strip())\n if g:\n return int(g.group(1))\n \n return 0\n\nif __name__ == '__main__':\n test_file_path = os.environ['TEST_FILE_NAME']\n if not test_file_path:\n print('test file name is empty', file=sys.stderr)\n exit(1)\n print(parse_test_jobs(test_file_path))\n " }, { "path": ".github/scripts/mutate/parse_mutate_log.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef parse_mutate_log(log_file):\n mutants = {}\n with open(log_file) as f:\n lines = f.readlines()\n for line in lines:\n # The mutation score is 0.326154 (106 passed, 180 failed, 31 duplicated, 39 skipped, total is 325)\n if line.strip().startswith(\"The mutation score is\"):\n result = re.match(r'(.+)\\((\\d+) passed, (\\d+) failed, (\\d+) duplicated, (\\d+) skipped, total is (\\d+)\\)', line)\n passed = result.group(2)\n failed = result.group(3)\n duplicated = result.group(4)\n skipped = result.group(5)\n total = result.group(6)\n score = int(passed) * 1.0 / (int(total) - int(skipped))\n return f'The mutation score is {score} ({passed} passed, {failed} failed, {duplicated} duplicated, {skipped} skipped, total is {total})'\n return ''\n\nif __name__ == '__main__':\n log_file = os.environ['LOG_FILE']\n if not log_file:\n print('log file is empty')\n exit(1)\n s = parse_mutate_log(log_file)\n if s:\n print(s)\n else:\n exit(1)\n " }, { "path": ".github/scripts/mutate/parse_test_cases.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nimport os\nimport re\n\n\ndef parse_test_cases(test_file_path):\n test_cases = []\n with open(test_file_path) as f:\n lines = f.readlines()\n for line in lines:\n # func TestXattr2(t *testing.T) {\n if re.search('^func\\s+Test.+', line.strip()):\n if 'skip mutate' in line:\n continue\n name = line.strip().split(' ')[1].split('(')[0]\n test_cases.append(name)\n return test_cases\n\n\nif __name__ == '__main__':\n test_file_path = os.environ['TEST_FILE_NAME']\n if not test_file_path:\n print('test file name is empty')\n exit(1)\n test_cases = parse_test_cases(test_file_path)\n if len(test_cases) == 0:\n print('test case is empty')\n exit(1)\n test_cases_str = '|'.join(test_cases)\n print(f'({test_cases_str})')" }, { "path": ".github/scripts/mutate/query_report.py", "content": "\nimport os\nimport sys\nimport MySQLdb\n\ndef query_report(repo, run_id):\n passowrd = os.environ['MYSQL_PASSWORD']\n db = MySQLdb.connect(host=\"8.210.231.144\", user=\"juicedata\", passwd=passowrd, db=\"mutate\")\n db.query(f\"\"\"SELECT job_name, github_job_url, passed, failed, compile_error, out_of_coverage, skip_by_comment, others FROM report \n WHERE github_repo=\"{repo}\" AND github_run_id={run_id}\"\"\")\n r=db.store_result()\n for i in range(r.num_rows()):\n row = r.fetch_row()[0]\n passed = int(row[2])\n failed = int(row[3])\n if passed+failed != 0:\n score = row[2]/(row[2]+row[3])\n else:\n score = 0\n print(f'{row[0]}: score:{score:.2f} failed:{row[3]}, passed:{row[2]}, compile error:{row[4]}, out of coverage:{row[5]}, skip by comment:{row[6]}, others:{row[7]}')\n print(f'Job detail: {row[1]}\\n')\n db.close()\n\nif __name__ == \"__main__\":\n repo = os.environ.get('GITHUB_REPOSITORY')\n run_id = os.environ.get('GITHUB_RUN_ID')\n # repo = 'juicedata/juicefs'\n # run_id = '3608212346'\n print(f'repo is {repo}, run_id is {run_id}', file=sys.stderr)\n query_report(repo, run_id)\n" }, { "path": ".github/scripts/mutate/save_report.py", "content": "\nimport json\nimport os\nfrom sys import argv\nimport MySQLdb\nfrom datetime import datetime\nimport argparse\n# CREATE DATABASE mutate\n# CREATE TABLE `report` (\n# `github_repo` varchar(128) DEFAULT NULL,\n# `github_ref_name` varchar(64) DEFAULT NULL,\n# `github_sha` varchar(128) DEFAULT NULL,\n# `github_run_id` varchar(64) DEFAULT NULL,\n# `github_job_url` varchar(1024) DEFAULT NULL,\n# `created_date` datetime DEFAULT NULL,\n# `job_name` varchar(64) DEFAULT NULL,\n# `passed` int,\n# `failed` int,\n# `compile_error` int, \n# `out_of_coverage` int, \n# `skip_by_comment` int,\n# `others` int\n# )\n\ndef save_report(job_name, report):\n passowrd = os.environ['MYSQL_PASSWORD']\n github_repo = os.environ.get('GITHUB_REPOSITORY')\n print(f'github_repo is: {github_repo}')\n github_ref_name = os.environ.get('GITHUB_REF_NAME')\n print(f'github_ref_name is: {github_ref_name}')\n github_sha = os.environ.get('GITHUB_SHA')\n print(f'github_sha is: {github_sha}')\n github_run_id = os.environ.get('GITHUB_RUN_ID')\n print(f'github_run_id is: {github_run_id}')\n github_job_url = os.environ.get('JOB_URL')\n print(f'github_job_url is: {github_job_url}')\n created_date = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n db = MySQLdb.connect(host=\"8.210.231.144\", user=\"juicedata\", passwd=passowrd, db=\"mutate\")\n c = db.cursor()\n c.execute(f\"insert into report(github_repo, github_ref_name, github_sha, github_run_id, github_job_url, created_date, job_name, passed, failed, compile_error, out_of_coverage, skip_by_comment, others) \\\n values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\", (github_repo, github_ref_name, github_sha, github_run_id, github_job_url, created_date, job_name, report['passed'], report['failed'], report['compile_error'], report['out_of_coverage'], report['skip_by_comment'], report['others']))\n db.commit()\n c.close()\n db.close()\n print(f'save report for {job_name} succeed')\n\nif __name__ == \"__main__\":\n job_name = os.environ.get('JOB_NAME')\n stat_result_file = os.environ.get('STAT_RESULT_FILE')\n print(f'save report for {job_name}, stat result file is {stat_result_file}')\n with open(stat_result_file) as f:\n report = json.load(f)\n save_report(job_name, report)\n\n" }, { "path": ".github/scripts/perf/ai.sh", "content": "#!/bin/bash\n# ai_format_benchmark.sh\nset -e\n\nMNT_POINT=$1\nRESULTS_FILE=$2\nVERSION=$3\n\n# Create Python virtual environment if needed\nif [ ! -d \"venv\" ]; then\n PY_VER=$(python3 -V 2>&1 | awk '{print $2}' | cut -d. -f1,2)\n PKG=\"python${PY_VER}-venv\"\n sudo apt install $PKG -y\n python3 -m venv venv\nfi\n\nsource venv/bin/activate\n\n# Install required packages\n#pip install --upgrade pip\npip install numpy pandas\n\n# Try to install optional dependencies\npip install h5py || echo \"h5py installation failed, HDF5 tests will be skipped\"\npip install torch || echo \"PyTorch installation failed, PyTorch tests will be skipped\"\npip install tensorflow || echo \"TensorFlow installation failed, TensorFlow tests will be skipped\"\npip install pyarrow || echo \"PyArrow installation failed, Parquet tests will be skipped\"\npip install onnx || echo \"OONX installation failed, ONNX tests will be skipped\"\npip install onnxruntime\npip install pillow\npip install lmdb\npip install tqdm\n\n\n# Run the benchmark\npython .github/scripts/perf/ai_format_benchmark.py \"$MNT_POINT\" \"$RESULTS_FILE\" \"$VERSION\"\n\ndeactivate\n" }, { "path": ".github/scripts/perf/ai_format_benchmark.py", "content": "#!/usr/bin/env python3\n\"\"\"\nAI Training Format Performance Benchmark Script - Fixed Version\nComprehensive performance testing for AI training file formats\n\"\"\"\n\nimport os\nimport sys\nimport json\nimport time\nimport tempfile\nimport subprocess\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple, Any, Callable, Optional\nimport argparse\nimport shutil\nfrom dataclasses import dataclass\nimport pickle\nimport random\nimport io\nfrom PIL import Image\nimport lmdb\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\nfrom tqdm import tqdm\n\ntry:\n import h5py\nexcept ImportError:\n h5py = None\n\ntry:\n import torch\nexcept ImportError:\n torch = None\n\ntry:\n import tensorflow as tf\nexcept ImportError:\n tf = None\n\ntry:\n import pyarrow.parquet as pq\n import pyarrow as pa\nexcept ImportError:\n pq = None\n pa = None\n\ntry:\n import onnx\n import onnxruntime as ort\nexcept ImportError:\n onnx = None\n ort = None\n\n@dataclass\nclass BenchmarkResult:\n \"\"\"Structured benchmark result\"\"\"\n min_time: float\n max_time: float\n mean_time: float\n std_time: float\n throughput_mb_s: Optional[float] = None\n file_size_bytes: Optional[int] = None\n operation_count: Optional[int] = None\n details: Dict[str, Any] = None\n\nclass AIFormatBenchmark:\n def __init__(self, mount_point: str, results_file: str, version: str):\n self.mount_point = Path(mount_point)\n self.results_file = Path(results_file)\n self.version = version\n self.results = {}\n self.verbose = False\n\n # Test configuration\n self.config = {\n 'small_file_mb': 50,\n 'medium_file_mb': 100,\n 'large_file_mb': 200,\n 'num_runs': 2,\n 'cool_down_time': 0.5,\n 'num_samples': 5000, # For dataset benchmarks\n 'image_size': (128, 128, 3), # For image dataset benchmarks\n 'lmdb_num_samples': 1000, # Reduced for CI testing\n 'lmdb_num_proc': 4, # Reduced for CI testing\n 'lmdb_image_size': (128, 128) # Smaller images for CI\n }\n\n # Create test directory\n self.test_dir = self.mount_point / \"ai_benchmark\"\n self.test_dir.mkdir(exist_ok=True)\n\n def clear_cache(self):\n \"\"\"Clear system cache silently\"\"\"\n try:\n subprocess.run([\"sudo\", \"sync\"], check=True,\n stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n subprocess.run(\n \"echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null\",\n shell=True,\n check=True,\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL\n )\n except (subprocess.CalledProcessError, FileNotFoundError):\n if self.verbose:\n print(\"Warning: Failed to clear cache (requires sudo privileges)\")\n\n def run_benchmark(self, name: str, func: Callable, file_size: int = None,\n description: str = \"\") -> Optional[BenchmarkResult]:\n \"\"\"Run a benchmark function multiple times and calculate statistics\"\"\"\n times = []\n size_results = []\n\n if self.verbose:\n print(f\" Running {name}: {description}\")\n\n for i in range(self.config['num_runs']):\n self.clear_cache()\n start_time = time.perf_counter()\n try:\n result = func()\n except Exception as e:\n print(f\" Error in run {i+1}: {e}\")\n result = None\n end_time = time.perf_counter()\n\n elapsed_time = end_time - start_time\n times.append(elapsed_time)\n \n if \"file_size\" in result:\n file_size = result[\"file_size\"]\n if result is not None:\n size_results.append(result)\n if tf is not None:\n tf.keras.backend.clear_session()\n if torch is not None:\n torch.cuda.empty_cache() if torch.cuda.is_available() else None\n import gc\n gc.collect()\n time.sleep(self.config['cool_down_time'])\n\n if not times:\n return None\n\n stats = BenchmarkResult(\n min_time=min(times),\n max_time=max(times),\n mean_time=np.mean(times),\n std_time=np.std(times),\n details=size_results[0] if size_results else {}\n )\n\n if file_size is not None and stats.mean_time > 0:\n if \"num_layers\" in result:\n stats.throughput_mb_s = file_size / result[\"num_layers\"] / stats.mean_time / (1024**2)\n else:\n stats.throughput_mb_s = file_size / stats.mean_time / (1024**2)\n stats.file_size_bytes = file_size\n\n if stats.details:\n for key in ['num_records', 'records_read', 'num_files', 'num_layers', 'num_samples']:\n if key in stats.details:\n stats.operation_count = stats.details[key]\n break\n\n if self.verbose:\n self._print_benchmark_result(name, stats)\n\n return stats\n\n def _print_benchmark_result(self, name: str, stats: BenchmarkResult):\n \"\"\"Print individual benchmark result in structured format\"\"\"\n print(f\" {name}:\")\n print(f\" Time: {stats.mean_time:.3f}s ± {stats.std_time:.3f}s \"\n f\"(min: {stats.min_time:.3f}s, max: {stats.max_time:.3f}s)\")\n\n if stats.throughput_mb_s is not None:\n print(f\" Throughput: {stats.throughput_mb_s:.2f} MB/s\")\n\n if stats.file_size_bytes is not None:\n size_mb = stats.file_size_bytes / (1024**2)\n print(f\" File size: {size_mb:.1f} MB\")\n\n if stats.operation_count is not None:\n print(f\" Operations: {stats.operation_count:,}\")\n\n if stats.details:\n details_str = \", \".join([f\"{k}: {v}\" for k, v in stats.details.items()])\n print(f\" Details: {details_str}\")\n \n def generate_random_image_bytes(self, width=64, height=64, format=\"JPEG\", quality=85):\n \"\"\"Generate random image bytes for LMDB testing\"\"\"\n image_np = np.random.randint(0, 255, (height, width, 3), dtype=np.uint8)\n img = Image.fromarray(image_np)\n img_bytes = io.BytesIO()\n img.save(img_bytes, format=format, quality=quality)\n return img_bytes.getvalue()\n\n def generate_lmdb_data_entry(self, idx, image_size=(64, 64)):\n \"\"\"Generate a single LMDB data entry\"\"\"\n img_bytes = self.generate_random_image_bytes(width=image_size[0], height=image_size[1])\n return {\n \"index\": idx,\n \"txt\": f\"Sample text for entry {idx}\",\n \"jpeg\": img_bytes\n }\n\n def write_lmdb_data(self, lmdb_path, num_samples, image_size=(64, 64)):\n \"\"\"Write data to LMDB database\"\"\"\n env = lmdb.open(str(lmdb_path), readonly=False, meminit=False, map_size=1024**4)\n total_bytes = 0\n \n with env.begin(write=True) as txn:\n for i in range(num_samples):\n data = self.generate_lmdb_data_entry(i, image_size)\n key = str(i).encode()\n value = pickle.dumps(data)\n txn.put(key, value)\n total_bytes += len(value)\n \n env.close()\n return total_bytes\n\n def read_lmdb_data_single_process(self, lmdb_path):\n \"\"\"Read LMDB data using single process\"\"\"\n env = lmdb.open(str(lmdb_path), readonly=True, lock=False, readahead=False, meminit=False)\n total_bytes = 0\n samples_read = 0\n \n with env.begin(write=False) as txn:\n cursor = txn.cursor()\n for key, value in cursor:\n total_bytes += len(value)\n samples_read += 1\n \n env.close()\n return total_bytes, samples_read\n\n def lmdb_batch_worker(self, lmdb_path, key_batch):\n \"\"\"Worker function for multi-process LMDB reading\"\"\"\n env = lmdb.open(str(lmdb_path), readonly=True, lock=False, readahead=False, meminit=False)\n total_bytes = 0\n samples_processed = 0\n \n with env.begin(write=False) as txn:\n for key_bytes in key_batch:\n data = txn.get(key_bytes)\n if data:\n total_bytes += len(data)\n samples_processed += 1\n \n env.close()\n return samples_processed, total_bytes\n\n def read_lmdb_data_multi_process(self, lmdb_path, num_processes=2):\n \"\"\"Read LMDB data using multiple processes\"\"\"\n env = lmdb.open(str(lmdb_path), readonly=True, lock=False, readahead=False, meminit=False)\n \n # Get all keys\n keys = []\n with env.begin(write=False) as txn:\n cursor = txn.cursor()\n for key, _ in cursor:\n keys.append(key)\n \n env.close()\n \n # Split keys into batches for each process\n batch_size = len(keys) // num_processes + 1\n key_batches = [keys[i:i + batch_size] for i in range(0, len(keys), batch_size)]\n \n total_bytes = 0\n total_samples = 0\n with ProcessPoolExecutor(max_workers=num_processes) as executor:\n futures = []\n for batch in key_batches:\n futures.append(executor.submit(self.lmdb_batch_worker, lmdb_path, batch))\n \n for future in as_completed(futures):\n samples, bytes_read = future.result()\n total_samples += samples\n total_bytes += bytes_read\n \n return total_bytes, total_samples\n \n def benchmark_lmdb(self):\n \"\"\"Benchmark LMDB format for datasets\"\"\"\n results = {}\n num_samples = self.config['lmdb_num_samples']\n num_proc = self.config['lmdb_num_proc']\n image_size = self.config['lmdb_image_size']\n \n # Estimate file size (approx 5KB per sample)\n estimated_file_size = num_samples * 5 * 1024\n \n for size_name, sample_multiplier in [('small', 1), ('medium', 2)]:\n actual_samples = num_samples * sample_multiplier\n lmdb_dir = self.test_dir / f\"lmdb_{size_name}_{actual_samples}samples\"\n lmdb_dir.mkdir(exist_ok=True)\n \n def write_func():\n total_bytes = self.write_lmdb_data(lmdb_dir, actual_samples, image_size)\n return {\"file_size\": total_bytes, \"num_samples\": actual_samples}\n \n def read_single_func():\n bytes_read, samples_read = self.read_lmdb_data_single_process(lmdb_dir)\n return {\"bytes_read\": bytes_read, \"samples_read\": samples_read}\n\n def read_multi_func():\n bytes_read, samples_read = self.read_lmdb_data_multi_process(lmdb_dir, num_proc)\n return {\"bytes_read\": bytes_read, \"samples_read\": samples_read, \"processes\": num_proc}\n \n # Write benchmark\n write_stats = self.run_benchmark(\n f\"lmdb_{size_name}_write\", write_func, file_size=estimated_file_size * sample_multiplier,\n description=f\"Write LMDB ({actual_samples} samples)\"\n )\n \n # Single process read benchmark\n read_single_stats = self.run_benchmark(\n f\"lmdb_{size_name}_read_single\", read_single_func, file_size=estimated_file_size * sample_multiplier,\n description=f\"Read LMDB single process ({actual_samples} samples)\"\n )\n read_multi_stats = self.run_benchmark(\n f\"lmdb_{size_name}_read_multi\", read_multi_func, file_size=estimated_file_size * sample_multiplier,\n description=f\"Read LMDB multi process ({num_proc} processes, {actual_samples} samples)\"\n )\n \n # Cleanup\n if lmdb_dir.exists():\n shutil.rmtree(lmdb_dir)\n \n if write_stats and read_single_stats and read_multi_stats:\n results[size_name] = {\n \"write\": write_stats,\n \"read_single\": read_single_stats,\n \"read_multi\": read_multi_stats\n }\n \n return results\n\n\n # ----------------------------------------------------------------------\n # Model Weights Benchmarks\n # ----------------------------------------------------------------------\n\n def benchmark_pytorch_weights(self):\n \"\"\"Benchmark PyTorch .pt/.pth format with multiple sizes\"\"\"\n if torch is None:\n print(\"PyTorch not available, skipping PyTorch benchmark\")\n return None\n\n results = {}\n for size_name, size_mb in [('small', 1000), ('large', 4000)]:\n file_path = self.test_dir / f\"pytorch_weights_{size_name}_{size_mb}mb.pt\"\n file_size = size_mb * 1024 * 1024\n\n layer_sizes = [file_size // 8 // 5] * 5 # Split into 5 layers\n dummy_data = {\n 'weights': {f'layer_{i}': torch.randn(size) for i, size in enumerate(layer_sizes)},\n 'optimizer': {'lr': 0.001, 'momentum': 0.9},\n 'metadata': {'epoch': 10, 'version': '1.0', 'created': time.time()}\n }\n\n def write_func():\n torch.save(dummy_data, file_path)\n actual_size = os.path.getsize(file_path)\n return {'file_size': actual_size, 'num_layers': len(dummy_data['weights'])}\n\n def read_func():\n loaded = torch.load(file_path)\n total_params = 0\n for layer_name, weights in loaded['weights'].items():\n total_params += weights.numel()\n _ = torch.sum(weights).item() % 1000\n return {'file_size': total_params, 'num_layers': len(loaded['weights'])}\n\n write_stats = self.run_benchmark(\n f\"pytorch_weights_{size_name}_write\", write_func, file_size=file_size / 2,\n description=f\"Write PyTorch weights ({size_mb}MB)\"\n )\n\n read_stats = self.run_benchmark(\n f\"pytorch_weights_{size_name}_read\", read_func, file_size=file_size / 2,\n description=f\"Read PyTorch weights ({size_mb}MB)\"\n )\n\n if file_path.exists():\n file_path.unlink()\n\n if write_stats and read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n\n return results\n\n def benchmark_tensorflow_h5(self):\n \"\"\"Benchmark TensorFlow HDF5 format with multiple sizes\"\"\"\n if tf is None or h5py is None:\n print(\"TensorFlow or h5py not available, skipping HDF5 benchmark\")\n return None\n\n results = {}\n for size_name, size_mb in [('small', 500), ('large', 2000)]:\n file_path = self.test_dir / f\"tf_h5_{size_name}_{size_mb}mb.h5\"\n file_size = size_mb * 1024 * 1024\n\n def write_func():\n total_data_size = 0\n with h5py.File(file_path, \"w\") as f:\n num_layers = 8\n target_data_size = file_size\n data_per_dataset = target_data_size // (num_layers * 2)\n for i in range(num_layers):\n weights_elements = data_per_dataset // 4\n weights_data = np.random.randn(weights_elements).astype(np.float32)\n f.create_dataset(f'conv_{i}_weights', data=weights_data)\n total_data_size += weights_data.nbytes\n bias_data = np.random.randn(256).astype(np.float32)\n f.create_dataset(f'conv_{i}_bias', data=bias_data)\n total_data_size += bias_data.nbytes\n\n actual_size = os.path.getsize(file_path)\n return {\"file_size\": actual_size, \"num_datasets\": num_layers * 2}\n\n def read_func():\n total_size = 0\n dataset_count = 0\n data_checksum = 0\n actual_size = os.path.getsize(file_path)\n with h5py.File(file_path, \"r\") as f:\n for key in f.keys():\n if isinstance(f[key], h5py.Dataset):\n # 实际读取数据\n data = f[key][:]\n total_size += data.nbytes\n dataset_count += 1\n # 处理数据确保实际读取\n data_checksum = (data_checksum + np.sum(data)) % 1000000\n return {\"file_size\": actual_size, \"num_datasets\": dataset_count}\n write_stats = self.run_benchmark(\n f\"tensorflow_h5_{size_name}_write\", write_func, file_size=file_size,\n description=f\"Write TensorFlow H5 ({size_mb}MB)\"\n )\n self.clear_cache()\n read_stats = self.run_benchmark(\n f\"tensorflow_h5_{size_name}_read\", read_func, file_size=file_size,\n description=f\"Read TensorFlow H5 ({size_mb}MB)\"\n )\n\n if file_path.exists():\n file_path.unlink()\n\n if write_stats and read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n\n return results\n\n def benchmark_onnx(self):\n \"\"\"Benchmark ONNX model format\"\"\"\n if onnx is None or ort is None:\n print(\"ONNX or ONNX Runtime not available, skipping ONNX benchmark\")\n return None\n\n results = {}\n for size_name, size_mb in [('small', 50), ('medium', 100), ('large', 200)]:\n file_path = self.test_dir / f\"onnx_model_{size_name}_{size_mb}mb.onnx\"\n file_size = size_mb * 1024 * 1024\n\n # Create a simple ONNX model\n def create_onnx_model():\n from onnx import helper, TensorProto, save\n\n # Calculate appropriate tensor sizes to match target file size\n tensor_size = max(100, int((file_size * 0.8) / 4 / 4)) # Rough estimation\n\n # Create a simple graph\n X = helper.make_tensor_value_info('X', TensorProto.FLOAT, [1, 3, 224, 224])\n Y = helper.make_tensor_value_info('Y', TensorProto.FLOAT, [1, 1000])\n\n # Create weights with appropriate size\n weights = helper.make_tensor(\n 'W',\n TensorProto.FLOAT,\n [3 * 224 * 224, 1000],\n np.random.randn(3 * 224 * 224 * 1000).astype(np.float32)[:3*224*224*1000]\n )\n\n node = helper.make_node(\n 'MatMul',\n ['X', 'W'],\n ['Y'],\n name='matmul'\n )\n\n graph = helper.make_graph(\n [node],\n 'simple_model',\n [X],\n [Y],\n [weights]\n )\n\n model = helper.make_model(graph, producer_name='benchmark')\n return model\n\n def write_func():\n model = create_onnx_model()\n onnx.save(model, file_path)\n actual_size = os.path.getsize(file_path)\n return {\"file_size\": actual_size, \"model_size\": file_size}\n\n def read_func():\n # Load and validate model\n model = onnx.load(file_path)\n onnx.checker.check_model(model)\n\n # Run inference with ONNX Runtime\n sess = ort.InferenceSession(file_path)\n input_data = np.random.randn(1, 3, 224, 224).astype(np.float32)\n outputs = sess.run(None, {'X': input_data})\n return {\"output_shape\": outputs[0].shape, \"model_valid\": True}\n\n write_stats = self.run_benchmark(\n f\"onnx_{size_name}_write\", write_func, file_size=file_size,\n description=f\"Write ONNX model ({size_mb}MB)\"\n )\n\n read_stats = self.run_benchmark(\n f\"onnx_{size_name}_read\", read_func, file_size=file_size,\n description=f\"Read ONNX model ({size_mb}MB)\"\n )\n\n if file_path.exists():\n file_path.unlink()\n\n if write_stats and read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n\n return results\n\n def benchmark_huggingface_bin(self):\n \"\"\"Benchmark HuggingFace .bin format\"\"\"\n if torch is None:\n print(\"PyTorch not available, skipping HuggingFace benchmark\")\n return None\n\n results = {}\n for size_name, size_mb in [('small', 10), ('medium', 50), ('large', 100)]:\n file_path = self.test_dir / f\"hf_model_{size_name}_{size_mb}mb.bin\"\n file_size = size_mb * 1024 * 1024\n\n # Create HuggingFace-style model weights\n def create_hf_weights():\n # Calculate layer sizes to approximate target file size\n num_layers = 12\n layer_size = max(100, int((file_size * 0.9) / num_layers / 4)) # Rough estimation\n\n weights = {}\n for i in range(num_layers):\n weights[f\"layer.{i}.attention.self.query.weight\"] = torch.randn(layer_size)\n weights[f\"layer.{i}.attention.self.key.weight\"] = torch.randn(layer_size)\n weights[f\"layer.{i}.attention.self.value.weight\"] = torch.randn(layer_size)\n weights[f\"layer.{i}.attention.output.dense.weight\"] = torch.randn(layer_size)\n weights[f\"layer.{i}.intermediate.dense.weight\"] = torch.randn(layer_size)\n weights[f\"layer.{i}.output.dense.weight\"] = torch.randn(layer_size)\n\n # Add embeddings\n weights[\"embeddings.word_embeddings.weight\"] = torch.randn(layer_size)\n weights[\"embeddings.position_embeddings.weight\"] = torch.randn(512, layer_size)\n weights[\"embeddings.token_type_embeddings.weight\"] = torch.randn(2, layer_size)\n\n return weights\n\n def write_func():\n weights = create_hf_weights()\n torch.save(weights, file_path)\n actual_size = os.path.getsize(file_path)\n return {\"file_size\": actual_size, \"num_tensors\": len(weights)}\n\n def read_func():\n weights = torch.load(file_path)\n total_params = sum(param.numel() for param in weights.values())\n return {\"loaded_params\": total_params, \"num_tensors\": len(weights)}\n\n write_stats = self.run_benchmark(\n f\"huggingface_{size_name}_write\", write_func, file_size=file_size,\n description=f\"Write HuggingFace weights ({size_mb}MB)\"\n )\n\n read_stats = self.run_benchmark(\n f\"huggingface_{size_name}_read\", read_func, file_size=file_size,\n description=f\"Read HuggingFace weights ({size_mb}MB)\"\n )\n\n if file_path.exists():\n file_path.unlink()\n\n if write_stats and read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n\n return results\n\n def benchmark_tensorflow_checkpoint(self):\n \"\"\"Benchmark TensorFlow checkpoint format\"\"\"\n if tf is None:\n print(\"TensorFlow not available, skipping TF checkpoint benchmark\")\n return None\n # physical_devices = tf.config.list_physical_devices('CPU')\n # if physical_devices:\n # try:\n # tf.config.set_logical_device_configuration(\n # physical_devices[0],\n # [tf.config.LogicalDeviceConfiguration(memory_limit=5 * 1024)] # 5GB\n # )\n # print(\"Set TensorFlow memory limit to 5GB\")\n # except RuntimeError as e:\n # print(f\"Could not set memory limit: {e}\")\n results = {}\n for size_name, size_mb in [('small', 10), ('large', 100)]:\n checkpoint_dir = self.test_dir / f\"tf_checkpoint_{size_name}_{size_mb}mb\"\n checkpoint_dir.mkdir(exist_ok=True)\n file_size = size_mb * 1024 * 1024\n\n def write_func():\n # Create a simple model\n if size_mb == 10:\n layer_sizes = [2048, 1024, 512, 256, 128, 64]\n else: # 100MB\n layer_sizes = [8192, 4096, 2048, 1024, 512, 256]\n layers = [tf.keras.layers.Dense(layer_sizes[0], activation='relu', input_shape=(784,))]\n for size in layer_sizes[1:]:\n layers.append(tf.keras.layers.Dense(size, activation='relu'))\n layers.append(tf.keras.layers.Dense(10, activation='softmax'))\n model = tf.keras.Sequential(layers)\n checkpoint = tf.train.Checkpoint(model=model)\n checkpoint_path = checkpoint_dir / \"model.ckpt\"\n checkpoint.write(str(checkpoint_path))\n del model\n tf.keras.backend.clear_session()\n \n total_size = sum(file.stat().st_size for file in checkpoint_dir.glob(\"*\"))\n return {\"file_size\": total_size, \"num_files\": len(list(checkpoint_dir.glob(\"*\")))}\n\n def read_func():\n if size_mb == 10:\n layer_sizes = [2048, 1024, 512, 256, 128, 64]\n else:\n layer_sizes = [8192, 4096, 2048, 1024, 512, 256]\n \n layers = [tf.keras.layers.Dense(layer_sizes[0], activation='relu', input_shape=(784,))]\n for size in layer_sizes[1:]:\n layers.append(tf.keras.layers.Dense(size, activation='relu'))\n layers.append(tf.keras.layers.Dense(10, activation='softmax'))\n \n model = tf.keras.Sequential(layers)\n checkpoint = tf.train.Checkpoint(model=model)\n checkpoint_path = checkpoint_dir / \"model.ckpt\"\n checkpoint.restore(str(checkpoint_path))\n\n batch_size = 64\n num_batches = 100\n \n for i in range(num_batches):\n test_input = tf.random.normal((batch_size, 784))\n output = model(test_input)\n _ = tf.reduce_mean(output)\n \n del model\n tf.keras.backend.clear_session()\n \n return {\"output_shape\": output.shape, \"restored\": True}\n tf.keras.backend.clear_session()\n import gc\n gc.collect()\n write_stats = self.run_benchmark(\n f\"tf_checkpoint_{size_name}_write\", write_func, file_size=file_size,\n description=f\"Write TF checkpoint ({size_mb}MB)\"\n )\n\n read_stats = self.run_benchmark(\n f\"tf_checkpoint_{size_name}_read\", read_func, file_size=file_size,\n description=f\"Read TF checkpoint ({size_mb}MB)\"\n )\n\n # Cleanup\n if checkpoint_dir.exists():\n shutil.rmtree(checkpoint_dir)\n\n if write_stats and read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n\n return results\n\n # ----------------------------------------------------------------------\n # Dataset Format Benchmarks\n # ----------------------------------------------------------------------\n\n def benchmark_tfrecord(self):\n \"\"\"Benchmark TFRecord format for datasets\"\"\"\n if tf is None:\n print(\"TensorFlow not available, skipping TFRecord benchmark\")\n return None\n\n results = {}\n num_samples = self.config['num_samples']\n image_size = self.config['image_size']\n \n for size_name, sample_multiplier in [('small', 1), ('medium', 2), ('large', 4)]:\n actual_samples = num_samples * sample_multiplier\n file_path = self.test_dir / f\"tfrecord_{size_name}_{actual_samples}samples.tfrecord\"\n \n image_data_size = np.prod(image_size) * 4 # 图像数据大小 (float32)\n sample_size_estimate = image_data_size + 100 # 图像 + 标签 + 元数据\n file_size_bytes = actual_samples * sample_size_estimate\n\n def create_example(image_data, label, extra_features):\n feature = {\n 'image': tf.train.Feature(\n bytes_list=tf.train.BytesList(value=[image_data])),\n 'label': tf.train.Feature(\n int64_list=tf.train.Int64List(value=[label])),\n 'extra_features': tf.train.Feature(\n float_list=tf.train.FloatList(value=extra_features))\n }\n return tf.train.Example(features=tf.train.Features(feature=feature))\n\n def write_func():\n with tf.io.TFRecordWriter(str(file_path)) as writer:\n for i in range(actual_samples):\n # 创建随机图像数据和额外特征\n image_data = np.random.rand(*image_size).astype(np.float32).tobytes()\n label = i % 100\n extra_features = np.random.randn(10).astype(np.float32).tolist()\n\n example = create_example(image_data, label, extra_features)\n writer.write(example.SerializeToString())\n\n actual_file_size = os.path.getsize(file_path)\n return {\"file_size\": actual_file_size, \"num_samples\": actual_samples}\n \n def read_func():\n def parse_example(example_proto):\n feature_description = {\n 'image': tf.io.FixedLenFeature([], tf.string),\n 'label': tf.io.FixedLenFeature([], tf.int64),\n 'extra_features': tf.io.FixedLenFeature([10], tf.float32),\n }\n return tf.io.parse_single_example(example_proto, feature_description)\n\n # 创建数据集\n dataset = tf.data.TFRecordDataset(str(file_path))\n dataset = dataset.map(parse_example)\n\n # 实际读取和处理所有样本\n total_samples = 0\n total_image_size = 0\n label_sum = 0\n feature_sum = 0.0\n for example in dataset:\n total_samples += 1\n\n # 实际处理图像数据（触发磁盘读取）\n image_data = tf.io.decode_raw(example['image'], tf.float32)\n total_image_size += image_data.shape[0] * 4 # 4 bytes per float32\n\n # 处理标签和特征数据\n label_sum += example['label'].numpy()\n feature_sum += tf.reduce_sum(example['extra_features']).numpy()\n\n # 验证处理结果（防止编译器优化）\n validation_value = (label_sum + int(feature_sum)) % 1000\n _ = validation_value # 确保值被使用\n\n file_size = os.path.getsize(file_path)\n return {\n \"samples_read\": total_samples,\n \"file_size\": file_size,\n \"total_data_processed\": total_image_size,\n \"validation_ok\": validation_value >= 0\n }\n\n write_stats = self.run_benchmark(\n f\"tfrecord_{size_name}_write\", write_func, file_size=file_size_bytes,\n description=f\"Write TFRecord ({actual_samples} samples)\"\n )\n\n if write_stats:\n self.clear_cache()\n # 读取测试\n read_stats = self.run_benchmark(\n f\"tfrecord_{size_name}_read\", read_func, file_size=file_size_bytes,\n description=f\"Read TFRecord ({actual_samples} samples)\"\n )\n\n # 清理文件\n if file_path.exists():\n file_path.unlink()\n\n if read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n elif file_path.exists():\n file_path.unlink()\n return results \n\n def benchmark_hdf5_dataset(self):\n \"\"\"Benchmark HDF5 format for datasets\"\"\"\n if h5py is None:\n print(\"h5py not available, skipping HDF5 dataset benchmark\")\n return None\n\n results = {}\n num_samples = self.config['num_samples']\n image_size = self.config['image_size']\n\n sample_size_estimate = np.prod(image_size) * 4 \n\n for size_name, sample_multiplier in [('small', 1), ('medium', 2)]:\n actual_samples = num_samples * sample_multiplier\n file_path = self.test_dir / f\"hdf5_dataset_{size_name}_{actual_samples}samples.h5\"\n file_size_bytes = actual_samples * sample_size_estimate\n\n def write_func():\n all_images = np.random.rand(actual_samples, *image_size).astype(np.float32)\n all_labels = np.arange(actual_samples) % 10\n\n with h5py.File(file_path, 'w') as f:\n images = f.create_dataset(\n 'images',\n data=all_images,\n dtype=np.float32,\n compression='gzip'\n )\n labels = f.create_dataset(\n 'labels',\n data=all_labels,\n dtype=np.int64\n )\n\n actual_file_size = os.path.getsize(file_path)\n return {\"file_size\": actual_file_size, \"num_samples\": actual_samples}\n def read_func():\n with h5py.File(file_path, 'r') as f:\n images = f['images'][:]\n labels = f['labels'][:]\n\n total_images = len(images)\n file_size = os.path.getsize(file_path)\n return {\"samples_read\": total_images, \"file_size\": file_size}\n\n write_stats = self.run_benchmark(\n f\"hdf5_dataset_{size_name}_write\", write_func, file_size=file_size_bytes,\n description=f\"Write HDF5 dataset ({actual_samples} samples)\"\n )\n self.clear_cache()\n read_stats = self.run_benchmark(\n f\"hdf5_dataset_{size_name}_read\", read_func, file_size=file_size_bytes,\n description=f\"Read HDF5 dataset ({actual_samples} samples)\"\n )\n\n if file_path.exists():\n file_path.unlink()\n\n if write_stats and read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n\n return results\n\n def benchmark_parquet(self):\n \"\"\"Benchmark Parquet format for datasets\"\"\"\n if pq is None or pa is None:\n print(\"PyArrow not available, skipping Parquet benchmark\")\n return None\n\n results = {}\n num_samples = self.config['num_samples']\n\n sample_size_estimate = 500\n\n for size_name, sample_multiplier in [('small', 2), ('medium', 4), ('large', 8)]:\n actual_samples = num_samples * sample_multiplier\n file_path = self.test_dir / f\"parquet_{size_name}_{actual_samples}samples.parquet\"\n file_size_bytes = actual_samples * sample_size_estimate\n\n def write_func():\n # Create sample data\n data = {\n 'id': list(range(actual_samples)),\n 'feature1': np.random.randn(actual_samples).astype(np.float32),\n 'feature2': np.random.randn(actual_samples).astype(np.float32),\n 'feature3': np.random.randn(actual_samples).astype(np.float32),\n 'label': np.random.randint(0, 10, actual_samples).astype(np.int64),\n 'timestamp': [time.time()] * actual_samples\n }\n\n table = pa.Table.from_pydict(data)\n pq.write_table(table, file_path, compression='snappy')\n\n actual_file_size = os.path.getsize(file_path)\n return {\"file_size\": actual_file_size, \"num_samples\": actual_samples}\n \n def read_func():\n parquet_file = pq.ParquetFile(file_path)\n total_rows = 0\n feature_sum = 0.0\n \n for i in range(parquet_file.num_row_groups):\n table = parquet_file.read_row_group(i)\n df = table.to_pandas()\n total_rows += len(df)\n feature_sum += df['feature1'].sum() + df['feature2'].sum()\n \n _ = feature_sum % 1000\n return {\"rows_read\": total_rows, \"file_size\": os.path.getsize(file_path)}\n\n write_stats = self.run_benchmark(\n f\"parquet_{size_name}_write\", write_func, file_size=file_size_bytes,\n description=f\"Write Parquet ({actual_samples} samples)\"\n )\n self.clear_cache()\n read_stats = self.run_benchmark(\n f\"parquet_{size_name}_read\", read_func, file_size=file_size_bytes,\n description=f\"Read Parquet ({actual_samples} samples)\"\n )\n\n if file_path.exists():\n file_path.unlink()\n\n if write_stats and read_stats:\n results[size_name] = {\"write\": write_stats, \"read\": read_stats}\n\n return results\n\n def benchmark_comprehensive(self):\n \"\"\"Run comprehensive benchmarks with multiple file sizes\"\"\"\n benchmarks = [\n (\"LMDB\", self.benchmark_lmdb),\n (\"PyTorch Weights\", self.benchmark_pytorch_weights),\n (\"TensorFlow H5\", self.benchmark_tensorflow_h5),\n # (\"ONNX\", self.benchmark_onnx),\n (\"HuggingFace Bin\", self.benchmark_huggingface_bin),\n (\"TensorFlow Checkpoint\", self.benchmark_tensorflow_checkpoint),\n # (\"TFRecord Dataset\", self.benchmark_tfrecord),\n (\"HDF5 Dataset\", self.benchmark_hdf5_dataset),\n (\"Parquet Dataset\", self.benchmark_parquet),\n ]\n\n comprehensive_results = {}\n\n for name, benchmark_func in benchmarks:\n try:\n print(f\"\\n{'='*60}\")\n print(f\"RUNNING COMPREHENSIVE {name.upper()} BENCHMARK\")\n print(f\"{'='*60}\")\n\n result = benchmark_func()\n if result:\n comprehensive_results[name.lower().replace(\" \", \"_\")] = result\n print(f\"✓ Completed comprehensive {name} benchmark\")\n else:\n print(f\"✗ {name} benchmark returned no results\")\n\n except Exception as e:\n print(f\"✗ Error running comprehensive {name} benchmark: {e}\")\n import traceback\n traceback.print_exc()\n comprehensive_results[name.lower().replace(\" \", \"_\")] = {\"error\": str(e)}\n\n return comprehensive_results\n\n def generate_report(self):\n \"\"\"Generate detailed performance report\"\"\"\n def default_serializer(obj):\n if isinstance(obj, BenchmarkResult):\n return {\n 'min_time': obj.min_time,\n 'max_time': obj.max_time,\n 'mean_time': obj.mean_time,\n 'std_time': obj.std_time,\n 'throughput_mb_s': obj.throughput_mb_s,\n 'file_size_bytes': obj.file_size_bytes,\n 'operation_count': obj.operation_count,\n 'details': obj.details\n }\n elif isinstance(obj, np.integer):\n return int(obj)\n elif isinstance(obj, np.floating):\n return float(obj)\n elif isinstance(obj, np.ndarray):\n return obj.tolist()\n elif hasattr(obj, '__dict__'):\n return obj.__dict__\n return str(obj)\n\n report = {\n 'version': self.version,\n 'timestamp': time.strftime('%Y-%m-%d %H:%M:%S'),\n 'mount_point': str(self.mount_point),\n 'config': self.config,\n 'environment': {\n 'python_version': sys.version,\n 'has_torch': torch is not None,\n 'has_tensorflow': tf is not None,\n 'has_h5py': h5py is not None,\n 'has_pyarrow': pq is not None,\n 'has_onnx': onnx is not None,\n 'has_onnxruntime': ort is not None\n },\n 'results': self.results\n }\n\n summary = {}\n for benchmark_name, benchmark_data in self.results.items():\n if isinstance(benchmark_data, dict) and 'error' not in benchmark_data:\n for size_name, size_data in benchmark_data.items():\n if isinstance(size_data, dict):\n for op_type, stats in size_data.items():\n if hasattr(stats, 'throughput_mb_s') and stats.throughput_mb_s:\n key = f\"{benchmark_name}_{size_name}_{op_type}\"\n summary[key] = {\n 'throughput_mb_s': stats.throughput_mb_s,\n 'time_s': stats.mean_time,\n 'file_size_mb': stats.file_size_bytes / (1024**2) if stats.file_size_bytes else None\n }\n\n report['summary'] = summary\n return report\n\n def save_results(self):\n \"\"\"Save results to JSON file with comprehensive report\"\"\"\n self.results_file.parent.mkdir(parents=True, exist_ok=True)\n\n report = self.generate_report()\n\n def default_serializer(obj):\n if isinstance(obj, BenchmarkResult):\n return {\n 'min_time': obj.min_time,\n 'max_time': obj.max_time,\n 'mean_time': obj.mean_time,\n 'std_time': obj.std_time,\n 'throughput_mb_s': obj.throughput_mb_s,\n 'file_size_bytes': obj.file_size_bytes,\n 'operation_count': obj.operation_count,\n 'details': obj.details\n }\n elif isinstance(obj, np.integer):\n return int(obj)\n elif isinstance(obj, np.floating):\n return float(obj)\n elif isinstance(obj, np.ndarray):\n return obj.tolist()\n elif hasattr(obj, '__dict__'):\n return obj.__dict__\n return str(obj)\n\n with open(self.results_file, 'w') as f:\n json.dump(report, f, indent=2, default=default_serializer)\n\n print(f\"\\nResults saved to {self.results_file}\")\n\n def print_detailed_summary(self):\n \"\"\"Print detailed summary of all benchmark results\"\"\"\n print(f\"\\n{'='*80}\")\n print(f\"COMPREHENSIVE AI FORMAT PERFORMANCE BENCHMARK SUMMARY\")\n print(f\"Version: {self.version}\")\n print(f\"Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}\")\n print(f\"{'='*80}\")\n\n for benchmark_name, benchmark_data in self.results.items():\n if isinstance(benchmark_data, dict) and 'error' in benchmark_data:\n print(f\"\\n{benchmark_name.upper()}: ERROR - {benchmark_data['error']}\")\n continue\n\n print(f\"\\n{benchmark_name.upper()}:\")\n for size_name, size_data in benchmark_data.items():\n print(f\" {size_name.upper()} FILES:\")\n for op_type, stats in size_data.items():\n if hasattr(stats, 'mean_time'):\n print(f\" {op_type.upper()}:\")\n print(f\" Time: {stats.mean_time:.3f}s ± {stats.std_time:.3f}s\")\n print(f\" Range: {stats.min_time:.3f}s - {stats.max_time:.3f}s\")\n\n if stats.throughput_mb_s:\n print(f\" Throughput: {stats.throughput_mb_s:.2f} MB/s\")\n\n if stats.file_size_bytes:\n size_mb = stats.file_size_bytes / (1024**2)\n print(f\" Size: {size_mb:.1f} MB\")\n\n if stats.details:\n details = \", \".join([f\"{k}: {v}\" for k, v in stats.details.items()])\n print(f\" Details: {details}\")\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Comprehensive AI Format Performance Benchmark\")\n parser.add_argument(\"mount_point\", help=\"Mount point to test\")\n parser.add_argument(\"results_file\", help=\"File to save results JSON\")\n parser.add_argument(\"version\", help=\"Version identifier\")\n parser.add_argument(\"--verbose\", \"-v\", action=\"store_true\", help=\"Verbose output\")\n parser.add_argument(\"--quick\", \"-q\", action=\"store_true\", help=\"Quick test (small files only)\")\n args = parser.parse_args()\n\n benchmark = AIFormatBenchmark(args.mount_point, args.results_file, args.version)\n benchmark.verbose = args.verbose\n\n if args.quick:\n benchmark.config.update({\n 'small_file_mb': 10,\n 'medium_file_mb': 20,\n 'large_file_mb': 50,\n 'num_runs': 1,\n 'cool_down_time': 0.3,\n 'num_samples': 500,\n 'lmdb_num_samples': 200,\n 'lmdb_num_proc': 1,\n 'lmdb_image_size': (32, 32)\n })\n\n print(\"Starting Comprehensive AI Format Performance Benchmark...\")\n print(f\"Configuration: {benchmark.config}\")\n\n # Run comprehensive benchmarks\n benchmark.results = benchmark.benchmark_comprehensive()\n\n # Save and display results\n benchmark.save_results()\n benchmark.print_detailed_summary()\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": ".github/scripts/perf/compare_ai.sh", "content": "#!/bin/bash\n# fixed_compare.sh\n\ncurrent_file=\"$1\"\nold_file=\"$2\"\nTOLERANCE=${TOLERANCE:-0.3}\nEXIT_ON_REGRESSION=${EXIT_ON_REGRESSION:-true}\n\necho \"====================================================================\"\necho \"Fixed Performance Comparison Summary ($(echo \"$TOLERANCE * 100\" | bc)% tolerance):\"\necho \"====================================================================\"\necho \"Current: $current_file\"\necho \"Old: $old_file\"\necho \"====================================================================\"\n\nregression_detected=false\n\nkeys=$(jq -r '.summary | keys[]' \"$current_file\")\n\ndeclare -A categories\ncategories[\"lmdb\"]=\"Lmdb\"\ncategories[\"pytorch_weights\"]=\"PyTorch Weights\"\ncategories[\"tensorflow_h5\"]=\"TensorFlow H5\"\ncategories[\"huggingface\"]=\"HuggingFace Bin\"\ncategories[\"tensorflow_checkpoint\"]=\"TensorFlow Checkpoint\"\ncategories[\"tfrecord\"]=\"TFRecord Dataset\"\ncategories[\"hdf5_dataset\"]=\"HDF5 Dataset\"\ncategories[\"parquet\"]=\"Parquet Dataset\"\n\necho \"Available keys in results:\"\necho \"$keys\"\necho \"\"\n\nfor category_pattern in \"${!categories[@]}\"; do\n category_name=\"${categories[$category_pattern]}\"\n echo \"=== $category_name ===\"\n category_keys=$(echo \"$keys\" | grep \"^${category_pattern}_\")\n if [ -z \"$category_keys\" ]; then\n echo \" No tests found for this category (pattern: $category_pattern)\"\n echo \"\"\n continue\n fi\n \n while read -r key; do\n current_throughput=$(jq -r \".summary.\\\"$key\\\".throughput_mb_s\" \"$current_file\")\n old_throughput=$(jq -r \".summary.\\\"$key\\\".throughput_mb_s\" \"$old_file\")\n\n if [ \"$current_throughput\" = \"null\" ] || [ \"$old_throughput\" = \"null\" ] || [ \"$current_throughput\" = \"\" ] || [ \"$old_throughput\" = \"\" ]; then\n continue\n fi\n\n diff=$(echo \"scale=1; $current_throughput - $old_throughput\" | bc)\n diff_pct=$(echo \"scale=1; ($diff / $old_throughput) * 100\" | bc)\n abs_diff_pct=$(echo $diff_pct | awk '{if ($1<0) print -$1; else print $1}')\n current_formatted=$(printf \"%.1f\" \"$current_throughput\")\n old_formatted=$(printf \"%.1f\" \"$old_throughput\")\n diff_pct_formatted=$(printf \"%.1f\" \"$diff_pct\")\n\n status=\"✓ OK\"\n if (( $(echo \"$abs_diff_pct > $TOLERANCE * 100\" | bc -l) )); then\n if (( $(echo \"$current_throughput < $old_throughput\" | bc -l) )); then\n status=\"❌ Worse\"\n regression_detected=true\n else\n status=\"✅ Better\"\n fi\n fi\n\n test_size=$(echo \"$key\" | awk -F_ '{print $(NF-1)}')\n test_operation=$(echo \"$key\" | awk -F_ '{print $NF}')\n \n echo \" ${test_size}_${test_operation}:\"\n echo \" Current: $current_formatted MB/s\"\n echo \" Old: $old_formatted MB/s\"\n echo \" Diff: $diff_pct_formatted%\"\n echo \" Status: $status\"\n echo \"\"\n\n done <<< \"$category_keys\"\ndone\n\necho \"====================================================================\"\necho \"Summary:\"\nif [ \"$regression_detected\" = true ]; then\n echo \"❌ PERFORMANCE REGRESSION DETECTED!\"\n if [ \"$EXIT_ON_REGRESSION\" = true ]; then\n exit 1\n else\n exit 0\n fi\nelse\n echo \"✅ No performance regression detected.\"\n exit 0\nfi\n" }, { "path": ".github/scripts/perf/compare_mdtest_fio.sh", "content": "#!/bin/bash\nset -e\n\nCURRENT_RESULTS=$1\nOLD_RESULTS=$2\nFILTER_OPS=(\"File read\" \"File stat\" \"File removal\" \"Tree removal\" \"Tree creation\")\n\n# Function to extract files/s from built-in mdtest output\nextract_files_per_sec() {\n local mdtest_output=$1\n local files_per_sec=$(grep -oP 'Created .*files.*\\(\\K[0-9]+(\\.[0-9]+)?(?= files/s\\))' <<< \"$mdtest_output\" | head -1)\n if [[ -z \"$files_per_sec\" ]]; then\n files_per_sec=$(grep -oP '\\(\\K[0-9]+(\\.[0-9]+)?(?= files/s\\))' <<< \"$mdtest_output\" | head -1)\n fi\n if [[ -z \"$files_per_sec\" ]]; then\n echo \"0\"\n else\n echo \"$files_per_sec\"\n fi\n}\n\n# Function to extract IOPS from fio output\nextract_iops() {\n local fio_output=$1\n local iops=$(grep -oP 'IOPS=\\K[\\d.]+[kMG]?' <<< \"$fio_output\" | head -1)\n # Convert to numeric value (handle k/M/G suffixes)\n if [[ \"$iops\" == *k ]]; then\n echo \"${iops%k} * 1000\" | bc -l\n elif [[ \"$iops\" == *M ]]; then\n echo \"${iops%M} * 1000000\" | bc -l\n elif [[ \"$iops\" == *G ]]; then\n echo \"${iops%G} * 1000000000\" | bc -l\n else\n echo \"$iops\"\n fi\n}\n\nextract_bw() {\n local fio_output=$1\n local bw=$(grep -oP 'BW=\\K[^, ]+' <<< \"$fio_output\" | head -1)\n if [[ -z \"$bw\" ]]; then\n echo \"N/A\"\n else\n echo \"$bw\"\n fi\n}\n\nextract_metrics() {\n awk '{\n op_description=$1;\n op_type=$2;\n for(i=3;i<=NF;i++) if($i == \":\") break;\n max=$(i+1); min=$(i+2); mean=$(i+3); stddev=$(i+4);\n print op_description, op_type, max, min, mean, stddev\n }' <<< \"$1\"\n}\n\nis_op_in_filter() {\n local op=\"$1\"\n for allowed_op in \"${FILTER_OPS[@]}\"; do\n if [[ \"$op\" == \"$allowed_op\" ]]; then\n return 0\n fi\n done\n return 1\n}\n\ncompare_with_tolerance() {\n local current=$1\n local old=$2\n local op_type=$3\n local direction=${4:-higher}\n tolerance=$(echo \"$old * 0.2\" | bc -l)\n lower_bound=$(echo \"$old - $tolerance\" | bc -l)\n upper_bound=$(echo \"$old + $tolerance\" | bc -l)\n\n # For time comparison, lower is better\n if is_op_in_filter \"$op_type\"; then\n echo \"skip\"\n elif (( $(echo \"$current <= $upper_bound && $current >= $lower_bound\" | bc -l) )); then\n echo \"same\"\n else\n if [[ \"$direction\" == \"lower\" ]]; then\n if (( $(echo \"$current < $old\" | bc -l) )); then\n echo \"better\"\n else\n echo \"worse\"\n fi\n else\n if (( $(echo \"$current > $old\" | bc -l) )); then\n echo \"better\"\n else\n echo \"worse\"\n fi\n fi\n fi\n}\n\ncompare_scenario() {\n local scenario=$1\n local current_file=\"${CURRENT_RESULTS}.${scenario}.summary\"\n local old_file=\"${OLD_RESULTS}.${scenario}.summary\"\n\n echo \"\"\n echo \"====================================================================\"\n echo \"Detailed Comparison for $scenario (with 20% tolerance)\"\n case \"$scenario\" in\n \"scenario1\")\n echo \"Command is : mpirun --use-hwthread-cpus --allow-run-as-root -np 4 mdtest -b 3 -z 1 -I 300\"\n ;;\n \"scenario2\")\n echo \"Command is : mpirun --use-hwthread-cpus --allow-run-as-root -np 4 mdtest -F -w 102400 -I 3000 -z 0\"\n ;;\n \"scenario3\")\n echo \"Command is : ./juicefs mdtest /mdtest_perf --threads 10 --dirs 3 --depth 3 --files 100\"\n ;;\n \"fio_scenario4\")\n echo \"Command is : fio --name=big-write --directory=/mnt/fio --group_reporting --rw=write --direct=1 --bs=64k --end_fsync=1 --numjobs=8 --nrfiles=1 --size=2G --runtime=120\"\n ;;\n \"fio_scenario5\")\n echo \"Command is : fio --name=big-write --group_reporting --rw=randwrite --direct=1 --bs=64k --end_fsync=1 --runtime=200 --numjobs=8 --nrfiles=1 --size=2G\"\n ;;\n \"fio_scenario6\")\n echo \"Command is : fio --name=big-read-multiple --group_reporting --runtime=300 --rw=read --direct=1 --bs=4k --numjobs=8 --nrfiles=1 --size=2G\"\n ;;\n \"fio_scenario7\")\n echo \"Command is : fio --name=big-read-multiple-concurrent --group_reporting --rw=randread --direct=1 --bs=4k --numjobs=8 --nrfiles=1 --openfiles=1 --size=2G --output-format=normal --runtime=120\"\n ;;\n \"fio_scenario8\")\n echo \"fio --name=big-write --directory=\"$MNT_POINT/fio\" --group_reporting \\\n --rw=write --direct=1 --bs=1m --end_fsync=1 --runtime=120 \\\n --numjobs=8 --nrfiles=8 --size=2G\"\n ;;\n \"fio_scenario9\")\n echo \"Command is : fio --name=big-read-multiple-concurrent --directory=\"$MNT_POINT/fio\" --group_reporting \\\n --rw=read --direct=1 --bs=1m --numjobs=8 --nrfiles=8 --openfiles=1 --size=2G --output-format=normal --runtime=120\"\n ;;\n esac\n echo \"====================================================================\"\n\n # Handle built-in mdtest scenario (scenario3)\n if [[ \"$scenario\" == \"scenario3\" ]]; then\n printf \"%-30s %-12s %-12s %-12s %-12s %-12s\\n\" \"Operation\" \"Current files/s\" \"Old files/s\" \"Diff\" \"Status\" \"Variance\"\n echo \"--------------------------------------------------------------------\"\n\n current_files_per_sec=$(extract_files_per_sec \"$(cat \"${current_file}\")\")\n old_files_per_sec=$(extract_files_per_sec \"$(cat \"${old_file}\")\")\n\n diff=$(echo \"$current_files_per_sec - $old_files_per_sec\" | bc -l)\n if (( $(echo \"$old_files_per_sec == 0\" | bc -l) )); then\n variance=\"N/A\"\n comparison=\"same\"\n else\n variance=$(echo \"scale=2; ($current_files_per_sec - $old_files_per_sec)*100/$old_files_per_sec\" | bc -l)\n comparison=$(compare_with_tolerance $current_files_per_sec $old_files_per_sec \"builtin_mdtest\")\n fi\n\n case $comparison in\n \"worse\") status=\"❌ Worse\" ;;\n \"better\") status=\"✅ Better\" ;;\n \"same\") status=\"⚖️ Same\" ;;\n \"skip\") status=\"⏭️ Skipped\" ;;\n *) status=\"⚠️ Unknown\" ;;\n esac\n\n if [[ \"$variance\" == \"N/A\" ]]; then\n printf \"%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s\\n\" \\\n \"Built-in mdtest\" \"$current_files_per_sec\" \"$old_files_per_sec\" \"$diff\" \"$status\" \"$variance\"\n else\n printf \"%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s%%\\n\" \\\n \"Built-in mdtest\" \"$current_files_per_sec\" \"$old_files_per_sec\" \"$diff\" \"$status\" \"$variance\"\n fi\n \n # Handle fio scenarios\n elif [[ \"$scenario\" =~ ^fio ]]; then\n printf \"%-30s %-12s %-12s %-12s %-12s %-12s\\n\" \"Operation\" \"Current IOPS\" \"Old IOPS\" \"Diff\" \"Status\" \"Variance\"\n echo \"--------------------------------------------------------------------\"\n\n current_iops=$(extract_iops \"$(cat \"${current_file}\")\")\n old_iops=$(extract_iops \"$(cat \"${old_file}\")\")\n current_bw=$(extract_bw \"$(cat \"${current_file}\")\")\n old_bw=$(extract_bw \"$(cat \"${old_file}\")\")\n\n diff=$(echo \"$current_iops - $old_iops\" | bc -l)\n variance=$(echo \"scale=2; ($current_iops - $old_iops)*100/$old_iops\" | bc -l)\n comparison=$(compare_with_tolerance $current_iops $old_iops \"fio_${scenario}\")\n\n case $comparison in\n \"worse\") status=\"❌ Worse\" ;;\n \"better\") status=\"✅ Better\" ;;\n \"same\") status=\"⚖️ Same\" ;;\n \"skip\") status=\"⏭️ Skipped\" ;;\n *) status=\"⚠️ Unknown\" ;;\n esac\n\n printf \"%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s%%\\n\" \\\n \"FIO ${scenario}\" \"$current_iops\" \"$old_iops\" \"$diff\" \"$status\" \"$variance\"\n printf \"%-30s %-12s %-12s\\n\" \"Bandwidth\" \"$current_bw\" \"$old_bw\"\n\n # Handle mdtest scenarios\n else\n printf \"%-30s %-12s %-12s %-12s %-12s %-12s\\n\" \"Operation\" \"Current Max\" \"Old Max\" \"Diff\" \"Status\" \"Variance\"\n echo \"--------------------------------------------------------------------\"\n\n while IFS= read -r current_line && IFS= read -r old_line <&3; do\n if [ -z \"$current_line\" ] || [ -z \"$old_line\" ]; then\n continue\n fi\n\n current_metrics=($(extract_metrics \"$current_line\"))\n old_metrics=($(extract_metrics \"$old_line\"))\n\n current_op=\"${current_metrics[0]} ${current_metrics[1]}\"\n old_op=\"${old_metrics[0]} ${old_metrics[1]}\"\n\n if [ \"$current_op\" != \"$old_op\" ]; then\n echo \"Warning: Operation mismatch ('$current_op' vs '$old_op'), skipping...\"\n continue\n fi\n\n current_max=${current_metrics[2]}\n old_max=${old_metrics[2]}\n\n if [[ \"$current_max\" =~ ^[0-9.]+$ ]] && [[ \"$old_max\" =~ ^[0-9.]+$ ]]; then\n diff=$(echo \"$current_max - $old_max\" | bc -l)\n variance=$(echo \"scale=2; ($current_max - $old_max)*100/$old_max\" | bc -l)\n comparison=$(compare_with_tolerance $current_max $old_max \"$current_op\")\n\n case $comparison in\n \"worse\") status=\"❌ Worse\" ;;\n \"better\") status=\"✅ Better\" ;;\n \"same\") status=\"⚖️ Same\" ;;\n \"skip\") status=\"⏭️ Skipped\" ;;\n *) status=\"⚠️ Unknown\" ;;\n esac\n\n printf \"%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s%%\\n\" \\\n \"$current_op\" \"$current_max\" \"$old_max\" \"$diff\" \"$status\" \"$variance\"\n else\n printf \"%-30s %-12s %-12s %-12s %-12s %-12s\\n\" \\\n \"$current_op\" \"N/A\" \"N/A\" \"N/A\" \"⚠️ Invalid\" \"N/A\"\n fi\n done < \"$current_file\" 3< \"$old_file\"\n fi\n}\n\n# Check if any scenario has \"worse\" results\ncheck_regression() {\n local scenario=$1\n local current_file=\"${CURRENT_RESULTS}.${scenario}.summary\"\n local old_file=\"${OLD_RESULTS}.${scenario}.summary\"\n local regression_detected=0\n\n # Handle built-in mdtest scenario (scenario3)\n if [[ \"$scenario\" == \"scenario3\" ]]; then\n current_files_per_sec=$(extract_files_per_sec \"$(cat \"${current_file}\")\")\n old_files_per_sec=$(extract_files_per_sec \"$(cat \"${old_file}\")\")\n if (( $(echo \"$old_files_per_sec == 0\" | bc -l) )); then\n comparison=\"same\"\n else\n comparison=$(compare_with_tolerance $current_files_per_sec $old_files_per_sec \"builtin_mdtest\")\n fi\n\n if [ \"$comparison\" == \"worse\" ]; then\n variance=$(echo \"scale=2; ($current_files_per_sec - $old_files_per_sec)*100/$old_files_per_sec\" | bc -l)\n echo \"Regression detected in $scenario for built-in mdtest (files/s): Current $current_files_per_sec vs Old $old_files_per_sec (Variance: ${variance}%)\"\n regression_detected=1\n fi\n \n # Handle fio scenarios\n elif [[ \"$scenario\" =~ ^fio ]]; then\n current_iops=$(extract_iops \"$(cat \"${current_file}\")\")\n old_iops=$(extract_iops \"$(cat \"${old_file}\")\")\n comparison=$(compare_with_tolerance $current_iops $old_iops \"fio_${scenario}\")\n\n if [ \"$comparison\" == \"worse\" ]; then\n variance=$(echo \"scale=2; ($current_iops - $old_iops)*100/$old_iops\" | bc -l)\n echo \"Regression detected in $scenario: Current $current_iops IOPS vs Old $old_iops IOPS (Variance: ${variance}%)\"\n regression_detected=1\n fi\n\n # Handle mdtest scenarios\n else\n while IFS= read -r current_line && IFS= read -r old_line <&3; do\n # Skip empty lines\n if [ -z \"$current_line\" ] || [ -z \"$old_line\" ]; then\n continue\n fi\n\n current_metrics=($(extract_metrics \"$current_line\"))\n old_metrics=($(extract_metrics \"$old_line\"))\n\n current_op=\"${current_metrics[0]} ${current_metrics[1]}\"\n old_op=\"${old_metrics[0]} ${old_metrics[1]}\"\n\n if [ \"$current_op\" != \"$old_op\" ]; then\n continue\n fi\n\n current_max=${current_metrics[2]}\n old_max=${old_metrics[2]}\n\n if [[ \"$current_max\" =~ ^[0-9.]+$ ]] && [[ \"$old_max\" =~ ^[0-9.]+$ ]]; then\n comparison=$(compare_with_tolerance $current_max $old_max \"$current_op\")\n if [ \"$comparison\" == \"worse\" ]; then\n variance=$(echo \"scale=2; ($current_max - $old_max)*100/$old_max\" | bc -l)\n echo \"Regression detected in $scenario for $current_op: Current $current_max vs Old $old_max (Variance: ${variance}%)\"\n regression_detected=1\n fi\n fi\n done < \"$current_file\" 3< \"$old_file\"\n fi\n\n return $regression_detected\n}\n\necho \"\"\necho \"====================================================================\"\necho \"Performance Comparison Summary (with 20% tolerance)\"\necho \"====================================================================\"\n\ncompare_scenario \"scenario1\"\ncompare_scenario \"scenario2\"\ncompare_scenario \"scenario3\"\ncompare_scenario \"fio_scenario4\"\ncompare_scenario \"fio_scenario5\"\ncompare_scenario \"fio_scenario6\"\ncompare_scenario \"fio_scenario7\"\ncompare_scenario \"fio_scenario8\"\ncompare_scenario \"fio_scenario9\"\n\necho \"\"\necho \"====================================================================\"\necho \"Regression Check Summary (with 20% tolerance)\"\necho \"====================================================================\"\n\nregression_found=0\nif ! check_regression \"scenario1\"; then\n regression_found=1\nfi\nif ! check_regression \"scenario2\"; then\n regression_found=1\nfi\nif ! check_regression \"scenario3\"; then\n regression_found=1\nfi\nif ! check_regression \"fio_scenario4\"; then\n regression_found=1\nfi\nif ! check_regression \"fio_scenario5\"; then\n regression_found=1\nfi\nif ! check_regression \"fio_scenario6\"; then\n regression_found=1\nfi\nif ! check_regression \"fio_scenario7\"; then\n regression_found=1\nfi\nif ! check_regression \"fio_scenario8\"; then\n regression_found=1\nfi\nif ! check_regression \"fio_scenario9\"; then\n regression_found=1\nfi\n\nif [ $regression_found -eq 1 ]; then\n echo \"\"\n echo \"ERROR: Performance regression detected compared to old version!\"\n exit 1\nelse\n echo \"\"\n echo \"SUCCESS: No performance regression detected.\"\n exit 0\nfi\n" }, { "path": ".github/scripts/perf/mdtest_fio.sh", "content": "#!/bin/bash\nset -e\n\nMNT_POINT=$1\nRESULTS_FILE=$2\nVERSION=$3\nMETA_URL=$4\n\nif [[ -z \"$META_URL\" ]]; then\n echo \"ERROR: META_URL is required as 4th argument for built-in mdtest scenario\"\n exit 1\nfi\n\nmkdir -p \"$(dirname \"$RESULTS_FILE\")\"\n\nprocess_run() {\n local output=$1\n local scenario=$2\n local attempt=$3\n\n # For built-in mdtest (scenario3) and fio tests, we just capture the output\n if [[ \"$scenario\" == \"scenario3\" || \"$scenario\" =~ ^fio ]]; then\n cp \"$output\" \"${output}.summary\"\n return\n fi\n\n grep -A 100 \"SUMMARY rate:\" \"$output\" | \\\n grep -v \"SUMMARY rate:\" | \\\n grep -v \"\\-\\-\\-\" | \\\n grep -v \"Command line used:\" | \\\n grep -v \"Path:\" | \\\n grep -v \"FS:\" | \\\n grep -v \"Nodemap:\" | \\\n grep -v \"tasks,\" | \\\n awk 'NF' > \"${output}.tmp\"\n\n # Convert to CSV format for easier processing\n awk '{\n # Skip lines that don'\\''t contain operation metrics\n if ($0 ~ /:/ && $0 !~ /^-+$/) {\n op=\"\";\n for(i=1;i<=NF;i++) {\n if ($i == \":\") {\n # Join all words before \":\" as operation name\n for(j=1;j1?\" \":\"\") $j;\n # Extract metrics\n max=$(i+1); min=$(i+2); mean=$(i+3); stddev=$(i+4);\n print op \",\" max \",\" min \",\" mean \",\" stddev;\n break;\n }\n }\n }\n }' \"${output}.tmp\" > \"${output}.csv\"\n\n rm -f \"${output}.tmp\"\n}\n\ncalculate_averages() {\n local scenario=$1\n local runs=$2\n\n # Skip averaging for built-in mdtest (scenario3) and fio tests\n if [[ \"$scenario\" == \"scenario3\" || \"$scenario\" =~ ^fio ]]; then\n return\n fi\n\n declare -A ops max_sum min_sum mean_sum stddev_sum count\n declare -a op_order # To maintain operation order\n\n for ((i=1; i<=runs; i++)); do\n while IFS=, read -r op max min mean stddev; do\n # Skip empty lines\n [ -z \"$op\" ] && continue\n\n # Add to op_order if not already present\n if [[ -z \"${ops[$op]}\" ]]; then\n ops[\"$op\"]=1\n op_order+=(\"$op\")\n fi\n\n max=$(echo \"$max\" | tr -d ',')\n min=$(echo \"$min\" | tr -d ',')\n mean=$(echo \"$mean\" | tr -d ',')\n stddev=$(echo \"$stddev\" | tr -d ',')\n\n max_sum[\"$op\"]=$(echo \"${max_sum[$op]:-0} + $max\" | bc -l)\n min_sum[\"$op\"]=$(echo \"${min_sum[$op]:-0} + $min\" | bc -l)\n mean_sum[\"$op\"]=$(echo \"${mean_sum[$op]:-0} + $mean\" | bc -l)\n stddev_sum[\"$op\"]=$(echo \"${stddev_sum[$op]:-0} + $stddev\" | bc -l)\n count[\"$op\"]=$(( ${count[$op]:-0} + 1 ))\n done < \"${RESULTS_FILE}.${scenario}.run${i}.csv\"\n done\n\n > \"${RESULTS_FILE}.${scenario}.summary\" # Clear the file\n for op in \"${op_order[@]}\"; do\n cnt=${count[$op]:-1} # Avoid division by zero\n avg_max=$(echo \"scale=2; ${max_sum[$op]:-0} / $cnt\" | bc -l)\n avg_min=$(echo \"scale=2; ${min_sum[$op]:-0} / $cnt\" | bc -l)\n avg_mean=$(echo \"scale=2; ${mean_sum[$op]:-0} / $cnt\" | bc -l)\n avg_stddev=$(echo \"scale=2; ${stddev_sum[$op]:-0} / $cnt\" | bc -l)\n\n printf \"%-25s : %12.2f %12.2f %12.2f %12.2f\\n\" \\\n \"$op\" \"$avg_max\" \"$avg_min\" \"$avg_mean\" \"$avg_stddev\" \\\n >> \"${RESULTS_FILE}.${scenario}.summary\"\n done\n}\n\n# Scenario 1: -b 3 -z 1 -I 1000\nfor i in {1..3}; do\n echo \"Running scenario 1 (attempt $i)...\"\n output_file=\"${RESULTS_FILE}.scenario1.run${i}\"\n echo 3 | sudo tee /proc/sys/vm/drop_caches\n mpirun --use-hwthread-cpus --allow-run-as-root -np 4 mdtest -b 3 -z 1 -I 300 -d \"$MNT_POINT/mdtest\" | tee \"$output_file\"\n process_run \"$output_file\" \"scenario1\" $i\n rm -rf \"$MNT_POINT/mdtest\"/*\ndone\n\n# Scenario 2: -F -w 102400 -I 1000 -z 0\nfor i in {1..3}; do\n echo \"Running scenario 2 (attempt $i)...\"\n output_file=\"${RESULTS_FILE}.scenario2.run${i}\"\n echo 3 | sudo tee /proc/sys/vm/drop_caches\n mpirun --use-hwthread-cpus --allow-run-as-root -np 4 mdtest -F -w 102400 -I 2000 -z 0 -d \"$MNT_POINT/mdtest\" | tee \"$output_file\"\n process_run \"$output_file\" \"scenario2\" $i\n rm -rf \"$MNT_POINT/mdtest\"/*\ndone\n\n# Scenario 3: JuiceFS built-in mdtest (run only once)\necho \"Running scenario 3 (built-in mdtest)...\"\noutput_file=\"${RESULTS_FILE}.scenario3.run1\"\necho 3 | sudo tee /proc/sys/vm/drop_caches\n{ time sudo ./juicefs mdtest \"$META_URL\" /mdtest_perf --threads 10 --dirs 3 --depth 3 --files 100; } 2>&1 | tee \"$output_file\"\nprocess_run \"$output_file\" \"scenario3\" 1\n\n# Fio Scenario 4: Concurrent sequential write of 1 big file per thread (16 threads)\necho \"Running fio scenario 4...\"\noutput_file=\"${RESULTS_FILE}.fio_scenario4.run1\"\necho 3 | sudo tee /proc/sys/vm/drop_caches\nmkdir -p \"$MNT_POINT/fio\"\n\nfio --name=big-write --filename=\"${MNT_POINT}/fio/fio_test_$(date +%Y%m%d_%H%M%S).dat\" --group_reporting \\\n --rw=write --direct=1 --bs=64k --end_fsync=1 --runtime=200 \\\n --numjobs=8 --nrfiles=1 --size=1G --output-format=normal | tee \"$output_file\"\nprocess_run \"$output_file\" \"fio_scenario4\" 1\nrm -rf \"$MNT_POINT/fio\"/*\n\n# Fio Scenario 5: Concurrent sequential write of multiple big files (16 threads, 64 files each)\necho \"Running fio scenario 5...\"\noutput_file=\"${RESULTS_FILE}.fio_scenario5.run1\"\necho 3 | sudo tee /proc/sys/vm/drop_caches\nmkdir -p \"$MNT_POINT/fio\"\nfio --name=big-write --filename=\"${MNT_POINT}/fio/fio_test_$(date +%Y%m%d_%H%M%S).dat\" --group_reporting \\\n --rw=randwrite --direct=1 --bs=64k --end_fsync=1 --runtime=200 \\\n --numjobs=8 --nrfiles=1 --size=1G --output-format=normal | tee \"$output_file\"\nprocess_run \"$output_file\" \"fio_scenario5\" 1\nrm -rf \"$MNT_POINT/fio\"/*\n\n# Fio Scenario 6: Sequential read of multiple big files (single thread)\necho \"Running fio scenario 6...\"\noutput_file=\"${RESULTS_FILE}.fio_scenario6.run1\"\necho 3 | sudo tee /proc/sys/vm/drop_caches\nmkdir -p \"$MNT_POINT/fio\"\nfio --name=big-read-multiple --filename=\"${MNT_POINT}/fio/fio_test_$(date +%Y%m%d_%H%M%S).dat\" --group_reporting --runtime=300 \\\n --rw=read --direct=1 --bs=4k --numjobs=8 --nrfiles=1 --size=1G --output-format=normal | tee \"$output_file\"\nprocess_run \"$output_file\" \"fio_scenario6\" 1\nrm -rf \"$MNT_POINT/fio\"/*\n\n# Fio Scenario 7: Concurrent sequential read of multiple big files (64 threads)\necho \"Running fio scenario 7...\"\noutput_file=\"${RESULTS_FILE}.fio_scenario7.run1\"\necho 3 | sudo tee /proc/sys/vm/drop_caches\nmkdir -p \"$MNT_POINT/fio\"\nfio --name=big-read-multiple-concurrent --filename=\"${MNT_POINT}/fio/fio_test_$(date +%Y%m%d_%H%M%S).dat\" --group_reporting \\\n --rw=randread --direct=1 --bs=4k --numjobs=8 --nrfiles=1 --openfiles=1 --size=1G --output-format=normal --runtime=120 | tee \"$output_file\"\nprocess_run \"$output_file\" \"fio_scenario7\" 1\nrm -rf \"$MNT_POINT/fio\"/*\n\n# Fio Scenario 8: Concurrent sequential write of multiple big files (8 threads, 8 files each)\necho \"Running fio scenario 8...\"\noutput_file=\"${RESULTS_FILE}.fio_scenario8.run1\"\necho 3 | sudo tee /proc/sys/vm/drop_caches\nmkdir -p \"$MNT_POINT/fio\"\nfio --name=big-write --directory=\"$MNT_POINT/fio\" --group_reporting \\\n --rw=write --direct=1 --bs=1m --end_fsync=1 --runtime=120 \\\n --numjobs=8 --nrfiles=8 --size=1G --output-format=normal | tee \"$output_file\"\nprocess_run \"$output_file\" \"fio_scenario8\" 1\nrm -rf \"$MNT_POINT/fio\"/*\n\n\n# Fio Scenario 9: Concurrent sequential read of multiple big files (8 threads)\necho \"Running fio scenario 9...\"\noutput_file=\"${RESULTS_FILE}.fio_scenario9.run1\"\necho 3 | sudo tee /proc/sys/vm/drop_caches\nmkdir -p \"$MNT_POINT/fio\"\nfio --name=big-read-multiple-concurrent --directory=\"$MNT_POINT/fio\" --group_reporting \\\n --rw=read --direct=1 --bs=1m --numjobs=8 --nrfiles=8 --openfiles=1 --size=1G --output-format=normal --runtime=120 | tee \"$output_file\"\nprocess_run \"$output_file\" \"fio_scenario9\" 1\nrm -rf \"$MNT_POINT/fio\"/*\n\n\n\n# Calculate averages for scenario1 and scenario2\ncalculate_averages \"scenario1\" 3\ncalculate_averages \"scenario2\" 3\n\n# For scenario3 and fio scenarios, just rename the single run file to .summary\nmv \"${RESULTS_FILE}.scenario3.run1.summary\" \"${RESULTS_FILE}.scenario3.summary\"\nfor scenario in fio_scenario4 fio_scenario5 fio_scenario6 fio_scenario7 fio_scenario8 fio_scenario9; do\n mv \"${RESULTS_FILE}.${scenario}.run1.summary\" \"${RESULTS_FILE}.${scenario}.summary\"\ndone\n\nrm -f \"${RESULTS_FILE}\"*.run*.csv \"${RESULTS_FILE}\"*.run[1-3]\n\n# Print summary results\necho \"\"\necho \"Summary Results for $VERSION:\"\nfor scenario in scenario1 scenario2; do\n echo \"\"\n echo \"$scenario Results:\"\n printf \"%-25s %-12s %-12s %-12s %-12s\\n\" \"Operation\" \"Max\" \"Min\" \"Mean\" \"Std Dev\"\n cat \"${RESULTS_FILE}.${scenario}.summary\"\ndone\n\n# Print built-in mdtest results\necho \"\"\necho \"Scenario3 (Built-in mdtest) Results:\"\ncat \"${RESULTS_FILE}.scenario3.summary\"\n\n# Print fio results\nfor scenario in fio_scenario4 fio_scenario5 fio_scenario6 fio_scenario7 fio_scenario8 fio_scenario9; do\n echo \"\"\n echo \"${scenario} Results:\"\n cat \"${RESULTS_FILE}.${scenario}.summary\"\ndone\n" }, { "path": ".github/scripts/prepare_db.sh", "content": "#!/bin/bash -e\nsource .github/scripts/start_meta_engine.sh\n[ -z \"$TEST\" ] && echo \"TEST is not set\" && exit 1\n\n# check port is ready until 60s, sleep 1s for each query\ncheck_port() {\n port=$1\n echo \"check for port:\" $port\n for i in {1..30}; do\n sudo lsof -i :$port && echo \"port is available: $port after $i sec\" && return 0 ||\n (echo \"port is not available after $i\" && sleep 1)\n done\n echo \"service not ready on: $port\" && exit 1\n}\n\ninstall_mysql() {\n sudo service mysql start\n sudo mysql -uroot -proot -e \"use mysql;alter user 'root'@'localhost' identified with mysql_native_password by '';\"\n sudo mysql -e \"create database dev;\"\n sudo mysql -e \"create database dev2;\"\n check_port 3306\n}\n\ninstall_postgres() {\n sudo service postgresql start\n sudo chmod 777 /etc/postgresql/*/main/pg_hba.conf\n sudo sed -i \"s?local.*all.*postgres.*peer?local all postgres trust?\" /etc/postgresql/*/main/pg_hba.conf\n sudo sed -i \"s?host.*all.*all.*32.*scram-sha-256?host all all 127.0.0.1/32 trust?\" /etc/postgresql/*/main/pg_hba.conf\n sudo sed -i \"s?host.*all.*all.*128.*scram-sha-256?host all all ::1/128 trust?\" /etc/postgresql/*/main/pg_hba.conf\n cat /etc/postgresql/*/main/pg_hba.conf\n sudo service postgresql restart\n psql -c \"create user runner superuser;\" -U postgres\n sudo service postgresql restart\n psql -c 'create database test;' -U postgres\n}\n\ninstall_etcd() {\n docker run -d \\\n -p 3379:2379 \\\n -p 3380:2380 \\\n --name etcd_3_5_7 \\\n quay.io/coreos/etcd:v3.5.7 \\\n /usr/local/bin/etcd --data-dir=/etcd-data --name node1 \\\n --listen-client-urls http://0.0.0.0:2379 \\\n --advertise-client-urls http://0.0.0.0:3379 \\\n --listen-peer-urls http://0.0.0.0:2380 \\\n --initial-advertise-peer-urls http://0.0.0.0:2380 \\\n --initial-cluster node1=http://0.0.0.0:2380\n check_port 3379\n check_port 3380\n}\n\ninstall_keydb() {\n echo \"deb https://download.keydb.dev/open-source-dist $(lsb_release -sc) main\" | sudo tee /etc/apt/sources.list.d/keydb.list\n sudo wget -O /etc/apt/trusted.gpg.d/keydb.gpg https://download.keydb.dev/open-source-dist/keyring.gpg\n sudo .github/scripts/apt_install.sh keydb\n keydb-server --storage-provider flash /tmp/ --port 6378 --bind 127.0.0.1 --daemonize yes\n keydb-server --port 6377 --bind 127.0.0.1 --daemonize yes\n check_port 6377\n check_port 6378\n}\n\ninstall_minio() {\n docker run -d -p 9000:9000 -p 9001:9001 -e \"MINIO_ROOT_USER=testUser\" -e \"MINIO_ROOT_PASSWORD=testUserPassword\" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address \":9001\"\n go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc alias set local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket\n}\n\ninstall_fdb() {\n wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb\n wget -O /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-server_6.3.23-1_amd64.deb\n sudo dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb\n check_port 4500\n}\n\ninstall_gluster() {\n sudo systemctl start glusterd.service\n mkdir -p /tmp/gluster/gv0\n sudo hostname jfstest\n sudo gluster volume create gv0 jfstest:/tmp/gluster/gv0 force\n sudo gluster volume start gv0\n sudo gluster volume info gv0\n}\n\ninstall_litmus() {\n wget -O /home/travis/.m2/litmus-0.13.tar.gz http://www.webdav.org/neon/litmus/litmus-0.13.tar.gz\n tar -zxvf /home/travis/.m2/litmus-0.13.tar.gz -C /home/travis/.m2/\n cd /home/travis/.m2/litmus-0.13/ && ./configure && make && cd -\n}\n\ninstall_webdav() {\n wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip\n unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2/\n nohup /home/travis/.m2/rclone-v1.57.0-linux-amd64/rclone serve webdav local --addr 127.0.0.1:9007 >>rclone.log 2>&1 &\n}\n\nprepare_db() {\n case \"$TEST\" in\n \"test.meta.core\")\n retry install_tikv\n install_mysql\n ;;\n \"test.meta.non-core\")\n install_postgres\n install_etcd\n install_keydb\n ;;\n \"test.cmd\")\n install_minio\n install_litmus\n ;;\n \"test.fdb\")\n install_fdb\n ;;\n \"test.pkg\")\n install_mysql\n retry install_tikv\n install_minio\n install_gluster\n install_webdav\n docker run -d --name sftp -p 2222:22 juicedata/ci-sftp\n docker run -d --name samba -p 4445:445 -e \"USER=samba\" -e \"PASS=secret\" dockurr/samba\n install_etcd\n .github/scripts/setup-hdfs.sh\n ;;\n *)\n echo \"Test: $TEST is not valid\" && exit 1\n ;;\n esac\n}\n\nprepare_db\n" }, { "path": ".github/scripts/pysdk/bench.py", "content": "import os\nimport random\nimport sys\nimport time\nimport argparse\nimport threading\nimport hashlib\n\nsys.path.append('.')\nfrom sdk.python.juicefs.juicefs import juicefs\n\ndef print_stats(stats, interval):\n while not stats['stop']:\n time.sleep(interval)\n elapsed_time = time.time() - stats['start_time']\n iops = stats['ops'] / elapsed_time\n print(f\"IOPS: {iops:.2f}\")\n\ndef seq_write(filename, client: juicefs.Client, protocol, block_size, buffering, run_time, file_size):\n stats = {'bytes': 0, 'ops': 0, 'start_time': time.time(), 'stop': False}\n stats_thread = threading.Thread(target=print_stats, args=(stats, 2))\n stats_thread.start()\n\n def perform_seq_writes(f):\n while time.time() - stats['start_time'] < run_time and stats['bytes'] < file_size:\n data = os.urandom(block_size) \n f.write(data)\n stats['bytes'] += block_size\n stats['ops'] += 1\n\n try:\n if protocol == 'pysdk':\n with client.open(filename, 'wb', buffering=buffering) as f:\n perform_seq_writes(f)\n else:\n with open(f'/tmp/jfs/{filename}', 'wb') as f:\n perform_seq_writes(f)\n finally:\n stats['stop'] = True\n stats_thread.join()\n\ndef random_write(filename, client: juicefs.Client, protocol, buffering, block_size, run_time, file_size, seed):\n random.seed(seed)\n stats = {'bytes': 0, 'ops': 0, 'start_time': time.time(), 'stop': False}\n stats_thread = threading.Thread(target=print_stats, args=(stats, 2))\n stats_thread.start()\n\n write_records = []\n\n def perform_random_writes(f):\n while time.time() - stats['start_time'] < run_time and stats['bytes'] < file_size:\n offset = random.randint(0, file_size - block_size)\n data = os.urandom(block_size) \n f.seek(offset)\n f.write(data)\n stats['bytes'] += block_size\n stats['ops'] += 1\n\n f.seek(offset)\n read_data = f.read(block_size)\n if hashlib.md5(read_data).hexdigest() != hashlib.md5(data).hexdigest():\n print(f\"data inconsistency: offset {offset}\")\n return False\n try:\n if protocol == 'pysdk':\n with client.open(filename, 'w+b', buffering=buffering) as f:\n perform_random_writes(f)\n else:\n with open(f'/tmp/jfs/{filename}', 'w+b') as f:\n perform_random_writes(f)\n finally:\n stats['stop'] = True\n stats_thread.join()\n\ndef seq_read(filename, client: juicefs.Client, protocol, block_size, buffering):\n stats = {'bytes': 0, 'ops': 0, 'start_time': time.time(), 'stop': False}\n stats_thread = threading.Thread(target=print_stats, args=(stats, 2))\n stats_thread.start()\n\n def perform_seq_reads(f):\n while True:\n buffer = f.read(block_size)\n if not buffer:\n break\n stats['bytes'] += len(buffer)\n stats['ops'] += 1\n\n try:\n if protocol == 'pysdk':\n with client.open(filename, 'rb', buffering=buffering) as f:\n perform_seq_reads(f)\n else:\n with open(f'/tmp/jfs/{filename}', 'rb') as f:\n perform_seq_reads(f)\n finally:\n stats['stop'] = True\n stats_thread.join()\n\ndef random_read(filename, client: juicefs.Client, protocol, buffering, block_size, seed, count):\n random.seed(seed)\n stats = {'bytes': 0, 'ops': 0, 'start_time': time.time(), 'stop': False}\n stats_thread = threading.Thread(target=print_stats, args=(stats, 2))\n stats_thread.start()\n\n def perform_random_reads(f):\n f.seek(0, 2)\n file_size = f.tell()\n for _ in range(count):\n length = random.randint(1, block_size)\n offset = random.randint(0, file_size - length)\n f.seek(offset)\n buffer = f.read(length)\n stats['bytes'] += len(buffer)\n stats['ops'] += 1\n\n try:\n if protocol == 'pysdk':\n with client.open(filename, 'rb', buffering=buffering) as f:\n perform_random_reads(f)\n else:\n with open(f'/tmp/jfs/{filename}', 'rb') as f:\n perform_random_reads(f)\n finally:\n stats['stop'] = True\n stats_thread.join()\n\ndef clean_page_cache():\n with open('/proc/sys/vm/drop_caches', 'w') as f:\n f.write('3')\n f.flush()\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser('benchmark on pysdk')\n parser.add_argument('operation', type=str, help='operation: [random_read|seq_read|random_write|seq_write]')\n parser.add_argument('filename', type=str, help='file name')\n parser.add_argument('--seed', type=int, default=0, help='seed of random read/write')\n parser.add_argument('--count', type=int, default=1000, help='count of random read')\n parser.add_argument('--buffer-size', type=int, default=300, help='buffer size')\n parser.add_argument('--block-size', type=int, default=128*1024, help='block size')\n parser.add_argument('--buffering', type=int, default=2*1024*1024, help='buffering')\n parser.add_argument('--run-time', type=int, default=10, help='run time in seconds')\n parser.add_argument('--file-size', type=int, default=1024*1024*1024, help='file size in bytes')\n parser.add_argument('-p', '--protocol', type=str, default='pysdk', help='protocol: [fuse|pysdk]')\n args = parser.parse_args()\n\n if args.protocol == 'pysdk':\n meta_url=os.environ.get('META_URL', 'redis://localhost')\n client = juicefs.Client(\"test-volume\", meta=meta_url, access_log=\"/tmp/access.log\")\n else:\n client = None\n start=time.time()\n if args.operation == 'seq_read':\n seq_read(client=client, filename=args.filename, protocol=args.protocol, block_size=args.block_size, buffering=args.buffering)\n elif args.operation == 'random_read':\n random_read(client=client, filename=args.filename, protocol=args.protocol, block_size=args.block_size, buffering=args.buffering, seed=args.seed, count=args.count)\n cold_read=time.time()-start\n clean_page_cache()\n start=time.time()\n if args.operation == 'seq_read':\n seq_read(client=client, filename=args.filename, protocol=args.protocol, block_size=args.block_size, buffering=args.buffering)\n hot_read=time.time()-start\n print(f\"{cold_read:.2f} {hot_read:.2f} \")\n elif args.operation == 'random_read':\n random_read(client=client, filename=args.filename, protocol=args.protocol, block_size=args.block_size, buffering=args.buffering, seed=args.seed, count=args.count)\n hot_read=time.time()-start\n print(f\"{cold_read:.2f} {hot_read:.2f} \")\n elif args.operation == 'seq_write':\n seq_write(client=client, filename=args.filename, protocol=args.protocol, block_size=args.block_size, buffering=args.buffering, run_time=args.run_time, file_size=args.file_size)\n elif args.operation == 'random_write':\n random_write(client=client, filename=args.filename, protocol=args.protocol, buffering=args.buffering, block_size=args.block_size, run_time=args.run_time, file_size=args.file_size, seed=args.seed)\n else:\n raise ValueError(f\"Unsupported operation: {args.operation}\")" }, { "path": ".github/scripts/pysdk/pysdk_test.py", "content": "import errno\nimport fractions\nimport unittest\nimport os\nimport pwd\nfrom os.path import dirname\nimport sys\nimport time\nsys.path.append('.')\nfrom sdk.python.juicefs.juicefs import juicefs\nfrom bench import seq_write, random_write, seq_read, random_read\n\nTESTFN='/test'\nTESTFILE='/test/file'\nos.makedirs('/tmp/jfsCache0', exist_ok=True)\nmeta_url=os.environ.get('META_URL', 'redis://localhost')\n\n\nclass FileTests(unittest.TestCase):\n def setUp(self):\n self.v = juicefs.Client(\"test-volume\", meta=meta_url, access_log=\"/tmp/access.log\")\n if not self.v.exists(TESTFN):\n self.v.mkdir(TESTFN)\n\n def tearDown(self):\n self.v.rmr(TESTFN)\n\n def create_file(self, filename, content=b'content'):\n with self.v.open(filename, \"xb\", 0) as fp:\n fp.write(content)\n\n def test_read(self):\n with self.v.open(TESTFILE, \"w+b\") as fobj:\n fobj.write(b\"spam\")\n fobj.flush()\n fd = fobj.fileno()\n fobj.seek(0,0)\n s = fobj.read(4)\n self.assertEqual(type(s), bytes)\n self.assertEqual(s, b\"spam\")\n\n def test_write(self):\n fd = self.v.open(TESTFILE, 'wb')\n self.assertRaises(TypeError, os.write, fd, \"beans\")\n fd.write(b\"bacon\\n\")\n fd.close()\n with self.v.open(TESTFILE, \"rb\") as fobj:\n self.assertEqual(fobj.read().splitlines(), [b\"bacon\"])\n\n\nclass UtimeTests(FileTests):\n def setUp(self):\n super().setUp()\n self.fname = os.path.join(TESTFN, \"f1\")\n if not self.v.exists(self.fname):\n self.create_file(self.fname)\n\n def _test_utime(self, set_time, filename=None):\n if not filename:\n filename = self.fname\n atime = 1.0 # 1.0 seconds\n mtime = 4.0 # 4.0 seconds\n set_time(filename, (atime, mtime))\n st = self.v.stat(filename)\n self.assertEqual(st.st_atime, atime)\n self.assertEqual(st.st_mtime, mtime)\n\n def test_utime(self):\n def set_time(filename, times):\n self.v.utime(filename, times)\n self._test_utime(set_time)\n\n def test_utime_by_times(self):\n self.test_utime()\n\n\nclass MakedirTests(FileTests):\n def test_makedir(self):\n base = TESTFN\n path = os.path.join(base, 'dir1', 'dir2', 'dir3')\n self.v.makedirs(path) # Should work\n path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4')\n self.v.makedirs(path)\n self.assertRaises(OSError, self.v.makedirs, os.curdir)\n path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', os.curdir)\n path = os.path.join(base, 'dir1', os.curdir, 'dir2', 'dir3', 'dir4',\n 'dir5', 'dir6')\n self.v.makedirs(path)\n\n\nclass ChownFileTests(FileTests):\n def test_chown_uid_gid_arguments_must_be_index(self):\n stat = self.v.stat(TESTFN)\n uid = stat.st_uid\n gid = stat.st_gid\n for value in (-1.0, -1j, fractions.Fraction(-2, 2)):\n self.assertRaises(TypeError, self.v.chown, TESTFN, value, gid)\n self.assertRaises(TypeError, self.v.chown, TESTFN, uid, value)\n self.assertIsNone(self.v.chown(TESTFN, uid, gid))\n\n def test_chown_with_root(self):\n try:\n all_users = [u.pw_uid for u in pwd.getpwall()]\n except (AttributeError):\n all_users = []\n uid_1, uid_2 = all_users[:2]\n gid = self.v.stat(TESTFN).st_gid\n self.v.chown(TESTFN, uid_1, gid)\n uid = self.v.stat(TESTFN).st_uid\n self.assertEqual(uid, uid_1)\n self.v.chown(TESTFN, uid_2, gid)\n uid = self.v.stat(TESTFN).st_uid\n self.assertEqual(uid, uid_2)\n\n\nclass LinkTests(FileTests):\n def setUp(self):\n super().setUp()\n self.file1 = os.path.join(TESTFN, \"1\")\n self.file2 = os.path.join(TESTFN, \"2\")\n\n def are_files_same(self, file1, file2):\n stat1 = self.v.lstat(file1)\n stat2 = self.v.lstat(file2)\n return stat1.st_ino == stat2.st_ino and stat1.st_dev == stat2.st_dev\n\n def _test_link(self, file1, file2):\n self.create_file(file1)\n\n try:\n self.v.link(file1, file2)\n except PermissionError as e:\n self.skipTest('os.link(): %s' % e)\n self.assertTrue(self.are_files_same(file1, file2))\n\n def test_link(self):\n self._test_link(self.file1, self.file2)\n\n\nclass SummaryTests(FileTests):\n # /test/dir1/file\n # /dir2\n # /file\n def setUp(self):\n super().setUp()\n self.create_file(TESTFILE)\n self.v.mkdir(TESTFN + '/dir1')\n self.create_file(TESTFN + '/dir1/file')\n self.v.mkdir(TESTFN + '/dir2')\n\n def test_summary(self):\n res = self.v.summary(TESTFILE, depth=258, entries=2)\n self.assertTrue(normalize(res)==normalize({\"Path\": \"file\", \"Type\": 2, \"Files\":1, \"Dirs\":0, \"Size\":4096}))\n res = self.v.summary(TESTFN)\n self.assertTrue(normalize(res)==normalize({\"Path\": \"test\", \"Type\": 2, \"Files\":2, \"Dirs\":3, \"Size\":20480}))\n res = self.v.summary(TESTFN, depth=257, entries=1)\n self.assertTrue(normalize(res)==normalize({\"Path\": \"test\", \"Type\": 2, \"Files\":2, \"Dirs\":3, \"Size\":20480, \"Children\":[\n {\"Path\": \"dir1\", \"Type\": 2, \"Files\":1, \"Dirs\":1, \"Size\":8192},{'Path': '...', 'Type': 1, 'Size': 8192, 'Files': 1, 'Dirs': 1}]}))\n res = self.v.summary(TESTFN, depth=258, entries=1)\n self.assertTrue(normalize(res)==normalize(\n {\n \"Path\": \"test\", \"Type\": 2, \"Files\":2, \"Dirs\":3, \"Size\":20480, \"Children\":\n [\n {\"Path\": \"dir1\", \"Type\": 2, \"Files\":1, \"Dirs\":1, \"Size\":8192, \"Children\": [\n {\"Path\": \"dir1/file\", \"Type\": 1, \"Size\": 4096, \"Files\": 1, \"Dirs\": 0}\n ]\n },{'Path': '...', 'Type': 1, 'Size': 8192, 'Files': 1, 'Dirs': 1}\n ]}\n ))\n res = self.v.summary(TESTFN, depth=259, entries=4)\n self.assertTrue(normalize(res)==normalize(\n {\n \"Path\": \"test\", \"Type\": 2, \"Files\":2, \"Dirs\":3, \"Size\":20480, \"Children\":\n [\n {\n \"Path\": \"dir1\", \"Type\": 2, \"Files\":1, \"Dirs\":1, \"Size\":8192, \"Children\":\n [{\"Path\": \"dir1/file\", \"Type\": 1, \"Size\": 4096, \"Files\": 1, \"Dirs\": 0}]\n },{\n 'Path': 'file', 'Type': 1, 'Size': 4096, 'Files': 1, 'Dirs': 0\n },{\n 'Path': 'dir2', 'Type': 2, 'Size': 4096, 'Files': 0, 'Dirs': 1\n }\n ]}\n ))\n\n\nclass QuotaTests(FileTests):\n def test_quota(self):\n # /test/dir1/file\n # /dir2\n # /file\n self.create_file(TESTFILE)\n self.v.mkdir(TESTFN + '/dir1')\n self.create_file(TESTFN + '/dir1/file')\n self.v.mkdir(TESTFN + '/dir2')\n\n # set quota\n self.v.set_quota(path=TESTFN, capacity=1024*1024*1024, inodes=1000, create=True)\n res = self.v.get_quota(path=TESTFN)\n self.assertTrue(normalize(res)==normalize({\"/test\": {\"MaxSpace\": 1024*1024*1024, \"MaxInodes\": 1000, \"UsedSpace\": 0, \"UsedInodes\": 3}}))\n\n res = self.v.list_quota()\n self.assertTrue(normalize(res)==normalize({\"/test\": {\"MaxSpace\": 1024*1024*1024, \"MaxInodes\": 1000, \"UsedSpace\": 0, \"UsedInodes\": 3}}))\n\n self.v.set_quota(path=TESTFN+\"/dir1\", capacity=1024*1024*1024, inodes=10000, create=True, strict=True)\n res = self.v.list_quota()\n self.assertTrue(normalize(res)==normalize({\"/test\": {\"MaxSpace\": 1024*1024*1024, \"MaxInodes\": 1000, \"UsedSpace\": 0, \"UsedInodes\": 3}, \"/test/dir1\": {\"MaxSpace\": 1024*1024*1024, \"MaxInodes\": 10000, \"UsedSpace\": 4096, \"UsedInodes\": 1}}))\n\n # check quota\n self.v.check_quota(path=TESTFN, strict=True, repair=True)\n\n # unset quota\n self.v.del_quota(path=TESTFN)\n res = self.v.get_quota(path=TESTFN)\n self.assertTrue(res=={})\n\n\ndef normalize(d):\n if isinstance(d, dict):\n if \"Children\" in d:\n d[\"Children\"].sort(key=lambda x: x[\"Path\"])\n return {k: normalize(v) for k, v in d.items()}\n elif isinstance(d, list):\n return sorted((normalize(x) for x in d), key=lambda x: x.get(\"Path\", \"\"))\n else:\n return d\n\n\nclass NonLocalSymlinkTests(FileTests):\n def test_directory_link_nonlocal(self):\n src = os.path.join(TESTFN, 'some_link')\n self.v.symlink('/some_dir', src)\n assert self.v.readlink(src) == '../some_dir'\n\n\nclass ExtendedAttributeTests(FileTests):\n def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwargs):\n fn = TESTFN + '_xattr'\n if self.v.exists(fn):\n self.v.unlink(fn)\n self.create_file(fn)\n\n # with self.assertRaises(OSError) as cm:\n # self.v.getxattr(fn, s(\"user.test\"), **kwargs)\n # self.assertEqual(cm.exception.errno, errno.ENODATA)\n\n init_xattr = self.v.listxattr(fn)\n self.assertIsInstance(init_xattr, list)\n\n self.v.setxattr(fn, s(\"user.test\"), b\"a\", **kwargs)\n xattr = set(init_xattr)\n xattr.add(\"user.test\")\n self.assertEqual(set(self.v.listxattr(fn)), xattr)\n self.assertEqual(self.v.getxattr(fn, b\"user.test\", **kwargs), b\"a\")\n self.v.setxattr(fn, s(\"user.test\"), b\"hello\", os.XATTR_REPLACE, **kwargs)\n self.assertEqual(self.v.getxattr(fn, b\"user.test\", **kwargs), b\"hello\")\n\n with self.assertRaises(OSError) as cm:\n self.v.setxattr(fn, s(\"user.test\"), b\"bye\", os.XATTR_CREATE, **kwargs)\n self.assertEqual(cm.exception.errno, errno.EEXIST)\n\n # with self.assertRaises(OSError) as cm:\n # self.v.setxattr(fn, s(\"user.test2\"), b\"bye\", os.XATTR_REPLACE, **kwargs)\n # self.assertEqual(cm.exception.errno, errno.ENODATA)\n\n self.v.setxattr(fn, s(\"user.test2\"), b\"foo\", os.XATTR_CREATE, **kwargs)\n xattr.add(\"user.test2\")\n self.assertEqual(set(self.v.listxattr(fn)), xattr)\n self.v.removexattr(fn, s(\"user.test\"), **kwargs)\n\n with self.assertRaises(OSError) as cm:\n self.v.getxattr(fn, s(\"user.test\"), **kwargs)\n self.assertEqual(cm.exception.errno, errno.ENODATA)\n\n xattr.remove(\"user.test\")\n self.assertEqual(set(self.v.listxattr(fn)), xattr)\n self.assertEqual(self.v.getxattr(fn, s(\"user.test2\"), **kwargs), b\"foo\")\n self.v.setxattr(fn, s(\"user.test\"), b\"a\"*1024, **kwargs)\n self.assertEqual(self.v.getxattr(fn, s(\"user.test\"), **kwargs), b\"a\"*1024)\n self.v.removexattr(fn, s(\"user.test\"), **kwargs)\n many = sorted(\"user.test{}\".format(i) for i in range(100))\n for thing in many:\n self.v.setxattr(fn, thing, b\"x\", **kwargs)\n self.assertEqual(set(self.v.listxattr(fn)), set(init_xattr) | set(many))\n\n def _check_xattrs(self, *args, **kwargs):\n self._check_xattrs_str(str, *args, **kwargs)\n self.v.unlink(TESTFN + '_xattr')\n\n self._check_xattrs_str(os.fsencode, *args, **kwargs)\n self.v.unlink(TESTFN + '_xattr')\n\n def test_simple(self):\n self._check_xattrs(self.v.getxattr, self.v.setxattr, self.v.removexattr,\n self.v.listxattr)\n\n def test_fds(self):\n def getxattr(path, *args):\n with self.v.open(path, \"rb\") as fp:\n return self.v.getxattr(fp.fileno(), *args)\n def setxattr(path, *args):\n with self.v.open(path, \"wb\", 0) as fp:\n self.v.setxattr(fp.fileno(), *args)\n def removexattr(path, *args):\n with self.v.open(path, \"wb\", 0) as fp:\n self.v.removexattr(fp.fileno(), *args)\n def listxattr(path, *args):\n with self.v.open(path, \"rb\") as fp:\n return self.v.listxattr(fp.fileno(), *args)\n self._check_xattrs(getxattr, setxattr, removexattr, listxattr)\n\n\nclass BenchTests(FileTests):\n test_file = TESTFILE + '_bench'\n block_size = 128 * 1024 # 128KB\n buffer_size = 300\n buffering = 2 * 1024 * 1024\n run_time = 30\n file_size = 100 * 1024 * 1024\n seed = 20\n count = 200\n\n def test_seq_write(self):\n print('test_seq_write')\n seq_write(\n filename=self.test_file,\n client=self.v,\n protocol='pysdk',\n block_size=self.block_size,\n buffering=self.buffering,\n run_time=self.run_time,\n file_size=self.file_size\n )\n self.assertTrue(self.v.exists(self.test_file))\n stat = self.v.stat(self.test_file)\n self.assertGreater(stat.st_size, 0)\n\n def test_random_write(self):\n print('test_random_write')\n random_write(\n filename=self.test_file,\n client=self.v,\n protocol='pysdk',\n buffering=self.buffering,\n block_size=self.block_size,\n run_time=self.run_time,\n file_size=self.file_size,\n seed=self.seed\n )\n self.assertTrue(self.v.exists(self.test_file))\n stat = self.v.stat(self.test_file)\n self.assertGreater(stat.st_size, 0)\n\n def test_seq_read(self):\n print('test_seq_read')\n with self.v.open(self.test_file, 'wb') as f:\n f.write(os.urandom(self.file_size))\n\n seq_read(\n filename=self.test_file,\n client=self.v,\n protocol='pysdk',\n block_size=self.block_size,\n buffering=self.buffering\n )\n\n def test_random_read(self):\n print('test_random_read')\n with self.v.open(self.test_file, 'wb') as f:\n f.write(os.urandom(self.file_size))\n\n random_read(\n filename=self.test_file,\n client=self.v,\n protocol='pysdk',\n buffering=self.buffering,\n block_size=self.block_size,\n seed=self.seed,\n count=self.count\n )\n\n\nclass ClientParamsTests(FileTests):\n testfile = TESTFN + '/testfile'\n\n def test_readonly_param(self):\n v = juicefs.Client(\n \"test-volume-ro\",\n meta=meta_url,\n read_only=True\n )\n with self.assertRaises(OSError):\n v.open(self.testfile, 'w')\n\n def test_cache_params(self):\n v = juicefs.Client(\n \"test-volume-cache\",\n meta=meta_url,\n cache_dir=\"/tmp/jfs_test_cache\",\n cache_size=\"100M\",\n cache_partial_only=False\n )\n\n size_mb = 48\n test_data = os.urandom(size_mb * 1024 * 1024)\n with v.open(self.testfile, 'wb') as f:\n f.write(test_data)\n\n with v.open(self.testfile, 'rb') as f:\n read_data = f.read()\n self.assertEqual(read_data, test_data)\n\n cache_dir = \"/tmp/jfs_test_cache\"\n cache_size = 0\n for root, dirs, files in os.walk(cache_dir):\n for file in files:\n cache_size += os.path.getsize(os.path.join(root, file))\n self.assertGreaterEqual(cache_size, size_mb * 1024 * 1024/2)\n\n def test_io_limits(self):\n v = juicefs.Client(\n \"test-volume-limited\",\n meta=meta_url,\n upload_limit=\"1M\",\n download_limit=\"1M\"\n )\n\n test_data = b\"x\" * (10 * 1024 * 1024) # 10MB\n start_time = time.time()\n with v.open(self.testfile, 'wb') as f:\n f.write(test_data)\n write_time = time.time() - start_time\n\n self.assertGreaterEqual(write_time, 10.0)\n\n\nclass CloneTests(FileTests):\n def setUp(self):\n super().setUp()\n self.source = TESTFN + '/source'\n self.target = TESTFN + '/target'\n self.test_data = b\"Hello JuiceFS!\" * 1024\n\n with self.v.open(self.source, 'wb') as f:\n f.write(self.test_data)\n\n def test_basic_clone(self):\n self.v.clone(self.source, self.target)\n\n self.assertTrue(self.v.exists(self.target))\n\n with self.v.open(self.target, 'rb') as f:\n cloned_data = f.read()\n self.assertEqual(cloned_data, self.test_data)\n\n source_stat = self.v.stat(self.source)\n target_stat = self.v.stat(self.target)\n self.assertEqual(source_stat.st_size, target_stat.st_size)\n\n def test_clone_with_preserve(self):\n self.v.chmod(self.source, 0o644)\n\n self.v.clone(self.source, self.target, preserve=True)\n source_stat = self.v.stat(self.source)\n target_stat = self.v.stat(self.target)\n self.assertEqual(source_stat.st_mode, target_stat.st_mode)\n\n\nclass WarmupTests(unittest.TestCase):\n @classmethod\n def setUpClass(self):\n self.v = juicefs.Client(\n \"test-warmup\",\n meta=meta_url,\n cache_dir=\"/tmp/jfs_test_warmup\",\n cache_size=\"1000M\",\n cache_partial_only=True\n )\n if self.v.exists(TESTFN):\n self.v.rmr(TESTFN)\n self.v.mkdir(TESTFN)\n self.test_files = [\n TESTFN + '/file1',\n TESTFN + '/file2'\n ]\n size_mb = 50\n test_data = os.urandom(size_mb * 1024 * 1024)\n for file in self.test_files:\n with self.v.open(file, 'wb') as f:\n f.write(test_data)\n\n @classmethod\n def tearDownClass(self):\n if self.v.exists(TESTFN):\n self.v.warmup(self.test_files, isEvict=True)\n self.v.rmr(TESTFN)\n\n def test_basic_warmup(self):\n result = self.v.warmup(self.test_files, numthreads=4)\n self.assertIn('FileCount', result)\n self.assertEqual(result['FileCount'], 2)\n self.assertIn('SliceCount', result)\n self.assertIn('TotalBytes', result)\n self.assertIn('MissBytes', result)\n # self.assertIn('Locations', result)\n cache_dir = \"/tmp/jfs_test_warmup\"\n size_mb = 100\n cache_size = 0\n time.sleep(2)\n for root, dirs, files in os.walk(cache_dir):\n for file in files:\n cache_size += os.path.getsize(os.path.join(root, file))\n self.assertGreaterEqual(cache_size, size_mb * 1024 * 1024)\n\n def test_warmup_check(self):\n self.v.warmup(self.test_files)\n result = self.v.warmup(self.test_files, isCheck=True)\n self.assertEqual(result['MissBytes'], 0)\n self.assertTrue(any('jfs_test_warmup' in path for path in result['Locations']),\n msg=f\"'jfs_test_warmup' not found in {result['Locations']}\")\n\n def test_warmup_evict(self):\n self.v.warmup(self.test_files)\n result = self.v.warmup(self.test_files, isEvict=True)\n time.sleep(2)\n cache_dir = \"/tmp/jfs_test_warmup\"\n size_mb = 1\n cache_size = 0\n for root, dirs, files in os.walk(cache_dir):\n for file in files:\n cache_size += os.path.getsize(os.path.join(root, file))\n self.assertLessEqual(cache_size, size_mb * 1024 * 1024)\n result = self.v.warmup(self.test_files, isCheck=True)\n self.assertEqual(result['MissBytes'], result['TotalBytes'])\n\n\nclass InfoTests(FileTests):\n def test_file_info(self):\n self.test_dir = TESTFN + '/infotest'\n self.test_file = self.test_dir + '/testfile'\n self.v.makedirs(self.test_dir)\n with self.v.open(self.test_file, 'w') as f:\n f.write(\"test content\")\n\n info = self.v.info(self.test_dir,recursive=True,strict=True)\n self.assertIn('Length', info)\n self.assertEqual(info['Files'], 1)\n self.assertEqual(info['Dirs'], 1)\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": ".github/scripts/random_read_write.py", "content": "import random\nimport os\n\ndef random_write(path1, path2, count=1000):\n if not os.path.exists(path1):\n os.system(f'touch {path1}')\n if not os.path.exists(path2):\n os.system(f'touch {path2}')\n with open(path1, 'r+b') as f1, open(path2, 'r+b') as f2:\n print(f1.seek(0, 2))\n for i in range(1, count):\n # Get the size of the file\n # size = os.path.getsize(path1)\n size = f1.seek(0, 2)\n # Generate a random position within the file that is not at the end\n pos = random.randint(0, size)\n f1.seek(pos, 0)\n f2.seek(pos, 0)\n # Generate random data\n length = random.randint(1, 1024*1024*5)\n data = os.urandom(length)\n # data = b\"abcdefg\"\n length = len(data)\n # Write data to the files\n f1.write(data)\n f2.write(data)\n f1.flush()\n f2.flush()\n assert f1.seek(0, 2) == pos+max(length, size-pos)\n assert f1.seek(0, 2) == f2.seek(0, 2)\n print(\"Wrote %d bytes at position %d\" % (length, pos))\n\ndef random_read(path1, path2):\n with open(path1, 'rb') as f1, open(path2, 'rb') as f2:\n size = f1.seek(0, 2)\n pos = random.randint(0, size)\n f1.seek(pos)\n f2.seek(pos)\n len = random.randint(1, 1024*1024)\n assert f1.read(len) == f2.read(len)\n print(\"Read %d bytes at position %d\" % (len, pos))\n\ndef read_all(path1, path2):\n with open(path1, 'rb') as f1, open(path2, 'rb') as f2:\n assert f1.read() == f2.read()\n print(\"Read all bytes\")\n \nif __name__ == '__main__':\n path1 = os.environ.get('PATH1', '/tmp/test1')\n path2 = os.environ.get('PATH2', '/tmp/test2')\n print(f'path1: {path1}, path2: {path2}')\n if os.path.exists(path1):\n os.remove(path1)\n if os.path.exists(path2):\n os.remove(path2)\n for i in range(10):\n random_write(path1, path2, count=100)\n\n for i in range(1000):\n random_read(path1, path2)\n\n read_all(path1, path2)" }, { "path": ".github/scripts/save_benchmark.sh", "content": "#/bin/bash -e\n\nmount_jfs(){\n mkdir -p /root/.juicefs\n wget -q s.juicefs.com/static/Linux/mount -O /root/.juicefs/jfsmount \n chmod +x /root/.juicefs/jfsmount\n curl -s -L https://juicefs.com/static/juicefs -o /usr/local/bin/juicefs && sudo chmod +x /usr/local/bin/juicefs\n juicefs auth ci-coverage --access-key $AWS_ACEESS_KEY --secret-key $AWS_SECRET_KEY --token $AWS_ACCESS_TOKEN --encrypt-keys\n juicefs mount ci-coverage --subdir juicefs/ci-benchmark/ --allow-other /ci-benchmark\n} \n\nsave_benchmark(){\n while [[ $# -gt 0 ]]; do\n key=\"$1\"\n case $key in\n --name)\n name=\"$2\"\n shift\n ;;\n --result)\n result=\"$2\"\n shift\n ;;\n --meta)\n meta=\"$2\"\n shift\n ;;\n --storage)\n storage=\"$2\"\n shift\n ;;\n --extra)\n extra=\"$2\"\n shift\n ;;\n *)\n # Unknown option\n ;;\n esac\n shift\n done\n [[ -z $name ]] && echo \"name is required\" && exit 1\n [[ -z $result ]] && echo \"result is required\" && exit 1\n [[ -z $meta ]] && echo \"meta is required\" && exit 1\n [[ -z $storage ]] && storage='unknown'\n\n version=$(./juicefs -V | cut -b 17- | sed 's/:/-/g')\n created_date=$(date +\"%Y-%m-%d\")\n cat < result.json\n {\n \"workflow\": \"$GITHUB_WORKFLOW\",\n \"name\": \"$name\",\n \"result\": \"$result\",\n \"meta\": \"$meta\",\n \"storage\": \"$storage\",\n \"extra\": \"$extra\",\n \"version\": \"$version\",\n \"created_date\": \"$created_date\",\n \"github_repo\": \"$GITHUB_REPOSITORY\",\n \"github_ref_name\": \"$GITHUB_REF_NAME\",\n \"github_run_id\": \"$GITHUB_RUN_ID\",\n \"github_sha\": \"$GITHUB_SHA\",\n \"workflow_url\": \"https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID\",\n }\nEOF\n cat result.json\n if [[ \"$GITHUB_EVENT_NAME\" == \"schedule\" || \"$GITHUB_EVENT_NAME\" == \"workflow_dispatch\" ]]; then\n mount_jfs\n echo \"save result.json to /ci-benchmark/$GITHUB_WORKFLOW/$name/$created_date/$meta-$storage.json\"\n mkdir -p /ci-benchmark/$GITHUB_WORKFLOW/$name/$created_date/\n cp result.json /ci-benchmark/$GITHUB_WORKFLOW/$name/$created_date/$meta-$storage.json\n fi\n}\n\nsave_benchmark $@\n" }, { "path": ".github/scripts/setup-hdfs.sh", "content": "#!/bin/bash\n\n# JuiceFS, Copyright 2021 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nset -e\nsudo apt-get update\nsudo apt-get install openjdk-8-jdk -y\n\nHADOOP_VERSION=\"2.10.2\"\nwget -q https://dlcdn.apache.org/hadoop/common/hadoop-2.10.2/hadoop-2.10.2.tar.gz\nmkdir ~/app\ntar -zxf hadoop-${HADOOP_VERSION}.tar.gz -C ~/app\n\nsudo tee -a ~/.bashrc <> ~/.ssh/authorized_keys\nchmod 700 ~/.ssh\nchmod 600 ~/.ssh/authorized_keys\necho \"StrictHostKeyChecking no\" >> ~/.ssh/config\n\nsed -i 's/${JAVA_HOME}/\\/usr\\/lib\\/jvm\\/java-8-openjdk-amd64/g' ~/app/hadoop-${HADOOP_VERSION}/etc/hadoop/hadoop-env.sh\n\nsudo tee ~/app/hadoop-${HADOOP_VERSION}/etc/hadoop/core-site.xml <\n \n fs.defaultFS\n hdfs://localhost:8020\n \n\n \n hadoop.tmp.dir\n ${HOME}/apps/tmp\n \n \nEOF\n\nsudo tee ~/app/hadoop-${HADOOP_VERSION}/etc/hadoop/hdfs-site.xml <\n \n dfs.replication\n 1\n \n \nEOF\n\ncd ~/app/hadoop-${HADOOP_VERSION}/bin\n./hdfs namenode -format\ncd ~/app/hadoop-${HADOOP_VERSION}/sbin\n./start-dfs.sh\n\nfor i in {1..3} ; do\n ProcNumber=$( jps |grep -w DataNode|wc -l)\n if [ ${ProcNumber} -lt 1 ];then\n echo \"current java process:\"\n jps\n echo \"The DataNode is not running, Retry for the $i time...\"\n ./start-dfs.sh\n fi\ndone\n\necho \"hello world\" > /tmp/testfile\ncd ~/app/hadoop-${HADOOP_VERSION}/bin\n./hdfs dfs -put /tmp/testfile /\n./hdfs dfs -rm /testfile\n./hdfs dfs -chmod 777 /\n\necho \"hdfs started successfully\"\n" }, { "path": ".github/scripts/ssh/Dockerfile", "content": "FROM ubuntu:latest\nRUN apt update && apt install openssh-server sudo -y\nRUN groupadd juicedata && useradd -ms /bin/bash -g juicedata juicedata -u 1024\nRUN mkdir /var/jfs\nRUN mkdir -p /home/juicedata/.ssh\nCOPY id_rsa.pub /home/juicedata/.ssh/authorized_keys\nRUN chown juicedata:juicedata /home/juicedata/.ssh/authorized_keys && chmod 600 /home/juicedata/.ssh/authorized_keys\nRUN service ssh start\nEXPOSE 22\nCMD [\"/usr/sbin/sshd\",\"-D\"]" }, { "path": ".github/scripts/ssh/docker-compose.yml", "content": "version: '2'\nservices:\n worker1:\n image: juicedata/ssh\n container_name: worker1\n restart: unless-stopped\n networks:\n static-network:\n ipv4_address: 172.20.0.2\n \n worker2:\n image: juicedata/ssh\n container_name: worker2\n restart: unless-stopped\n networks:\n static-network:\n ipv4_address: 172.20.0.3\n \nnetworks:\n static-network:\n ipam:\n config:\n - subnet: 172.20.0.0/16\n" }, { "path": ".github/scripts/start_meta_engine.sh", "content": "#!/bin/bash -e\nREDIS_CSC_QUERY=\"client-cache=true&client-cache-size=500&client-cache-expire=60s&client-cache-preload=100\"\n\nretry() {\n local retries=5\n local delay=3\n for i in $(seq 1 $retries); do\n set +e\n ( set -e; \"$@\" )\n exit=$?\n set -e\n if [ $exit == 0 ]; then\n echo \"run $@ succceed\"\n return $exit\n elif [ $i == $retries ]; then\n echo \"Retry failed after $i attempts.\"\n exit $exit\n else\n echo \"Retry in $delay seconds...\"\n sleep $delay\n fi\n done\n}\n\ninstall_tikv(){\n [[ ! -d tcli ]] && git clone https://github.com/c4pt0r/tcli\n make -C tcli && sudo cp tcli/bin/tcli /usr/local/bin\n # retry because of: https://github.com/pingcap/tiup/issues/2057\n echo 'head -1' > /tmp/head.txt\n if lsof -i:2379 && pgrep pd-server && tcli -pd 127.0.0.1:2379 < /tmp/head.txt; then\n echo \"TiKV is already running and healthy\"\n return 0\n fi\n user=$(whoami)\n echo user is $user\n if [[ \"$user\" == \"root\" ]]; then\n curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sudo sh\n export PATH=/root/.tiup/bin:$PATH\n tiup=/root/.tiup/bin/tiup\n elif [[ \"$user\" == \"runner\" ]]; then\n curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh\n export PATH=/home/runner/.tiup/bin:$PATH\n tiup=/home/runner/.tiup/bin/tiup\n else\n echo \"Unknown user $user\"\n exit 1\n fi\n echo tiup is $tiup\n echo $(whoami) $(pwd)\n $tiup playground --mode tikv-slim > tikv.log 2>&1 &\n pid=$!\n timeout=60\n count=0\n while true; do\n echo 'head -1' > /tmp/head.txt\n lsof -i:2379 && pgrep pd-server && tcli -pd 127.0.0.1:2379 < /tmp/head.txt && exit_code=0 || exit_code=$?\n if [ $exit_code -eq 0 ]; then\n echo \"TiDB is running.\"\n exit 0\n fi\n sleep 1\n count=$((count+1))\n if [ $count -eq $timeout ]; then\n echo \"TiDB failed to start within $timeout seconds.\"\n kill -9 $pid || true\n exit 1\n fi\n done\n}\n\ninstall_tidb(){\n user=$(whoami)\n echo user is $user\n if [[ \"$user\" == \"root\" ]]; then\n curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sudo sh\n tiup=/root/.tiup/bin/tiup\n elif [[ \"$user\" == \"runner\" ]]; then\n curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh\n tiup=/home/runner/.tiup/bin/tiup\n else\n echo \"Unknown user $user\"\n exit 1\n fi\n echo tiup is $tiup\n \n $tiup playground 5.4.0 > tidb.log 2>&1 &\n pid=$!\n timeout=60\n count=0\n while true; do\n lsof -i:4000 && pgrep pd-server && mysql -h127.0.0.1 -P4000 -uroot -e \"select version();\" && exit_code=0 || exit_code=$?\n if [ $exit_code -eq 0 ]; then\n echo \"TiDB is running.\"\n exit 0\n fi\n sleep 1\n count=$((count+1))\n if [ $count -eq $timeout ]; then\n echo \"TiDB failed to start within $timeout seconds.\"\n kill -9 $pid || true\n exit 1\n fi\n done\n}\n\nstart_meta_engine(){\n meta=$1\n storage=$2\n if [ \"$meta\" == \"mysql\" ]; then\n sudo /etc/init.d/mysql start\n elif [ \"$meta\" == \"redis\" ]; then\n sudo .github/scripts/apt_install.sh redis-tools redis-server\n elif [ \"$meta\" == \"tikv\" ]; then\n retry install_tikv\n elif [ \"$meta\" == \"badger\" ]; then\n sudo go get github.com/dgraph-io/badger/v3\n elif [ \"$meta\" == \"mariadb\" ]; then\n if lsof -i:3306; then\n echo \"mariadb is already running\"\n else\n docker run -p 127.0.0.1:3306:3306 --name mdb -e MARIADB_ROOT_PASSWORD=root -d mariadb:latest\n sleep 10\n fi\n elif [ \"$meta\" == \"tidb\" ]; then\n retry install_tidb\n mysql -h127.0.0.1 -P4000 -uroot -e \"set global tidb_enable_noop_functions=1;\"\n elif [ \"$meta\" == \"etcd\" ]; then\n sudo .github/scripts/apt_install.sh etcd\n elif [ \"$meta\" == \"fdb\" ]; then\n if lsof -i:4500; then\n echo \"fdb is already running\"\n else \n docker run --name fdb --rm -d -p 4500:4500 foundationdb/foundationdb:6.3.23\n sleep 5\n docker exec fdb fdbcli --exec \"configure new single memory\"\n echo \"docker:docker@127.0.0.1:4500\" > /home/runner/fdb.cluster\n fdbcli -C /home/runner/fdb.cluster --exec \"status\"\n fi\n elif [ \"$meta\" == \"ob\" ]; then\n docker rm obstandalone --force || echo \"remove obstandalone failed\"\n docker run -p 2881:2881 --name obstandalone -e MINI_MODE=1 -d oceanbase/oceanbase-ce\n sleep 60\n mysql -h127.0.0.1 -P2881 -uroot -e \"ALTER SYSTEM SET _ob_enable_prepared_statement=TRUE;\"\n elif [ \"$meta\" == \"postgres\" ]; then\n echo \"start postgres\"\n lsof -i:5432 || true\n if lsof -i:5432; then\n echo \"postgres is already running\"\n else\n # default max_connections is 100.\n docker run --name postgresql \\\n -e POSTGRES_USER=postgres \\\n -e POSTGRES_PASSWORD=postgres \\\n -p 5432:5432 \\\n -v /tmp/postgresql:/var/lib/postgresql \\\n -d postgres \\\n -N 300\n sleep 10\n docker exec -i postgresql psql -U postgres -c \"SHOW max_connections;\"\n fi\n fi\n \n if [ \"$storage\" == \"minio\" ]; then\n if ! docker ps | grep \"minio/minio\"; then\n docker run -d -p 9000:9000 --name minio \\\n -e \"MINIO_ACCESS_KEY=minioadmin\" \\\n -e \"MINIO_SECRET_KEY=minioadmin\" \\\n -v /tmp/data:/data \\\n -v /tmp/config:/root/.minio \\\n minio/minio server /data\n sleep 3s\n fi\n [ ! -x mc ] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x mc\n ./mc alias set myminio http://localhost:9000 minioadmin minioadmin || ./mc alias set myminio http://127.0.0.1:9000 minioadmin minioadmin\n elif [ \"$storage\" == \"gluster\" ]; then\n dpkg -s glusterfs-server || .github/scripts/apt_install.sh glusterfs-server\n systemctl start glusterd.service\n elif [ \"$meta\" != \"postgres\" ] && [ \"$storage\" == \"postgres\" ]; then\n echo \"start postgres\"\n if lsof -i:5432; then\n echo \"postgres is already running\"\n else\n docker run --name postgresql \\\n -e POSTGRES_USER=postgres \\\n -e POSTGRES_PASSWORD=postgres \\\n -p 5432:5432 \\\n -v /tmp/data:/var/lib/postgresql/data \\\n -d postgres\n sleep 10\n fi\n elif [ \"$meta\" != \"mysql\" ] && [ \"$storage\" == \"mysql\" ]; then\n echo \"start mysql\"\n sudo /etc/init.d/mysql start\n fi\n}\n\nget_meta_url(){\n meta=$1\n if [ \"$meta\" == \"postgres\" ]; then\n meta_url=\"postgres://postgres:postgres@127.0.0.1:5432/test?sslmode=disable\"\n elif [ \"$meta\" == \"mysql\" ]; then\n meta_url=\"mysql://root:root@(127.0.0.1)/test?max_open_conns=30\"\n elif [ \"$meta\" == \"redis\" ]; then\n meta_url=\"redis://127.0.0.1:6379/1?${REDIS_CSC_QUERY}\"\n elif [ \"$meta\" == \"sqlite3\" ]; then\n meta_url=\"sqlite3://test.db\"\n elif [ \"$meta\" == \"tikv\" ]; then\n meta_url=\"tikv://127.0.0.1:2379/test\"\n elif [ \"$meta\" == \"badger\" ]; then\n meta_url=\"badger:///tmp/test\"\n elif [ \"$meta\" == \"mariadb\" ]; then\n meta_url=\"mysql://root:root@(127.0.0.1)/test?max_open_conns=30\"\n elif [ \"$meta\" == \"tidb\" ]; then\n meta_url=\"mysql://root:@(127.0.0.1:4000)/test\"\n elif [ \"$meta\" == \"etcd\" ]; then\n meta_url=\"etcd://localhost:2379/test\"\n elif [ \"$meta\" == \"fdb\" ]; then\n meta_url=\"fdb:///home/runner/fdb.cluster?prefix=jfs\"\n elif [ \"$meta\" == \"ob\" ]; then\n meta_url=\"mysql://root:@\\\\(127.0.0.1:2881\\\\)/test\"\n else\n echo >&2 \": meta $meta is not supported\"\n meta_url=\"\"\n return 1\n fi\n echo $meta_url\n return 0\n}\n\nget_meta_url2(){\n meta=$1\n if [ \"$meta\" == \"postgres\" ]; then\n meta_url=\"postgres://postgres:postgres@127.0.0.1:5432/test2?sslmode=disable\"\n elif [ \"$meta\" == \"mysql\" ]; then\n meta_url=\"mysql://root:root@(127.0.0.1)/test2?max_open_conns=30\"\n elif [ \"$meta\" == \"redis\" ]; then\n meta_url=\"redis://127.0.0.1:6379/2?${REDIS_CSC_QUERY}\"\n elif [ \"$meta\" == \"sqlite3\" ]; then\n meta_url=\"sqlite3://test2.db\"\n elif [ \"$meta\" == \"tikv\" ]; then\n meta_url=\"tikv://127.0.0.1:2379/jfs2\"\n elif [ \"$meta\" == \"badger\" ]; then\n meta_url=\"badger:///tmp/test2\"\n elif [ \"$meta\" == \"mariadb\" ]; then\n meta_url=\"mysql://root:root@(127.0.0.1)/test2?max_open_conns=30\"\n elif [ \"$meta\" == \"tidb\" ]; then\n meta_url=\"mysql://root:@(127.0.0.1:4000)/test2\"\n elif [ \"$meta\" == \"etcd\" ]; then\n meta_url=\"etcd://localhost:2379/test2\"\n elif [ \"$meta\" == \"fdb\" ]; then\n meta_url=\"fdb:///home/runner/fdb.cluster?prefix=jfs2\"\n elif [ \"$meta\" == \"ob\" ]; then\n meta_url=\"mysql://root:@\\\\(127.0.0.1:2881\\\\)/test2\"\n else\n echo >&2 \": meta $meta is not supported\"\n meta_url=\"\"\n return 1\n fi\n echo $meta_url\n return 0\n}\n\ncreate_database(){\n meta_url=$1\n db_name=$(basename $meta_url | awk -F? '{print $1}')\n if [[ \"$meta_url\" == mysql* ]]; then\n user=$(echo $meta_url | awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $1}')\n password=$(echo $meta_url | awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $2}')\n test -n \"$password\" && password=\"-p$password\" || password=\"\"\n host=$(basename $(dirname $meta_url) | awk -F@ '{print $2}'| sed 's/(//g' | sed 's/)//g' | awk -F: '{print $1}')\n port=$(basename $(dirname $meta_url) | awk -F@ '{print $2}'| sed 's/(//g' | sed 's/)//g' | awk -F: '{print $2}')\n test -z \"$port\" && port=\"3306\"\n echo user=$user, password=$password, host=$host, port=$port, db_name=$db_name\n if [ \"$#\" -eq 2 ]; then\n echo isolation_level=$2\n mysql -u$user $password -h $host -P $port -e \"set global transaction isolation level $2;\"\n mysql -u$user $password -h $host -P $port -e \"show variables like '%isolation%;'\"\n fi\n mysql -u$user $password -h $host -P $port -e \"drop database if exists $db_name; create database $db_name;\"\n elif [[ \"$meta_url\" == postgres* ]]; then\n export PGPASSWORD=\"postgres\"\n printf \"\\set AUTOCOMMIT on\\ndrop database if exists $db_name; create database $db_name; \" | psql -U postgres -h localhost\n if [ \"$#\" -eq 2 ]; then\n echo isolation_level=$2\n printf \"\\set AUTOCOMMIT on\\nALTER DATABASE $db_name SET DEFAULT_TRANSACTION_ISOLATION TO '$2';\" | psql -U postgres -h localhost\n fi\n fi\n}\n" }, { "path": ".github/scripts/sync/sync.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$ENCRYPT\" ]] && ENCRYPT=false\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\nFORMAT_OPTIONS=\"\"\nif [ \"$ENCRYPT\" == \"true\" ]; then\n export JFS_RSA_PASSPHRASE=the-passwd-for-rsa\n openssl genrsa -aes256 -passout pass:$JFS_RSA_PASSPHRASE -out my-priv-key.pem 2048\n FORMAT_OPTIONS=\"--encrypt-rsa-key my-priv-key.pem\"\nfi\n\ngenerate_source_dir(){\n rm -rf jfs_source\n git clone https://github.com/juicedata/juicefs.git jfs_source --depth 1\n chmod 777 jfs_source\n mkdir jfs_source/empty_dir\n dd if=/dev/urandom of=jfs_source/file bs=5M count=1\n chmod 777 jfs_source/file\n ln -sf file jfs_source/symlink_to_file\n ln -f jfs_source/file jfs_source/hard_link_to_file\n id -u juicefs && sudo userdel juicefs\n sudo useradd -u 1101 juicefs\n sudo -u juicefs touch jfs_source/file2\n ln -s ../cmd jfs_source/pkg/symlink_to_cmd\n}\n\ngenerate_source_dir\n\ngenerate_fsrand(){\n seed=$(date +%s)\n python3 .github/scripts/fsrand.py -a -c 2000 -s $seed fsrand\n}\n\ntest_sync_with_mount_point(){\n do_sync_with_mount_point \n do_sync_with_mount_point --list-threads 10 --list-depth 5\n do_sync_with_mount_point --dirs --update --perms --check-all \n do_sync_with_mount_point --dirs --update --perms --check-all --list-threads 10 --list-depth 5\n}\n\ntest_sync_without_mount_point(){\n do_sync_without_mount_point \n do_sync_without_mount_point --list-threads 10 --list-depth 5\n do_sync_without_mount_point --dirs --update --perms --check-all \n do_sync_without_mount_point --dirs --update --perms --check-all --list-threads 10 --list-depth 5\n}\n\ndo_sync_without_mount_point(){\n prepare_test\n options=$@\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n meta_url=$META_URL ./juicefs sync jfs_source/ jfs://meta_url/jfs_source/ $options --links\n\n ./juicefs mount -d $META_URL /jfs\n if [[ ! \"$options\" =~ \"--dirs\" ]]; then\n find jfs_source -type d -empty -delete\n fi\n find /jfs/jfs_source -type f -name \".*.tmp*\" -delete\n diff -ur --no-dereference jfs_source/ /jfs/jfs_source\n}\n\ndo_sync_with_mount_point(){\n prepare_test\n options=$@\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n ./juicefs mount -d $META_URL /jfs\n ./juicefs sync jfs_source/ /jfs/jfs_source/ $options --links\n\n if [[ ! \"$options\" =~ \"--dirs\" ]]; then\n find jfs_source -type d -empty -delete\n fi\n find /jfs/jfs_source -type f -name \".*.tmp*\" -delete\n diff -ur --no-dereference jfs_source/ /jfs/jfs_source/\n}\n\ntest_sync_with_loop_link(){\n prepare_test\n options=\"--dirs --update --perms --check-all --list-threads 10 --list-depth 5\"\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n ./juicefs mount -d $META_URL /jfs\n ln -s looplink jfs_source/looplink\n ./juicefs sync jfs_source/ /jfs/jfs_source/ $options 2>&1 | tee err.log || true\n grep -i \"failed to handle 1 objects\" err.log || (echo \"grep failed\" && exit 1)\n rm -rf jfs_source/looplink\n}\n\ntest_sync_with_deep_link(){\n prepare_test\n options=\"--dirs --update --perms --check-all --list-threads 10 --list-depth 5\"\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n ./juicefs mount -d $META_URL /jfs\n touch jfs_source/symlink_1\n for i in {1..41}; do\n ln -s symlink_$i jfs_source/symlink_$((i+1))\n done\n ./juicefs sync jfs_source/ /jfs/jfs_source/ $options 2>&1 | tee err.log || true\n grep -i \"failed to handle 1 objects\" err.log || (echo \"grep failed\" && exit 1)\n rm -rf jfs_source/symlink_*\n}\n\nskip_test_sync_fsrand_with_mount_point(){\n generate_fsrand\n do_test_sync_fsrand_with_mount_point \n do_test_sync_fsrand_with_mount_point --list-threads 10 --list-depth 5\n do_test_sync_fsrand_with_mount_point --dirs --update --perms --check-all \n do_test_sync_fsrand_with_mount_point --dirs --update --perms --check-all --list-threads 10 --list-depth 5\n}\n\ndo_test_sync_fsrand_with_mount_point(){\n prepare_test\n options=$@\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n ./juicefs mount -d $META_URL /jfs\n ./juicefs sync fsrand/ /jfs/fsrand/ $options --links\n\n if [[ ! \"$options\" =~ \"--dirs\" ]]; then\n find jfs_source -type d -empty -delete\n fi\n diff -ur --no-dereference fsrand/ /jfs/fsrand/\n}\n\ntest_sync_include_exclude_option(){\n prepare_test\n ./juicefs format --trash-days 0 $FORMAT_OPTIONS $META_URL myjfs\n ./juicefs mount $META_URL /jfs -d\n ./juicefs sync jfs_source/ /jfs/\n for source_dir in \"/jfs/\" \"jfs_source/\" ; do \n while IFS=, read -r jfs_option rsync_option status; do\n printf '\\n%s, %s, %s\\n' \"$jfs_option\" \"$rsync_option\" \"$status\"\n status=$(echo $status| xargs)\n if [[ -z \"$status\" || \"$status\" = \"disable\" ]]; then \n continue\n fi\n if [ \"$source_dir\" == \"/jfs/\" ]; then \n jfs_option=\"--exclude .stats --exclude .config $jfs_option \" \n rsync_option=\"--exclude .stats --exclude .config $rsync_option \" \n fi\n rm rsync_dir/ -rf && mkdir rsync_dir\n set -o noglob\n rsync -a $source_dir rsync_dir/ $rsync_option\n rm jfs_sync_dir/ -rf && mkdir jfs_sync_dir/\n ./juicefs sync $source_dir jfs_sync_dir/ $jfs_option --list-threads 2\n set -u noglob\n printf 'juicefs sync %s %s %s\\n' \"$source_dir\" \"jfs_sync_dir/\" \"$jfs_option\" \n printf 'rsync %s %s %s\\n' \"$source_dir\" \"rsync_dir/\" \"$rsync_option\" \n printf 'diff between juicefs sync and rsync:\\n'\n diff -ur jfs_sync_dir rsync_dir\n done < .github/workflows/resources/sync-options.txt\n done\n}\n\ntest_sync_with_time(){\n prepare_test\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n ./juicefs mount $META_URL /jfs -d\n rm -rf data/\n mkdir data\n echo \"old\" > data/file1\n echo \"old\" > data/file2\n echo \"old\" > data/file3\n sleep 1\n start_time=$(date \"+%Y-%m-%d %H:%M:%S\")\n sleep 1\n echo \"new\" > data/file2\n sleep 1\n mid_time=$(date \"+%Y-%m-%d %H:%M:%S\")\n sleep 1\n echo \"new\" > data/file3\n sleep 1\n end_time=$(date \"+%Y-%m-%d %H:%M:%S\")\n mkdir -p sync_dst1 sync_dst2\n ./juicefs sync --start-time \"$start_time\" data/ /jfs/sync_dst1/\n [ \"$(cat /jfs/sync_dst1/file1 2>/dev/null)\" = \"\" ] || (echo \"file1 should not exist\" && exit 1)\n [ \"$(cat /jfs/sync_dst1/file2)\" = \"new\" ] || (echo \"file2 should be new\" && exit 1)\n [ \"$(cat /jfs/sync_dst1/file3)\" = \"new\" ] || (echo \"file3 should be new\" && exit 1)\n ./juicefs sync --start-time \"$start_time\" --end-time \"$mid_time\" data/ /jfs/sync_dst2/\n [ \"$(cat /jfs/sync_dst2/file1 2>/dev/null)\" = \"\" ] || (echo \"file1 should not exist\" && exit 1)\n [ \"$(cat /jfs/sync_dst2/file2)\" = \"new\" ] || (echo \"file2 should be new\" && exit 1)\n [ \"$(cat /jfs/sync_dst2/file3 2>/dev/null)\" = \"\" ] || (echo \"file3 should not exist\" && exit 1)\n}\n\ntest_sync_check_change()\n{\n prepare_test\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n ./juicefs mount $META_URL /jfs -d\n rm -rf data/\n mkdir data\n nohup bash -c 'for i in `seq 1 1000000`; do echo $i >> data/echo; done' > /dev/null 2>&1 &\n pid=$!\n sleep 0.5\n ./juicefs sync --check-change data/ /jfs/data/ 2>&1 | grep \"changed during sync\" || (echo \"should detect file changes during sync\" && exit 1 )\n kill $pid || true\n}\n\ntest_ignore_existing()\n{\n prepare_test\n rm -rf /tmp/src_dir /tmp/rsync_dir /tmp/jfs_sync_dir\n mkdir -p /tmp/src_dir/d1\n mkdir -p /tmp/jfs_sync_dir/d1\n echo abc > /tmp/src_dir/file1\n echo 1234 > /tmp/jfs_sync_dir/file1\n echo abcde > /tmp/src_dir/d1/d1file1\n echo 123456 > /tmp/jfs_sync_dir/d1/d1file1\n cp -rf /tmp/jfs_sync_dir/ /tmp/rsync_dir\n \n mkdir /tmp/src_dir/no-exist-dir\n echo 1111 > /tmp/src_dir/no-exist-dir/f1\n echo 123456 > /tmp/src_dir/d1/no-exist-file\n\n ./juicefs sync /tmp/src_dir /tmp/jfs_sync_dir --existing\n rsync -r /tmp/src_dir/ /tmp/rsync_dir --existing --size-only\n diff -ur /tmp/jfs_sync_dir /tmp/rsync_dir\n \n rm -rf /tmp/src_dir /tmp/rsync_dir\n mkdir -p /tmp/src_dir/d1\n mkdir -p /tmp/jfs_sync_dir/d1\n echo abc > /tmp/src_dir/file1\n echo 1234 > /tmp/jfs_sync_dir/file1\n echo abcde > /tmp/src_dir/d1/d1file1\n echo 123456 > /tmp/jfs_sync_dir/d1/d1file1\n echo abc > /tmp/src_dir/file2\n echo abcde > /tmp/src_dir/d1/d1file2\n cp -rf /tmp/jfs_sync_dir/ /tmp/rsync_dir\n \n ./juicefs sync /tmp/src_dir /tmp/jfs_sync_dir --ignore-existing \n rsync -r /tmp/src_dir/ /tmp/rsync_dir --ignore-existing --size-only\n diff -ur /tmp/jfs_sync_dir /tmp/rsync_dir\n}\ntest_file_head(){\n # issue link: https://github.com/juicedata/juicefs/issues/2125\n ./juicefs format $META_URL $FORMAT_OPTIONS myjfs\n ./juicefs mount $META_URL /jfs -d\n mkdir /jfs/jfs_source/\n [[ ! -d jfs_source ]] && git clone https://github.com/juicedata/juicefs.git jfs_source\n ./juicefs sync jfs_source/ /jfs/jfs_source/ --update --perms --check-all --bwlimit=81920 --dirs --threads=30 --list-threads=3 --debug\n echo \"test\" > jfs_source/test_file\n mkdir -p jfs_source/test_dir\n ./juicefs sync jfs_source/ /jfs/jfs_source/ --update --perms --check-all --bwlimit=81920 --dirs --threads=30 --list-threads=2 --debug\n find /jfs/jfs_source -type f -name \".*.tmp*\" -delete\n diff -ur jfs_source/ /jfs/jfs_source\n}\n\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/sync/sync_cluster.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n[[ -z \"$CI\" ]] && CI=false\n[[ -z \"$META\" ]] && META=redis\n[[ -z \"$KEY_TYPE\" ]] && KEY_TYPE=ed25519\n[[ -z \"$FILE_COUNT\" ]] && FILE_COUNT=600\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\ndpkg -s gawk || .github/scripts/apt_install.sh gawk\nstart_minio(){\n if ! docker ps | grep \"minio/minio\"; then\n docker run -d -p 9000:9000 --name minio \\\n -e \"MINIO_ACCESS_KEY=minioadmin\" \\\n -e \"MINIO_SECRET_KEY=minioadmin\" \\\n -v /tmp/data:/data \\\n -v /tmp/config:/root/.minio \\\n minio/minio server /data\n sleep 3s\n fi\n [ ! -x mc ] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x mc\n ./mc alias set myminio http://localhost:9000 minioadmin minioadmin || ./mc alias set myminio http://127.0.0.1:9000 minioadmin minioadmin\n}\nstart_minio\nstart_worker(){\n if getent group juicedata ; then groupdel -f juicedata; echo delete juicedata group; fi\n if getent passwd juicedata ; then rm -rf /home/juicedata && userdel -f juicedata; echo delete juicedata user; fi\n groupadd juicedata && useradd -ms /bin/bash -g juicedata juicedata -u 1024\n if [ \"$CI\" != \"true\" ] && [ -f ~/.ssh/id_rsa ]; then\n echo \"ssh key already exists, don't overwrite it in non ci environment\"\n else\n echo \"generating ssh key with type $KEY_TYPE\"\n yes |sudo -u juicedata ssh-keygen -t $KEY_TYPE -C \"default\" -f /home/juicedata/.ssh/id_rsa -q -N \"\"\n chmod 600 /home/juicedata/.ssh/id_rsa\n fi\n cp -f /home/juicedata/.ssh/id_rsa.pub .github/scripts/ssh/id_rsa.pub\n docker build -t juicedata/ssh -f .github/scripts/ssh/Dockerfile .github/scripts/ssh\n docker rm worker1 worker2 -f\n docker compose -f .github/scripts/ssh/docker-compose.yml up -d\n sleep 3s\n sudo -u juicedata ssh -o BatchMode=yes -o StrictHostKeyChecking=no juicedata@172.20.0.2 exit\n sudo -u juicedata ssh -o BatchMode=yes -o StrictHostKeyChecking=no juicedata@172.20.0.3 exit\n}\nstart_worker\n\nsed -i 's/bind 127.0.0.1 ::1/bind 0.0.0.0 ::1/g' /etc/redis/redis.conf\nsystemctl restart redis\nMETA_URL=$(echo $META_URL | sed 's/127\\.0\\.0\\.1/172.20.0.1/g')\n# github runner 22.04 will set /home/runner to 750, which make juicefs binary not accessed by other users.\nchmod 755 /home/runner/\n\ntest_sync_without_mount_point(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n file_count=$FILE_COUNT\n mkdir -p /jfs/data\n for i in $(seq 1 $file_count); do\n dd if=/dev/urandom of=/jfs/data/file$i bs=1M count=1 status=none\n done\n dd if=/dev/urandom of=/jfs/data/file$file_count bs=1M count=1024\n (./mc rb myminio/data1 > /dev/null 2>&1 --force || true) && ./mc mb myminio/data1\n sudo -u juicedata meta_url=$META_URL ./juicefs sync -v jfs://meta_url/data/ minio://minioadmin:minioadmin@172.20.0.1:9000/data1/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-change \\\n >sync.log 2>&1\n # diff data/ /jfs/data1/\n check_sync_log $file_count\n ./mc rm -r --force myminio/data1\n}\n\ntest_sync_without_mount_point2(){\n prepare_test\n file_count=$FILE_COUNT\n rm -rf data/\n mkdir -p data/\n for i in $(seq 1 $file_count); do\n dd if=/dev/urandom of=data/file$i bs=1M count=1 status=none\n done\n dd if=/dev/urandom of=data/file$file_count bs=1M count=1024\n (./mc rb myminio/data > /dev/null 2>&1 --force || true) && ./mc mb myminio/data\n ./mc cp -r data myminio/data\n \n # (./mc rb myminio/data1 > /dev/null 2>&1 --force || true) && ./mc mb myminio/data1\n set -o pipefail\n sudo -u juicedata meta_url=$META_URL ./juicefs sync -v minio://minioadmin:minioadmin@172.20.0.1:9000/data/ jfs://meta_url/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-change \\\n >sync.log 2>&1\n set +o pipefail\n check_sync_log $file_count\n set -o pipefail\n sudo -u juicedata meta_url=$META_URL ./juicefs sync -v minio://minioadmin:minioadmin@172.20.0.1:9000/data/ jfs://meta_url/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 --start-time 2020-01-01 \\\n --list-threads 10 --list-depth 5 --check-all \\\n >sync.log 2>&1\n set +o pipefail\n ./juicefs mount -d $META_URL /jfs\n diff data/ /jfs/data/\n current_time=$(date -d \"1 minute ago\" \"+%Y-%m-%d %H:%M:%S\")\n for i in $(seq 1 $file_count); do\n dd if=/dev/urandom of=data/file$i bs=1M count=2 status=none\n done\n dd if=/dev/urandom of=data/file$file_count bs=1M count=10\n ./mc cp -r data myminio/data\n sleep 2\n set -o pipefail\n sudo -u juicedata meta_url=$META_URL ./juicefs sync minio://minioadmin:minioadmin@172.20.0.1:9000/data/ jfs://meta_url/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 --start-time \"$current_time\" \\\n --list-threads 10 --list-depth 5 --update \\\n >sync.log 2>&1\n set +o pipefail\n diff data/ /jfs/data/\n ./mc rm -r --force myminio/data\n rm -rf data\n grep \"panic:\\|\\|ERROR\" sync.log && echo \"panic or fatal or ERROR in sync.log\" && exit 1 || true\n} \n\ntest_sync_delete_src_and_update(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n file_count=$FILE_COUNT\n rm -rf data\n mkdir -p data\n for i in $(seq 1 $file_count); do\n echo \"test-$i\" > data/test-$i\n done\n ./mc cp -r data myminio/data\n set -o pipefail\n sudo -u juicedata meta_url=$META_URL ./juicefs sync minio://minioadmin:minioadmin@172.20.0.1:9000/data/ jfs://meta_url/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --dirs --check-change \\\n >sync.log 2>&1\n set +o pipefail\n diff data/ /jfs/data/\n rm sync.log\n for i in $(seq 1 $file_count); do\n echo \"test-update-$i\" > data/test-$i\n done\n ./mc cp -r data myminio/data\n set -o pipefail\n sudo -u juicedata meta_url=$META_URL ./juicefs sync minio://minioadmin:minioadmin@172.20.0.1:9000/data/ jfs://meta_url/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --delete-src --update --dirs --check-change \\\n >sync.log 2>&1\n set +o pipefail\n diff data/ /jfs/data/\n set -o pipefail\n sudo -u juicedata meta_url=$META_URL ./juicefs sync minio://minioadmin:minioadmin@172.20.0.1:9000/data/ jfs://meta_url/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --delete-src --dirs --check-change \\\n >sync.log 2>&1\n set +o pipefail\n if ./mc ls myminio/data/ | grep -q .; then\n echo \"Error: MinIO bucket /data is not empty\"\n exit 1\n fi\n diff data/ /jfs/data/\n grep \"panic:\\|\\|ERROR\" sync.log && echo \"panic or fatal or ERROR in sync.log\" && exit 1 || true\n}\n\ntest_sync_delete_dst(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n file_count=$FILE_COUNT\n rm -rf data\n mkdir -p /jfs/data\n for i in $(seq 1 $file_count); do\n dd if=/dev/urandom of=/jfs/data/file$i bs=1M count=1 status=none\n done\n dd if=/dev/urandom of=/jfs/data/file$file_count bs=1M count=1024\n echo \"retain\" > /jfs/data/retain\n chmod -R 777 /jfs/data\n rm -rf empty && mkdir empty\n sudo -u juicedata meta_url=$META_URL ./juicefs sync --delete-dst --match-full-path --exclude='retain' --include='*' \\\n ./empty/ jfs://meta_url/data/ --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-new \\\n >sync.log 2>&1\n grep \"panic:\\|\\|ERROR\" sync.log && echo \"panic or fatal in sync.log\" && exit 1 || true\n [ ! -f /jfs/data/retain ] && echo \"Error: retain file was incorrectly deleted\" && exit 1 || true\n}\n\ntest_sync_with_random_test(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs/test || true\n mkdir /jfs/test2 || true\n current_time=$(date -d \"1 minute ago\" \"+%Y-%m-%d %H:%M:%S\")\n ./random-test runOp -baseDir /jfs/test -files 100000 -ops 1000000 -threads 50 -dirSize 100 -duration 30s -createOp 30,uniform \\\n -deleteOp 5,end --linkOp 10,uniform --symlinkOp 20,uniform --setXattrOp 10,uniform --truncateOp 10,uniform\n chmod -R 777 /jfs/test\n chmod -R 777 /jfs/test2\n sudo -u juicedata meta_url=$META_URL ./juicefs sync jfs://meta_url/test/ jfs://meta_url/test2/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-new --links --dirs --start-time \"$current_time\" \\\n >sync.log 2>&1\n grep \"panic:\\|\\|ERROR\" sync.log && echo \"panic or fatal in sync.log\" && exit 1 || true\n sudo -u juicedata meta_url=$META_URL ./juicefs sync --delete-src --match-full-path jfs://meta_url/test/ jfs://meta_url/test2/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-all --links --start-time 2199-12-30 \\\n >sync.log 2>&1\n grep \"panic:\\|\\|ERROR\" sync.log && echo \"panic or fatal in sync.log\" && exit 1 || true \n sudo -u juicedata meta_url=$META_URL ./juicefs sync --delete-src --match-full-path jfs://meta_url/test/ jfs://meta_url/test2/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 --dirs \\\n --list-threads 10 --list-depth 5 --check-all --links --start-time \"$current_time\" \\\n >sync.log 2>&1\n grep \"panic:\\|\\|ERROR\" sync.log && echo \"panic or fatal in sync.log\" && exit 1 || true\n [ -z \"$(ls -A /jfs/test)\" ] || exit 1\n rm -rf empty || mkdir empty\n sudo -u juicedata meta_url=$META_URL ./juicefs sync --delete-dst --match-full-path --include='*' \\\n ./empty/ jfs://meta_url/test2/ --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-change --dirs --links --start-time \"$current_time\" \\\n >sync.log 2>&1\n grep \"panic:\\|\" sync.log && echo \"panic or fatal in sync.log\" && exit 1 || true\n [ -z \"$(ls -A /jfs/test2)\" ] || exit 1\n}\n\ntest_sync_files_from_file(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs/test || true\n mkdir /jfs/test2 || true\n ./random-test runOp -baseDir /jfs/test -files 50000 -ops 500000 -threads 50 -dirSize 100 -duration 30s -createOp 30,uniform \\\n -deleteOp 5,end --linkOp 10,uniform --symlinkOp 20,uniform --setXattrOp 10,uniform --truncateOp 10,uniform\n chmod -R 777 /jfs/test\n chmod -R 777 /jfs/test2\n ls /jfs/test > files | tee files\n sudo -u juicedata meta_url=$META_URL ./juicefs sync jfs://meta_url/test/ jfs://meta_url/test2/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-all --check-change --links --dirs --files-from files \\\n >sync.log 2>&1\n grep \"panic\\|\\|ERROR\" sync.log && echo \"panic or fatal or error in sync.log\" && exit 1 || true\n}\n\ntest_sync_chown_perms(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n mkdir /jfs/data\n for i in $(seq 1 $FILE_COUNT); do\n mkdir /jfs/data/test$i\n dd if=/dev/urandom of=/jfs/data/test$i/file$i bs=1M count=1 status=none\n done\n sudo chown 1000:1000 /jfs/data -R\n sudo chmod -R 777 /jfs/data\n sudo -u juicedata meta_url=$META_URL ./juicefs sync jfs://meta_url/data/ jfs://meta_url/data2/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-all --links --dirs --perms \\\n >sync.log 2>&1\n grep \"panic\\|\\|ERROR\" sync.log && echo \"panic or fatal or error in sync.log\" && exit 1 || true\n diff /jfs/data/ /jfs/data2/\n}\n\nskip_test_sync_between_oss(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n mkdir -p /jfs/test\n file_count=$FILE_COUNT\n for i in $(seq 1 $file_count); do\n dd if=/dev/urandom of=/jfs/file$i bs=1M count=1 status=none\n done\n start_gateway\n sudo -u juicedata ./juicefs sync -v minio://minioadmin:minioadmin@172.20.0.1:9005/myjfs/ \\\n minio://minioadmin:minioadmin@172.20.0.1:9000/myjfs/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 \\\n > sync.log 2>&1\n count1=$(./mc ls myminio/myjfs/test -r | wc -l)\n count2=$(./mc ls juicegw/myjfs/test -r | awk '$4==\"5MiB\"' | wc -l)\n if [ \"$count1\" != \"$count2\" ]; then\n echo \"count not equal, $count1, $count2\"\n exit 1\n fi\n check_sync_log $file_count\n}\n\ntest_sync_worker_down(){\n prepare_test\n ./juicefs mount -d $META_URL /jfs\n file_count=$FILE_COUNT \n mkdir -p /jfs/data\n for i in $(seq 1 $file_count); do\n echo \"test-$i\" > /jfs/data/test-$i\n done\n docker stop worker1\n sudo -u juicedata meta_url=$META_URL ./juicefs sync jfs://meta_url/data/ jfs://meta_url/data2/ \\\n --manager-addr 172.20.0.1:8081 --worker juicedata@172.20.0.2,juicedata@172.20.0.3 \\\n --list-threads 10 --list-depth 5 --check-new \\\n >sync.log 2>&1\n diff /jfs/data/ /jfs/data2/\n docker start worker1\n}\n\ncheck_sync_log(){\n grep \"panic:\\|\" sync.log && echo \"panic or fatal in sync.log\" && exit 1 || true\n file_count=$1\n if tail -1 sync.log | grep -q \"close session\"; then\n file_copied=$(tail -n 3 sync.log | head -n 1 | sed 's/.*copied: \\([0-9]*\\).*/\\1/' )\n else\n file_copied=$(tail -1 sync.log | sed 's/.*copied: \\([0-9]*\\).*/\\1/' )\n fi\n if [ \"$file_copied\" != \"$file_count\" ]; then\n echo \"file_copied not equal, $file_copied, $file_count\"\n exit 1\n fi\n count2=$(cat sync.log | grep 172.20.0.2 | grep \"receive stats\" | gawk '{sum += gensub(/.*Copied:([0-9]+).*/, \"\\\\1\", \"g\");} END {print sum;}')\n [ -z \"$count2\" ] && count2=0\n count3=$(cat sync.log | grep 172.20.0.3 | grep \"receive stats\" | gawk '{sum += gensub(/.*Copied:([0-9]+).*/, \"\\\\1\", \"g\");} END {print sum;}')\n [ -z \"$count3\" ] && count3=0\n count1=$((file_count - count2 - count3))\n echo \"count1, $count1, count2, $count2, count3, $count3\"\n min_count=10\n # check if count1 is less than min_count\n if [ \"$count1\" -lt \"$min_count\" ] || [ \"$count2\" -lt \"$min_count\" ] || [ \"$count3\" -lt \"$min_count\" ]; then\n echo \"count is less than min_count, $count1, $count2, $count3, $min_count\"\n exit 1\n fi\n}\n\nprepare_test(){\n umount_jfs /jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs\n rm -rf /var/jfsCache/myjfs\n (./mc rb myminio/myjfs > /dev/null 2>&1 --force || true) && ./mc mb myminio/myjfs\n ./juicefs format $META_URL myjfs --storage minio --access-key minioadmin --secret-key minioadmin --bucket http://172.20.0.1:9000/myjfs\n}\nstart_gateway(){\n lsof -i :9005 | awk 'NR!=1 {print $2}' | xargs -r kill -9\n MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ./juicefs gateway $META_URL 172.20.0.1:9005 &\n ./mc alias set juicegw http://172.20.0.1:9005 minioadmin minioadmin --api S3v4\n}\n\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/sync/sync_fsrand.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\n\n[[ -z \"$SEED\" ]] && SEED=$(date +%s)\n[[ -z \"$DERANDOMIZE\" ]] && DERANDOMIZE=false\n[[ -z \"$MAX_EXAMPLE\" ]] && MAX_EXAMPLE=100\n[[ -z \"$GOCOVERDIR\" ]] && GOCOVERDIR=/tmp/cover\n[[ -z \"$USER\" ]] && USER=root\nif [ ! -d \"$GOCOVERDIR\" ]; then\n mkdir -p $GOCOVERDIR\nfi\ntrap \"echo random seed is $SEED\" EXIT\nSOURCE_DIR1=/tmp/fsrand1/\nSOURCE_DIR2=/tmp/fsrand2/\nDEST_DIR1=/tmp/jfs/fsrand1/\nDEST_DIR2=/tmp/jfs/fsrand2/\n\nrm $SOURCE_DIR1 -rf && sudo -u $USER mkdir $SOURCE_DIR1\nrm $SOURCE_DIR2 -rf && sudo -u $USER mkdir $SOURCE_DIR2\nEXCLUDE_RULES=\"utime\"\nPROFILE=generate EXCLUDE_RULES=$EXCLUDE_RULES MAX_EXAMPLE=$MAX_EXAMPLE SEED=$SEED ROOT_DIR1=$SOURCE_DIR1 ROOT_DIR2=$SOURCE_DIR2 python3 .github/scripts/hypo/fs.py || true\nprepare_test()\n{\n umount_jfs /tmp/jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs || true\n rm -rf /var/jfsCache/myjfs || true\n ./juicefs format $META_URL myjfs\n}\n\ntest_cmp_cp(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option 2>&1| tee sync.log || true\n do_copy $sync_option\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ntest_cmp_cp_without_perms(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option=\"--dirs --check-all --links --list-threads 10 --list-depth 5\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option 2>&1| tee sync.log || true\n do_copy $sync_option\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ntest_cmp_cp_without_links(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option=\"--dirs --check-all --perms --list-threads 10 --list-depth 5\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option 2>&1| tee sync.log || true\n do_copy $sync_option\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ntest_no_mount_point(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option 2>&1| tee sync1.log || true\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync -v $SOURCE_DIR1 jfs://meta_url/fsrand2/ $sync_option 2>&1| tee sync2.log || true\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ntest_inplace(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option1=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5\"\n sync_option2=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5 --inplace\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync -v $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option1 2>&1| tee sync1.log || true\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync -v $SOURCE_DIR1 jfs://meta_url/fsrand2/ $sync_option2 2>&1| tee sync2.log || true\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ntest_list_threads(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option1=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5\"\n sync_option2=\"--dirs --perms --check-all --links\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option1 2>&1| tee sync1.log || true\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR2 $sync_option2 2>&1| tee sync2.log || true\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ntest_update(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option 2>&1| tee sync.log || true\n do_copy $sync_option\n check_diff $DEST_DIR1 $DEST_DIR2\n \n sudo -u $USER PROFILE=generate EXCLUDE_RULES=$EXCLUDE_RULES MAX_EXAMPLE=$MAX_EXAMPLE SEED=$SEED ROOT_DIR1=$SOURCE_DIR1 ROOT_DIR2=$SOURCE_DIR2 python3 .github/scripts/hypo/fs.py || true\n # chmod 777 $SOURCE_DIR1\n # chmod 777 $SOURCE_DIR2\n do_copy $sync_option\n for i in {1..5}; do\n sync_option+=\" --update --delete-dst\"\n echo sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option 2>&1| tee sync.log || true\n if grep -q \"Failed to delete\" sync.log; then\n echo \"failed to delete, retry sync\"\n else\n echo \"sync delete success\"\n break\n fi\n done\n diff -ur --no-dereference $DEST_DIR1 $DEST_DIR2\n}\n\ntest_files_from(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option1=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5\"\n sync_option2=\"--dirs --perms --check-all --links --list-threads 10 --list-depth 5 --files-from files\"\n ls -A \"$SOURCE_DIR1\" | while read file; do \n full_path=\"$SOURCE_DIR1/$file\"\n if [ -L \"$full_path\" ] && [ ! -e \"$full_path\" ]; then\n rm \"$full_path\"\n else\n echo \"$file\"\n fi \n done > files\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option1 2>&1| tee sync1.log || true\n SOURCE_PERM=$(sudo stat -c \"%a\" \"$DEST_DIR1\")\n SOURCE_OWNER=$(sudo stat -c \"%U\" \"$DEST_DIR1\")\n SOURCE_GROUP=$(sudo stat -c \"%G\" \"$DEST_DIR1\")\n sudo mkdir -p $DEST_DIR2\n sudo chmod $SOURCE_PERM $DEST_DIR2\n sudo chown $SOURCE_OWNER:$SOURCE_GROUP $DEST_DIR2\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR2 $sync_option2 2>&1| tee sync2.log || true\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ntest_check_change(){\n prepare_test\n ./juicefs mount $META_URL /tmp/jfs -d\n sync_option=\"--dirs --check-change --links --perms --list-threads 10 --list-depth 5\"\n sudo -u $USER GOCOVERDIR=$GOCOVERDIR ./juicefs sync -v $SOURCE_DIR1 $DEST_DIR1 $sync_option 2>&1| tee sync.log || true\n do_copy $sync_option\n check_diff $DEST_DIR1 $DEST_DIR2\n}\n\ndo_copy(){\n local sync_option=$@\n local preserve=\"timestamps\"\n local no_preserve=\"\"\n if [[ \"$sync_option\" =~ \"--perms\" ]]; then\n preserve+=\",mode,ownership\"\n else\n no_preserve+=\"mode,ownership\"\n fi\n if [[ \"$sync_option\" =~ \"--links\" ]]; then\n preserve+=\",links\"\n fi\n local cp_option=\"--recursive --preserve=$preserve\"\n if [[ -n \"$no_preserve\" ]]; then\n cp_option+=\" --no-preserve=$no_preserve\"\n fi\n if [[ \"$sync_option\" =~ \"--links\" ]]; then\n cp_option+=\" --no-dereference\"\n else\n cp_option+=\" --dereference\"\n fi\n rm -rf $DEST_DIR2 \n sudo -u $USER cp $SOURCE_DIR1 $DEST_DIR2 $cp_option || true\n echo sudo -u $USER cp $SOURCE_DIR1 $DEST_DIR2 $cp_option\n}\n\ncheck_diff(){\n local dir1=$1\n local dir2=$2\n diff -ur --no-dereference $dir1 $dir2\n pushd . && diff <(cd $dir1 && find . -printf \"%p:%m:%u:%g:%y\\n\" | sort) <(cd $dir2 && find . -printf \"%p:%m:%u:%g:%y\\n\" | sort) && popd\n if [ $? -ne 0 ]; then\n echo \"permission or owner or group not equal\"\n exit 1\n fi\n # pushd . && diff <(cd $dir1 && find . ! -type d -printf \"%p:%.23T+\\n\" | sort) <(cd $dir2 && find . ! -type d -printf \"%p:%.23T+\\n\" | sort) && popd\n # if [ $? -ne 0 ]; then\n # echo \"mtime not equal\"\n # exit 1\n # fi\n # TODO: uncomment this after xattr is supported\n # pushd . && diff <(cd $dir1 && find . -exec getfattr -dm- {} + | sort) <(cd $dir2 && find . -exec getfattr -dm- {} + | sort) && popd\n # if [ $? -ne 0 ]; then\n # echo \"xattr not equal\"\n # exit 1\n # fi\n echo \"check diff success\"\n}\n\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/sync/sync_minio.sh", "content": "#!/bin/bash -e\nsource .github/scripts/common/common.sh\n\n[[ -z \"$META\" ]] && META=sqlite3\nsource .github/scripts/start_meta_engine.sh\nstart_meta_engine $META minio\nMETA_URL=$(get_meta_url $META)\n\ntest_sync_small_files(){\n prepare_test\n ./juicefs mdtest $META_URL /test --dirs 10 --depth 3 --files 5 --threads 10\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ --list-threads 100 --list-depth 10\n count1=$(./mc ls -r juicegw/myjfs/ | wc -l)\n count2=$(./mc ls -r myminio/myjfs/ | wc -l)\n [ $count1 -eq $count2 ]\n}\n\ntest_sync_big_file_with_jfs(){\n prepare_test\n [[ ! -f \"/tmp/bigfile\" ]] && dd if=/dev/urandom of=/tmp/bigfile bs=1M count=1024\n ./mc cp /tmp/bigfile myminio/myjfs/bigfile\n export dst_jfs=$META_URL \n timeout 10 ./juicefs sync minio://minioadmin:minioadmin@localhost:9000/myjfs/bigfile jfs://dst_jfs/bigfile --threads=64 --force-update\n cmp /tmp/bigfile /jfs/bigfile\n}\n\ntest_sync_big_file(){\n prepare_test\n dd if=/dev/urandom of=/tmp/bigfile bs=1M count=1024\n cp /tmp/bigfile /jfs/bigfile\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n ./mc cp myminio/myjfs/bigfile /tmp/bigfile2\n cmp /tmp/bigfile /tmp/bigfile2\n}\n\ntest_sync_with_limit(){\n prepare_test\n ./juicefs mdtest $META_URL /test --dirs 10 --depth 2 --files 5 --threads 10\n ./juicefs sync --limit 1000 minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ \n count=$(./mc ls myminio/myjfs -r | wc -l)\n echo count is $count\n [ $count -eq 1000 ]\n}\ntest_sync_with_existing(){\n prepare_test\n echo abc > /jfs/abc\n ./juicefs sync --existing minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ \n ./mc find myminio/myjfs/abc && echo \"myminio/myjfs/abc should not exist\" && exit 1 || true\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n ./mc find myminio/myjfs/abc\n}\ntest_sync_with_update(){\n prepare_test\n echo abc > /jfs/abc\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n echo def > def\n ./mc cp def myminio/myjfs/abc\n ./juicefs sync --update minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ \n ./mc cat myminio/myjfs/abc | grep def || (echo \"content should be def\" && exit 1)\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ \n ./mc cat myminio/myjfs/abc | grep def || (echo \"content should be def\" && exit 1)\n ./juicefs sync --force-update minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n ./mc cat myminio/myjfs/abc | grep abc || (echo \"content should be abc\" && exit 1)\n echo hijk > hijk\n ./mc cp hijk myminio/myjfs/abc\n ./juicefs sync --update minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ \n ./mc cat myminio/myjfs/abc | grep hijk || (echo \"content should be hijk\" && exit 1)\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ \n ./mc cat myminio/myjfs/abc | grep abc || (echo \"content should be abc\" && exit 1)\n}\n\ntest_sync_hard_link(){\n prepare_test\n echo abc > /jfs/abc\n ln /jfs/abc /jfs/def\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ \n ./mc cat myminio/myjfs/def | grep abc || (echo \"content should be abc\" && exit 1)\n echo abcd > /jfs/abc\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n ./mc cat myminio/myjfs/def | grep abcd || (echo \"content should be abcd\" && exit 1)\n}\n\ntest_sync_external_link(){\n prepare_test\n touch hello\n ln -s $(realpath hello) /jfs/hello\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n [ -z $(./mc cat myminio/myjfs/hello) ]\n}\n\n# list object should be skipped when encountering a loop symlink\ntest_sync_loop_symlink(){\n prepare_test\n touch hello\n ln -s hello /jfs/hello\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n rm -rf /jfs/hello\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n}\n\ntest_sync_deep_symlink(){\n prepare_test\n cd /jfs\n echo hello > hello\n ln -s hello symlink_1\n for i in {1..40}; do\n ln -s symlink_$i symlink_$((i+1))\n done\n cat symlink_40 | grep hello\n cat symlink_41 && echo \"cat symlink_41 fail\" && exit 1 || true\n cd -\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/\n for i in {1..40}; do\n ./mc cat myminio/myjfs/symlink_$i | grep \"^hello$\"\n done\n}\n\ntest_sync_list_object_symlink(){\n prepare_test\n cd /jfs\n mkdir dir1\n mkdir -p dir2/src_dir\n echo abc > dir2/src_dir/afile\n ln -s ./../dir2/src_dir dir1/symlink_dir\n cd -\n ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/dir1/ minio://minioadmin:minioadmin@localhost:9000/myjfs/dir3/\n ./mc cat myminio/myjfs/dir3/symlink_dir/afile | grep abc || (echo \"content should be abc\" && exit 1)\n}\n\nprepare_test(){\n umount_jfs /jfs $META_URL\n python3 .github/scripts/flush_meta.py $META_URL\n rm -rf /var/jfs/myjfs\n rm -rf /var/jfsCache/myjfs\n (./mc rb myminio/myjfs > /dev/null 2>&1 --force || true) && ./mc mb myminio/myjfs\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL /jfs\n lsof -i :9005 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true\n MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ./juicefs gateway $META_URL localhost:9005 &\n wait_gateway_ready\n ./mc alias set juicegw http://localhost:9005 minioadmin minioadmin --api S3v4\n}\n\nwait_gateway_ready(){\n timeout=30\n for i in $(seq 1 $timeout); do\n if [[ -z $(lsof -i :9005) ]]; then\n echo \"$i Waiting for port 9005 to be ready...\"\n sleep 1\n else\n echo \"gateway is now ready on port 9005\"\n break\n fi\n done\n if [[ -z $(lsof -i :9005) ]]; then\n echo \"gateway is not ready after $timeout seconds\"\n exit 1\n fi\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/test-mac/mac_commands.sh", "content": "#!/bin/bash -e\n\nsource .github/scripts/common/common.sh\nsource .github/scripts/test-mac/start_meta_engine.sh\n\n\n[[ -z \"$META\" ]] && META=redis\nstart_meta_engine $META\nMETA_URL=$(get_meta_url $META)\nuser=$(whoami)\nmount_point=\"/Users/$user/jfs\"\nHEARTBEAT_INTERVAL=3\nHEARTBEAT_SLEEP=3\nDIR_QUOTA_FLUSH_INTERVAL=4\nVOLUME_QUOTA_FLUSH_INTERVAL=2\n\nwget https://dl.min.io/client/mc/release/darwin-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O mc\nchmod +x mc\nexport MINIO_ROOT_USER=admin\nexport MINIO_ROOT_PASSWORD=admin123\nexport MINIO_REFRESH_IAM_INTERVAL=10s\n\n[[ ! -f my-priv-key.pem ]] && openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048\n\n\nskip_test_modify_acl_config()\n{\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount -d $META_URL $mount_point\n touch $mount_point/test\n sudo chmod +a \"$user allow read,write\" $mount_point/test && echo \"setfacl should failed\" && exit 1\n ./juicefs config $META_URL --enable-acl=true\n ./juicefs umount $mount_point\n sleep 2\n ./juicefs mount -d $META_URL $mount_point\n sudo chmod +a \"$user allow read,write\" $mount_point/test\n ./juicefs config $META_URL --enable-acl\n umount_jfs $mount_point $META_URL\n ./juicefs mount -d $META_URL $mount_point\n sudo chmod +a \"$user allow read,write\" $mount_point/test\n ./juicefs config $META_URL --enable-acl=false && echo \"should not disable acl\" && exit 1 || true \n ./juicefs config $META_URL | grep EnableACL | grep \"true\" || (echo \"EnableACL should be true\" && exit 1) \n}\n\ntest_clone_with_jfs_source()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL $mount_point\n [[ ! -d $mount_point/juicefs ]] && git clone https://github.com/juicedata/juicefs.git $mount_point/juicefs --depth 1\n do_clone true\n do_clone false\n}\n\ndo_clone()\n{\n is_preserve=$1\n rm -rf $mount_point/juicefs1\n rm -rf $mount_point/juicefs2\n [[ \"$is_preserve\" == \"true\" ]] && preserve=\"-p\" || preserve=\"\"\n cp -r $preserve $mount_point/juicefs $mount_point/juicefs1\n ./juicefs clone $mount_point/juicefs $mount_point/juicefs2 --preserve\n diff -r $mount_point/juicefs1 $mount_point/juicefs2\n cd $mount_point/juicefs1/ && find . -exec stat -f \"%p %u %g %N\" {} \\; | sort >/tmp/log1 && cd -\n cd $mount_point/juicefs2/ && find . -exec stat -f \"%p %u %g %N\" {} \\; | sort >/tmp/log2 && cd -\n diff /tmp/log1 /tmp/log2\n}\n\ncheck_debug_file(){\n files=(\"system-info.log\" \"juicefs.log\" \"config.txt\" \"stats.txt\" \"stats.5s.txt\" \"pprof\")\n debug_dir=\"debug\"\n if [ ! -d \"$debug_dir\" ]; then\n echo \"error:no debug dir\"\n exit 1\n fi\n all_files_exist=true\n for file in \"${files[@]}\"; do\n exist=`find \"$debug_dir\" -name $file | wc -l`\n if [ \"$exist\" == 0 ]; then\n echo \"no $file\"\n all_files_exist=false\n fi\n done\n if [ \"$all_files_exist\" = true ]; then\n echo \"pass\"\n else\n exit 1\n fi\n}\n\ntest_debug_juicefs(){\n ./juicefs format $META_URL myjfs \n ./juicefs mount -d $META_URL $mount_point\n dd if=/dev/urandom of=$mount_point/bigfile bs=1M count=128\n ./juicefs debug $mount_point/\n check_debug_file\n ./juicefs rmr $mount_point/bigfile\n}\n\ntest_sync_dir_stat()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL $mount_point\n ./juicefs mdtest $META_URL /d --depth 15 --dirs 2 --files 100 --threads 10 & \n pid=$!\n sleep 10\n kill -9 $pid\n pkill -P \"$pid\" 2>/dev/null || true\n ./juicefs info -r $mount_point/d\n ./juicefs info -r $mount_point/d --strict \n ./juicefs fsck $META_URL --path /d --sync-dir-stat --repair -r\n ./juicefs info -r $mount_point/d | tee info1.log\n ./juicefs info -r $mount_point/d --strict | tee info2.log\n diff info1.log info2.log\n rm info*.log\n ./juicefs fsck $META_URL --path / --sync-dir-stat --repair -r\n ./juicefs info -r $mount_point | tee info1.log\n ./juicefs info -r $mount_point --strict | tee info2.log\n diff info1.log info2.log\n}\n\ntest_gc_trash_slices(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL $mount_point\n PATH1=/tmp/test PATH2=$mount_point/test python3 .github/scripts/random_read_write.py \n ./juicefs status --more $META_URL\n ./juicefs config $META_URL --trash-days 0 --yes\n ./juicefs gc $META_URL \n ./juicefs gc $META_URL --delete\n ./juicefs status --more $META_URL\n}\n\ntest_update_non_fuse_option(){\n prepare_test\n JFS_RSA_PASSPHRASE=12345678 ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem\n JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL $mount_point\n echo abc | tee $mount_point/test\n JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL $mount_point --read-only\n echo abc | tee $mount_point/test && (echo \"should not write read-only file system\" && exit 1) || true\n JFS_RSA_PASSPHRASE=12345678 ./juicefs mount -d $META_URL $mount_point \n echo abc | tee $mount_point/test\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l | tr -d ' ')\n [[ $count -ne 2 ]] && echo \"mount process count should be 2, count=$count\" && exit 1 || true\n umount $mount_point\n sleep 2\n ps -ef | grep juicefs | grep mount | grep -v grep || true\n count=$(ps -ef | grep juicefs | grep mount | grep -v grep | wc -l | tr -d ' ')\n [[ $count -ne 0 ]] && echo \"mount process count should be 0, count=$count\" && exit 1 || true\n}\n\ntest_info_big_file(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL $mount_point\n dd if=/dev/zero of=$mount_point/bigfile bs=1M count=4096\n ./juicefs info $mount_point/bigfile\n ./juicefs rmr $mount_point/bigfile\n df -h $mount_point\n}\n\ntest_list_large_dir()\n{\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL $mount_point\n local files_count=100000\n if [[ \"$META_URL\" == redis://* ]]; then\n files_count=130000\n fi\n ./juicefs mdtest $META_URL /test --depth 0 --dirs 1 --files $files_count --threads 1\n du $mount_point/test & du_pid=$!\n sleep 2\n kill -INT $du_pid || true\n wait $du_pid || true\n if ! [ -d \"$mount_point/test\" ]; then\n echo >&2 \": directory $mount_point/test is not accessible after ls interruption\"\n exit 1\n fi\n}\n\ntest_total_inodes(){\n prepare_test\n ./juicefs format $META_URL myjfs --inodes 1000\n ./juicefs mount -d $META_URL $mount_point --heartbeat $HEARTBEAT_INTERVAL\n set +x\n for i in {1..1000}; do\n echo $i | tee $mount_point/test$i > /dev/null\n done\n set -x\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee $mount_point/test1001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n grep \"No space left on device\" error.log\n ./juicefs config $META_URL --inodes 2000\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n set +x\n for i in {1001..2000}; do\n echo $i | tee $mount_point/test$i > /dev/null || (df -i $mount_point && ls $mount_point/ -l | wc -l && exit 1)\n done\n set -x\n sleep $VOLUME_QUOTA_FLUSH_INTERVAL\n echo a | tee $mount_point/test2001 2>error.log && echo \"write should fail on out of inodes\" && exit 1 || true\n}\n\ntest_remove_and_restore(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL $mount_point --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p $mount_point/d\n ./juicefs quota set $META_URL --path /d --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=$mount_point/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n echo a | tee -a $mount_point/d/test2 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n grep -i \"Disc quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n\n echo \"remove test1\" && rm -rf $mount_point/d/test1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"0%\" ]] && echo \"used should be 0%\" && exit 1 || true\n\n trash_dir=$(ls $mount_point/.trash)\n sudo ./juicefs restore $META_URL $trash_dir --put-back\n ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log\n used=$(cat quota.log | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n echo a | tee -a $mount_point/d/test2 2>error.log && echo \"write should fail on out of space\" && exit 1 || true\n grep -i \"Disc quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n\n echo \"remove test1\" && rm -rf $mount_point/d/test1\n dd if=/dev/zero of=$mount_point/d/test2 bs=1M count=1\n trash_dir=$(ls $mount_point/.trash)\n sudo ./juicefs restore $META_URL $trash_dir --put-back 2>&1 | tee restore.log\n grep \"disc quota exceeded\" restore.log || (echo \"check restore log failed\" && exit 1)\n}\n\ntest_dir_capacity(){\n prepare_test\n ./juicefs format $META_URL myjfs\n ./juicefs mount -d $META_URL $mount_point --heartbeat $HEARTBEAT_INTERVAL\n mkdir -p $mount_point/d\n ./juicefs quota set $META_URL --path /d --capacity 1\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=$mount_point/d/test1 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota get $META_URL --path /d\n used=$(./juicefs quota get $META_URL --path /d 2>&1 | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"100%\" ]] && echo \"used should be 100%\" && exit 1 || true\n echo a | tee -a $mount_point/d/test2 2>error.log && echo \"echo should fail on out of space\" && exit 1 || true\n grep -i \"Disc quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n\n ./juicefs quota set $META_URL --path /d --capacity 2\n sleep $((HEARTBEAT_INTERVAL+HEARTBEAT_SLEEP))\n dd if=/dev/zero of=$mount_point/d/test2 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n echo a | tee -a $mount_point/d/test3 2>error.log && echo \"echo should fail on out of space\" && exit 1 || true\n grep -i \"Disc quota exceeded\" error.log || (echo \"grep failed\" && exit 1)\n rm -rf $mount_point/d/test1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n used=$(./juicefs quota get $META_URL --path /d 2>&1 | grep \"/d\" | awk -F'|' '{print $5}' | tr -d '[:space:]')\n [[ $used != \"50%\" ]] && echo \"used should be 50%\" && exit 1 || true\n dd if=/dev/zero of=$mount_point/d/test3 bs=1G count=1\n sleep $DIR_QUOTA_FLUSH_INTERVAL\n ./juicefs quota check $META_URL --path /d --strict\n}\n\nkill_gateway() {\n port=$1\n lsof -i:$port || true\n lsof -t -i :$port | xargs -r kill -9 || true\n}\n\ntrap 'kill_gateway 9001; kill_gateway 9002' EXIT\n\nstart_two_gateway()\n{ \n kill_gateway 9001\n kill_gateway 9002\n prepare_test\n ./juicefs format $META_URL myjfs --trash-days 0\n ./juicefs mount -d $META_URL $mount_point\n export MINIO_ROOT_USER=admin\n export MINIO_ROOT_PASSWORD=admin123\n ./juicefs gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag -background\n sleep 1\n ./juicefs gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag -background\n sleep 2\n ./mc alias set gateway1 http://127.0.0.1:9001 admin admin123\n ./mc alias set gateway2 http://127.0.0.1:9002 admin admin123\n}\n\ntest_user_management()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n sleep 12\n user=$(./mc admin user list gateway2 | grep user1) || true\n if [ -z \"$user\" ]\n then\n echo \"user synchronization error\"\n exit 1\n fi\n ./mc mb gateway1/test1\n ./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n if ./mc cp mc gateway1_user1/test1/file1\n then\n echo \"By default, the user has no read and write permission\"\n exit 1\n fi\n ./mc admin policy set gateway1 readwrite user=user1\n if ./mc cp mc gateway1_user1/test1/file1\n then \n echo \"readwrite policy can read and write objects\" \n else\n echo \"set readwrite policy fail\"\n exit 1\n fi\n ./mc cp gateway2/test1/file1 .\n compare_md5sum file1 mc \n ./mc admin user disable gateway1 user1\n ./mc admin user remove gateway2 user1\n sleep 12\n user=$(./mc admin user list gateway1 | grep user1) || true\n if [ ! -z \"$user\" ]\n then\n echo \"remove user user1 fail\"\n echo $user\n exit 1\n fi\n}\n\ntest_group_management()\n{\n prepare_test\n start_two_gateway\n ./mc admin user add gateway1 user1 admin123\n ./mc admin user add gateway1 user2 admin123\n ./mc admin user add gateway1 user3 admin123\n ./mc admin group add gateway1 testcents user1 user2 user3\n result=$(./mc admin group info gateway1 testcents | grep Members |awk '{print $2}') || true\n if [ \"$result\" != \"user1,user2,user3\" ]\n then\n echo \"error,result is '$result'\"\n exit 1\n fi\n ./mc admin policy set gateway1 readwrite group=testcents\n sleep 5\n ./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123\n ./mc mb gateway1/test1\n if ./mc cp mc gateway1_user1/test1/file1\n then\n echo \"readwrite policy can read write\"\n else\n echo \"the readwrite group has no read and write permission\"\n exit 1\n fi\n ./mc admin policy set gateway1 readonly group=testcents\n sleep 5\n if ./mc cp mc gateway1_user1/test1/file1\n then\n echo \"readonly group policy can not write\"\n exit 1\n else\n echo \"the readonly group has no write permission\"\n fi\n\n ./mc admin group remove gateway1 testcents user1 user2 user3 \n ./mc admin group remove gateway1 testcents\n}\n\nsource .github/scripts/common/run_test.sh && run_test $@\n" }, { "path": ".github/scripts/test-mac/start_meta_engine.sh", "content": "#!/bin/bash -e\n\nREDIS_CSC_QUERY=\"client-cache=true&client-cache-size=500&client-cache-expire=60s&client-cache-preload=100\"\n\n# Helper function to install packages via Homebrew\nbrew_install() {\n if ! brew list \"$1\" &>/dev/null; then\n echo \"Installing $1...\"\n brew install \"$1\"\n fi\n}\n\nstart_redis() {\n if pgrep redis-server >/dev/null; then\n echo \"Redis is already running\"\n return 0\n fi\n\n if brew services start redis 2>/dev/null; then\n echo \"Redis started via brew services\"\n elif [ -f /usr/local/bin/redis-server ]; then\n echo \"Starting Redis directly...\"\n /usr/local/bin/redis-server /usr/local/etc/redis.conf &\n else\n echo \"Failed to start Redis\"\n return 1\n fi\n\n sleep 2\n if ! pgrep redis-server >/dev/null; then\n echo \"Redis failed to start\"\n return 1\n fi\n}\n\nclean_minio() {\n if command -v mc >/dev/null; then\n mc ls local/ 2>/dev/null | awk '{print $5}' | while read -r bucket; do\n if [ -n \"$bucket\" ]; then\n echo \"Cleaning bucket: $bucket\"\n mc rb --force local/\"$bucket\" 2>/dev/null || true\n fi\n done\n fi\n}\n\nstart_minio() {\n if ! command -v minio >/dev/null; then\n brew_install minio/stable/minio\n fi\n \n if ! command -v mc >/dev/null; then\n brew_install minio/stable/mc\n fi\n\n clean_minio\n \n if ! pgrep minio >/dev/null; then\n mkdir -p /tmp/data\n rm -rf /tmp/data/*\n minio server /tmp/data --console-address :9001 &\n sleep 3\n fi\n\n mc alias set local http://127.0.0.1:9000 minioadmin minioadmin || true\n \n mc mb local/jfs || true\n mc mb local/test || true\n}\n\nstart_meta_engine() {\n local meta=$1\n local storage=$2\n\n case \"$meta\" in\n redis)\n brew_install redis\n if ! start_redis; then\n echo >&2 \"Failed to start Redis\"\n return 1\n fi\n ;;\n sqlite3)\n brew_install sqlite3\n echo \"SQLite3 ready to use\"\n ;;\n *)\n echo >&2 \": Unsupported meta engine: $meta\"\n return 1\n ;;\n esac\n\n if [ \"$storage\" = \"minio\" ]; then\n if ! start_minio; then\n echo >&2 \"Failed to start MinIO\"\n return 1\n fi\n fi\n}\n\nget_meta_url() {\n case \"$1\" in\n redis) echo \"redis://127.0.0.1:6379/1?${REDIS_CSC_QUERY}\" ;;\n sqlite3) echo \"sqlite3://test.db\" ;;\n *) echo >&2 \": Unsupported meta: $1\"; return 1 ;;\n esac\n}\n\nget_meta_url2() {\n case \"$1\" in\n redis) echo \"redis://127.0.0.1:6379/2?${REDIS_CSC_QUERY}\" ;;\n sqlite3) echo \"sqlite3://test2.db\" ;;\n *) echo >&2 \": Unsupported meta: $1\"; return 1 ;;\n esac\n}\n\nretry() {\n local retries=5\n local delay=3\n local exit=0\n\n for i in $(seq 1 \"$retries\"); do\n if \"$@\"; then\n return 0\n else\n exit=$?\n if [ \"$i\" -eq \"$retries\" ]; then\n return \"$exit\"\n fi\n sleep \"$delay\"\n fi\n done\n}" }, { "path": ".github/scripts/testVersionCompatible.py", "content": "import subprocess\ntry:\n __import__(\"hypothesis\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"hypothesis\"])\nfrom datetime import datetime\nimport json\nimport os\nfrom pickle import FALSE\nimport platform\nimport shutil\nimport sys\nfrom termios import TIOCPKT_DOSTOP\nimport time\nimport unittest\nfrom xmlrpc.client import boolean\nimport hypothesis\nfrom hypothesis.stateful import rule, precondition, RuleBasedStateMachine\nfrom hypothesis import Phase, Verbosity, assume, strategies as st\nfrom hypothesis import seed\nfrom packaging import version\nimport subprocess\ntry:\n __import__(\"minio\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"minio\"])\nfrom minio import Minio\nimport uuid\nfrom utils import *\nfrom fsrand import *\nfrom cmptree import *\nimport random\n\n@seed(random.randint(10000, 1000000))\n@hypothesis.settings(\n verbosity=Verbosity.debug, \n max_examples=100, \n stateful_step_count=30, \n deadline=None, \n report_multiple_bugs=False, \n phases=[Phase.explicit, Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])\nclass JuicefsMachine(RuleBasedStateMachine):\n MIN_CLIENT_VERSIONS = ['0.0.1', '0.0.17','1.0.0-beta1', '1.0.0-rc1']\n MAX_CLIENT_VERSIONS = ['1.2.0', '2.0.0']\n JFS_BINS = ['./'+os.environ.get('OLD_JFS_BIN'), './'+os.environ.get('NEW_JFS_BIN')]\n meta_dict = {'redis':'redis://localhost/1', 'mysql':'mysql://root:root@(127.0.0.1)/test', 'postgres':'postgres://postgres:postgres@127.0.0.1:5432/test?sslmode=disable', \\\n 'tikv':'tikv://127.0.0.1:2379', 'badger':'badger://badger-data', 'mariadb': 'mysql://root:root@(127.0.0.1)/test', \\\n 'sqlite3': 'sqlite3://test.db', 'fdb':'fdb:///home/runner/fdb.cluster?prefix=jfs'}\n META_URL = meta_dict[os.environ.get('META')]\n STORAGE = os.environ.get('STORAGE')\n MOUNT_POINT = '/tmp/sync-test/'\n VOLUME_NAME = 'test-volume'\n # valid_file_name = st.text(st.characters(max_codepoint=1000, blacklist_categories=('Cc', 'Cs')), min_size=2).map(lambda s: s.strip()).filter(lambda s: len(s) > 0)\n\n def __init__(self):\n super(JuicefsMachine, self).__init__()\n print(f\"seed is: {self._hypothesis_internal_use_seed}\")\n self.run_id = uuid.uuid4().hex\n print(f'\\ninit with run_id: {self.run_id}')\n with open(os.path.expanduser('~/command.log'), 'a') as f:\n f.write(f'init with run_id: {self.run_id}\\n')\n self.formatted = False\n self.mounted = False\n # mount at least once, see ref: https://github.com/juicedata/juicefs/issues/2717\n self.mounted_by = []\n self.formatted_by = ''\n self.dumped_by = ''\n if JuicefsMachine.META_URL.startswith('badger://'):\n # change url for each run\n JuicefsMachine.META_URL = f'badger://badger-{uuid.uuid4().hex}'\n if JuicefsMachine.STORAGE == 'minio':\n run_cmd(f'mc alias set myminio http://localhost:9000 minioadmin minioadmin')\n if os.path.isfile('dump.json'):\n os.remove('dump.json')\n os.environ['PGPASSWORD'] = 'postgres'\n\n @rule(\n juicefs=st.sampled_from(JFS_BINS),\n block_size=st.integers(min_value=1, max_value=4096*10), \n capacity=st.integers(min_value=0, max_value=1024),\n inodes=st.integers(min_value=1024*1024, max_value=1024*1024*1024),\n compress=st.sampled_from(['lz4', 'zstd', 'none']),\n shards=st.integers(min_value=0, max_value=1),\n storage=st.just(STORAGE), \n encrypt_rsa_key = st.booleans(), \n encrypt_algo = st.sampled_from(['aes256gcm-rsa','chacha20-rsa']),\n trash_days=st.integers(min_value=0, max_value=10000), \n hash_prefix=st.booleans(), \n force = st.booleans(), \n no_update = st.booleans()\n )\n def format(self, juicefs, block_size, capacity, inodes, compress, shards, storage, encrypt_rsa_key, encrypt_algo, trash_days, hash_prefix, force, no_update):\n assume (self.greater_than_version_formatted(juicefs))\n print('start format')\n options = [juicefs, 'format', JuicefsMachine.META_URL, JuicefsMachine.VOLUME_NAME]\n if not self.formatted:\n options.extend(['--block-size', str(block_size)])\n options.extend(['--compress', compress])\n options.extend(['--shards', str(shards)])\n options.extend(['--storage', storage])\n if hash_prefix and run_cmd(f'{juicefs} format --help | grep hash-prefix') == 0:\n options.append('--hash-prefix')\n options.extend(['--capacity', str(capacity)])\n options.extend(['--inodes', str(inodes)])\n if run_cmd(f'{juicefs} format --help | grep trash-days') == 0:\n options.extend(['--trash-days', str(trash_days)])\n \n if force:\n options.append('--force')\n if no_update:\n options.append('--no-update')\n if encrypt_rsa_key:\n if not os.path.exists('my-priv-key.pem'):\n subprocess.check_call('openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048'.split())\n os.environ['JFS_RSA_PASSPHRASE'] = '12345678'\n options.extend(['--encrypt-rsa-key', 'my-priv-key.pem'])\n if run_cmd(f'{juicefs} format --help | grep encrypt-algo') == 0:\n options.extend(['--encrypt-algo', encrypt_algo])\n \n if storage == 'minio':\n bucket = 'http://localhost:9000/testbucket'\n options.extend(['--bucket', bucket])\n options.extend(['--access-key', 'minioadmin'])\n options.extend(['--secret-key', 'minioadmin'])\n if self.formatted and version.parse('-'.join(juicefs.split('-')[1:])) <= version.parse('1.0.0-rc1'):\n # use the latest version to change secret-key because rc1 has a bug for secret-key\n options[0] = JuicefsMachine.JFS_BINS[1]\n elif storage == 'file':\n bucket = os.path.expanduser('~/.juicefs/local/')\n options.extend(['--bucket', bucket])\n elif storage == 'mysql':\n bucket = '(localhost:3306)/testbucket'\n options.extend(['--bucket', bucket])\n options.extend(['--access-key', 'root'])\n options.extend(['--secret-key', 'root'])\n elif storage == 'postgres':\n bucket = 'localhost:5432/testbucket?sslmode=disable'\n options.extend(['--bucket', bucket])\n options.extend(['--access-key', 'postgres'])\n options.extend(['--secret-key', 'postgres'])\n else:\n print(f'storage is {storage}')\n raise Exception(f'storage value error: {storage}')\n\n if not self.formatted:\n if os.path.exists(JuicefsMachine.MOUNT_POINT) and os.path.exists(JuicefsMachine.MOUNT_POINT+'.accesslog'):\n run_cmd('umount %s'%JuicefsMachine.MOUNT_POINT)\n print(f'umount {JuicefsMachine.MOUNT_POINT} succeed')\n clear_storage(storage, bucket, JuicefsMachine.VOLUME_NAME)\n flush_meta(JuicefsMachine.META_URL)\n print(f'format options: {\" \".join(options)}' )\n run_jfs_cmd(options)\n self.formatted = True\n self.formatted_by = juicefs\n print('format succeed')\n\n\n @rule(\n juicefs=st.sampled_from(JFS_BINS),\n capacity=st.integers(min_value=0, max_value=1024), \n inodes=st.integers(min_value=1024*1024, max_value=1024*1024*1024),\n change_bucket=st.booleans(), \n change_aksk=st.booleans(), \n encrypt_secret = st.booleans(), \n trash_days = st.integers(min_value=0, max_value=10000),\n min_client_version = st.sampled_from(MIN_CLIENT_VERSIONS), \n max_client_version = st.sampled_from(MAX_CLIENT_VERSIONS), \n force = st.booleans(),\n )\n @precondition(lambda self: self.formatted)\n def config(self, juicefs, capacity, inodes, change_bucket, change_aksk, encrypt_secret, trash_days, min_client_version, max_client_version, force):\n assume (self.greater_than_version_formatted(juicefs))\n assume(run_cmd(f'{juicefs} --help | grep config') == 0)\n print('start config')\n options = [juicefs, 'config', JuicefsMachine.META_URL]\n options.extend(['--trash-days', str(trash_days)])\n options.extend(['--capacity', str(capacity)])\n options.extend(['--inodes', str(inodes)])\n assert version.parse(min_client_version) <= version.parse(max_client_version)\n if run_cmd(f'{juicefs} config --help | grep min-client-version') == 0:\n options.extend(['--min-client-version', min_client_version])\n if run_cmd(f'{juicefs} config --help | grep max-client-version') == 0:\n options.extend(['--max-client-version', max_client_version])\n storage = get_storage(juicefs, JuicefsMachine.META_URL)\n \n if change_bucket:\n if storage == 'file':\n options.extend(['--bucket', os.path.expanduser('~/.juicefs/local2')])\n elif storage == 'minio': \n c = Minio('localhost:9000', access_key='minioadmin', secret_key='minioadmin', secure=False)\n if not c.bucket_exists('testbucket2'):\n run_cmd('mc mb myminio/testbucket2')\n # assert c.bucket_exists('testbucket2')\n options.extend(['--bucket', 'http://localhost:9000/testbucket2'])\n elif storage == 'mysql':\n create_mysql_db('mysql://root:root@(localhost:3306)/testbucket2')\n options.extend(['--bucket', '(localhost:3306)/testbucket2'])\n elif storage == 'postgres':\n create_postgres_db('postgres://postgres:postgres@localhost:5432/testbucket2?sslmode=disable')\n options.extend(['--bucket', 'localhost:5432/testbucket2?sslmode=disable'])\n if change_aksk and storage == 'minio':\n output = subprocess.check_output('mc admin user list myminio'.split())\n if not output:\n run_cmd('mc admin user add myminio juicedata 12345678')\n run_cmd('mc admin policy attach myminio consoleAdmin --user juicedata')\n options.extend(['--access-key', 'juicedata'])\n options.extend(['--secret-key', '12345678'])\n if version.parse('-'.join(juicefs.split('-')[1:])) <= version.parse('1.0.0-rc1'):\n # use the latest version to set secret-key because rc1 has a bug for secret-key\n options[0] = JuicefsMachine.JFS_BINS[1]\n if encrypt_secret and run_cmd(f'{juicefs} config --help | grep encrypt-secret') == 0:\n # 0.17.5 store the secret without encrypt, ref: https://github.com/juicedata/juicefs/issues/2721\n #if version.parse('-'.join(juicefs.split('-')[1:])) > version.parse('0.17.5'):\n options.append('--encrypt-secret')\n options.append('--force')\n run_jfs_cmd(options)\n if change_bucket:\n # change bucket back to avoid fsck fail.\n if storage == 'file':\n run_jfs_cmd([juicefs, 'config', JuicefsMachine.META_URL, '--bucket', os.path.expanduser('~/.juicefs/local')])\n elif storage == 'minio':\n run_jfs_cmd([juicefs, 'config', JuicefsMachine.META_URL, '--bucket', 'http://localhost:9000/testbucket'])\n elif storage == 'mysql':\n run_jfs_cmd([juicefs, 'config', JuicefsMachine.META_URL, '--bucket', '(localhost:3306)/testbucket'])\n elif storage == 'postgres':\n run_jfs_cmd([juicefs, 'config', JuicefsMachine.META_URL, '--bucket', 'localhost:5432/testbucket?sslmode=disable'])\n self.formatted_by = juicefs\n print('config succeed')\n\n\n @rule(juicefs=st.sampled_from(JFS_BINS))\n @precondition(lambda self: self.formatted )\n def status(self, juicefs):\n assume (self.greater_than_version_formatted(juicefs))\n print('start status')\n output = subprocess.run([juicefs, 'status', JuicefsMachine.META_URL], check=True, stdout=subprocess.PIPE).stdout.decode()\n if 'get timestamp too slow' in output: \n # remove the first line caust it is tikv log message\n output = '\\n'.join(output.split('\\n')[1:])\n print(f'status output: {output}')\n try:\n uuid = json.loads(output.replace(\"'\", '\"'))['Setting']['UUID']\n except:\n raise Exception(f'parse uuid failed, output: {output}')\n assert len(uuid) != 0\n if self.mounted and not is_readonly(JuicefsMachine.MOUNT_POINT) and self.greater_than_version_mounted(juicefs):\n sessions = json.loads(output.replace(\"'\", '\"'))['Sessions']\n assert len(sessions) != 0 \n print('status succeed')\n\n\n @rule(juicefs=st.sampled_from(JFS_BINS), \n no_syslog=st.booleans(),\n other_fuse_options=st.lists(st.sampled_from(['debug', 'allow_other', 'writeback_cache']), unique=True), \n enable_xattr=st.booleans(),\n attr_cache=st.integers(min_value=1, max_value=10), \n entry_cache=st.integers(min_value=1, max_value=10), \n dir_entry_cache=st.integers(min_value=1, max_value=10), \n get_timeout=st.integers(min_value=30, max_value=60), \n put_timeout=st.integers(min_value=30, max_value=60), \n io_retries=st.integers(min_value=5, max_value=15), \n max_uploads=st.integers(min_value=5, max_value=100), \n max_deletes=st.integers(min_value=5, max_value=100), \n buffer_size=st.integers(min_value=100, max_value=1000), \n upload_limit=st.integers(min_value=100, max_value=1000), \n download_limit=st.integers(min_value=100, max_value=1000), \n prefetch=st.integers(min_value=0, max_value=100), \n writeback=st.just(False),\n upload_delay=st.sampled_from([0, 2]), \n cache_dir=st.sampled_from(['cache1', 'cache2']),\n cache_size=st.integers(min_value=0, max_value=1024000), \n free_space_ratio=st.floats(min_value=0.1, max_value=0.5), \n cache_partial_only=st.booleans(),\n backup_meta=st.integers(min_value=300, max_value=1000),\n heartbeat=st.integers(min_value=5, max_value=12), \n read_only=st.booleans(),\n no_bgjob=st.booleans(),\n open_cache=st.integers(min_value=0, max_value=100),\n sub_dir=st.sampled_from(['dir1', 'dir2']),\n metrics=st.sampled_from(['127.0.0.1:9567', '127.0.0.1:9568']), \n consul=st.sampled_from(['127.0.0.1:8500', '127.0.0.1:8501']), \n )\n @precondition(lambda self: self.formatted )\n def mount(self, juicefs, no_syslog, other_fuse_options, enable_xattr, attr_cache, entry_cache, dir_entry_cache,\n get_timeout, put_timeout, io_retries, max_uploads, max_deletes, buffer_size, upload_limit, download_limit, prefetch, \n writeback, upload_delay, cache_dir, cache_size, free_space_ratio, cache_partial_only, backup_meta, heartbeat, read_only,\n no_bgjob, open_cache, sub_dir, metrics, consul):\n assume (self.greater_than_version_formatted(juicefs))\n if JuicefsMachine.META_URL.startswith('badger://'):\n assume(not self.mounted)\n retry = 3\n while os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog') and retry > 0:\n os.system(f'umount {JuicefsMachine.MOUNT_POINT}')\n retry = retry - 1 \n time.sleep(1)\n if os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'):\n print(f'FATAL: umount {JuicefsMachine.MOUNT_POINT} failed.')\n assume(not os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'))\n print('start mount')\n options = [juicefs, 'mount', '-d', JuicefsMachine.META_URL, JuicefsMachine.MOUNT_POINT]\n if no_syslog:\n options.append('--no-syslog')\n options.extend(['--log', os.path.expanduser(f'~/.juicefs/juicefs.log')])\n if other_fuse_options:\n options.extend(['-o', ','.join(other_fuse_options)])\n if 'allow_other' in other_fuse_options:\n if os.path.exists('/etc/fuse.conf'):\n # subprocess.check_call(['sudo', 'bash', '-c', '\"echo user_allow_other >>/etc/fuse.conf\"' ])\n os.system('sudo bash -c \"echo user_allow_other >>/etc/fuse.conf\"')\n print('add user_allow_other to /etc/fuse.conf succeed')\n if enable_xattr:\n options.append('--enable-xattr')\n options.extend(['--attr-cache', str(attr_cache)])\n options.extend(['--entry-cache', str(entry_cache)])\n options.extend(['--dir-entry-cache', str(dir_entry_cache)])\n options.extend(['--get-timeout', str(get_timeout)])\n options.extend(['--put-timeout', str(put_timeout)])\n options.extend(['--io-retries', str(io_retries)])\n options.extend(['--max-uploads', str(max_uploads)])\n if run_cmd(f'{juicefs} mount --help | grep max-deletes') == 0:\n options.extend(['--max-deletes', str(max_deletes)])\n options.extend(['--buffer-size', str(buffer_size)])\n options.extend(['--upload-limit', str(upload_limit)])\n options.extend(['--download-limit', str(download_limit)])\n options.extend(['--prefetch', str(prefetch)])\n if writeback:\n options.append('--writeback')\n upload_delay = str(upload_delay)\n if version.parse('-'.join(juicefs.split('-')[1:])) <= version.parse('1.0.0-beta2'):\n upload_delay = upload_delay + 's'\n options.extend(['--upload-delay', str(upload_delay)])\n options.extend(['--cache-dir', os.path.expanduser(f'~/.juicefs/{cache_dir}')])\n options.extend(['--cache-size', str(cache_size)])\n options.extend(['--free-space-ratio', str(free_space_ratio)])\n if cache_partial_only:\n options.append('--cache-partial-only')\n backup_meta = str(backup_meta)\n if version.parse('-'.join(juicefs.split('-')[1:])) <= version.parse('1.0.0-beta2'):\n backup_meta = '1h0m0s'\n if run_cmd(f'{juicefs} mount --help | grep backup-meta') == 0:\n options.extend(['--backup-meta', backup_meta])\n if run_cmd(f'{juicefs} mount --help | grep heartbeat') == 0:\n options.extend(['--heartbeat', str(heartbeat)])\n if read_only:\n options.append('--read-only')\n if no_bgjob and run_cmd(f'{juicefs} mount --help | grep no-bgjob') == 0:\n options.append('--no-bgjob')\n\n options.extend(['--open-cache', str(open_cache)])\n print('TODO: subdir')\n # options.extend('--subdir', str(sub_dir))\n if not is_port_in_use( int(metrics.split(':')[1])):\n options.extend(['--metrics', str(metrics)])\n # if run_cmd(f'{juicefs} mount --help | grep consul') == 0:\n # options.extend(['--consul', str(consul)])\n options.append('--no-usage-report')\n if os.path.exists(JuicefsMachine.MOUNT_POINT):\n run_cmd(f'stat {JuicefsMachine.MOUNT_POINT}')\n run_jfs_cmd(options)\n time.sleep(2)\n if platform.system() == 'Linux':\n inode = subprocess.check_output(f'stat -c %i {JuicefsMachine.MOUNT_POINT}'.split())\n elif platform.system() == 'Darwin':\n inode = subprocess.check_output(f'stat -f %i {JuicefsMachine.MOUNT_POINT}'.split())\n print(f'inode number: {inode}')\n assert(inode.decode()[:-1] == '1')\n output = subprocess.run([juicefs, 'status', JuicefsMachine.META_URL], check=True, stdout=subprocess.PIPE).stdout.decode()\n if 'get timestamp too slow' in output: \n # remove the first line caust it is tikv log message\n output = '\\n'.join(output.split('\\n')[1:])\n print(f'status output: {output}')\n sessions = json.loads(output.replace(\"'\", '\"'))['Sessions']\n if not read_only: \n assert len(sessions) != 0 \n self.mounted = True\n if not read_only:\n self.mounted_by.append(juicefs)\n print('mount succeed')\n\n @rule(juicefs=st.sampled_from(JFS_BINS), \n file_name=st.just('file_to_info'), \n data = st.binary())\n @precondition(lambda self: self.formatted and self.mounted )\n def info(self, juicefs, file_name, data):\n assume (self.greater_than_version_formatted(juicefs))\n assume (self.greater_than_version_mounted(juicefs))\n assume(not is_readonly(f'{JuicefsMachine.MOUNT_POINT}'))\n assert(os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'))\n print('start info')\n path = JuicefsMachine.MOUNT_POINT+file_name\n write_data(JuicefsMachine.MOUNT_POINT, path, data)\n options = [juicefs, 'info', path]\n run_jfs_cmd(options)\n print('info succeed')\n\n @rule(juicefs=st.sampled_from(JFS_BINS), \n file_name=st.just('file_to_rmr'))\n @precondition(lambda self: self.formatted and self.mounted )\n def rmr(self, juicefs, file_name):\n assume (self.greater_than_version_formatted(juicefs))\n assume (self.greater_than_version_mounted(juicefs))\n assume(not is_readonly(f'{JuicefsMachine.MOUNT_POINT}'))\n # ref: https://github.com/juicedata/juicefs/pull/2776\n assert(len(self.mounted_by) > 0)\n assume(version.parse('-'.join(self.mounted_by[-1].split('-')[1:])) >= version.parse('1.1.0-dev'))\n assume(version.parse('-'.join(juicefs.split('-')[1:])) >= version.parse('1.1.0-dev'))\n # TODO: should test upload delay.\n assume(get_upload_delay_seconds(JuicefsMachine.MOUNT_POINT) == 0)\n assert(os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'))\n print('start rmr')\n path = f'{JuicefsMachine.MOUNT_POINT}{file_name}'\n write_block(JuicefsMachine.MOUNT_POINT, path, 1048576, 3)\n os.system(f'ls -l {path}')\n assert(os.path.exists(path))\n run_cmd(f'stat {path}')\n options = [juicefs, 'rmr', path]\n run_jfs_cmd(options)\n # TODO: should uncomment the assert\n # assert(not os.path.exists(path))\n print('rmr succeed')\n\n @rule(juicefs=st.sampled_from(JFS_BINS), \n force=st.booleans())\n @precondition(lambda self: self.mounted)\n def umount(self, juicefs, force):\n assume (self.greater_than_version_formatted(juicefs))\n print('start umount')\n options = [juicefs, 'umount', JuicefsMachine.MOUNT_POINT]\n # don't force umount because it may not unmounted succeed.\n # if force:\n # options.append('--force')\n run_jfs_cmd(options)\n self.mounted = False\n print('umount succeed')\n\n @rule(juicefs=st.sampled_from(JFS_BINS))\n @precondition(lambda self: self.formatted and not self.mounted)\n def destroy(self, juicefs):\n assume (self.greater_than_version_formatted(juicefs))\n assume(run_cmd(f'{juicefs} --help | grep destroy') == 0)\n print('start destroy')\n output = subprocess.run([juicefs, 'status', JuicefsMachine.META_URL], check=True, stdout=subprocess.PIPE).stdout.decode()\n if 'get timestamp too slow' in output: \n # remove the first line caust it is tikv log message\n output = '\\n'.join(output.split('\\n')[1:]) \n print(f'status output: {output}')\n uuid = json.loads(output.replace(\"'\", '\"'))['Setting']['UUID']\n print(f'uuid is: {uuid}')\n assert len(uuid) != 0\n options = [juicefs, 'destroy', JuicefsMachine.META_URL, uuid]\n options.append('--force')\n run_jfs_cmd(options)\n self.formatted = False\n self.mounted = False\n self.mounted_by = []\n self.formatted_by = ''\n print('destroy succeed')\n\n @rule(file_name=st.sampled_from(['myfile1', 'myfile2']), \n data=st.binary() )\n @precondition(lambda self: self.mounted )\n def write_and_read(self, file_name, data):\n assume(not is_readonly(f'{JuicefsMachine.MOUNT_POINT}'))\n assert(os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'))\n print('start write and read')\n path = JuicefsMachine.MOUNT_POINT+file_name\n write_data(JuicefsMachine.MOUNT_POINT, path, data)\n with open(path, \"rb\") as f:\n result = f.read()\n assert str(result) == str(data)\n print('write and read succeed')\n \n def write_rand_files(self, path, seed):\n count = 50\n if os.path.isdir(path):\n shutil.rmtree(path)\n os.mkdir(path)\n fsrand = FsRandomizer(path, count, seed)\n fsrand.stdout = sys.stdout\n fsrand.stderr = sys.stderr\n fsrand.verbose = False\n fsrand.randomize()\n\n # @rule()\n @precondition(lambda self: self.mounted )\n def write_rand_files_and_compare(self):\n start = time.time()\n assume(not is_readonly(f'{JuicefsMachine.MOUNT_POINT}'))\n assert(os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'))\n seed = int(time.time())\n self.write_rand_files(JuicefsMachine.MOUNT_POINT+'fsrand', seed)\n self.write_rand_files('/tmp/fsrand', seed)\n tcmp = TreeComparator(JuicefsMachine.MOUNT_POINT+'fsrand', '/tmp/fsrand')\n tcmp.compare()\n res = len(tcmp.left_only) + len(tcmp.right_only) + \\\n len(tcmp.common_funny) + len(tcmp.funny_files) + len(tcmp.diff_files)\n if res > 0:\n raise Exception(\"compare failed\")\n os.system(f\"rm -rf {JuicefsMachine.MOUNT_POINT}/fsrand\")\n os.system(f\"rm -rf /tmp/fsrand\")\n print('write_rand_files_and_compare execution time:', time.time()-start, 'seconds')\n\n @rule(juicefs = st.sampled_from(JFS_BINS))\n @precondition(lambda self: self.formatted )\n def dump(self, juicefs):\n assume (self.greater_than_version_formatted(juicefs))\n # check this because of: https://github.com/juicedata/juicefs/issues/2717\n assume(juicefs in self.mounted_by)\n print('start dump')\n run_jfs_cmd([juicefs, 'dump', JuicefsMachine.META_URL, 'dump.json'])\n self.dumped_by = juicefs\n print('dump succeed')\n\n @rule(juicefs = st.sampled_from(JFS_BINS))\n @precondition(lambda self: self.formatted and os.path.exists('dump.json'))\n def load(self, juicefs):\n assume (self.greater_than_version_formatted(juicefs))\n assume (self.greater_than_version_dumped(juicefs))\n print('start load')\n if os.path.exists(JuicefsMachine.MOUNT_POINT) and os.path.exists(JuicefsMachine.MOUNT_POINT+'.accesslog'):\n run_cmd('umount %s'%JuicefsMachine.MOUNT_POINT)\n print(f'umount {JuicefsMachine.MOUNT_POINT} succeed')\n self.mounted = False\n flush_meta(JuicefsMachine.META_URL)\n run_jfs_cmd([juicefs, 'load', JuicefsMachine.META_URL, 'dump.json'])\n print('load succeed')\n options = [juicefs, 'config', JuicefsMachine.META_URL]\n if version.parse('-'.join(juicefs.split('-')[1:])) <= version.parse('1.0.0-rc1'):\n # use the latest version to change secret-key because rc1 has a bug for secret-key\n options[0] = JuicefsMachine.JFS_BINS[1]\n storage = get_storage(juicefs, JuicefsMachine.META_URL)\n if storage == 'minio':\n run_jfs_cmd([JuicefsMachine.JFS_BINS[1], 'config', JuicefsMachine.META_URL, '--access-key', 'minioadmin', '--secret-key', 'minioadmin'])\n elif storage == 'mysql':\n run_jfs_cmd([JuicefsMachine.JFS_BINS[1], 'config', JuicefsMachine.META_URL, '--access-key', 'root', '--secret-key', 'root'])\n elif storage == 'postgres':\n run_jfs_cmd([JuicefsMachine.JFS_BINS[1], 'config', JuicefsMachine.META_URL, '--access-key', 'postgres', '--secret-key', 'postgres'])\n \n os.remove('dump.json')\n\n @rule(juicefs=st.sampled_from(JFS_BINS))\n @precondition(lambda self: self.formatted)\n def fsck(self, juicefs):\n assume (self.greater_than_version_formatted(juicefs))\n assume(juicefs in self.mounted_by)\n print('start fsck')\n run_jfs_cmd([juicefs, 'fsck', JuicefsMachine.META_URL])\n print('fsck succeed')\n\n # @rule(juicefs=st.sampled_from(JFS_BINS),\n # block_size=st.integers(min_value=1, max_value=32),\n # big_file_size=st.integers(min_value=100, max_value=200),\n # small_file_size=st.integers(min_value=1, max_value=256),\n # small_file_count=st.integers(min_value=100, max_value=256), \n # threads=st.integers(min_value=1, max_value=100))\n @precondition(lambda self: self.mounted and False)\n def bench(self, juicefs, block_size, big_file_size, small_file_size, small_file_count, threads):\n assume (self.greater_than_version_formatted(juicefs))\n assert(os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'))\n print('start bench')\n run_cmd(f'df | grep {JuicefsMachine.MOUNT_POINT}')\n options = [juicefs, 'bench', JuicefsMachine.MOUNT_POINT]\n options.extend(['--block-size', str(block_size)])\n options.extend(['--big-file-size', str(big_file_size)])\n options.extend(['--small-file-size', str(small_file_size)])\n options.extend(['--small-file-count', str(small_file_count)])\n options.extend(['--threads', str(threads)])\n output = run_jfs_cmd(options)\n summary = output.decode('utf8').split('\\n')[2]\n expected = f'BlockSize: {block_size} MiB, BigFileSize: {big_file_size} MiB, SmallFileSize: {small_file_size} KiB, SmallFileCount: {small_file_count}, NumThreads: {threads}'\n assert summary == expected\n print('bench succeed')\n\n @rule(juicefs=st.sampled_from(JFS_BINS),\n threads=st.integers(min_value=1, max_value=100), \n background = st.booleans(), \n from_file = st.booleans(),\n directory = st.booleans() )\n @precondition(lambda self: self.mounted)\n def warmup(self, juicefs, threads, background, from_file, directory):\n assume (self.greater_than_version_formatted(juicefs))\n assume (self.greater_than_version_mounted(juicefs))\n assume(not is_readonly(f'{JuicefsMachine.MOUNT_POINT}'))\n assert(os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/.accesslog'))\n print('start warmup')\n clear_cache()\n options = [juicefs, 'warmup']\n options.extend(['--threads', str(threads)])\n if background:\n options.append('--background')\n if from_file:\n path_list = [JuicefsMachine.MOUNT_POINT+'file1', JuicefsMachine.MOUNT_POINT+'file2', JuicefsMachine.MOUNT_POINT+'file3']\n for filepath in path_list:\n if not os.path.exists(filepath):\n write_block(JuicefsMachine.MOUNT_POINT, filepath, 4096, 100)\n with open('file.list', 'w') as f:\n for path in path_list:\n f.write(path+'\\n')\n time.sleep(get_upload_delay_seconds(f'{JuicefsMachine.MOUNT_POINT}')+1)\n while(get_stage_blocks(JuicefsMachine.MOUNT_POINT) != 0):\n print('sleep for stage')\n time.sleep(1)\n options.extend(['--file', 'file.list'])\n else:\n if directory:\n options.append(JuicefsMachine.MOUNT_POINT)\n else:\n write_block(JuicefsMachine.MOUNT_POINT, f'{JuicefsMachine.MOUNT_POINT}/file_to_warmup', 1048576, 100)\n assert os.path.exists(f'{JuicefsMachine.MOUNT_POINT}/file_to_warmup')\n options.append(f'{JuicefsMachine.MOUNT_POINT}/file_to_warmup')\n \n run_jfs_cmd(options)\n # print(output)\n print('warmup succeed')\n # assert output.decode('utf8').split('\\n')[0].startswith('Warming up count: ')\n # assert output.decode('utf8').split('\\n')[0].startswith('Warming up bytes: ')\n\n @rule(\n juicefs = st.sampled_from(JFS_BINS), \n compact=st.booleans(), \n delete=st.booleans(),\n threads=st.integers(min_value=1, max_value=100) )\n @precondition(lambda self: self.formatted)\n def gc(self, juicefs, compact, delete, threads):\n assume (self.greater_than_version_formatted(juicefs))\n assume(juicefs in self.mounted_by)\n print('start gc')\n options = [juicefs, 'gc', JuicefsMachine.META_URL]\n if compact:\n options.append('--compact')\n if delete:\n options.append('--delete')\n options.extend(['--threads', str(threads)])\n run_jfs_cmd(options)\n # print(output)\n print('gc succeed')\n\n\n @rule(juicefs=st.sampled_from(JFS_BINS), \n get_timeout=st.integers(min_value=30, max_value=59), \n put_timeout=st.integers(min_value=30, max_value=59), \n io_retries=st.integers(min_value=5, max_value=15), \n max_uploads=st.integers(min_value=1, max_value=100), \n max_deletes=st.integers(min_value=1, max_value=100), \n buffer_size=st.integers(min_value=100, max_value=1000), \n upload_limit=st.integers(min_value=0, max_value=1000), \n download_limit=st.integers(min_value=0, max_value=1000), \n prefetch=st.integers(min_value=0, max_value=100), \n writeback=st.just(False),\n upload_delay=st.sampled_from([0, 2]), \n cache_dir=st.sampled_from(['cache1', 'cache2']),\n cache_size=st.integers(min_value=0, max_value=1024000), \n free_space_ratio=st.floats(min_value=0.1, max_value=0.5), \n cache_partial_only=st.booleans(),\n backup_meta=st.integers(min_value=300, max_value=1000),\n heartbeat=st.integers(min_value=5, max_value=30), \n read_only=st.booleans(),\n no_bgjob=st.booleans(),\n open_cache=st.integers(min_value=0, max_value=100),\n attr_cache=st.integers(min_value=1, max_value=10), \n entry_cache=st.integers(min_value=1, max_value=10), \n dir_entry_cache=st.integers(min_value=1, max_value=10), \n access_log=st.sampled_from(['accesslog1', 'accesslog2']),\n no_banner=st.booleans(),\n multi_buckets=st.booleans(), \n keep_etag=st.booleans(),\n umask=st.sampled_from(['022', '755']), \n metrics=st.sampled_from(['127.0.0.1:9567', '127.0.0.1:9568']), \n consul=st.sampled_from(['127.0.0.1:8500', '127.0.0.1:8501']), \n sub_dir=st.sampled_from(['dir1', 'dir2']),\n port=st.integers(min_value=9001, max_value=10000)\n )\n @precondition(lambda self: self.formatted and False)\n def gateway(self, juicefs, get_timeout, put_timeout, io_retries, max_uploads, max_deletes, buffer_size, upload_limit, \n download_limit, prefetch, writeback, upload_delay, cache_dir, cache_size, free_space_ratio, cache_partial_only, \n backup_meta,heartbeat, read_only, no_bgjob, open_cache, attr_cache, entry_cache, dir_entry_cache, access_log, \n no_banner, multi_buckets, keep_etag, umask, metrics, consul, sub_dir, port):\n assume (self.greater_than_version_formatted(juicefs))\n assume(not is_port_in_use(port))\n if JuicefsMachine.META_URL.startswith('badger://'):\n assume(not self.mounted)\n print('start gateway')\n os.environ['MINIO_ROOT_USER'] = 'admin'\n os.environ['MINIO_ROOT_PASSWORD'] = '12345678'\n options = [juicefs, 'gateway', JuicefsMachine.META_URL, f'localhost:{port}']\n \n options.extend(['--attr-cache', str(attr_cache)])\n options.extend(['--entry-cache', str(entry_cache)])\n options.extend(['--dir-entry-cache', str(dir_entry_cache)])\n options.extend(['--get-timeout', str(get_timeout)])\n options.extend(['--put-timeout', str(put_timeout)])\n options.extend(['--io-retries', str(io_retries)])\n options.extend(['--max-uploads', str(max_uploads)])\n if run_cmd(f'{juicefs} gateway --help | grep max-deletes') == 0:\n options.extend(['--max-deletes', str(max_deletes)])\n options.extend(['--buffer-size', str(buffer_size)])\n options.extend(['--upload-limit', str(upload_limit)])\n options.extend(['--download-limit', str(download_limit)])\n options.extend(['--prefetch', str(prefetch)])\n if writeback:\n options.append('--writeback')\n upload_delay = str(upload_delay)\n if version.parse('-'.join(juicefs.split('-')[1:])) <= version.parse('1.0.0-beta2'):\n upload_delay = upload_delay + 's'\n options.extend(['--upload-delay', upload_delay])\n options.extend(['--cache-dir', os.path.expanduser(f'~/.juicefs/{cache_dir}')])\n options.extend(['--access-log', os.path.expanduser(f'~/.juicefs/{access_log}')])\n options.extend(['--cache-size', str(cache_size)])\n options.extend(['--free-space-ratio', str(free_space_ratio)])\n if cache_partial_only:\n options.append('--cache-partial-only')\n backup_meta = str(backup_meta)\n if version.parse('-'.join(juicefs.split('-')[1:])) <= version.parse('1.0.0-beta2'):\n backup_meta = '1h0m0s'\n if run_cmd(f'{juicefs} gateway --help | grep backup-meta') == 0:\n options.extend(['--backup-meta', backup_meta])\n if run_cmd(f'{juicefs} gateway --help | grep heartbeat') == 0:\n options.extend(['--heartbeat', str(heartbeat)])\n if read_only:\n options.append('--read-only')\n if no_bgjob and run_cmd(f'{juicefs} gateway --help | grep no-bgjob') == 0:\n options.append('--no-bgjob')\n if no_banner:\n options.append('--no-banner')\n if multi_buckets and run_cmd(f'{juicefs} gateway --help | grep multi-buckets') == 0:\n options.append('--multi-buckets')\n if keep_etag and run_cmd(f'{juicefs} gateway --help | grep keep-etag') == 0:\n options.append('--keep-etag')\n if run_cmd(f'{juicefs} gateway --help | grep umask') == 0:\n options.extend(['--umask', umask])\n\n options.extend(['--open-cache', str(open_cache)])\n print(f'TODO: subdir:{sub_dir}')\n # options.extend('--subdir', str(sub_dir))\n if not is_port_in_use( int(metrics.split(':')[1])):\n options.extend(['--metrics', str(metrics)])\n # if run_cmd(f'{juicefs} mount --help | grep consul') == 0:\n # options.extend(['--consul', str(consul)])\n options.append('--no-usage-report')\n\n proc=subprocess.Popen(options)\n time.sleep(2.0)\n subprocess.Popen.kill(proc)\n print('gateway succeed')\n\n\n @rule(juicefs = st.sampled_from(JFS_BINS), \n port=st.integers(min_value=10001, max_value=11000)) \n @precondition(lambda self: self.formatted and False)\n def webdav(self, juicefs, port):\n assume (self.greater_than_version_formatted(juicefs))\n assert version.parse('-'.join(juicefs.split('-')[1:])) >= version.parse('-'.join(self.formatted_by.split('-')[1:]))\n assume (not is_port_in_use(port))\n if JuicefsMachine.META_URL.startswith('badger://'):\n assume(not self.mounted)\n print('start webdav')\n \n options = [juicefs, 'webdav', JuicefsMachine.META_URL, f'localhost:{port}']\n proc = subprocess.Popen(options)\n time.sleep(2.0)\n subprocess.Popen.kill(proc)\n print('webdav succeed')\n\n def greater_than_version_formatted(self, ver):\n print(f'ver is {ver}, formatted_by is {self.formatted_by}')\n if not self.formatted_by:\n return True\n return version.parse('-'.join(ver.split('-')[1:])) >= version.parse('-'.join(self.formatted_by.split('-')[1:]))\n\n def greater_than_version_dumped(self, ver):\n if not self.dumped_by:\n return True\n return version.parse('-'.join(ver.split('-')[1:])) >= version.parse('-'.join(self.dumped_by.split('-')[1:]))\n\n def greater_than_version_mounted(self, ver):\n for mounted_version in self.mounted_by:\n if version.parse('-'.join(ver.split('-')[1:])) < version.parse('-'.join(mounted_version.split('-')[1:])):\n return False\n return True\n\n\nTestJuiceFS = JuicefsMachine.TestCase\n\nif __name__ == \"__main__\":\n unittest.main(failfast=True)" }, { "path": ".github/scripts/upload_coverage_report.sh", "content": "#!/bin/bash\n\n# 参数检查\nif [ \"$#\" -ne 3 ]; then\n echo \"Usage: $0 \"\n exit 1\nfi\n\nCOVERAGE_FILE=$1\nUPLOAD_PATH=$2\nTOKEN=$3\nattempt=1\nmax_attempts=3\n\nwhile [ $attempt -le $max_attempts ]; do\n response=$(curl -w '%{http_code}' -s -o /dev/null --form \"file=@${COVERAGE_FILE}\" \"https://juicefs.com/upload-file-u80sdvuke/${UPLOAD_PATH}?token=${TOKEN}\")\n if [ \"$response\" -eq 200 ]; then\n echo \"Coverage Report: https://i.juicefs.io/ci-coverage/${UPLOAD_PATH}\"\n break\n else\n echo \"Upload attempt $attempt failed with status code $response. Retrying...\"\n attempt=$((attempt + 1))\n sleep 5 # 等待5秒钟后重试\n fi\ndone\n\nif [ \"$response\" -ne 200 ]; then\n echo \"Upload failed after $max_attempts attempts with status code $response\"\n exit 1\nfi" }, { "path": ".github/scripts/utils.py", "content": "import subprocess\ntry:\n __import__(\"minio\")\nexcept ImportError:\n subprocess.check_call([\"pip\", \"install\", \"minio\"])\nimport json\nimport os\nfrom posixpath import expanduser\nimport shutil\nimport subprocess\nimport sys\nimport time\nfrom urllib.parse import urlparse\nfrom minio import Minio\n\ndef flush_meta(meta_url:str):\n print(f'start flush meta: {meta_url}')\n if meta_url.startswith('sqlite3://'):\n path = meta_url[len('sqlite3://'):]\n if os.path.isfile(path):\n os.remove(path)\n print(f'remove meta file {path} succeed')\n elif meta_url.startswith('badger://'):\n path = meta_url[len('badger://'):]\n if os.path.isdir(path):\n shutil.rmtree(path)\n print(f'remove badger dir {path} succeed')\n elif meta_url.startswith('redis://') or meta_url.startswith('tikv://'):\n default_port = {\"redis\": 6379, \"tikv\": 2379}\n parsed = urlparse(meta_url)\n protocol = parsed.scheme\n host = parsed.hostname\n port = parsed.port if parsed.port else default_port[protocol]\n db = parsed.path.lstrip('/').split('/')[0]\n assert db\n print(f'flushing {protocol}://{host}:{port}/{db}')\n if protocol == 'redis':\n run_cmd(f'redis-cli -h {host} -p {port} -n {db} flushdb')\n elif protocol == 'tikv':\n # TODO: should only flush the specified db\n run_cmd(f'echo \"delall --yes\" |tcli -pd {host}:{port}')\n else:\n raise Exception(f'{protocol} not supported')\n print(f'flush {protocol}://{host}:{port}/{db} succeed')\n elif meta_url.startswith('mysql://'):\n create_mysql_db(meta_url)\n elif meta_url.startswith('postgres://'): \n create_postgres_db(meta_url)\n elif meta_url.startswith('fdb://'):\n # fdb:///home/runner/fdb.cluster?prefix=jfs2\n prefix = meta_url.split('?prefix=')[1] if '?prefix=' in meta_url else \"\"\n cluster_file = meta_url.split('fdb://')[1].split('?')[0]\n print(f'flushing fdb: cluster_file: {cluster_file}, prefix: {prefix}')\n run_cmd(f'echo \"writemode on; clearrange {prefix} {prefix}\\\\xff\" | fdbcli -C {cluster_file}')\n print(f'flush fdb succeed')\n else:\n raise Exception(f'{meta_url} not supported')\n print('flush meta succeed')\n\ndef create_mysql_db(meta_url):\n db_name = meta_url[8:].split('@')[1].split('/')[1].split('?')[0]\n user = meta_url[8:].split('@')[0].split(':')[0]\n password = meta_url[8:].split('@')[0].split(':')[1]\n if password: \n password = f'-p{password}'\n host_port= meta_url[8:].split('@')[1].split('/')[0].replace('(', '').replace(')', '')\n if ':' in host_port:\n host = host_port.split(':')[0]\n port = host_port.split(':')[1]\n else:\n host = host_port\n port = '3306'\n run_cmd(f'mysql -u{user} {password} -h {host} -P {port} -e \"drop database if exists {db_name}; create database {db_name};\"')\n\ndef create_postgres_db(meta_url):\n os.environ['PGPASSWORD'] = 'postgres'\n db_name = meta_url[8:].split('@')[1].split('/')[1]\n if '?' in db_name:\n db_name = db_name.split('?')[0]\n run_cmd(f'printf \"\\set AUTOCOMMIT on\\ndrop database if exists {db_name}; create database {db_name}; \" | psql -U postgres -h localhost')\n\ndef clear_storage(storage, bucket, volume):\n print('start clear storage')\n if storage == 'file':\n storage_dir = os.path.join(bucket, volume) \n if os.path.exists(storage_dir):\n try:\n shutil.rmtree(storage_dir)\n print(f'remove cache dir {storage_dir} succeed')\n except OSError as e:\n print(\"Error: %s : %s\" % (storage_dir, e.strerror))\n elif storage == 'minio':\n from urllib.parse import urlparse\n url = urlparse(bucket)\n c = Minio('localhost:9000', access_key='minioadmin', secret_key='minioadmin', secure=False)\n bucket_name = url.path[1:]\n while c.bucket_exists(bucket_name) and list(c.list_objects(bucket_name)) :\n print(f'try to remove bucket {url.path[1:]}')\n result = run_cmd(f'mc rm --recursive --force myminio/{bucket_name}')\n if result != 0:\n raise Exception(f'remove {bucket_name} failed')\n if c.bucket_exists(url.path[1:]) and list(c.list_objects(bucket_name)):\n time.sleep(1)\n print(f'remove bucket {bucket_name} succeed')\n if c.bucket_exists(bucket_name):\n assert not list(c.list_objects(bucket_name))\n elif storage == 'mysql':\n db_name = bucket.split('/')[-1]\n run_cmd(f'mysql -uroot -proot -h localhost -P 3306 -e \"drop database if exists {db_name};create database {db_name};\"')\n elif storage == 'postgres':\n db_name = bucket.split('/')[1]\n if '?' in db_name:\n db_name = db_name.split('?')[0]\n run_cmd(f'printf \"\\set AUTOCOMMIT on\\ndrop database if exists {db_name}; create database {db_name}; \" | psql -U postgres -h localhost')\n print('clear storage succeed')\n\n\ndef clear_cache():\n run_cmd('sudo rm -rf /var/jfsCache')\n run_cmd(f'sudo rm -rf {os.path.expanduser(\"~/.juicefs/cache\")}')\n if sys.platform.startswith('linux') :\n os.system('sudo bash -c \"echo 3> /proc/sys/vm/drop_caches\"')\n\ndef is_readonly(filesystem):\n if not os.path.exists(f'{filesystem}/.config'):\n return False\n with open(f'{filesystem}/.config') as f:\n config = json.load(f)\n return config['Meta']['ReadOnly']\n\ndef get_upload_delay_seconds(filesystem):\n if not os.path.exists(f'{filesystem}/.config'):\n return 0\n with open(f'{filesystem}/.config') as f:\n config = json.load(f)\n return config['Chunk']['UploadDelay']/1000000000\n \ndef get_stage_blocks(filesystem):\n try:\n ps = subprocess.Popen(('cat', f'{filesystem}/.stats'), stdout=subprocess.PIPE)\n output = subprocess.check_output(('grep', 'juicefs_staging_blocks'), stdin=ps.stdout)\n ps.wait()\n return int(output.decode().split()[1])\n except subprocess.CalledProcessError:\n print('get_stage_blocks: no juicefs_staging_blocks find')\n return 0\n\ndef write_data(filesystem, path, data):\n with open(path, \"wb\") as f:\n f.write(data)\n retry = get_upload_delay_seconds(filesystem) + 10\n while get_stage_blocks(filesystem) != 0 and retry > 0:\n print('sleep for stage')\n retry = retry - 1\n time.sleep(1)\n # assert get_stage_blocks(filesystem) == 0\n\ndef write_block(filesystem, filepath, bs, count):\n run_cmd(f'dd if=/dev/urandom of={filepath} bs={bs} count={count}')\n retry = get_upload_delay_seconds(filesystem) + 10\n while get_stage_blocks(filesystem) != 0 and retry > 0:\n print('sleep for stage')\n retry = retry - 1\n time.sleep(1)\n # assert get_stage_blocks(filesystem) == 0\n\ndef mdtest(filesystem, meta_url):\n juicefs_new = './'+os.environ.get('NEW_JFS_BIN')\n cwd = os.getcwd()\n if not os.path.exists(f'{filesystem}/{juicefs_new}'):\n run_cmd(f'ln -s {cwd}/{juicefs_new} {filesystem}/{juicefs_new}')\n os.chdir(filesystem)\n run_jfs_cmd(f'{juicefs_new} mdtest {meta_url} mdtest --dirs 5 --depth 2 --files 5 --threads 5 --write 8192'.split())\n os.chdir(cwd)\n time.sleep(get_upload_delay_seconds(filesystem)+1)\n retry = 5\n while get_stage_blocks(filesystem) != 0 and retry > 0:\n print('sleep for stage')\n retry = retry - 1\n time.sleep(1)\n assert os.path.exists(filesystem+'mdtest')\n\ndef run_jfs_cmd( options):\n # options.append('--debug')\n print('run_jfs_cmd:'+' '.join(options))\n with open(os.path.expanduser('~/command.log'), 'a') as f:\n f.write(' '.join(options).replace('/home/runner', '~'))\n f.write('\\n')\n try:\n output = subprocess.run(options, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as e:\n print(f': subprocess run error, return code: {e.returncode} , error message: {e.output.decode()}')\n raise Exception('subprocess run error')\n print(f'run_jfs_cmd return code: {output.returncode}, output: {output.stdout.decode()}')\n print('run_jfs_cmd succeed')\n return output.stdout.decode()\n\ndef run_cmd(command):\n print('run_cmd:'+command)\n if '|' in command or '\"' in command:\n return os.system(command)\n try:\n output = subprocess.run(command.split(), check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as e:\n print(f': subprocess run error, return code: {e.returncode} , error message: {e.output.decode()}')\n return e.returncode\n if output.stdout:\n print(output.stdout.decode())\n print('run_cmd succeed')\n return output.returncode\n\ndef is_port_in_use(port: int) -> bool:\n import socket\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n return s.connect_ex(('localhost', port)) == 0\n\ndef get_storage(juicefs, meta_url):\n output = subprocess.run([juicefs, 'status', meta_url], check=True, stdout=subprocess.PIPE).stdout.decode()\n if 'get timestamp too slow' in output: \n # remove the first line caust it is tikv log message\n output = '\\n'.join(output.split('\\n')[1:])\n print(f'status output: {output}')\n storage = json.loads(output.replace(\"'\", '\"'))['Setting']['Storage']\n return storage\n\nif __name__ == \"__main__\":\n run_jfs_cmd(['./juicefs-1.1.0-dev', 'rmr', '/tmp/sync-test/file_to_rmr', '--debug'])" }, { "path": ".github/scripts/wins_fs_test.py", "content": "import os\nimport sys\nimport time\nimport shutil\nimport random\nimport string\nimport threading\nimport unittest\nfrom pathlib import Path\n\nclass WindowsFSTest(unittest.TestCase):\n def setUp(self):\n self.test_dir = \"Z:\\\\test_fs\"\n self.ensure_clean_dir(self.test_dir)\n \n def tearDown(self):\n if os.path.exists(self.test_dir):\n shutil.rmtree(self.test_dir, ignore_errors=True)\n \n def ensure_clean_dir(self, path):\n if os.path.exists(path):\n shutil.rmtree(path, ignore_errors=True)\n os.makedirs(path)\n \n def random_string(self, length=10):\n return ''.join(random.choices(string.ascii_letters + string.digits, k=length))\n \n def test_basic_operations(self):\n test_file = os.path.join(self.test_dir, \"test.txt\")\n content = \"Hello, Windows!\"\n with open(test_file, 'w') as f:\n f.write(content)\n \n with open(test_file, 'r') as f:\n self.assertEqual(f.read(), content)\n \n new_file = os.path.join(self.test_dir, \"new.txt\")\n os.rename(test_file, new_file)\n self.assertTrue(os.path.exists(new_file))\n \n os.remove(new_file)\n self.assertFalse(os.path.exists(new_file))\n \n def test_rename_case_change(self):\n test_file = os.path.join(self.test_dir, \"a\")\n content = \"Hello, Windows!\"\n with open(test_file, 'w') as f:\n f.write(content)\n new_file_lower = os.path.join(self.test_dir, \"A\")\n os.rename(test_file, new_file_lower)\n self.assertTrue(os.path.exists(new_file_lower))\n new_file_upper = os.path.join(self.test_dir, \"a\")\n os.rename(new_file_lower, new_file_upper)\n self.assertTrue(os.path.exists(new_file_upper))\n os.remove(new_file_upper)\n\n def test_directory_operations(self):\n nested_dir = os.path.join(self.test_dir, \"dir1\", \"dir2\", \"dir3\")\n os.makedirs(nested_dir)\n \n self.assertTrue(os.path.exists(nested_dir))\n \n test_file = os.path.join(nested_dir, \"test.txt\")\n Path(test_file).touch()\n \n files = list(Path(self.test_dir).rglob(\"*\"))\n self.assertTrue(len(files) > 0)\n \n new_dir = os.path.join(self.test_dir, \"new_dir\")\n shutil.move(os.path.join(self.test_dir, \"dir1\"), new_dir)\n self.assertTrue(os.path.exists(new_dir))\n \n def test_concurrent_operations(self):\n file_count = 10\n thread_count = 5\n \n def write_files(start_idx):\n for i in range(start_idx, start_idx + file_count):\n file_path = os.path.join(self.test_dir, f\"concurrent_{i}.txt\")\n with open(file_path, 'w') as f:\n f.write(self.random_string(100))\n \n threads = []\n for i in range(thread_count):\n t = threading.Thread(target=write_files, args=(i * file_count,))\n threads.append(t)\n t.start()\n \n for t in threads:\n t.join()\n \n files = os.listdir(self.test_dir)\n self.assertEqual(len(files), file_count * thread_count)\n \n def test_special_characters(self):\n special_chars = [\n \"test with spaces\",\n \"test_with_unicode_中文\",\n \"test_with_symbols_!@#$%\",\n \"test.with.multiple.dots\"\n ]\n \n for name in special_chars:\n file_path = os.path.join(self.test_dir, name)\n with open(file_path, 'w') as f:\n f.write(\"test\")\n self.assertTrue(os.path.exists(file_path))\n with open(file_path, 'r') as f:\n self.assertEqual(f.read(), \"test\")\n \n def test_large_files(self):\n large_file = os.path.join(self.test_dir, \"large_file.dat\")\n size_mb = 10\n chunk_size = 1024 * 1024 # 1MB\n \n with open(large_file, 'wb') as f:\n for _ in range(size_mb):\n f.write(os.urandom(chunk_size))\n \n self.assertEqual(os.path.getsize(large_file), size_mb * chunk_size)\n \n with open(large_file, 'rb') as f:\n chunks = 0\n while f.read(chunk_size):\n chunks += 1\n self.assertEqual(chunks, size_mb)\n \n def test_file_attributes(self):\n test_file = os.path.join(self.test_dir, \"attrs.txt\")\n with open(test_file, 'w') as f:\n f.write(\"test\")\n \n \n os.system(f'attrib +R \"{test_file}\"')\n \n with self.assertRaises(PermissionError):\n with open(test_file, 'w') as f:\n f.write(\"new content\")\n \n os.system(f'attrib -R \"{test_file}\"')\n @unittest.skip(\"Windows Do not support\")\n def test_symlinks(self):\n source_dir_root = os.path.join(self.test_dir, \"source\")\n link_dir_root = os.path.join(self.test_dir, \"links\")\n os.makedirs(source_dir_root)\n os.makedirs(link_dir_root)\n \n source_file = os.path.join(source_dir_root, \"source_file.txt\")\n source_dir = os.path.join(source_dir_root, \"source_dir\")\n with open(source_file, 'w') as f:\n f.write(\"test content\")\n os.makedirs(source_dir)\n \n link_file = os.path.join(link_dir_root, \"link_file.txt\")\n os.symlink(source_file, link_file)\n self.assertTrue(os.path.exists(link_file))\n with open(link_file, 'r') as f:\n self.assertEqual(f.read(), \"test content\")\n \n link_dir = os.path.join(link_dir_root, \"link_dir\")\n os.symlink(source_dir, link_dir, target_is_directory=True)\n self.assertTrue(os.path.exists(link_dir))\n \n link_test_file = os.path.join(link_dir, \"test.txt\")\n with open(link_test_file, 'w') as f:\n f.write(\"test through link\")\n \n source_test_file = os.path.join(source_dir, \"test.txt\")\n self.assertTrue(os.path.exists(source_test_file))\n with open(source_test_file, 'r') as f:\n self.assertEqual(f.read(), \"test through link\")\n \n os.remove(source_file)\n self.assertFalse(os.path.exists(link_file))\n\n def test_long_paths(self):\n deep_dir = self.test_dir\n for i in range(10): \n deep_dir = os.path.join(deep_dir, f\"dir_{i}\")\n \n os.makedirs(deep_dir, exist_ok=True)\n test_file = os.path.join(deep_dir, \"test.txt\")\n \n with open(test_file, 'w') as f:\n f.write(\"test\")\n \n self.assertTrue(os.path.exists(test_file))\n\nif __name__ == '__main__':\n unittest.main(verbosity=2)" }, { "path": ".github/workflows/bash/rm_fs", "content": "gf01 growfiles -W gf01 -b -e 1 -u -i 0 -L 20 -w -C 1 -l -I r -T 10 -f glseek20 -S 2 -d $TMPDIR\ngf02 growfiles -W gf02 -b -e 1 -L 10 -i 100 -I p -S 2 -u -f gf03_ -d $TMPDIR\ngf03 growfiles -W gf03 -b -e 1 -g 1 -i 1 -S 150 -u -f gf05_ -d $TMPDIR\ngf04 growfiles -W gf04 -b -e 1 -g 4090 -i 500 -t 39000 -u -f gf06_ -d $TMPDIR\ngf05 growfiles -W gf05 -b -e 1 -g 5000 -i 500 -t 49900 -T10 -c9 -I p -u -f gf07_ -d $TMPDIR\ngf06 growfiles -W gf06 -b -e 1 -u -r 1-5000 -R 0--1 -i 0 -L 30 -C 1 -f g_rand10 -S 2 -d $TMPDIR\ngf07 growfiles -W gf07 -b -e 1 -u -r 1-5000 -R 0--2 -i 0 -L 30 -C 1 -I p -f g_rand13 -S 2 -d $TMPDIR\ngf08 growfiles -W gf08 -b -e 1 -u -r 1-5000 -R 0--2 -i 0 -L 30 -C 1 -f g_rand11 -S 2 -d $TMPDIR\ngf09 growfiles -W gf09 -b -e 1 -u -r 1-5000 -R 0--1 -i 0 -L 30 -C 1 -I p -f g_rand12 -S 2 -d $TMPDIR\ngf10 growfiles -W gf10 -b -e 1 -u -r 1-5000 -i 0 -L 30 -C 1 -I l -f g_lio14 -S 2 -d $TMPDIR\ngf11 growfiles -W gf11 -b -e 1 -u -r 1-5000 -i 0 -L 30 -C 1 -I L -f g_lio15 -S 2 -d $TMPDIR\ngf12 mkfifo $TMPDIR/gffifo17; growfiles -b -W gf12 -e 1 -u -i 0 -L 30 $TMPDIR/gffifo17\ngf13 mkfifo $TMPDIR/gffifo18; growfiles -b -W gf13 -e 1 -u -i 0 -L 30 -I r -r 1-4096 $TMPDIR/gffifo18\ngf14 growfiles -W gf14 -b -e 1 -u -i 0 -L 20 -w -l -C 1 -T 10 -f glseek19 -S 2 -d $TMPDIR\ngf15 growfiles -W gf15 -b -e 1 -u -r 1-49600 -I r -u -i 0 -L 120 -f Lgfile1 -d $TMPDIR\ngf16 growfiles -W gf16 -b -e 1 -i 0 -L 120 -u -g 4090 -T 101 -t 408990 -l -C 10 -c 1000 -S 10 -f Lgf02_ -d $TMPDIR\ngf17 growfiles -W gf17 -b -e 1 -i 0 -L 120 -u -g 5000 -T 101 -t 499990 -l -C 10 -c 1000 -S 10 -f Lgf03_ -d $TMPDIR\ngf18 growfiles -W gf18 -b -e 1 -i 0 -L 120 -w -u -r 10-5000 -I r -l -S 2 -f Lgf04_ -d $TMPDIR\ngf19 growfiles -W gf19 -b -e 1 -g 5000 -i 500 -t 49900 -T10 -c9 -I p -o O_RDWR,O_CREAT,O_TRUNC -u -f gf08i_ -d $TMPDIR\ngf20 growfiles -W gf20 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 1-256000:512 -R 512-256000 -T 4 -f gfbigio-$$ -d $TMPDIR\ngf21 growfiles -W gf21 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -T 10 -t 20480 -f gf-bld-$$ -d $TMPDIR\ngf22 growfiles -W gf22 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -T 10 -t 20480 -f gf-bldf-$$ -d $TMPDIR\ngf23 growfiles -W gf23 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 512-64000:1024 -R 1-384000 -T 4 -f gf-inf-$$ -d $TMPDIR\ngf24 growfiles -W gf24 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -f gf-jbld-$$ -d $TMPDIR\ngf25 growfiles -W gf25 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 1024000-2048000:2048 -R 4095-2048000 -T 1 -f gf-large-gs-$$ -d $TMPDIR\ngf26 growfiles -W gf26 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 128-32768:128 -R 512-64000 -T 4 -f gfsmallio-$$ -d $TMPDIR\ngf27 growfiles -W gf27 -b -D 0 -w -g 8b -C 1 -b -i 1000 -u -f gfsparse-1-$$ -d $TMPDIR\ngf28 growfiles -W gf28 -b -D 0 -w -g 16b -C 1 -b -i 1000 -u -f gfsparse-2-$$ -d $TMPDIR\ngf29 growfiles -W gf29 -b -D 0 -r 1-4096 -R 0-33554432 -i 0 -L 60 -C 1 -u -f gfsparse-3-$$ -d $TMPDIR\ngf30 growfiles -W gf30 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -o O_RDWR,O_CREAT,O_SYNC -g 20480 -T 10 -t 20480 -f gf-sync-$$ -d $TMPDIR\nrwtest01 export LTPROOT; rwtest -N rwtest01 -c -q -i 60s -f sync 10%25000:$TMPDIR/rw-sync-$$\nrwtest02 export LTPROOT; rwtest -N rwtest02 -c -q -i 60s -f buffered 10%25000:$TMPDIR/rw-buffered-$$\nrwtest03 export LTPROOT; rwtest -N rwtest03 -c -q -i 60s -n 2 -f buffered -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-buff-$$\nrwtest04 export LTPROOT; rwtest -N rwtest04 -c -q -i 60s -n 2 -f sync -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-sync-$$\nrwtest05 export LTPROOT; rwtest -N rwtest05 -c -q -i 50 -T 64b 500b:$TMPDIR/rwtest01%f\niogen01 export LTPROOT; rwtest -N iogen01 -i 120s -s read,write -Da -Dv -n 2 500b:$TMPDIR/doio.f1.$$ 1000b:$TMPDIR/doio.f2.$$\nquota_remount_test01 quota_remount_test01.sh\nisofs isofs.sh\nfs_fill fs_fill" }, { "path": ".github/workflows/bash/rm_list.sh", "content": "#!/bin/bash\nLIST=`cat $1`\n\nfor LINE in $LIST; do\n # should remove empty line and comment line\n sed -i -e \"\\!^${LINE}.*!d\" -e \"\\!^#!d\" -e \"\\!^\\s*\\$!d\" $2\ndone\n\n" }, { "path": ".github/workflows/bash/rm_syscalls", "content": "alarm02 alarm02\nalarm03 alarm03\nalarm05 alarm05\nalarm06 alarm06\nalarm07 alarm07\nbind01 bind01\nbind02 bind02\nbind03 bind03\nbind04 bind04\nbind05 bind05\nbind06 bind06\nbpf_prog05 bpf_prog05\ncacheflush01 cacheflush01\nchown01_16 chown01_16\nchown02_16 chown02_16\nchown03_16 chown03_16\nchown04_16 chown04_16\nchown05_16 chown05_16\nclock_adjtime01 clock_adjtime01\nclock_adjtime02 clock_adjtime02\nclock_getres01 clock_getres01\nclock_nanosleep01 clock_nanosleep01\nclock_nanosleep02 clock_nanosleep02\nclock_nanosleep03 clock_nanosleep03\nclock_nanosleep04 clock_nanosleep04\nclock_gettime01 clock_gettime01\nclock_gettime02 clock_gettime02\nclock_gettime03 clock_gettime03\nclock_gettime04 clock_gettime04\nleapsec01 leapsec01\nclock_settime01 clock_settime01\nclock_settime02 clock_settime02\nclock_settime03 clock_settime03\nclose_range01 close_range01\nclose_range02 close_range02\nfallocate06 fallocate06\nfanotify01 fanotify01\nfanotify02 fanotify02\nfanotify03 fanotify03\nfanotify04 fanotify04\nfanotify05 fanotify05\nfanotify06 fanotify06\nfanotify07 fanotify07\nfanotify08 fanotify08\nfanotify09 fanotify09\nfanotify10 fanotify10\nfanotify11 fanotify11\nfanotify12 fanotify12\nfanotify13 fanotify13\nfanotify14 fanotify14\nfanotify15 fanotify15\nfanotify16 fanotify16\nfanotify17 fanotify17\nfanotify18 fanotify18\nfanotify19 fanotify19\nfchown01_16 fchown01_16\nfchown02_16 fchown02_16\nfchown03_16 fchown03_16\nfchown04_16 fchown04_16\nfchown05_16 fchown05_16\nfcntl06 fcntl06\nfcntl06_64 fcntl06_64\nfork01 fork01\nfork02 fork02\nfork03 fork03\nfork04 fork04\nfork05 fork05\nfork06 fork06\nfork07 fork07\nfork08 fork08\nfork09 fork09\nfork10 fork10\nfork11 fork11\nfork13 fork13 -i 1000000\nfork14 fork14\ngetegid01_16 getegid01_16\ngetegid02_16 getegid02_16\ngeteuid01_16 geteuid01_16\ngeteuid02_16 geteuid02_16\ngetgid01_16 getgid01_16\ngetgid03_16 getgid03_16\ngetgroups01_16 getgroups01_16\ngetgroups03_16 getgroups03_16\ngetresgid01_16 getresgid01_16\ngetresgid02_16 getresgid02_16\ngetresgid03_16 getresgid03_16\ngetresuid01_16 getresuid01_16\ngetresuid02_16 getresuid02_16\ngetresuid03_16 getresuid03_16\ngetrusage04 getrusage04\ngettimeofday01 gettimeofday01\ngettimeofday02 gettimeofday02\ngetuid01_16 getuid01_16\ngetuid03_16 getuid03_16\nioctl03 ioctl03\nioctl_sg01 ioctl_sg01\nfanotify16 fanotify16\nfanotify18 fanotify18\nfanotify19 fanotify19\nkeyctl01 keyctl01\nkeyctl02 keyctl02\nkeyctl03 keyctl03\nkeyctl04 keyctl04\nkeyctl05 keyctl05\nkeyctl06 keyctl06\nkeyctl07 keyctl07\nkeyctl08 keyctl08\nkill02 kill02\nkill03 kill03\nkill05 kill05\nkill06 kill06\nkill07 kill07\nkill08 kill08\nkill09 kill09\nkill10 kill10\nkill11 kill11\nkill12 kill12\nkill13 kill13\nlchown01_16 lchown01_16\nlchown02_16 lchown02_16\nlchown03_16 lchown03_16\nmbind02 mbind02\nmbind03 mbind03\nmbind04 mbind04\nmigrate_pages02 migrate_pages02\nmigrate_pages03 migrate_pages03\nmodify_ldt01 modify_ldt01\nmodify_ldt02 modify_ldt02\nmodify_ldt03 modify_ldt03\nmove_pages01 move_pages01\nmove_pages02 move_pages02\nmove_pages03 move_pages03\nmove_pages04 move_pages04\nmove_pages05 move_pages05\nmove_pages06 move_pages06\nmove_pages07 move_pages07\nmove_pages09 move_pages09\nmove_pages10 move_pages10\nmove_pages11 move_pages11\nmove_pages12 move_pages12\nmsgctl05 msgctl05\nmsgstress04 msgstress04\nnanosleep01 nanosleep01\nnanosleep02 nanosleep02\nnanosleep04 nanosleep04\nopenat201 openat201\nopenat202 openat202\nopenat203 openat203\nmadvise06 madvise06\nmadvise09 madvise09\npselect01 pselect01\npselect01_64 pselect01_64\nptrace04 ptrace04\nquotactl01 quotactl01\nquotactl04 quotactl04\nquotactl06 quotactl06\nreaddir21 readdir21\nrecvmsg03 recvmsg03\nrt_sigaction01 rt_sigaction01\nrt_sigaction02 rt_sigaction02\nrt_sigaction03 rt_sigaction03\nrt_sigprocmask01 rt_sigprocmask01\nrt_sigprocmask02 rt_sigprocmask02\nrt_sigqueueinfo01 rt_sigqueueinfo01\nrt_sigsuspend01 rt_sigsuspend01\nrt_sigtimedwait01 rt_sigtimedwait01\nrt_tgsigqueueinfo01 rt_tgsigqueueinfo01\nsbrk03 sbrk03\nselect02 select02\nsemctl08 semctl08\nsemctl09 semctl09\nsendfile09_64 sendfile09_64\nset_mempolicy01 set_mempolicy01\nset_mempolicy02 set_mempolicy02\nset_mempolicy03 set_mempolicy03\nset_mempolicy04 set_mempolicy04\nset_thread_area01 set_thread_area01\nsetfsgid01_16 setfsgid01_16\nsetfsgid02_16 setfsgid02_16\nsetfsgid03_16 setfsgid03_16\nsetfsuid01_16 setfsuid01_16\nsetfsuid02_16 setfsuid02_16\nsetfsuid03_16 setfsuid03_16\nsetfsuid04_16 setfsuid04_16\nsetgid01_16 setgid01_16\nsetgid02_16 setgid02_16\nsetgid03_16 setgid03_16\nsgetmask01 sgetmask01\nsetgroups01_16 setgroups01_16\nsetgroups02_16 setgroups02_16\nsetgroups03_16 setgroups03_16\nsetgroups04_16 setgroups04_16\nsetregid01_16 setregid01_16\nsetregid02_16 setregid02_16\nsetregid03_16 setregid03_16\nsetregid04_16 setregid04_16\nsetresgid01_16 setresgid01_16\nsetresgid02_16 setresgid02_16\nsetresgid03_16 setresgid03_16\nsetresgid04_16 setresgid04_16\nsetresuid01_16 setresuid01_16\nsetresuid02_16 setresuid02_16\nsetresuid03_16 setresuid03_16\nsetresuid04_16 setresuid04_16\nsetresuid05_16 setresuid05_16\nsetreuid01_16 setreuid01_16\nsetreuid02_16 setreuid02_16\nsetreuid03_16 setreuid03_16\nsetreuid04_16 setreuid04_16\nsetreuid05_16 setreuid05_16\nsetreuid06_16 setreuid06_16\nsetreuid07_16 setreuid07_16\nsetsockopt06 setsockopt06\nsetsockopt07 setsockopt07\nsetuid01_16 setuid01_16\nsetuid03_16 setuid03_16\nsetuid04_16 setuid04_16\nshmctl05 shmctl05\nshmctl06 shmctl06\nsocketcall01 socketcall01\nsocketcall02 socketcall02\nsocketcall03 socketcall03\nssetmask01 ssetmask01\nswapoff01 swapoff01\nswapoff02 swapoff02\nswapon01 swapon01\nswapon02 swapon02\nswapon03 swapon03\nswitch01 endian_switch01\nsysinfo03 sysinfo03\nsyslog01 syslog01\nsyslog02 syslog02\nsyslog03 syslog03\nsyslog04 syslog04\nsyslog05 syslog05\nsyslog06 syslog06\nsyslog07 syslog07\nsyslog08 syslog08\nsyslog09 syslog09\nsyslog10 syslog10\nsyslog11 syslog11\nsyslog12 syslog12\ntimes03 times03\ntimerfd04 timerfd04\ntimerfd_settime02 timerfd_settime02\nperf_event_open02 perf_event_open02\nstatx07 statx07\nio_uring02 io_uring02\nioctl_loop05 ioctl_loop05\n# all local filesystems\nchdir01 chdir01\ncopy_file_range01 copy_file_range01\nfallocate04 fallocate04\nfallocate05 fallocate05\nfdatasync03 fdatasync03\nfgetxattr01 fgetxattr01\nfremovexattr01 fremovexattr01\nfremovexattr02 fremovexattr02\nfsconfig01 fsconfig01\nfsetxattr01 fsetxattr01\nfsmount01 fsmount01\nfsmount02 fsmount02\nfsopen01 fsopen01\nfspick01 fspick01\nfspick02 fspick02\nfsync01 fsync01\nfsync04 fsync04\nlremovexattr01 lremovexattr01\nmove_mount01 move_mount01\nmove_mount02 move_mount02\nmsync04 msync04\nopen_tree01 open_tree01\nopen_tree02 open_tree02\npreadv03 preadv03\npreadv03_64 preadv03_64\npreadv203 preadv203\npreadv203_64 preadv203_64\npwritev03 pwritev03\npwritev03_64 pwritev03_64\nsetxattr01 setxattr01\nstatx04 statx04\nsync01 sync01\nsync_file_range02 sync_file_range02\nsyncfs01 syncfs01\nutime03 utime03\nwritev03 writev03\n# cross mount (may fail on multi-zones meta)\ninotify03 inotify03\ninotify07 inotify07\ninotify08 inotify08\nlchown03 lchown03\nlinkat02 linkat02\nmadvise01 madvise01\nmknod07 mknod07\nmknodat02 mknodat02\nmmap16 mmap16\nmount03 mount03\nmount05 mount05\nmount06 mount06\nopen12 open12\npivot_root01 pivot_root01\nreadahead02 readahead02\nrename11 rename11\nrenameat01 renameat01\nstatx05 statx05\numount01 umount01\numount02 umount02\numount03 umount03\numount2_01 umount2_01\numount2_02 umount2_02\numount2_03 umount2_03\nutime06 utime06\n# not supported\nioctl_loop05 ioctl_loop05\nfcntl17 fcntl17\nfcntl17_64 fcntl17_64\nsetxattr03 setxattr03\ngetxattr05 getxattr05\n# not stable\nfinit_module02 finit_module02\nmsgstress03 msgstress03\nkill11 kill11\n# failed after upgrade github runner from ubuntu 20.04 to 22.04\ninotify02 inotify02\nioprio_set03 ioprio_set03" }, { "path": ".github/workflows/cache.yml", "content": "name: \"cache\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/cache.yml'\n - '**/cache.sh'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/cache.yml'\n - '**/cache.sh'\n schedule:\n - cron: '30 20 * * *'\n workflow_dispatch:\n\njobs:\n cache:\n timeout-minutes: 60\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Remove unused software\n timeout-minutes: 3\n continue-on-error: true\n run: |\n echo \"before remove unused software\"\n sudo df -h\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n echo \"after remove unused software\"\n sudo df -h\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n\n - name: Test\n run: |\n sudo GOCOVERDIR=$(pwd)/cover .github/scripts/cache.sh\n\n - name: Check juicefs log\n if: always()\n run: |\n sudo .github/scripts/check_juicefs_log.sh\n \n - name: Check /tmp/juicefs.log\n if: always()\n run: |\n [[ -f /tmp/juicefs.log ]] && sudo tail -n 1000 /tmp/juicefs.log || true\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\" \n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/cancel_outdate_runs.yml", "content": "name: cancel_outdate_runs\non:\n pull_request:\n branches:\n - main\n - release**\n\njobs:\n cancel-outdate-runs:\n if: github.event.pull_request.head.repo.full_name == github.repository\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/checkout@v3\n with :\n fetch-depth: 1\n\n - name: mount jfs dir\n timeout-minutes: 3\n run: |\n sudo mkdir -p /root/.juicefs\n sudo wget -q s.juicefs.com/static/Linux/mount -O /root/.juicefs/jfsmount \n sudo chmod +x /root/.juicefs/jfsmount\n sudo curl -s -L https://juicefs.com/static/juicefs -o /usr/local/bin/juicefs && sudo chmod +x /usr/local/bin/juicefs\n sudo juicefs auth ci-coverage --access-key ${{ secrets.CI_COVERAGE_AWS_AK }} --secret-key ${{ secrets.CI_COVERAGE_AWS_SK }} --token ${{ secrets.CI_COVERAGE_AWS_TOKEN }} --encrypt-keys\n sudo juicefs mount ci-coverage --subdir juicefs/cancel-outdate-runs /jfs --allow-other\n \n - name: Get previous head_sha\n timeout-minutes: 1\n run: |\n echo get previous head sha from /jfs/${{ github.event.pull_request.number }}/head_sha\n previous_head_sha=/jfs/${{ github.event.pull_request.number }}/head_sha\n if [ ! -f ${previous_head_sha} ]; then\n echo \"no previous head sha found\"\n exit 0\n else\n previous_head_sha=$(cat ${previous_head_sha})\n echo \"previous head sha is ${previous_head_sha}\"\n echo \"previous_head_sha=${previous_head_sha}\" >> $GITHUB_ENV\n fi\n\n - name: Save head_sha \n timeout-minutes: 1\n run: |\n mkdir -p /jfs/${{ github.event.pull_request.number }}\n echo ${{ github.event.pull_request.head.sha }} | tee /jfs/${{ github.event.pull_request.number }}/head_sha\n echo save head sha to /jfs/${{ github.event.pull_request.number }}/head_sha\n \n - name : Cancel Outdate Runs\n uses: ./.github/actions/cancel-outdate-runs\n with: \n per_page: 8\n page: 1\n head_sha: ${{ env.previous_head_sha }}\n github_token: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Wait Runs Cancelled\n run: |\n sleep 10s\n\n - name : Cancel Outdate Runs\n uses: ./.github/actions/cancel-outdate-runs\n with: \n per_page: 8\n page: 1\n head_sha: ${{ env.previous_head_sha }}\n github_token: ${{ secrets.GITHUB_TOKEN }}" }, { "path": ".github/workflows/chaos.yml", "content": "name: \"chaos-test\"\n\non:\n push:\n branches:\n - 'release-**'\n - 'main'\n paths:\n - '**/chaos.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/chaos.yml'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n schedule:\n - cron: '0 20 * * *' \n\njobs:\n chaos-test:\n timeout-minutes: 60\n runs-on: ubuntu-22.04\n strategy:\n fail-fast: false\n matrix:\n # chaos: [\"minio-io\", \"minio-memory\", \"minio-cpu\", \"minio-bandwidth\", \"redis-bandwidth\", \"redis-io\", \"redis-delay\", \"redis-memory\", \"redis-cpu\", \"juicefs-bandwidth\", \"juicefs-memory\", \"juicefs-cpu\", \"juicefs-delay\"]\n chaos: [\"minio-io\", \"minio-memory\", \"minio-cpu\", \"minio-bandwidth\", \"redis-io\", \"redis-delay\", \"redis-memory\", \"redis-cpu\", \"juicefs-bandwidth\", \"juicefs-memory\", \"juicefs-cpu\", \"juicefs-delay\"]\n # chaos: [\"minio-io\"]\n steps: \n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - uses: actions/setup-go@v3\n with:\n go-version: 'oldstable'\n cache: true\n\n - name: Build\n timeout-minutes: 10\n run: | \n sudo .github/scripts/apt_install.sh musl-tools upx-ucl\n export STATIC=1\n make juicefs\n\n - name: Creating kind cluster\n uses: helm/kind-action@v1.5.0\n\n - name: Print cluster information\n run: |\n kubectl config view\n kubectl cluster-info\n kubectl get nodes\n kubectl get pods -n kube-system\n helm version\n kubectl version\n\n # - name: Build And Load CSI Docker Image\n # run: |\n # echo GITHUB_REF is $GITHUB_REF\n # echo GITHUB_SHA is $GITHUB_SHA\n # helm repo add juicefs https://juicedata.github.io/charts/\n # helm repo update\n # APP_VERSION=$(helm search repo juicefs/juicefs-csi-driver --versions | grep juicefs | head -1 | awk -F\" \" '{print $3}')\n # echo APP_VERSION is $APP_VERSION\n # docker build --build-arg GITHUB_REF=$GITHUB_REF --build-arg GITHUB_SHA=$GITHUB_SHA -f .github/scripts/chaos/juicefs-csi-driver.Dockerfile -t juicedata/juicefs-csi-driver:v$APP_VERSION .\n # kind load docker-image juicedata/juicefs-csi-driver:v$APP_VERSION --name chart-testing\n\n - name: Build And Load CSI Docker Image\n run: |\n version=`./juicefs version |awk '{print $3}' | cut -d '-' -f1`\n docker build -f .github/scripts/chaos/juicefs.Dockerfile -t juicedata/mount:ce-v${version} .\n helm repo add juicefs https://juicedata.github.io/charts/\n helm repo update\n kind load docker-image juicedata/mount:ce-v${version} --name chart-testing\n \n - name: Install JuiceFS CSI Driver\n run: |\n CHART_VERSION=$(helm search repo juicefs/juicefs-csi-driver --versions | grep juicefs | head -1 | awk -F\" \" '{print $2}')\n echo CHART_VERSION is $CHART_VERSION\n helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system --version $CHART_VERSION\n kubectl -n kube-system get pods -l app.kubernetes.io/name=juicefs-csi-driver\n\n - name: Deploy redis\n run: |\n kubectl apply -f .github/scripts/chaos/redis.yaml\n\n - name: Deploy minio\n run: |\n rm -rf /data/minio-data/*\n kubectl apply -f .github/scripts/chaos/minio.yaml\n\n - name: Mount Juicefs \n run: |\n version=`./juicefs version |awk '{print $3}' | cut -d '-' -f1`\n sed -i \"s/mount:ci/mount:ce-v$version/\" .github/scripts/chaos/sc.yaml\n kubectl apply -f .github/scripts/chaos/sc.yaml\n kubectl apply -f .github/scripts/chaos/pvc.yaml\n\n - name: Start vdbenh\n run: |\n kubectl apply -f .github/scripts/chaos/dynamic.yaml\n\n - name: Install Chaos Mesh\n run: |\n helm version\n kubectl version\n helm repo add chaos-mesh https://charts.chaos-mesh.org\n kubectl create ns chaos-mesh\n helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-mesh --version 2.5.1 \\\n --set chaosDaemon.runtime=containerd \\\n --set chaosDaemon.socketPath=/run/containerd/containerd.sock \\\n --set controllerManager.replicaCount=1\n echo \"wait pod status to running\"\n for ((k=0; k<120; k++)); do\n kubectl get pods --namespace chaos-mesh -l app.kubernetes.io/instance=chaos-mesh > pods.status\n cat pods.status\n\n run_num=`grep Running pods.status | wc -l`\n pod_num=$((`cat pods.status | wc -l` - 1))\n if [ $run_num == $pod_num ]; then\n break\n fi\n\n sleep 1\n done\n\n - name: Run chaos mesh action\n run: | \n chaos=${{matrix.chaos}}\n sed -i \"s/# - $chaos/- $chaos/g\" .github/scripts/chaos/workflow.yaml \n cat .github/scripts/chaos/workflow.yaml \n kubectl apply -f .github/scripts/chaos/workflow.yaml \n\n - name: Verify \n run: |\n for i in {1..1200}; do \n if kubectl get pods --all-namespaces | grep dynamic-ce | grep -i \"Completed\"; then\n echo \"dynamic-ce is completed in $i seconds\"\n break\n else\n if [ $((i % 10)) -eq 0 ]; then\n echo \"dynamic-ce is not completed in $i seconds\"\n fi\n sleep 1\n fi\n done\n kubectl get pods --all-namespaces\n apps=(\"dynamic-ce\" \"juicefs-csi-node\" \"juicefs-csi-controller\" \"juicefs-chart-testing-control-plane-pvc\" \"redis\" \"minio\")\n for app in ${apps[@]}; do\n echo app is $app\n kubectl get pods --all-namespaces | grep $app | grep -i \"Running\\|Completed\"\n if [ $? != 0 ]; then\n echo status of $app is not expected.\n exit 1\n fi\n done\n \n - name: Check mount pod\n if: always()\n run: |\n POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}' | grep juicefs-chart-testing-control-plane-pvc)\n echo POD_NAME is $POD_NAME\n for pod in $POD_NAME;do\n kubectl -n kube-system describe po $pod\n kubectl logs -n kube-system $pod > juicefs.log\n cat juicefs.log\n grep \":\" juicefs.log | grep -v format.go && exit 1 || true\n done\n\n - name: Mount pod upgrade\n timeout-minutes: 5\n run: |\n chaos=${{matrix.chaos}}\n skip_conditions=(\"minio-io\")\n if [[ \"${skip_conditions[*]}\" =~ \"$chaos\" ]]; then\n echo \"skip mount pod upgrade\"\n exit 0\n else\n CSI_POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}' | grep juicefs-csi-node) \n PVC_POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}' | grep juicefs-chart-testing-control-plane-pvc)\n kubectl exec $CSI_POD_NAME -n kube-system -- juicefs-csi-driver upgrade $PVC_POD_NAME 2>&1 | tee upgrade.log\n sleep 5\n if ! grep \"SUCCESS\" upgrade.log;then exit -1;fi\n rm upgrade.log\n kubectl exec $CSI_POD_NAME -n kube-system -- juicefs-csi-driver upgrade $PVC_POD_NAME --restart 2>&1 | tee upgrade.log || true \n sleep 5\n fi\n kubectl delete -f .github/scripts/chaos/workflow.yaml\n\n - name: Check csi controller log\n if: always()\n run: |\n kubectl describe pvc dynamic-ce\n kubectl -n kube-system get po -l app=juicefs-csi-controller\n kubectl -n kube-system logs juicefs-csi-controller-0 juicefs-plugin\n\n - name: Check csi node log\n if: always()\n run: |\n POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}' | grep juicefs-csi-node)\n echo POD_NAME is $POD_NAME\n kubectl -n kube-system describe po $POD_NAME\n kubectl -n kube-system logs $POD_NAME -c juicefs-plugin > csi_node.log\n cat csi_node.log\n # grep -i \"error\" csi_node.log && exit 1 || true\n\n - name: Check mount point pod\n if: always()\n run: |\n POD_NAME=$(kubectl get pods -n kube-system | grep juicefs-chart-testing-control-plane-pvc | grep Running | awk '{print $1}')\n echo POD_NAME is $POD_NAME\n for pod in $POD_NAME;do\n kubectl -n kube-system describe po $pod\n kubectl logs -n kube-system $pod > juicefs.log\n cat juicefs.log\n grep \":\" juicefs.log | grep -v format.go && exit 1 || true\n done\n\n - name: Check vdbench log\n if: always()\n run: | \n POD_NAME=$(kubectl get pods -n default -o go-template --template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}' | grep dynamic-ce )\n echo POD_NAME is $POD_NAME\n kubectl -n default describe po $POD_NAME\n kubectl logs -n default $POD_NAME > vdbench.log\n cat vdbench.log\n # grep -i \"error\" vdbench.log && exit 1 || true\n\n - name: Check Redis log\n if: always()\n run: | \n POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}' | grep redis )\n echo POD_NAME is $POD_NAME\n kubectl -n kube-system describe po $POD_NAME\n kubectl logs -n kube-system $POD_NAME > redis.log\n cat redis.log\n # grep -i \"error\" redis.log && exit 1 || true\n\n - name: Check Minio log\n if: always()\n run: | \n POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}' | grep minio )\n echo POD_NAME is $POD_NAME\n kubectl -n kube-system describe po $POD_NAME\n kubectl logs -n kube-system $POD_NAME > minio.log\n cat minio.log\n # grep -i \"error\" minio.log && exit 1 || true\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [chaos-test]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch' \n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success() \n run: echo \"All Done\"\n" }, { "path": ".github/workflows/check-doc.yaml", "content": "name: Check document\n\non:\n push:\n branches: [main]\n paths:\n - 'README*.md'\n - 'docs/**'\n - 'package.json'\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - '.github/workflows/check-doc.yaml'\n pull_request:\n branches: [main]\n paths:\n - 'README*.md'\n - 'docs/**'\n - 'package.json'\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - '.github/workflows/check-doc.yaml'\n\njobs:\n check-doc:\n name: Check document\n runs-on: ubuntu-latest\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n - name: Use Node.js\n uses: actions/setup-node@v4\n with:\n node-version: '18.x'\n cache: 'npm'\n - name: Install dependencies\n run: |\n npm ci\n - name: Lint Markdown files (markdownlint)\n run: |\n npm run markdown-lint\n - name: Lint Markdown files (autocorrect)\n uses: huacnlee/autocorrect-action@main\n with:\n args: --lint --no-diff-bg-color ./docs/\n - name: Check broken link (including broken anchor)\n run: |\n npm run check-broken-link\n" }, { "path": ".github/workflows/codeql-analysis.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n push:\n branches: [ main ]\n paths-ignore:\n - 'docs/**'\n# pull_request:\n# The branches below must be a subset of the branches above\n# branches: [ main ]\n schedule:\n - cron: '28 20 * * 0'\n \n workflow_dispatch:\n\njobs:\n analyze:\n name: Analyze\n timeout-minutes: 30\n runs-on: ubuntu-22.04\n permissions:\n actions: read\n contents: read\n security-events: write\n\n strategy:\n fail-fast: false\n matrix:\n language: ['java','go']\n # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]\n # Learn more:\n # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v3\n\n - name: Set up Java\n uses: actions/setup-java@v3\n with:\n distribution: 'temurin'\n java-version: '8'\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v2\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n #- name: Autobuild\n # uses: github/codeql-action/autobuild@v2\n\n # ℹ️ Command-line programs to run using the OS shell.\n # 📚 https://git.io/JvXDl\n\n # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines\n # and modify them (or add more) to build your code if your project\n # uses a compiled language\n - if: matrix.language == 'go'\n name: Autobuild\n uses: github/codeql-action/autobuild@v2\n\n - if: matrix.language == 'java'\n name: build-java\n run: mvn clean package -Dmaven.test.skip=true\n working-directory: sdk/java/\n #- run: |\n # make bootstrap\n # make release\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v2\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n" }, { "path": ".github/workflows/command-win.yml", "content": "name: \"command-win\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/command-win.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/command-win.yml'\n workflow_dispatch:\n inputs:\n debug_enabled:\n type: boolean\n description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'\n required: false\n default: false\n schedule:\n - cron: '0 17 * * 0'\n\njobs:\n command-win:\n runs-on: windows-2022\n env:\n Actions_Allow_Unsecure_Commands: true\n steps:\n - name: Set up Go\n uses: actions/setup-go@v3\n with:\n go-version: '1.21'\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Setup MSBuild.exe\n if: false\n uses: microsoft/setup-msbuild@v1.0.3\n\n - name: Change Winsdk Version\n if: false\n uses: GuillaumeFalourd/setup-windows10-sdk-action@v1\n with:\n sdk-version: 18362\n\n - name: Download WinFsp\n run: |\n choco install wget\n mkdir \"C:\\wfsp\\\"\n wget -O winfsp.msi https://github.com/winfsp/winfsp/releases/download/v2.0/winfsp-2.0.23075.msi\n copy winfsp.msi \"C:\\wfsp\\\"\n choco install 7zip -y\n\n - name: Install WinFsp\n run: |\n # call start-process to install winfsp.msi\n Start-Process -Wait -FilePath \"C:\\wfsp\\winfsp.msi\" -ArgumentList \"/quiet /norestart\"\n ls \"C:\\Program Files (x86)\\WinFsp\"\n ls \"C:\\Program Files (x86)\\WinFsp\\bin\"\n\n - name: Set up Include Headers\n run: |\n mkdir \"C:\\WinFsp\\inc\\fuse\"\n copy .\\hack\\winfsp_headers\\* C:\\WinFsp\\inc\\fuse\\\n dir \"C:\\WinFsp\\inc\\fuse\"\n set CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n go env\n go env -w CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n go env\n\n - name: Install Scoop\n run: |\n dir \"C:\\Program Files (x86)\\WinFsp\"\n Set-ExecutionPolicy RemoteSigned -scope CurrentUser\n iwr -useb 'https://raw.githubusercontent.com/scoopinstaller/install/master/install.ps1' -outfile 'install.ps1'\n .\\install.ps1 -RunAsAdmin\n echo $env:USERNAME\n scoop\n $redisUrl = \"https://github.com/tporadowski/redis/releases/download/v5.0.14.1/Redis-x64-5.0.14.1.zip\"\n $redisRoot = Join-Path $env:USERPROFILE \"scoop\\apps\\redis\\current\"\n New-Item -ItemType Directory -Force -Path $redisRoot | Out-Null\n Invoke-WebRequest -Uri $redisUrl -OutFile redis.zip\n Expand-Archive -Path redis.zip -DestinationPath $redisRoot -Force\n $redisCli = Join-Path $redisRoot \"redis-cli.exe\"\n if (-not (Test-Path $redisCli)) {\n throw \"redis-cli.exe not found after downloading from $redisUrl\"\n }\n $env:Path += \";$redisRoot\"\n $redisRoot | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append\n echo \"Updated PATH: $env:Path\"\n & $redisCli --version\n scoop install minio@2021-12-10T23-03-39Z\n scoop install runasti\n\n - name: Download winsw\n run: |\n wget https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW-x64.exe -q --show-progress -O winsw.exe\n ls winsw.exe\n\n - name: Start Redis\n run: |\n copy winsw.exe redis-service.exe\n $redisRoot = Join-Path $env:USERPROFILE \"scoop\\apps\\redis\\current\"\n $redisExe = Join-Path $redisRoot \"redis-server.exe\"\n if (-not (Test-Path $redisExe)) {\n throw \"redis-server.exe not found: $redisExe\"\n }\n @\"\n \n redisredis\n redisredis\n redisredis\n $redisExe\n --bind 127.0.0.1 --port 6379 --save \\\"\\\" --appendonly no\n rotate\n \n \"@ | Set-Content redis-service.xml -Encoding utf8\n .\\redis-service.exe install\n net start redisredis\n Start-Sleep -Seconds 2\n $redisCli = Join-Path $redisRoot \"redis-cli.exe\"\n & $redisCli -h 127.0.0.1 -p 6379 ping\n\n - name: Download MinGW\n run: |\n wget https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev1/x86_64-14.2.0-release-win32-seh-msvcrt-rt_v12-rev1.7z -q --show-progress -O mingw.7z\n 7z.exe x mingw.7z -oC:\\mingw64\n ls C:\\mingw64\\bin\n\n - name: Install Git\n run: |\n if (-not (Get-Command git -ErrorAction SilentlyContinue)) {\n Write-Host \"Installing Git...\"\n $gitInstaller = \"$env:TEMP\\Git-Installer.exe\"\n Invoke-WebRequest -Uri \"https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe\" -OutFile $gitInstaller\n \n Start-Process -Wait -FilePath $gitInstaller -ArgumentList \"/VERYSILENT\", \"/NORESTART\", \"/NOCANCEL\", \"/SP-\", \"/CLOSEAPPLICATIONS\", \"/RESTARTAPPLICATIONS\", \"/COMPONENTS=\"\"icons,ext\\reg\\shellhere,assoc,assoc_sh\"\"\"\n $env:Path += \";C:\\Program Files\\Git\\bin\"\n }\n \n\n - name: Build Juicefs\n run: |\n $env:CGO_ENABLED=1\n $env:PATH+=\";C:\\mingw64\\bin\"\n go build -ldflags=\"-s -w\" -o juicefs.exe .\n\n - name: Install Python3\n run: |\n choco install python3 -y\n\n - name: Wins_fs_test\n run: |\n ./juicefs.exe format redis://127.0.0.1:6379/1 myjfs\n $env:PATH+=\";C:\\Program Files (x86)\\WinFsp\\bin\"\n ./juicefs.exe mount -d redis://127.0.0.1:6379/1 z:\n python3 .github/scripts/wins_fs_test.py\n\n - name: Test Gc\n timeout-minutes: 10\n shell: bash\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/gc.sh\n \n - name: Test Debug\n timeout-minutes: 10\n shell: bash\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/debug.sh\n\n - name: Test dump load\n timeout-minutes: 10\n shell: bash\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/dump_load.sh\n\n - name: Test acl\n timeout-minutes: 10\n shell: bash\n if: false\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/acl.sh\n\n - name: Test clone\n timeout-minutes: 10\n shell: bash\n# if: false\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/clone.sh\n\n - name: Test fsck\n timeout-minutes: 10\n shell: bash\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/fsck.sh\n\n - name: Test profile\n timeout-minutes: 10\n shell: bash\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/profile.sh\n \n - name: Test gateway\n timeout-minutes: 10\n shell: bash\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/gateway.sh\n\n - name: Test quota\n timeout-minutes: 10\n shell: bash\n run: |\n export PATH=\"$HOME/scoop/shims:$PATH\"\n export LANG=C.UTF-8\n export LC_ALL=C.UTF-8\n META_URL=redis://127.0.0.1:6379/1 .github/scripts/command-win/quota.sh\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: mxschmitt/action-tmate@v3\n \n" }, { "path": ".github/workflows/command.yml", "content": "name: \"command-test\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '.github/scripts/command/*.sh'\n - '**/command.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '.github/scripts/command/*.sh'\n - '**/command.yml'\n schedule:\n - cron: '30 20 * * *'\n\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n echo \"GITHUB_REF_NAME is ${GITHUB_REF_NAME}\"\n if [ \"${{github.event_name}}\" == \"schedule\" ]; then\n echo 'meta_matrix=[\"mysql\", \"redis\", \"tikv\"]' >> $GITHUB_OUTPUT\n elif [ \"${{github.event_name}}\" == \"pull_request\" ]; then\n echo 'meta_matrix=[\"mysql\", \"redis\", \"tikv\"]' >> $GITHUB_OUTPUT\n elif [ \"${{github.event_name}}\" == \"workflow_dispatch\" ]; then\n echo 'meta_matrix=[\"mysql\", \"redis\", \"tikv\"]' >> $GITHUB_OUTPUT\n elif [[ \"${{ github.event_name }}\" == \"issue_comment\" ]] &&\n [[ \"${{ github.event.comment.body }}\" == *\"/run-command-tests\"* ]];then \n echo 'meta_matrix=[\"mysql\", \"redis\", \"tikv\"]' >> $GITHUB_OUTPUT\n else\n echo 'meta_matrix=[\"redis\"]' >> $GITHUB_OUTPUT\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n command_test1:\n timeout-minutes: 60\n needs: [build-matrix]\n strategy:\n fail-fast: false\n matrix:\n # meta: [ 'sqlite3', 'redis', 'tikv']\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n steps:\n - name: Remove unused software\n shell: bash\n run: |\n echo \"before remove unused software\"\n sudo df -h\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n echo \"after remove unused software\"\n sudo df -h\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Build \n timeout-minutes: 10\n uses: ./.github/actions/build\n \n - name: Download Random Test\n run: |\n wget https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/random-test/random-test\n chmod +x random-test\n \n - name: Test Mount\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/mount.sh\n\n - name: Test Gc\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/gc.sh \n\n - name: Test Config\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/config.sh\n \n - name: Test acl\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/acl.sh \n\n - name: Test Clone\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/clone.sh\n\n - name: Test fsck\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/fsck.sh\n\n - name: Test Gateway\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/gateway.sh\n \n - name: Test Debug\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/debug.sh\n \n - name: Test Info\n timeout-minutes: 10\n run: |\n free -g\n if [ \"${{matrix.meta}}\" == \"tikv\" ]; then\n ps -ef | grep tikv-server || true\n fi\n sudo META=${{matrix.meta}} .github/scripts/command/info.sh\n\n - name: Test Format\n timeout-minutes: 10\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/format.sh\n \n - name: Log\n if: always()\n run: |\n echo \"juicefs log\"\n sudo tail -n 1000 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log || true\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n command_test2:\n needs: [build-matrix]\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Remove unused software\n run: |\n echo \"before remove unused software\"\n sudo df -h\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n echo \"after remove unused software\"\n sudo df -h\n \n - name: Build \n uses: ./.github/actions/build\n\n - name: Test Quota\n timeout-minutes: 30\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/quota.sh \n\n - name: Log\n if: always()\n run: |\n echo \"juicefs log\"\n sudo tail -n 1000 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n command_test3:\n needs: [build-matrix]\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Build \n uses: ./.github/actions/build\n\n - name: Test Graceful upgrade\n timeout-minutes: 30\n run: |\n if [ \"${{matrix.meta}}\" == \"redis\" ]; then\n sudo META=${{matrix.meta}} .github/scripts/command/graceful_upgrade.sh\n fi\n \n - name: Test Interface\n timeout-minutes: 20\n run: |\n sudo META=${{matrix.meta}} .github/scripts/command/interface.sh\n\n - name: Log\n if: always()\n run: |\n if [ \"${{matrix.meta}}\" == \"redis\" ]; then\n echo \"juicefs log\"\n sudo tail -n 1000 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n fi\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [command_test1, command_test2, command_test3]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/command2.yml", "content": "name: \"command-random-test\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '.github/scripts/command/random.sh'\n - '.github/scripts/hypo/command*.py'\n - '**/command2.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '.github/scripts/command/random.sh'\n - '.github/scripts/hypo/command*.py'\n - '**/command2.yml'\n schedule:\n - cron: '30 20 * * *'\n\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n echo \"GITHUB_REF_NAME is ${GITHUB_REF_NAME}\"\n if [ \"${{github.event_name}}\" == \"schedule\" ]; then\n echo 'meta_matrix=[\"sqlite3\", \"mysql\", \"tikv\", \"postgres\", \"mariadb\", \"fdb\"]' >> $GITHUB_OUTPUT\n elif [ \"${{github.event_name}}\" == \"pull_request\" ]; then\n echo 'meta_matrix=[\"sqlite3\"]' >> $GITHUB_OUTPUT\n elif [ \"${{github.event_name}}\" == \"workflow_dispatch\" ]; then\n echo 'meta_matrix=[\"mysql\", \"tikv\"]' >> $GITHUB_OUTPUT\n else\n echo 'meta_matrix=[\"mysql\", \"tikv\"]' >> $GITHUB_OUTPUT\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n test:\n needs: [build-matrix]\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Run Example\n timeout-minutes: 60\n run: |\n sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_examples\n \n - name: Remove unused software\n run: |\n if [ \"${{ github.event_name }}\" == \"schedule\" ]; then\n echo \"before remove unused software\"\n sudo df -h\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n echo \"after remove unused software\"\n sudo df -h\n fi\n\n - name: Download example database\n timeout-minutes: 5\n uses: dawidd6/action-download-artifact@v9\n if: false\n with:\n name: command2-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n if_no_artifact_found: ignore\n workflow_conclusion: \"\"\n check_artifacts: true\n\n - name: Run All\n continue-on-error: true\n timeout-minutes: 60\n run: |\n sudo -E LOG_LEVEL=WARNING META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_all 2>&1 | tee fsrand.log\n \n - name: check fsrand.log\n if: always()\n run: | \n [[ -f \"fsrand.log\" ]] && tail -n 1000 fsrand.log \n grep -i \"AssertionError\" fsrand.log && exit 1 || true\n\n - name: chmod example directory\n if: always()\n timeout-minutes: 5\n run: |\n if [[ -e \".hypothesis/examples\" ]]; then\n echo \"chmod for .hypothesis/examples\" && sudo chmod -R 755 .hypothesis/examples\n fi\n\n - name: Upload example database\n uses: actions/upload-artifact@v4\n if: false\n with:\n include-hidden-files: true\n name: command2-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n\n - name: Check client log\n if: always()\n run: |\n echo \"juicefs log\"\n sudo tail -n 1000 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [test]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/compile.yml", "content": "name: \"compile\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/compile.yml'\n pull_request:\n branches:\n - main\n - release**\n paths:\n - '**/compile.yml'\n schedule:\n - cron: '0 20 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n echo \"GITHUB_REF_NAME is ${GITHUB_REF_NAME}\"\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]]; then\n echo 'meta_matrix=[\"sqlite3\", \"redis\", \"mysql\", \"tikv\", \"postgres\", \"badger\", \"mariadb\", \"fdb\"]' >> $GITHUB_OUTPUT\n elif [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n echo 'meta_matrix=[\"redis\", \"mysql\", \"tikv\"]' >> $GITHUB_OUTPUT\n else\n echo \"event_name is not supported\" && exit 1\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n compile:\n timeout-minutes: 120\n needs: build-matrix\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n \n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n \n - name: Remove unused software\n timeout-minutes: 10\n run: |\n echo \"before remove unused software\"\n sudo df -h\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n echo \"after remove unused software\"\n sudo df -h\n\n - name: Prepare meta db\n run: | \n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo ./juicefs format $meta_url --trash-days 0 --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo ./juicefs mount -d $meta_url /jfs --no-usage-report\n stat /jfs/.accesslog\n \n - name: Build Redis\n timeout-minutes: 10\n working-directory: /jfs\n run: |\n wget -O /tmp/redis.tar.gz https://github.com/redis/redis/archive/refs/tags/6.0.16.tar.gz\n mkdir redis\n tar -xvf /tmp/redis.tar.gz -C redis --strip-components 1\n make -C redis\n \n - name: Install Depenency for Kernel\n run: |\n sudo apt-get install bison flex libelf-dev bc -y\n\n - name: Build Kernel\n timeout-minutes: 90\n working-directory: /jfs\n run: |\n wget -O /tmp/linux.tar.gz https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.5.tar.gz\n mkdir linux\n tar xzf /tmp//linux.tar.gz -C linux --strip-components 1\n make -C linux defconfig\n make -C linux -j`grep -c processor /proc/cpuinfo`\n \n - name: Build Spack\n if: false\n run: |\n git clone https://github.com/spack/spack.git\n source spack/share/spack/setup-env.sh\n spack --version\n spack bootstrap now\n spack compiler find \n spack compilers\n spack config get config > ~/.spack/config.yaml \n sed -i '/build_stage:/,+2d' ~/.spack/config.yaml\n echo -e \"build_stage:\\n - /jfs/spack-stage\" >> ~/.spack/config.yaml\n spack install spack\n\n - name: Log\n if: always()\n run: |\n echo \"juicefs log\"\n sudo tail -n 1000 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [compile]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success() \n run: echo \"All Done\"" }, { "path": ".github/workflows/coverage-report.yml", "content": "name: \"coverage-report\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/coverage-report.yml'\n pull_request:\n branches:\n - main\n - release**\n paths:\n - '**/coverage-report.yml'\n\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n last_date:\n type: string\n description: \"last date of coverage data\"\n required: false\n default: \"\"\n schedule:\n - cron: '0 23 * * *'\n \njobs:\n coverage-report:\n strategy:\n fail-fast: false\n matrix:\n branch: ['main']\n test: ['ut', 'it', 'all']\n \n timeout-minutes: 60\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n timeout-minutes: 1\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir for cleanup\n if: ${{ matrix.test == 'all' }}\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: /ci-coverage\n subdir: juicefs/\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: clean up old coverage data\n if: ${{ matrix.test == 'all' }}\n continue-on-error: true\n timeout-minutes: 10\n run: |\n sudo find /ci-coverage -type f \\( -name 'covcounters*' -o -name 'covmeta*' \\) -mtime +2 -print -exec rm -f {} +\n umount /jfs-coverage\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: schedule\n subdir: juicefs/schedule\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n \n - name: Determine lastdate dir\n timeout-minutes: 120\n run: |\n if [[ -n \"${{github.event.inputs.last_date}}\" ]]; then\n last_date=${{github.event.inputs.last_date}}\n else\n last_date=$(ls -t schedule | head -n 1)\n [[ -z \"$last_date\" ]] && echo \"no data found in schedule\" && exit 1\n fi\n [[ ! -d \"schedule/$last_date\" ]] && echo \"schedule/$last_date not found\" && exit 1\n echo \"last_date=$last_date\" >> $GITHUB_ENV\n\n - name: Generate today's coverage report\n timeout-minutes: 30\n working-directory: schedule/${{env.last_date}}\n run: |\n echo \"current dir is $(pwd)\"\n coverdirs=\"\"\n for dir in $(find . -mindepth 1 -maxdepth 1 -type d -exec basename {} \\;); do\n if [[ ${{matrix.test}} == \"ut\" ]]; then\n if [[ \"$dir\" == \"unittests\" ]]; then\n coverdirs+=\"$dir,\"\n fi\n elif [[ ${{matrix.test}} == \"it\" ]]; then\n if [[ \"$dir\" != \"unittests\" ]]; then\n coverdirs+=\"$dir,\"\n fi\n elif [[ ${{matrix.test}} == \"all\" ]]; then\n coverdirs+=\"$dir,\"\n fi\n done\n coverdirs=${coverdirs%,}\n echo coverdirs is $coverdirs\n [[ -z \"$coverdirs\" ]] && echo \"no coverage dir found\" && exit 0\n name=cover_${{matrix.test}}\n sudo go tool covdata percent -i=$coverdirs | sudo tee ${name}.percent\n echo \"generated coverage percent report:\" $(realpath ${name}.percent)\n sudo go tool covdata textfmt -i=$coverdirs -o ${name}.txt \n echo \"generated coverage report in text format:\" $(realpath ${name}.txt)\n sudo go tool cover -html=${name}.txt -o ${name}.html\n echo \"generated coverage report in html format:\" $(realpath ${name}.html)\n ls -l cover_*\n \n - name: upload coverage report\n working-directory: schedule/${{env.last_date}}\n timeout-minutes: 10\n run: |\n echo \"current dir is $(pwd)\"\n [[ ! -f \"cover_${{matrix.test}}.html\" ]] && echo \"no coverage report found\" && exit 0\n UPLOAD_PATH=${{github.workflow}}_${{github.run_id}}_${{matrix.test}}.html\n response=$(curl -w '%{http_code}' -s -o /dev/null --form 'file=@cover_${{matrix.test}}.html' https://juicefs.com/upload-file-u80sdvuke/${UPLOAD_PATH}?token=${{secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN}})\n if [ \"$response\" -eq 200 ]; then\n echo Coverage Report for ${{matrix.test}}: https://i.juicefs.io/ci-coverage/${UPLOAD_PATH}\n else\n echo \"Upload failed with status code $response\"\n exit 1\n fi\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 30\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/dependency-review.yml", "content": "# Dependency Review Action\n#\n# This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging.\n#\n# Source repository: https://github.com/actions/dependency-review-action\n# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement\nname: 'Dependency Review'\non: [pull_request]\n\npermissions:\n contents: read\n\njobs:\n dependency-review:\n runs-on: ubuntu-22.04\n steps:\n - name: 'Checkout Repository'\n uses: actions/checkout@v3\n - name: 'Dependency Review'\n uses: actions/dependency-review-action@v2\n# with:\n# fail-on-severity: high\n" }, { "path": ".github/workflows/dockerfile-sftp", "content": "FROM debian:stable-slim\nRUN apt-get clean\nRUN apt-get update\n\nRUN apt-get install openssh-server -y\n\nRUN mkdir /run/sshd\nRUN sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config\nRUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config\nRUN echo \"root:password\"|chpasswd\nRUN useradd -m testUser1\nRUN echo \"testUser1:password\"|chpasswd\nEXPOSE 22\nCMD [\"/usr/sbin/sshd\", \"-D\"]\n" }, { "path": ".github/workflows/dump_load.yml", "content": "name: \"dump_load\"\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/start_meta_engine.sh'\n - '**/dump_load.yml'\n - '**/dump_load.sh'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/start_meta_engine.sh'\n - '**/dump_load.yml'\n - '**/dump_load.sh'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n echo \"GITHUB_REF_NAME is ${GITHUB_REF_NAME}\"\n if [ \"${{github.event_name}}\" == \"schedule\" ] || [ \"${{github.event_name}}\" == \"workflow_dispatch\" ]; then\n echo 'meta_matrix=[\"sqlite3\", \"redis\", \"mysql\", \"tikv\", \"tidb\", \"postgres\", \"mariadb\", \"fdb\"]' >> $GITHUB_OUTPUT\n else\n echo 'meta_matrix=[\"redis\", \"mysql\", \"tikv\"]' >> $GITHUB_OUTPUT\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n dump_load:\n timeout-minutes: 90\n needs: [build-matrix]\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Install nodejs\n run: |\n curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash\n export NVM_DIR=\"$HOME/.nvm\"\n [ -s \"$NVM_DIR/nvm.sh\" ] && \\. \"$NVM_DIR/nvm.sh\" # This loads nvm\n [ -s \"$NVM_DIR/bash_completion\" ] && \\. \"$NVM_DIR/bash_completion\" # This loads nvm bash_completion\n nvm install 22\n nvm use 22\n node -v \n nvm current\n\n - name: Download Random Test\n run: |\n wget https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/random-test/random-test\n chmod +x random-test\n\n - name: Test Load & Dump with Binary\n timeout-minutes: 30\n continue-on-error: true\n run: |\n sudo BINARY=true GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/dump_load.sh\n\n - name: Test Load & Dump with Json Fast Mode\n timeout-minutes: 30\n run: |\n sudo FAST=true GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/dump_load.sh \n\n - name: Test Load & Dump with Json\n timeout-minutes: 30\n run: |\n sudo GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/dump_load.sh \n\n - name: log\n if: always()\n run: | \n tail -500 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [dump_load]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: upload total coverage report\n timeout-minutes: 30\n continue-on-error: true\n uses: ./.github/actions/upload-total-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch' \n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n " }, { "path": ".github/workflows/dump_load_bench.yml", "content": "name: \"dump_load_bench\"\non:\n push:\n branches:\n - \"main\"\n - \"release-**\"\n paths:\n - \"**/dump_load_bench.yml\"\n - \"**/dump_load_bench.sh\"\n pull_request:\n branches:\n - \"main\"\n - \"release-**\"\n paths:\n - \"**/dump_load_bench.yml\"\n - \"**/dump_load_bench.sh\"\n schedule:\n - cron: \"0 19 * * *\"\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n echo \"GITHUB_REF_NAME is ${GITHUB_REF_NAME}\"\n if [ \"${{github.event_name}}\" == \"schedule\" ] || [ \"${{github.event_name}}\" == \"workflow_dispatch\" ]; then\n echo 'meta_matrix=[\"sqlite3\", \"redis\", \"mysql\", \"tikv\", \"tidb\", \"postgres\", \"mariadb\", \"fdb\"]' >> $GITHUB_OUTPUT\n else\n echo 'meta_matrix=[\"redis\", \"sqlite3\", \"tikv\"]' >> $GITHUB_OUTPUT\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n dump_load_bench:\n timeout-minutes: 90\n needs: [build-matrix]\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Clean up\n run: |\n sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/local/.ghcup\n sudo docker system prune -af\n sudo df -h\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with:\n target: ${{steps.vars.outputs.target}}\n\n - name: Install nodejs\n run: |\n curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash\n export NVM_DIR=\"$HOME/.nvm\"\n [ -s \"$NVM_DIR/nvm.sh\" ] && \\. \"$NVM_DIR/nvm.sh\" # This loads nvm\n [ -s \"$NVM_DIR/bash_completion\" ] && \\. \"$NVM_DIR/bash_completion\" # This loads nvm bash_completion\n nvm install 22\n nvm use 22\n node -v \n nvm current\n\n - name: Benchmark dump load in binary format\n timeout-minutes: 60\n env:\n AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}\n AWS_SECRET_ACCESS_KEY: ${{secrets.CI_COVERAGE_AWS_SK}}\n AWS_ACCESS_TOKEN: ${{secrets.CI_COVERAGE_AWS_TOKEN}}\n META: ${{matrix.meta}}\n START_META: true\n run: |\n sudo -HE GOCOVERDIR=$(pwd)/cover .github/scripts/command/dump_load_bench.sh test_dump_load_in_binary\n\n - name: Benchmark dump load fast\n timeout-minutes: 60\n env:\n AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}\n AWS_SECRET_ACCESS_KEY: ${{secrets.CI_COVERAGE_AWS_SK}}\n AWS_ACCESS_TOKEN: ${{secrets.CI_COVERAGE_AWS_TOKEN}}\n META: ${{matrix.meta}}\n START_META: false\n run: |\n sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/dump_load_bench.sh test_dump_load_fast\n\n - name: Benchmark dump load\n timeout-minutes: 60\n env:\n AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}\n AWS_SECRET_ACCESS_KEY: ${{secrets.CI_COVERAGE_AWS_SK}}\n AWS_ACCESS_TOKEN: ${{secrets.CI_COVERAGE_AWS_TOKEN}}\n META: ${{matrix.meta}}\n START_META: false\n run: |\n sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/dump_load_bench.sh test_dump_load\n\n - name: log\n if: always()\n run: |\n tail -500 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Setup upterm session\n if: failure()\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [dump_load_bench]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: upload total coverage report\n timeout-minutes: 30\n continue-on-error: true\n uses: ./.github/actions/upload-total-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/dump_load_cross_meta.yml", "content": "name: \"dump_load_cross_meta\"\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/start_meta_engine.sh'\n - '**/dump_load_cross_meta.yml'\n - '**/dump_load_cross_meta.sh'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/start_meta_engine.sh'\n - '**/dump_load_cross_meta.yml'\n - '**/dump_load_cross_meta.sh'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n dump_load_cross_meta:\n timeout-minutes: 90\n strategy:\n fail-fast: false\n matrix:\n meta1: [redis]\n meta2: [sqlite3]\n include:\n - meta1: mysql\n meta2: redis\n - meta1: mysql\n meta2: tikv\n # - meta1: tikv\n # meta2: mysql\n # - meta1: redis\n # meta2: tikv\n # - meta1: tikv\n # meta2: redis\n\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [[ \"${{matrix.meta1}}\" == \"fdb\" || \"${{matrix.meta2}}\" == \"fdb\" ]]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Test Load & Dump with Json Fast Mode\n timeout-minutes: 30\n run: |\n sudo GOCOVERDIR=$(pwd)/cover META1=${{matrix.meta1}} META2=${{matrix.meta2}} FAST=true .github/scripts/command/dump_load_cross_meta.sh \n \n - name: Test Load & Dump with Binary\n timeout-minutes: 30\n run: |\n sudo GOCOVERDIR=$(pwd)/cover META1=${{matrix.meta1}} META2=${{matrix.meta2}} BINARY=true .github/scripts/command/dump_load_cross_meta.sh\n \n - name: Test Load & Dump with Json\n timeout-minutes: 30\n run: |\n sudo GOCOVERDIR=$(pwd)/cover META1=${{matrix.meta1}} META2=${{matrix.meta2}} .github/scripts/command/dump_load_cross_meta.sh \n\n - name: log\n if: always()\n run: | \n tail -500 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n\n - name: Upload dumpfiies\n uses: actions/upload-artifact@v4\n timeout-minutes: 5\n if: failure()\n with:\n name: dump-files-${{ github.run_id }}-${{matrix.meta1}}-${{matrix.meta2}}\n path: |\n ${{github.workspace}}/*.json\n ${{github.workspace}}/*.db\n if-no-files-found: warn\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [dump_load_cross_meta]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: upload total coverage report\n timeout-minutes: 30\n continue-on-error: true\n uses: ./.github/actions/upload-total-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch' \n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n \n " }, { "path": ".github/workflows/fsrand.yml", "content": "name: \"fsrand\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/fsrand.yml'\n - '**/fs.py'\n - '**/fs_test.py'\n - '**/fs_acl_test.py'\n pull_request:\n branches:\n - main\n - release**\n paths:\n - '**/fsrand.yml'\n - '**/fs.py'\n - '**/fs_test.py'\n - '**/fs_acl_test.py'\n schedule:\n - cron: '0 17 * * 0'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\njobs:\n fsrand:\n timeout-minutes: 60\n strategy:\n fail-fast: false\n matrix:\n meta: ['redis', 'mysql', 'tikv']\n # meta: ['redis']\n \n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n \n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Install tool\n run: | \n sudo .github/scripts/apt_install.sh attr\n sudo pip install xattr\n sudo pip install minio\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --enable-acl --trash-days 0 --bucket=/mnt/jfs myjfs\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr \n if [ ! -f /tmp/jfs/.accesslog ]; then\n echo \": mount failed\"\n exit 1\n fi\n \n - name: Run Examples\n timeout-minutes: 10\n run: |\n sudo -E python3 .github/scripts/hypo/fs_test.py\n sudo -E python3 .github/scripts/hypo/fs_acl_test.py\n\n - name: Download example database\n timeout-minutes: 5\n uses: dawidd6/action-download-artifact@v9\n if: false\n with:\n name: fsrand-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n if_no_artifact_found: ignore\n workflow_conclusion: \"\"\n check_artifacts: true\n\n - name: Test\n continue-on-error: true\n timeout-minutes: 120\n run: |\n sudo -E LOG_LEVEL=WARNING python3 .github/scripts/hypo/fs.py 2>&1 | tee fsrand.log\n \n - name: check fsrand.log\n if: always()\n run: | \n [[ -f \"fsrand.log\" ]] && tail -n 1000 fsrand.log \n grep -i \"AssertionError\" fsrand.log && exit 1 || true\n\n - name: chmod example directory\n if: always()\n timeout-minutes: 5\n run: |\n if [[ -e \".hypothesis/examples\" ]]; then\n echo \"chmod for .hypothesis/examples\" && sudo chmod -R 755 .hypothesis/examples\n fi\n\n - name: Upload example database\n uses: actions/upload-artifact@v4\n if: false\n with:\n include-hidden-files: true\n name: fsrand-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n\n - name: check juicefs.log\n if: always()\n run: | \n if [ -f ~/.juicefs/juicefs.log ]; then\n tail -300 ~/.juicefs/juicefs.log\n grep \":\" ~/.juicefs/juicefs.log && exit 1 || true\n fi\n if [ -f /var/log/juicefs.log ]; then\n tail -300 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n fi\n\n - name: upload coverage report\n timeout-minutes: 5\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure() \n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [fsrand]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"" }, { "path": ".github/workflows/fsspec.yml", "content": "name: \"fsspec\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/fsspec.yml'\n pull_request:\n branches:\n - main\n - release**\n paths:\n - '**/fsspec.yml'\n\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n schedule:\n - cron: '0 16 * * *'\n \njobs:\n fsspec:\n strategy:\n fail-fast: false\n matrix:\n meta: ['redis', 'mysql', 'tikv']\n timeout-minutes: 60\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n timeout-minutes: 1\n uses: actions/checkout@v3\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n \n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n \n - name: Create venv\n run: |\n python3 -m venv venv\n source venv/bin/activate\n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n \n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --enable-acl --trash-days 0 --bucket=/mnt/jfs myjfs\n\n - name: Build and install SDK\n timeout-minutes: 5\n run: |\n make -C sdk/python/ libjfs.so\n sudo python3 sdk/python/juicefs/setup.py install\n\n - name: Build and install juicefs spec\n timeout-minutes: 10\n working-directory: sdk/python/juicefs\n run: |\n pip install build\n python3 -m build -w\n ls dist/\n pip install dist/juicefs-*.whl\n\n - name: Run Test\n timeout-minutes: 10\n working-directory: sdk/python/juicefs\n run: |\n sudo pip install pytest\n sudo pip install fsspec\n source ../../../.github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo JUICEFS_META=${meta_url} python3 -m pytest tests/test.py\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 30\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [fsspec]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"" }, { "path": ".github/workflows/gateway-random.yml", "content": "name: \"gateway-random\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/gateway-random.yml'\n - '**/gateway-random.sh'\n - '.github/scripts/hypo/s3**.py'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/gateway-random.yml'\n - '**/gateway-random.sh'\n - '.github/scripts/hypo/s3**.py'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n echo \"GITHUB_REF_NAME is ${GITHUB_REF_NAME}\"\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]]; then\n echo 'meta_matrix=[\"mysql\", \"redis\", \"tikv\"]' >> $GITHUB_OUTPUT\n else\n echo 'meta_matrix=[\"redis\"]' >> $GITHUB_OUTPUT\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n gateway-random:\n timeout-minutes: 90\n needs: build-matrix\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Test with example\n run: |\n if [[ ${{matrix.meta}} == \"tikv\" ]]; then\n subdir=true\n else\n subdir=false\n fi\n sudo -E SUBDIR=$subdir .github/scripts/command/gateway-random.sh test_run_example\n \n - name: Download example database\n timeout-minutes: 5\n uses: dawidd6/action-download-artifact@v9\n if: false\n with:\n name: gateway-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n if_no_artifact_found: ignore\n workflow_conclusion: \"\"\n check_artifacts: true\n\n - name: Test randomly\n continue-on-error: true\n timeout-minutes: 60\n run: |\n if [[ ${{matrix.meta}} == \"tikv\" ]]; then\n subdir=true\n else\n subdir=false\n fi\n sudo -E LOG_LEVEL=WARNING SUBDIR=$subdir .github/scripts/command/gateway-random.sh test_run_all 2>&1 | tee fsrand.log\n \n - name: check fsrand.log\n if: always()\n run: | \n [[ -f \"fsrand.log\" ]] && tail -n 1000 fsrand.log \n grep -i \"AssertionError\" fsrand.log && exit 1 || true\n\n - name: chmod example directory\n if: always()\n timeout-minutes: 5\n run: |\n if [[ -e \".hypothesis/examples\" ]]; then\n echo \"chmod for .hypothesis/examples\" && sudo chmod -R 755 .hypothesis/examples\n fi\n\n - name: Upload example database\n uses: actions/upload-artifact@v4\n if: false\n with:\n include-hidden-files: true\n name: gateway-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n\n - name: check log\n if: always()\n run: | \n if [ -f /var/log/juicefs-gateway.log ]; then\n tail -300 /var/log/juicefs-gateway.log\n grep \":\" /var/log/juicefs-gateway.log && exit 1 || true\n fi\n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [gateway-random]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success() \n run: echo \"All Done\"" }, { "path": ".github/workflows/gateway.yml", "content": "name: \"gateway-test\"\n\non:\n push:\n branches: \n - release-**\n paths-ignore:\n - 'docs/**'\n - '**.md'\n pull_request:\n #The branches below must be a subset of the branches above\n branches: \n - release-**\n paths-ignore:\n - 'docs/**'\n - '**.md'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n schedule:\n - cron: '0 19 * * *'\n\njobs:\n gateway:\n timeout-minutes: 60\n strategy:\n fail-fast: false\n matrix:\n meta: [ 'sqlite3', 'redis','tikv', 'badger', 'etcd', 'fdb']\n file_size: ['100M']\n isolation_level: ['']\n include:\n - meta: 'mariadb'\n file_size: '100M'\n isolation_level: \"read committed\"\n\n - meta: 'mysql'\n file_size: '100M'\n isolation_level: \"read committed\"\n - meta: 'mysql'\n file_size: '100M'\n isolation_level: \"repeatable read\"\n - meta: 'mysql'\n file_size: '100M'\n isolation_level: \"serializable\"\n\n - meta: 'postgres'\n file_size: '100M'\n isolation_level: \"read committed\"\n - meta: 'postgres'\n file_size: '100M'\n isolation_level: \"repeatable read\"\n - meta: 'postgres'\n file_size: '100M'\n isolation_level: \"serializable\"\n\n - meta: 'tidb'\n file_size: '100M'\n isolation_level: \"read committed\"\n - meta: 'tidb'\n file_size: '100M'\n isolation_level: \"repeatable read\"\n\n runs-on: ubuntu-22.04\n\n steps: \n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n \n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n \n - name: Start meta\n run: | \n sudo chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n\n - name: Install tools\n run: | \n wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc\n chmod +x mc \n shell: bash\n \n - name: start gateway\n shell: bash\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url ${{matrix.isolation_level}}\n mp=\"/tmp/myjfs\"\n volume=\"myjfs\"\n export MINIO_ROOT_USER=minioadmin\n export MINIO_ROOT_PASSWORD=minioadmin\n sudo chmod 777 /mnt\n ./juicefs format $meta_url $volume --trash-days 0 --bucket=/mnt/jfs\n ./juicefs gateway $meta_url localhost:8080 --no-usage-report --access-log /tmp/access1.log &\n \n - name: Sync with multiple process\n shell: bash\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n file_size=${{matrix.file_size}}\n if [ $file_size=\"100M\" ]; then \n file_count=5\n else\n file_count=2000\n fi\n threads=20\n mp=/tmp/myjfs\n volume=myjfs\n \n dd if=/dev/urandom of=file iflag=fullblock,count_bytes bs=4k count=\"$file_size\" > /dev/null\n mkdir data\n for i in $(seq 1 $file_count); do\n cp file data/file$i\n done\n start=`date +%s`\n declare -a pids \n ./juicefs sync --dirs data/ s3://minioadmin:minioadmin@localhost:8080/$volume/data/ --no-https -p $threads &\n pids+=($!)\n ./juicefs sync --dirs data/ s3://minioadmin:minioadmin@localhost:8080/$volume/data/ --no-https -p $threads &\n pids+=($!)\n ./juicefs sync --dirs data/ s3://minioadmin:minioadmin@localhost:8080/$volume/data/ --no-https -p $threads &\n pids+=($!)\n wait \"${pids[@]}\"\n rm -rf $HOME/.juicefs/cache/ || true\n # ./mc alias set minio http://localhost:9000 minioadmin minioadmin --api S3v4\n # ./mc mb minio/$volume\n # ./mc cp --recursive data/ minio/$volume/data\n end=`date +%s`\n time=$((end-start))\n echo time cost is: $time second\n killall juicefs \n sleep 3\n ./juicefs mount -d $meta_url $mp --no-usage-report\n diff -ur data/ $mp/data/ \n echo \"diff succeed\"\n ./juicefs umount $mp --force\n \n - name: Sync with empty dir\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n threads=20\n mp=/tmp/myjfs\n volume=myjfs\n export MINIO_ROOT_USER=minioadmin\n export MINIO_ROOT_PASSWORD=minioadmin\n ./juicefs mdtest $meta_url test --dirs 10 --depth 2 --files 10 --threads 10 --no-usage-report\n ./juicefs gateway $meta_url localhost:8080 --access-log /tmp/access1.log &\n sleep 10\n mkdir emptydir\n declare -a pids \n ./juicefs sync emptydir/ s3://minioadmin:minioadmin@localhost:8080/$volume/test/ --delete-dst --no-https -p $threads &\n pids+=($!)\n ./juicefs sync emptydir/ s3://minioadmin:minioadmin@localhost:8080/$volume/test/ --delete-dst --no-https -p $threads &\n pids+=($!)\n ./juicefs sync emptydir/ s3://minioadmin:minioadmin@localhost:8080/$volume/test/ --delete-dst --no-https -p $threads &\n pids+=($!)\n wait \"${pids[@]}\"\n killall juicefs\n sleep 3\n ./juicefs mount -d $meta_url $mp --no-usage-report\n [ -d \"$mp/test/\" ] && exit 1 \n ./juicefs umount $mp --force\n shell: bash\n\n - name: log\n if: always()\n shell: bash\n run: | \n if [ -f ~/.juicefs/juicefs.log ]; then\n tail -300 ~/.juicefs/juicefs.log\n grep \":\" ~/.juicefs/juicefs.log && exit 1 || true\n fi\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 1\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [gateway]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/integrationtests.yml", "content": "name: \"integrationtests\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**.c'\n - '**.go'\n - 'Makefile'\n - '**/integrationtests.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**.c'\n - '**.go'\n - 'Makefile'\n - '**/integrationtests.yml'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n echo \"GITHUB_REF_NAME is ${GITHUB_REF_NAME}\"\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]]; then\n echo 'meta_matrix=[\"sqlite3\", \"redis\", \"mysql\", \"tikv\", \"tidb\", \"postgres\", \"badger\", \"mariadb\", \"fdb\"]' >> $GITHUB_OUTPUT\n elif [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n echo 'meta_matrix=[\"redis\", \"mysql\", \"tikv\"]' >> $GITHUB_OUTPUT\n else\n echo \"event_name is not supported\" && exit 1\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n integrationtests:\n timeout-minutes: 120\n runs-on: ubuntu-22.04\n needs: build-matrix\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n \n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --trash-days 0 --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr\n stat /jfs/.accesslog\n\n - name: Fslock Test\n timeout-minutes: 5\n run: |\n cd /jfs/\n git clone https://github.com/danjacques/gofslock.git\n cd gofslock && git checkout cc7f001fe0e7df1710adc8f0cd9e9d6d21fdb3a9\n go test -v ./fslock/...\n stat /jfs/\n\n - name: flock test\n timeout-minutes: 5\n run: |\n git clone https://github.com/gofrs/flock.git\n mkdir /jfs/tmp\n cd flock && go mod tidy && TMPDIR=/jfs/tmp go test .\n\n - name: make secfs.test\n run: |\n sudo .github/scripts/apt_install.sh libacl1-dev\n git clone https://github.com/billziss-gh/secfs.test.git\n make -C secfs.test tools tools/bin/fsx\n make -C secfs.test tools tools/bin/fsracer\n \n - name: Fsx Test\n timeout-minutes: 16\n run: |\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]] ; then\n duration=900\n else\n duration=300\n fi\n sudo touch /jfs/fsx.out\n sudo rm -f /tmp/fsx.out\n sudo ln -s /jfs/fsx.out /tmp/fsx.out\n sudo secfs.test/tools/bin/fsx -d $duration -p 10000 -F 10000000 /tmp/fsx.out\n\n - name: Fsracer Test\n if: false\n timeout-minutes: 16\n shell: 'script -q -e -c \"bash {0}\"'\n run: |\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]] ; then\n duration=600\n else\n duration=300\n fi\n sudo secfs.test/tools/bin/fsracer $duration /jfs\n \n - name: log\n if: always()\n run: |\n tail -300 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [integrationtests]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success() \n run: echo \"All Done\"" }, { "path": ".github/workflows/ltpfs.yml", "content": "name: \"ltpfs\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/ltpfs.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/ltpfs.yml'\n schedule:\n - cron: '30 20 * * *'\n workflow_dispatch:\n\njobs:\n ltpfs:\n timeout-minutes: 60\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n\n - name: Copy\n run: |\n cp .github/workflows/bash/rm_fs /tmp/\n cp .github/workflows/bash/rm_list.sh /tmp/\n\n - name: Run Redis\n run: |\n sudo docker run -d --name redis -v redis-data:/data \\\n -p 6379:6379 redis redis-server --appendonly yes\n\n - name: Juicefs Format\n run: |\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs format --trash-days 0 redis://127.0.0.1:6379/1 --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n run: |\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d redis://127.0.0.1:6379/1 /tmp/jfs --no-usage-report\n\n - name: LTP FS\n timeout-minutes: 50\n run: |\n sudo .github/scripts/apt_install.sh libaio-dev libacl1-dev attr\n wget -O ltp.tar.gz https://github.com/linux-test-project/ltp/archive/refs/tags/20210927.tar.gz\n mkdir ltp\n tar -xvf ltp.tar.gz -C ltp --strip-components 1\n cd ltp\n ls -lh\n make autotools\n ./configure\n make\n sudo make install\n cd /opt/ltp\n sudo chmod +x /tmp/rm_list.sh\n sudo chmod 777 runtest/fs\n sudo /tmp/rm_list.sh /tmp/rm_fs /opt/ltp/runtest/fs\n sudo ./runltp -d /tmp/jfs -f fs,fs_perms_simple,fsx,io,fcntl-locktests -C result.log.failed -T result.log.tconf -l result.log\n\n - name: tconf Log\n if: always()\n run: |\n cat /opt/ltp/output/result.log.tconf\n\n - name: check ltpsyscall failed log\n if: always()\n run: |\n cat /opt/ltp/output/result.log.failed\n \n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\" \n" }, { "path": ".github/workflows/ltpsyscalls.yml", "content": "name: \"ltp-syscalls\"\n\non:\n push:\n branches:\n - 'release-**'\n paths-ignore:\n - 'docs/**'\n pull_request:\n #The branches below must be a subset of the branches above\n branches:\n - 'release-**'\n paths-ignore:\n - 'docs/**'\n schedule:\n - cron: '30 20 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n ltpsyscalls:\n timeout-minutes: 60\n strategy:\n fail-fast: false\n matrix:\n # meta: [ 'sqlite3', 'redis', 'mysql', 'tikv', 'tidb', 'postgres', 'mariadb', 'badger', 'fdb']\n meta: ['redis']\n type: [ 'head', 'middle', 'tail']\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Copy\n run: |\n cp .github/workflows/bash/rm_syscalls /tmp/\n cp .github/workflows/bash/rm_list.sh /tmp/\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs format --trash-days 0 $meta_url --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount --enable-xattr -d $meta_url /tmp/jfs --no-usage-report\n\n - name: Install LTP Syscalls\n run: |\n sudo .github/scripts/apt_install.sh libaio-dev libacl1-dev attr\n wget -O ltp.tar.gz https://github.com/linux-test-project/ltp/archive/refs/tags/20210927.tar.gz\n mkdir ltp\n tar -xvf ltp.tar.gz -C ltp --strip-components 1\n cd ltp\n make autotools\n ./configure\n make\n sudo make install\n\n - name: Run LTP Syscalls\n timeout-minutes: 60\n run: |\n cd /opt/ltp\n sudo chmod +x /tmp/rm_list.sh\n sudo chmod 777 runtest/syscalls\n sudo /tmp/rm_list.sh /tmp/rm_syscalls /opt/ltp/runtest/syscalls\n split -a 1 -d -l $(( $(wc -l < /opt/ltp/runtest/syscalls) / 3 + 1 )) /opt/ltp/runtest/syscalls /tmp/syscalls_\n sudo chmod 777 /tmp/syscalls_*\n if [ \"${{matrix.type}}\" == \"head\" ]; then\n cat /tmp/syscalls_0\n sudo ./runltp -d /tmp/jfs -C result.log.failed -T result.log.tconf -l result0.log -f /tmp/syscalls_0\n elif [ \"${{matrix.type}}\" == \"middle\" ]; then\n cat /tmp/syscalls_1\n sudo ./runltp -d /tmp/jfs -C result.log.failed -T result.log.tconf -l result1.log -f /tmp/syscalls_1\n elif [ \"${{matrix.type}}\" == \"tail\" ]; then\n cat /tmp/syscalls_2\n sudo ./runltp -d /tmp/jfs -C result.log.failed -T result.log.tconf -l result2.log -f /tmp/syscalls_2\n else\n echo \"matrix.type: ${{matrix.type}} is not valid\" && exit 1\n fi\n\n - name: tconf Log\n if: always()\n run: |\n cat /opt/ltp/output/result.log.tconf\n\n - name: check ltpsyscall failed log\n if: always()\n run: |\n cat /opt/ltp/output/result.log.failed\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\" \n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/mutate-test-sdk.yml", "content": "name: mutate-test-sdk\non:\n pull_request:\n branches:\n - 'main'\n paths:\n - '**/JuiceFileSystemTest.java'\n\n workflow_dispatch:\n inputs:\n targetTests:\n type: string\n description: \"Target tests, eg: io.juicefs.JuiceFileSystemTest\"\n required: true\n default: \"\" \n targetClasses:\n type: string\n description: \"Target classes, eg: io.juicefs.JuiceFileSystemImpl*\"\n required: true\n default: \"\" \n timeoutConstant:\n type: int\n description: \"Timeout constant\"\n required: true\n default: 1000 \n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false \n\njobs:\n mutate-test-sdk:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - name: Build\n uses: ./.github/actions/build\n\n - name: Set up Java\n uses: actions/setup-java@v3\n with:\n distribution: 'temurin'\n java-version: '8'\n\n - name: Run Redis\n run: |\n sudo docker run -d --name redis -v redis-data:/data \\\n -p 6379:6379 redis redis-server --appendonly yes\n\n - name: Juicefs Format\n run: |\n sudo ./juicefs format localhost --bucket=/mnt/jfs dev\n\n - name: Juicefs Mount\n run: |\n sudo ./juicefs mount -d localhost /jfs\n touch /jfs/inner_sym_target\n echo \"hello juicefs\" > /jfs/inner_sym_target\n cd /jfs\n ln -s inner_sym_target inner_sym_link \n mkdir etc\n chmod 777 etc\n echo `hostname` > etc/nodes\n\n - name: Make SDK\n run: |\n cd sdk/java\n make\n cd -\n\n - name: Change pom\n run: |\n if [ \"${{github.event_name}}\" == \"pull_request\" ]; then\n targetTests=\"io.juicefs.JuiceFileSystemTest\"\n targetClasses=\"io.juicefs.JuiceFileSystemImpl*\"\n timeConstant=1000\n elif [ \"${{github.event_name}}\" == \"workflow_dispatch\" ]; then\n targetTests=\"${{github.event.inputs.targetTests}}\"\n echo \"targetTests is $targetTests\"\n targetClasses=\"${{github.event.inputs.targetClasses}}\"\n echo \"targetClasses is $targetClasses\"\n timeConstant=\"${{github.event.inputs.timeConstant}}\"\n fi\n POM_XML_PATH=\"sdk/java/pom.xml\" TARGET_TESTS=$targetTests TARGET_CLASSES=$targetClasses TIME_CONSTANT=$timeConstant python3 .github/scripts/mutate/modify_sdk_pom.py\n cat sdk/java/pom.xml\n\n - name: Test SDK\n run: |\n cd sdk/java\n sudo mvn --no-transfer-progress test-compile org.pitest:pitest-maven:mutationCoverage\n cd -\n\n - name: Upload Pit Report\n uses: actions/upload-artifact@v4\n with:\n name: pit-reports\n path: sdk/java/target/pit-reports\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1" }, { "path": ".github/workflows/mutate-test.yml", "content": "name: mutate-test\non:\n pull_request:\n branches:\n - 'main'\n paths:\n - '**/*_test.go'\n\n workflow_dispatch:\n inputs:\n test_file:\n type: string\n description: \"the go test file relative path you want to mutate, eg cmd/meta/xattr_test.go\"\n required: true\n default: \"\" \n job_total:\n type: string\n description: \"number of job to run mutation test\"\n required: true\n default: \"1\"\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false \n\njobs:\n\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/checkout@v3\n with:\n fetch-depth: 0\n\n - name: Set up Go\n uses: actions/setup-go@v3\n with:\n go-version: 'oldstable'\n\n - name: install go-mutesting\n run: |\n go install github.com/zimmski/go-mutesting/cmd/go-mutesting@latest\n\n - id: set-matrix\n run: |\n sudo .github/scripts/apt_install.sh jq\n if [ \"${{github.event_name}}\" == \"pull_request\" ]; then\n echo github.event.pull_request.base.sha is ${{github.event.pull_request.base.sha}}\n echo github.event.pull_request.head.sha is ${{github.event.pull_request.head.sha}}\n echo github.sha is ${{ github.sha }}\n changed_file_str=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | grep _test.go$ | xargs)\n echo \"added or changed test files: $changed_file_str\"\n changed_file_array=($changed_file_str)\n declare -a Jobs=();\n for test_file_name in \"${changed_file_array[@]}\"\n do\n echo \"test_file_name is $test_file_name\"\n if grep -q \"//mutate:disable\" $test_file_name; then\n echo \"found //mutate:disable in $test_file_name\" \n continue\n fi\n source_file_name=${test_file_name%\"_test.go\"}.go\n echo \"source_file_name is :\" $source_file_name\n black_list_file=black.list\n TEST_FILE_NAME=\"$test_file_name\" BLACK_LIST_FILE=$black_list_file python3 .github/scripts/mutate/parse_black_list.py \n echo \"black list checksum: \"\n cat $black_list_file\n total_count=$(go-mutesting $source_file_name --debug --no-exec --blacklist $black_list_file| grep \"Save mutation into\" | wc -l)\n echo \"total_count is $total_count\"\n job_total=$(TEST_FILE_NAME=$test_file_name python3 .github/scripts/mutate/parse_job_total.py)\n echo \"job_total specified: $job_total\"\n if [ $job_total -eq 0 ]; then\n if [ $total_count -gt 200 ]; then\n job_total=4\n else\n job_total=1\n fi\n fi\n echo \"job_total: $job_total\"\n for i in `seq 1 $job_total` \n do\n Jobs=(\"${Jobs[@]}\" \"$test_file_name-$i-$job_total\")\n done\n done\n value=`printf '%s\\n' \"${Jobs[@]}\" | jq -R . | jq -cs .`\n echo \"value: $value\"\n echo \"matrix=$value\" >> $GITHUB_OUTPUT\n elif [ \"${{github.event_name}}\" == \"workflow_dispatch\" ]; then\n test_file_name=${{github.event.inputs.test_file}}\n echo \"test file is $test_file_name\"\n job_total=${{github.event.inputs.job_total}}\n echo \"job_total is $job_total\"\n declare -a Jobs=();\n for i in `seq 1 $job_total` \n do\n Jobs=(\"${Jobs[@]}\" \"$test_file_name-$i-$job_total\")\n done\n value=`printf '%s\\n' \"${Jobs[@]}\" | jq -R . | jq -cs .`\n echo \"value: $value\"\n echo \"matrix=$value\" >> $GITHUB_OUTPUT\n fi\n\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n\n mutate-test:\n timeout-minutes: 120\n if: \"!github.event.pull_request.draft\"\n name: ${{matrix.test_file}}\n needs: build-matrix\n strategy:\n fail-fast: false\n matrix:\n test_file: ${{ fromJson(needs.build-matrix.outputs.matrix) }}\n runs-on: ubuntu-22.04\n permissions:\n pull-requests: write\n steps:\n - uses: actions/checkout@v3\n\n - name: Get Current Job Log URL\n uses: Tiryoh/gha-jobid-action@v0\n id: jobs\n with:\n github_token: ${{ secrets.GITHUB_TOKEN }}\n job_name: ${{matrix.test_file}}\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n\n - name: Remove problem matcher for go\n run: |\n # https://github.com/actions/setup-go/blob/main/matchers.json\n echo \"::remove-matcher owner=go::\"\n\n - name: Install Packages\n run: |\n go install github.com/zimmski/go-mutesting/cmd/go-mutesting@latest\n sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr python3-tk\n sudo pip install mysqlclient\n apt -y install glusterfs-server\n\n - name: Prepare Database\n timeout-minutes: 10\n run: |\n docker run -d -p 9000:9000 -p 9001:9001 -e \"MINIO_ROOT_USER=testUser\" -e \"MINIO_ROOT_PASSWORD=testUserPassword\" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address \":9001\"\n go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc alias set local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket\n make\n # sudo make -C fstests setup\n\n - name: run mutate test\n # timeout-minutes: 120\n run: |\n sudo chmod 777 /var/jfsCache\n test_file=$(echo ${{matrix.test_file}} | awk -F'-' '{print $1}')\n job_index=$(echo ${{matrix.test_file}} | awk -F'-' '{print $2}')\n job_total=$(echo ${{matrix.test_file}} | awk -F'-' '{print $3}')\n echo \"test file is: $test_file, job_index is $job_index, job_total is $job_total\"\n if [ -z \"$test_file\" ]; then \n echo \"test file is empty, will not run mutate test\"\n exit 0\n fi\n source_file=${test_file%\"_test.go\"}.go\n echo \"source file is :\" $source_file\n package_path=$(dirname $test_file)\n echo \"package path is :\" $package_path\n\n test_cases=$(TEST_FILE_NAME=$test_file python3 .github/scripts/mutate/parse_test_cases.py || true)\n if [ \"$?\" -ne 0 ]; then\n echo \"no test cases in test file, will not run mutate test\"\n exit 0\n fi\n echo \"test cases: $test_cases\"\n\n if [[ \"$test_file\" =~ ^pkg/.* ]]; then\n go test ./$package_path/... -v -run \"$test_cases\" -count=1 -cover -timeout=5m -coverpkg=./$package_path/... -coverprofile=mutest-cov.out\n elif [[ \"$test_file\" =~ ^cmd/.* ]]; then\n sudo JFS_GC_SKIPPEDTIME=1 MINIO_ACCESS_KEY=testUser MINIO_SECRET_KEY=testUserPassword go test ./cmd/... -v -run \"$test_cases\" -count=1 -cover -timeout=5m -coverpkg=./pkg/...,./cmd/... -coverprofile=mutest-cov.out \n else\n echo \"test file location error: $test_file\"\n exit 0\n fi\n \n black_list_file=black.list\n TEST_FILE_NAME=\"$test_file\" BLACK_LIST_FILE=$black_list_file python3 .github/scripts/mutate/parse_black_list.py \n echo \"black list checksum: \"\n cat $black_list_file\n\n go-mutesting $source_file --debug --no-exec --do-not-remove-tmp-folder --blacklist $black_list_file | tee -a mutate.log\n mutation_dir=$(cat mutate.log | grep \"Save mutations into\" | awk -F' ' '{print $4}' | sed -e 's:\"::g')\n echo \"mutation dir is $mutation_dir\"\n JOB_INDEX=$job_index JOB_TOTAL=$job_total MUTATE_ORIGINAL=$source_file MUTATION_DIR=$mutation_dir COVERAGE_FILE=mutest-cov.out TEST_FILE_NAME=\"$test_file\" PACKAGE_PATH=\"$package_path\" STAT_RESULT_FILE=stat_result.log python3 .github/scripts/mutate/mutesting.py\n # COVERAGE_FILE=mutest-cov.out TEST_FILE_NAME=\"$test_file\" PACKAGE_PATH=\"$package_path\" go-mutesting $source_file --debug --exec=.github/scripts/mutate/mutest.sh --do-not-remove-tmp-folder --blacklist $black_list_file \n if [ $? != 0 ]; then echo \"run mutesting.py failed\" && exit 1; fi\n \n [[ -z \"${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} \" ]] && echo \": MYSQL_PASSWORD is empty\" && exit 0\n export MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} \n JOB_NAME=${{matrix.test_file}} JOB_URL=${{steps.jobs.outputs.html_url}} STAT_RESULT_FILE=stat_result.log python3 .github/scripts/mutate/save_report.py\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-22.04\n needs: [build-matrix, mutate-test]\n if: always() && !github.event.pull_request.draft\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Install tools\n run: |\n sudo pip install mysqlclient\n\n - name: Generate mutate report\n run: |\n [[ -z \"${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} \" ]] && echo \": MYSQL_PASSWORD is empty\" && exit 0\n export MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} \n mutate_report=$(python3 .github/scripts/mutate/query_report.py)\n echo \"mutate_report is $mutate_report\"\n # echo \"mutate_report=$mutate_report\" >> $GITHUB_ENV\n MY_STRING=$(cat << EOF\n $mutate_report\n EOF\n )\n echo \"MY_STRING<> $GITHUB_ENV\n echo \"$MY_STRING\" >> $GITHUB_ENV\n echo \"EOF\" >> $GITHUB_ENV\n \n - uses: mshick/add-pr-comment@v2\n with:\n allow-repeats: true\n message: |\n *Mutate Test Report*\n ${{env.MY_STRING}} \n \n Usage: https://github.com/juicedata/juicefs/blob/main/.github/scripts/mutate/how_to_use_mutate_test.md \n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n\n" }, { "path": ".github/workflows/perf-test.yml", "content": "name: \"JuiceFS mdtest Performance Comparison\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**/perf-test.yml'\n - '**/perf/**'\n pull_request:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**/perf-test.yml'\n - '**/perf/**'\n schedule:\n - cron: '0 20 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n current_version_commit:\n type: string\n description: \"Commit hash for current version to test (default: HEAD)\"\n required: false\n default: ''\n old_version_commit:\n type: string\n description: \"Commit hash for old version to compare (default: latest release)\"\n required: false\n default: ''\n\njobs:\n mdtest-perf-test:\n timeout-minutes: 60\n strategy:\n fail-fast: false\n matrix:\n meta: ['redis', 'redis-nocache', 'mysql', 'tikv']\n cases: [\"mdtest_fio\"]\n runs-on: ubuntu-22.04\n\n steps:\n - name: Clean up\n run: |\n sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/local/.ghcup\n sudo docker system prune -af\n sudo df -h\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 0 # Need full history for checking out specific commits\n\n - name: Install dependencies\n run: |\n sudo apt-get update\n sudo apt-get install -y build-essential bc\n sudo pip install minio\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Prepare meta db\n run: |\n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n engine_meta=${{matrix.meta}}\n if [[ \"$engine_meta\" == \"redis-nocache\" ]]; then\n engine_meta=\"redis\"\n fi\n start_meta_engine $engine_meta\n meta_url=$(get_meta_url $engine_meta)\n create_database $meta_url\n\n - name: Install mdtest & fio\n run: |\n sudo apt-get install mpich openmpi-bin libopenmpi-dev fio -y\n wget https://github.com/hpc/ior/releases/download/3.3.0/ior-3.3.0.tar.gz\n tar -xzvf ior-3.3.0.tar.gz\n cd ior-3.3.0\n ./configure && make && sudo make install\n\n # Build and test current version (either specified commit or HEAD)\n - name: Checkout and build current version\n if: ${{ inputs.current_version_commit != '' }}\n run: |\n mkdir -p ../juicefs-build\n cd ../juicefs-build\n git clone $GITHUB_SERVER_URL/$GITHUB_REPOSITORY .\n git checkout ${{ inputs.current_version_commit }}\n make\n cp juicefs ../juicefs/\n cd ../juicefs\n\n - name: Build current version (default)\n if: ${{ inputs.current_version_commit == '' }}\n timeout-minutes: 10\n uses: ./.github/actions/build\n with:\n target: ${{steps.vars.outputs.target}}\n\n - name: Run benckmark with current version\n run: |\n mkdir -p /tmp/jfs/mdtest\n sudo chmod 777 /tmp/jfs\n \n # Mount current version\n engine_meta=${{matrix.meta}}\n if [[ \"$engine_meta\" == \"redis-nocache\" ]]; then\n engine_meta=\"redis\"\n fi\n meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)\n sudo chmod 777 /mnt\n if [[ \"${{matrix.meta}}\" == \"redis-nocache\" ]]; then\n meta_url=${meta_url%%\\?*}\n echo \"Removed redis query parameters for redis-nocache profile\"\n fi\n ./juicefs format $meta_url current-version-test --trash-days 0 --bucket=/mnt/jfs\n ./juicefs mount -d \"$meta_url\" /tmp/jfs --no-usage-report\n \n # Run tests\n chmod +x .github/scripts/perf/*.sh\n .github/scripts/perf/${{matrix.cases}}.sh /tmp/jfs \"./results/current_${{matrix.meta}}\" \"current\" \"$meta_url\"\n\n - name: Cleanup current version\n run: |\n source .github/scripts/common/common.sh\n source .github/scripts/start_meta_engine.sh\n engine_meta=${{matrix.meta}}\n if [[ \"$engine_meta\" == \"redis-nocache\" ]]; then\n engine_meta=\"redis\"\n fi\n meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)\n if [[ \"${{matrix.meta}}\" == \"redis-nocache\" ]]; then\n meta_url=${meta_url%%\\?*}\n echo \"Removed redis query parameters for redis-nocache profile\"\n fi\n META_URL=$meta_url \n uuid=$(./juicefs status $meta_url | grep UUID | cut -d '\"' -f 4)\n ./juicefs destroy --force $meta_url $uuid\n ./juicefs umount /tmp/jfs\n rm -rf /mnt/jfs\n start_meta_engine $engine_meta\n create_database $meta_url\n prepare_test\n\n # Build and test old version (either specified commit or latest release)\n - name: Checkout and build old version from commit\n if: ${{ inputs.old_version_commit != '' }}\n run: |\n mkdir -p ../juicefs-build2\n cd ../juicefs-build2\n git clone $GITHUB_SERVER_URL/$GITHUB_REPOSITORY .\n git checkout ${{ inputs.old_version_commit }}\n make\n engine_meta=${{matrix.meta}}\n if [[ \"$engine_meta\" == \"redis-nocache\" ]]; then\n engine_meta=\"redis\"\n fi\n meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)\n mkdir -p /tmp/jfs_old/mdtest\n sudo chmod 777 /tmp/jfs_old\n if [[ \"$engine_meta\" == \"mysql\" || \"$engine_meta\" == \"redis\" ]]; then\n meta_url=${meta_url%%\\?*}\n echo \"Removed query parameters for old version compatibility\"\n fi\n ./juicefs format \"$meta_url\" old-version-test --trash-days 0 --bucket=/mnt/jfs\n ./juicefs mount -d \"$meta_url\" /tmp/jfs_old --no-usage-report\n cd ../juicefs\n\n - name: Install old JuiceFS version (default)\n if: ${{ inputs.old_version_commit == '' }}\n run: |\n curl -sSL https://d.juicefs.com/install | sh -\n JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '\"' -f 4 | tr -d 'v')\n engine_meta=${{matrix.meta}}\n if [[ \"$engine_meta\" == \"redis-nocache\" ]]; then\n engine_meta=\"redis\"\n fi\n meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)\n if [[ \"$engine_meta\" == \"mysql\" || \"$engine_meta\" == \"redis\" ]]; then\n meta_url=${meta_url%%\\?*}\n echo \"Removed query parameters for old version compatibility\"\n fi\n juicefs format \"$meta_url\" old-version-test --trash-days 0 --bucket=/mnt/jfs\n mkdir -p /tmp/jfs_old/mdtest\n sudo chmod 777 /tmp/jfs_old\n juicefs mount -d \"$meta_url\" /tmp/jfs_old --no-usage-report\n\n - name: Run benchmark with old version\n run: |\n engine_meta=${{matrix.meta}}\n if [[ \"$engine_meta\" == \"redis-nocache\" ]]; then\n engine_meta=\"redis\"\n fi\n meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)\n if [[ \"$engine_meta\" == \"mysql\" || \"$engine_meta\" == \"redis\" ]]; then\n meta_url=${meta_url%%\\?*}\n echo \"Removed query parameters for old version compatibility\"\n fi\n .github/scripts/perf/${{matrix.cases}}.sh /tmp/jfs_old \"./results/old_${{matrix.meta}}\" \"old\" \"$meta_url\"\n\n - name: Compare results\n run: |\n .github/scripts/perf/compare_${{matrix.cases}}.sh \"./results/current_${{matrix.meta}}\" \"./results/old_${{matrix.meta}}\" || true\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/permission-check.yaml", "content": "name: \"permission-check\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n - '**.java'\n - '**/pom.xml'\n pull_request:\n #The branches below must be a subset of the branches above\n branches:\n - 'main'\n - 'release-**'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n - '.github/**'\n - '**.java'\n - '**/pom.xml'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n pjdfstest:\n timeout-minutes: 30\n strategy:\n fail-fast: false\n matrix:\n meta: [ 'sqlite3', 'redis', 'badger' ]\n\n runs-on: ubuntu-22.04\n steps:\n - uses: shogo82148/actions-setup-perl@v1\n with:\n perl-version: '5.34'\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Set Variable\n id: vars\n run: echo \"target=juicefs\" >> $GITHUB_OUTPUT\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with:\n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: |\n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo ./juicefs format $meta_url --trash-days 0 pics\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n # sudo mkdir /var/jfs\n # sudo chmod 777 /var/jfs\n sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --attr-cache 0 --entry-cache 0 --dir-entry-cache 0 --non-default-permission &\n sleep 5\n if [ ! -f /tmp/jfs/.accesslog ]; then\n echo \": mount failed\"\n exit 1\n fi\n\n - name: Pjdfstest\n run: |\n sudo .github/scripts/apt_install.sh libtap-harness-archive-perl\n cd /tmp/jfs/\n git clone https://github.com/hexilee/pjdfstest.git\n cd pjdfstest\n autoreconf -ifs\n ./configure\n make pjdfstest\n sudo prove -rv tests/\n\n - name: log\n if: always()\n shell: bash\n run: |\n if [ -f ~/.juicefs/juicefs.log ]; then\n tail -300 ~/.juicefs/juicefs.log\n grep \":\" ~/.juicefs/juicefs.log && exit 1 || true\n fi\n if [ -f /var/log/juicefs.log ]; then\n tail -300 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n fi\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [pjdfstest]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/pjdfstest.yml", "content": "name: \"pjdfstest\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n - '.github/**'\n - '**.java'\n - '**/pom.xml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n - '.github/**'\n - '**.java'\n - '**/pom.xml'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]]; then\n echo 'meta_matrix=[\"sqlite3\", \"redis\", \"mysql\", \"tikv\", \"tidb\", \"postgres\", \"badger\", \"mariadb\", \"fdb\"]' >> $GITHUB_OUTPUT\n elif [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n echo 'meta_matrix=[\"redis\"]' >> $GITHUB_OUTPUT\n else\n echo 'event name is not supported' && exit 1\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n pjdfstest:\n needs: build-matrix\n strategy:\n fail-fast: false\n matrix:\n # [ 'sqlite3', 'redis', 'mysql', 'tikv', 'tidb', 'postgres', 'badger', 'mariadb', 'fdb']\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n\n steps:\n - uses: shogo82148/actions-setup-perl@v1\n with:\n perl-version: '5.34'\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n uses: ./.github/actions/build\n with:\n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: |\n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo ./juicefs format $meta_url --trash-days 0 --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n # sudo mkdir /var/jfs\n # sudo chmod 777 /var/jfs\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report\n stat /tmp/jfs/.accesslog\n\n - name: Pjdfstest\n run: |\n sudo .github/scripts/apt_install.sh libtap-harness-archive-perl\n cd /tmp/jfs/\n git clone https://github.com/sanwan/pjdfstest.git\n cd pjdfstest\n autoreconf -ifs\n ./configure\n make pjdfstest\n sudo prove -rv tests/\n\n - name: log\n if: always()\n shell: bash\n run: |\n if [ -f ~/.juicefs/juicefs.log ]; then\n tail -300 ~/.juicefs/juicefs.log\n grep \":\" ~/.juicefs/juicefs.log && exit 1 || true\n fi\n if [ -f /var/log/juicefs.log ]; then\n tail -300 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n fi\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [pjdfstest]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/pysdk.yml", "content": "name: \"pysdk\"\n\non:\n push:\n branches:\n - main\n - release**\n paths:\n - '**/hypo/fs_op.py'\n - '**/hypo/fs.py'\n - '**/hypo/fs_sdk_test.py'\n - '**/pysdk_test.py'\n - '**/juicefs.py'\n - '**/pysdk.yml'\n pull_request:\n branches:\n - main\n - release**\n paths:\n - '**/hypo/fs_op.py'\n - '**/hypo/fs.py'\n - '**/hypo/fs_sdk_test.py'\n - '**/pysdk_test.py'\n - '**/juicefs.py'\n - '**/pysdk.yml'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n seed:\n type: string\n description: \"Seed for random test\"\n required: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n # TODO: add fdb when bugfix: https://github.com/juicedata/juicefs/issues/5910\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]]; then\n echo 'meta_matrix=[\"sqlite3\", \"redis\", \"mysql\", \"tikv\", \"tidb\", \"postgres\", \"mariadb\"]' >> $GITHUB_OUTPUT\n elif [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n echo 'meta_matrix=[\"redis\", \"tikv\", \"mysql\"]' >> $GITHUB_OUTPUT\n else\n echo 'event name is not supported' && exit 1\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n pysdk:\n needs: build-matrix\n strategy:\n fail-fast: false\n matrix:\n # [ 'sqlite3', 'redis', 'mysql', 'tikv', 'tidb', 'postgres', 'badger', 'mariadb', 'fdb']\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Download example database\n timeout-minutes: 5\n uses: dawidd6/action-download-artifact@v9\n if: false\n with:\n name: pysdk-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n if_no_artifact_found: ignore\n workflow_conclusion: \"\"\n check_artifacts: true\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n \n - name: Build\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo ./juicefs format $meta_url --trash-days 0 --enable-acl --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n if: false\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr\n stat /tmp/jfs/.accesslog\n\n - name: Build and install SDK\n timeout-minutes: 5\n run: |\n make -C sdk/python/ libjfs.so\n sudo python3 sdk/python/juicefs/setup.py install\n df -h\n\n - name: Run juicefs.py\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo JFS_VOLUME=test-volume JFS_META=$meta_url python3 sdk/python/juicefs/juicefs/juicefs.py\n df -h\n\n - name: Run pysdk_test.py\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo META_URL=$meta_url python3 .github/scripts/pysdk/pysdk_test.py\n df -h\n\n - name: Run file_test.py\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo USE_SDK=true META_URL=$meta_url python3 .github/scripts/hypo/file_test.py\n df -h\n\n - name: Run file.py without read\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo USE_SDK=true META_URL=$meta_url EXCLUDE_RULES=\"read,readline,readlines\" python3 .github/scripts/hypo/file.py\n df -h\n\n - name: Run file.py without write\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo USE_SDK=true META_URL=$meta_url EXCLUDE_RULES=\"write,writelines,truncate\" python3 .github/scripts/hypo/file.py\n df -h\n\n - name: Run fs_sdk_test.py\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo USE_SDK=true META_URL=$meta_url python3 .github/scripts/hypo/fs_sdk_test.py\n df -h\n\n - name: Run fs.py\n timeout-minutes: 60\n run: |\n if [[ -n \"${{ github.event.inputs.seed }}\" ]]; then \n seed=${{ github.event.inputs.seed }}\n elif [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n seed=0\n else\n seed=$RANDOM\n fi\n echo seed is $seed\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo IGNORE_DIFF_ERRORS=true MAX_EXAMPLE=1000 STEP_COUNT=200 USE_SDK=true SEED=$seed META_URL=$meta_url EXCLUDE_RULES=\"readlines,readline,clone_cp_file,clone_cp_dir\" python3 .github/scripts/hypo/fs.py 2>&1 | tee fsrand.log\n exit ${PIPESTATUS[0]}\n df -h\n\n - name: check fsrand.log\n if: always()\n run: |\n sudo tail -n 500 fsrand.log || true\n\n - name: chmod example directory\n if: always()\n timeout-minutes: 5\n run: |\n if [[ -e \".hypothesis/examples\" ]]; then\n echo \"chmod for .hypothesis/examples\" && sudo chmod -R 755 .hypothesis/examples\n fi\n\n - name: Upload example database\n uses: actions/upload-artifact@v4\n if: false\n with:\n include-hidden-files: true\n name: pysdk-hypothesis-example-db-${{ matrix.meta }}\n path: .hypothesis/examples\n overwrite: true\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\" \n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [pysdk]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success() \n run: echo \"All Done\"" }, { "path": ".github/workflows/random-test.yml", "content": "name: \"random-test\"\non:\n pull_request:\n types: [opened, synchronize, reopened, ready_for_review]\n branches:\n - main\n - release**\n paths:\n - '**/workflows/random-test.yml'\n\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n duration:\n type: string\n description: \"duration in seconds\"\n required: false\n default: \"1800\"\n\n schedule:\n - cron: '0 16 * * *'\n\njobs:\n random-test:\n if: \"!github.event.pull_request.draft\"\n strategy:\n fail-fast: false\n matrix:\n meta: [\"redis\"]\n basedir: [ \"random-test\" ]\n subdir: [ \"true\" ]\n removeOp: [ \"rm\"]\n zipf: [\"\"]\n include:\n - basedir: \"random-test\"\n meta: \"mysql\"\n subdir: \"false\"\n removeOp: \"rmr\"\n zipf: \"1.02\"\n - basedir: \"\"\n meta: \"tikv\"\n subdir: \"false\"\n removeOp: \"noOp\"\n zipf: \"1.04\"\n - basedir: \"random-test\"\n meta: \"postgres\"\n subdir: \"false\"\n removeOp: \"rm\"\n zipf: \"1.06\"\n\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n uses: ./.github/actions/build\n with:\n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: |\n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo ./juicefs format $meta_url --trash-days 0 --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n [[ \"${{matrix.subdir}}\" == \"true\" ]] && subdir_option=\"--subdir=subdir\" || subdir_option=\"\"\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report $subdir_option\n stat /tmp/jfs/.accesslog\n \n - name: Test\n timeout-minutes: 125\n run: |\n if [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n duration=600\n elif [[ -n \"${{github.event.inputs.duration}}\" ]]; then\n duration=${{github.event.inputs.duration}}\n else\n duration=1800\n fi\n [[ \"${{matrix.zipf}}\" == \"\" ]] && merge_option=\"-mergeOp 1,uniform\" || merge_option=\"\"\n [[ \"${{matrix.zipf}}\" != \"\" ]] && zipf_option=\"-zipf ${{matrix.zipf}}\" || zipf_option=\"\"\n wget -q s.juicefs.com/static/Linux/mount -O mount && chmod +x mount\n set -x\n timeout $((duration + 60))s sudo ./mount random-test runOp --baseDir /tmp/jfs/${{matrix.basedir}} --logDir random-test-log \\\n --duration ${duration}s --files 1000000 --ops 1000000000 --threads 200 --dirSize 100 ${zipf_option} --skewInterval 30s --hotPoints $((RANDOM%5+1)) \\\n --mkdirOp 10,uniform -createOp 10,uniform -readOp 1,uniform -lsOp 1,uniform -deleteOp 0.1,uniform -rmrOp 0.03,end -renameOp 1,uniform -linkOp 3,uniform --truncateOp 1,uniform --truncateSize 1G,1G -inspectOp 30,uniform --cmdCloneOp 5,1s,10s --cmdRmrOp 5,1s,10s -walkOp 1,10s -walkThreads 5\n set +x\n\n - name: Remove test dir\n timeout-minutes: 30\n run: |\n echo \"Removing test dir /tmp/jfs/${{matrix.basedir}} with ${{matrix.removeOp}}\"\n if [[ \"${{matrix.removeOp}}\" == \"rm\" ]]; then\n sudo rm -rf /tmp/jfs/${{matrix.basedir}} || (find /tmp/jfs/${{matrix.basedir}} -exec stat -c '%n %i' {} + && exit 1)\n elif [[ \"${{matrix.removeOp}}\" == \"rmr\" ]]; then\n sudo ./juicefs rmr /tmp/jfs/${{matrix.basedir}} || (find /tmp/jfs/${{matrix.basedir}} -exec stat -c '%n %i' {} + && exit 1)\n elif [[ \"${{matrix.removeOp}}\" == \"rmr-skip-trash\" ]]; then\n sudo ./juicefs rmr /tmp/jfs/${{matrix.basedir}} --skip-trash || (find /tmp/jfs/${{matrix.basedir}} -exec stat -c '%n %i' {} + && exit 1)\n else\n echo \"no removeOp specified, skip removing test dir\"\n fi\n if [[ \"${{matrix.removeOp}}\" == \"rm\" || \"${{matrix.removeOp}}\" == \"rmr\" || \"${{matrix.removeOp}}\" == \"rmr-skip-trash\" ]]; then\n ls -ali /tmp/jfs/${{matrix.basedir}} && find /tmp/jfs/${{matrix.basedir}} -exec stat -c '%n %i' {} + && echo \"Error: /tmp/jfs/${{matrix.basedir}} still exists after remove\" && exit 1 || echo \"/tmp/jfs/${{matrix.basedir}} removed successfully\"\n fi\n \n - name: Check file-op.log\n timeout-minutes: 5\n if: always() && !github.event.pull_request.draft\n run: |\n sudo chmod -R a+r random-test-log\n ls -l random-test-log/\n [[ -f random-test-log/file-op.log ]] && tail -n 500 random-test-log/file-op.log || true\n\n - name: Check log\n if: always()\n shell: bash\n run: |\n if [ -f /var/log/juicefs.log ]; then\n tail -300 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n fi\n \n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Setup upterm session\n if: failure() && github.event.inputs.debug == 'true'\n # if: failure()\n timeout-minutes: 30\n uses: owenthereal/action-upterm@v1\n with:\n wait-timeout-minutes: 10\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [random-test]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"" }, { "path": ".github/workflows/release.yml", "content": "name: release\n\non:\n push:\n tags:\n - v*\n\njobs:\n releaser:\n runs-on: ubuntu-22.04\n steps:\n - name: Clean up\n run: |\n sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/local/.ghcup\n sudo docker system prune -af\n sudo df -h\n\n - name: Set up Go\n uses: actions/setup-go@v3\n with:\n go-version: \"oldstable\"\n\n - name: Set up Java\n uses: actions/setup-java@v3\n with:\n distribution: \"temurin\"\n java-version: \"8\"\n server-id: central\n server-username: MAVEN_USERNAME\n server-password: MAVEN_PASSWORD\n gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}\n gpg-passphrase: MAVEN_GPG_PASSPHRASE\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: setup release environment\n run: |-\n echo 'GITHUB_TOKEN=${{secrets.GH_PERSONAL_ACCESS_TOKEN}}' > .release-env\n\n - name: goreleaser release\n run: make release\n\n - name: Cache local Maven repository\n id: cache-maven\n uses: actions/cache@v3\n with:\n path: ~/.m2/repository\n key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}\n restore-keys: |\n ${{ runner.os }}-maven-\n\n - name: Chown go module cache\n run: sudo chown -R $USER $HOME/go/pkg/mod\n\n - name: Build SDK\n run: |\n make -C sdk/java package-all && sudo chown -R $USER sdk/java/target\n echo \"JUICEFS_VERSION=$(mvn -f sdk/java/pom.xml help:evaluate -Dexpression=project.version -q -DforceStdout)\" >> $GITHUB_ENV\n\n - name: Upload SDK\n uses: softprops/action-gh-release@v1\n with:\n files: |\n sdk/java/target/juicefs-hadoop-${{ env.JUICEFS_VERSION }}.jar\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Publish package\n run: mvn -f sdk/java/pom.xml deploy -DskipTests\n env:\n MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}\n MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}\n MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}\n\n - name: Setup upterm session\n if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/resources/core-site.xml", "content": "\n\n\n \n fs.defaultFS\n jfs://dev/\n \n \n fs.jfs.impl\n io.juicefs.JuiceFileSystem\n \n \n juicefs.cache-size\n 0\n \n \n juicefs.no-usage-report\n true\n \n \n juicefs.access-log\n /tmp/juicefs-access.log\n \n \n juicefs.dev.meta\n redis://127.0.0.1:6379/1\n \n\n" }, { "path": ".github/workflows/resources/load-balancer.conf", "content": " upstream backend {\n server 127.0.0.1:9000;\n server 127.0.0.1:9001;\n }\n\n # This server accepts all traffic to port 80 and passes it to the upstream.\n # Notice that the upstream name and the proxy_pass need to match.\n\n server {\n listen 8080;\n server_name localhost;\n location / {\n proxy_set_header Host $http_host;\n proxy_pass http://backend;\n }\n }\n\n client_max_body_size 100M;" }, { "path": ".github/workflows/resources/sync-options.txt", "content": "--dirs --include .* , -r --include .* , enable \n--dirs --exclude .* , -r --exclude .* , enable \n--dirs --exclude .* --exclude docs/ --exclude *.png , -r --exclude .* --exclude docs/ --exclude *.png , enable\n--dirs --include docs/ --include *.png --exclude * , -r --include docs/ --include *.png --exclude * , enable \n--dirs --exclude * --include docs/ --include *.png , -r --exclude * --include docs/ --include *.png , enable\n--dirs --include .github --include *.png --exclude .* , -r --include .github --include *.png --exclude .* , enable\n--dirs --include .github --include *.png --exclude .* , -r --include .github --include *.png --exclude .* , enable\n--dirs --exclude .* --include .github --include *.png , -r --exclude .* --include .github --include *.png , enable\n--dirs --include [a-f]*.go --exclude *.go ,-r --include [a-f]*.go --exclude *.go ,enable \n--dirs --include *_test.go --exclude *.go ,-r --include *_test.go --exclude *.go ,enable \n--dirs --include cmd/ --exclude *.go ,-r --include cmd/ --exclude *.go ,enable \n--dirs --include pk*/chu*/ --exclude *.go ,-r --include pk*/chu*/ --exclude *.go ,enable \n--dirs --include chu*/ --exclude pk*/ --exclude *.go ,-r --include chu*/ --exclude pk*/ --exclude *.go ,enable\n--dirs --include chun?/ --exclude pk*/ --exclude *.go ,-r --include chun?/ --exclude pk*/ --exclude *.go ,enable" }, { "path": ".github/workflows/resources/tpcds_datagen.scala", "content": "// Copyright 2015 Databricks\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nval scaleFactor = \"5\"\n\n// data format.\nval format = \"parquet\"\n// If false, float type will be used instead of decimal.\nval useDecimal = true\n// If false, string type will be used instead of date.\nval useDate = true\n// If true, rows with nulls in partition key will be thrown away.\nval filterNull = false\n// If true, partitions will be coalesced into a single file during generation.\nval shuffle = true\n\n// s3/dbfs path to generate the data to.\nval rootDir = s\"jfs:///tmp/performance-datasets/tpcds/sf$scaleFactor-$format/\"\n// name of database to be created.\nval databaseName = s\"tpcds_sf${scaleFactor}\" +\n s\"\"\"_${if (useDecimal) \"with\" else \"no\"}decimal\"\"\" +\n s\"\"\"_${if (useDate) \"with\" else \"no\"}date\"\"\" +\n s\"\"\"_${if (filterNull) \"no\" else \"with\"}nulls\"\"\"\n\n\nimport com.databricks.spark.sql.perf.tpcds.TPCDSTables\nval sqlContext = new org.apache.spark.sql.SQLContext(sc)\nval tables = new TPCDSTables(sqlContext, dsdgenDir = \"/tmp/tpcds-kit/tools\", scaleFactor = scaleFactor, useDoubleForDecimal = !useDecimal, useStringForDate = !useDate)\n\n\nimport org.apache.spark.deploy.SparkHadoopUtil\n// Limit the memory used by parquet writer\n// Compress with snappy:\nsqlContext.sparkContext.hadoopConfiguration.set(\"parquet.memory.pool.ratio\", \"0.1\")\nsqlContext.setConf(\"spark.sql.parquet.compression.codec\", \"snappy\")\n// TPCDS has around 2000 dates.\nspark.conf.set(\"spark.sql.shuffle.partitions\", \"10\")\n// Don't write too huge files.\nsqlContext.setConf(\"spark.sql.files.maxRecordsPerFile\", \"20000000\")\n\nval dsdgen_partitioned=10 // recommended for SF10000+.\nval dsdgen_nonpartitioned=10 // small tables do not need much parallelism in generation.\n\n// generate all the small dimension tables\nval nonPartitionedTables = Array(\"call_center\", \"catalog_page\", \"customer\", \"customer_address\", \"customer_demographics\", \"date_dim\", \"household_demographics\", \"income_band\", \"item\", \"promotion\", \"reason\", \"ship_mode\", \"store\", \"time_dim\", \"warehouse\", \"web_page\", \"web_site\")\nnonPartitionedTables.foreach { t => {\n tables.genData(\n location = rootDir,\n format = format,\n overwrite = true,\n partitionTables = true,\n clusterByPartitionColumns = shuffle,\n filterOutNullPartitionValues = filterNull,\n tableFilter = t,\n numPartitions = dsdgen_nonpartitioned)\n}}\nprintln(\"Done generating non partitioned tables.\")\n\n// leave the biggest/potentially hardest tables to be generated last.\nval partitionedTables = Array(\"inventory\", \"web_returns\", \"catalog_returns\", \"store_returns\", \"web_sales\", \"catalog_sales\", \"store_sales\") \npartitionedTables.foreach { t => {\n tables.genData(\n location = rootDir,\n format = format,\n overwrite = true,\n partitionTables = true,\n clusterByPartitionColumns = shuffle,\n filterOutNullPartitionValues = filterNull,\n tableFilter = t,\n numPartitions = dsdgen_partitioned)\n}}\nprintln(\"Done generating partitioned tables.\")\n\n// COMMAND ----------\n\nsql(s\"drop database if exists $databaseName cascade\")\nsql(s\"create database $databaseName\")\n\n// COMMAND ----------\n\nsql(s\"use $databaseName\")\n\n// COMMAND ----------\n\ntables.createExternalTables(rootDir, format, databaseName, overwrite = true, discoverPartitions = true)\n\n// COMMAND ----------\n\n// MAGIC %md\n// MAGIC Analyzing tables is needed only if cbo is to be used.\n\n// COMMAND ----------\n\ntables.analyzeTables(databaseName, analyzeColumns = true)\n\nSystem.exit(0)\n" }, { "path": ".github/workflows/resources/tpcds_run.scala", "content": "// Copyright 2015 Databricks\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Databricks notebook source\n// MAGIC %md \n// MAGIC This notebook runs spark-sql-perf TPCDS benchmark on and saves the result.\n\n// COMMAND ----------\n\n// Database to be used:\n// TPCDS Scale factor\nval scaleFactor = \"5\"\n// If false, float type will be used instead of decimal.\nval useDecimal = true\n// If false, string type will be used instead of date.\nval useDate = true\n// name of database to be used.\nval filterNull = false\n\nval databaseName = s\"tpcds_sf${scaleFactor}\" +\n s\"\"\"_${if (useDecimal) \"with\" else \"no\"}decimal\"\"\" +\n s\"\"\"_${if (useDate) \"with\" else \"no\"}date\"\"\" +\n s\"\"\"_${if (filterNull) \"no\" else \"with\"}nulls\"\"\"\n\nval iterations = 2 // how many times to run the whole set of queries.\n\nval timeout = 60 // timeout in hours\n\nval query_filter = Seq(\"q1-v2.4\", \"q2-v2.4\", \"q3-v2.4\", \"q4-v2.4\", \"q5-v2.4\", \"q6-v2.4\", \"q7-v2.4\", \"q8-v2.4\", \"q9-v2.4\", \"q10-v2.4\") // Seq() == all queries\nval randomizeQueries = false // run queries in a random order. Recommended for parallel runs.\n\n// detailed results will be written as JSON to this location.\nval resultLocation = \"file:///tmp/performance-datasets/tpcds/results\"\n\n// COMMAND ----------\n\n// Spark configuration\nspark.conf.set(\"spark.sql.broadcastTimeout\", \"10000\") // good idea for Q14, Q88.\n\n// ... + any other configuration tuning\n\n// COMMAND ----------\n\nsql(s\"use `$databaseName`\")\n\n// COMMAND ----------\n\nimport com.databricks.spark.sql.perf.tpcds.TPCDS\nval sqlContext = new org.apache.spark.sql.SQLContext(sc)\nval tpcds = new TPCDS (sqlContext = sqlContext)\ndef queries = {\n val filtered_queries = query_filter match {\n case Seq() => tpcds.tpcds2_4Queries\n case _ => tpcds.tpcds2_4Queries.filter(q => query_filter.contains(q.name))\n }\n if (randomizeQueries) scala.util.Random.shuffle(filtered_queries) else filtered_queries\n}\nval experiment = tpcds.runExperiment(\n queries,\n iterations = iterations,\n resultLocation = resultLocation,\n tags = Map(\"runtype\" -> \"benchmark\", \"database\" -> databaseName, \"scale_factor\" -> scaleFactor))\n\nexperiment.waitForFinish(timeout*60*60)\n\nexperiment.getCurrentResults.createOrReplaceTempView(\"result\")\nspark.sql(\"select substring(name,1,100) as Name, bround((parsingTime+analysisTime+optimizationTime+planningTime+executionTime)/1000.0,1) as Runtime_sec from result\").show()\n\nSystem.exit(0)\n//display(summary)\n" }, { "path": ".github/workflows/resources/vdbench_big_file.conf", "content": "data_errors=1\nfsd=fsd1,anchor=/tmp/vdbench/vdbench-big,depth=1,width=1,files=4,size=1g,openflags=o_direct\n\nfwd=fwd1,fsd=fsd1,operation=write,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4\nfwd=fwd2,fsd=fsd1,operation=read,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4\n\nrd=rd1,fwd=fwd1,fwdrate=max,format=restart,elapsed=10,interval=1\nrd=rd2,fwd=fwd2,fwdrate=max,format=restart,elapsed=10,interval=1\n" }, { "path": ".github/workflows/resources/vdbench_long_run.conf", "content": "data_errors=1\nfsd=fsd1,anchor=/tmp/jfs,depth=1,width=2,files=2,sizes=(10m,0),shared=yes,openflags=o_direct\nfwd=fwd1,fsd=fsd1,threads=4,xfersize=(512,20,4k,20,64k,20,512k,20,1024k,20),fileio=random,fileselect=random,rdpct=70\nrd=rd1,fwd=fwd*,fwdrate=max,format=restart,elapsed=1500,interval=1\n" }, { "path": ".github/workflows/resources/vdbench_small_file.conf", "content": "data_errors=1\nfsd=fsd1,anchor=/tmp/vdbench/vdbench-small,depth=3,width=10,files=10,size=128k,openflags=o_direct\n\nfwd=fwd1,fsd=fsd1,operation=write,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd2,fsd=fsd1,operation=read,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd3,fsd=fsd1,rdpct=70,xfersize=128k,fileio=random,fileselect=random,threads=4\n\nrd=rd1,fwd=fwd1,fwdrate=max,format=restart,elapsed=60,interval=1\nrd=rd2,fwd=fwd2,fwdrate=max,format=restart,elapsed=60,interval=1\nrd=rd3,fwd=fwd3,fwdrate=max,format=restart,elapsed=60,interval=1\n" }, { "path": ".github/workflows/rmfiles.yml", "content": "name: \"rmr-test\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/rmfiles.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/rmfiles.yml'\n schedule:\n - cron: '0 20 * * *'\n workflow_dispatch:\n\njobs:\n rmr-test:\n timeout-minutes: 30\n strategy:\n fail-fast: false\n matrix:\n meta: [ 'sqlite3', 'redis', 'mysql', 'postgres', 'tikv', 'fdb', 'badger', 'etcd']\n\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} \n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: | \n sudo chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n\n - name: Rmr\n shell: bash\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n mp=/tmp/jfs\n # wget -q https://s.juicefs.com/static/bench/500K_empty_files.dump.gz\n # gzip -dk 500K_empty_files.dump.gz\n # ./juicefs load $meta_url 500K_empty_files.dump\n sudo chmod 777 /mnt\n GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --bucket=/mnt/jfs jfs\n GOCOVERDIR=$(pwd)/cover ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 10 --threads 10 --no-usage-report\n GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url $mp --no-usage-report\n sleep 3\n ls -l $mp/test\n GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/\n sleep 3 \n ls -l $mp/test && exit 1 || true\n \n - name: Clear\n run: | \n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n mp=/tmp/jfs\n volume=jfs\n test -d $mp && ./juicefs umount -f $mp\n ./juicefs status $meta_url && UUID=$(./juicefs status $meta_url | grep UUID | cut -d '\"' -f 4) || echo \"meta not exist\"\n if [ -n \"$UUID\" ];then\n ./juicefs destroy --yes $meta_url $UUID\n fi\n test -d /var/jfs/$volume && rm -rf /var/jfs/$volume || true\n shell: bash\n\n - name: Rmr Parallel\n shell: bash\n run: |\n sudo chmod 777 /var\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n mp=/tmp/jfs\n # wget -q https://s.juicefs.com/static/bench/500K_empty_files.dump.gz\n # gzip -dk 500K_empty_files.dump.gz\n # ./juicefs load $meta_url 500K_empty_files.dump\n GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --bucket=/mnt/jfs jfs\n GOCOVERDIR=$(pwd)/cover ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 15 --threads 10 --no-usage-report\n GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url $mp --no-usage-report\n sleep 3\n declare -a pidlist\n GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true &\n pidlist+=($!)\n GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true &\n pidlist+=($!)\n GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true &\n pidlist+=($!)\n wait \"${pidlist[@]}\"\n ls -l $mp/test && exit 1 || true\n\n - name: log\n if: always()\n shell: bash\n run: | \n tail -300 ~/.juicefs/juicefs.log\n grep \":\" ~/.juicefs/juicefs.log && exit 1 || true\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} \n \n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [rmr-test]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"" }, { "path": ".github/workflows/sdktest.yml", "content": "name: \"sdktest\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n pull_request:\n #The branches below must be a subset of the branches above\n branches:\n - 'main'\n - 'release-*'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n - '.github/**'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n schedule:\n - cron: '0 17 * * *'\n\njobs:\n sdktest:\n timeout-minutes: 50\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n\n - name: Set up Java\n uses: actions/setup-java@v3\n with:\n distribution: 'temurin'\n java-version: '8'\n\n - name: Run Redis\n run: |\n sudo docker run -d --name redis -v redis-data:/data \\\n -p 6379:6379 redis redis-server --appendonly yes\n\n - name: Juicefs Format\n run: |\n sudo ./juicefs format localhost --bucket=/mnt/jfs dev\n\n - name: Juicefs Mount\n run: |\n sudo ./juicefs mount -d localhost /jfs\n touch /jfs/inner_sym_target\n echo \"hello juicefs\" > /jfs/inner_sym_target\n cd /jfs\n ln -s inner_sym_target inner_sym_link\n mkdir etc\n chmod 777 etc\n echo `hostname` > etc/nodes\n echo \"tom:3001\" > users\n echo \"g1:2001:tom\" > groups\n mkdir /jfs/tmp\n chmod 777 /jfs/tmp\n\n - name: Sdk Test\n run: |\n sudo sh sdk/java/kerberos.sh\n make -C sdk/java/libjfs\n cd sdk/java\n sudo mvn test -B -Dtest=\\!io.juicefs.permission.**,\\!io.juicefs.kerberos.**\n sudo mvn test -B -Dflink.version=1.17.2 -Dtest=io.juicefs.JuiceFileSystemTest#testFlinkHadoopRecoverableWriter\n # ranger test\n sudo JUICEFS_RANGER_TEST=1 mvn test -B -Dtest=io.juicefs.permission.RangerPermissionCheckerTest,\\!io.juicefs.permission.RangerPermissionCheckerTest#testRangerCheckerInitFailed\n sudo mvn test -B -Dtest=io.juicefs.permission.RangerPermissionCheckerTest#testRangerCheckerInitFailed\n # kerberos test\n sudo cp src/test/resources/kerberos.cfg /tmp/kerberos.cfg\n sudo sh -c \"echo \"dev.keytab=`base64 /tmp/server.keytab -w 0`\" >> /tmp/kerberos.cfg\"\n sudo ../../juicefs config localhost --kerberos-config-file /tmp/kerberos.cfg\n sudo mvn test -B -Dtest=io.juicefs.kerberos.KerberosTest\n \n sudo mvn package -B -Dmaven.test.skip=true --quiet -Dmaven.javadoc.skip=true\n expect=$(git rev-parse --short HEAD | cut -b 1-7)\n ls /tmp/libjfs* | grep $expect\n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/sync.yml", "content": "name: \"sync\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**.go'\n - 'Makefile'\n - '**/sync.yml'\n - '.github/scripts/sync/**'\n - '.github/scripts/hypo/sync.py'\n - '.github/scripts/hypo/sync_test.py'\n - '.github/actions/upload-coverage/action.yml'\n - '.github/actions/mount-coverage-dir/action.yml'\n - '.github/actions/upload-total-coverage/action.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**.go'\n - 'Makefile'\n - '**/sync.yml'\n - '.github/scripts/sync/**'\n - '.github/scripts/hypo/sync.py'\n - '.github/scripts/hypo/sync_test.py'\n - '.github/actions/upload-coverage/action.yml'\n - '.github/actions/mount-coverage-dir/action.yml'\n - '.github/actions/upload-total-coverage/action.yml'\n \n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n seed:\n type: string\n description: \"Seed for random test\"\n required: false\n schedule:\n - cron: '0 17 * * *'\n\njobs:\n sync:\n runs-on: ubuntu-22.04\n strategy:\n fail-fast: false\n matrix:\n type: ['sync', 'sync_encrypt', 'sync_fsrand', 'sync_minio', 'sync_cluster', 'sync_exclude']\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Build \n uses: ./.github/actions/build\n\n - name: Clean up\n run: |\n sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/local/.ghcup\n sudo docker system prune -af\n sudo df -h\n\n - name: Test Sync\n timeout-minutes: 20\n run: |\n if [[ \"${{matrix.type}}\" == 'sync' ]]; then\n sudo GOCOVERDIR=$(pwd)/cover META=redis .github/scripts/sync/sync.sh\n elif [[ \"${{matrix.type}}\" == 'sync_encrypt' ]]; then\n sudo GOCOVERDIR=$(pwd)/cover ENCRYPT=true META=redis .github/scripts/sync/sync.sh\n elif [[ \"${{matrix.type}}\" == 'sync_fsrand' ]]; then\n if [[ -n \"${{ github.event.inputs.seed }}\" ]]; then \n seed=${{ github.event.inputs.seed }}\n elif [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n seed=0\n else\n seed=$RANDOM\n fi\n echo \"using seed: $seed\"\n sudo GOCOVERDIR=$(pwd)/cover META=redis SEED=\"$seed\" .github/scripts/sync/sync_fsrand.sh \n elif [[ \"${{matrix.type}}\" == 'sync_minio' ]]; then\n sudo GOCOVERDIR=$(pwd)/cover META=redis .github/scripts/sync/sync_minio.sh \n elif [[ \"${{matrix.type}}\" == 'sync_cluster' ]]; then\n wget https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/random-test/random-test\n chmod +x random-test\n types=(\"ecdsa\" \"ed25519\" \"rsa\")\n random_type=${types[$RANDOM % ${#types[@]}]}\n sudo CI=true GOCOVERDIR=$(pwd)/cover META=redis KEY_TYPE=$random_type .github/scripts/sync/sync_cluster.sh\n elif [[ \"${{matrix.type}}\" == 'sync_exclude' ]]; then\n sudo GOCOVERDIR=$(pwd)/cover python3 .github/scripts/hypo/sync_test.py\n sudo GOCOVERDIR=$(pwd)/cover MAX_EXAMPLE=50 STEP_COUNT=30 PROFILE=ci python3 .github/scripts/hypo/sync.py\n else\n echo \"Unknown type: ${{matrix.type}}\"\n exit 1\n fi\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} \n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [sync]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n \n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: upload total coverage report\n timeout-minutes: 30\n continue-on-error: true\n uses: ./.github/actions/upload-total-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch' \n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/unit-random-tests.yml", "content": "name: \"unit-random-tests\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - 'pkg/meta/random_test.go'\n - '**/unit-random-tests.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - 'pkg/meta/random_test.go'\n - '**/unit-random-tests.yml'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n schedule:\n - cron: '0 17 * * *'\n\njobs:\n unit-random-tests:\n strategy:\n fail-fast: false\n matrix:\n meta: ['redis', 'sqlite3', 'tikv']\n timeout-minutes: 120\n runs-on: ubuntu-22.04\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Test\n timeout-minutes: 60\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n number=10000\n if [[ ${{matrix.meta}} == \"tikv\" ]]; then\n number=1500\n fi\n make unit-random-test seed=$RANDOM checks=${number} steps=200 meta=\"$meta_url\"\n\n - name: print failfile content\n if: failure()\n run: |\n pwd\n cat pkg/meta/testdata/rapid/TestFSOps/TestFSOps-*.fail\n continue-on-error: true\n\n - name: upload coverage report\n if: always()\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} \n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [unit-random-tests]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run:\n exit 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: upload total coverage report\n timeout-minutes: 30\n continue-on-error: true\n uses: ./.github/actions/upload-total-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch' \n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n\n" }, { "path": ".github/workflows/unittests.yml", "content": "name: \"unittests\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n - '**.java'\n - '**/pom.xml'\n pull_request:\n #The branches below must be a subset of the branches above\n branches:\n - 'main'\n - 'release-*'\n paths-ignore:\n - '.autocorrectrc'\n - '.markdownlint-cli2.jsonc'\n - 'package*.json'\n - 'docs/**'\n - '**.md'\n - '.github/**'\n - '!.github/workflows/unittests.yml'\n - '**.java'\n - '**/pom.xml'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n schedule:\n - cron: '0 17 * * *'\njobs:\n unittests:\n strategy:\n fail-fast: false\n matrix:\n test: [ 'test.meta.core','test.meta.non-core','test.pkg','test.cmd', 'test.fdb' ]\n timeout-minutes: 60\n runs-on: ubuntu-22.04\n env:\n MINIO_TEST_BUCKET: 127.0.0.1:9000/testbucket\n MINIO_ACCESS_KEY: testUser\n MINIO_SECRET_KEY: testUserPassword\n GLUSTER_VOLUME: jfstest/gv0\n DISPLAY_PROGRESSBAR: false\n HDFS_ADDR: localhost:8020\n SFTP_HOST: localhost:2222:/home/testUser1/upload/\n SFTP_USER: testUser1\n SFTP_PASS: password\n CIFS_ADDR: localhost:4445/Data\n CIFS_USER: samba\n CIFS_PASSWORD: secret\n WEBDAV_TEST_BUCKET: 127.0.0.1:9007\n TIKV_ADDR: 127.0.0.1\n REDIS_ADDR: redis://127.0.0.1:6379/13\n ETCD_ADDR: 127.0.0.1:3379\n MYSQL_ADDR: (127.0.0.1:3306)/dev\n MYSQL_USER: root\n NFS_ADDR: 127.0.0.1:/srv/nfs/\n steps:\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n\n - name: Install Packages\n run: |\n sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr glusterfs-server libglusterfs-dev nfs-kernel-server\n sudo mkdir -p /home/travis/.m2/\n - if: matrix.test == 'test.pkg'\n name: Set up nfs-server\n run: |\n sudo mkdir -p /srv/nfs\n sudo chown nobody:nogroup /srv/nfs\n sudo chmod 777 /srv/nfs\n echo \"/srv/nfs 127.0.0.1(rw,sync,insecure)\" | sudo tee -a /etc/exports\n sudo systemctl start nfs-kernel-server.service\n sudo exportfs -arv\n\n - if: matrix.test == 'test.meta.non-core'\n name: Install redis-cluster\n uses: vishnudxb/redis-cluster@1.0.5\n with:\n master1-port: 7000\n master2-port: 7001\n master3-port: 7002\n slave1-port: 7003\n slave2-port: 7004\n slave3-port: 7005\n\n - name: Prepare Database\n run: |\n TEST=${{matrix.test}} ./.github/scripts/prepare_db.sh\n\n - name: Clean up\n run: |\n sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/local/.ghcup\n sudo docker system prune -af\n sudo df -h\n\n - name: Unit Test\n timeout-minutes: 30\n run: |\n test=${{matrix.test}}\n make $test\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Code Coverage\n uses: codecov/codecov-action@v3\n with:\n files: cover/cover.txt\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 120\n uses: lhotari/action-upterm@v1\n\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [unittests]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"\n" }, { "path": ".github/workflows/vdbench.yml", "content": "name: \"vdbench\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**/vdbench.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-*'\n paths:\n - '**/vdbench.yml'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n schedule:\n - cron: '0 17 * * *'\n\njobs:\n vdbench:\n timeout-minutes: 60\n strategy:\n fail-fast: false\n matrix:\n meta: [ 'redis', 'mysql', 'fdb', 'tikv']\n #storage: [ 'local', 'minio', 'cifs' ]\n storage: [ 'cifs' ]\n runs-on: ubuntu-22.04\n\n steps: \n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Remove unused software\n timeout-minutes: 10\n run: |\n echo \"before remove unused software\"\n sudo df -h\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /opt/ghc\n echo \"after remove unused software\"\n sudo df -h\n\n - name: Build\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}} \n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Install tools\n shell: bash\n run: |\n wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc\n chmod +x mc \n wget -q https://s.juicefs.com/static/bench/vdbench50407.zip\n unzip vdbench50407.zip -d vdbench\n\n - name: vdbench-long-run\n shell: bash\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n storage='${{matrix.storage}}'\n format_opts='--bucket=/mnt/jfs'\n\n if [ \"$storage\" == \"minio\" ]; then\n start_meta_engine none minio\n ./mc alias set myminio http://127.0.0.1:9000 minioadmin minioadmin\n ./mc mb myminio/vdbench-long-run || true\n format_opts='--storage minio --bucket=http://127.0.0.1:9000/vdbench-long-run --access-key=minioadmin --secret-key=minioadmin'\n elif [ \"$storage\" == \"cifs\" ]; then\n SMB_CONTAINER_NAME=\"juicefs-ci-smb\"\n SMB_USER=\"juicefs\"\n SMB_PASSWORD=\"juicefs\"\n SMB_SHARE=\"share\" \n SMB_DATA_DIR=\"/mnt/jfs/${SMB_CONTAINER_NAME}-data\"\n docker rm -f \"$SMB_CONTAINER_NAME\" >/dev/null 2>&1 || true\n sudo mkdir -p $SMB_DATA_DIR\n sudo chmod 0777 $SMB_DATA_DIR -R\n docker run -d --name \"$SMB_CONTAINER_NAME\" \\\n -v \"$SMB_DATA_DIR\":/mount \\\n dperson/samba \\\n -r \\\n -u \"$SMB_USER;$SMB_PASSWORD\" \\\n -s \"$SMB_SHARE;/mount;yes;no;no;$SMB_USER\" >/dev/null\n container_ip=$(docker container inspect \"$SMB_CONTAINER_NAME\" --format '{{ .NetworkSettings.IPAddress }}')\n for _ in $(seq 1 40); do\n if (echo > /dev/tcp/${container_ip}/445) >/dev/null 2>&1; then\n break\n fi\n sleep 1\n done\n SMB_ENDPOINT=\"${container_ip}/${SMB_SHARE}\"\n format_opts=\"--storage cifs --bucket=${SMB_ENDPOINT} --access-key=${SMB_USER} --secret-key=${SMB_PASSWORD}\"\n fi\n\n sudo chmod 777 /mnt\n GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url vdbench-long-run --trash-days 0 $format_opts\n GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --cache-size 1024 --max-deletes 50\n vdbench/vdbench -f .github/workflows/resources/vdbench_long_run.conf -jn\n\n - uses: actions/upload-artifact@v4\n with:\n name: output-long-run-${{ matrix.meta }}-${{ matrix.storage }}\n path: output\n\n - name: check vdbench log\n if: always()\n run: |\n grep -i \"java.lang.RuntimeException\" output/errorlog.html && exit 1 || true\n if ! grep -q \"Vdbench execution completed successfully\" output/logfile.html; then\n echo \"vdbench not completed succeed\"\n exit 1\n fi\n\n - name: log\n if: always()\n run: | \n tail -300 ~/.juicefs/juicefs.log\n grep \":\" ~/.juicefs/juicefs.log && exit 1 || true\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} \n\n - name: Send Slack Notification\n if: failure()\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\" \n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/verify.yml", "content": "name: verify\n\non:\n push:\n branches:\n - main\n - \"release-**\"\n paths-ignore:\n - \".autocorrectrc\"\n - \".markdownlint-cli2.jsonc\"\n - \"package*.json\"\n - \"docs/**\"\n - \"**.md\"\n - \".github/**\"\n - \"!.github/workflows/verify.yml\"\n pull_request:\n branches:\n - \"main\"\n - \"release-**\"\n paths-ignore:\n - \".autocorrectrc\"\n - \".markdownlint-cli2.jsonc\"\n - \"package*.json\"\n - \"docs/**\"\n - \"**.md\"\n - \".github/**\"\n - \"!.github/workflows/verify.yml\"\n workflow_dispatch:\n schedule:\n - cron: \"0 17 * * 0\"\n\njobs:\n lint:\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/setup-go@v3\n with:\n go-version: \"1.23\"\n\n - uses: actions/checkout@v3\n - name: golangci-lint\n uses: golangci/golangci-lint-action@v6\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n build:\n strategy:\n fail-fast: false\n matrix:\n version: [\"1.21\", \"1.22\", \"1.23\"]\n runs-on: ubuntu-22.04\n steps:\n - name: Check out code\n uses: actions/checkout@v3\n\n - name: Set up Go\n uses: actions/setup-go@v3\n with:\n go-version: ${{ matrix.version }}\n cache: true\n id: go\n\n - name: Install dependencies\n run: |\n sudo .github/scripts/apt_install.sh g++-multilib gcc-mingw-w64\n\n - name: Go mod tidy\n run: |\n go mod tidy\n\n - name: Build linux target\n timeout-minutes: 10\n run: |\n make\n ./juicefs version\n\n - name: build lite\n timeout-minutes: 10\n run: |\n make juicefs.lite\n ./juicefs.lite version\n\n - name: build windows\n timeout-minutes: 10\n run: make juicefs.exe\n\n - name: build libjfs.dll\n timeout-minutes: 10\n run: make -C sdk/java/libjfs libjfs.dll\n\n - name: build ceph\n timeout-minutes: 10\n run: |\n sudo .github/scripts/apt_install.sh librados-dev\n make juicefs.ceph\n ./juicefs.ceph version\n\n - name: build fdb\n timeout-minutes: 10\n run: |\n wget https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb\n sudo dpkg -i foundationdb-clients_6.3.23-1_amd64.deb\n make juicefs.fdb\n ./juicefs.fdb version\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n # if: failure()\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n" }, { "path": ".github/workflows/version_compatible_hypo.yml", "content": "name: \"version-compatible-test-hypo\"\n\non:\n push:\n branches: \n - main\n paths:\n - '**/testVersionCompatible.py'\n - '**/version_compatible_hypo.yml'\n pull_request:\n branches: \n - main\n paths:\n - '**/testVersionCompatible.py'\n - '**/version_compatible_hypo.yml'\n schedule:\n - cron: '0 19 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n vc-hypo:\n timeout-minutes: 120\n continue-on-error: false\n strategy:\n fail-fast: false\n matrix:\n old_juicefs_version: ['main', 'release-1.0']\n meta: ['redis', 'mysql', 'tikv']\n storage: ['minio']\n include:\n - old_juicefs_version: 'main'\n meta: 'fdb'\n storage: 'minio'\n - old_juicefs_version: 'release-1.0'\n meta: 'postgres'\n storage: 'minio'\n\n runs-on: ubuntu-22.04\n\n steps:\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build new version\n timeout-minutes: 10\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Checkout old version\n uses: actions/checkout@v3\n with:\n ref: ${{matrix.old_juicefs_version}}\n path: ${{matrix.old_juicefs_version}}\n\n - name: Make old build\n timeout-minutes: 10\n run: | \n cd ${{matrix.old_juicefs_version}}\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"build juicefs.fdb\"\n wget -q https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb\n sudo dpkg -i foundationdb-clients_6.3.23-1_amd64.deb\n make juicefs.fdb\n mv juicefs.fdb juicefs\n else\n echo \"build juicefs\"\n make juicefs \n fi\n cd -\n\n - name: Prepare meta database\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}} ${{matrix.storage}}\n # meta_url=$(get_meta_url ${{matrix.meta}})\n # create_database $meta_url\n \n - name: Install tools\n run: | \n wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc\n chmod +x mc\n sudo mv mc /usr/local/bin\n sudo .github/scripts/apt_install.sh redis-tools\n sudo pip install hypothesis\n sudo pip install minio\n sudo pip install xattr\n \n - name: Test\n timeout-minutes: 90\n run: | \n export META=${{matrix.meta}}\n export STORAGE=${{matrix.storage}}\n new_version=`./juicefs --version | awk -F\" \" '{print $3}' | awk -F+ '{print $1}'`\n echo new_version is $new_version\n mv juicefs juicefs-$new_version\n export NEW_JFS_BIN=\"juicefs-$new_version\"\n old_version=`${{matrix.old_juicefs_version}}/juicefs --version | awk -F\" \" '{print $3}' | awk -F+ '{print $1}'`\n echo old_version is $old_version\n mv ${{matrix.old_juicefs_version}}/juicefs juicefs-$old_version\n export OLD_JFS_BIN=\"juicefs-$old_version\"\n timeout 3600 python3 .github/scripts/testVersionCompatible.py 2>&1 | tee result.log || code=$?; if [[ $code -eq 124 ]]; then echo test timeout with $code && exit 0; else echo exit with $code && exit $code; fi\n \n - name: Display result log\n if: always()\n run: | \n if [ -f \"result.log\" ]; then\n tail -n 500 result.log\n fi\n\n - name: Upload command log\n if: always()\n uses: actions/upload-artifact@v4\n with:\n name: ${{matrix.meta}}-${{matrix.old_juicefs_version}}.command.log\n path: ~/command.log\n\n - name: Display log\n if: always()\n shell: bash\n run: | \n if [ -f \"/home/runner/.juicefs/juicefs.log\" ]; then\n tail -1000 /home/runner/.juicefs/juicefs.log\n grep \":\" /home/runner/.juicefs/juicefs.log && exit 1 || true\n fi\n\n - name: Display command\n if: always()\n shell: bash\n run: | \n if [ -f \"$HOME/command.log\" ]; then\n tail -100 ~/command.log\n fi\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [vc-hypo]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success()\n run: echo \"All Done\"" }, { "path": ".github/workflows/wintest.yml", "content": "name: \"wintest\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/wintest.yml'\n - 'pkg/winfsp/*.go'\n - '**/*_windows.go'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**/wintest.yml'\n - 'pkg/winfsp/*.go'\n - '**/*_windows.go'\n workflow_dispatch:\n inputs:\n debug_enabled:\n type: boolean\n description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'\n required: false\n default: false\n schedule:\n - cron: '0 17 * * 0'\n\njobs:\n wintest:\n runs-on: windows-2022\n env:\n Actions_Allow_Unsecure_Commands: true\n steps:\n - name: Set up Go\n uses: actions/setup-go@v3\n with:\n go-version: '1.21'\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: Setup MSBuild.exe\n if: false\n uses: microsoft/setup-msbuild@v1.0.3\n\n - name: Change Winsdk Version\n if: false\n uses: GuillaumeFalourd/setup-windows10-sdk-action@v1\n with:\n sdk-version: 18362\n\n - name: Download WinFsp\n run: |\n choco install wget\n mkdir \"C:\\wfsp\\\"\n wget -O winfsp.msi https://github.com/winfsp/winfsp/releases/download/v2.0/winfsp-2.0.23075.msi\n copy winfsp.msi \"C:\\wfsp\\\"\n choco install 7zip -y\n\n - name: Install WinFsp\n run: |\n # call start-process to install winfsp.msi\n Start-Process -Wait -FilePath \"C:\\wfsp\\winfsp.msi\" -ArgumentList \"/quiet /norestart\"\n ls \"C:\\Program Files (x86)\\WinFsp\"\n ls \"C:\\Program Files (x86)\\WinFsp\\bin\"\n\n - name: Set up Include Headers\n run: |\n mkdir \"C:\\WinFsp\\inc\\fuse\"\n copy .\\hack\\winfsp_headers\\* C:\\WinFsp\\inc\\fuse\\\n dir \"C:\\WinFsp\\inc\\fuse\"\n set CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n go env\n go env -w CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n go env\n\n - name: Install Scoop\n run: |\n dir \"C:\\Program Files (x86)\\WinFsp\"\n Set-ExecutionPolicy RemoteSigned -scope CurrentUser\n iwr -useb 'https://raw.githubusercontent.com/scoopinstaller/install/master/install.ps1' -outfile 'install.ps1'\n .\\install.ps1 -RunAsAdmin\n echo $env:USERNAME\n scoop\n $redisUrl = \"https://github.com/tporadowski/redis/releases/download/v5.0.14.1/Redis-x64-5.0.14.1.zip\"\n $redisRoot = Join-Path $env:USERPROFILE \"scoop\\apps\\redis\\current\"\n New-Item -ItemType Directory -Force -Path $redisRoot | Out-Null\n Invoke-WebRequest -Uri $redisUrl -OutFile redis.zip\n Expand-Archive -Path redis.zip -DestinationPath $redisRoot -Force\n $redisCli = Join-Path $redisRoot \"redis-cli.exe\"\n if (-not (Test-Path $redisCli)) {\n throw \"redis-cli.exe not found after downloading from $redisUrl\"\n }\n $env:Path += \";$redisRoot\"\n $redisRoot | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append\n & $redisCli --version\n scoop install minio@2021-12-10T23-03-39Z\n scoop install runasti\n\n - name: Download winsw\n run: |\n wget https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW-x64.exe -q --show-progress -O winsw.exe\n ls winsw.exe\n\n - name: Start Redis\n run: |\n copy winsw.exe redis-service.exe\n $redisExe = Join-Path $env:USERPROFILE \"scoop\\apps\\redis\\current\\redis-server.exe\"\n if (!(Test-Path $redisExe)) {\n throw \"redis-server.exe not found: $redisExe\"\n }\n @\"\n \n redisredis\n redisredis\n redisredis\n $redisExe\n --bind 127.0.0.1 --port 6379 --save \\\"\\\" --appendonly no\n rotate\n \n \"@ | Set-Content redis-service.xml -Encoding utf8\n .\\redis-service.exe install\n net start redisredis\n Start-Sleep -Seconds 2\n redis-cli -h 127.0.0.1 -p 6379 ping\n\n - name: Download MinGW\n run: |\n wget https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev1/x86_64-14.2.0-release-win32-seh-msvcrt-rt_v12-rev1.7z -q --show-progress -O mingw.7z\n 7z.exe x mingw.7z -oC:\\mingw64\n ls C:\\mingw64\\bin\n\n\n - name: Build Juicefs\n run: |\n $env:CGO_ENABLED=1\n $env:PATH+=\";C:\\mingw64\\bin\"\n go build -ldflags=\"-s -w\" -o juicefs.exe .\n \n - name: Install Python2\n run: |\n choco install python2 -y\n\n - name: Juicefs Format\n run: |\n ./juicefs.exe format redis://127.0.0.1:6379/1 myjfs\n\n - name: Juicefs Mount\n run: |\n $env:PATH+=\";C:\\Program Files (x86)\\WinFsp\\bin\"\n ./juicefs.exe mount -d redis://127.0.0.1:6379/1 z: --fuse-trace-log c:/fuse.log\n\n - name: Run Winfsp Tests\n run: |\n wget https://github.com/juicedata/winfsp/releases/download/testing_suit_20250324/winfsp-tests-x64.exe -q --show-progress -O \"C:\\Program Files (x86)\\WinFsp\\bin\\winfsp-tests-x64.exe\"\n ls \"C:\\Program Files (x86)\\WinFsp\\bin\\winfsp-tests-x64.exe\"\n cd Z:\n & \"C:\\Program Files (x86)\\WinFsp\\bin\\winfsp-tests-x64.exe\" --fuse-external --resilient --case-insensitive-cmp\n \n - name: Run winfstest\n run: |\n wget https://github.com/juicedata/winfstest/releases/download/testing_20250313/TestSuite-x64-v4.zip -q --show-progress -O Z:\\TestSuite-x64.zip\n ls Z:\\TestSuite-x64.zip\n cd Z:\\\n Expand-Archive -Path .\\TestSuite-x64.zip -DestinationPath .\\TestSuite\n ls Z:\\TestSuite\n cd Z:\\TestSuite\\TestSuite\n ./run-winfstest.ps1\n\n - name: Run FSX Test\n run: |\n cd Z:\\\n wget https://github.com/chenjie4255/fstools/releases/download/v0.0.1/fsx-x64.exe -q --show-progress -O fsx.exe\n ls fsx.exe\n ./fsx.exe -d 180 -p 10000 -F 100000 fsxtest\n\n - name: Run basic subcommand tests\n run: |\n echo hi > Z:\\hi.txt\n ./juicefs.exe info Z:\\hi.txt\n ./juicefs.exe status redis://127.0.0.1:6379/1\n ./juicefs.exe debug Z:\\\n New-Item -Path 'Z:\\summary' -ItemType Directory\n echo hi > Z:\\summary\\1.txt\n echo hi > Z:\\summary\\2.txt\n ./juicefs.exe summary Z:\\summary\n ./juicefs.exe info Z:\\summary\\1.txt\n ./juicefs.exe stats Z: -c 5\n - name: Setup tmate session\n if: ${{ failure() && github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}\n uses: mxschmitt/action-tmate@v3\n" }, { "path": ".github/workflows/xattr.yml", "content": "name: \"xattr\"\n\non:\n push:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**.go'\n - '**.c'\n - '**/xattr.yml'\n pull_request:\n branches:\n - 'main'\n - 'release-**'\n paths:\n - '**.go'\n - '**.c'\n - '**/xattr.yml'\n schedule:\n - cron: '0 17 * * *'\n workflow_dispatch:\n inputs:\n debug:\n type: boolean\n description: \"Run the build with tmate debugging enabled\"\n required: false\n default: false\n\njobs:\n build-matrix:\n runs-on: ubuntu-22.04\n steps:\n - id: set-matrix\n run: |\n echo \"github.event_name is ${{github.event_name}}\"\n if [[ \"${{github.event_name}}\" == \"schedule\" || \"${{github.event_name}}\" == \"workflow_dispatch\" ]]; then\n echo 'meta_matrix=[\"sqlite3\", \"redis\", \"mysql\", \"tikv\", \"badger\", \"postgres\", \"mariadb\", \"fdb\"]' >> $GITHUB_OUTPUT\n elif [[ \"${{github.event_name}}\" == \"pull_request\" || \"${{github.event_name}}\" == \"push\" ]]; then\n echo 'meta_matrix=[\"redis\"]' >> $GITHUB_OUTPUT\n else\n echo 'event name is not supported' && exit 1\n fi\n outputs:\n meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}\n\n xattr:\n needs: build-matrix\n strategy:\n fail-fast: false\n matrix:\n meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}\n runs-on: ubuntu-22.04\n\n steps:\n - uses: shogo82148/actions-setup-perl@v1\n with:\n perl-version: '5.34'\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: 1\n\n - name: mount coverage dir\n timeout-minutes: 5\n uses: ./.github/actions/mount-coverage-dir\n with:\n mount_point: cover\n access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}\n secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}\n token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}\n\n - name: Set Variable\n id: vars\n run: |\n if [ \"${{matrix.meta}}\" == \"fdb\" ]; then\n echo \"target=juicefs.fdb\" >> $GITHUB_OUTPUT\n else\n echo \"target=juicefs\" >> $GITHUB_OUTPUT\n fi\n\n - name: Build\n uses: ./.github/actions/build\n with: \n target: ${{steps.vars.outputs.target}}\n\n - name: Prepare meta db\n run: | \n chmod +x .github/scripts/start_meta_engine.sh\n source .github/scripts/start_meta_engine.sh\n start_meta_engine ${{matrix.meta}}\n meta_url=$(get_meta_url ${{matrix.meta}})\n create_database $meta_url\n\n - name: Juicefs Format\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --trash-days 0 --bucket=/mnt/jfs pics\n\n - name: Juicefs Mount\n run: |\n source .github/scripts/start_meta_engine.sh\n meta_url=$(get_meta_url ${{matrix.meta}})\n # sudo mkdir /var/jfs\n # sudo chmod 777 /var/jfs\n sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr\n stat /tmp/jfs/.accesslog\n\n - name: Test\n run: |\n git clone https://github.com/iustin/pyxattr.git\n cd pyxattr\n pip3 install pytest\n pip3 install pyxattr\n stat /tmp/jfs/\n if [[ \"${{matrix.meta}}\" == \"tikv\" || \"${{matrix.meta}}\" == \"badger\" ]]; then\n TEST_DIR=/tmp/jfs/ python3 -m pytest tests -k \"not test_empty_value\"\n else\n TEST_DIR=/tmp/jfs/ python3 -m pytest tests\n fi\n \n - name: log\n if: always()\n run: | \n if [ -f /var/log/juicefs.log ]; then\n tail -300 /var/log/juicefs.log\n grep \":\" /var/log/juicefs.log && exit 1 || true\n fi\n\n - name: upload coverage report\n timeout-minutes: 5\n continue-on-error: true\n uses: ./.github/actions/upload-coverage\n with:\n UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}\n\n - name: Setup upterm session\n if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)\n timeout-minutes: 60\n uses: lhotari/action-upterm@v1\n\n success-all-test:\n runs-on: ubuntu-latest\n needs: [xattr]\n if: always()\n steps:\n - uses: technote-space/workflow-conclusion-action@v3\n - uses: actions/checkout@v3\n\n - name: Check Failure\n if: env.WORKFLOW_CONCLUSION == 'failure'\n run: exit 1\n\n - name: Send Slack Notification\n if: failure() && github.event_name != 'workflow_dispatch'\n uses: juicedata/slack-notify-action@main\n with:\n channel-id: \"${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}\"\n slack_bot_token: \"${{ secrets.SLACK_BOT_TOKEN }}\"\n\n - name: Success\n if: success() \n run: echo \"All Done\"" }, { "path": ".gitignore", "content": "*.o\n*.sw[po]\nltmain.sh\n*.orig\n*.rej\n.deps\n.dirstamp\njfs\n*.rdb\n.release-env\n*.so\nlibjfs.h\ndocs/node_modules\ncmd/cmd\n.hypothesis\n/node_modules\n\n# os\n.DS_Store\n\n# ide\n.vscode\n.idea\n\n# lang\n__pycache__\n\n# temp\npkg/meta/badger\npkg/meta/testdata\n*.dump\n*.out\n\n# gen\n/juicefs\n/juicefs.ceph\n/juicefs.exe\n/juicefsd.exe\n/juicefs.exe~\n/juicefsd.exe~\n/juicefs.lite\ndist/\n" }, { "path": ".golangci.yml", "content": "run:\n timeout: 5m\n tests: false\n" }, { "path": ".goreleaser.yml", "content": "project_name: juicefs\nenv:\n - GO111MODULE=on\n - CGO_ENABLED=1\n - REVISIONDATE={{ .Env.REVISIONDATE }}\nbefore:\n hooks:\n - go mod download\nbuilds:\n - id: juicefs-windows-amd64\n hooks:\n pre:\n - sh -c 'mkdir -p /usr/local/include/winfsp && cp hack/winfsp_headers/* /usr/local/include/winfsp'\n env:\n - CC=x86_64-w64-mingw32-gcc\n - CXX=x86_64-w64-mingw32-g++\n ldflags: -s -w -X github.com/juicedata/juicefs/pkg/version.version={{.Version}} -X github.com/juicedata/juicefs/pkg/version.revision={{.ShortCommit}} -X github.com/juicedata/juicefs/pkg/version.revisionDate={{.Env.REVISIONDATE}}\n flags:\n - -buildmode\n - exe\n main: .\n goos:\n - windows\n goarch:\n - amd64\n - id: juicefs-darwin-amd64\n env:\n - CC=o64-clang\n - CXX=o64-clang++\n ldflags: -s -w -X github.com/juicedata/juicefs/pkg/version.version={{.Version}} -X github.com/juicedata/juicefs/pkg/version.revision={{.ShortCommit}} -X github.com/juicedata/juicefs/pkg/version.revisionDate={{.Env.REVISIONDATE}}\n main: .\n goos:\n - darwin\n goarch:\n - amd64\n - id: juicefs-darwin-arm64\n env:\n - CC=oa64-clang\n - CXX=oa64-clang++\n ldflags: -s -w -X github.com/juicedata/juicefs/pkg/version.version={{.Version}} -X github.com/juicedata/juicefs/pkg/version.revision={{.ShortCommit}} -X github.com/juicedata/juicefs/pkg/version.revisionDate={{.Env.REVISIONDATE}}\n main: .\n goos:\n - darwin\n goarch:\n - arm64\n - id: juicefs-linux-amd64\n env:\n - CC=/usr/bin/musl-gcc\n ldflags: -s -w -X github.com/juicedata/juicefs/pkg/version.version={{.Version}} -X github.com/juicedata/juicefs/pkg/version.revision={{.ShortCommit}} -X github.com/juicedata/juicefs/pkg/version.revisionDate={{.Env.REVISIONDATE}} -linkmode external -extldflags '-static'\n main: .\n goos:\n - linux\n goarch:\n - amd64\n - id: juicefs-linux-arm64\n env:\n - CC=/usr/local/aarch64-linux-musl-cross/bin/aarch64-linux-musl-cc\n ldflags: -s -w -X github.com/juicedata/juicefs/pkg/version.version={{.Version}} -X github.com/juicedata/juicefs/pkg/version.revision={{.ShortCommit}} -X github.com/juicedata/juicefs/pkg/version.revisionDate={{.Env.REVISIONDATE}} -linkmode external -extldflags '-static'\n main: .\n goos:\n - linux\n goarch:\n - arm64\nchecksum:\n name_template: 'checksums.txt'\nsnapshot:\n name_template: \"{{ .Tag }}-next\"\nchangelog:\n sort: asc\n filters:\n exclude:\n - '^docs:'\n - '^test:'\narchives:\n - name_template: \"{{ .ProjectName }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}\"\n" }, { "path": ".markdownlint-cli2.jsonc", "content": "{\n \"customRules\": [\n \"markdownlint-rule-enhanced-proper-names/src/enhanced-proper-names.js\",\n \"markdownlint-rule-no-trailing-slash-in-links/src/no-trailing-slash-in-links.js\"\n ],\n \"config\": {\n \"default\": true,\n \"first-heading-h1\": false,\n \"heading-style\": {\n \"style\": \"atx\"\n },\n \"ul-style\": false,\n \"link-image-style\": {\n \"autolink\": false\n },\n \"no-hard-tabs\": {\n \"spaces_per_tab\": 4\n },\n \"line-length\": false,\n \"no-duplicate-heading\": {\n \"siblings_only\": true\n },\n \"no-inline-html\": {\n \"allowed_elements\": [\n \"Badge\",\n \"TabItem\",\n \"Tabs\",\n \"a\",\n \"br\",\n \"div\",\n \"img\",\n \"li\",\n \"ul\",\n \"kbd\",\n \"p\",\n \"span\",\n \"sup\",\n \"iframe\",\n \"VersionAdd\"\n ]\n },\n \"fenced-code-language\": false,\n \"first-line-heading\": false,\n \"no-alt-text\": true,\n \"code-block-style\": {\n \"style\": \"fenced\"\n },\n \"code-fence-style\": {\n \"style\": \"backtick\"\n },\n \"link-fragments\": false,\n \"no-trailing-slash-in-links\": true,\n \"enhanced-proper-names\": {\n \"code_blocks\": false,\n \"html_elements\": false,\n \"heading_id\": false,\n \"names\": [\n \"ACL\",\n \"AI\",\n \"API\",\n \"ARM\",\n \"ARM64\",\n \"AWS\",\n \"Amazon\",\n \"Ansible\",\n \"Apache\",\n \"Azure\",\n \"BSD\",\n \"BadgerDB\",\n \"CDH\",\n \"CPU\",\n \"CSI Driver\",\n \"CSI\",\n \"CentOS\",\n \"Ceph\",\n \"CephFS\",\n \"ClickHouse\",\n \"Cloud SQL\",\n \"Colab\",\n \"Consul\",\n \"Debian\",\n \"DevOps\",\n \"DigitalOcean\",\n \"DistCp\",\n \"Docker Compose\",\n \"Docker\",\n \"Dockerfile\",\n \"Doris\",\n \"ECI\",\n \"Elasticsearch\",\n \"FTP\",\n \"FUSE\",\n \"Flink\",\n \"Fluid\",\n \"FoundationDB\",\n \"GCC\",\n \"GID\",\n \"Git\",\n \"GitHub\",\n \"Google\",\n \"Grafana\",\n \"Graphite\",\n \"HBase\",\n \"HDFS\",\n \"HDP\",\n \"HTTP\",\n \"HTTPS\",\n \"Hadoop\",\n \"Hive Metastore\",\n \"Hive\",\n \"Hudi\",\n \"IAM\",\n \"ID\",\n \"IOPS\",\n \"IP\",\n \"Iceberg\",\n \"JAR\",\n \"JDK\",\n \"JSON\",\n \"Java\",\n \"JuiceFS\",\n \"JuiceFSRuntime\",\n \"Juicedata\",\n \"K3s\",\n \"K8s\",\n \"Kerberos\",\n \"KeyDB\",\n \"KubeSphere\",\n \"Kubernetes\",\n \"LDAP\",\n \"LZ4\",\n \"Linux\",\n \"M1\",\n \"MariaDB\",\n \"Maven\",\n \"MinIO\",\n \"MySQL\",\n \"NFS\",\n \"NGINX\",\n \"POSIX\",\n \"PV\",\n \"PVC\",\n \"PostgreSQL\",\n \"PowerShell\",\n \"Prometheus\",\n \"Pushgateway\",\n \"Python\",\n \"QPS\",\n \"QoS\",\n \"RADOS\",\n \"RESTful\",\n \"RGW\",\n \"RPC\",\n \"Raft\",\n \"Rancher\",\n \"Ranger\",\n \"Redis\",\n \"S3\",\n \"S3QL\",\n \"SDK\",\n \"SFTP\",\n \"SID\",\n \"SMB\",\n \"SQL\",\n \"SQLite\",\n \"SSH\",\n \"Samba\",\n \"Scala\",\n \"Spark\",\n \"StarRocks\",\n \"ThriftServer\",\n \"TiKV\",\n \"Trino\",\n \"UID\",\n \"UUID\",\n \"Ubuntu\",\n \"Unix\",\n \"VFS\",\n \"WSL\",\n \"WebDAV\",\n \"WinFsp\",\n \"Windows\",\n \"YAML\",\n \"YARN\",\n \"Zstandard\",\n \"etcd\",\n \"macFUSE\",\n \"macOS\"\n ]\n }\n }\n}\n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v2.3.0\n hooks:\n - id: check-yaml\n args: [--allow-multiple-documents]\n - id: end-of-file-fixer\n - id: trailing-whitespace\n - repo: https://github.com/golangci/golangci-lint\n rev: v1.52.2\n hooks:\n - id: golangci-lint\n" }, { "path": "ADOPTERS.md", "content": "# JuiceFS Adopters\n\nPlease visit [JuiceFS Official Documentation](https://juicefs.com/docs/community/adopters) for details.\n" }, { "path": "ADOPTERS_CN.md", "content": "# JuiceFS 使用者\n\n请访问 [JuiceFS 官方文档](https://juicefs.com/docs/zh/community/adopters)了解详情。\n" }, { "path": "CODEOWNERS", "content": "/docs/ @CaitinChen\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "\n# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make participation in our\ncommunity a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, caste, color, religion, or sexual\nidentity and orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive, and healthy community.\n\n## Our Standards\n\nExamples of behavior that contributes to a positive environment for our\ncommunity include:\n\n* Demonstrating empathy and kindness toward other people\n* Being respectful of differing opinions, viewpoints, and experiences\n* Giving and gracefully accepting constructive feedback\n* Accepting responsibility and apologizing to those affected by our mistakes,\n and learning from the experience\n* Focusing on what is best not just for us as individuals, but for the overall\n community\n\nExamples of unacceptable behavior include:\n\n* The use of sexualized language or imagery, and sexual attention or advances of\n any kind\n* Trolling, insulting or derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or email address,\n without their explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive,\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces, and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official e-mail address,\nposting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at team@juicedata.io.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to respect the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action they deem in violation of this Code of Conduct:\n\n### 1. Correction\n\n**Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community.\n\n**Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n\n**Community Impact**: A violation through a single incident or series of\nactions.\n\n**Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or permanent\nban.\n\n### 3. Temporary Ban\n\n**Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n**Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n\n**Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior, harassment of an\nindividual, or aggression toward or disparagement of classes of individuals.\n\n**Consequence**: A permanent ban from any sort of public interaction within the\ncommunity.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.1, available at\n[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].\n\nCommunity Impact Guidelines were inspired by\n[Mozilla's code of conduct enforcement ladder][Mozilla CoC].\n\nFor answers to common questions about this code of conduct, see the FAQ at\n[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at\n[https://www.contributor-covenant.org/translations][translations].\n\n[homepage]: https://www.contributor-covenant.org\n[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html\n[Mozilla CoC]: https://github.com/mozilla/diversity\n[FAQ]: https://www.contributor-covenant.org/faq\n[translations]: https://www.contributor-covenant.org/translations\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to JuiceFS\n\n## Guidelines\n\n- Before starting work on a feature or bug fix, please search GitHub or reach out to us via GitHub, Slack etc. The purpose of this step is make sure no one else is already working on it and we'll ask you to open a GitHub issue if necessary.\n- We will use the GitHub issue to discuss the feature and come to agreement. This is to prevent your time being wasted, as well as ours.\n- If it is a major feature update, we highly recommend you also write a design document to help the community understand your motivation and solution.\n- A good way to find a project properly sized for a first time contributor is to search for open issues with the label [\"kind/good-first-issue\"](https://github.com/juicedata/juicefs/labels/kind%2Fgood-first-issue) or [\"kind/help-wanted\"](https://github.com/juicedata/juicefs/labels/kind%2Fhelp-wanted).\n\n## Coding Style\n\n- We're following [\"Effective Go\"](https://go.dev/doc/effective_go) and [\"Go Code Review Comments\"](https://github.com/golang/go/wiki/CodeReviewComments).\n- Use `go fmt` to format your code before committing. You can find information in editor support for Go tools in [\"IDEs and Plugins for Go\"](https://github.com/golang/go/wiki/IDEsAndTextEditorPlugins).\n- If you see any code which clearly violates the style guide, please fix it and send a pull request.\n- Every new source file must begin with a license header.\n- Install [pre-commit](https://pre-commit.com) and use it to set up a pre-commit hook for static analysis. Just run `pre-commit install` in the root of the repo.\n\n## Sign the CLA\n\nBefore you can contribute to JuiceFS, you will need to sign the [Contributor License Agreement](https://cla-assistant.io/juicedata/juicefs). There're a CLA assistant to guide you when you first time submit a pull request.\n\n## What is a Good PR\n\n- Presence of unit tests\n- Adherence to the coding style\n- Adequate in-line comments\n- Explanatory commit message\n\n## Contribution Flow\n\nThis is a rough outline of what a contributor's workflow looks like:\n\n1. Create a topic branch from where to base the contribution. This is usually `main`.\n1. Make commits of logical units.\n1. Make sure commit messages are in the proper format.\n1. Push changes in a topic branch to a personal fork of the repository.\n1. Submit a pull request to [juicedata/juicefs](https://github.com/juicedata/juicefs/compare). The PR should link to one issue which either created by you or others.\n1. The PR must receive approval from at least one maintainer before it be merged.\n\nHappy hacking!\n" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "Makefile", "content": "export GO111MODULE=on\n\nall: juicefs\n\nREVISION := $(shell git rev-parse --short HEAD 2>/dev/null)\nREVISIONDATE := $(shell git log -1 --pretty=format:'%cd' --date short 2>/dev/null)\nPKG := github.com/juicedata/juicefs/pkg/version\nGCFLAGS =\nBUILD ?= release\nifneq ($(strip $(REVISION)),) # Use git clone\n\tLDFLAGS += -X $(PKG).revision=$(REVISION) \\\n\t\t -X $(PKG).revisionDate=$(REVISIONDATE)\nendif\n\nifeq ($(BUILD),release)\n\tLDFLAGS += -s -w\nelse ifeq ($(BUILD),debug)\n\tGCFLAGS := all=-N -l\nendif\n\nSHELL = /bin/sh\n\nifdef STATIC\n\tLDFLAGS += -linkmode external -extldflags '-static'\n\tCC = /usr/bin/musl-gcc\n\texport CC\nendif\n\njuicefs: Makefile cmd/*.go pkg/*/*.go go.*\n\tgo version\n\tgo build -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs .\n\njuicefs.cover: Makefile cmd/*.go pkg/*/*.go go.*\n\tgo version\n\tgo build -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -cover -o juicefs .\n\njuicefs.lite: Makefile cmd/*.go pkg/*/*.go\n\tgo build -tags nogateway,nowebdav,nocos,nobos,nohdfs,noibmcos,noobs,nooss,noqingstor,nosftp,noswift,noazure,nogs,noufile,nob2,nonfs,nodragonfly,nosqlite,nomysql,nopg,notikv,nobadger,noetcd,nocifs \\\n\t\t-gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs.lite .\n\njuicefs.ceph: Makefile cmd/*.go pkg/*/*.go\n\tgo build -tags ceph -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs.ceph .\n\njuicefs.fdb: Makefile cmd/*.go pkg/*/*.go\n\tgo build -tags fdb -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs.fdb .\n\njuicefs.fdb.cover: Makefile cmd/*.go pkg/*/*.go\n\tgo build -tags fdb -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -cover -o juicefs.fdb .\n\njuicefs.gluster: Makefile cmd/*.go pkg/*/*.go\n\tgo build -tags gluster -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs.gluster .\n\njuicefs.gluster.cover: Makefile cmd/*.go pkg/*/*.go\n\tgo build -tags gluster -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -cover -o juicefs.gluster .\n\njuicefs.all: Makefile cmd/*.go pkg/*/*.go\n\tgo build -tags ceph,fdb,gluster -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs.all .\n\n# This is cross-compiling LoongArch in a Linux environment on x86_64 (amd64) or aarch64 (arm64) architecture.\n# 1. Install LoongArch64 cross-compile toolchain from https://github.com/loong64/cross-tools\n# 2. Set CC to your toolchain path.\n# 3. Run `STATIC=1 make juicefs.loongarch` to build the LoongArch binary.\njuicefs.loongarch: Makefile cmd/*.go pkg/*/*.go go.*\n\tCC=bin/loongarch64-unknown-linux-musl-cc CGO_ENABLED=1 GOARCH=loong64 go build -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs .\n\n# This is the script for compiling the Linux version on the MacOS platform.\n# Please execute the `brew install FiloSottile/musl-cross/musl-cross` command before using it.\njuicefs.linux:\n\tCGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-musl-gcc CGO_LDFLAGS=\"-static\" go build -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -o juicefs .\n\n/usr/local/include/winfsp:\n\tsudo mkdir -p /usr/local/include/winfsp\n\tsudo cp hack/winfsp_headers/* /usr/local/include/winfsp\n\n# This is the script for compiling the Windows version on the MacOS platform.\n# Please execute the `brew install mingw-w64` command before using it.\njuicefs.exe: /usr/local/include/winfsp cmd/*.go pkg/*/*.go\n\tGOOS=windows CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \\\n\t go build -gcflags=\"$(GCFLAGS)\" -ldflags=\"$(LDFLAGS)\" -buildmode exe -o juicefs.exe .\n\n# This is the script for compiling the Windows version on Windows platform.\n# Please ensure mingw64 is in PATH and WinFsp SDK is installed at C:/WinFsp\n_juicefs.exe:\n\tpowershell -Command \"$$env:PATH+=';C:\\mingw64\\bin'; $$env:CGO_ENABLED='1'; $$env:CGO_CFLAGS='-IC:/WinFsp/inc/fuse'; go build -ldflags='-s -w' -o juicefs.exe .\"\n\n.PHONY: snapshot release debug test\nsnapshot:\n\tdocker run --rm --privileged \\\n\t\t-e REVISIONDATE=$(REVISIONDATE) \\\n\t\t-e PRIVATE_KEY=${PRIVATE_KEY} \\\n\t\t-v ~/go/pkg/mod:/go/pkg/mod \\\n\t\t-v `pwd`:/go/src/github.com/juicedata/juicefs \\\n\t\t-v /var/run/docker.sock:/var/run/docker.sock \\\n\t\t-w /go/src/github.com/juicedata/juicefs \\\n\t\tjuicedata/golang-cross:latest release --snapshot --rm-dist --skip-publish\n\nrelease:\n\tdocker run --rm --privileged \\\n\t\t-e REVISIONDATE=$(REVISIONDATE) \\\n\t\t-e PRIVATE_KEY=${PRIVATE_KEY} \\\n\t\t--env-file .release-env \\\n\t\t-v ~/go/pkg/mod:/go/pkg/mod \\\n\t\t-v `pwd`:/go/src/github.com/juicedata/juicefs \\\n\t\t-v /var/run/docker.sock:/var/run/docker.sock \\\n\t\t-w /go/src/github.com/juicedata/juicefs \\\n\t\tjuicedata/golang-cross:latest release --rm-dist\n\ndebug:\n\t$(MAKE) BUILD=debug all\n\ntest.meta.core:\n\tSKIP_NON_CORE=true go test -v -cover -count=1 -failfast -timeout=12m ./pkg/meta/... -args -test.gocoverdir=\"$(shell realpath cover/)\"\n\ntest.meta.non-core:\n\tgo test -v -cover -run='TestRedisCluster|TestPostgreSQLClient|TestLoadDumpSlow|TestEtcdClient|TestKeyDB' -count=1 -failfast -timeout=12m ./pkg/meta/... -args -test.gocoverdir=\"$(shell realpath cover/)\"\n\ntest.pkg:\n\tgo test -tags gluster -v -cover -count=1 -failfast -timeout=12m $$(go list ./pkg/... | grep -v /meta) -args -test.gocoverdir=\"$(shell realpath cover/)\"\n\ntest.cmd:\n\tsudo JFS_GC_SKIPPEDTIME=1 MINIO_ACCESS_KEY=testUser MINIO_SECRET_KEY=testUserPassword GOMAXPROCS=8 go test -v -count=1 -failfast -cover -timeout=8m ./cmd/... -coverpkg=./pkg/...,./cmd/... -args -test.gocoverdir=\"$(shell realpath cover/)\"\n\ntest.fdb:\n\tgo test -v -cover -count=1 -failfast -timeout=4m ./pkg/meta/ -tags fdb -run=TestFdb -args -test.gocoverdir=\"$(shell realpath cover/)\"\n\nunit-random-test:\n\techo \"Using meta:$(meta), seed: $(seed), checks:${checks}, steps: $(steps)\"\n\tgo test ./pkg/meta/... -rapid.meta=\"$(meta)\" -rapid.seed=$(seed) -rapid.checks=$(checks) -rapid.steps=$(steps) -run \"TestFSOps\" -v -failfast -count=1 -timeout=60m -cover -coverpkg=./pkg/... -args -test.gocoverdir=\"$(shell realpath cover/)\"\n" }, { "path": "README.md", "content": "

\"JuiceFS

\n

\n \"Latest\n \"GitHub\n \"GitHub\n \"Go\n \"English\n \"Join\n

\n\n**JuiceFS** is a high-performance [POSIX](https://en.wikipedia.org/wiki/POSIX) file system released under Apache License 2.0, particularly designed for the cloud-native environment. The data, stored via JuiceFS, will be persisted in Object Storage _(e.g. Amazon S3)_, and the corresponding metadata can be persisted in various compatible database engines such as Redis, MySQL, and TiKV based on the scenarios and requirements.\n\nWith JuiceFS, massive cloud storage can be directly connected to big data, machine learning, artificial intelligence, and various application platforms in production environments. Without modifying code, the massive cloud storage can be used as efficiently as local storage.\n\n📖 **Document**: [Quick Start Guide](https://juicefs.com/docs/community/quick_start_guide)\n\n## Highlighted Features\n\n1. **Fully POSIX-compatible**: Use as a local file system, seamlessly docking with existing applications without breaking business workflow.\n2. **Fully Hadoop-compatible**: JuiceFS' [Hadoop Java SDK](https://juicefs.com/docs/community/hadoop_java_sdk) is compatible with Hadoop 2.x and Hadoop 3.x as well as a variety of components in the Hadoop ecosystems.\n3. **S3-compatible**: JuiceFS' [S3 Gateway](https://juicefs.com/docs/community/s3_gateway) provides an S3-compatible interface.\n4. **Cloud Native**: A [Kubernetes CSI Driver](https://juicefs.com/docs/community/how_to_use_on_kubernetes) is provided for easily using JuiceFS in Kubernetes.\n5. **Shareable**: JuiceFS is a shared file storage that can be read and written by thousands of clients.\n6. **Strong Consistency**: The confirmed modification will be immediately visible on all the servers mounted with the same file system.\n7. **Outstanding Performance**: The latency can be as low as a few milliseconds, and the throughput can be expanded nearly unlimitedly _(depending on the size of the Object Storage)_. [Test results](https://juicefs.com/docs/community/benchmark)\n8. **Data Encryption**: Supports data encryption in transit and at rest (please refer to [the guide](https://juicefs.com/docs/community/security/encrypt) for more information).\n9. **Global File Locks**: JuiceFS supports both BSD locks (flock) and POSIX record locks (fcntl).\n10. **Data Compression**: JuiceFS supports [LZ4](https://lz4.github.io/lz4) or [Zstandard](https://facebook.github.io/zstd) to compress all your data.\n\n---\n\n[Architecture](#architecture) | [Getting Started](#getting-started) | [Advanced Topics](#advanced-topics) | [POSIX Compatibility](#posix-compatibility) | [Performance Benchmark](#performance-benchmark) | [Supported Object Storage](#supported-object-storage) | [Who is using](#who-is-using) | [Roadmap](#roadmap) | [Reporting Issues](#reporting-issues) | [Contributing](#contributing) | [Community](#community) | [Usage Tracking](#usage-tracking) | [License](#license) | [Credits](#credits) | [FAQ](#faq)\n\n---\n\n## Architecture\n\nJuiceFS consists of three parts:\n\n1. **JuiceFS Client**: Coordinates Object Storage and metadata storage engine as well as implementation of file system interfaces such as POSIX, Hadoop, Kubernetes, and S3 gateway.\n2. **Data Storage**: Stores data, with supports of a variety of data storage media, e.g., local disk, public or private cloud Object Storage, and HDFS.\n3. **Metadata Engine**: Stores the corresponding metadata that contains information of file name, file size, permission group, creation and modification time and directory structure, etc., with supports of different metadata engines, e.g., Redis, MySQL, SQLite and TiKV.\n\n![JuiceFS Architecture](docs/en/images/juicefs-arch-new.png)\n\nJuiceFS can store the metadata of file system on different metadata engines, like Redis, which is a fast, open-source, in-memory key-value data storage, particularly suitable for storing metadata; meanwhile, all the data will be stored in Object Storage through JuiceFS client. [Learn more](https://juicefs.com/docs/community/architecture)\n\n![data-structure-diagram](docs/en/images/data-structure-diagram.svg)\n\nEach file stored in JuiceFS is split into **\"Chunk\"** s at a fixed size with the default upper limit of 64 MiB. Each Chunk is composed of one or more **\"Slice\"**(s), and the length of the slice varies depending on how the file is written. Each slice is composed of size-fixed **\"Block\"** s, which are 4 MiB by default. These blocks will be stored in Object Storage in the end; at the same time, the metadata information of the file and its Chunks, Slices, and Blocks will be stored in metadata engines via JuiceFS. [Learn more](https://juicefs.com/docs/community/architecture/#how-juicefs-store-files)\n\n![How JuiceFS stores your files](docs/en/images/how-juicefs-stores-files.svg)\n\nWhen using JuiceFS, files will eventually be split into Chunks, Slices and Blocks and stored in Object Storage. Therefore, the source files stored in JuiceFS cannot be found in the file browser of the Object Storage platform; instead, there are only a chunks directory and a bunch of digitally numbered directories and files in the bucket. Don't panic! This is just the secret of the high-performance operation of JuiceFS!\n\n## Getting Started\n\nBefore you begin, make sure you have:\n\n1. One supported metadata engine, see [How to Set Up Metadata Engine](https://juicefs.com/docs/community/databases_for_metadata)\n2. One supported Object Storage for storing data blocks, see [Supported Object Storage](https://juicefs.com/docs/community/how_to_setup_object_storage)\n3. [JuiceFS Client](https://juicefs.com/docs/community/installation) downloaded and installed\n\nPlease refer to [Quick Start Guide](https://juicefs.com/docs/community/quick_start_guide) to start using JuiceFS right away!\n\n### Command Reference\n\nCheck out all the command line options in [command reference](https://juicefs.com/docs/community/command_reference).\n\n### Containers\n\nJuiceFS can be used as a persistent volume for Docker and Podman, please check [here](https://juicefs.com/docs/community/juicefs_on_docker) for details.\n\n### Kubernetes\n\nIt is also very easy to use JuiceFS on Kubernetes. Please find more information [here](https://juicefs.com/docs/community/how_to_use_on_kubernetes).\n\n### Hadoop Java SDK\n\nIf you wanna use JuiceFS in Hadoop, check [Hadoop Java SDK](https://juicefs.com/docs/community/hadoop_java_sdk).\n\n## Advanced Topics\n\n- [Redis Best Practices](https://juicefs.com/docs/community/redis_best_practices)\n- [How to Setup Object Storage](https://juicefs.com/docs/community/how_to_setup_object_storage)\n- [Cache](https://juicefs.com/docs/community/cache)\n- [Fault Diagnosis and Analysis](https://juicefs.com/docs/community/fault_diagnosis_and_analysis)\n- [FUSE Mount Options](https://juicefs.com/docs/community/fuse_mount_options)\n- [Using JuiceFS on Windows](https://juicefs.com/docs/community/installation#windows)\n- [S3 Gateway](https://juicefs.com/docs/community/s3_gateway)\n\nPlease refer to [JuiceFS Document Center](https://juicefs.com/docs/community/introduction) for more information.\n\n## POSIX Compatibility\n\nJuiceFS has passed all of the compatibility tests (8813 in total) in the latest [pjdfstest](https://github.com/pjd/pjdfstest) .\n\n```\nAll tests successful.\n\nTest Summary Report\n-------------------\n/root/soft/pjdfstest/tests/chown/00.t (Wstat: 0 Tests: 1323 Failed: 0)\n TODO passed: 693, 697, 708-709, 714-715, 729, 733\nFiles=235, Tests=8813, 233 wallclock secs ( 2.77 usr 0.38 sys + 2.57 cusr 3.93 csys = 9.65 CPU)\nResult: PASS\n```\n\nAside from the POSIX features covered by pjdfstest, JuiceFS also provides:\n\n- **Close-to-open consistency**. Once a file is written _and_ closed, it is guaranteed to view the written data in the following opens and reads from any client. Within the same mount point, all the written data can be read immediately.\n- Rename and all other metadata operations are atomic, which are guaranteed by supported metadata engine transaction.\n- Opened files remain accessible after unlink from same mount point.\n- Mmap (tested with FSx).\n- Fallocate with punch hole support.\n- Extended attributes (xattr).\n- BSD locks (flock).\n- POSIX record locks (fcntl).\n\n## Performance Benchmark\n\n### Basic benchmark\n\nJuiceFS provides a subcommand that can run a few basic benchmarks to help you understand how it works in your environment:\n\n![JuiceFS Bench](docs/en/images/juicefs-bench.png)\n\n### Throughput\n\nA sequential read/write benchmark has also been performed on JuiceFS, [EFS](https://aws.amazon.com/efs) and [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) by [fio](https://github.com/axboe/fio).\n\n![Sequential Read Write Benchmark](docs/en/images/sequential-read-write-benchmark.svg)\n\nAbove result figure shows that JuiceFS can provide 10X more throughput than the other two (see [more details](https://juicefs.com/docs/community/fio)).\n\n### Metadata IOPS\n\nA simple mdtest benchmark has been performed on JuiceFS, [EFS](https://aws.amazon.com/efs) and [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) by [mdtest](https://github.com/hpc/ior).\n\n![Metadata Benchmark](docs/en/images/metadata-benchmark.svg)\n\nThe result shows that JuiceFS can provide significantly more metadata IOPS than the other two (see [more details](https://juicefs.com/docs/community/mdtest)).\n\n### Analyze performance\n\nSee [Real-Time Performance Monitoring](https://juicefs.com/docs/community/fault_diagnosis_and_analysis#performance-monitor) if you encountered performance issues.\n\n## Supported Object Storage\n\n- Amazon S3 _(and other S3 compatible Object Storage services)_\n- Google Cloud Storage\n- Azure Blob Storage\n- Alibaba Cloud Object Storage Service (OSS)\n- Tencent Cloud Object Storage (COS)\n- Qiniu Cloud Object Storage (Kodo)\n- QingStor Object Storage\n- Ceph RGW\n- MinIO\n- Local disk\n- Redis\n- ...\n\nJuiceFS supports numerous Object Storage services. [Learn more](https://juicefs.com/docs/community/how_to_setup_object_storage#supported-object-storage).\n\n## Who is using\n\nJuiceFS is production ready and used by thousands of machines in production. A list of users has been assembled and documented [here](https://juicefs.com/docs/community/adopters). In addition JuiceFS has several collaborative projects that integrate with other open source projects, which we have documented [here](https://juicefs.com/docs/community/integrations). If you are also using JuiceFS, please feel free to let us know, and you are welcome to share your specific experience with everyone.\n\nThe storage format is stable, and will be supported by all future releases.\n\n## Roadmap\n\n- Gateway Optimization\n- Resumable Sync\n- Read-ahead Optimization\n- Optimization for Large-scale Scenarios\n- Snapshots\n\n## Reporting Issues\n\nWe use [GitHub Issues](https://github.com/juicedata/juicefs/issues) to track community reported issues. You can also [contact](#community) the community for any questions.\n\n## Contributing\n\nThank you for your contribution! Please refer to the [JuiceFS Contributing Guide](https://juicefs.com/docs/community/development/contributing_guide) for more information.\n\n## Community\n\nWelcome to join the [Discussions](https://github.com/juicedata/juicefs/discussions) and the [Slack channel](https://go.juicefs.com/slack) to connect with JuiceFS team members and other users.\n\n## Usage Tracking\n\nJuiceFS collects **anonymous** usage data by default to help us better understand how the community is using JuiceFS. Only core metrics (e.g. version number) will be reported, and user data and any other sensitive data will not be included. The related code can be viewed [here](pkg/usage/usage.go).\n\nYou could also disable reporting easily by command line option `--no-usage-report`:\n\n```bash\njuicefs mount --no-usage-report\n```\n\n## License\n\nJuiceFS is open-sourced under Apache License 2.0, see [LICENSE](LICENSE).\n\n## Credits\n\nThe design of JuiceFS was inspired by [Google File System](https://research.google/pubs/pub51), [HDFS](https://hadoop.apache.org) and [MooseFS](https://moosefs.com). Thanks for their great work!\n\n## FAQ\n\n### Why doesn't JuiceFS support XXX Object Storage?\n\nJuiceFS supports many Object Storage services. Please check out [this list](https://juicefs.com/docs/community/how_to_setup_object_storage#supported-object-storage) first. If the Object Storage you want to use is compatible with S3, you could treat it as S3. Otherwise, try reporting any issue.\n\n### Can I use Redis Cluster as metadata engine?\n\nYes. Since [v1.0.0 Beta3](https://github.com/juicedata/juicefs/releases/tag/v1.0.0-beta3) JuiceFS supports the use of [Redis Cluster](https://redis.io/docs/manual/scaling) as the metadata engine, but it should be noted that Redis Cluster requires that the keys of all operations in a transaction must be in the same hash slot, so a JuiceFS file system can only use one hash slot.\n\nSee [\"Redis Best Practices\"](https://juicefs.com/docs/community/redis_best_practices) for more information.\n\n### What's the difference between JuiceFS and XXX?\n\nSee [\"Comparison with Others\"](https://juicefs.com/docs/community/comparison/juicefs_vs_alluxio) for more information.\n\nFor more FAQs, please see the [full list](https://juicefs.com/docs/community/faq).\n\n## Stargazers over time\n\n[![Star History Chart](https://api.star-history.com/svg?repos=juicedata/juicefs&type=Date)](https://star-history.com/#juicedata/juicefs&Date)\n" }, { "path": "README_CN.md", "content": "

\"JuiceFS

\n

\n \"GitHub\n \"GitHub\n \"Go\n \"English\n \"Join\n

\n\nJuiceFS 是一款高性能 [POSIX](https://en.wikipedia.org/wiki/POSIX) 文件系统，针对云原生环境特别优化设计，在 Apache 2.0 开源协议下发布。使用 JuiceFS 存储数据，数据本身会被持久化在对象存储（例如 Amazon S3），而数据所对应的元数据可以根据场景需求被持久化在 Redis、MySQL、TiKV 等多种数据库引擎中。\n\nJuiceFS 可以简单便捷的将海量云存储直接接入已投入生产环境的大数据、机器学习、人工智能以及各种应用平台，无需修改代码即可像使用本地存储一样高效使用海量云端存储。\n\n📺 **视频**：[什么是 JuiceFS?](https://www.bilibili.com/video/BV1HK4y197va)\n\n📖 **文档**：[快速上手指南](https://juicefs.com/docs/zh/community/quick_start_guide)\n\n## 核心特性\n\n1. **POSIX 兼容**：像本地文件系统一样使用，无缝对接已有应用，无业务侵入性；\n2. **HDFS 兼容**：完整兼容 [HDFS API](https://juicefs.com/docs/zh/community/hadoop_java_sdk)，提供更强的元数据性能；\n3. **S3 兼容**：提供 [S3 网关](https://juicefs.com/docs/zh/community/s3_gateway) 实现 S3 协议兼容的访问接口；\n4. **云原生**：通过 [Kubernetes CSI 驱动](https://juicefs.com/docs/zh/community/how_to_use_on_kubernetes) 可以很便捷地在 Kubernetes 中使用 JuiceFS；\n5. **多端共享**：同一文件系统可在上千台服务器同时挂载，高性能并发读写，共享数据；\n6. **强一致性**：确认的修改会在所有挂载了同一文件系统的服务器上立即可见，保证强一致性；\n7. **强悍性能**：毫秒级的延迟，近乎无限的吞吐量（取决于对象存储规模），查看[性能测试结果](https://juicefs.com/docs/zh/community/benchmark)；\n8. **数据安全**：支持传输中加密（encryption in transit）以及静态加密（encryption at rest），[查看详情](https://juicefs.com/docs/zh/community/security/encrypt)；\n9. **文件锁**：支持 BSD 锁（flock）及 POSIX 锁（fcntl）；\n10. **数据压缩**：支持使用 [LZ4](https://lz4.github.io/lz4) 或 [Zstandard](https://facebook.github.io/zstd) 压缩数据，节省存储空间。\n\n---\n\n[架构](#架构) | [开始使用](#开始使用) | [进阶主题](#进阶主题) | [POSIX 兼容性](#posix-兼容性测试) | [性能测试](#性能测试) | [支持的对象存储](#支持的对象存储) | [谁在使用](#谁在使用) | [产品路线图](#产品路线图) | [反馈问题](#反馈问题) | [贡献](#贡献) | [社区](#社区) | [使用量收集](#使用量收集) | [开源协议](#开源协议) | [致谢](#致谢) | [FAQ](#faq)\n\n---\n\n## 架构\n\nJuiceFS 由三个部分组成：\n\n1. **JuiceFS 客户端**：协调对象存储和元数据存储引擎，以及 POSIX、Hadoop、Kubernetes、S3 Gateway 等文件系统接口的实现；\n2. **数据存储**：存储数据本身，支持本地磁盘、对象存储；\n3. **元数据引擎**：存储数据对应的元数据，支持 Redis、MySQL、SQLite 等多种引擎；\n\n![JuiceFS Architecture](docs/zh_cn/images/juicefs-arch-new.png)\n\nJuiceFS 依靠 Redis 来存储文件的元数据。Redis 是基于内存的高性能的键值数据存储，非常适合存储元数据。与此同时，所有数据将通过 JuiceFS 客户端存储到对象存储中。[了解详情](https://juicefs.com/docs/zh/community/architecture)\n\n![Data structure diagram](docs/en/images/data-structure-diagram.svg)\n\n任何存入 JuiceFS 的文件都会被拆分成固定大小的 **\"Chunk\"**，默认的容量上限是 64 MiB。每个 Chunk 由一个或多个 **\"Slice\"** 组成，Slice 的长度不固定，取决于文件写入的方式。每个 Slice 又会被进一步拆分成固定大小的 **\"Block\"**，默认为 4 MiB。最后，这些 Block 会被存储到对象存储。与此同时，JuiceFS 会将每个文件以及它的 Chunks、Slices、Blocks 等元数据信息存储在元数据引擎中。[了解详情](https://juicefs.com/docs/zh/community/architecture#%E5%A6%82%E4%BD%95%E5%AD%98%E5%82%A8%E6%96%87%E4%BB%B6)\n\n![How JuiceFS stores your files](docs/zh_cn/images/how-juicefs-stores-files.svg)\n\n使用 JuiceFS，文件最终会被拆分成 Chunks、Slices 和 Blocks 存储在对象存储。因此，你会发现在对象存储平台的文件浏览器中找不到存入 JuiceFS 的源文件，存储桶中只有一个 chunks 目录和一堆数字编号的目录和文件。不要惊慌，这正是 JuiceFS 高性能运作的秘诀！\n\n## 开始使用\n\n创建 JuiceFS，需要以下 3 个方面的准备：\n\n1. 准备 Redis 数据库\n2. 准备对象存储\n3. 下载安装 [JuiceFS 客户端](https://juicefs.com/docs/zh/community/installation)\n\n请参照 [快速上手指南](https://juicefs.com/docs/zh/community/quick_start_guide) 立即开始使用 JuiceFS！\n\n### 命令索引\n\n请点击 [这里](https://juicefs.com/docs/zh/community/command_reference) 查看所有子命令以及命令行参数。\n\n### 容器\n\nJuiceFS 可以为 Docker、Podman 等容器化技术提供持久化存储，请查阅 [文档](https://juicefs.com/docs/community/juicefs_on_docker) 了解详情。\n\n### Kubernetes\n\n在 Kubernetes 中使用 JuiceFS 非常便捷，请查看 [这个文档](https://juicefs.com/docs/zh/community/how_to_use_on_kubernetes) 了解更多信息。\n\n### Hadoop Java SDK\n\nJuiceFS 使用 [Hadoop Java SDK](https://juicefs.com/docs/zh/community/hadoop_java_sdk) 与 Hadoop 生态结合。\n\n## 进阶主题\n\n- [Redis 最佳实践](https://juicefs.com/docs/zh/community/redis_best_practices)\n- [如何设置对象存储](https://juicefs.com/docs/zh/community/how_to_setup_object_storage)\n- [缓存](https://juicefs.com/docs/zh/community/cache)\n- [故障诊断和分析](https://juicefs.com/docs/zh/community/fault_diagnosis_and_analysis)\n- [FUSE 挂载选项](https://juicefs.com/docs/zh/community/fuse_mount_options)\n- [在 Windows 中使用 JuiceFS](https://juicefs.com/docs/zh/community/installation#windows-系统)\n- [S3 网关](https://juicefs.com/docs/zh/community/s3_gateway)\n\n请查阅 [JuiceFS 文档中心](https://juicefs.com/docs/zh/community/introduction) 了解更多信息。\n\n## POSIX 兼容性测试\n\nJuiceFS 通过了 [pjdfstest](https://github.com/pjd/pjdfstest) 最新版所有 8813 项兼容性测试。\n\n```\nAll tests successful.\n\nTest Summary Report\n-------------------\n/root/soft/pjdfstest/tests/chown/00.t (Wstat: 0 Tests: 1323 Failed: 0)\n TODO passed: 693, 697, 708-709, 714-715, 729, 733\nFiles=235, Tests=8813, 233 wallclock secs ( 2.77 usr 0.38 sys + 2.57 cusr 3.93 csys = 9.65 CPU)\nResult: PASS\n```\n\n除了 pjdfstest 覆盖的那些 POSIX 特性外，JuiceFS 还支持：\n\n- 关闭再打开（close-to-open）一致性。一旦一个文件写入完成并关闭，之后的打开和读操作保证可以访问之前写入的数据。如果是在同一个挂载点，所有写入的数据都可以立即读。\n- 重命名以及所有其他元数据操作都是原子的，由 Redis 的事务机制保证。\n- 当文件被删除后，同一个挂载点上如果已经打开了，文件还可以继续访问。\n- 支持 mmap\n- 支持 fallocate 以及空洞\n- 支持扩展属性\n- 支持 BSD 锁（flock）\n- 支持 POSIX 记录锁（fcntl）\n\n## 性能测试\n\n### 基础性能测试\n\nJuiceFS 提供一个性能测试的子命令来帮助你了解它在你的环境中的性能表现：\n\n![JuiceFS Bench](docs/zh_cn/images/juicefs-bench.png)\n\n### 顺序读写性能\n\n使用 [fio](https://github.com/axboe/fio) 测试了 JuiceFS、[EFS](https://aws.amazon.com/efs) 和 [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 的顺序读写性能，结果如下：\n\n![Sequential Read Write Benchmark](docs/zh_cn/images/sequential-read-write-benchmark.svg)\n\n上图显示 JuiceFS 可以比其他两者提供 10 倍以上的吞吐，详细结果请看[这里](https://juicefs.com/docs/zh/community/fio)。\n\n### 元数据性能\n\n使用 [mdtest](https://github.com/hpc/ior) 测试了 JuiceFS、[EFS](https://aws.amazon.com/efs) 和 [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 的元数据性能，结果如下：\n\n![Metadata Benchmark](docs/zh_cn/images/metadata-benchmark.svg)\n\n上图显示 JuiceFS 的元数据性能显著优于其他两个，详细的测试报告请看[这里](https://juicefs.com/docs/zh/community/mdtest)。\n\n### 性能分析\n\n如遇性能问题，查看[「实时性能监控」](https://juicefs.com/docs/zh/community/fault_diagnosis_and_analysis#performance-monitor)。\n\n## 支持的对象存储\n\n- 亚马逊 S3\n- 谷歌云存储\n- 微软云存储\n- 阿里云 OSS\n- 腾讯云 COS\n- 青云 QingStor 对象存储\n- Ceph RGW\n- MinIO\n- 本地目录\n- Redis\n- ……\n\nJuiceFS 支持几乎所有主流的对象存储服务，[查看详情](https://juicefs.com/docs/zh/community/how_to_setup_object_storage/#%E6%94%AF%E6%8C%81%E7%9A%84%E5%AD%98%E5%82%A8%E6%9C%8D%E5%8A%A1)。\n\n## 谁在使用\n\nJuiceFS 已经可以用于生产环境，目前有几千个节点在生产环境中使用它。我们收集汇总了一份使用者名单，记录在[这里](https://juicefs.com/docs/zh/community/adopters)。另外 JuiceFS 还有不少与其他开源项目进行集成的合作项目，我们将其记录在[这里](https://juicefs.com/docs/zh/community/integrations)。如果你也在使用 JuiceFS，请随时告知我们，也欢迎你向大家分享具体的使用经验。\n\nJuiceFS 的存储格式已经稳定，会被后续发布的所有版本支持。\n\n## 产品路线图\n\n- 基于用户和组的配额\n- 快照\n- 一次写入多次读取（WORM）\n\n## 反馈问题\n\n我们使用 [GitHub Issues](https://github.com/juicedata/juicefs/issues) 来管理社区反馈的问题，你也可以通过其他[渠道](#社区)跟社区联系。\n\n## 贡献\n\n感谢你对 JuiceFS 社区的贡献！请参考 [JuiceFS 贡献指南](https://juicefs.com/docs/zh/community/development/contributing_guide) 了解更多信息。\n\n## 社区\n\n欢迎加入 [Discussions](https://github.com/juicedata/juicefs/discussions) 和 [Slack 频道](https://go.juicefs.com/slack) 跟我们的团队和其他社区成员交流。\n\n## 使用量收集\n\nJuiceFS 的客户端会收集 **匿名** 使用数据来帮助我们更好地了解大家如何使用它，它只上报诸如版本号等使用量数据，不包含任何用户信息，完整的代码在 [这里](pkg/usage/usage.go)。\n\n你也可以通过下面的方式禁用它：\n\n```bash\njuicefs mount --no-usage-report\n```\n\n## 开源协议\n\n使用 Apache License 2.0 开源，详见 [LICENSE](LICENSE)。\n\n## 致谢\n\nJuiceFS 的设计参考了 [Google File System](https://research.google/pubs/pub51)、[HDFS](https://hadoop.apache.org) 以及 [MooseFS](https://moosefs.com)，感谢他们的杰出工作。\n\n## FAQ\n\n### 为什么不支持某个对象存储？\n\n已经支持了绝大部分对象存储，参考这个[列表](https://juicefs.com/docs/zh/community/how_to_setup_object_storage#支持的存储服务)。如果它跟 S3 兼容的话，也可以当成 S3 来使用。否则，请创建一个 issue 来增加支持。\n\n### 是否可以使用 Redis 集群版作为元数据引擎？\n\n可以。自 [v1.0.0 Beta3](https://github.com/juicedata/juicefs/releases/tag/v1.0.0-beta3) 版本开始 JuiceFS 支持使用 [Redis 集群版](https://redis.io/docs/manual/scaling)作为元数据引擎，不过需要注意的是 Redis 集群版要求一个事务中所有操作的 key 必须在同一个 hash slot 中，因此一个 JuiceFS 文件系统只能使用一个 hash slot。\n\n请查看[「Redis 最佳实践」](https://juicefs.com/docs/zh/community/redis_best_practices)了解更多信息。\n\n### JuiceFS 与 XXX 的区别是什么？\n\n请查看[「同类技术对比」](https://juicefs.com/docs/zh/community/comparison/juicefs_vs_alluxio)文档了解更多信息。\n\n更多 FAQ 请查看[完整列表](https://juicefs.com/docs/zh/community/faq)。\n\n## 历史加星\n\n[![Stargazers over time](https://starchart.cc/juicedata/juicefs.svg)](https://starchart.cc/juicedata/juicefs)\n" }, { "path": "check-changed.sh", "content": "#!/bin/bash\n\nset -e\n\nif [ x\"${TRAVIS_COMMIT_RANGE}\" == x ] ; then\n CHANGED_FILES=`git diff --name-only HEAD~1`\nelse\n CHANGED_FILES=`git diff --name-only $TRAVIS_COMMIT_RANGE`\nfi\necho $CHANGED_FILES\nDOCS_DIR=\"docs/\"\nGITHUB_DIR=\".github/\"\nSKIP_TEST=true\n\nfor CHANGED_FILE in $CHANGED_FILES; do\n if ! [[ $CHANGED_FILE =~ $DOCS_DIR ]] && ! [[ $CHANGED_FILE =~ $GITHUB_DIR ]] ; then\n SKIP_TEST=false\n break\n fi\ndone" }, { "path": "cmd/bench.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdBench() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"bench\",\n\t\tAction: bench,\n\t\tCategory: \"TOOL\",\n\t\tUsage: \"Run benchmarks on a path\",\n\t\tArgsUsage: \"PATH\",\n\t\tDescription: `\nRun basic benchmarks on the target PATH to test if it works as expected. Results are colored with\ngreen/yellow/red to indicate whether they are in a normal range. If you see any red value, please\ndouble check relevant configuration before further test.\n\nExamples:\n# Run benchmarks with 4 threads\n$ juicefs bench /mnt/jfs -p 4\n\n# Run benchmarks of only small files\n$ juicefs bench /mnt/jfs --big-file-size 0\n\nDetails: https://juicefs.com/docs/community/performance_evaluation_guide#juicefs-bench`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"block-size\",\n\t\t\t\tValue: \"1M\",\n\t\t\t\tUsage: \"size of each IO block in MiB\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"big-file-size\",\n\t\t\t\tValue: \"1G\",\n\t\t\t\tUsage: \"size of each big file in MiB\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"small-file-size\",\n\t\t\t\tValue: \"128K\",\n\t\t\t\tUsage: \"size of each small file in KiB\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"small-file-count\",\n\t\t\t\tValue: 100,\n\t\t\t\tUsage: \"number of small files per thread\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tValue: 1,\n\t\t\t\tUsage: \"number of concurrent threads\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nvar resultRange = map[string][4]float64{\n\t\"bigwr\": {100, 200, 10, 50},\n\t\"bigrd\": {100, 200, 10, 50},\n\t\"smallwr\": {12.5, 20, 50, 80},\n\t\"smallrd\": {50, 100, 10, 20},\n\t\"stat\": {20, 1000, 1, 5},\n\t\"fuse\": {0, 0, 0.5, 2},\n\t\"meta\": {0, 0, 2, 5},\n\t\"put\": {0, 0, 100, 200},\n\t\"get\": {0, 0, 100, 200},\n\t\"delete\": {0, 0, 30, 100},\n\t\"cachewr\": {0, 0, 10, 20},\n\t\"cacherd\": {0, 0, 1, 5},\n}\n\ntype benchCase struct {\n\tbm *benchmark\n\tname string\n\tfsize, bsize int // file/block size in Bytes\n\tfcount, bcount int // file/block count\n\twbar, rbar, sbar *utils.Bar // progress bar for write/read/stat\n}\n\ntype benchmark struct {\n\tcolorful bool\n\tbig, small *benchCase\n\tthreads int\n\ttmpdir string\n}\n\nfunc (bc *benchCase) writeFiles(index int) {\n\tfor i := 0; i < bc.fcount; i++ {\n\t\tfname := filepath.Join(bc.bm.tmpdir, fmt.Sprintf(\"%s.%d.%d\", bc.name, index, i))\n\t\tfp, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"Failed to open file %s: %s\", fname, err)\n\t\t}\n\t\tbuf := make([]byte, bc.bsize)\n\t\tutils.RandRead(buf)\n\t\tfor j := 0; j < bc.bcount; j++ {\n\t\t\tif _, err = fp.Write(buf); err != nil {\n\t\t\t\tlogger.Fatalf(\"Failed to write file %s: %s\", fname, err)\n\t\t\t}\n\t\t\tbc.wbar.Increment()\n\t\t}\n\t\t_ = fp.Close()\n\t}\n}\n\nfunc (bc *benchCase) readFiles(index int) {\n\tfor i := 0; i < bc.fcount; i++ {\n\t\tfname := filepath.Join(bc.bm.tmpdir, fmt.Sprintf(\"%s.%d.%d\", bc.name, index, i))\n\t\tfp, err := os.Open(fname)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"Failed to open file %s: %s\", fname, err)\n\t\t}\n\t\tbuf := make([]byte, bc.bsize)\n\t\tfor j := 0; j < bc.bcount; j++ {\n\t\t\tif n, err := fp.Read(buf); err != nil || n != bc.bsize {\n\t\t\t\tlogger.Fatalf(\"Failed to read file %s: %d %s\", fname, n, err)\n\t\t\t}\n\t\t\tbc.rbar.Increment()\n\t\t}\n\t\t_ = fp.Close()\n\t}\n}\n\nfunc (bc *benchCase) statFiles(index int) {\n\tfor i := 0; i < bc.fcount; i++ {\n\t\tfname := filepath.Join(bc.bm.tmpdir, fmt.Sprintf(\"%s.%d.%d\", bc.name, index, i))\n\t\tif _, err := os.Stat(fname); err != nil {\n\t\t\tlogger.Fatalf(\"Failed to stat file %s: %s\", fname, err)\n\t\t}\n\t\tbc.sbar.Increment()\n\t}\n}\n\nfunc (bc *benchCase) run(test string) float64 {\n\tvar fn func(int)\n\tswitch test {\n\tcase \"write\":\n\t\tfn = bc.writeFiles\n\tcase \"read\":\n\t\tfn = bc.readFiles\n\tcase \"stat\":\n\t\tfn = bc.statFiles\n\t} // default: fatal\n\tvar wg sync.WaitGroup\n\tstart := time.Now()\n\tfor i := 0; i < bc.bm.threads; i++ {\n\t\tindex := i\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tfn(index)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\twg.Wait()\n\treturn time.Since(start).Seconds()\n}\n\nfunc newBenchmark(tmpdir string, blockSize, bigSize, smallSize, smallCount, threads int) *benchmark {\n\tbm := &benchmark{threads: threads, tmpdir: tmpdir}\n\tif bigSize > 0 {\n\t\tbm.big = bm.newCase(\"bigfile\", bigSize, 1, blockSize)\n\t}\n\tif smallSize > 0 && smallCount > 0 {\n\t\tbm.small = bm.newCase(\"smallfile\", smallSize, smallCount, blockSize)\n\t}\n\treturn bm\n}\n\nfunc (bm *benchmark) newCase(name string, fsize, fcount, bsize int) *benchCase {\n\tbc := &benchCase{\n\t\tbm: bm,\n\t\tname: name,\n\t\tfsize: fsize,\n\t\tfcount: fcount,\n\t\tbsize: bsize,\n\t}\n\tif fsize <= bsize {\n\t\tbc.bcount = 1\n\t\tbc.bsize = fsize\n\t} else {\n\t\tbc.bcount = (fsize-1)/bsize + 1\n\t\tbc.fsize = bc.bcount * bsize\n\t}\n\treturn bc\n}\n\nfunc (bm *benchmark) colorize(item string, value, cost float64, prec int) (string, string) {\n\tsvalue := strconv.FormatFloat(value, 'f', prec, 64)\n\tscost := strconv.FormatFloat(cost, 'f', 2, 64)\n\tif bm.colorful {\n\t\tr, ok := resultRange[item]\n\t\tif !ok {\n\t\t\tlogger.Fatalf(\"Invalid item: %s\", item)\n\t\t}\n\t\tif item == \"smallwr\" || item == \"smallrd\" || item == \"stat\" {\n\t\t\tr[0] *= float64(bm.threads)\n\t\t\tr[1] *= float64(bm.threads)\n\t\t}\n\t\tvar color int\n\t\tif value > r[1] { // max\n\t\t\tcolor = GREEN\n\t\t} else if value > r[0] { // min\n\t\t\tcolor = YELLOW\n\t\t} else {\n\t\t\tcolor = RED\n\t\t}\n\t\tsvalue = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, color, svalue, RESET_SEQ)\n\t\tif cost < r[2] { // min\n\t\t\tcolor = GREEN\n\t\t} else if cost < r[3] { // max\n\t\t\tcolor = YELLOW\n\t\t} else {\n\t\t\tcolor = RED\n\t\t}\n\t\tscost = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, color, scost, RESET_SEQ)\n\t}\n\treturn svalue, scost\n}\n\nfunc printResult(result [][]string, leftAlign int, colorful bool) {\n\tif len(result) < 2 {\n\t\tlogger.Fatalf(\"result must not be empty\")\n\t}\n\tcolNum := len(result[0])\n\trawmax, max := make([]int, colNum), make([]int, colNum)\n\tfor _, l := range result {\n\t\tfor i := 0; i < colNum; i++ {\n\t\t\tif len(l[i]) > rawmax[i] {\n\t\t\t\trawmax[i] = len(l[i])\n\t\t\t}\n\t\t}\n\t}\n\tcopy(max, rawmax)\n\tif colorful {\n\t\tfor i := 1; i < colNum; i++ {\n\t\t\tmax[i] -= 11\n\t\t}\n\t}\n\n\tvar b strings.Builder\n\tfor i := 0; i < colNum; i++ {\n\t\tb.WriteByte('+')\n\t\tb.WriteString(strings.Repeat(\"-\", max[i]+2))\n\t}\n\tb.WriteByte('+')\n\tdivider := b.String()\n\tfmt.Println(divider)\n\n\tb.Reset()\n\theader := result[0]\n\tfor i := 0; i < colNum; i++ {\n\t\tb.WriteString(\" | \")\n\t\tb.WriteString(padding(header[i], max[i], ' '))\n\t}\n\tb.WriteString(\" |\")\n\tfmt.Println(b.String()[1:])\n\tfmt.Println(divider)\n\n\tfor _, l := range result[1:] {\n\t\tb.Reset()\n\t\tfor i := 0; i < colNum; i++ {\n\t\t\tb.WriteString(\" | \")\n\t\t\tif i == leftAlign {\n\t\t\t\tb.WriteString(l[i])\n\t\t\t}\n\t\t\tif spaces := rawmax[i] - len(l[i]); spaces > 0 {\n\t\t\t\tb.WriteString(strings.Repeat(\" \", spaces))\n\t\t\t}\n\t\t\tif i != leftAlign {\n\t\t\t\tb.WriteString(l[i])\n\t\t\t}\n\t\t}\n\t\tb.WriteString(\" |\")\n\t\tfmt.Println(b.String()[1:])\n\t}\n\tfmt.Println(divider)\n}\n\nfunc bench(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\t/* --- Pre-check --- */\n\tblockSize := utils.ParseBytes(ctx, \"block-size\", 'M')\n\tif blockSize == 0 || ctx.Uint(\"threads\") == 0 {\n\t\treturn os.ErrInvalid\n\t}\n\ttmpdir, err := filepath.Abs(ctx.Args().First())\n\tif err != nil {\n\t\tlogger.Fatalf(\"Failed to get absolute path of %s: %s\", ctx.Args().First(), err)\n\t}\n\tbigSize := utils.ParseBytes(ctx, \"big-file-size\", 'M')\n\tsmallSize := utils.ParseBytes(ctx, \"small-file-size\", 'K')\n\ttmpdir = filepath.Join(tmpdir, fmt.Sprintf(\"__juicefs_benchmark_%d__\", time.Now().UnixNano()))\n\tbm := newBenchmark(tmpdir, int(blockSize), int(bigSize), int(smallSize),\n\t\tint(ctx.Uint(\"small-file-count\")), int(ctx.Uint(\"threads\")))\n\tif bm.big == nil && bm.small == nil {\n\t\treturn os.ErrInvalid\n\t}\n\tvar purgeArgs []string\n\tif os.Getuid() != 0 {\n\t\tpurgeArgs = append(purgeArgs, \"sudo\")\n\t}\n\tswitch runtime.GOOS {\n\tcase \"darwin\":\n\t\tpurgeArgs = append(purgeArgs, \"purge\")\n\tcase \"linux\":\n\t\tpurgeArgs = append(purgeArgs, \"/bin/sh\", \"-c\", \"echo 3 > /proc/sys/vm/drop_caches\")\n\tcase \"windows\":\n\t\tbreak\n\tdefault:\n\t\tlogger.Fatal(\"Currently only support Linux/MacOS/Windows\")\n\t}\n\n\t/* --- Prepare --- */\n\tif _, err := os.Stat(bm.tmpdir); os.IsNotExist(err) {\n\t\tif err = os.MkdirAll(bm.tmpdir, 0777); err != nil {\n\t\t\tlogger.Fatalf(\"Failed to create %s: %s\", bm.tmpdir, err)\n\t\t}\n\t}\n\tmp, _ := findMountpoint(bm.tmpdir)\n\tdropCaches := func() {\n\t\tif os.Getenv(\"SKIP_DROP_CACHES\") != \"true\" && runtime.GOOS != \"windows\" {\n\t\t\tif err := exec.Command(purgeArgs[0], purgeArgs[1:]...).Run(); err != nil {\n\t\t\t\tlogger.Warnf(\"Failed to clean kernel caches: %s\", err)\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"Clear cache operation has been skipped\")\n\t\t}\n\t}\n\tif os.Getuid() != 0 {\n\t\tfmt.Println(\"Cleaning kernel cache, may ask for root privilege...\")\n\t}\n\tdropCaches()\n\tbm.colorful = utils.SupportANSIColor(os.Stdout.Fd())\n\tprogress := utils.NewProgress(false)\n\t/* --- Run Benchmark --- */\n\tvar stats map[string]float64\n\tif mp != \"\" {\n\t\tstats = readStats(mp)\n\t}\n\tvar result [][]string\n\tresult = append(result, []string{\"ITEM\", \"VALUE\", \"COST\"})\n\tif b := bm.big; b != nil {\n\t\ttotal := int64(bm.threads * b.fcount * b.bcount)\n\t\tb.wbar = progress.AddCountBar(\"Write big blocks\", total)\n\t\tcost := b.run(\"write\")\n\t\tb.wbar.Done()\n\t\tline := make([]string, 3)\n\t\tline[0] = \"Write big file\"\n\t\tline[1], line[2] = bm.colorize(\"bigwr\", float64(b.fsize)/1024/1024*float64(b.fcount*bm.threads)/cost, cost/float64(b.fcount), 2)\n\t\tline[1] += \" MiB/s\"\n\t\tline[2] += \" s/file\"\n\t\tresult = append(result, line)\n\t\tdropCaches()\n\n\t\tb.rbar = progress.AddCountBar(\"Read big blocks\", total)\n\t\tcost = b.run(\"read\")\n\t\tb.rbar.Done()\n\t\tline = make([]string, 3)\n\t\tline[0] = \"Read big file\"\n\t\tline[1], line[2] = bm.colorize(\"bigrd\", float64(b.fsize)/1024/1024*float64(b.fcount*bm.threads)/cost, cost/float64(b.fcount), 2)\n\t\tline[1] += \" MiB/s\"\n\t\tline[2] += \" s/file\"\n\t\tresult = append(result, line)\n\t}\n\tif s := bm.small; s != nil {\n\t\ttotal := int64(bm.threads * s.fcount * s.bcount)\n\t\ts.wbar = progress.AddCountBar(\"Write small blocks\", total)\n\t\tcost := s.run(\"write\")\n\t\ts.wbar.Done()\n\t\tline := make([]string, 3)\n\t\tline[0] = \"Write small file\"\n\t\tline[1], line[2] = bm.colorize(\"smallwr\", float64(s.fcount*bm.threads)/cost, cost*1000/float64(s.fcount), 1)\n\t\tline[1] += \" files/s\"\n\t\tline[2] += \" ms/file\"\n\t\tresult = append(result, line)\n\t\tdropCaches()\n\n\t\ts.rbar = progress.AddCountBar(\"Read small blocks\", total)\n\t\tcost = s.run(\"read\")\n\t\ts.rbar.Done()\n\t\tline = make([]string, 3)\n\t\tline[0] = \"Read small file\"\n\t\tline[1], line[2] = bm.colorize(\"smallrd\", float64(s.fcount*bm.threads)/cost, cost*1000/float64(s.fcount), 1)\n\t\tline[1] += \" files/s\"\n\t\tline[2] += \" ms/file\"\n\t\tresult = append(result, line)\n\t\tdropCaches()\n\n\t\ts.sbar = progress.AddCountBar(\"Stat small files\", int64(bm.threads*s.fcount))\n\t\tcost = s.run(\"stat\")\n\t\ts.sbar.Done()\n\t\tline = make([]string, 3)\n\t\tline[0] = \"Stat file\"\n\t\tline[1], line[2] = bm.colorize(\"stat\", float64(s.fcount*bm.threads)/cost, cost*1000/float64(s.fcount), 1)\n\t\tline[1] += \" files/s\"\n\t\tline[2] += \" ms/file\"\n\t\tresult = append(result, line)\n\t}\n\tprogress.Done()\n\n\t/* --- Clean-up --- */\n\tif runtime.GOOS == \"windows\" {\n\t\tif err := exec.Command(\"cmd\", \"/C\", \"rd\", \"/s\", \"/q\", bm.tmpdir).Run(); err != nil {\n\t\t\tlogger.Warnf(\"Failed to cleanup %s: %s\", bm.tmpdir, err)\n\t\t}\n\t} else {\n\t\tif err := exec.Command(\"rm\", \"-rf\", bm.tmpdir).Run(); err != nil {\n\t\t\tlogger.Warnf(\"Failed to cleanup %s: %s\", bm.tmpdir, err)\n\t\t}\n\t}\n\n\t/* --- Report --- */\n\tfmt.Println(\"Benchmark finished!\")\n\tfmt.Printf(\"BlockSize: %s, BigFileSize: %s, SmallFileSize: %s, SmallFileCount: %d, NumThreads: %d\\n\",\n\t\thumanize.IBytes(blockSize), humanize.IBytes(bigSize), humanize.IBytes(smallSize),\n\t\tctx.Uint(\"small-file-count\"), ctx.Uint(\"threads\"))\n\tif stats != nil {\n\t\tstats2 := readStats(mp)\n\t\tdiff := func(item string) float64 {\n\t\t\treturn stats2[\"juicefs_\"+item] - stats[\"juicefs_\"+item]\n\t\t}\n\t\tshow := func(title, nick, item string) {\n\t\t\tcount := diff(item + \"_total\")\n\t\t\tvar cost float64\n\t\t\tif count > 0 {\n\t\t\t\tcost = diff(item+\"_sum\") * 1000 / count\n\t\t\t}\n\t\t\tline := make([]string, 3)\n\t\t\tline[0] = title\n\t\t\tline[1], line[2] = bm.colorize(nick, count, cost, 0)\n\t\t\tline[1] += \" operations\"\n\t\t\tline[2] += \" ms/op\"\n\t\t\tresult = append(result, line)\n\t\t}\n\t\tshow(\"FUSE operation\", \"fuse\", \"fuse_ops_durations_histogram_seconds\")\n\t\tshow(\"Update meta\", \"meta\", \"transaction_durations_histogram_seconds\")\n\t\tshow(\"Put object\", \"put\", \"object_request_durations_histogram_seconds_PUT\")\n\t\tshow(\"Get object\", \"get\", \"object_request_durations_histogram_seconds_GET\")\n\t\tshow(\"Delete object\", \"delete\", \"object_request_durations_histogram_seconds_DELETE\")\n\t\tshow(\"Write into cache\", \"cachewr\", \"blockcache_write_hist_seconds\")\n\t\tshow(\"Read from cache\", \"cacherd\", \"blockcache_read_hist_seconds\")\n\t\tvar fmtString string\n\t\tif bm.colorful {\n\t\t\tgreenSeq := fmt.Sprintf(\"%s%dm\", COLOR_SEQ, GREEN)\n\t\t\tfmtString = fmt.Sprintf(\"Time used: %s%%.1f%s s, CPU: %s%%.1f%s%%%%, Memory: %s%%.1f%s MiB\\n\",\n\t\t\t\tgreenSeq, RESET_SEQ, greenSeq, RESET_SEQ, greenSeq, RESET_SEQ)\n\t\t} else {\n\t\t\tfmtString = \"Time used: %.1f s, CPU: %.1f%%, Memory: %.1f MiB\\n\"\n\t\t}\n\t\tfmt.Printf(fmtString, diff(\"uptime\"), diff(\"cpu_usage\")*100/diff(\"uptime\"), stats2[\"juicefs_memory\"]/1024/1024)\n\t}\n\tprintResult(result, -1, bm.colorful)\n\treturn nil\n}\n" }, { "path": "cmd/bench_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestBench(t *testing.T) {\n\tmountTemp(t, nil, []string{\"--trash-days=0\"}, nil)\n\tdefer umountTemp(t)\n\n\tos.Setenv(\"SKIP_DROP_CACHES\", \"true\")\n\tdefer os.Unsetenv(\"SKIP_DROP_CACHES\")\n\tif err := Main([]string{\"\", \"bench\", testMountPoint}); err != nil {\n\t\tt.Fatalf(\"test bench failed: %s\", err)\n\t}\n}\n\nfunc TestBenchForObject(t *testing.T) {\n\tif err := Main([]string{\"\", \"objbench\", testMountPoint + \"/\", \"-p\", \"4\"}); err != nil {\n\t\tt.Fatalf(\"test bench failed: %s\", err)\n\t}\n}\n" }, { "path": "cmd/clone.go", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdClone() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"clone\",\n\t\tAction: clone,\n\t\tUsage: \"clone a file or directory without copying the underlying data\",\n\t\tArgsUsage: \"SRC DST\",\n\t\tCategory: \"TOOL\",\n\t\tDescription: `\nThis command can clone a file or directory without copying the underlying data,similar to the cp command but very fast.\nExamples:\n# Clone a file\n$ juicefs clone /mnt/jfs/file1 /mnt/jfs/file2\n\n# Clone a directory\n$ juicefs clone /mnt/jfs/dir1 /mnt/jfs/dir2\n\n# Clone with preserving the uid, gid, and mode of the file\n$ juicefs clone -p /mnt/jfs/file1 /mnt/jfs/file2`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"preserve\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tUsage: \"preserve the uid, gid, and mode of the file. (This is forced on Windows)\",\n\t\t\t},\n\t\t\t&cli.IntFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tValue: meta.CLONE_DEFAULT_CONCURRENCY,\n\t\t\t\tUsage: \"number of concurrent workers for cloning directories\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc clone(ctx *cli.Context) error {\n\tsetup(ctx, 2)\n\tsrcPath := ctx.Args().Get(0)\n\tsrcAbsPath, err := filepath.Abs(srcPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"abs of %s: %s\", srcPath, err)\n\t}\n\tsrcIno, err := utils.GetFileInode(srcPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"lookup inode for %s: %s\", srcPath, err)\n\t}\n\tsrcParentIno, err := utils.GetFileInode(filepath.Dir(srcAbsPath))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"lookup inode for %s: %s\", filepath.Dir(srcAbsPath), err)\n\t}\n\tdst := ctx.Args().Get(1)\n\tif strings.HasSuffix(dst, string(filepath.Separator)) {\n\t\tdst = filepath.Join(dst, filepath.Base(srcPath))\n\t}\n\tif _, err := os.Stat(dst); err == nil {\n\t\treturn fmt.Errorf(\"%s already exists\", dst)\n\t} else if !os.IsNotExist(err) {\n\t\treturn fmt.Errorf(\"stat %s: %s\", dst, err)\n\t}\n\tdstAbsPath, err := filepath.Abs(dst)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"abs of %s: %s\", dst, err)\n\t}\n\n\tsrcMp, err := findMountpoint(srcAbsPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdstMp, err := findMountpoint(filepath.Dir(dstAbsPath))\n\tif err != nil {\n\t\treturn err\n\t}\n\tif srcMp != dstMp {\n\t\treturn fmt.Errorf(\"the clone DST path should be at the same mount point as the SRC path\")\n\t}\n\tif strings.HasPrefix(dstAbsPath, path.Clean(srcAbsPath)+\"/\") {\n\t\treturn fmt.Errorf(\"the clone DST path should not be under the SRC path\")\n\t}\n\n\tdstParent := filepath.Dir(dstAbsPath)\n\tdstName := filepath.Base(dstAbsPath)\n\tdstParentIno, err := utils.GetFileInode(dstParent)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"lookup inode for %s: %s\", dstParent, err)\n\t}\n\tvar cmode uint8\n\tumask := utils.GetUmask()\n\tif ctx.Bool(\"preserve\") || runtime.GOOS == \"windows\" {\n\t\tcmode |= meta.CLONE_MODE_PRESERVE_ATTR\n\t}\n\tthreads := ctx.Int(\"threads\")\n\tif threads < 1 {\n\t\tthreads = 1\n\t} else if threads > 255 {\n\t\tthreads = 255\n\t}\n\theaderSize := 4 + 4\n\tcontentSize := 8 + 8 + 8 + 1 + uint32(len(dstName)) + 2 + 1 + 1 // +1 for threads\n\twb := utils.NewBuffer(uint32(headerSize) + contentSize)\n\twb.Put32(meta.Clone)\n\twb.Put32(contentSize)\n\twb.Put64(srcIno)\n\twb.Put64(srcParentIno)\n\twb.Put64(dstParentIno)\n\twb.Put8(uint8(len(dstName)))\n\twb.Put([]byte(dstName))\n\twb.Put16(uint16(umask))\n\twb.Put8(cmode)\n\twb.Put8(uint8(threads))\n\tf, err := openController(srcMp)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer f.Close()\n\tif _, err = f.Write(wb.Bytes()); err != nil {\n\t\treturn fmt.Errorf(\"write message: %s\", err)\n\t}\n\n\tprogress := utils.NewProgress(false)\n\tdefer progress.Done()\n\tbar := progress.AddCountBar(\"Cloning entries\", 0)\n\tif _, errno := readProgress(f, func(count uint64, total uint64) {\n\t\tbar.SetTotal(int64(total))\n\t\tbar.SetCurrent(int64(count))\n\t}); errno != 0 {\n\t\treturn fmt.Errorf(\"clone failed: %v\", errno)\n\t}\n\treturn nil\n}\n\nfunc findMountpoint(fpath string) (string, error) {\n\tfor p := fpath; p != \"/\"; p = filepath.Dir(p) {\n\t\tinode, err := utils.GetFileInode(p)\n\t\tif err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"get inode of %s: %s\", p, err)\n\t\t}\n\t\tif inode == uint64(meta.RootInode) {\n\t\t\treturn p, nil\n\t\t}\n\t}\n\treturn \"\", fmt.Errorf(\"%s is not inside JuiceFS\", fpath)\n}\n" }, { "path": "cmd/compact.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"path/filepath\"\n\t\"syscall\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdCompact() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"compact\",\n\t\tAction: compact,\n\t\tCategory: \"TOOL\",\n\t\tUsage: \"Trigger compaction of chunks\",\n\t\tArgsUsage: \"PATH...\",\n\t\tDescription: `\n Examples:\n # compact with path\n $ juicefs compact /mnt/jfs/foo\n `,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"compact concurrency\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc compact(ctx *cli.Context) error {\n\tsetup0(ctx, 1, 0)\n\n\tcoCnt := ctx.Int(\"threads\")\n\tif coCnt <= 0 {\n\t\tlogger.Warn(\"threads should be > 0\")\n\t\tcoCnt = 1\n\t} else if coCnt >= math.MaxUint16 {\n\t\tlogger.Warn(\"threads should be < MaxUint16\")\n\t\tcoCnt = math.MaxUint16\n\t}\n\n\tpaths := ctx.Args().Slice()\n\tfor i := 0; i < len(paths); i++ {\n\t\tpath, err := filepath.Abs(paths[i])\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"get absolute path of %s error: %v\", paths[i], err)\n\t\t}\n\n\t\tinodeNo, err := utils.GetFileInode(path)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"lookup inode for %s error: %v\", path, err)\n\t\t\tcontinue\n\t\t}\n\t\tinode := meta.Ino(inodeNo)\n\n\t\tif !inode.IsValid() {\n\t\t\tlogger.Fatalf(\"inode numbe %d not valid\", inode)\n\t\t}\n\n\t\tif err = doCompact(inode, path, uint16(coCnt)); err != nil {\n\t\t\tlogger.Error(err)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc doCompact(inode meta.Ino, path string, coCnt uint16) error {\n\tf, err := openController(path)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"open control file for [%d:%s]: %w\", inode, path, err)\n\t}\n\tdefer f.Close()\n\n\theaderLen, bodyLen := uint32(8), uint32(8+2)\n\twb := utils.NewBuffer(headerLen + bodyLen)\n\twb.Put32(meta.CompactPath)\n\twb.Put32(bodyLen)\n\twb.Put64(uint64(inode))\n\twb.Put16(coCnt)\n\n\t_, err = f.Write(wb.Bytes())\n\tif err != nil {\n\t\tlogger.Fatalf(\"write message: %s\", err)\n\t}\n\n\tprogress := utils.NewProgress(false)\n\tbar := progress.AddCountBar(\"Compacted chunks\", 0)\n\t_, errno := readProgress(f, func(totalChunks, currChunks uint64) {\n\t\tbar.SetTotal(int64(totalChunks))\n\t\tbar.SetCurrent(int64(currChunks))\n\t})\n\n\tbar.Done()\n\tprogress.Done()\n\n\tif errno == syscall.EINVAL {\n\t\tlogger.Fatalf(\"compact is not supported, please upgrade and mount again\")\n\t}\n\tif errno != 0 {\n\t\treturn fmt.Errorf(\"compact [%d:%s] error: %s\", inode, path, errno)\n\t}\n\n\tlogger.Infof(\"compact [%d:%s] success.\", inode, path)\n\treturn nil\n}\n" }, { "path": "cmd/compact_test.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc createTestFile(path string, size int, partCnt int) error {\n\tfile, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer file.Close()\n\n\tcontent := []byte(strings.Repeat(\"a\", size/partCnt))\n\tfor i := 0; i < partCnt; i++ {\n\t\tif _, err = file.Write(content); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err = file.Sync(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\ntype testDir struct {\n\tpath string\n\tfileCnt int\n\tfileSize int\n\tfilePart int\n}\n\nfunc initForCompactTest(mountDir string, dirs map[string]testDir) {\n\tfor _, d := range dirs {\n\t\tdirPath := filepath.Join(mountDir, d.path)\n\n\t\terr := os.MkdirAll(dirPath, 0755)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tfor i := 0; i < d.fileCnt; i++ {\n\t\t\tif err := createTestFile(filepath.Join(dirPath, fmt.Sprintf(\"%d\", i)), d.fileSize, d.filePart); err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestCompact(t *testing.T) {\n\tvar bucket string\n\tmountTemp(t, &bucket, []string{\"--trash-days=0\"}, nil)\n\tdefer umountTemp(t)\n\n\tdirs := map[string]testDir{\n\t\t\"d1/d11\": {\n\t\t\tpath: \"d1/d11\",\n\t\t\tfileCnt: 10,\n\t\t\tfileSize: 10,\n\t\t\tfilePart: 2,\n\t\t},\n\t\t\"d1\": {\n\t\t\tpath: \"d1\",\n\t\t\tfileCnt: 20,\n\t\t\tfileSize: 10,\n\t\t\tfilePart: 5,\n\t\t},\n\t\t\"d2\": {\n\t\t\tpath: \"d2\",\n\t\t\tfileCnt: 5,\n\t\t\tfileSize: 20,\n\t\t\tfilePart: 4,\n\t\t},\n\t}\n\tinitForCompactTest(testMountPoint, dirs)\n\tdataDir := filepath.Join(bucket, testVolume, \"chunks\")\n\n\tsumChunks := 0\n\tfor _, d := range dirs {\n\t\tsumChunks += d.fileCnt * d.filePart\n\t}\n\n\tchunkCnt := getFileCount(dataDir)\n\tassert.Equal(t, sumChunks, chunkCnt)\n\n\torderedDirs := []string{\"d1/d11\", \"d1\", \"d2\"}\n\tfor _, path := range orderedDirs {\n\t\td := dirs[path]\n\n\t\terr := Main([]string{\"\", \"compact\", filepath.Join(testMountPoint, d.path)})\n\t\tassert.Nil(t, err)\n\n\t\tchunkCnt = getFileCount(dataDir)\n\t\tsumChunks -= d.fileCnt * (d.filePart - 1)\n\t\tassert.Equal(t, sumChunks, chunkCnt)\n\t}\n}\n" }, { "path": "cmd/config.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdConfig() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"config\",\n\t\tAction: config,\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"Change configuration of a volume\",\n\t\tArgsUsage: \"META-URL\",\n\t\tDescription: `\nOnly flags explicitly specified are changed.\n\nExamples:\n# Show the current configurations\n$ juicefs config redis://localhost\n\n# Change volume \"quota\"\n$ juicefs config redis://localhost --inodes 10000000 --capacity 1048576\n\n# Change maximum days before files in trash are deleted\n$ juicefs config redis://localhost --trash-days 7\n\n# Limit client version that is allowed to connect\n$ juicefs config redis://localhost --min-client-version 1.0.0 --max-client-version 1.1.0`,\n\t\tFlags: expandFlags(\n\t\t\tformatStorageFlags(),\n\t\t\taddCategories(\"DATA STORAGE\", []cli.Flag{\n\t\t\t\t&cli.StringFlag{\n\t\t\t\t\tName: \"upload-limit\",\n\t\t\t\t\tUsage: \"default bandwidth limit of a client for upload in Mbps\",\n\t\t\t\t},\n\t\t\t\t&cli.StringFlag{\n\t\t\t\t\tName: \"download-limit\",\n\t\t\t\t\tUsage: \"default bandwidth limit of a client for download in Mbps\",\n\t\t\t\t},\n\t\t\t}),\n\t\t\tformatManagementFlags(),\n\t\t\tconfigManagementFlags(),\n\t\t\tconfigFlags()),\n\t}\n}\n\nfunc configManagementFlags() []cli.Flag {\n\treturn addCategories(\"MANAGEMENT\", []cli.Flag{\n\t\t&cli.BoolFlag{\n\t\t\tName: \"encrypt-secret\",\n\t\t\tUsage: \"encrypt the secret key if it was previously stored in plain format\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"min-client-version\",\n\t\t\tUsage: \"minimum client version allowed to connect\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"max-client-version\",\n\t\t\tUsage: \"maximum client version allowed to connect\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"dir-stats\",\n\t\t\tUsage: \"enable dir stats, which is necessary for fast summary and dir quota\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"user-group-quota\",\n\t\t\tUsage: \"enable user and group quota management\",\n\t\t},\n\t})\n}\n\nfunc configFlags() []cli.Flag {\n\treturn []cli.Flag{\n\t\t&cli.BoolFlag{\n\t\t\tName: \"yes\",\n\t\t\tAliases: []string{\"y\"},\n\t\t\tUsage: \"automatically answer 'yes' to all prompts and run non-interactively\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"force\",\n\t\t\tUsage: \"skip sanity check and force update the configurations\",\n\t\t},\n\t}\n}\n\nfunc warn(format string, a ...interface{}) {\n\tfmt.Printf(\"\\033[1;33mWARNING\\033[0m: \"+format+\"\\n\", a...)\n}\n\nfunc userConfirmed() bool {\n\tfmt.Print(\"Proceed anyway? [y/N]: \")\n\tscanner := bufio.NewScanner(os.Stdin)\n\tfor scanner.Scan() {\n\t\tif text := strings.ToLower(scanner.Text()); text == \"y\" || text == \"yes\" {\n\t\t\treturn true\n\t\t} else if text == \"\" || text == \"n\" || text == \"no\" {\n\t\t\treturn false\n\t\t} else {\n\t\t\tfmt.Print(\"Please input y(yes) or n(no): \")\n\t\t}\n\t}\n\treturn false\n}\n\nfunc config(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tremovePassword(ctx.Args().Get(0))\n\tm := meta.NewClient(ctx.Args().Get(0), nil)\n\n\tformat, err := m.Load(false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(ctx.LocalFlagNames()) == 0 {\n\t\tfmt.Println(format)\n\t\treturn nil\n\t}\n\n\toriginDirStats := format.DirStats\n\toriginUGQuota := format.UserGroupQuota\n\tvar quota, storage, trash, clientVer bool\n\tvar msg strings.Builder\n\tencrypted := format.KeyEncrypted\n\tfor _, flag := range ctx.LocalFlagNames() {\n\t\tswitch flag {\n\t\tcase \"capacity\":\n\t\t\tif new := utils.ParseBytes(ctx, flag, 'G'); new != format.Capacity {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag,\n\t\t\t\t\thumanize.IBytes(format.Capacity), humanize.IBytes(new)))\n\t\t\t\tformat.Capacity = new\n\t\t\t\tquota = true\n\t\t\t}\n\t\tcase \"inodes\":\n\t\t\tif new := ctx.Uint64(flag); new != format.Inodes {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag,\n\t\t\t\t\thumanize.Comma(int64(format.Inodes)), humanize.Comma(int64(new))))\n\t\t\t\tformat.Inodes = new\n\t\t\t\tquota = true\n\t\t\t}\n\t\tcase \"storage\":\n\t\t\tif new := ctx.String(flag); new != format.Storage {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag, format.Storage, new))\n\t\t\t\tformat.Storage = new\n\t\t\t\tstorage = true\n\t\t\t}\n\t\tcase \"bucket\":\n\t\t\t// bucket will be accessed before storage, so it is necessary to determine if storage is a file\n\t\t\tif new := ctx.String(flag); new != format.Bucket {\n\t\t\t\teffectiveStorage := format.Storage\n\t\t\t\tif ctx.IsSet(\"storage\") {\n\t\t\t\t\teffectiveStorage = ctx.String(\"storage\")\n\t\t\t\t}\n\t\t\t\tif effectiveStorage == \"file\" {\n\t\t\t\t\tif p, err := filepath.Abs(new); err == nil {\n\t\t\t\t\t\tnew = p + \"/\"\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlogger.Fatalf(\"Failed to get absolute path of %s: %s\", new, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag, format.Bucket, new))\n\t\t\t\tformat.Bucket = new\n\t\t\t\tstorage = true\n\t\t\t}\n\t\tcase \"access-key\":\n\t\t\tif new := ctx.String(flag); new != format.AccessKey {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag, format.AccessKey, new))\n\t\t\t\tformat.AccessKey = new\n\t\t\t\tstorage = true\n\t\t\t}\n\t\tcase \"secret-key\": // always update\n\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: updated\\n\", flag))\n\t\t\tif err := format.Decrypt(); err != nil && strings.Contains(err.Error(), \"secret was removed\") {\n\t\t\t\tlogger.Warnf(\"decrypt secrets: %s\", err)\n\t\t\t}\n\t\t\tformat.SecretKey = ctx.String(flag)\n\t\t\tstorage = true\n\t\tcase \"session-token\": // always update\n\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: updated\\n\", flag))\n\t\t\tif err := format.Decrypt(); err != nil && strings.Contains(err.Error(), \"secret was removed\") {\n\t\t\t\tlogger.Warnf(\"decrypt secrets: %s\", err)\n\t\t\t}\n\t\t\tformat.SessionToken = ctx.String(flag)\n\t\t\tstorage = true\n\t\tcase \"storage-class\": // always update\n\t\t\tif new := ctx.String(flag); new != format.StorageClass {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag, format.StorageClass, new))\n\t\t\t\tformat.StorageClass = new\n\t\t\t\tstorage = true\n\t\t\t}\n\t\tcase \"upload-limit\":\n\t\t\tif new := utils.ParseMbps(ctx, flag); new != format.UploadLimit {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag, utils.Mbps(format.UploadLimit), utils.Mbps(new)))\n\t\t\t\tformat.UploadLimit = new\n\t\t\t}\n\t\tcase \"download-limit\":\n\t\t\tif new := utils.ParseMbps(ctx, flag); new != format.DownloadLimit {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %s -> %s\\n\", flag, utils.Mbps(format.DownloadLimit), utils.Mbps(new)))\n\t\t\t\tformat.DownloadLimit = new\n\t\t\t}\n\t\tcase \"trash-days\":\n\t\t\tif new := ctx.Int(flag); new != format.TrashDays {\n\t\t\t\tif new < 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid trash days: %d\", new)\n\t\t\t\t}\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %d -> %d\\n\", flag, format.TrashDays, new))\n\t\t\t\tformat.TrashDays = new\n\t\t\t\ttrash = true\n\t\t\t}\n\t\tcase \"dir-stats\":\n\t\t\tif new := ctx.Bool(flag); new != format.DirStats {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %t -> %t\\n\", flag, format.DirStats, new))\n\t\t\t\tformat.DirStats = new\n\t\t\t}\n\t\tcase \"user-group-quota\":\n\t\t\tif new := ctx.Bool(flag); new != format.UserGroupQuota {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%10s: %t -> %t\\n\", flag, format.UserGroupQuota, new))\n\t\t\t\tformat.UserGroupQuota = new\n\t\t\t}\n\t\tcase \"min-client-version\":\n\t\t\tif new := ctx.String(flag); new != format.MinClientVersion {\n\t\t\t\tif version.Parse(new) == nil {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid version string: %s\", new)\n\t\t\t\t}\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%s: %s -> %s\\n\", flag, format.MinClientVersion, new))\n\t\t\t\tformat.MinClientVersion = new\n\t\t\t\tclientVer = true\n\t\t\t}\n\t\tcase \"max-client-version\":\n\t\t\tif new := ctx.String(flag); new != format.MaxClientVersion {\n\t\t\t\tif version.Parse(new) == nil {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid version string: %s\", new)\n\t\t\t\t}\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%s: %s -> %s\\n\", flag, format.MaxClientVersion, new))\n\t\t\t\tformat.MaxClientVersion = new\n\t\t\t\tclientVer = true\n\t\t\t}\n\t\tcase \"enable-acl\":\n\t\t\tif enableACL := ctx.Bool(flag); enableACL != format.EnableACL {\n\t\t\t\tif enableACL {\n\t\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%s: %v -> %v\\n\", flag, format.EnableACL, true))\n\t\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%s: %s -> %s\\n\", \"min-client-version\", format.MinClientVersion, \"1.2.0-A\"))\n\t\t\t\t\tformat.EnableACL = true\n\t\t\t\t\tformat.MinClientVersion = \"1.2.0-A\"\n\t\t\t\t\tclientVer = true\n\t\t\t\t} else {\n\t\t\t\t\treturn errors.New(\"cannot disable acl\")\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"ranger-rest-url\":\n\t\t\tif newUrl := ctx.String(flag); newUrl != format.RangerRestUrl {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%s: %s -> %s\\n\", flag, format.RangerRestUrl, newUrl))\n\t\t\t\tformat.RangerRestUrl = newUrl\n\t\t\t\tformat.MinClientVersion = \"1.3.0-A\"\n\t\t\t\tclientVer = true\n\t\t\t}\n\t\tcase \"ranger-service\":\n\t\t\tif newService := ctx.String(flag); newService != format.RangerService {\n\t\t\t\tmsg.WriteString(fmt.Sprintf(\"%s: %s -> %s\\n\", flag, format.RangerService, newService))\n\t\t\t\tformat.RangerService = newService\n\t\t\t\tformat.MinClientVersion = \"1.3.0-A\"\n\t\t\t\tclientVer = true\n\t\t\t}\n\t\tcase \"kerberos-config-file\":\n\t\t\tmsg.WriteString(fmt.Sprintf(\"%s: updated\\n\", flag))\n\t\t\tformat.KerbConf = readKerbConf(ctx.String(flag))\n\t\t\tformat.MinClientVersion = \"1.4.0-A\"\n\t\t\tclientVer = true\n\t\t}\n\t}\n\tif msg.Len() == 0 {\n\t\tfmt.Println(\"Nothing changed.\")\n\t\treturn nil\n\t}\n\n\tif !ctx.Bool(\"force\") {\n\t\tyes := ctx.Bool(\"yes\")\n\t\tif storage {\n\t\t\tblob, err := createStorage(*format)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err = test(blob); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif quota {\n\t\t\tvar totalSpace, availSpace, iused, iavail uint64\n\t\t\t_ = m.StatFS(meta.Background(), meta.RootInode, &totalSpace, &availSpace, &iused, &iavail)\n\t\t\tusedSpace := totalSpace - availSpace\n\t\t\tif format.Capacity > 0 && usedSpace >= format.Capacity ||\n\t\t\t\tformat.Inodes > 0 && iused >= format.Inodes {\n\t\t\t\twarn(\"New quota is too small (used / quota): %d / %d bytes, %d / %d inodes.\",\n\t\t\t\t\tusedSpace, format.Capacity, iused, format.Inodes)\n\t\t\t\tif !yes && !userConfirmed() {\n\t\t\t\t\treturn fmt.Errorf(\"Aborted.\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif trash && format.TrashDays == 0 {\n\t\t\twarn(\"The current trash will be emptied and future removed files will purged immediately.\")\n\t\t\tif !yes && !userConfirmed() {\n\t\t\t\treturn fmt.Errorf(\"Aborted.\")\n\t\t\t}\n\t\t}\n\t\tif originDirStats && !format.DirStats {\n\t\t\tqs := make(map[string]*meta.Quota)\n\t\t\terr := m.HandleQuota(meta.Background(), meta.QuotaList, \"\", 0, 0, qs, false, false, false)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"list quotas\")\n\t\t\t}\n\t\t\tif len(qs) != 0 {\n\t\t\t\tpaths := make([]string, 0, len(qs))\n\t\t\t\tfor path := range qs {\n\t\t\t\t\tpaths = append(paths, path)\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"cannot disable dir stats when there are still %d dir quotas: %v\", len(qs), paths)\n\t\t\t}\n\t\t}\n\t\tif clientVer {\n\t\t\tif format.CheckVersion() != nil {\n\t\t\t\twarn(\"Clients with the same version of this will be rejected after modification.\")\n\t\t\t\tif !yes && !userConfirmed() {\n\t\t\t\t\treturn fmt.Errorf(\"Aborted.\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// check all clients\n\t\t\tif sessions, err := m.ListSessions(); err == nil {\n\t\t\t\twarnMsg := \"\"\n\t\t\t\tfor _, session := range sessions {\n\t\t\t\t\tif err := format.CheckCliVersion(version.Parse(session.Version)); err != nil {\n\t\t\t\t\t\twarnMsg += fmt.Sprintf(\"host %s pid %d client version error: %s\\n\", session.HostName, session.ProcessID, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif warnMsg != \"\" {\n\t\t\t\t\tfmt.Println(warnMsg)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif encrypted || ctx.Bool(\"encrypt-secret\") {\n\t\tif err = format.Encrypt(); err != nil {\n\t\t\tlogger.Fatalf(\"Format encrypt: %s\", err)\n\t\t}\n\t}\n\tif err = m.Init(format, false); err == nil {\n\t\tfmt.Println(msg.String()[:msg.Len()-1])\n\t}\n\n\tif !originUGQuota && format.UserGroupQuota {\n\t\tif err = m.ScanUserGroupUsage(meta.Background()); err != nil {\n\t\t\tlogger.Warnf(\"Scan user group usage: %s\", err)\n\t\t}\n\t}\n\n\treturn err\n}\n" }, { "path": "cmd/config_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/agiledragon/gomonkey/v2\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\n// mutate_test_job_number: 3\nfunc getStdout(args []string) ([]byte, error) {\n\ttmp, err := os.CreateTemp(\"/tmp\", \"jfstest-*\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer tmp.Close()\n\tdefer os.Remove(tmp.Name())\n\tpatch := gomonkey.ApplyGlobalVar(os.Stdout, *tmp)\n\tdefer patch.Reset()\n\n\tif err = Main(args); err != nil {\n\t\treturn nil, err\n\t}\n\treturn os.ReadFile(tmp.Name())\n}\n\nfunc TestConfig(t *testing.T) {\n\t_ = resetTestMeta()\n\tbucketPath := \"/tmp/testBucket\"\n\t_ = os.RemoveAll(bucketPath)\n\tif err := Main([]string{\"\", \"format\", testMeta, \"--bucket\", bucketPath, testVolume}); err != nil {\n\t\tt.Fatalf(\"format: %s\", err)\n\t}\n\n\tif err := Main([]string{\"\", \"config\", testMeta, \"--trash-days\", \"2\"}); err != nil {\n\t\tt.Fatalf(\"config: %s\", err)\n\t}\n\tdata, err := getStdout([]string{\"\", \"config\", testMeta})\n\tif err != nil {\n\t\tt.Fatalf(\"getStdout: %s\", err)\n\t}\n\tvar format meta.Format\n\tif err = json.Unmarshal(data, &format); err != nil {\n\t\tt.Fatalf(\"json unmarshal: %s\", err)\n\t}\n\tif format.TrashDays != 2 {\n\t\tt.Fatalf(\"trash-days %d != expect 2\", format.TrashDays)\n\t}\n\n\tif err = Main([]string{\"\", \"config\", testMeta, \"--capacity\", \"10\", \"--inodes\", \"1000000\"}); err != nil {\n\t\tt.Fatalf(\"config: %s\", err)\n\t}\n\tif err = Main([]string{\"\", \"config\", testMeta, \"--bucket\", \"/tmp/newBucket\", \"--access-key\", \"testAK\", \"--secret-key\", \"testSK\", \"--session-token\", \"token\"}); err != nil {\n\t\tt.Fatalf(\"config: %s\", err)\n\t}\n\tif data, err = getStdout([]string{\"\", \"config\", testMeta}); err != nil {\n\t\tt.Fatalf(\"getStdout: %s\", err)\n\t}\n\tif err = json.Unmarshal(data, &format); err != nil {\n\t\tt.Fatalf(\"json unmarshal: %s\", err)\n\t}\n\tif format.Capacity != 10<<30 || format.Inodes != 1000000 ||\n\t\tformat.Bucket != \"/tmp/newBucket/\" || format.AccessKey != \"testAK\" || format.SecretKey != \"removed\" || format.SessionToken != \"removed\" {\n\t\tt.Fatalf(\"unexpect format: %+v\", format)\n\t}\n\n\tif err = Main([]string{\"\", \"config\", testMeta, \"--bucket\", \"http://localhost:9000/miniofs\", \"--storage\", \"minio\", \"--force\"}); err != nil {\n\t\tt.Fatalf(\"config: %s\", err)\n\t}\n\tif data, err = getStdout([]string{\"\", \"config\", testMeta}); err != nil {\n\t\tt.Fatalf(\"getStdout: %s\", err)\n\t}\n\tif err = json.Unmarshal(data, &format); err != nil {\n\t\tt.Fatalf(\"json unmarshal: %s\", err)\n\t}\n\tif format.Bucket != \"http://localhost:9000/miniofs\" || format.Storage != \"minio\" {\n\t\tt.Fatalf(\"unexpect format: %+v\", format)\n\t}\n\n\tif err = Main([]string{\"\", \"config\", testMeta, \"--bucket\", \"http://localhost:9000/miniofs2\", \"--force\"}); err != nil {\n\t\tt.Fatalf(\"config: %s\", err)\n\t}\n\tif data, err = getStdout([]string{\"\", \"config\", testMeta}); err != nil {\n\t\tt.Fatalf(\"getStdout: %s\", err)\n\t}\n\tif err = json.Unmarshal(data, &format); err != nil {\n\t\tt.Fatalf(\"json unmarshal: %s\", err)\n\t}\n\tif format.Bucket != \"http://localhost:9000/miniofs2\" || format.Storage != \"minio\" {\n\t\tt.Fatalf(\"unexpect format: %+v\", format)\n\t}\n}\n" }, { "path": "cmd/debug.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"archive/zip\"\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nvar defaultOutDir = filepath.Join(\".\", \"debug\")\n\nfunc cmdDebug() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"debug\",\n\t\tAction: debug,\n\t\tCategory: \"INSPECTOR\",\n\t\tArgsUsage: \"MOUNTPOINT\",\n\t\tUsage: \"Collect and display system static and runtime information\",\n\t\tDescription: `\nIt collects and displays information from multiple dimensions such as the running environment and system logs, etc.\n\nExamples:\n$ juicefs debug /mnt/jfs\n\n# Result will be output to /var/log/\n$ juicefs debug --out-dir=/var/log /mnt/jfs\n\n# Get the last up to 1000 log entries\n$ juicefs debug --out-dir=/var/log --limit=1000 /mnt/jfs\n`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"out-dir\",\n\t\t\t\tValue: defaultOutDir,\n\t\t\t\tUsage: \"the output directory of the result file\",\n\t\t\t},\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"limit\",\n\t\t\t\tUsage: \"the number of last entries to be collected\",\n\t\t\t\tValue: 5000,\n\t\t\t},\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"stats-sec\",\n\t\t\t\tValue: 5,\n\t\t\t\tUsage: \"stats sampling duration\",\n\t\t\t},\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"trace-sec\",\n\t\t\t\tValue: 5,\n\t\t\t\tUsage: \"trace sampling duration\",\n\t\t\t},\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"profile-sec\",\n\t\t\t\tValue: 30,\n\t\t\t\tUsage: \"profile sampling duration\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc copyFileOnWindows(srcPath, destPath string) error {\n\tsrcFile, err := os.Open(srcPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer closeFile(srcFile)\n\tdestFile, err := os.Create(destPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer closeFile(destFile)\n\tif _, err := io.Copy(destFile, srcFile); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc copyFile(srcPath, destPath string, requireRootPrivileges bool) error {\n\tif runtime.GOOS == \"windows\" {\n\t\treturn utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\t\treturn copyFileOnWindows(srcPath, destPath)\n\t\t}, 3*time.Second)\n\t}\n\n\tvar copyArgs []string\n\tif requireRootPrivileges {\n\t\tcopyArgs = append(copyArgs, \"sudo\")\n\t}\n\tctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)\n\tdefer cancel()\n\tcopyArgs = append(copyArgs, \"/bin/sh\", \"-c\", fmt.Sprintf(\"cat %s > %s\", srcPath, destPath))\n\treturn exec.CommandContext(ctx, copyArgs[0], copyArgs[1:]...).Run()\n}\n\nvar logArg = regexp.MustCompile(`--log(\\s*=?\\s*)(\\S+)`)\n\nfunc getLogPath(cmd string) (string, error) {\n\tvar logPath string\n\ttmp := logArg.FindStringSubmatch(cmd)\n\tif len(tmp) == 3 {\n\t\tlogPath = tmp[2]\n\t} else {\n\t\tlogPath = filepath.Join(getDefaultLogDir(), \"juicefs.log\")\n\t}\n\n\treturn logPath, nil\n}\n\nfunc closeFile(file *os.File) {\n\tif err := file.Close(); err != nil {\n\t\tlogger.Fatalf(\"failed to close file %s: %v\", file.Name(), err)\n\t}\n}\n\nfunc getPprofPort(pid, amp string, requireRootPrivileges bool) (int, error) {\n\tcfg := vfs.Config{}\n\t_ = utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\tcontent, err := readConfig(amp)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"failed to read config file: %v\", err)\n\t\t}\n\t\tif err := json.Unmarshal(content, &cfg); err != nil {\n\t\t\tlogger.Warnf(\"failed to unmarshal config file: %v\", err)\n\t\t}\n\t\treturn nil\n\t}, 3*time.Second)\n\n\tif cfg.Port != nil {\n\t\tif len(strings.Split(cfg.Port.DebugAgent, \":\")) >= 2 {\n\t\t\tif port, err := strconv.Atoi(strings.Split(cfg.Port.DebugAgent, \":\")[1]); err != nil {\n\t\t\t\tlogger.Warnf(\"failed to parse debug agent port: %v\", err)\n\t\t\t} else {\n\t\t\t\treturn port, nil\n\t\t\t}\n\t\t}\n\t}\n\n\tvar lsofArgs []string\n\tif requireRootPrivileges {\n\t\tlsofArgs = append(lsofArgs, \"sudo\")\n\t}\n\tlsofArgs = append(lsofArgs, \"/bin/sh\", \"-c\", \"lsof -i -nP | grep -v grep | grep LISTEN | grep \"+pid)\n\tret, err := exec.Command(lsofArgs[0], lsofArgs[1:]...).CombinedOutput()\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"failed to execute command `%s`: %v\", strings.Join(lsofArgs, \" \"), err)\n\t}\n\tlogger.Debugf(\"lsof output: \\n%s\", string(ret))\n\tlines := strings.Split(string(ret), \"\\n\")\n\tif len(lines) == 0 {\n\t\treturn 0, fmt.Errorf(\"pprof will be collected, but no listen port\")\n\t}\n\n\tvar listenPort = -1\n\tfor _, line := range lines {\n\t\tfields := strings.Fields(line)\n\t\tif len(fields) != 0 {\n\t\t\tport, err := func() (port int, err error) {\n\t\t\t\tdefer func() {\n\t\t\t\t\te := recover()\n\t\t\t\t\tif e != nil {\n\t\t\t\t\t\terr = fmt.Errorf(\"failed to parse listen port: %v\", e)\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t\tport, err = strconv.Atoi(strings.Split(fields[len(fields)-2], \":\")[1])\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"failed to parse port %v: %v\", port, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}()\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif port >= 6060 && port <= 6099 && port > listenPort {\n\t\t\t\tif err := checkPort(port, amp); err == nil {\n\t\t\t\t\tlistenPort = port\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\n\tif listenPort == -1 {\n\t\treturn 0, fmt.Errorf(\"no valid pprof port found\")\n\t}\n\treturn listenPort, nil\n}\n\nfunc getRequest(url string, timeout time.Duration) ([]byte, error) {\n\tctx, cancel := context.WithTimeout(context.Background(), timeout)\n\tdefer cancel()\n\treq, err := http.NewRequestWithContext(ctx, \"GET\", url, nil)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error creating GET request: %v\", err)\n\t}\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error GET request: %v\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\treturn nil, fmt.Errorf(\"error GET request, status code %d\", resp.StatusCode)\n\t}\n\n\tdefer func(body io.ReadCloser) {\n\t\tif err := body.Close(); err != nil {\n\t\t\tlogger.Errorf(\"error closing body: %v\", err)\n\t\t}\n\t}(resp.Body)\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading response: %v\", err)\n\t}\n\n\treturn body, nil\n}\n\n// check pprof service status\nfunc checkPort(port int, amp string) error {\n\turl := fmt.Sprintf(\"http://localhost:%d/debug/pprof/cmdline?debug=1\", port)\n\tresp, err := getRequest(url, 3*time.Second)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error checking pprof alive: %v\", err)\n\t}\n\tresp = bytes.ReplaceAll(resp, []byte{0}, []byte{' '})\n\tfields := strings.Fields(string(resp))\n\tflag := false\n\tfor _, field := range fields {\n\t\tif amp == field {\n\t\t\tflag = true\n\t\t}\n\t}\n\tif !flag {\n\t\treturn fmt.Errorf(\"mount point mismatch: \\n%s\\n%s\", resp, amp)\n\t}\n\treturn nil\n}\n\ntype metricItem struct {\n\tname, url string\n}\n\nfunc reqAndSaveMetric(name string, metric metricItem, outDir string, timeout time.Duration) error {\n\tresp, err := getRequest(metric.url, timeout)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error getting metric: %v\", err)\n\t}\n\tretPath := filepath.Join(outDir, fmt.Sprintf(\"juicefs.%s\", metric.name))\n\tretFile, err := os.Create(retPath)\n\tif err != nil {\n\t\tlogger.Fatalf(\"error creating metric file %s: %v\", retPath, err)\n\t}\n\tdefer closeFile(retFile)\n\n\tif name == \"cmdline\" {\n\t\tresp = bytes.ReplaceAll(resp, []byte{0}, []byte{' '})\n\t}\n\n\twriter := bufio.NewWriter(retFile)\n\tif _, err := writer.Write(resp); err != nil {\n\t\treturn fmt.Errorf(\"failed to write metric %s: %v\", name, err)\n\t}\n\treturn writer.Flush()\n}\n\nfunc checkAgent(cmd string) bool {\n\tfor _, field := range strings.Fields(cmd) {\n\t\tif field == \"--no-agent\" {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc geneZipFile(srcPath, destPath string) error {\n\tzipFile, err := os.Create(destPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer closeFile(zipFile)\n\tarchive := zip.NewWriter(zipFile)\n\tdefer func() {\n\t\tif err := archive.Close(); err != nil {\n\t\t\tlogger.Fatalf(\"error closing zip archive: %v\", err)\n\t\t}\n\t}()\n\n\treturn filepath.Walk(srcPath, func(path string, info os.FileInfo, _ error) error {\n\t\tif path == srcPath {\n\t\t\treturn nil\n\t\t}\n\n\t\theader, err := zip.FileInfoHeader(info)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\theader.Name = strings.TrimPrefix(path, srcPath+`/`)\n\t\tif info.IsDir() {\n\t\t\theader.Name += `/`\n\t\t} else {\n\t\t\theader.Method = zip.Deflate\n\t\t}\n\n\t\twriter, err := archive.CreateHeader(header)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !info.IsDir() {\n\t\t\tfile, err := os.Open(path)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdefer closeFile(file)\n\t\t\tif _, err := io.Copy(writer, file); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc collectPprof(ctx *cli.Context, cmd string, pid string, amp string, requireRootPrivileges bool, currDir string, wg *sync.WaitGroup) error {\n\tif !checkAgent(cmd) {\n\t\tlogger.Warnf(\"No agent found, the pprof metrics will not be collected\")\n\t\treturn nil\n\t}\n\n\tport, err := getPprofPort(pid, amp, requireRootPrivileges)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get pprof port: %v\", err)\n\t}\n\tbaseUrl := fmt.Sprintf(\"http://localhost:%d/debug/pprof/\", port)\n\tlogger.Infof(\"The pprof base url: %s\", baseUrl)\n\ttrace := ctx.Uint64(\"trace-sec\")\n\tprofile := ctx.Uint64(\"profile-sec\")\n\tmetrics := map[string]metricItem{\n\t\t\"allocs\": {name: \"allocs.pb.gz\", url: baseUrl + \"allocs\"},\n\t\t\"blocks\": {name: \"block.pb.gz\", url: baseUrl + \"block\"},\n\t\t\"cmdline\": {name: \"cmdline.txt\", url: baseUrl + \"cmdline\"},\n\t\t\"goroutine\": {name: \"goroutine.pb.gz\", url: baseUrl + \"goroutine\"},\n\t\t\"stack\": {name: \"goroutine.stack.txt\", url: baseUrl + \"goroutine?debug=1\"},\n\t\t\"stack-detail\": {name: \"goroutine.stack.detail.txt\", url: baseUrl + \"goroutine?debug=2\"},\n\t\t\"heap\": {name: \"heap.pb.gz\", url: baseUrl + \"heap\"},\n\t\t\"mutex\": {name: \"mutex.pb.gz\", url: baseUrl + \"mutex\"},\n\t\t\"threadcreate\": {name: \"threadcreate.pb.gz\", url: baseUrl + \"threadcreate\"},\n\t\t\"trace\": {name: fmt.Sprintf(\"trace.%ds.pb.gz\", trace), url: fmt.Sprintf(\"%strace?seconds=%d\", baseUrl, trace)},\n\t\t\"profile\": {name: fmt.Sprintf(\"profile.%ds.pb.gz\", profile), url: fmt.Sprintf(\"%sprofile?seconds=%d\", baseUrl, profile)},\n\t}\n\n\tpprofOutDir := filepath.Join(currDir, \"pprof\")\n\tif err := os.Mkdir(pprofOutDir, os.ModePerm); err != nil {\n\t\treturn fmt.Errorf(\"failed to create out directory: %v\", err)\n\t}\n\n\tfor name, metric := range metrics {\n\t\twg.Add(1)\n\t\tgo func(name string, metric metricItem) {\n\t\t\ttimeout := 3 * time.Second\n\t\t\tdefer wg.Done()\n\t\t\tif name == \"profile\" {\n\t\t\t\tlogger.Infof(\"Profile metrics are being sampled, sampling duration: %ds\", profile)\n\t\t\t\ttimeout = time.Duration(profile+5) * time.Second\n\t\t\t}\n\t\t\tif name == \"trace\" {\n\t\t\t\tlogger.Infof(\"Trace metrics are being sampled, sampling duration: %ds\", trace)\n\t\t\t\ttimeout = time.Duration(trace+5) * time.Second\n\t\t\t}\n\t\t\tif err := reqAndSaveMetric(name, metric, pprofOutDir, timeout); err != nil {\n\t\t\t\tlogger.Errorf(\"Failed to get and save metric %s: %v\", name, err)\n\t\t\t}\n\t\t}(name, metric)\n\t}\n\treturn nil\n}\n\nfunc collectLog(ctx *cli.Context, cmd string, requireRootPrivileges bool, currDir string, uid string) error {\n\tmountdByWinSystem := runtime.GOOS == \"windows\" && uid == \"S-1-5-18\" // https://learn.microsoft.com/en-us/windows/win32/secauthz/well-known-sids\n\tif !(strings.Contains(cmd, \"-d\") || strings.Contains(cmd, \"--background\")) && !mountdByWinSystem {\n\t\tlogger.Warnf(\"The juicefs mount by foreground, the log will not be collected\")\n\t\treturn nil\n\t}\n\tlogPath, err := getLogPath(cmd)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get log path: %v\", err)\n\t}\n\tlimit := ctx.Uint64(\"limit\")\n\tretLogPath := filepath.Join(currDir, \"juicefs.log\")\n\n\tif runtime.GOOS == \"windows\" {\n\t\t// check powershell is installed\n\t\t_, err = exec.LookPath(\"powershell\")\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"Powershell is not installed, the log will not be collected\")\n\t\t\treturn nil\n\t\t}\n\n\t\tcopyArgs := []string{\"powershell\", \"-Command\", fmt.Sprintf(\"Get-Content -Tail %d %s > %s\", limit, logPath, retLogPath)}\n\t\tlogger.Infof(\"The last %d lines of %s will be collected\", limit, logPath)\n\t\ttimeoutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\tdefer cancel()\n\t\treturn exec.CommandContext(timeoutCtx, copyArgs[0], copyArgs[1:]...).Run()\n\t} else {\n\t\tvar copyArgs []string\n\t\tif requireRootPrivileges {\n\t\t\tcopyArgs = append(copyArgs, \"sudo\")\n\t\t}\n\t\tcopyArgs = append(copyArgs, \"/bin/sh\", \"-c\", fmt.Sprintf(\"tail -n %d %s > %s\", limit, logPath, retLogPath))\n\t\tlogger.Infof(\"The last %d lines of %s will be collected\", limit, logPath)\n\t\ttimeoutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\tdefer cancel()\n\t\treturn exec.CommandContext(timeoutCtx, copyArgs[0], copyArgs[1:]...).Run()\n\t}\n}\n\nfunc collectSysInfo(ctx *cli.Context, currDir string) error {\n\tsysInfo := utils.GetSysInfo()\n\tresult := fmt.Sprintf(`Platform: \n%s %s\n%s`, runtime.GOOS, runtime.GOARCH, sysInfo)\n\n\tsysPath := filepath.Join(currDir, \"system-info.log\")\n\tsysFile, err := os.Create(sysPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create system info file %s: %v\", sysPath, err)\n\t}\n\tdefer closeFile(sysFile)\n\tif _, err = sysFile.WriteString(result); err != nil {\n\t\treturn fmt.Errorf(\"failed to write system info file %s: %v\", sysPath, err)\n\t}\n\n\tfmt.Printf(\"\\n%s\\n\", result)\n\treturn nil\n}\n\nfunc collectSpecialFile(ctx *cli.Context, amp string, currDir string, requireRootPrivileges bool, wg *sync.WaitGroup) error {\n\tprefixed := true\n\tconfigName := \".jfs.config\"\n\t_ = utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\tif !utils.Exists(filepath.Join(amp, configName)) {\n\t\t\tconfigName = \".config\"\n\t\t\tprefixed = false\n\t\t}\n\t\treturn nil\n\t}, 3*time.Second)\n\tif err := copyFile(filepath.Join(amp, configName), filepath.Join(currDir, \"config.txt\"), requireRootPrivileges); err != nil {\n\t\treturn fmt.Errorf(\"failed to get volume config %s: %v\", configName, err)\n\t}\n\n\tstatsName := \".jfs.stats\"\n\tif !prefixed {\n\t\tstatsName = statsName[4:]\n\t}\n\tstats := ctx.Uint64(\"stats-sec\")\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tsrcPath := filepath.Join(amp, statsName)\n\t\tdestPath := filepath.Join(currDir, \"stats.txt\")\n\t\tif err := copyFile(srcPath, destPath, requireRootPrivileges); err != nil {\n\t\t\tlogger.Errorf(\"Failed to get volume config %s: %v\", statsName, err)\n\t\t}\n\n\t\tlogger.Infof(\"Stats metrics are being sampled, sampling duration: %ds\", stats)\n\t\ttime.Sleep(time.Second * time.Duration(stats))\n\t\tdestPath = filepath.Join(currDir, fmt.Sprintf(\"stats.%ds.txt\", stats))\n\t\tif err := copyFile(srcPath, destPath, requireRootPrivileges); err != nil {\n\t\t\tlogger.Errorf(\"Failed to get volume config %s: %v\", statsName, err)\n\t\t}\n\t}()\n\treturn nil\n}\n\nfunc debug(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tmp := ctx.Args().First()\n\tvar inode uint64\n\tif err := utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\tvar err error\n\t\tif inode, err = utils.GetFileInode(mp); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to lookup inode for %s: %s\", mp, err)\n\t\t}\n\t\treturn nil\n\t}, 3*time.Second); err != nil {\n\t\tlogger.Warnf(err.Error())\n\t\tlogger.Warnf(\"assuming the mount point is JuiceFS mount point\")\n\t} else {\n\t\tif inode != uint64(meta.RootInode) {\n\t\t\treturn fmt.Errorf(\"path %s is not a mount point\", mp)\n\t\t}\n\t}\n\n\tamp, err := filepath.Abs(mp)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get absolute path: %v\", err)\n\t}\n\ttimestamp := time.Now().Format(\"20060102150405\")\n\tprefix := strings.Trim(strings.Join(strings.Split(amp, \"/\"), \"-\"), \"-\")\n\tif runtime.GOOS == \"windows\" {\n\t\tprefix = strings.ReplaceAll(prefix, \":\", \"\")\n\t}\n\toutDir := ctx.String(\"out-dir\")\n\tcurrDir := filepath.Join(outDir, fmt.Sprintf(\"%s-%s\", prefix, timestamp))\n\tif err := os.MkdirAll(currDir, os.ModePerm); err != nil {\n\t\treturn fmt.Errorf(\"failed to create current out dir %s: %v\", currDir, err)\n\t}\n\n\tif err := collectSysInfo(ctx, currDir); err != nil {\n\t\tlogger.Errorf(\"Failed to collect system info: %v\", err)\n\t}\n\n\tuid, pid, cmd, err := getCmdMount(amp)\n\tlogger.Infof(\"mount point:%s pid:%s uid:%s\", amp, pid, uid)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get mount command: %v\", err)\n\t}\n\tfmt.Printf(\"\\nMount Command:\\n%s\\n\\n\", cmd)\n\n\trequireRootPrivileges := false\n\tif (uid == \"0\" || uid == \"root\") && os.Getuid() != 0 {\n\t\tfmt.Println(\"Mount point is mounted by the root user, may ask for root privilege...\")\n\t\trequireRootPrivileges = true\n\t}\n\n\tvar wg sync.WaitGroup\n\tif err := collectSpecialFile(ctx, amp, currDir, requireRootPrivileges, &wg); err != nil {\n\t\tlogger.Errorf(\"Failed to collect special file: %v\", err)\n\t}\n\n\tif err := collectLog(ctx, cmd, requireRootPrivileges, currDir, uid); err != nil {\n\t\tlogger.Errorf(\"Failed to collect log: %v\", err)\n\t}\n\n\tif err := collectPprof(ctx, cmd, pid, amp, requireRootPrivileges, currDir, &wg); err != nil {\n\t\tlogger.Errorf(\"Failed to collect pprof: %v\", err)\n\t}\n\n\twg.Wait()\n\tabs, _ := filepath.Abs(currDir)\n\tlogger.Infof(\"All files are collected to %s\", abs)\n\treturn geneZipFile(currDir, filepath.Join(outDir, fmt.Sprintf(\"%s-%s.zip\", prefix, timestamp)))\n}\n" }, { "path": "cmd/debug_test.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestDebug(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\trequire.NotNil(t, Main([]string{\"\", \"debug\", \"/jfs/test/mp\"}), \"mount point does not exist\")\n\trequire.NotNil(t, Main([]string{\"\", \"debug\", \"./\"}), \"directory is not a mount point\")\n\trequire.NotNil(t, Main([]string{\"\", \"debug\", \"--out-dir\", \"./debug_test.go\", testMountPoint}), \"specify a file as out dir\")\n\n\tcases := []struct {\n\t\targ string\n\t\tval string\n\t}{\n\t\t{\"--log /var/log/jfs.log\", \"/var/log/jfs.log\"},\n\t\t{\"--log=/var/log/jfs.log\", \"/var/log/jfs.log\"},\n\t\t{\"--log = /var/log/jfs.log\", \"/var/log/jfs.log\"},\n\t\t{\"--log= /var/log/jfs.log\", \"/var/log/jfs.log\"},\n\t\t{\"--log =/var/log/jfs.log\", \"/var/log/jfs.log\"},\n\t\t{\"--log /var/log/jfs.log\", \"/var/log/jfs.log\"},\n\t}\n\tfor i, c := range cases {\n\t\trequire.True(t, logArg.FindStringSubmatch(c.arg)[2] == c.val, fmt.Sprintf(\"valid log arg %d\", i))\n\t}\n}\n" }, { "path": "cmd/debug_unix.go", "content": "//go:build !windows\r\n// +build !windows\r\n\r\n/*\r\n * JuiceFS, Copyright 2025 Juicedata, Inc.\r\n *\r\n * Licensed under the Apache License, Version 2.0 (the \"License\");\r\n * you may not use this file except in compliance with the License.\r\n * You may obtain a copy of the License at\r\n *\r\n * http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an \"AS IS\" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\n\r\npackage cmd\r\n\r\nimport (\r\n\t\"context\"\r\n\t\"encoding/json\"\r\n\t\"fmt\"\r\n\t\"os/exec\"\r\n\t\"strconv\"\r\n\t\"strings\"\r\n\t\"time\"\r\n\r\n\t\"github.com/juicedata/juicefs/pkg/utils\"\r\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\r\n)\r\n\r\nfunc getCmdMount(mp string) (uid, pid, cmd string, err error) {\r\n\tvar tmpPid string\r\n\t_ = utils.WithTimeout(context.TODO(), func(context.Context) error {\r\n\t\tcontent, err := readConfig(mp)\r\n\t\tif err != nil {\r\n\t\t\tlogger.Warnf(\"failed to read config file: %v\", err)\r\n\t\t}\r\n\t\tcfg := vfs.Config{}\r\n\t\tif err := json.Unmarshal(content, &cfg); err != nil {\r\n\t\t\tlogger.Warnf(\"failed to unmarshal config file: %v\", err)\r\n\t\t}\r\n\t\tif cfg.Pid != 0 {\r\n\t\t\ttmpPid = strconv.Itoa(cfg.Pid)\r\n\t\t}\r\n\t\treturn nil\r\n\t}, 3*time.Second)\r\n\r\n\tvar psArgs []string\r\n\tif tmpPid != \"\" {\r\n\t\tpid = tmpPid\r\n\t\tpsArgs = []string{\"/bin/sh\", \"-c\", fmt.Sprintf(\"ps -f -p %s\", pid)}\r\n\t} else {\r\n\t\tpsArgs = []string{\"/bin/sh\", \"-c\", fmt.Sprintf(\"ps -ef | grep -v grep | grep mount | grep %s\", mp)}\r\n\t}\r\n\tret, err := exec.Command(psArgs[0], psArgs[1:]...).CombinedOutput()\r\n\tif err != nil {\r\n\t\treturn \"\", \"\", \"\", fmt.Errorf(\"failed to execute command `%s`: %v\", strings.Join(psArgs, \" \"), err)\r\n\t}\r\n\tvar find bool\r\n\tvar ppid string\r\n\tlines := strings.Split(string(ret), \"\\n\")\r\n\tfor i := len(lines) - 1; i >= 0; i-- {\r\n\t\tline := lines[i]\r\n\t\tfields := strings.Fields(line)\r\n\t\tif len(fields) <= 7 {\r\n\t\t\tcontinue\r\n\t\t}\r\n\t\tcmdFields := fields[7:]\r\n\t\tfor _, arg := range cmdFields {\r\n\t\t\tif mp == arg {\r\n\t\t\t\tif find {\r\n\t\t\t\t\tnewCmd := strings.Join(fields[7:], \" \")\r\n\t\t\t\t\tnewUid, newPid, newPpid := strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1]), strings.TrimSpace(fields[2])\r\n\t\t\t\t\tif newPid == ppid {\r\n\t\t\t\t\t\treturn uid, pid, cmd, nil\r\n\t\t\t\t\t} else if pid == newPpid {\r\n\t\t\t\t\t\treturn newUid, newPid, newCmd, nil\r\n\t\t\t\t\t} else {\r\n\t\t\t\t\t\treturn \"\", \"\", \"\", fmt.Errorf(\"find more than one mount process for %s\", mp)\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t\tcmd = strings.Join(fields[7:], \" \")\r\n\t\t\t\tuid, pid, ppid = strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1]), strings.TrimSpace(fields[2])\r\n\t\t\t\tfind = true\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\tif cmd == \"\" {\r\n\t\treturn \"\", \"\", \"\", fmt.Errorf(\"no mount command found for %s\", mp)\r\n\t}\r\n\treturn uid, pid, cmd, nil\r\n}\r\n" }, { "path": "cmd/debug_windows.go", "content": "package cmd\r\n\r\n/*\r\n * JuiceFS, Copyright 2025 Juicedata, Inc.\r\n *\r\n * Licensed under the Apache License, Version 2.0 (the \"License\");\r\n * you may not use this file except in compliance with the License.\r\n * You may obtain a copy of the License at\r\n *\r\n * http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an \"AS IS\" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\n\r\nimport (\r\n\t\"context\"\r\n\t\"encoding/json\"\r\n\t\"fmt\"\r\n\t\"os\"\r\n\t\"os/exec\"\r\n\t\"path/filepath\"\r\n\t\"strconv\"\r\n\t\"strings\"\r\n\t\"time\"\r\n\r\n\t\"github.com/juicedata/juicefs/pkg/utils\"\r\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\r\n\t\"golang.org/x/sys/windows\"\r\n)\r\n\r\nfunc getprocessCommandLine(pid int) (string, error) {\r\n\tcmd := exec.Command(\"wmic\", \"process\", \"where\", \"ProcessID=\"+strconv.Itoa(pid), \"get\", \"CommandLine\")\r\n\tout, err := cmd.CombinedOutput()\r\n\tif err != nil {\r\n\t\treturn \"\", fmt.Errorf(\"failed to run command line: %s, %v\", cmd.String(), err)\r\n\t}\r\n\r\n\tlines := strings.Split(string(out), \"\\r\\n\")\r\n\tif len(lines) < 2 {\r\n\t\treturn \"\", fmt.Errorf(\"failed to find command line for pid: %d\", pid)\r\n\t}\r\n\r\n\tfor _, line := range lines[1:] {\r\n\t\tsline := strings.TrimSpace(line)\r\n\t\tif sline == \"\" {\r\n\t\t\tcontinue\r\n\t\t}\r\n\t\treturn sline, nil\r\n\t}\r\n\r\n\treturn \"\", fmt.Errorf(\"cannot find command line for pid %d. If the juicefs are mounted at background, Please rerun this with the admin permission.\", pid)\r\n}\r\n\r\nfunc findMountProcess(mp string) (int, error) {\r\n\tprocessName := filepath.Base(os.Args[0])\r\n\tcmd := exec.Command(\"wmic\", \"process\", \"where\", fmt.Sprintf(\"name='%s'\", processName), \"get\", \"CommandLine,ProcessId\")\r\n\tout, err := cmd.CombinedOutput()\r\n\tif err != nil {\r\n\t\treturn 0, fmt.Errorf(\"failed to exec command line: %s, %s\", cmd.String(), err)\r\n\t}\r\n\r\n\tlines := strings.Split(string(out), \"\\r\\n\")\r\n\tif len(lines) < 2 {\r\n\t\treturn 0, fmt.Errorf(\"failed to find mount process\")\r\n\t}\r\n\r\n\tmp = strings.TrimRight(mp, \"\\\\\")\r\n\tfor _, line := range lines[1:] {\r\n\t\tsline := strings.TrimSpace(line)\r\n\r\n\t\tif sline == \"\" {\r\n\t\t\tcontinue\r\n\t\t}\r\n\r\n\t\t// the first part of commandline contains 'xxx/mount.exe\"'\r\n\t\tslines := strings.SplitN(sline, \".exe\\\" \", 2)\r\n\t\tif len(slines) < 2 {\r\n\t\t\tlogger.Warnf(\"failed to split command line: %s\", sline)\r\n\t\t\tcontinue\r\n\t\t}\r\n\r\n\t\tsline = slines[1]\r\n\t\tlogger.Infof(\"sline: %s\", sline)\r\n\r\n\t\targs := strings.Split(sline, \" \")\r\n\t\tif len(args) < 3 {\r\n\t\t\tcontinue\r\n\t\t}\r\n\t\tmpFound := false\r\n\t\tmountFound := false\r\n\t\tfor _, arg := range args {\r\n\t\t\targ = strings.TrimSpace(arg)\r\n\t\t\tif arg == \"\" {\r\n\t\t\t\tcontinue\r\n\t\t\t}\r\n\r\n\t\t\tif arg == \"mount\" {\r\n\t\t\t\tmountFound = true\r\n\t\t\t\tcontinue\r\n\t\t\t}\r\n\r\n\t\t\targ = strings.TrimRight(arg, \"\\\\\")\r\n\r\n\t\t\tif strings.EqualFold(arg, mp) {\r\n\t\t\t\tmpFound = true\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tif mpFound && mountFound {\r\n\t\t\t// THE LAST PART IS PID\r\n\t\t\tpid, err := strconv.Atoi(args[len(args)-1])\r\n\t\t\tif err != nil {\r\n\t\t\t\treturn 0, fmt.Errorf(\"failed to parse pid: %s\", args[len(args)-1])\r\n\t\t\t}\r\n\t\t\treturn pid, nil\r\n\t\t}\r\n\t}\r\n\r\n\treturn 0, fmt.Errorf(\"cannot find the mount process for %s\", mp)\r\n}\r\n\r\nfunc getProcessUserSid(pid int) (string, error) {\r\n\th, err := windows.OpenProcess(windows.PROCESS_QUERY_INFORMATION, false, uint32(pid))\r\n\tif err != nil {\r\n\t\treturn \"\", err\r\n\t}\r\n\tdefer windows.CloseHandle(h)\r\n\r\n\tvar token windows.Token\r\n\terr = windows.OpenProcessToken(h, windows.TOKEN_QUERY, &token)\r\n\tif err != nil {\r\n\t\treturn \"\", err\r\n\t}\r\n\tdefer token.Close()\r\n\r\n\tuser, err := token.GetTokenUser()\r\n\tif err != nil {\r\n\t\treturn \"\", err\r\n\t}\r\n\r\n\treturn user.User.Sid.String(), nil\r\n\r\n}\r\n\r\nfunc getCmdMount(mp string) (uid, pid, cmd string, err error) {\r\n\tvar tmpPid string\r\n\t_ = utils.WithTimeout(context.TODO(), func(context.Context) error {\r\n\t\tcontent, err := readConfig(mp)\r\n\t\tif err != nil {\r\n\t\t\tlogger.Warnf(\"failed to read config file: %v\", err)\r\n\t\t}\r\n\t\tcfg := vfs.Config{}\r\n\t\tif err := json.Unmarshal(content, &cfg); err != nil {\r\n\t\t\tlogger.Warnf(\"failed to unmarshal config file: %v\", err)\r\n\t\t}\r\n\t\tif cfg.Pid != 0 {\r\n\t\t\ttmpPid = strconv.Itoa(cfg.Pid)\r\n\t\t}\r\n\t\treturn nil\r\n\t}, 3*time.Second)\r\n\r\n\tfoundPid := 0\r\n\tif tmpPid != \"\" {\r\n\t\tpid = tmpPid\r\n\t\tfoundPid, err = strconv.Atoi(pid)\r\n\t\tif err != nil {\r\n\t\t\treturn \"\", \"\", \"\", fmt.Errorf(\"failed to parse pid: %s\", pid)\r\n\t\t}\r\n\t} else {\r\n\t\tfoundPid, err = findMountProcess(mp)\r\n\t\tif err != nil {\r\n\t\t\treturn \"\", \"\", \"\", err\r\n\t\t}\r\n\r\n\t\tpid = strconv.Itoa(foundPid)\r\n\t}\r\n\r\n\tcmd, err = getprocessCommandLine(foundPid)\r\n\tif err != nil {\r\n\t\treturn \"\", \"\", \"\", err\r\n\t}\r\n\r\n\tuid, err = getProcessUserSid(foundPid)\r\n\tif err != nil {\r\n\t\treturn \"\", \"\", \"\", err\r\n\t}\r\n\r\n\treturn uid, pid, cmd, nil\r\n}\r\n" }, { "path": "cmd/destroy.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdDestroy() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"destroy\",\n\t\tAction: destroy,\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"Destroy an existing volume\",\n\t\tArgsUsage: \"META-URL UUID\",\n\t\tDescription: `\nDestroy the target volume, removing all objects in the data storage and all entries in its metadata engine.\n\nWARNING: BE CAREFUL! This operation cannot be undone.\n\nExamples:\n$ juicefs destroy redis://localhost e94d66a8-2339-4abd-b8d8-6812df737892\n\nDetails: https://juicefs.com/docs/community/administration/destroy`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"yes\",\n\t\t\t\tAliases: []string{\"y\"},\n\t\t\t\tUsage: \"automatically answer 'yes' to all prompts and run non-interactively\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"force\",\n\t\t\t\tUsage: \"skip sanity check and force destroy the volume\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc printSessions(ss [][3]string) string {\n\theader := [3]string{\"SID\", \"HostName\", \"MountPoint\"}\n\tvar max [3]int\n\tfor i := 0; i < 3; i++ {\n\t\tmax[i] = len(header[i])\n\t}\n\tfor _, s := range ss {\n\t\tfor i := 0; i < 3; i++ {\n\t\t\tif l := len(s[i]); l > max[i] {\n\t\t\t\tmax[i] = l\n\t\t\t}\n\t\t}\n\t}\n\n\tvar ret, b strings.Builder\n\tfor i := 0; i < 3; i++ {\n\t\tb.WriteByte('+')\n\t\tb.WriteString(strings.Repeat(\"-\", max[i]+2))\n\t}\n\tb.WriteString(\"+\\n\")\n\tdivider := b.String()\n\tret.WriteString(divider)\n\n\tb.Reset()\n\tfor i := 0; i < 3; i++ {\n\t\tb.WriteString(\" | \")\n\t\tb.WriteString(padding(header[i], max[i], ' '))\n\t}\n\tb.WriteString(\" |\\n\")\n\tret.WriteString(b.String()[1:])\n\tret.WriteString(divider)\n\n\tfor _, s := range ss {\n\t\tb.Reset()\n\t\tfor i := 0; i < 3; i++ {\n\t\t\tb.WriteString(\" | \")\n\t\t\tif spaces := max[i] - len(s[i]); spaces > 0 {\n\t\t\t\tb.WriteString(strings.Repeat(\" \", spaces))\n\t\t\t}\n\t\t\tb.WriteString(s[i])\n\t\t}\n\t\tb.WriteString(\" |\\n\")\n\t\tret.WriteString(b.String()[1:])\n\t}\n\tret.WriteString(divider)\n\n\treturn ret.String()\n}\n\nfunc destroy(ctx *cli.Context) error {\n\tsetup(ctx, 2)\n\turi := ctx.Args().Get(0)\n\tif !strings.Contains(uri, \"://\") {\n\t\turi = \"redis://\" + uri\n\t}\n\tremovePassword(uri)\n\tm := meta.NewClient(uri, meta.DefaultConf())\n\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"load setting: %s\", err)\n\t}\n\tif uuid := ctx.Args().Get(1); uuid != format.UUID {\n\t\tlogger.Fatalf(\"UUID %s != expected %s\", uuid, format.UUID)\n\t}\n\tblob, err := createStorage(*format)\n\tif err != nil {\n\t\tlogger.Fatalf(\"create object storage: %s\", err)\n\t}\n\n\tif !ctx.Bool(\"force\") {\n\t\tm.CleanStaleSessions(meta.Background())\n\t\tsessions, err := m.ListSessions()\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"list sessions: %s\", err)\n\t\t}\n\t\tif num := len(sessions); num > 0 {\n\t\t\tss := make([][3]string, num)\n\t\t\tfor i, s := range sessions {\n\t\t\t\tss[i] = [3]string{strconv.FormatUint(s.Sid, 10), s.HostName, s.MountPoint}\n\t\t\t}\n\t\t\tlogger.Fatalf(\"%d sessions are active, please disconnect them first:\\n%s\", num, printSessions(ss))\n\t\t}\n\t\tvar totalSpace, availSpace, iused, iavail uint64\n\t\t_ = m.StatFS(meta.Background(), meta.RootInode, &totalSpace, &availSpace, &iused, &iavail)\n\n\t\tfmt.Printf(\" volume name: %s\\n\", format.Name)\n\t\tfmt.Printf(\" volume UUID: %s\\n\", format.UUID)\n\t\tfmt.Printf(\"data storage: %s\\n\", blob)\n\t\tfmt.Printf(\" used bytes: %d\\n\", totalSpace-availSpace)\n\t\tfmt.Printf(\" used inodes: %d\\n\", iused)\n\t\twarn(\"The target volume will be permanently destroyed, including:\")\n\t\twarn(\"1. ALL objects in the data storage: %s\", blob)\n\t\twarn(\"2. ALL entries in the metadata engine: %s\", utils.RemovePassword(uri))\n\t\tif !ctx.Bool(\"yes\") && !userConfirmed() {\n\t\t\tlogger.Fatalln(\"Aborted.\")\n\t\t}\n\t}\n\n\tobjs, err := object.ListAll(ctx.Context, blob, \"\", \"\", true, false)\n\tif err != nil {\n\t\tlogger.Fatalf(\"list all objects: %s\", err)\n\t}\n\tprogress := utils.NewProgress(false)\n\tspin := progress.AddCountSpinner(\"Deleted objects\")\n\tvar failed int\n\tvar dirs []string\n\tvar mu sync.Mutex\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 8; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor obj := range objs {\n\t\t\t\tif obj == nil {\n\t\t\t\t\tbreak // failed listing\n\t\t\t\t}\n\t\t\t\tif obj.IsDir() {\n\t\t\t\t\tmu.Lock()\n\t\t\t\t\tdirs = append(dirs, obj.Key())\n\t\t\t\t\tmu.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif err := blob.Delete(ctx.Context, obj.Key()); err == nil {\n\t\t\t\t\tspin.Increment()\n\t\t\t\t} else {\n\t\t\t\t\tfailed++\n\t\t\t\t\tlogger.Warnf(\"delete %s: %s\", obj.Key(), err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tsort.Strings(dirs)\n\tfor i := len(dirs) - 1; i >= 0; i-- {\n\t\tif err := blob.Delete(ctx.Context, dirs[i]); err == nil {\n\t\t\tspin.Increment()\n\t\t} else {\n\t\t\tfailed++\n\t\t\tlogger.Warnf(\"delete %s: %s\", dirs[i], err)\n\t\t}\n\t}\n\tprogress.Done()\n\tif progress.Quiet {\n\t\tlogger.Infof(\"Deleted %d objects\", spin.Current())\n\t}\n\tif failed > 0 {\n\t\tlogger.Errorf(\"%d objects are failed to delete, please do it manually.\", failed)\n\t}\n\n\tif err = m.Reset(); err != nil {\n\t\tlogger.Fatalf(\"reset meta: %s\", err)\n\t}\n\n\tlogger.Infof(\"The volume has been destroyed! You may need to delete cache directory manually.\")\n\treturn nil\n}\n" }, { "path": "cmd/dump.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"compress/gzip\"\n\t\"errors\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/DataDog/zstd\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdDump() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"dump\",\n\t\tAction: dump,\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"Dump metadata into a file\",\n\t\tArgsUsage: \"META-URL [FILE]\",\n\t\tDescription: `\nSupports two formats: JSON format and binary format.\n1. Dump metadata of the volume in JSON format so users are able to see its content in an easy way.\nOutput of this command can be loaded later into an empty database, serving as a method to backup\nmetadata or to change metadata engine.\n\nExamples:\n$ juicefs dump redis://localhost meta-dump.json\n$ juicefs dump redis://localhost meta-dump.json.gz\n\n# Dump only a subtree of the volume to STDOUT\n$ juicefs dump redis://localhost --subdir /dir/in/jfs\n\n2. Binary format is more compact, faster, and memory-efficient.\n\nExamples:\n$ juicefs dump redis://localhost meta-dump.bin --binary\n\nDetails: https://juicefs.com/docs/community/metadata_dump_load`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"subdir\",\n\t\t\t\tUsage: \"only dump a sub-directory\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"keep-secret-key\",\n\t\t\t\tUsage: \"keep secret keys intact (WARNING: Be careful as they may be leaked)\",\n\t\t\t},\n\t\t\t&cli.IntFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"number of threads to dump metadata\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"fast\",\n\t\t\t\tUsage: \"speedup dump by load all metadata into memory (only works with JSON format and DB/KV engine)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"skip-trash\",\n\t\t\t\tUsage: \"skip files in trash\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"binary\",\n\t\t\t\tUsage: \"dump metadata into a binary file (different from original JSON format, subdir/fast/skip-trash will be ignored)\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc dumpMeta(m meta.Meta, dst string, threads int, keepSecret, fast, skipTrash, isBinary bool) (err error) {\n\tvar w io.WriteCloser\n\tif dst == \"\" {\n\t\tw = os.Stdout\n\t} else {\n\t\ttmp := dst + \".tmp\"\n\t\tfp, e := os.Create(tmp)\n\t\tif e != nil {\n\t\t\treturn e\n\t\t}\n\t\tdefer func() {\n\t\t\terr = errors.Join(err, fp.Close())\n\t\t\tif err == nil {\n\t\t\t\terr = os.Rename(tmp, dst)\n\t\t\t} else {\n\t\t\t\t_ = os.Remove(tmp)\n\t\t\t}\n\t\t}()\n\n\t\tif strings.HasSuffix(dst, \".gz\") {\n\t\t\tw, _ = gzip.NewWriterLevel(fp, gzip.BestSpeed)\n\t\t\tdefer func() {\n\t\t\t\terr = errors.Join(err, w.Close())\n\t\t\t}()\n\t\t} else if strings.HasSuffix(dst, \".zstd\") {\n\t\t\tw = zstd.NewWriterLevel(fp, zstd.BestSpeed)\n\t\t\tdefer func() {\n\t\t\t\terr = errors.Join(err, w.Close())\n\t\t\t}()\n\t\t} else {\n\t\t\tw = fp\n\t\t}\n\t}\n\tif isBinary {\n\t\tprogress := utils.NewProgress(false)\n\t\tdefer progress.Done()\n\n\t\tbars := make(map[string]*utils.Bar)\n\t\tfor _, name := range meta.SegType2Name {\n\t\t\tbars[name] = progress.AddCountSpinner(name)\n\t\t}\n\n\t\treturn m.DumpMetaV2(meta.Background(), w, &meta.DumpOption{\n\t\t\tKeepSecret: keepSecret,\n\t\t\tThreads: threads,\n\t\t\tProgress: func(name string, cnt int) {\n\t\t\t\tbars[name].IncrBy(cnt)\n\t\t\t},\n\t\t})\n\t}\n\treturn m.DumpMeta(w, 1, threads, keepSecret, fast, skipTrash)\n}\n\nfunc dump(ctx *cli.Context) error {\n\tsetup0(ctx, 1, 2)\n\tmetaUri := ctx.Args().Get(0)\n\tvar dst string\n\tif ctx.Args().Len() > 1 {\n\t\tdst = ctx.Args().Get(1)\n\t}\n\tremovePassword(metaUri)\n\n\tmetaConf := meta.DefaultConf()\n\tmetaConf.Subdir = ctx.String(\"subdir\")\n\tm := meta.NewClient(metaUri, metaConf)\n\tif _, err := m.Load(true); err != nil {\n\t\treturn err\n\t}\n\tif st := m.Chroot(meta.Background(), metaConf.Subdir); st != 0 {\n\t\treturn st\n\t}\n\n\tthreads := ctx.Int(\"threads\")\n\tif threads <= 0 {\n\t\tlogger.Warnf(\"Invalid threads number %d, reset to 1\", threads)\n\t\tthreads = 1\n\t}\n\n\terr := dumpMeta(m, dst, threads, ctx.Bool(\"keep-secret-key\"), ctx.Bool(\"fast\"), ctx.Bool(\"skip-trash\"), ctx.Bool(\"binary\"))\n\tif err == nil {\n\t\tif dst == \"\" {\n\t\t\tdst = \"STDOUT\"\n\t\t}\n\t\tlogger.Infof(\"Dump metadata into %s succeed\", dst)\n\t}\n\treturn err\n}\n" }, { "path": "cmd/dump_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/redis/go-redis/v9\"\n)\n\nfunc TestDumpAndLoad(t *testing.T) {\n\tmetaUrl := \"redis://127.0.0.1:6379/15\"\n\topt, err := redis.ParseURL(metaUrl)\n\tif err != nil {\n\t\tt.Fatalf(\"ParseURL: %v\", err)\n\t}\n\trdb := redis.NewClient(opt)\n\trdb.FlushDB(context.Background())\n\n\tt.Run(\"Test Load\", func(t *testing.T) {\n\t\tloadArgs := []string{\"\", \"load\", metaUrl, \"./../pkg/meta/metadata.sample\"}\n\t\terr = Main(loadArgs)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"load failed: %v\", err)\n\t\t}\n\t\tif rdb.DBSize(context.Background()).Val() == 0 {\n\t\t\tt.Fatalf(\"load error: %v\", err)\n\t\t}\n\t})\n\tt.Run(\"Test dump\", func(t *testing.T) {\n\t\tdumpArgs := []string{\"\", \"dump\", metaUrl, \"/tmp/dump_test.json.gz\"}\n\t\terr := Main(dumpArgs)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"dump error: %v\", err)\n\t\t}\n\t\t_, err = os.Stat(\"/tmp/dump_test.json.gz\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"dump error: %v\", err)\n\t\t}\n\t})\n\n\trdb.FlushDB(context.Background())\n\tt.Run(\"Test load compressed\", func(t *testing.T) {\n\t\tloadArgs := []string{\"\", \"load\", metaUrl, \"/tmp/dump_test.json.gz\"}\n\t\terr := Main(loadArgs)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"load error: %v\", err)\n\t\t}\n\t\tif rdb.DBSize(context.Background()).Val() == 0 {\n\t\t\tt.Fatalf(\"load error: %v\", err)\n\t\t}\n\t})\n\n\tt.Run(\"Test dump with subdir\", func(t *testing.T) {\n\t\tdumpArgs := []string{\"\", \"dump\", metaUrl, \"/tmp/dump_subdir_test.json\", \"--subdir\", \"d1\"}\n\t\terr := Main(dumpArgs)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"dump error: %v\", err)\n\t\t}\n\t\t_, err = os.Stat(\"/tmp/dump_subdir_test.json\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"dump error: %v\", err)\n\t\t}\n\t})\n\trdb.FlushDB(context.Background())\n}\n" }, { "path": "cmd/flags.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc globalFlags() []cli.Flag {\n\treturn []cli.Flag{\n\t\t&cli.BoolFlag{\n\t\t\tName: \"verbose\",\n\t\t\tAliases: []string{\"debug\", \"v\"},\n\t\t\tUsage: \"enable debug log\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"quiet\",\n\t\t\tAliases: []string{\"q\"},\n\t\t\tUsage: \"show warning and errors only\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"trace\",\n\t\t\tUsage: \"enable trace log\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"log-level\",\n\t\t\tUsage: \"set log level (trace, debug, info, warn, error, fatal, panic)\",\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"log-id\",\n\t\t\tUsage: \"append the given log id in log, use \\\"random\\\" to use random uuid\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"no-agent\",\n\t\t\tUsage: \"disable pprof (:6060) agent\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"pyroscope\",\n\t\t\tUsage: \"pyroscope address\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"no-color\",\n\t\t\tUsage: \"disable colors\",\n\t\t},\n\t}\n}\n\nfunc addCategory(f cli.Flag, cat string) {\n\tswitch ff := f.(type) {\n\tcase *cli.StringFlag:\n\t\tff.Category = cat\n\tcase *cli.BoolFlag:\n\t\tff.Category = cat\n\tcase *cli.IntFlag:\n\t\tff.Category = cat\n\tcase *cli.Int64Flag:\n\t\tff.Category = cat\n\tcase *cli.Uint64Flag:\n\t\tff.Category = cat\n\tcase *cli.Float64Flag:\n\t\tff.Category = cat\n\tcase *cli.StringSliceFlag:\n\t\tff.Category = cat\n\tdefault:\n\t\tpanic(f)\n\t}\n}\n\nfunc addCategories(cat string, flags []cli.Flag) []cli.Flag {\n\tfor _, f := range flags {\n\t\taddCategory(f, cat)\n\t}\n\treturn flags\n}\n\nfunc storageFlags() []cli.Flag {\n\treturn addCategories(\"DATA STORAGE\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"storage\",\n\t\t\tUsage: \"customized storage type (e.g. s3, gs, oss, cos) to access object store\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"bucket\",\n\t\t\tUsage: \"customized endpoint to access object store\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"storage-class\",\n\t\t\tUsage: \"the storage class for data written by current client\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"get-timeout\",\n\t\t\tValue: \"60s\",\n\t\t\tUsage: \"the timeout to download an object\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"put-timeout\",\n\t\t\tValue: \"60s\",\n\t\t\tUsage: \"the timeout to upload an object\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"io-retries\",\n\t\t\tValue: 10,\n\t\t\tUsage: \"number of retries after network failure\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"max-uploads\",\n\t\t\tValue: 20,\n\t\t\tUsage: \"number of connections to upload\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"max-downloads\",\n\t\t\tValue: 200,\n\t\t\tUsage: \"number of connections to download\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"max-stage-write\",\n\t\t\tValue: 1000, // large enough for normal cases, also prevents unlimited concurrency in abnormal cases\n\t\t\tUsage: \"number of threads allowed to write staged files, other requests will be uploaded directly (this option is only effective when 'writeback' mode is enabled)\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"max-deletes\",\n\t\t\tValue: 10,\n\t\t\tUsage: \"number of threads to delete objects\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"upload-limit\",\n\t\t\tUsage: \"bandwidth limit for upload in Mbps\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"download-limit\",\n\t\t\tUsage: \"bandwidth limit for download in Mbps\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"check-storage\",\n\t\t\t// AK/SK should have been checked before creating volume, here checks client access to the storage\n\t\t\tUsage: \"test storage before mounting to expose access issues early\",\n\t\t},\n\t})\n}\n\nfunc getDefaultCacheDir() string {\n\tvar defaultCacheDir = \"/var/jfsCache\"\n\tswitch runtime.GOOS {\n\tcase \"linux\":\n\t\tif os.Getuid() == 0 {\n\t\t\tbreak\n\t\t}\n\t\tfallthrough\n\tcase \"darwin\":\n\t\thomeDir, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"%v\", err)\n\t\t\thomeDir = defaultCacheDir\n\t\t}\n\t\tdefaultCacheDir = path.Join(homeDir, \".juicefs\", \"cache\")\n\tcase \"windows\":\n\t\thomeDir, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"%v\", err)\n\t\t\treturn \"\"\n\t\t}\n\t\tdefaultCacheDir = path.Join(homeDir, \".juicefs\", \"cache\")\n\t}\n\treturn defaultCacheDir\n}\n\nfunc dataCacheFlags() []cli.Flag {\n\tvar defaultCacheDir = getDefaultCacheDir()\n\treturn addCategories(\"DATA CACHE\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"buffer-size\",\n\t\t\tValue: \"300M\",\n\t\t\tUsage: \"total read/write buffering in MiB\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"max-readahead\",\n\t\t\tUsage: \"max buffering for read ahead in MiB per read session\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"prefetch\",\n\t\t\tValue: 1,\n\t\t\tUsage: \"prefetch N blocks in parallel\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"writeback\",\n\t\t\tUsage: \"upload blocks in background\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"writeback-threshold-size\",\n\t\t\tValue: \"0\",\n\t\t\tUsage: \"blocks smaller than this size will be staged, 0 means all staged.\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"upload-delay\",\n\t\t\tValue: \"0s\",\n\t\t\tUsage: \"delayed duration for uploading blocks\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"upload-hours\",\n\t\t\tUsage: \"(start-end) hour of a day between which the delayed blocks can be uploaded\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"cache-dir\",\n\t\t\tValue: defaultCacheDir,\n\t\t\tUsage: \"directory paths of local cache, use colon to separate multiple paths\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"cache-mode\",\n\t\t\tValue: \"0600\", // only owner can read/write cache\n\t\t\tUsage: \"file permissions for cached blocks\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"cache-size\",\n\t\t\tValue: \"100G\",\n\t\t\tUsage: \"size of cached object for read in MiB\",\n\t\t},\n\t\t&cli.Int64Flag{\n\t\t\tName: \"cache-items\",\n\t\t\tValue: 0,\n\t\t\tUsage: \"max number of cached items (0 will be automatically calculated based on the `free‑space‑ratio`.)\",\n\t\t},\n\t\t&cli.Float64Flag{\n\t\t\tName: \"free-space-ratio\",\n\t\t\tValue: 0.1,\n\t\t\tUsage: \"min free space (ratio)\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"cache-partial-only\",\n\t\t\tUsage: \"cache only random/small read\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"cache-large-write\",\n\t\t\tUsage: \"cache full blocks after uploading\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"verify-cache-checksum\",\n\t\t\tValue: \"extend\",\n\t\t\tUsage: \"checksum level (none, full, shrink, extend)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"cache-eviction\",\n\t\t\tValue: chunk.Eviction2Random,\n\t\t\tUsage: fmt.Sprintf(\"cache eviction policy [%s, %s, %s]\", chunk.EvictionNone, chunk.Eviction2Random, chunk.EvictionLRU),\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"cache-scan-interval\",\n\t\t\tValue: \"1h\",\n\t\t\tUsage: \"interval to scan cache-dir to rebuild in-memory index\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"cache-expire\",\n\t\t\tValue: \"0s\",\n\t\t\tUsage: \"cached blocks not accessed for longer than this option will be automatically evicted (0 means never)\",\n\t\t},\n\t})\n}\n\nfunc metaFlags() []cli.Flag {\n\treturn addCategories(\"META\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"subdir\",\n\t\t\tUsage: \"mount a sub-directory as root\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"backup-meta\",\n\t\t\tValue: \"1h\",\n\t\t\tUsage: \"interval to automatically backup metadata in the object storage (0 means disable backup)\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"backup-skip-trash\",\n\t\t\tUsage: \"skip files in trash when backup metadata\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"heartbeat\",\n\t\t\tValue: \"12s\",\n\t\t\tUsage: \"interval to send heartbeat; it's recommended that all clients use the same heartbeat value\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"read-only\",\n\t\t\tUsage: \"allow lookup/read operations only\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"no-bgjob\",\n\t\t\tUsage: \"disable background jobs (clean-up, backup, etc.)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"atime-mode\",\n\t\t\tValue: \"noatime\",\n\t\t\tUsage: \"when to update atime, supported mode includes: noatime, relatime, strictatime\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"skip-dir-nlink\",\n\t\t\tValue: 20,\n\t\t\tUsage: \"number of retries after which the update of directory nlink will be skipped (used for tkv only, 0 means never)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"skip-dir-mtime\",\n\t\t\tValue: \"100ms\",\n\t\t\tUsage: \"skip updating attribute of a directory if the mtime difference is smaller than this value\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"sort-dir\",\n\t\t\tUsage: \"sort entries within a directory by name\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"fast-statfs\",\n\t\t\tValue: false,\n\t\t\tUsage: \"Use local counters for statfs instead of querying metadata service\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"network-interfaces\",\n\t\t\tUsage: \"comma-separated list of network interfaces to use for IP discovery (e.g. eth0,en0), empty means all\",\n\t\t},\n\t})\n}\n\nfunc clientFlags(defaultEntryCache float64) []cli.Flag {\n\treturn expandFlags(\n\t\tmetaFlags(),\n\t\tmetaCacheFlags(defaultEntryCache),\n\t\tstorageFlags(),\n\t\tdataCacheFlags(),\n\t)\n}\n\nfunc shareInfoFlags() []cli.Flag {\n\treturn addCategories(\"METRICS\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"metrics\",\n\t\t\tValue: \"127.0.0.1:9567\",\n\t\t\tUsage: \"address to export metrics\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"custom-labels\",\n\t\t\tUsage: \"custom labels for metrics\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"consul\",\n\t\t\tValue: \"127.0.0.1:8500\",\n\t\t\tUsage: \"consul address to register\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"no-usage-report\",\n\t\t\tUsage: \"do not send usage report\",\n\t\t},\n\t})\n}\n\nfunc metaCacheFlags(defaultEntryCache float64) []cli.Flag {\n\treturn addCategories(\"META CACHE\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"attr-cache\",\n\t\t\tValue: \"1.0s\",\n\t\t\tUsage: \"attributes cache timeout\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"entry-cache\",\n\t\t\tValue: fmt.Sprintf(\"%.1fs\", defaultEntryCache),\n\t\t\tUsage: \"file entry cache timeout\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"dir-entry-cache\",\n\t\t\tValue: \"1.0s\",\n\t\t\tUsage: \"dir entry cache timeout\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"negative-entry-cache\",\n\t\t\tUsage: \"cache timeout for negative entry lookups\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"readdir-cache\",\n\t\t\tUsage: \"enable kernel caching of readdir entries, with timeout controlled by attr-cache flag (require linux kernel 4.20+)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"open-cache\",\n\t\t\tValue: \"0s\",\n\t\t\tUsage: \"The cache time to reuse open file without checking update (0 means disable this feature)\",\n\t\t},\n\t\t&cli.Uint64Flag{\n\t\t\tName: \"open-cache-limit\",\n\t\t\tValue: 10000,\n\t\t\tUsage: \"max number of open files to cache (soft limit, 0 means unlimited)\",\n\t\t},\n\t})\n}\n\nfunc expandFlags(compoundFlags ...[]cli.Flag) []cli.Flag {\n\tvar flags []cli.Flag\n\tfor _, flag := range compoundFlags {\n\t\tflags = append(flags, flag...)\n\t}\n\treturn flags\n}\n" }, { "path": "cmd/flags_test.go", "content": "package cmd\n\nimport (\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc Test_duration(t *testing.T) {\n\ttype args struct {\n\t\ts string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant time.Duration\n\t}{\n\t\t{\n\t\t\tname: \"DurationWithSeconds\",\n\t\t\targs: args{s: \"60\"},\n\t\t\twant: time.Minute,\n\t\t},\n\t\t{\n\t\t\tname: \"DurationWithHours\",\n\t\t\targs: args{s: \"2h\"},\n\t\t\twant: 2 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tname: \"DurationWithDays\",\n\t\t\targs: args{s: \"1d\"},\n\t\t\twant: 24 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tname: \"DurationWithDaysAndTime\",\n\t\t\targs: args{s: \"1d2h\"},\n\t\t\twant: 26 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tname: \"DurationWithInvalidInput\",\n\t\t\targs: args{s: \"invalid\"},\n\t\t\twant: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"DurationWithEmptyString\",\n\t\t\targs: args{s: \"\"},\n\t\t\twant: 0,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tassert.Equalf(t, tt.want, utils.Duration(tt.args.s), \"duration(%v)\", tt.args.s)\n\t\t})\n\t}\n}\n" }, { "path": "cmd/format.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net/http\"\n\t_ \"net/http/pprof\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/google/uuid\"\n\t\"github.com/juicedata/juicefs/pkg/compress\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdFormat() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"format\",\n\t\tAction: format,\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"Format a volume\",\n\t\tArgsUsage: \"META-URL NAME\",\n\t\tDescription: `\nCreate a new JuiceFS volume. Here META-URL is used to set up the metadata engine (Redis, TiKV, MySQL, etc.),\nand NAME is the prefix of all objects in data storage.\n\nDEPRECATED: It was also used to change configuration of an existing volume, but now this function is\ndeprecated, instead please use the \"config\" command.\n\nExamples:\n# Create a simple test volume (data will be stored in a local directory)\n$ juicefs format sqlite3://myjfs.db myjfs\n\n# Create a volume with Redis and S3\n$ juicefs format redis://localhost myjfs --storage s3 --bucket https://mybucket.s3.us-east-2.amazonaws.com\n\n# Create a volume with password protected MySQL\n$ juicefs format mysql://jfs:mypassword@(127.0.0.1:3306)/juicefs myjfs\n# A safer alternative\n$ META_PASSWORD=mypassword juicefs format mysql://jfs:@(127.0.0.1:3306)/juicefs myjfs\n\n# Create a volume with \"quota\" enabled\n$ juicefs format sqlite3://myjfs.db myjfs --inodes 1000000 --capacity 102400\n\n# Create a volume with \"trash\" disabled\n$ juicefs format sqlite3://myjfs.db myjfs --trash-days 0\n\nDetails: https://juicefs.com/docs/community/quick_start_guide`,\n\t\tFlags: expandFlags(\n\t\t\tformatStorageFlags(),\n\t\t\tformatFlags(),\n\t\t\tformatManagementFlags(),\n\t\t\t[]cli.Flag{\n\t\t\t\t&cli.BoolFlag{\n\t\t\t\t\tName: \"force\",\n\t\t\t\t\tUsage: \"overwrite existing format\",\n\t\t\t\t},\n\t\t\t\t&cli.BoolFlag{\n\t\t\t\t\tName: \"no-update\",\n\t\t\t\t\tUsage: \"don't update existing volume\",\n\t\t\t\t},\n\t\t\t}),\n\t}\n}\n\nfunc formatStorageFlags() []cli.Flag {\n\tvar defaultBucket = \"/var/jfs\"\n\tswitch runtime.GOOS {\n\tcase \"linux\":\n\t\tif os.Getuid() == 0 {\n\t\t\tbreak\n\t\t}\n\t\tfallthrough\n\tcase \"darwin\":\n\t\thomeDir, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tlogger.Warn(err)\n\t\t\thomeDir = defaultBucket\n\t\t}\n\t\tdefaultBucket = path.Join(homeDir, \".juicefs\", \"local\")\n\tcase \"windows\":\n\t\tdefaultBucket = path.Join(\"C:/jfs/local\")\n\t}\n\treturn addCategories(\"DATA STORAGE\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"storage\",\n\t\t\tValue: \"file\",\n\t\t\tUsage: \"object storage type (e.g. s3, gs, oss, cos)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"bucket\",\n\t\t\tValue: defaultBucket,\n\t\t\tUsage: \"the bucket URL of object storage to store data\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"access-key\",\n\t\t\tUsage: \"access key for object storage (env ACCESS_KEY)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"secret-key\",\n\t\t\tUsage: \"secret key for object storage (env SECRET_KEY)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"session-token\",\n\t\t\tUsage: \"session token for object storage\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"storage-class\",\n\t\t\tUsage: \"the default storage class\",\n\t\t},\n\t})\n}\n\nfunc formatFlags() []cli.Flag {\n\treturn addCategories(\"DATA FORMAT\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"block-size\",\n\t\t\tValue: \"4M\",\n\t\t\tUsage: \"size of block in KiB\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"compress\",\n\t\t\tValue: \"none\",\n\t\t\tUsage: \"compression algorithm (lz4, zstd, none)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"encrypt-rsa-key\",\n\t\t\tUsage: \"a path to RSA private key (PEM)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"encrypt-algo\",\n\t\t\tUsage: \"encrypt algorithm (aes256gcm-rsa, chacha20-rsa)\",\n\t\t\tValue: object.AES256GCM_RSA,\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"hash-prefix\",\n\t\t\tUsage: \"add a hash prefix to name of objects\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"shards\",\n\t\t\tUsage: \"store the blocks into N buckets by hash of key\",\n\t\t},\n\t})\n}\n\nfunc formatManagementFlags() []cli.Flag {\n\treturn addCategories(\"MANAGEMENT\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"capacity\",\n\t\t\tUsage: \"hard quota of the volume limiting its usage of space in GiB\",\n\t\t},\n\t\t&cli.Uint64Flag{\n\t\t\tName: \"inodes\",\n\t\t\tUsage: \"hard quota of the volume limiting its number of inodes\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"trash-days\",\n\t\t\tValue: 1,\n\t\t\tUsage: \"number of days after which removed files will be permanently deleted\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"enable-acl\",\n\t\t\tUsage: \"enable POSIX ACL (this flag is irreversible once enabled)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"ranger-rest-url\",\n\t\t\tUsage: \"URL of the RangerAdmin\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"ranger-service\",\n\t\t\tUsage: \"Name of the Ranger service used For JuiceFS\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"kerberos-config-file\",\n\t\t\tUsage: \"Path to Kerberos configuration file\",\n\t\t},\n\t})\n}\n\nfunc fixObjectSize(s uint64) uint64 {\n\tconst min, max = 64 << 10, 16 << 20\n\tvar bits uint\n\tfor s > 1 {\n\t\tbits++\n\t\ts >>= 1\n\t}\n\ts = s << bits\n\tif s < min {\n\t\tlogger.Warnf(\"block size is too small: %s, use %s instead\", humanize.IBytes(s), humanize.IBytes(min))\n\t\ts = min\n\t} else if s > max {\n\t\tlogger.Warnf(\"block size is too large: %s, use %s instead\", humanize.IBytes(s), humanize.IBytes(max))\n\t\ts = max\n\t}\n\treturn s\n}\n\nfunc createStorage(format meta.Format) (object.ObjectStorage, error) {\n\n\tif err := format.Decrypt(); err != nil {\n\t\treturn nil, fmt.Errorf(\"format decrypt: %s\", err)\n\t}\n\tobject.UserAgent = \"JuiceFS-\" + version.Version()\n\tvar blob object.ObjectStorage\n\tvar err error\n\tif u, err := url.Parse(format.Bucket); err == nil {\n\t\tvalues := u.Query()\n\t\tif values.Get(\"tls-insecure-skip-verify\") != \"\" {\n\t\t\tvar tlsSkipVerify bool\n\t\t\tif tlsSkipVerify, err = strconv.ParseBool(values.Get(\"tls-insecure-skip-verify\")); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tobject.GetHttpClient().Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify = tlsSkipVerify\n\t\t\tvalues.Del(\"tls-insecure-skip-verify\")\n\t\t\tu.RawQuery = values.Encode()\n\t\t\tformat.Bucket = u.String()\n\t\t}\n\n\t\t// Configure client TLS when params are provided\n\t\tif values.Get(\"ca-certs\") != \"\" && values.Get(\"ssl-cert\") != \"\" && values.Get(\"ssl-key\") != \"\" {\n\n\t\t\tclientTLSCert, err := tls.LoadX509KeyPair(values.Get(\"ssl-cert\"), values.Get(\"ssl-key\"))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"error loading certificate and key file: %s\", err.Error())\n\t\t\t}\n\n\t\t\tcertPool := x509.NewCertPool()\n\t\t\tcaCertPEM, err := os.ReadFile(values.Get(\"ca-certs\"))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"error loading CA cert file: %s\", err.Error())\n\t\t\t}\n\n\t\t\tif certAdded := certPool.AppendCertsFromPEM(caCertPEM); !certAdded {\n\t\t\t\treturn nil, fmt.Errorf(\"error appending CA cert to pool\")\n\t\t\t}\n\n\t\t\tobject.GetHttpClient().Transport.(*http.Transport).TLSClientConfig.RootCAs = certPool\n\t\t\tobject.GetHttpClient().Transport.(*http.Transport).TLSClientConfig.Certificates = []tls.Certificate{clientTLSCert}\n\t\t}\n\t}\n\n\tif format.Shards > 1 {\n\t\tblob, err = object.NewSharded(strings.ToLower(format.Storage), format.Bucket, format.AccessKey, format.SecretKey, format.SessionToken, format.Shards)\n\t} else {\n\t\tblob, err = object.CreateStorage(strings.ToLower(format.Storage), format.Bucket, format.AccessKey, format.SecretKey, format.SessionToken)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tblob = object.WithPrefix(blob, format.Name+\"/\")\n\tif format.StorageClass != \"\" {\n\t\tif os, ok := blob.(object.SupportStorageClass); ok {\n\t\t\terr := os.SetStorageClass(format.StorageClass)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"set storage class %q: %v\", format.StorageClass, err)\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"Storage class is not supported by %q, will ignore\", format.Storage)\n\t\t}\n\t}\n\tif format.EncryptKey != \"\" {\n\t\tprivKey, err := object.ParsePrivateKeyFromPem([]byte(format.EncryptKey), []byte(os.Getenv(\"JFS_RSA_PASSPHRASE\")))\n\t\tif err != nil {\n\t\t\tif errors.Is(err, object.ErrKeyNeedPasswd) {\n\t\t\t\treturn nil, fmt.Errorf(\"%w: please set the 'JFS_RSA_PASSPHRASE' environment variable\", err)\n\t\t\t}\n\t\t\treturn nil, fmt.Errorf(\"parse private key: %s\", err)\n\t\t}\n\t\tencryptor, err := object.NewDataEncryptor(object.NewKeyEncryptor(privKey), format.EncryptAlgo)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tblob = object.NewEncrypted(blob, encryptor)\n\t}\n\treturn blob, nil\n}\n\nvar letters = []rune(\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\")\n\nfunc randSeq(n int) string {\n\tb := make([]rune, n)\n\tr := rand.New(rand.NewSource(time.Now().UnixNano()))\n\tfor i := range b {\n\t\tb[i] = letters[r.Intn(len(letters))]\n\t}\n\treturn string(b)\n}\n\nfunc doTesting(store object.ObjectStorage, key string, data []byte) error {\n\tctx := context.Background()\n\tif err := store.Put(ctx, key, bytes.NewReader(data)); err != nil {\n\t\tif strings.Contains(strings.ToLower(err.Error()), \"denied\") {\n\t\t\treturn fmt.Errorf(\"Failed to put: %s\", err)\n\t\t}\n\t\tif err2 := store.Create(ctx); err2 != nil {\n\t\t\tif strings.Contains(err.Error(), \"NoSuchBucket\") {\n\t\t\t\treturn fmt.Errorf(\"Failed to create bucket %s: %s, previous error: %s\\nPlease create bucket %s manually, then format again.\",\n\t\t\t\t\tstore, err2, err, store)\n\t\t\t} else {\n\t\t\t\treturn fmt.Errorf(\"Failed to create bucket %s: %s, previous error: %s\",\n\t\t\t\t\tstore, err2, err)\n\t\t\t}\n\t\t}\n\t\tif err := store.Put(ctx, key, bytes.NewReader(data)); err != nil {\n\t\t\treturn fmt.Errorf(\"Failed to put: %s\", err)\n\t\t}\n\t}\n\tp, err := store.Get(ctx, key, 0, -1)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Failed to get: %s\", err)\n\t}\n\tdata2, err := io.ReadAll(p)\n\t_ = p.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !bytes.Equal(data, data2) {\n\t\treturn fmt.Errorf(\"read wrong data: expected %x, got %x\", data, data2)\n\t}\n\terr = store.Delete(ctx, key)\n\tif err != nil {\n\t\t// it's OK to don't have delete permission, but we should warn user explicitly\n\t\tlogger.Warnf(\"Failed to delete, err: %s\", err)\n\t}\n\treturn nil\n}\n\nfunc test(store object.ObjectStorage) error {\n\tkey := \"testing/\" + randSeq(10)\n\tdata := make([]byte, 100)\n\tutils.RandRead(data)\n\tnRetry := 3\n\tvar err error\n\tfor i := 0; i < nRetry; i++ {\n\t\terr = doTesting(store, key, data)\n\t\tif err == nil {\n\t\t\tbreak\n\t\t}\n\t\tlogger.Warnf(\"Test storage %s failed: %s, tries: #%d\", store, err, i+1)\n\t\ttime.Sleep(time.Second * time.Duration(i*3+1))\n\t}\n\tif err == nil {\n\t\t_ = store.Delete(ctx, \"testing/\")\n\t}\n\treturn err\n}\n\nfunc loadEncrypt(keyPath string) string {\n\tif keyPath == \"\" {\n\t\treturn \"\"\n\t}\n\tpem, err := os.ReadFile(keyPath)\n\tif err != nil {\n\t\tlogger.Fatalf(\"load RSA key from %s: %s\", keyPath, err)\n\t}\n\treturn string(pem)\n}\n\nfunc readKerbConf(file string) string {\n\tif file == \"\" {\n\t\treturn \"\"\n\t}\n\tdata, err := os.ReadFile(file)\n\tif err != nil {\n\t\tlogger.Fatalf(\"load Kerberos config from %s: %s\", file, err)\n\t}\n\treturn string(data)\n}\n\nfunc format(c *cli.Context) error {\n\tsetup(c, 2)\n\tremovePassword(c.Args().Get(0))\n\tm := meta.NewClient(c.Args().Get(0), nil)\n\tname := c.Args().Get(1)\n\tvalidName := regexp.MustCompile(`^[a-z0-9][a-z0-9\\-]{1,61}[a-z0-9]$`)\n\tif !validName.MatchString(name) {\n\t\tlogger.Fatalf(\"invalid name: %s, only alphabet, number and - are allowed, and the length should be 3 to 63 characters.\", name)\n\t}\n\tif v := c.String(\"compress\"); compress.NewCompressor(v) == nil {\n\t\tlogger.Fatalf(\"Unsupported compress algorithm: %s\", v)\n\t}\n\tif v := c.Int(\"trash-days\"); v < 0 {\n\t\tlogger.Fatalf(\"Invalid trash days: %d\", v)\n\t}\n\tif v := c.Int(\"shards\"); v > 256 {\n\t\tlogger.Fatalf(\"too many shards: %d\", v)\n\t}\n\n\tvar create, encrypted bool\n\tformat, err := m.Load(false)\n\tif err == nil {\n\t\tif c.Bool(\"no-update\") {\n\t\t\treturn nil\n\t\t}\n\t\tformat.Name = name\n\t\tfor _, flag := range c.LocalFlagNames() {\n\t\t\tswitch flag {\n\t\t\tcase \"capacity\":\n\t\t\t\tformat.Capacity = utils.ParseBytes(c, flag, 'G')\n\t\t\tcase \"inodes\":\n\t\t\t\tformat.Inodes = c.Uint64(flag)\n\t\t\tcase \"bucket\":\n\t\t\t\tformat.Bucket = c.String(flag)\n\t\t\tcase \"access-key\":\n\t\t\t\tformat.AccessKey = c.String(flag)\n\t\t\tcase \"secret-key\":\n\t\t\t\tencrypted = format.KeyEncrypted\n\t\t\t\tif err := format.Decrypt(); err != nil && strings.Contains(err.Error(), \"secret was removed\") {\n\t\t\t\t\tlogger.Warnf(\"decrypt secrets: %s\", err)\n\t\t\t\t}\n\t\t\t\tformat.SecretKey = c.String(flag)\n\t\t\tcase \"session-token\":\n\t\t\t\tencrypted = format.KeyEncrypted\n\t\t\t\tif err := format.Decrypt(); err != nil && strings.Contains(err.Error(), \"secret was removed\") {\n\t\t\t\t\tlogger.Warnf(\"decrypt secrets: %s\", err)\n\t\t\t\t}\n\t\t\t\tformat.SessionToken = c.String(flag)\n\t\t\tcase \"trash-days\":\n\t\t\t\tformat.TrashDays = c.Int(flag)\n\t\t\tcase \"block-size\":\n\t\t\t\tformat.BlockSize = int(fixObjectSize(utils.ParseBytes(c, flag, 'K')) >> 10)\n\t\t\tcase \"compress\":\n\t\t\t\tformat.Compression = c.String(flag)\n\t\t\tcase \"shards\":\n\t\t\t\tformat.Shards = c.Int(flag)\n\t\t\tcase \"hash-prefix\":\n\t\t\t\tformat.HashPrefix = c.Bool(flag)\n\t\t\tcase \"storage\":\n\t\t\t\tformat.Storage = c.String(flag)\n\t\t\tcase \"encrypt-rsa-key\", \"encrypt-algo\":\n\t\t\t\tlogger.Warnf(\"Flag %s is ignored since it cannot be updated\", flag)\n\t\t\tcase \"ranger-rest-url\":\n\t\t\t\tformat.RangerRestUrl = c.String(flag)\n\t\t\tcase \"ranger-service\":\n\t\t\t\tformat.RangerService = c.String(flag)\n\t\t\tcase \"kerberos-config-file\":\n\t\t\t\tformat.KerbConf = readKerbConf(c.String(flag))\n\t\t\t}\n\t\t}\n\t} else if strings.HasPrefix(err.Error(), \"database is not formatted\") {\n\t\tcreate = true\n\t\tformat = &meta.Format{\n\t\t\tName: name,\n\t\t\tUUID: uuid.New().String(),\n\t\t\tStorage: c.String(\"storage\"),\n\t\t\tStorageClass: c.String(\"storage-class\"),\n\t\t\tBucket: c.String(\"bucket\"),\n\t\t\tAccessKey: c.String(\"access-key\"),\n\t\t\tSecretKey: c.String(\"secret-key\"),\n\t\t\tSessionToken: c.String(\"session-token\"),\n\t\t\tEncryptKey: loadEncrypt(c.String(\"encrypt-rsa-key\")),\n\t\t\tEncryptAlgo: c.String(\"encrypt-algo\"),\n\t\t\tShards: c.Int(\"shards\"),\n\t\t\tHashPrefix: c.Bool(\"hash-prefix\"),\n\t\t\tCapacity: utils.ParseBytes(c, \"capacity\", 'G'),\n\t\t\tInodes: c.Uint64(\"inodes\"),\n\t\t\tBlockSize: int(fixObjectSize(utils.ParseBytes(c, \"block-size\", 'K')) >> 10),\n\t\t\tCompression: c.String(\"compress\"),\n\t\t\tTrashDays: c.Int(\"trash-days\"),\n\t\t\tDirStats: true,\n\t\t\tUserGroupQuota: false,\n\t\t\tMetaVersion: meta.MaxVersion,\n\t\t\tMinClientVersion: \"1.1.0-A\",\n\t\t\tEnableACL: c.Bool(\"enable-acl\"),\n\t\t\tRangerRestUrl: c.String(\"ranger-rest-url\"),\n\t\t\tRangerService: c.String(\"ranger-service\"),\n\t\t\tKerbConf: readKerbConf(c.String(\"kerberos-config-file\")),\n\t\t}\n\t\tif format.EnableACL {\n\t\t\tformat.MinClientVersion = \"1.2.0-A\"\n\t\t}\n\t\tif format.RangerRestUrl != \"\" || format.RangerService != \"\" {\n\t\t\tformat.MinClientVersion = \"1.3.0-A\"\n\t\t}\n\t\tif format.KerbConf != \"\" {\n\t\t\tformat.MinClientVersion = \"1.4.0-A\"\n\t\t}\n\n\t\tif format.AccessKey == \"\" && os.Getenv(\"ACCESS_KEY\") != \"\" {\n\t\t\tformat.AccessKey = os.Getenv(\"ACCESS_KEY\")\n\t\t\t_ = os.Unsetenv(\"ACCESS_KEY\")\n\t\t}\n\t\tif format.SecretKey == \"\" && os.Getenv(\"SECRET_KEY\") != \"\" {\n\t\t\tformat.SecretKey = os.Getenv(\"SECRET_KEY\")\n\t\t\t_ = os.Unsetenv(\"SECRET_KEY\")\n\t\t}\n\t\tif format.SessionToken == \"\" && os.Getenv(\"SESSION_TOKEN\") != \"\" {\n\t\t\tformat.SessionToken = os.Getenv(\"SESSION_TOKEN\")\n\t\t\t_ = os.Unsetenv(\"SESSION_TOKEN\")\n\t\t}\n\t} else {\n\t\tlogger.Fatalf(\"Load metadata: %s\", err)\n\t}\n\tif format.Storage == \"file\" || format.Storage == \"sqlite3\" {\n\t\tp, err := filepath.Abs(format.Bucket)\n\t\tif err == nil {\n\t\t\tformat.Bucket = p\n\t\t} else {\n\t\t\tlogger.Fatalf(\"Failed to get absolute path of %s: %s\", format.Bucket, err)\n\t\t}\n\t\tif format.Storage == \"file\" {\n\t\t\tformat.Bucket += \"/\"\n\t\t}\n\t}\n\n\tblob, err := createStorage(*format)\n\tif err != nil {\n\t\tlogger.Fatalf(\"object storage: %s\", err)\n\t}\n\tlogger.Infof(\"Data use %s\", blob)\n\tif os.Getenv(\"JFS_NO_CHECK_OBJECT_STORAGE\") == \"\" {\n\t\tif err := test(blob); err != nil {\n\t\t\tlogger.Fatalf(\"Storage %s is not configured correctly: %s\", blob, err)\n\t\t}\n\t\tif create {\n\t\t\tif objs, err := object.ListAll(c.Context, blob, \"\", \"\", true, false); err == nil {\n\t\t\t\tfor o := range objs {\n\t\t\t\t\tif o == nil {\n\t\t\t\t\t\tlogger.Warnf(\"List storage %s failed\", blob)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t} else if o.IsDir() || o.Size() == 0 {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t} else if o.Key() != \"testing\" && !strings.HasPrefix(o.Key(), \"testing/\") {\n\t\t\t\t\t\tlogger.Fatalf(\"Storage %s is not empty; please clean it up or pick another volume name\", blob)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tlogger.Warnf(\"List storage %s failed: %s\", blob, err)\n\t\t\t}\n\t\t\tif err = blob.Put(ctx, \"juicefs_uuid\", strings.NewReader(format.UUID)); err != nil {\n\t\t\t\tlogger.Warnf(\"Put uuid object: %s\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tif create || encrypted {\n\t\tif err = format.Encrypt(); err != nil {\n\t\t\tlogger.Fatalf(\"Format encrypt: %s\", err)\n\t\t}\n\t}\n\tif err = m.Init(format, c.Bool(\"force\")); err != nil {\n\t\tif create {\n\t\t\t_ = blob.Delete(ctx, \"juicefs_uuid\")\n\t\t}\n\t\tlogger.Fatalf(\"format: %s\", err)\n\t}\n\tlogger.Infof(\"Volume is formatted as %s\", format)\n\treturn nil\n}\n" }, { "path": "cmd/format_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\nfunc TestFixObjectSize(t *testing.T) {\n\tt.Run(\"Should make sure the size is in range\", func(t *testing.T) {\n\t\tcases := []struct {\n\t\t\tinput, expected uint64\n\t\t}{\n\t\t\t{30 << 10, 64 << 10},\n\t\t\t{0, 64 << 10},\n\t\t\t{2 << 40, 16 << 20},\n\t\t\t{16 << 21, 16 << 20},\n\t\t}\n\t\tfor _, c := range cases {\n\t\t\tif size := fixObjectSize(c.input); size != c.expected {\n\t\t\t\tt.Fatalf(\"Expected %d, got %d\", c.expected, size)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"Should use powers of two\", func(t *testing.T) {\n\t\tcases := []struct {\n\t\t\tinput, expected uint64\n\t\t}{\n\t\t\t{150 << 10, 128 << 10},\n\t\t\t{99 << 10, 64 << 10},\n\t\t\t{1077 << 10, 1024 << 10},\n\t\t}\n\t\tfor _, c := range cases {\n\t\t\tif size := fixObjectSize(c.input); size != c.expected {\n\t\t\t\tt.Fatalf(\"Expected %d, got %d\", c.expected, size)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFormat(t *testing.T) {\n\trdb := resetTestMeta()\n\tif err := Main([]string{\"\", \"format\", \"--bucket\", t.TempDir(), testMeta, testVolume}); err != nil {\n\t\tt.Fatalf(\"format error: %s\", err)\n\t}\n\tbody, err := rdb.Get(context.Background(), \"setting\").Bytes()\n\tif err != nil {\n\t\tt.Fatalf(\"get setting: %s\", err)\n\t}\n\tf := meta.Format{}\n\tif err = json.Unmarshal(body, &f); err != nil {\n\t\tt.Fatalf(\"json unmarshal: %s\", err)\n\t}\n\tif f.Name != testVolume {\n\t\tt.Fatalf(\"volume name %s != expected %s\", f.Name, testVolume)\n\t}\n\n\tif err = Main([]string{\"\", \"format\", testMeta, testVolume, \"--capacity\", \"1\", \"--inodes\", \"1000\"}); err != nil {\n\t\tt.Fatalf(\"format error: %s\", err)\n\t}\n\tif body, err = rdb.Get(context.Background(), \"setting\").Bytes(); err != nil {\n\t\tt.Fatalf(\"get setting: %s\", err)\n\t}\n\tif err = json.Unmarshal(body, &f); err != nil {\n\t\tt.Fatalf(\"json unmarshal: %s\", err)\n\t}\n\tif f.Capacity != 1<<30 || f.Inodes != 1000 {\n\t\tt.Fatalf(\"unexpected volume: %+v\", f)\n\t}\n}\n" }, { "path": "cmd/fsck.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdFsck() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"fsck\",\n\t\tAction: fsck,\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"Check consistency of a volume\",\n\t\tArgsUsage: \"META-URL\",\n\t\tDescription: `\nIt scans all objects in data storage and slices in metadata, comparing them to see if there is any\nlost object or broken file.\n\nExamples:\n$ juicefs fsck redis://localhost\n\n# Repair broken directories\n$ juicefs fsck redis://localhost --path /d1/d2 --repair\n\n# recursively check\n$ juicefs fsck redis://localhost --path /d1/d2 --recursive`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"path\",\n\t\t\t\tUsage: \"absolute path within JuiceFS to check\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"repair\",\n\t\t\t\tUsage: \"repair specified path if it's broken\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"recursive\",\n\t\t\t\tAliases: []string{\"r\"},\n\t\t\t\tUsage: \"recursively check or repair\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"sync-dir-stat\",\n\t\t\t\tUsage: \"sync stat of all directories, even if they are existed and not broken (NOTE: it may take a long time for huge trees)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"repair-dir-mode\",\n\t\t\t\tValue: \"0755\",\n\t\t\t\tUsage: \"permission mode for repaired directories (octal, e.g., 0755)\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc fsck(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tif ctx.Bool(\"repair\") && ctx.String(\"path\") == \"\" {\n\t\tlogger.Fatalf(\"Please provide the path to repair with `--path` option\")\n\t}\n\tremovePassword(ctx.Args().Get(0))\n\tm := meta.NewClient(ctx.Args().Get(0), nil)\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"load setting: %s\", err)\n\t}\n\tvar c = meta.NewContext(0, 0, []uint32{0})\n\tprogress := utils.NewProgress(false)\n\t// prepare slices\n\tsliceCSpin := progress.AddCountSpinner(\"Listed slices\")\n\tslices := make(map[meta.Ino][]meta.Slice)\n\tpath := ctx.String(\"path\")\n\trepairDirMode, err := strconv.ParseUint(ctx.String(\"repair-dir-mode\"), 8, 16) // base 8 (octal), 16-bit result\n\tif err != nil {\n\t\tlogger.Fatalf(\"invalid repair-dir-mode: %s\", err)\n\t}\n\tif path != \"\" {\n\t\tif !strings.HasPrefix(path, \"/\") {\n\t\t\tlogger.Fatalf(\"File path should be the absolute path within JuiceFS\")\n\t\t}\n\t\terr := m.Check(c, path, &meta.CheckOpt{\n\t\t\tRepair: ctx.Bool(\"repair\"),\n\t\t\tRecursive: ctx.Bool(\"recursive\"),\n\t\t\tSyncDirStat: ctx.Bool(\"sync-dir-stat\"),\n\t\t\tRepairDirMode: uint16(repairDirMode),\n\t\t\tShowProgress: sliceCSpin.IncrBy,\n\t\t\tSlices: slices,\n\t\t})\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"check: %s\", err)\n\t\t}\n\t} else {\n\t\tr := m.ListSlices(c, slices, false, false, sliceCSpin.Increment)\n\t\tif r != 0 {\n\t\t\tlogger.Fatalf(\"list all slices: %s\", r)\n\t\t}\n\t}\n\tsliceCSpin.Done()\n\n\tchunkConf := *getDefaultChunkConf(format)\n\tchunkConf.CacheDir = \"memory\"\n\n\tblob, err := createStorage(*format)\n\tif err != nil {\n\t\tlogger.Fatalf(\"object storage: %s\", err)\n\t}\n\tlogger.Infof(\"Data use %s\", blob)\n\tblob = object.WithPrefix(blob, \"chunks/\")\n\n\t// Find all blocks in object storage\n\tblockDSpin := progress.AddDoubleSpinner(\"Found blocks\")\n\tvar blocks = make(map[string]int64)\n\tif path == \"\" {\n\t\tobjs, err := object.ListAll(ctx.Context, blob, \"\", \"\", true, false)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"list all blocks: %s\", err)\n\t\t}\n\t\tfor obj := range objs {\n\t\t\tif obj == nil {\n\t\t\t\tbreak // failed listing\n\t\t\t}\n\t\t\tif obj.IsDir() {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tlogger.Debugf(\"found block %s\", obj.Key())\n\t\t\tparts := strings.Split(obj.Key(), \"/\")\n\t\t\tif len(parts) != 3 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tname := parts[2]\n\t\t\tblocks[name] = obj.Size()\n\t\t\tblockDSpin.IncrInt64(obj.Size())\n\t\t}\n\t\tblockDSpin.Done()\n\t}\n\tdelfiles := make(map[meta.Ino]bool)\n\terr = m.ScanDeletedObject(c, nil, nil, nil, func(ino meta.Ino, size uint64, ts int64) (clean bool, err error) {\n\t\tdelfiles[ino] = true\n\t\treturn false, nil\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"scan deleted objects: %s\", err)\n\t}\n\t// Scan all slices to find lost blocks\n\tdelfilesSpin := progress.AddCountSpinner(\"Deleted files\")\n\tskippedSlices := progress.AddCountSpinner(\"Skipped slices\")\n\tsliceCBar := progress.AddCountBar(\"Scanned slices\", sliceCSpin.Current())\n\tsliceBSpin := progress.AddByteSpinner(\"Scanned slices\")\n\tlostDSpin := progress.AddDoubleSpinner(\"Lost blocks\")\n\tbrokens := make(map[meta.Ino]string)\n\tfor inode, ss := range slices {\n\t\tif delfiles[inode] {\n\t\t\tdelfilesSpin.Increment()\n\t\t\tskippedSlices.IncrBy(len(ss))\n\t\t\tcontinue\n\t\t}\n\t\tfor _, s := range ss {\n\t\t\tn := (s.Size - 1) / uint32(chunkConf.BlockSize)\n\t\t\tfor i := uint32(0); i <= n; i++ {\n\t\t\t\tsz := chunkConf.BlockSize\n\t\t\t\tif i == n {\n\t\t\t\t\tsz = int(s.Size) - int(i)*chunkConf.BlockSize\n\t\t\t\t}\n\t\t\t\tkey := fmt.Sprintf(\"%d_%d_%d\", s.Id, i, sz)\n\t\t\t\tif _, ok := blocks[key]; !ok {\n\t\t\t\t\tvar objKey string\n\t\t\t\t\tif format.HashPrefix {\n\t\t\t\t\t\tobjKey = fmt.Sprintf(\"%02X/%v/%s\", s.Id%256, s.Id/1000/1000, key)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tobjKey = fmt.Sprintf(\"%v/%v/%s\", s.Id/1000/1000, s.Id/1000, key)\n\t\t\t\t\t}\n\t\t\t\t\tobj, err := blob.Head(ctx.Context, objKey)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tif _, ok := brokens[inode]; !ok {\n\t\t\t\t\t\t\tif ps := m.GetPaths(meta.Background(), inode); len(ps) > 0 {\n\t\t\t\t\t\t\t\tbrokens[inode] = ps[0]\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tbrokens[inode] = fmt.Sprintf(\"inode:%d\", inode)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tlogger.Errorf(\"can't find block %s for file %s: %s\", objKey, brokens[inode], err)\n\t\t\t\t\t\tlostDSpin.IncrInt64(int64(sz))\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tblockDSpin.IncrInt64(obj.Size())\n\t\t\t\t}\n\t\t\t}\n\t\t\tsliceCBar.Increment()\n\t\t\tsliceBSpin.IncrInt64(int64(s.Size))\n\t\t}\n\t}\n\tprogress.Done()\n\tif progress.Quiet {\n\t\tc, b := blockDSpin.Current()\n\t\tlogger.Infof(\"Found %d blocks (%d bytes)\", c, b)\n\t\tlogger.Infof(\"Used by %d slices (%d bytes)\", sliceCBar.Current(), sliceBSpin.Current())\n\t}\n\tif lc, lb := lostDSpin.Current(); lc > 0 {\n\t\tmsg := fmt.Sprintf(\"%d objects are lost (%d bytes), %d broken files:\\n\", lc, lb, len(brokens))\n\t\tmsg += fmt.Sprintf(\"%13s: PATH\\n\", \"INODE\")\n\t\tvar fileList []string\n\t\tfor i, p := range brokens {\n\t\t\tfileList = append(fileList, fmt.Sprintf(\"%13d: %s\", i, p))\n\t\t}\n\t\tsort.Strings(fileList)\n\t\tmsg += strings.Join(fileList, \"\\n\")\n\t\tlogger.Fatal(msg)\n\t}\n\n\treturn nil\n}\n" }, { "path": "cmd/fsck_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestFsck(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tfor i := 0; i < 10; i++ {\n\t\tfilename := fmt.Sprintf(\"%s/f%d.txt\", testMountPoint, i)\n\t\tif err := os.WriteFile(filename, []byte(\"test\"), 0644); err != nil {\n\t\t\tt.Fatalf(\"write file failed: %s\", err)\n\t\t}\n\t}\n\tif err := Main([]string{\"\", \"fsck\", testMeta}); err != nil {\n\t\tt.Fatalf(\"fsck failed: %s\", err)\n\t}\n\tif err := Main([]string{\"\", \"fsck\", testMeta, \"--path\", \"/f3.txt\"}); err != nil {\n\t\tt.Fatalf(\"fsck failed: %s\", err)\n\t}\n}\n\nfunc TestFsckRepairDirMode(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tif err := os.MkdirAll(testMountPoint+\"/testdir\", 0755); err != nil {\n\t\tt.Fatalf(\"mkdir failed: %s\", err)\n\t}\n\n\tif err := Main([]string{\"\", \"fsck\", testMeta, \"--path\", \"/testdir\", \"--repair-dir-mode\", \"0700\"}); err != nil {\n\t\tt.Fatalf(\"fsck with repair-dir-mode 0700 failed: %s\", err)\n\t}\n\n\tif err := Main([]string{\"\", \"fsck\", testMeta, \"--path\", \"/testdir\", \"--repair-dir-mode\", \"0755\"}); err != nil {\n\t\tt.Fatalf(\"fsck with repair-dir-mode 0755 failed: %s\", err)\n\t}\n}\n" }, { "path": "cmd/gateway.go", "content": "//go:build !nogateway\n// +build !nogateway\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t_ \"net/http/pprof\"\n\t\"os\"\n\t\"os/signal\"\n\t\"path\"\n\t\"strconv\"\n\t\"syscall\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\n\tjfsgateway \"github.com/juicedata/juicefs/pkg/gateway\"\n\t\"github.com/urfave/cli/v2\"\n\n\tmcli \"github.com/minio/cli\"\n\tminio \"github.com/minio/minio/cmd\"\n)\n\nfunc cmdGateway() *cli.Command {\n\tselfFlags := []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"log\",\n\t\t\tUsage: \"path for gateway log\",\n\t\t\tValue: path.Join(getDefaultLogDir(), \"juicefs-gateway.log\"),\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"access-log\",\n\t\t\tUsage: \"path for JuiceFS access log\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"background\",\n\t\t\tAliases: []string{\"d\"},\n\t\t\tUsage: \"run in background\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"no-banner\",\n\t\t\tUsage: \"disable MinIO startup information\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"multi-buckets\",\n\t\t\tUsage: \"use top level of directories as buckets\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"keep-etag\",\n\t\t\tUsage: \"keep the ETag for uploaded objects\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"umask\",\n\t\t\tValue: \"022\",\n\t\t\tUsage: \"umask for new files and directories in octal\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"object-tag\",\n\t\t\tUsage: \"enable object tagging api\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"object-meta\",\n\t\t\tUsage: \"enable object metadata api\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"head-dir\",\n\t\t\tUsage: \"allow HEAD request on directories\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"hide-dir-object\",\n\t\t\tUsage: \"hide the directories created by PUT Object API\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"domain\",\n\t\t\tUsage: \"domain for virtual-host-style requests\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"refresh-iam-interval\",\n\t\t\tValue: \"5m\",\n\t\t\tUsage: \"interval to reload gateway IAM from configuration\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"mountpoint\",\n\t\t\tValue: \"s3gateway\",\n\t\t\tUsage: \"the mount point for current volume (to follow symlink)\",\n\t\t},\n\t}\n\n\treturn &cli.Command{\n\t\tName: \"gateway\",\n\t\tAction: gateway,\n\t\tCategory: \"SERVICE\",\n\t\tUsage: \"Start an S3-compatible gateway\",\n\t\tArgsUsage: \"META-URL ADDRESS\",\n\t\tDescription: `\nIt is implemented based on the MinIO S3 Gateway. Before starting the gateway, you need to set\nMINIO_ROOT_USER and MINIO_ROOT_PASSWORD environment variables, which are the access key and secret\nkey used for accessing S3 APIs.\n\nExamples:\n$ export MINIO_ROOT_USER=admin\n$ export MINIO_ROOT_PASSWORD=12345678\n$ juicefs gateway redis://localhost localhost:9000\n\nDetails: https://juicefs.com/docs/community/s3_gateway`,\n\t\tFlags: expandFlags(selfFlags, clientFlags(0), shareInfoFlags()),\n\t}\n}\n\nfunc gateway(c *cli.Context) error {\n\tsetup(c, 2)\n\tak := os.Getenv(\"MINIO_ROOT_USER\")\n\tif ak == \"\" {\n\t\tak = os.Getenv(\"MINIO_ACCESS_KEY\")\n\t}\n\tif len(ak) < 3 {\n\t\tlogger.Fatalf(\"MINIO_ROOT_USER should be specified as an environment variable with at least 3 characters\")\n\t}\n\tsk := os.Getenv(\"MINIO_ROOT_PASSWORD\")\n\tif sk == \"\" {\n\t\tsk = os.Getenv(\"MINIO_SECRET_KEY\")\n\t}\n\tif len(sk) < 8 {\n\t\tlogger.Fatalf(\"MINIO_ROOT_PASSWORD should be specified as an environment variable with at least 8 characters\")\n\t}\n\tif c.IsSet(\"domain\") {\n\t\tos.Setenv(\"MINIO_DOMAIN\", c.String(\"domain\"))\n\t}\n\n\tif c.IsSet(\"refresh-iam-interval\") {\n\t\tos.Setenv(\"MINIO_REFRESH_IAM_INTERVAL\", c.String(\"refresh-iam-interval\"))\n\t}\n\n\tmetaAddr := c.Args().Get(0)\n\tlistenAddr := c.Args().Get(1)\n\tconf, jfs := initForSvc(c, c.String(\"mountpoint\"), \"s3gateway\", metaAddr, listenAddr)\n\n\tumask, err := strconv.ParseUint(c.String(\"umask\"), 8, 16)\n\tif err != nil {\n\t\tlogger.Fatalf(\"invalid umask %s: %s\", c.String(\"umask\"), err)\n\t}\n\n\treadonly := c.Bool(\"read-only\")\n\tjfsGateway, err = jfsgateway.NewJFSGateway(\n\t\tjfs,\n\t\tconf,\n\t\t&jfsgateway.Config{\n\t\t\tMultiBucket: c.Bool(\"multi-buckets\"),\n\t\t\tKeepEtag: c.Bool(\"keep-etag\"),\n\t\t\tUmask: uint16(umask),\n\t\t\tObjTag: c.Bool(\"object-tag\"),\n\t\t\tObjMeta: c.Bool(\"object-meta\"),\n\t\t\tHeadDir: c.Bool(\"head-dir\"),\n\t\t\tHideDir: c.Bool(\"hide-dir-object\"),\n\t\t\tReadOnly: readonly,\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif readonly {\n\t\tos.Setenv(\"JUICEFS_META_READ_ONLY\", \"1\")\n\t} else {\n\t\tif _, err := jfsGateway.GetBucketInfo(context.Background(), minio.MinioMetaBucket); errors.As(err, &minio.BucketNotFound{}) {\n\t\t\tif err := jfsGateway.MakeBucketWithLocation(context.Background(), minio.MinioMetaBucket, minio.BucketOptions{}); err != nil {\n\t\t\t\tlogger.Fatalf(\"init MinioMetaBucket error %s: %s\", minio.MinioMetaBucket, err)\n\t\t\t}\n\t\t}\n\t}\n\n\targs := []string{\"server\", \"--address\", listenAddr, \"--anonymous\"}\n\tif c.Bool(\"no-banner\") {\n\t\targs = append(args, \"--quiet\")\n\t}\n\tapp := &mcli.App{\n\t\tAction: gateway2,\n\t\tFlags: []mcli.Flag{\n\t\t\tmcli.StringFlag{\n\t\t\t\tName: \"address\",\n\t\t\t\tValue: \":9000\",\n\t\t\t\tUsage: \"bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname\",\n\t\t\t},\n\t\t\tmcli.BoolFlag{\n\t\t\t\tName: \"anonymous\",\n\t\t\t\tUsage: \"hide sensitive information from logging\",\n\t\t\t},\n\t\t\tmcli.BoolFlag{\n\t\t\t\tName: \"json\",\n\t\t\t\tUsage: \"output server logs and startup information in json format\",\n\t\t\t},\n\t\t\tmcli.BoolFlag{\n\t\t\t\tName: \"quiet\",\n\t\t\t\tUsage: \"disable MinIO startup information\",\n\t\t\t},\n\t\t},\n\t}\n\treturn app.Run(args)\n}\n\nvar jfsGateway minio.ObjectLayer\n\nfunc gateway2(ctx *mcli.Context) error {\n\tminio.ServerMainForJFS(ctx, jfsGateway)\n\treturn nil\n}\n\nfunc initForSvc(c *cli.Context, mp string, svcType, metaUrl, listenAddr string) (*vfs.Config, *fs.FileSystem) {\n\tremovePassword(metaUrl)\n\tmetaConf := getMetaConf(c, mp, c.Bool(\"read-only\"))\n\tmetaCli := meta.NewClient(metaUrl, metaConf)\n\tif c.Bool(\"background\") {\n\t\tif err := makeDaemonForSvc(c, metaCli, metaUrl, listenAddr); err != nil {\n\t\t\tlogger.Fatalf(\"make daemon: %s\", err)\n\t\t}\n\t}\n\n\tformat, err := metaCli.Load(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"load setting: %s\", err)\n\t}\n\tif st := metaCli.Chroot(meta.Background(), metaConf.Subdir); st != 0 {\n\t\tlogger.Fatalf(\"Chroot to %s: %s\", metaConf.Subdir, st)\n\t}\n\tregisterer, registry := wrapRegister(c, svcType, format.Name)\n\n\tblob, err := NewReloadableStorage(format, metaCli, updateFormat(c))\n\tif err != nil {\n\t\tlogger.Fatalf(\"object storage: %s\", err)\n\t}\n\tlogger.Infof(\"Data use %s\", blob)\n\n\tchunkConf := getChunkConf(c, format)\n\tstore := chunk.NewCachedStore(blob, *chunkConf, registerer)\n\tregisterMetaMsg(metaCli, store, chunkConf)\n\n\terr = metaCli.NewSession(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"new session: %s\", err)\n\t}\n\tmetaCli.OnReload(func(fmt *meta.Format) {\n\t\tupdateFormat(c)(fmt)\n\t\tstore.UpdateLimit(fmt.UploadLimit, fmt.DownloadLimit)\n\t})\n\n\t// Go will catch all the signals\n\tsignal.Ignore(syscall.SIGPIPE)\n\tsignalChan := make(chan os.Signal, 1)\n\tsignal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)\n\tgo func() {\n\t\tsig := <-signalChan\n\t\tlogger.Infof(\"Received signal %s, exiting...\", sig.String())\n\t\tif err := metaCli.CloseSession(); err != nil {\n\t\t\tlogger.Fatalf(\"close session failed: %s\", err)\n\t\t}\n\t\tobject.Shutdown(blob)\n\t\tos.Exit(0)\n\t}()\n\tvfsConf := getVfsConf(c, metaConf, format, chunkConf)\n\tvfsConf.AccessLog = c.String(\"access-log\")\n\tvfsConf.AttrTimeout = utils.Duration(c.String(\"attr-cache\"))\n\tvfsConf.EntryTimeout = utils.Duration(c.String(\"entry-cache\"))\n\tvfsConf.DirEntryTimeout = utils.Duration(c.String(\"dir-entry-cache\"))\n\tvfsConf.Mountpoint = mp\n\n\tinitBackgroundTasks(c, vfsConf, metaConf, metaCli, blob, registerer, registry)\n\tjfs, err := fs.NewFileSystem(vfsConf, metaCli, store, registry)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Initialize failed: %s\", err)\n\t}\n\tjfs.InitMetrics(registerer)\n\n\treturn vfsConf, jfs\n}\n" }, { "path": "cmd/gateway_noop.go", "content": "//go:build nogateway\n// +build nogateway\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage cmd\n\nimport (\n\t\"errors\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdGateway() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"gateway\",\n\t\tCategory: \"SERVICE\",\n\t\tUsage: \"Start an S3-compatible gateway (not included)\",\n\t\tDescription: `This feature is not included. If you want it, recompile juicefs without \"nogateway\" flag`,\n\t\tAction: func(*cli.Context) error {\n\t\t\treturn errors.New(\"not supported\")\n\t\t},\n\t}\n}\n" }, { "path": "cmd/gc.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdGC() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"gc\",\n\t\tAction: gc,\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"Garbage collector of objects in data storage\",\n\t\tArgsUsage: \"META-URL\",\n\t\tDescription: `\nIt scans all objects in data storage and slices in metadata, comparing them to see if there is any\nleaked object. It can also actively trigger compaction of slices and the cleanup of delayed deleted slices or files.\nUse this command if you find that data storage takes more than expected.\n\nExamples:\n# Check only, no writable change\n$ juicefs gc redis://localhost\n\n# Trigger compaction of all slices\n$ juicefs gc redis://localhost --compact\n\n# Delete leaked objects or metadata and delayed deleted slices or files\n$ juicefs gc redis://localhost --delete`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"compact\",\n\t\t\t\tUsage: \"compact small slices into bigger ones\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"delete\",\n\t\t\t\tUsage: \"delete leaked objects or metadata and delayed deleted slices or files\",\n\t\t\t},\n\t\t\t&cli.IntFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"number threads to delete leaked objects\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc gc(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tremovePassword(ctx.Args().Get(0))\n\tmetaConf := meta.DefaultConf()\n\tmetaConf.MaxDeletes = ctx.Int(\"threads\")\n\tmetaConf.NoBGJob = true\n\tm := meta.NewClient(ctx.Args().Get(0), metaConf)\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"load setting: %s\", err)\n\t}\n\tif err = m.NewSession(false); err == nil { // To sync all stats periodically\n\t\tdefer m.CloseSession() //nolint:errcheck\n\t} else {\n\t\tlogger.Fatalf(\"create session: %v\", err)\n\t}\n\n\tchunkConf := *getDefaultChunkConf(format)\n\tchunkConf.CacheDir = \"memory\"\n\n\tblob, err := createStorage(*format)\n\tif err != nil {\n\t\tlogger.Fatalf(\"object storage: %s\", err)\n\t}\n\tlogger.Infof(\"Data use %s\", blob)\n\tstore := chunk.NewCachedStore(blob, chunkConf, nil)\n\n\t// Scan all chunks first and do compaction if necessary\n\tprogress := utils.NewProgress(false)\n\t// Delete pending slices while listing all slices\n\tdelete := ctx.Bool(\"delete\")\n\tthreads := ctx.Int(\"threads\")\n\tcompact := ctx.Bool(\"compact\")\n\tif (delete || compact) && threads <= 0 {\n\t\tlogger.Fatal(\"threads should be greater than 0 to delete or compact objects\")\n\t}\n\tmaxMtime := time.Now().Add(time.Hour * -1)\n\tstrDuration := os.Getenv(\"JFS_GC_SKIPPEDTIME\")\n\tif strDuration != \"\" {\n\t\tiDuration, err := strconv.Atoi(strDuration)\n\t\tif err == nil {\n\t\t\tmaxMtime = time.Now().Add(time.Second * -1 * time.Duration(iDuration))\n\t\t} else {\n\t\t\tlogger.Errorf(\"parse JFS_GC_SKIPPEDTIME=%s: %s\", strDuration, err)\n\t\t}\n\t}\n\n\tvar wg sync.WaitGroup\n\tvar delSpin *utils.Bar\n\n\tif delete || compact {\n\t\tdelSpin = progress.AddCountSpinner(\"Cleaned pending slices\")\n\t\tm.OnMsg(meta.DeleteSlice, func(args ...interface{}) error {\n\t\t\tdelSpin.Increment()\n\t\t\treturn store.Remove(args[0].(uint64), int(args[1].(uint32)))\n\t\t})\n\t}\n\n\tc := meta.WrapContext(ctx.Context)\n\tdelayedFileSpin := progress.AddDoubleSpinnerTwo(\"Pending deleted files\", \"Pending deleted data\")\n\tcleanedFileSpin := progress.AddDoubleSpinnerTwo(\"Cleaned pending files\", \"Cleaned pending data\")\n\tedge := time.Now().Add(-time.Duration(format.TrashDays) * 24 * time.Hour)\n\tif delete {\n\t\tcleanTrashSpin := progress.AddCountSpinner(\"Cleaned trash\")\n\t\t_ = m.CleanupTrashBefore(c, edge, cleanTrashSpin.IncrBy, nil)\n\t\tcleanTrashSpin.Done()\n\n\t\tcleanDetachedNodeSpin := progress.AddCountSpinner(\"Cleaned detached nodes\")\n\t\tm.CleanupDetachedNodesBefore(c, time.Now().Add(-time.Hour*24), cleanDetachedNodeSpin.Increment)\n\t\tcleanDetachedNodeSpin.Done()\n\t}\n\n\terr = m.ScanDeletedObject(\n\t\tc,\n\t\tnil, nil, nil,\n\t\tfunc(_ meta.Ino, size uint64, ts int64) (bool, error) {\n\t\t\tdelayedFileSpin.IncrInt64(int64(size))\n\t\t\tif delete {\n\t\t\t\tcleanedFileSpin.IncrInt64(int64(size))\n\t\t\t\treturn true, nil\n\t\t\t}\n\t\t\treturn false, nil\n\t\t},\n\t)\n\tif err != nil {\n\t\tlogger.Fatalf(\"scan deleted object: %s\", err)\n\t}\n\tdelayedFileSpin.Done()\n\tcleanedFileSpin.Done()\n\n\tif compact {\n\t\tbar := progress.AddCountBar(\"Compacted chunks\", 0)\n\t\tspin := progress.AddDoubleSpinnerTwo(\"Compacted slices\", \"Compacted data\")\n\t\tm.OnMsg(meta.CompactChunk, func(args ...interface{}) error {\n\t\t\tslices := args[0].([]meta.Slice)\n\t\t\terr := vfs.Compact(chunkConf, store, slices, args[1].(uint64))\n\t\t\tfor _, s := range slices {\n\t\t\t\tspin.IncrInt64(int64(s.Len))\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t\tif st := m.CompactAll(meta.Background(), ctx.Int(\"threads\"), bar); st == 0 {\n\t\t\tif progress.Quiet {\n\t\t\t\tc, b := spin.Current()\n\t\t\t\tlogger.Infof(\"Compacted %d chunks (%d slices, %d bytes).\", bar.Current(), c, b)\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Errorf(\"compact all chunks: %s\", st)\n\t\t}\n\t\tbar.Done()\n\t\tspin.Done()\n\t} else {\n\t\tm.OnMsg(meta.CompactChunk, func(args ...interface{}) error {\n\t\t\treturn nil // ignore compaction\n\t\t})\n\t}\n\n\t// put it above delete count spinner\n\tsliceCSpin := progress.AddCountSpinner(\"Listed slices\")\n\n\t// List all slices in metadata engine\n\tslices := make(map[meta.Ino][]meta.Slice)\n\tr := m.ListSlices(c, slices, true, delete, sliceCSpin.Increment)\n\tif r != 0 {\n\t\tlogger.Fatalf(\"list all slices: %s\", r)\n\t}\n\n\tdelayedSliceSpin := progress.AddDoubleSpinnerTwo(\"Trash slices\", \"Trash data\")\n\tcleanedSliceSpin := progress.AddDoubleSpinnerTwo(\"Cleaned trash slices\", \"Cleaned trash data\")\n\n\terr = m.ScanDeletedObject(\n\t\tc,\n\t\tfunc(ss []meta.Slice, ts int64) (bool, error) {\n\t\t\tfor _, s := range ss {\n\t\t\t\tdelayedSliceSpin.IncrInt64(int64(s.Size))\n\t\t\t\tif delete && ts < edge.Unix() {\n\t\t\t\t\tcleanedSliceSpin.IncrInt64(int64(s.Size))\n\t\t\t\t}\n\t\t\t}\n\t\t\tif delete && ts < edge.Unix() {\n\t\t\t\treturn true, nil\n\t\t\t}\n\t\t\treturn false, nil\n\t\t},\n\t\tnil, nil, nil,\n\t)\n\tif err != nil {\n\t\tlogger.Fatalf(\"statistic: %s\", err)\n\t}\n\tdelayedSliceSpin.Done()\n\tcleanedSliceSpin.Done()\n\n\t// Scan all objects to find leaked ones\n\tblob = object.WithPrefix(blob, \"chunks/\")\n\tobjs, err := object.ListAll(ctx.Context, blob, \"\", \"\", true, false)\n\tif err != nil {\n\t\tlogger.Fatalf(\"list all blocks: %s\", err)\n\t}\n\tvkeys := make(map[uint64]uint32)\n\tpkeys := make(map[uint64]uint32)\n\tckeys := make(map[uint64]uint32)\n\tvar total int64\n\tvar totalBytes uint64\n\tfor _, s := range slices[0] {\n\t\tpkeys[s.Id] = s.Size\n\t\ttotal += int64(int(s.Size-1)/chunkConf.BlockSize) + 1\n\t\ttotalBytes += uint64(s.Size)\n\t}\n\tslices[0] = nil\n\tfor _, s := range slices[1] {\n\t\tckeys[s.Id] = s.Size\n\t\ttotal += int64(int(s.Size-1)/chunkConf.BlockSize) + 1\n\t\ttotalBytes += uint64(s.Size)\n\t}\n\tslices[1] = nil\n\tfor _, ss := range slices {\n\t\tfor _, s := range ss {\n\t\t\tvkeys[s.Id] = s.Size\n\t\t\ttotal += int64(int(s.Size-1)/chunkConf.BlockSize) + 1 // s.Size should be > 0\n\t\t\ttotalBytes += uint64(s.Size)\n\t\t}\n\t}\n\tif progress.Quiet {\n\t\tlogger.Infof(\"using %d slices (%d bytes)\", len(vkeys)+len(ckeys), totalBytes)\n\t}\n\n\tbar := progress.AddCountBar(\"Scanned objects\", total)\n\tvalid := progress.AddDoubleSpinnerTwo(\"Valid objects\", \"Valid data\")\n\tpending := progress.AddDoubleSpinnerTwo(\"Pending delete objects\", \"Pending delete data\")\n\tcompacted := progress.AddDoubleSpinnerTwo(\"Compacted objects\", \"Compacted data\")\n\tleaked := progress.AddDoubleSpinnerTwo(\"Leaked objects\", \"Leaked data\")\n\tskipped := progress.AddDoubleSpinnerTwo(\"Skipped objects\", \"Skipped data\")\n\n\tvar leakedObj = make(chan string, 10240)\n\tfor i := 0; i < threads; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor key := range leakedObj {\n\t\t\t\tif err := blob.Delete(ctx.Context, key); err != nil {\n\t\t\t\t\tlogger.Warnf(\"delete %s: %s\", key, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tfoundLeaked := func(obj object.Object) {\n\t\tbar.IncrTotal(1)\n\t\tleaked.IncrInt64(obj.Size())\n\t\tif delete {\n\t\t\tleakedObj <- obj.Key()\n\t\t}\n\t}\n\n\tfor obj := range objs {\n\t\tif obj == nil {\n\t\t\tbreak // failed listing\n\t\t}\n\t\tif obj.IsDir() {\n\t\t\tcontinue\n\t\t}\n\t\tif obj.Mtime().After(maxMtime) || obj.Mtime().Unix() == 0 {\n\t\t\tlogger.Debugf(\"ignore new block: %s %s\", obj.Key(), obj.Mtime())\n\t\t\tbar.Increment()\n\t\t\tskipped.IncrInt64(obj.Size())\n\t\t\tcontinue\n\t\t}\n\n\t\tlogger.Debugf(\"found block %s\", obj.Key())\n\t\tparts := strings.Split(obj.Key(), \"/\")\n\t\tif len(parts) != 3 {\n\t\t\tcontinue\n\t\t}\n\t\tname := parts[2]\n\t\tparts = strings.Split(name, \"_\")\n\t\tif len(parts) != 3 {\n\t\t\tcontinue\n\t\t}\n\t\tbar.Increment()\n\t\tcid, _ := strconv.Atoi(parts[0])\n\t\tsize := vkeys[uint64(cid)]\n\t\tvar pobj, cobj bool\n\t\tif size == 0 {\n\t\t\tsize, pobj = pkeys[uint64(cid)]\n\t\t}\n\t\tif size == 0 {\n\t\t\tsize, cobj = ckeys[uint64(cid)]\n\t\t}\n\t\tif size == 0 {\n\t\t\tlogger.Debugf(\"find leaked object: %s, size: %d\", obj.Key(), obj.Size())\n\t\t\tfoundLeaked(obj)\n\t\t\tcontinue\n\t\t}\n\t\tindx, _ := strconv.Atoi(parts[1])\n\t\tcsize, _ := strconv.Atoi(parts[2])\n\t\tif csize == chunkConf.BlockSize {\n\t\t\tif (indx+1)*csize > int(size) {\n\t\t\t\tlogger.Warnf(\"size of slice %d is larger than expected: %d > %d\", cid, indx*chunkConf.BlockSize+csize, size)\n\t\t\t\tfoundLeaked(obj)\n\t\t\t} else if pobj {\n\t\t\t\tpending.IncrInt64(obj.Size())\n\t\t\t} else if cobj {\n\t\t\t\tcompacted.IncrInt64(obj.Size())\n\t\t\t} else {\n\t\t\t\tvalid.IncrInt64(obj.Size())\n\t\t\t}\n\t\t} else {\n\t\t\tif indx*chunkConf.BlockSize+csize != int(size) {\n\t\t\t\tlogger.Warnf(\"size of slice %d is %d, but expect %d\", cid, indx*chunkConf.BlockSize+csize, size)\n\t\t\t\tfoundLeaked(obj)\n\t\t\t} else if pobj {\n\t\t\t\tpending.IncrInt64(obj.Size())\n\t\t\t} else if cobj {\n\t\t\t\tcompacted.IncrInt64(obj.Size())\n\t\t\t} else {\n\t\t\t\tvalid.IncrInt64(obj.Size())\n\t\t\t}\n\t\t}\n\t}\n\tm.OnMsg(meta.DeleteSlice, func(args ...interface{}) error {\n\t\treturn errors.New(\"stop deleting slice\")\n\t})\n\tclose(leakedObj)\n\twg.Wait()\n\tif delete || compact {\n\t\tdelSpin.Done()\n\t\tif progress.Quiet {\n\t\t\tlogger.Infof(\"Deleted %d pending slices\", delSpin.Current())\n\t\t}\n\t}\n\tsliceCSpin.Done()\n\tprogress.Done()\n\n\tvc, _ := valid.Current()\n\tpc, pb := pending.Current()\n\tcc, cb := compacted.Current()\n\tlc, lb := leaked.Current()\n\tsc, sb := skipped.Current()\n\tdsc, dsb := cleanedSliceSpin.Current()\n\tfc, fb := cleanedFileSpin.Current()\n\tlogger.Infof(\"scanned %d objects, %d valid, %d pending delete (%d bytes), %d compacted (%d bytes), %d leaked (%d bytes), %d delslices (%d bytes), %d delfiles (%d bytes), %d skipped (%d bytes)\",\n\t\tbar.Current(), vc, pc, pb, cc, cb, lc, lb, dsc, dsb, fc, fb, sc, sb)\n\tif lc > 0 && !delete {\n\t\tlogger.Infof(\"Please add `--delete` to clean leaked objects\")\n\t}\n\treturn nil\n}\n" }, { "path": "cmd/gc_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc writeSmallBlocks(mountDir string) error {\n\tfile, err := os.OpenFile(\n\t\tfilepath.Join(mountDir, \"test.txt\"),\n\t\tos.O_WRONLY|os.O_TRUNC|os.O_CREATE,\n\t\t0666,\n\t)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer file.Close()\n\n\tcontent := []byte(strings.Repeat(\"aaaaaaaabbbbbbbb\", 256))\n\tfor k := 0; k < 64; k++ {\n\t\tif _, err = file.Write(content); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err = file.Sync(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc getFileCount(dir string) int {\n\tfiles, _ := os.ReadDir(dir)\n\tcount := 0\n\tfor _, f := range files {\n\t\tif f.IsDir() {\n\t\t\tcount += getFileCount(filepath.Join(dir, f.Name()))\n\t\t} else {\n\t\t\tcount++\n\t\t}\n\t}\n\n\treturn count\n}\n\nfunc TestGc(t *testing.T) {\n\tvar bucket string\n\tmountTemp(t, &bucket, []string{\"--trash-days=0\", \"--hash-prefix\"}, nil)\n\tdefer umountTemp(t)\n\n\tif err := writeSmallBlocks(testMountPoint); err != nil {\n\t\tt.Fatalf(\"write small blocks failed: %s\", err)\n\t}\n\tdataDir := filepath.Join(bucket, testVolume, \"chunks\")\n\tbeforeCompactFileNum := getFileCount(dataDir)\n\tif err := Main([]string{\"\", \"gc\", \"--compact\", testMeta}); err != nil {\n\t\tt.Fatalf(\"gc compact failed: %s\", err)\n\t}\n\tafterCompactFileNum := getFileCount(dataDir)\n\tif beforeCompactFileNum <= afterCompactFileNum {\n\t\tt.Fatalf(\"blocks before gc compact %d <= after %d\", beforeCompactFileNum, afterCompactFileNum)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tfilename := fmt.Sprintf(\"%s/f%d.txt\", testMountPoint, i)\n\t\tif err := os.WriteFile(filename, []byte(\"test\"), 0644); err != nil {\n\t\t\tt.Fatalf(\"write file failed: %s\", err)\n\t\t}\n\t}\n\n\tos.Setenv(\"JFS_GC_SKIPPEDTIME\", \"0\")\n\tdefer os.Unsetenv(\"JFS_GC_SKIPPEDTIME\")\n\tt.Logf(\"JFS_GC_SKIPPEDTIME is %s\", os.Getenv(\"JFS_GC_SKIPPEDTIME\"))\n\n\tleaked := filepath.Join(dataDir, \"0\", \"0\", \"123456789_0_1048576\")\n\tos.WriteFile(leaked, []byte(strings.Repeat(\"aaaaaaaabbbbbbbb\", 64*1024)), 0644)\n\ttime.Sleep(time.Second * 3)\n\n\tif err := Main([]string{\"\", \"gc\", \"--delete\", testMeta}); err != nil {\n\t\tt.Fatalf(\"gc delete failed: %s\", err)\n\t}\n\n\trequire.False(t, utils.Exists(leaked))\n\n\tif err := Main([]string{\"\", \"gc\", testMeta}); err != nil {\n\t\tt.Fatalf(\"gc failed: %s\", err)\n\t}\n}\n" }, { "path": "cmd/info.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdInfo() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"info\",\n\t\tAction: info,\n\t\tCategory: \"INSPECTOR\",\n\t\tUsage: \"Show internal information of a path or inode\",\n\t\tArgsUsage: \"PATH/INODE\",\n\t\tDescription: `\nIt is used to inspect internal metadata values of the target file.\n\nExamples:\n$ Check a path\n$ juicefs info /mnt/jfs/foo\n\n# Check an inode\n$ cd /mnt/jfs\n$ juicefs info -i 100`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"inode\",\n\t\t\t\tAliases: []string{\"i\"},\n\t\t\t\tUsage: \"use inode instead of path (current dir should be inside JuiceFS)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"recursive\",\n\t\t\t\tAliases: []string{\"r\"},\n\t\t\t\tUsage: \"get summary of directories recursively (NOTE: it may be inaccurate, use --strict to get accurate result)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"strict\",\n\t\t\t\tUsage: \"get accurate summary of directories (NOTE: it may take a long time for huge trees)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"raw\",\n\t\t\t\tUsage: \"show internal raw information\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc info(ctx *cli.Context) error {\n\tsetup0(ctx, 1, 0)\n\tvar recursive, strict, raw uint8\n\tif ctx.Bool(\"recursive\") {\n\t\trecursive = 1\n\t}\n\tif ctx.Bool(\"strict\") {\n\t\tstrict = 1\n\t}\n\tif ctx.Bool(\"raw\") {\n\t\traw = 1\n\t}\n\tfor i := 0; i < ctx.Args().Len(); i++ {\n\t\tprogress := utils.NewProgress(recursive == 0) // only show progress for recursive info\n\t\tpath := ctx.Args().Get(i)\n\t\tdspin := progress.AddDoubleSpinner(path)\n\t\tvar d string\n\t\tvar inode uint64\n\t\tvar err error\n\t\tif ctx.Bool(\"inode\") {\n\t\t\tinode, err = strconv.ParseUint(path, 10, 64)\n\t\t\td, _ = os.Getwd()\n\t\t} else {\n\t\t\td, err = filepath.Abs(path)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Fatalf(\"abs of %s: %s\", path, err)\n\t\t\t}\n\t\t\tinode, err = utils.GetFileInode(d)\n\t\t}\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"lookup inode for %s: %s\", path, err)\n\t\t\tcontinue\n\t\t}\n\t\tif inode < uint64(meta.RootInode) {\n\t\t\tlogger.Fatalf(\"inode number shouldn't be less than %d\", meta.RootInode)\n\t\t}\n\t\tf, err := openController(d)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Open control file for %s: %s\", d, err)\n\t\t\tcontinue\n\t\t}\n\n\t\twb := utils.NewBuffer(8 + 11)\n\t\twb.Put32(meta.InfoV2)\n\t\twb.Put32(11)\n\t\twb.Put64(inode)\n\t\twb.Put8(recursive)\n\t\twb.Put8(raw)\n\t\twb.Put8(strict)\n\t\t_, err = f.Write(wb.Bytes())\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"write message: %s\", err)\n\t\t}\n\t\tdata, errno := readProgress(f, func(count, size uint64) {\n\t\t\tdspin.SetCurrent(int64(count), int64(size))\n\t\t})\n\t\tif errno == syscall.EINVAL {\n\t\t\tlegacyInfo(d, path, inode, recursive, raw)\n\t\t\tcontinue\n\t\t} else if errno != 0 {\n\t\t\tlogger.Errorf(\"failed to get info: %s\", syscall.Errno(errno))\n\t\t}\n\t\tdspin.Done()\n\t\tprogress.Done()\n\n\t\tvar resp vfs.InfoResponse\n\t\terr = json.Unmarshal(data, &resp)\n\t\t_ = f.Close()\n\t\tif err == nil && resp.Failed {\n\t\t\terr = errors.New(resp.Reason)\n\t\t}\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"info: %s\", err)\n\t\t}\n\t\tfmt.Println(path, \":\")\n\t\tfmt.Printf(\" inode: %d\\n\", resp.Ino)\n\t\tfmt.Printf(\" files: %d\\n\", resp.Summary.Files)\n\t\tfmt.Printf(\" dirs: %d\\n\", resp.Summary.Dirs)\n\t\tfmt.Printf(\" length: %s\\n\", utils.FormatBytes(resp.Summary.Length))\n\t\tfmt.Printf(\" size: %s\\n\", utils.FormatBytes(resp.Summary.Size))\n\t\tswitch len(resp.Paths) {\n\t\tcase 0:\n\t\t\tfmt.Printf(\" path: %s\\n\", \"unknown\")\n\t\tcase 1:\n\t\t\tfmt.Printf(\" path: %s\\n\", resp.Paths[0])\n\t\tdefault:\n\t\t\tfmt.Printf(\" paths:\\n\")\n\t\t\tfor _, p := range resp.Paths {\n\t\t\t\tfmt.Printf(\"\\t%s\\n\", p)\n\t\t\t}\n\t\t}\n\t\tif len(resp.Chunks) > 0 {\n\t\t\tfmt.Println(\" chunks:\")\n\t\t\tresults := make([][]string, 0, 1+len(resp.Chunks))\n\t\t\tresults = append(results, []string{\"chunkIndex\", \"sliceId\", \"size\", \"offset\", \"length\"})\n\t\t\tfor _, c := range resp.Chunks {\n\t\t\t\tresults = append(results, []string{\n\t\t\t\t\tstrconv.FormatUint(c.ChunkIndex, 10),\n\t\t\t\t\tstrconv.FormatUint(c.Id, 10),\n\t\t\t\t\tstrconv.FormatUint(uint64(c.Size), 10),\n\t\t\t\t\tstrconv.FormatUint(uint64(c.Off), 10),\n\t\t\t\t\tstrconv.FormatUint(uint64(c.Len), 10),\n\t\t\t\t})\n\t\t\t}\n\t\t\tprintResult(results, -1, false)\n\t\t}\n\t\tif len(resp.Objects) > 0 {\n\t\t\tfmt.Println(\" objects:\")\n\t\t\tresults := make([][]string, 0, 1+len(resp.Objects))\n\t\t\tresults = append(results, []string{\"chunkIndex\", \"objectName\", \"size\", \"offset\", \"length\", \"pos\"})\n\t\t\tvar chunkOffset, lastChunk uint64\n\t\t\tfor _, o := range resp.Objects {\n\t\t\t\tif lastChunk != o.ChunkIndex {\n\t\t\t\t\tchunkOffset = 0\n\t\t\t\t}\n\t\t\t\tlastChunk = o.ChunkIndex\n\t\t\t\tresults = append(results, []string{\n\t\t\t\t\tstrconv.FormatUint(o.ChunkIndex, 10),\n\t\t\t\t\to.Key,\n\t\t\t\t\tstrconv.FormatUint(uint64(o.Size), 10),\n\t\t\t\t\tstrconv.FormatUint(uint64(o.Off), 10),\n\t\t\t\t\tstrconv.FormatUint(uint64(o.Len), 10),\n\t\t\t\t\tstrconv.FormatUint(chunkOffset+o.ChunkIndex*meta.ChunkSize, 10),\n\t\t\t\t})\n\t\t\t\tchunkOffset += uint64(o.Len)\n\t\t\t}\n\t\t\tprintResult(results, 1, false)\n\t\t}\n\t\tif len(resp.FLocks) > 0 {\n\t\t\tfmt.Println(\" flocks:\")\n\t\t\tresults := make([][]string, 0, 1+len(resp.FLocks))\n\t\t\tresults = append(results, []string{\"Sid\", \"Owner\", \"Type\"})\n\t\t\tfor _, l := range resp.FLocks {\n\t\t\t\tresults = append(results, []string{\n\t\t\t\t\tstrconv.FormatUint(l.Sid, 10),\n\t\t\t\t\tstrconv.FormatUint(l.Owner, 10),\n\t\t\t\t\tl.Type,\n\t\t\t\t})\n\t\t\t}\n\t\t\tprintResult(results, 0, false)\n\t\t}\n\t\tif len(resp.PLocks) > 0 {\n\t\t\tfmt.Println(\" plocks:\")\n\t\t\tresults := make([][]string, 0, 1+len(resp.PLocks))\n\t\t\tresults = append(results, []string{\"Sid\", \"Owner\", \"Type\", \"Pid\", \"Start\", \"End\"})\n\t\t\tfor _, l := range resp.PLocks {\n\t\t\t\tresults = append(results, []string{\n\t\t\t\t\tstrconv.FormatUint(l.Sid, 10),\n\t\t\t\t\tstrconv.FormatUint(l.Owner, 10),\n\t\t\t\t\tltypeToString(l.Type),\n\t\t\t\t\tstrconv.FormatUint(uint64(l.Pid), 10),\n\t\t\t\t\tstrconv.FormatUint(l.Start, 10),\n\t\t\t\t\tstrconv.FormatUint(l.End, 10),\n\t\t\t\t})\n\t\t\t}\n\t\t\tprintResult(results, 0, false)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc ltypeToString(t uint32) string {\n\tswitch t {\n\tcase meta.F_RDLCK:\n\t\treturn \"R\"\n\tcase meta.F_WRLCK:\n\t\treturn \"W\"\n\tdefault:\n\t\treturn \"UNKNOWN\"\n\t}\n}\n\nfunc legacyInfo(d, path string, inode uint64, recursive, raw uint8) {\n\tf, err := openController(d)\n\tif err != nil {\n\t\tlogger.Errorf(\"Open control file for %s: %s\", d, err)\n\t\treturn\n\t}\n\tdefer f.Close()\n\twb := utils.NewBuffer(8 + 10)\n\twb.Put32(meta.LegacyInfo)\n\twb.Put32(10)\n\twb.Put64(inode)\n\twb.Put8(recursive)\n\twb.Put8(raw)\n\t_, err = f.Write(wb.Bytes())\n\tif err != nil {\n\t\tlogger.Fatalf(\"write message: %s\", err)\n\t}\n\tdata := make([]byte, 4)\n\tn := readControl(f, data)\n\tif n == 1 && data[0] == byte(syscall.EINVAL&0xff) {\n\t\tlogger.Fatalf(\"info is not supported, please upgrade and mount again\")\n\t}\n\tr := utils.ReadBuffer(data)\n\tsize := r.Get32()\n\tdata = make([]byte, size)\n\tn, err = f.Read(data)\n\tif err != nil {\n\t\tlogger.Fatalf(\"read info: %s\", err)\n\t}\n\tfmt.Println(path, \":\")\n\tresp := string(data[:n])\n\tvar p int\n\tif p = strings.Index(resp, \"chunks:\\n\"); p > 0 {\n\t\tp += 8\n\t\traw = 1 // legacy clients always return chunks\n\t} else if p = strings.Index(resp, \"objects:\\n\"); p > 0 {\n\t\tp += 9\n\t}\n\tif p <= 0 {\n\t\tfmt.Println(resp)\n\t} else {\n\t\tfmt.Println(resp[:p-1])\n\t\tif len(resp[p:]) > 0 {\n\t\t\tlegacyPrintChunks(resp[p:], raw == 1)\n\t\t}\n\t}\n}\n\nfunc legacyPrintChunks(resp string, raw bool) {\n\tcs := strings.Split(resp, \"\\n\")\n\tresult := make([][]string, len(cs))\n\tresult[0] = []string{\"chunkIndex\", \"objectName\", \"size\", \"offset\", \"length\"}\n\tleftAlign := 1\n\tif raw {\n\t\tresult[0][1] = \"sliceId\"\n\t\tleftAlign = -1\n\t}\n\tfor i := 1; i < len(result); i++ {\n\t\tresult[i] = make([]string, 5) // len(result[0])\n\t}\n\n\tfor i, c := range cs[:len(cs)-1] { // remove the last empty string\n\t\tps := strings.Split(c, \"\\t\")[1:] // remove the first empty string\n\t\tfor j, p := range ps {\n\t\t\tif j == 0 {\n\t\t\t\tp = p[:len(p)-1] // remove the last ':'\n\t\t\t}\n\t\t\tresult[i+1][j] = p\n\t\t}\n\t}\n\tprintResult(result, leftAlign, false)\n}\n" }, { "path": "cmd/info_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/agiledragon/gomonkey/v2\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestInfo(t *testing.T) {\n\ttmpFile, err := os.CreateTemp(\"/tmp\", \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create temporary file: %s\", err)\n\t}\n\tdefer tmpFile.Close()\n\tdefer os.Remove(tmpFile.Name())\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\t// mock os.Stdout\n\tpatches := gomonkey.ApplyGlobalVar(os.Stdout, *tmpFile)\n\tdefer patches.Reset()\n\n\tif err = os.MkdirAll(fmt.Sprintf(\"%s/dir1\", testMountPoint), 0777); err != nil {\n\t\tt.Fatalf(\"mkdirAll failed: %s\", err)\n\t}\n\tfor i := 0; i < 10; i++ {\n\t\tfilename := fmt.Sprintf(\"%s/dir1/f%d.txt\", testMountPoint, i)\n\t\tif err = os.WriteFile(filename, []byte(\"test\"), 0644); err != nil {\n\t\t\tt.Fatalf(\"write file failed: %s\", err)\n\t\t}\n\t}\n\n\tif err = Main([]string{\"\", \"info\", fmt.Sprintf(\"%s/dir1\", testMountPoint), \"--strict\"}); err != nil {\n\t\tt.Fatalf(\"info failed: %s\", err)\n\t}\n\tcontent, err := os.ReadFile(tmpFile.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"read file failed: %s\", err)\n\t}\n\treplacer := strings.NewReplacer(\"\\n\", \"\", \" \", \"\")\n\tres := replacer.Replace(string(content))\n\tanswer := fmt.Sprintf(\"%s/dir1: inode: 2 files: 10 dirs: 1 length: 40 Bytes size: 44.00 KiB (45056 Bytes) path: /dir1\", testMountPoint)\n\tanswer = replacer.Replace(answer)\n\trequire.Equal(t, answer, res)\n}\n" }, { "path": "cmd/integration_test.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"os/exec\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/redis/go-redis/v9\"\n)\n\nconst gatewayMeta = \"redis://127.0.0.1:6379/14\"\nconst gatewayVolume = \"gateway-volume\"\nconst gatewayAddr = \"localhost:9008\"\nconst webdavMeta = \"redis://127.0.0.1:6379/15\"\nconst webdavVolume = \"webdav-volume\"\nconst webdavAddr = \"localhost:9009\"\n\nfunc startGateway(t *testing.T) {\n\topt, _ := redis.ParseURL(gatewayMeta)\n\trdb := redis.NewClient(opt)\n\t_ = rdb.FlushDB(context.Background())\n\ttestDir := t.TempDir()\n\tif err := Main([]string{\"\", \"format\", \"--bucket\", testDir, gatewayMeta, gatewayVolume}); err != nil {\n\t\tt.Fatalf(\"format failed: %s\", err)\n\t}\n\n\t// must do reset, otherwise will panic\n\tResetHttp()\n\n\tgo func() {\n\t\tif err := Main([]string{\"\", \"gateway\", gatewayMeta, gatewayAddr, \"--multi-buckets\", \"--keep-etag\", \"--object-tag\", \"--no-usage-report\"}); err != nil {\n\t\t\tt.Errorf(\"gateway failed: %s\", err)\n\t\t}\n\t}()\n\ttime.Sleep(2 * time.Second)\n}\n\nfunc startWebdav(t *testing.T) {\n\topt, _ := redis.ParseURL(webdavMeta)\n\trdb := redis.NewClient(opt)\n\t_ = rdb.FlushDB(context.Background())\n\ttestDir := t.TempDir()\n\tif err := Main([]string{\"\", \"format\", \"--bucket\", testDir, webdavMeta, webdavVolume}); err != nil {\n\t\tt.Fatalf(\"format failed: %s\", err)\n\t}\n\n\t// must do reset, otherwise will panic\n\tResetHttp()\n\n\tgo func() {\n\t\tos.Setenv(\"WEBDAV_USER\", \"root\")\n\t\tos.Setenv(\"WEBDAV_PASSWORD\", \"1234\")\n\t\tif err := Main([]string{\"\", \"webdav\", webdavMeta, webdavAddr, \"--no-usage-report\"}); err != nil {\n\t\t\tt.Errorf(\"gateway failed: %s\", err)\n\t\t}\n\t}()\n\ttime.Sleep(2 * time.Second)\n}\n\nfunc TestIntegration(t *testing.T) {\n\tmountTemp(t, nil, nil, []string{\"--enable-ioctl\"})\n\tdefer umountTemp(t)\n\tstartGateway(t)\n\tstartWebdav(t)\n\t_ = os.Chdir(\"../integration\")\n\tmakeCmd := exec.Command(\"make\")\n\tout, err := makeCmd.CombinedOutput()\n\tif err != nil {\n\t\tt.Logf(\"std out:\\n%s\\n\", string(out))\n\t\tt.Fatalf(\"std err failed with %s\\n\", err)\n\t} else {\n\t\tt.Logf(\"std out:\\n%s\\n\", string(out))\n\t}\n}\n" }, { "path": "cmd/load.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"compress/gzip\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/DataDog/zstd\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdLoad() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"load\",\n\t\tAction: load,\n\t\tCategory: \"ADMIN\",\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"encrypt-rsa-key\",\n\t\t\t\tUsage: \"a path to RSA private key (PEM)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"encrypt-algo\",\n\t\t\t\tUsage: \"encrypt algorithm (aes256gcm-rsa, chacha20-rsa)\",\n\t\t\t\tValue: object.AES256GCM_RSA,\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"binary\",\n\t\t\t\tUsage: \"load metadata from a binary file (different from original JSON format)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"stat\",\n\t\t\t\tUsage: \"show statistics of the metadata binary file\",\n\t\t\t},\n\t\t\t&cli.Int64Flag{\n\t\t\t\tName: \"offset\",\n\t\t\t\tUsage: \"offset of binary backup's segment (works with --stat and --binary). Use -1 to show all offsets, or specify one for details\",\n\t\t\t},\n\t\t\t&cli.IntFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"number of threads to load binary metadata, only works with --binary\",\n\t\t\t},\n\t\t},\n\t\tUsage: \"Load metadata from a previously dumped file\",\n\t\tArgsUsage: \"META-URL [FILE]\",\n\t\tDescription: `\nLoad metadata into an empty metadata engine or show statistics of the backup file.\n\nWARNING: Do NOT use new engine and the old one at the same time, otherwise it will probably break\nconsistency of the volume.\n\nExamples:\n$ juicefs load redis://localhost/1 meta-dump.json.gz\n$ juicefs load redis://localhost/1 meta-dump.bin --binary --threads 10\n$ juicefs load meta-dump.bin --binary --stat\n\nDetails: https://juicefs.com/docs/community/metadata_dump_load`,\n\t}\n}\n\ntype reader struct {\n\tencryptR io.ReadCloser\n\tcompressR io.ReadCloser\n}\n\nfunc (r *reader) Read(p []byte) (n int, err error) {\n\treturn r.compressR.Read(p)\n}\n\nfunc (r *reader) Close() error {\n\tif err := r.compressR.Close(); err != nil {\n\t\treturn err\n\t}\n\tif r.encryptR != r.compressR {\n\t\treturn r.encryptR.Close()\n\t}\n\treturn nil\n}\n\nfunc open(src string, key string, algo string) (io.ReadCloser, error) {\n\tvar r io.ReadCloser\n\tvar ioErr error\n\tvar fp io.ReadCloser\n\tif key != \"\" {\n\t\tprivKey, err := object.ParsePrivateKeyFromPem([]byte(loadEncrypt(key)), []byte(os.Getenv(\"JFS_RSA_PASSPHRASE\")))\n\t\tif err != nil {\n\t\t\tif errors.Is(err, object.ErrKeyNeedPasswd) {\n\t\t\t\treturn nil, fmt.Errorf(\"%w: please set the 'JFS_RSA_PASSPHRASE' environment variable\", err)\n\t\t\t}\n\t\t\treturn nil, fmt.Errorf(\"parse private key: %s\", err)\n\t\t}\n\t\tencryptor, err := object.NewDataEncryptor(object.NewKeyEncryptor(privKey), algo)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif _, err := os.Stat(src); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to stat %s: %s\", src, err)\n\t\t}\n\t\tvar srcAbsPath string\n\t\tsrcAbsPath, err = filepath.Abs(src)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to get absolute path of %s: %s\", src, err)\n\t\t}\n\t\tfileBlob, err := object.CreateStorage(\"file\", strings.TrimSuffix(src, filepath.Base(srcAbsPath)), \"\", \"\", \"\")\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tblob := object.NewEncrypted(fileBlob, encryptor)\n\t\tfp, ioErr = blob.Get(context.Background(), filepath.Base(srcAbsPath), 0, -1)\n\t} else {\n\t\tfp, ioErr = os.Open(src)\n\t}\n\tif ioErr != nil {\n\t\treturn nil, ioErr\n\t}\n\tif strings.HasSuffix(src, \".gz\") {\n\t\tvar err error\n\t\tr, err = gzip.NewReader(fp)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if strings.HasSuffix(src, \".zstd\") {\n\t\tr = zstd.NewReader(fp)\n\t} else {\n\t\tr = fp\n\t}\n\treturn &reader{compressR: r, encryptR: fp}, nil\n}\n\nfunc convert(path string, key, algo string) (string, error) {\n\tisCompress := false\n\tif strings.HasSuffix(path, \".gz\") || strings.HasSuffix(path, \".zstd\") {\n\t\tisCompress = true\n\t}\n\n\tif key == \"\" && !isCompress {\n\t\treturn path, nil\n\t}\n\n\tnPath := path[:strings.LastIndex(path, \".\")]\n\tif utils.Exists(nPath) {\n\t\tlogger.Infof(\"plain backup %s already exists, skip conversion\", nPath)\n\t\treturn nPath, nil\n\t}\n\n\tr, err := open(path, key, algo)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer r.Close()\n\n\tw, err := os.Create(nPath)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to create plain backup %s: %w\", nPath, err)\n\t}\n\tdefer w.Close()\n\n\tif _, err = io.Copy(w, r); err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to convert %s to %s: %w\", path, nPath, err)\n\t}\n\tlogger.Infof(\"converted backup %s to %s\", path, nPath)\n\treturn nPath, nil\n}\n\nfunc load(ctx *cli.Context) error {\n\tsetup0(ctx, 1, 2)\n\n\tkey, algo := ctx.String(\"encrypt-rsa-key\"), ctx.String(\"encrypt-algo\")\n\tsrc := ctx.Args().Get(1)\n\tvar err error\n\tif ctx.Bool(\"binary\") {\n\t\tif ctx.Bool(\"stat\") {\n\t\t\tsrc = ctx.Args().Get(0)\n\t\t}\n\t\tif src, err = convert(src, key, algo); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif ctx.Bool(\"stat\") {\n\t\t\treturn statBak(ctx, src)\n\t\t}\n\t}\n\n\tmetaUri := ctx.Args().Get(0)\n\tremovePassword(metaUri)\n\tvar r io.ReadCloser\n\tif ctx.Args().Len() == 1 {\n\t\tr = os.Stdin\n\t\tsrc = \"STDIN\"\n\t} else {\n\t\tr, err = open(src, key, algo)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer r.Close()\n\t}\n\n\tm := meta.NewClient(metaUri, nil)\n\tif format, err := m.Load(false); err == nil {\n\t\treturn fmt.Errorf(\"database %s is used by volume %s\", utils.RemovePassword(metaUri), format.Name)\n\t}\n\n\tif ctx.Bool(\"binary\") {\n\t\tprogress := utils.NewProgress(false)\n\t\tbars := make(map[string]*utils.Bar)\n\t\tfor _, name := range meta.SegType2Name {\n\t\t\tbars[name] = progress.AddCountSpinner(name)\n\t\t}\n\n\t\topt := &meta.LoadOption{\n\t\t\tThreads: ctx.Int(\"threads\"),\n\t\t\tProgress: func(name string, cnt int) {\n\t\t\t\tbars[name].IncrBy(cnt)\n\t\t\t},\n\t\t}\n\t\tif err := m.LoadMetaV2(meta.WrapContext(ctx.Context), r, opt); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tprogress.Done()\n\t} else {\n\t\tif err := m.LoadMeta(r); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif format, err := m.Load(true); err == nil {\n\t\tif format.SecretKey == \"removed\" {\n\t\t\tlogger.Warnf(\"secret key was removed; please correct it with `config` command\")\n\t\t}\n\t} else {\n\t\treturn err\n\t}\n\tlogger.Infof(\"load metadata from %s succeed\", src)\n\treturn nil\n}\n\nfunc statBak(ctx *cli.Context, path string) error {\n\tlogger.Infof(\"load backup from %s\", path)\n\tfp, err := os.Open(path)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to open file %s: %w\", path, err)\n\t}\n\tdefer fp.Close()\n\n\tif !ctx.IsSet(\"offset\") {\n\t\treturn showBakSummary(ctx, fp, false)\n\t}\n\n\toffset := ctx.Int64(\"offset\")\n\tif offset == -1 {\n\t\treturn showBakSummary(ctx, fp, true)\n\t}\n\n\treturn showBakDetail(ctx, fp, offset)\n}\n\nfunc showBakSummary(ctx *cli.Context, fp *os.File, withOffset bool) error {\n\tbak := &meta.BakFormat{}\n\tfooter, err := bak.ReadFooter(fp)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to read footer: %w\", err)\n\t}\n\n\tfmt.Printf(\"Backup Version: %d\\n\", footer.Msg.Version)\n\tdata := make([][]string, 0, len(footer.Msg.Infos))\n\tfor name, info := range footer.Msg.Infos {\n\t\tif withOffset {\n\t\t\tdata = append(data, []string{name, fmt.Sprintf(\"%d\", info.Num), fmt.Sprintf(\"%d\", info.Offset)})\n\t\t} else {\n\t\t\tdata = append(data, []string{name, fmt.Sprintf(\"%d\", info.Num)})\n\t\t}\n\t}\n\tsort.Slice(data, func(i, j int) bool {\n\t\treturn data[i][0] < data[j][0]\n\t})\n\n\tif withOffset {\n\t\tfmt.Println(strings.Repeat(\"-\", 34))\n\t\tfmt.Printf(\"%-10s| %-10s| %-10s\\n\", \"Name\", \"Num\", \"Offset\")\n\t\tfmt.Println(strings.Repeat(\"-\", 34))\n\t} else {\n\t\tfmt.Println(strings.Repeat(\"-\", 23))\n\t\tfmt.Printf(\"%-10s| %-10s\\n\", \"Name\", \"Num\")\n\t\tfmt.Println(strings.Repeat(\"-\", 23))\n\t}\n\tfor _, v := range data {\n\t\tfmt.Printf(\"%-10s| %-10s|\", v[0], v[1])\n\t\tif withOffset {\n\t\t\tfmt.Printf(\" %-10s\", v[2])\n\t\t}\n\t\tfmt.Println()\n\t}\n\treturn nil\n}\n\nfunc showBakDetail(ctx *cli.Context, fp *os.File, offset int64) error {\n\tbak := &meta.BakFormat{}\n\tif _, err := fp.Seek(offset, io.SeekStart); err != nil {\n\t\treturn err\n\t}\n\n\tseg, err := bak.ReadSegment(fp)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to read segment: %w\", err)\n\t}\n\n\tfmt.Printf(\"Segment: %s\\n\", seg.Name())\n\tfmt.Printf(\"Value: %s\\n\", seg)\n\treturn nil\n}\n" }, { "path": "cmd/main.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t_ \"net/http/pprof\"\n\t\"os\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/grafana/pyroscope-go\"\n\t_ \"github.com/grafana/pyroscope-go/godeltaprof/http/pprof\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/urfave/cli/v2\"\n\t\"go.uber.org/automaxprocs/maxprocs\"\n)\n\nvar logger = utils.GetLogger(\"juicefs\")\nvar debugAgent string\nvar debugAgentOnce sync.Once\n\nfunc Main(args []string) error {\n\t// we have to call this because gspt removes all arguments\n\tutils.SetProcTitle(os.Args)\n\tcli.VersionFlag = &cli.BoolFlag{\n\t\tName: \"version\", Aliases: []string{\"V\"},\n\t\tUsage: \"print version only\",\n\t}\n\tapp := &cli.App{\n\t\tName: \"juicefs\",\n\t\tUsage: \"A POSIX file system built on Redis and object storage.\",\n\t\tVersion: version.Version(),\n\t\tCopyright: \"Apache License 2.0\",\n\t\tHideHelpCommand: true,\n\t\tEnableBashCompletion: true,\n\t\tFlags: globalFlags(),\n\t\tCommands: []*cli.Command{\n\t\t\tcmdFormat(),\n\t\t\tcmdConfig(),\n\t\t\tcmdQuota(),\n\t\t\tcmdDestroy(),\n\t\t\tcmdGC(),\n\t\t\tcmdFsck(),\n\t\t\tcmdRestore(),\n\t\t\tcmdDump(),\n\t\t\tcmdLoad(),\n\t\t\tcmdVersion(),\n\t\t\tcmdStatus(),\n\t\t\tcmdStats(),\n\t\t\tcmdProfile(),\n\t\t\tcmdInfo(),\n\t\t\tcmdMount(),\n\t\t\tcmdUmount(),\n\t\t\tcmdGateway(),\n\t\t\tcmdWebDav(),\n\t\t\tcmdBench(),\n\t\t\tcmdObjbench(),\n\t\t\tcmdMdtest(),\n\t\t\tcmdWarmup(),\n\t\t\tcmdRmr(),\n\t\t\tcmdSync(),\n\t\t\tcmdDebug(),\n\t\t\tcmdClone(),\n\t\t\tcmdSummary(),\n\t\t\tcmdCompact(),\n\t\t},\n\t}\n\n\tif runtime.GOOS == \"windows\" {\n\t\tapp.Commands = append(app.Commands, cmdPrintSID())\n\t}\n\n\tif calledViaMount(args) {\n\t\tvar err error\n\t\targs, err = handleSysMountArgs(args)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(args) < 1 {\n\t\t\targs = []string{\"mount\", \"--help\"}\n\t\t}\n\t}\n\terr := app.Run(reorderOptions(app, args))\n\tif errno, ok := err.(syscall.Errno); ok && errno == 0 {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc calledViaMount(args []string) bool {\n\tif os.Getenv(\"CALL_VIA_MOUNT\") != \"\" {\n\t\treturn true\n\t}\n\tif strings.HasSuffix(args[0], \"/mount.juicefs\") {\n\t\tos.Setenv(\"CALL_VIA_MOUNT\", \"1\")\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc handleSysMountArgs(args []string) ([]string, error) {\n\toptionToCmdFlag := map[string]string{\n\t\t\"attrcacheto\": \"attr-cache\",\n\t\t\"entrycacheto\": \"entry-cache\",\n\t\t\"direntrycacheto\": \"dir-entry-cache\",\n\t}\n\tnewArgs := []string{\"juicefs\", \"mount\", \"-d\"}\n\tif len(args) < 3 {\n\t\treturn nil, nil\n\t}\n\tmountOptions := args[3:]\n\tsysOptions := []string{\"_netdev\", \"nofail\", \"rw\", \"defaults\", \"remount\"}\n\tfuseOptions := make([]string, 0, 20)\n\tcmdFlagsLookup := make(map[string]bool, 20)\n\tfor _, f := range append(cmdMount().Flags, globalFlags()...) {\n\t\tfor _, name := range f.Names() {\n\t\t\tif len(name) > 1 {\n\t\t\t\t_, cmdFlagsLookup[name] = f.(*cli.BoolFlag)\n\t\t\t}\n\t\t}\n\t}\n\n\tparseFlag := false\n\tfor _, option := range mountOptions {\n\t\tif option == \"-o\" {\n\t\t\tparseFlag = true\n\t\t\tcontinue\n\t\t}\n\t\tif !parseFlag {\n\t\t\tcontinue\n\t\t}\n\n\t\topts := strings.Split(option, \",\")\n\t\tfor _, opt := range opts {\n\t\t\topt = strings.TrimSpace(opt)\n\t\t\tif opt == \"\" || opt == \"background\" || utils.StringContains(sysOptions, opt) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Lower case option name is preferred, but if it's the same as flag name, we also accept it\n\t\t\tif strings.Contains(opt, \"=\") {\n\t\t\t\tfields := strings.SplitN(opt, \"=\", 2)\n\t\t\t\tif flagName, ok := optionToCmdFlag[fields[0]]; ok {\n\t\t\t\t\tnewArgs = append(newArgs, fmt.Sprintf(\"--%s=%s\", flagName, fields[1]))\n\t\t\t\t} else if _, ok := cmdFlagsLookup[fields[0]]; ok {\n\t\t\t\t\tnewArgs = append(newArgs, fmt.Sprintf(\"--%s=%s\", fields[0], fields[1]))\n\t\t\t\t} else {\n\t\t\t\t\tfuseOptions = append(fuseOptions, opt)\n\t\t\t\t}\n\t\t\t} else if flagName, ok := optionToCmdFlag[opt]; ok {\n\t\t\t\tnewArgs = append(newArgs, fmt.Sprintf(\"--%s\", flagName))\n\t\t\t} else if isBool, ok := cmdFlagsLookup[opt]; ok {\n\t\t\t\tif !isBool {\n\t\t\t\t\treturn nil, fmt.Errorf(\"option %s requires a value\", opt)\n\t\t\t\t}\n\t\t\t\tnewArgs = append(newArgs, fmt.Sprintf(\"--%s\", opt))\n\t\t\t\tif opt == \"debug\" {\n\t\t\t\t\tfuseOptions = append(fuseOptions, opt)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfuseOptions = append(fuseOptions, opt)\n\t\t\t}\n\t\t}\n\n\t\tparseFlag = false\n\t}\n\tif len(fuseOptions) > 0 {\n\t\tnewArgs = append(newArgs, \"-o\", strings.Join(fuseOptions, \",\"))\n\t}\n\tnewArgs = append(newArgs, args[1], args[2])\n\tlogger.Debug(\"Parsed mount args: \", strings.Join(newArgs, \" \"))\n\treturn newArgs, nil\n}\n\nfunc isFlag(flags []cli.Flag, option string) (bool, bool) {\n\tif !strings.HasPrefix(option, \"-\") {\n\t\treturn false, false\n\t}\n\t// --V or -v work the same\n\toption = strings.TrimLeft(option, \"-\")\n\tfor _, flag := range flags {\n\t\t_, isBool := flag.(*cli.BoolFlag)\n\t\tfor _, name := range flag.Names() {\n\t\t\tif option == name || strings.HasPrefix(option, name+\"=\") {\n\t\t\t\treturn true, !isBool && !strings.Contains(option, \"=\")\n\t\t\t}\n\t\t}\n\t}\n\treturn false, false\n}\n\nfunc reorderOptions(app *cli.App, args []string) []string {\n\tvar newArgs = []string{args[0]}\n\tvar others []string\n\tglobalFlags := append(app.Flags, cli.VersionFlag)\n\tfor i := 1; i < len(args); i++ {\n\t\toption := args[i]\n\t\tif ok, hasValue := isFlag(globalFlags, option); ok {\n\t\t\tnewArgs = append(newArgs, option)\n\t\t\tif hasValue {\n\t\t\t\ti++\n\t\t\t\tif i >= len(args) {\n\t\t\t\t\tlogger.Fatalf(\"option %s requires value\", option)\n\t\t\t\t}\n\t\t\t\tnewArgs = append(newArgs, args[i])\n\t\t\t}\n\t\t} else {\n\t\t\tothers = append(others, option)\n\t\t}\n\t}\n\t// no command\n\tif len(others) == 0 {\n\t\treturn newArgs\n\t}\n\tcmdName := others[0]\n\tvar cmd *cli.Command\n\tfor _, c := range app.Commands {\n\t\tif c.Name == cmdName {\n\t\t\tcmd = c\n\t\t\tbreak\n\t\t}\n\t}\n\tif cmd == nil {\n\t\t// can't recognize the command, skip it\n\t\treturn append(newArgs, others...)\n\t}\n\n\tnewArgs = append(newArgs, cmdName)\n\targs, others = others[1:], nil\n\t// -h is valid for all the commands\n\tcmdFlags := append(cmd.Flags, cli.HelpFlag)\n\tfor i := 0; i < len(args); i++ {\n\t\toption := args[i]\n\t\tif ok, hasValue := isFlag(cmdFlags, option); ok {\n\t\t\tnewArgs = append(newArgs, option)\n\t\t\tif hasValue && len(args[i+1:]) > 0 {\n\t\t\t\ti++\n\t\t\t\tnewArgs = append(newArgs, args[i])\n\t\t\t}\n\t\t} else {\n\t\t\tif strings.HasPrefix(option, \"-\") && !utils.StringContains(args, \"--generate-bash-completion\") {\n\t\t\t\tlogger.Fatalf(\"unknown option: %s\", option)\n\t\t\t}\n\t\t\tothers = append(others, option)\n\t\t}\n\t}\n\treturn append(newArgs, others...)\n}\n\n// Check number of positional arguments, set logger level and setup agent if needed\nfunc setup(c *cli.Context, n int) {\n\tsetup0(c, n, n)\n}\n\nfunc setup0(c *cli.Context, min, max int) {\n\tif c.NArg() < min {\n\t\tfmt.Printf(\"ERROR: This command requires at least %d arguments\\n\", min)\n\t\tfmt.Printf(\"USAGE:\\n juicefs %s [command options] %s\\n\", c.Command.Name, c.Command.ArgsUsage)\n\t\tos.Exit(1)\n\t} else if max > 0 && c.NArg() > max {\n\t\tfmt.Printf(\"ERROR: This command accept at most %d arguments but got %+v\\n\", max, c.Args().Slice())\n\t\tfmt.Printf(\"USAGE:\\n juicefs %s [command options] %s\\n\", c.Command.Name, c.Command.ArgsUsage)\n\t\tlogger.Exit(1)\n\t}\n\n\tswitch c.String(\"log-level\") {\n\tcase \"trace\":\n\t\tutils.SetLogLevel(logrus.TraceLevel)\n\tcase \"debug\":\n\t\tutils.SetLogLevel(logrus.DebugLevel)\n\tcase \"info\":\n\t\tutils.SetLogLevel(logrus.InfoLevel)\n\tcase \"warn\":\n\t\tutils.SetLogLevel(logrus.WarnLevel)\n\tcase \"error\":\n\t\tutils.SetLogLevel(logrus.ErrorLevel)\n\tcase \"fatal\":\n\t\tutils.SetLogLevel(logrus.FatalLevel)\n\tcase \"panic\":\n\t\tutils.SetLogLevel(logrus.PanicLevel)\n\tdefault:\n\t\tif c.Bool(\"trace\") {\n\t\t\tutils.SetLogLevel(logrus.TraceLevel)\n\t\t} else if c.Bool(\"verbose\") {\n\t\t\tutils.SetLogLevel(logrus.DebugLevel)\n\t\t} else if c.Bool(\"quiet\") {\n\t\t\tutils.SetLogLevel(logrus.WarnLevel)\n\t\t} else {\n\t\t\tutils.SetLogLevel(logrus.InfoLevel)\n\t\t}\n\t}\n\tif c.Bool(\"no-color\") {\n\t\tutils.DisableLogColor()\n\t}\n\t// set the correct value when it runs inside container\n\tif undo, err := maxprocs.Set(maxprocs.Logger(logger.Debugf)); err != nil {\n\t\tundo()\n\t}\n\n\tlogID := c.String(\"log-id\")\n\tif logID != \"\" {\n\t\tif logID == \"random\" {\n\t\t\tlogID = uuid.New().String()\n\t\t}\n\t\tutils.SetLogID(\"[\" + logID + \"] \")\n\t}\n\n\tif !c.Bool(\"no-agent\") {\n\t\tgo debugAgentOnce.Do(func() {\n\t\t\tfor port := 6060; port < 6100; port++ {\n\t\t\t\tdebugAgent = fmt.Sprintf(\"127.0.0.1:%d\", port)\n\t\t\t\tlogger.Debugf(\"Debug agent listening on %s\", debugAgent)\n\t\t\t\t_ = http.ListenAndServe(debugAgent, nil)\n\t\t\t}\n\t\t})\n\t}\n\n\tif c.IsSet(\"pyroscope\") {\n\t\ttags := make(map[string]string)\n\t\tappName := fmt.Sprintf(\"juicefs.%s\", c.Command.Name)\n\t\tif c.Command.Name == \"mount\" {\n\t\t\ttags[\"mountpoint\"] = c.Args().Get(1)\n\t\t}\n\t\tif hostname, err := os.Hostname(); err == nil {\n\t\t\ttags[\"hostname\"] = hostname\n\t\t}\n\t\ttags[\"pid\"] = strconv.Itoa(os.Getpid())\n\t\ttags[\"version\"] = version.Version()\n\n\t\ttypes := []pyroscope.ProfileType{pyroscope.ProfileCPU, pyroscope.ProfileInuseObjects, pyroscope.ProfileAllocObjects,\n\t\t\tpyroscope.ProfileInuseSpace, pyroscope.ProfileAllocSpace, pyroscope.ProfileGoroutines, pyroscope.ProfileMutexCount,\n\t\t\tpyroscope.ProfileMutexDuration, pyroscope.ProfileBlockCount, pyroscope.ProfileBlockDuration}\n\t\tif _, err := pyroscope.Start(pyroscope.Config{\n\t\t\tApplicationName: appName,\n\t\t\tServerAddress: c.String(\"pyroscope\"),\n\t\t\tLogger: logger,\n\t\t\tTags: tags,\n\t\t\tAuthToken: os.Getenv(\"PYROSCOPE_AUTH_TOKEN\"),\n\t\t\tProfileTypes: types,\n\t\t}); err != nil {\n\t\t\tlogger.Errorf(\"start pyroscope agent: %v\", err)\n\t\t}\n\t}\n}\n\nfunc removePassword(uris ...string) {\n\targs := make([]string, len(os.Args))\n\tcopy(args, os.Args)\n\tvar idx int\n\tfor _, uri := range uris {\n\t\turi2 := utils.RemovePassword(uri)\n\t\tif uri2 != uri {\n\t\t\tfor i := idx; i < len(os.Args); i++ {\n\t\t\t\tif os.Args[i] == uri {\n\t\t\t\t\targs[i] = uri2\n\t\t\t\t\tidx = i + 1\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tutils.SetProcTitle(args)\n}\n" }, { "path": "cmd/main_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc TestArgsOrder(t *testing.T) {\n\tvar app = &cli.App{\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"verbose\",\n\t\t\t\tAliases: []string{\"v\"},\n\t\t\t},\n\t\t\t&cli.Int64Flag{\n\t\t\t\tName: \"key\",\n\t\t\t\tAliases: []string{\"k\"},\n\t\t\t},\n\t\t},\n\t\tCommands: []*cli.Command{\n\t\t\t{\n\t\t\t\tName: \"cmd\",\n\t\t\t\tFlags: []cli.Flag{\n\t\t\t\t\t&cli.Int64Flag{\n\t\t\t\t\t\tName: \"k2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tvar cases = [][]string{\n\t\t{\"test\", \"cmd\", \"a\", \"-k2\", \"v2\", \"b\", \"--v\"},\n\t\t{\"test\", \"--v\", \"cmd\", \"-k2\", \"v2\", \"a\", \"b\"},\n\t\t{\"test\", \"cmd\", \"a\", \"-k2=v\", \"--h\"},\n\t\t{\"test\", \"cmd\", \"-k2=v\", \"--h\", \"a\"},\n\t}\n\tfor i := 0; i < len(cases); i += 2 {\n\t\toreded := reorderOptions(app, cases[i])\n\t\tif !reflect.DeepEqual(cases[i+1], oreded) {\n\t\t\tt.Fatalf(\"expecte %v, but got %v\", cases[i+1], oreded)\n\t\t}\n\t}\n}\n\nfunc TestHandleSysMountArgs(t *testing.T) {\n\tvar cases = []struct {\n\t\targs []string\n\t\tnewArgs string\n\t\tfail bool\n\t}{\n\t\t{\n\t\t\t[]string{\"/mount.juicefs\", \"memkv://\", \"/jfs\", \"-o\", \"no-usage-report\"},\n\t\t\t\"juicefs mount -d --no-usage-report memkv:// /jfs\",\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t[]string{\"/mount.juicefs\", \"memkv://\", \"/jfs\", \"-o\", \"no-usage-report=true\"},\n\t\t\t\"juicefs mount -d --no-usage-report=true memkv:// /jfs\",\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t[]string{\"/mount.juicefs\", \"memkv://\", \"/jfs\", \"-o\", \"cache-size=204800\"},\n\t\t\t\"juicefs mount -d --cache-size=204800 memkv:// /jfs\",\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t[]string{\"/mount.juicefs\", \"memkv://\", \"/jfs\", \"-o\", \"verbose\"},\n\t\t\t\"juicefs mount -d --verbose memkv:// /jfs\",\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t[]string{\"/mount.juicefs\", \"memkv://\", \"/jfs\", \"-o\", \"debug\"},\n\t\t\t\"juicefs mount -d --debug -o debug memkv:// /jfs\",\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t[]string{\"/mount.juicefs\", \"memkv://\", \"/jfs\", \"-o\", \"cache-size=204800,no-usage-report=false,free-space-ratio=0.5,cache-dir=/data/juicfs,metrics=0.0.0.0:9567\"},\n\t\t\t\"juicefs mount -d --cache-size=204800 --no-usage-report=false --free-space-ratio=0.5 --cache-dir=/data/juicfs --metrics=0.0.0.0:9567 memkv:// /jfs\",\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t[]string{\"/mount.juicefs\", \"memkv://\", \"/jfs\", \"-o\", \"cache-size\"},\n\t\t\t\"\",\n\t\t\ttrue,\n\t\t},\n\t}\n\tfor _, c := range cases {\n\t\trawNewArgs, err := handleSysMountArgs(c.args)\n\t\tif c.fail && err == nil {\n\t\t\tt.Fatalf(\"expect error, but got nil\")\n\t\t}\n\t\tif !c.fail && err != nil {\n\t\t\tt.Fatalf(\"expect nil, but got %v\", err)\n\t\t}\n\t\tnewArgs := strings.Join(rawNewArgs, \" \")\n\t\tif c.newArgs != newArgs {\n\t\t\tt.Fatalf(\"expect `%v`, but got `%v`\", c.newArgs, newArgs)\n\t\t}\n\t}\n}\n" }, { "path": "cmd/mdtest.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t_ \"net/http/pprof\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/metric\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nvar ctx = meta.NewContext(1, uint32(utils.GetCurrentUID()), []uint32{uint32(utils.GetCurrentGID())})\nvar umask = uint16(utils.GetUmask())\n\nfunc init() {\n\t// For all the juicefs command, we treat admin/elevated privilege user as root(0) on Windows\n\t// just like the mount option '-adminasroot' does for the mounted filesystem.\n\tif runtime.GOOS == \"windows\" && utils.IsWinAdminOrElevatedPrivilege() {\n\t\tctx = meta.NewContext(1, 0, []uint32{0})\n\t}\n}\n\nfunc createDir(jfs *fs.FileSystem, root string, d int, width int) error {\n\tif err := jfs.Mkdir(ctx, root, 0777, umask); err != 0 {\n\t\treturn fmt.Errorf(\"Mkdir %s: %s\", root, err)\n\t}\n\tif d > 0 {\n\t\tfor i := 0; i < width; i++ {\n\t\t\tdn := path.Join(root, fmt.Sprintf(\"mdtest_tree.%d\", i))\n\t\t\tif err := createDir(jfs, dn, d-1, width); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc createFile(jfs *fs.FileSystem, bar *utils.Bar, np int, root string, d int, width, files, bytes int) error {\n\tm := jfs.Meta()\n\tfor i := 0; i < files; i++ {\n\t\tfn := path.Join(root, fmt.Sprintf(\"file.mdtest.%d.%d\", np, i))\n\t\tf, err := jfs.Create(ctx, fn, 0666, umask)\n\t\tif err != 0 {\n\t\t\treturn fmt.Errorf(\"create %s: %s\", fn, err)\n\t\t}\n\t\tif bytes > 0 {\n\t\t\tfor indx := 0; indx*meta.ChunkSize < bytes; indx++ {\n\t\t\t\tvar id uint64\n\t\t\t\tif st := m.NewSlice(ctx, &id); st != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"writechunk %s: %s\", fn, st)\n\t\t\t\t}\n\t\t\t\tsize := meta.ChunkSize\n\t\t\t\tif bytes < (indx+1)*meta.ChunkSize {\n\t\t\t\t\tsize = bytes - indx*meta.ChunkSize\n\t\t\t\t}\n\t\t\t\tif st := m.Write(ctx, f.Inode(), uint32(indx), 0, meta.Slice{Id: id, Size: uint32(size), Len: uint32(size)}, time.Now()); st != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"writeend %s: %s\", fn, st)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tf.Close(ctx)\n\t\tbar.Increment()\n\t}\n\tif d > 0 {\n\t\tdirs := make([]int, width)\n\t\tfor i := 0; i < width; i++ {\n\t\t\tdirs[i] = i\n\t\t}\n\t\trand.Shuffle(width, func(i, j int) {\n\t\t\tdirs[i], dirs[j] = dirs[j], dirs[i]\n\t\t})\n\t\tfor i := range dirs {\n\t\t\tdn := path.Join(root, fmt.Sprintf(\"mdtest_tree.%d\", dirs[i]))\n\t\t\tif err := createFile(jfs, bar, np, dn, d-1, width, files, bytes); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc runTest(jfs *fs.FileSystem, rootDir string, np, width, depth, files, bytes int) {\n\tdirs := 1\n\tw := width\n\tz := depth\n\tfor z > 0 {\n\t\tdirs += w\n\t\tw = w * width\n\t\tz--\n\t}\n\tvar total = dirs * np * files\n\tprogress := utils.NewProgress(!isatty.IsTerminal(os.Stdout.Fd()))\n\tbar := progress.AddCountBar(\"create file\", int64(total))\n\tlogger.Infof(\"Create %d files in %d dirs\", total, dirs)\n\n\tstart := time.Now()\n\tif err := jfs.Mkdir(ctx, rootDir, 0777, umask); err != 0 {\n\t\tlogger.Errorf(\"mkdir %s: %s\", rootDir, err)\n\t}\n\troot := path.Join(rootDir, \"test-dir.0-0\")\n\tif err := jfs.Mkdir(ctx, root, 0777, umask); err != 0 {\n\t\tlogger.Fatalf(\"Mkdir %s: %s\", root, err)\n\t}\n\troot = path.Join(root, \"mdtest_tree.0\")\n\tif err := createDir(jfs, root, depth, width); err != nil {\n\t\tlogger.Fatalf(\"initialize: %s\", err)\n\t}\n\tt1 := time.Since(start)\n\tlogger.Infof(\"Created %d dirs in %s (%d dirs/s)\", dirs, t1, int(float64(dirs)/t1.Seconds()))\n\n\tvar g sync.WaitGroup\n\tfor i := 0; i < np; i++ {\n\t\tg.Add(1)\n\t\tgo func(np int) {\n\t\t\tif err := createFile(jfs, bar, np, root, depth, width, files, bytes); err != nil {\n\t\t\t\tlogger.Errorf(\"Create: %s\", err)\n\t\t\t}\n\t\t\tg.Done()\n\t\t}(i)\n\t}\n\tg.Wait()\n\tprogress.Done()\n\tused := time.Since(start) - t1\n\tlogger.Infof(\"Created %d files in %s (%d files/s)\", total, used, int(float64(total)/used.Seconds()))\n}\n\nfunc cmdMdtest() *cli.Command {\n\tselfFlags := []cli.Flag{\n\t\t&cli.IntFlag{\n\t\t\tName: \"threads\",\n\t\t\tValue: 1,\n\t\t\tUsage: \"number of threads\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"dirs\",\n\t\t\tValue: 3,\n\t\t\tUsage: \"number of subdir\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"depth\",\n\t\t\tValue: 2,\n\t\t\tUsage: \"levels of tree\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"files\",\n\t\t\tValue: 10,\n\t\t\tUsage: \"number of files\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"write\",\n\t\t\tValue: 0,\n\t\t\tUsage: \"number of bytes\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"access-log\",\n\t\t\tUsage: \"path for JuiceFS access log\",\n\t\t},\n\t}\n\treturn &cli.Command{\n\t\tName: \"mdtest\",\n\t\tAction: mdtest,\n\t\tCategory: \"TOOL\",\n\t\tHidden: true,\n\t\tUsage: \"run test on meta engines\",\n\t\tArgsUsage: \"META-URL PATH\",\n\t\tDescription: `\nExamples:\n$ juicefs mdtest redis://localhost /test1`,\n\t\tFlags: expandFlags(selfFlags, clientFlags(0), shareInfoFlags()),\n\t}\n}\n\nfunc initForMdtest(c *cli.Context, mp string, metaUrl string) *fs.FileSystem {\n\tmetaConf := getMetaConf(c, mp, c.Bool(\"read-only\"))\n\tm := meta.NewClient(metaUrl, metaConf)\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"load setting: %s\", err)\n\t}\n\tif st := m.Chroot(meta.Background(), metaConf.Subdir); st != 0 {\n\t\tlogger.Fatalf(\"Chroot to %s: %s\", metaConf.Subdir, st)\n\t}\n\tregisterer, registry := wrapRegister(c, mp, format.Name)\n\n\tblob, err := NewReloadableStorage(format, m, updateFormat(c))\n\tif err != nil {\n\t\tlogger.Fatalf(\"object storage: %s\", err)\n\t}\n\tlogger.Infof(\"Data use %s\", blob)\n\n\tchunkConf := getChunkConf(c, format)\n\tstore := chunk.NewCachedStore(blob, *chunkConf, registerer)\n\tregisterMetaMsg(m, store, chunkConf)\n\n\terr = m.NewSession(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"new session: %s\", err)\n\t}\n\n\tconf := getVfsConf(c, metaConf, format, chunkConf)\n\tconf.AccessLog = c.String(\"access-log\")\n\tconf.AttrTimeout = utils.Duration(c.String(\"attr-cache\"))\n\tconf.EntryTimeout = utils.Duration(c.String(\"entry-cache\"))\n\tconf.DirEntryTimeout = utils.Duration(c.String(\"dir-entry-cache\"))\n\n\tmetricsAddr := exposeMetrics(c, registerer, registry)\n\tm.InitMetrics(registerer)\n\tvfs.InitMetrics(registerer)\n\tif c.IsSet(\"consul\") {\n\t\tmetadata := make(map[string]string)\n\t\tmetadata[\"mountPoint\"] = conf.Meta.MountPoint\n\t\tmetric.RegisterToConsul(c.String(\"consul\"), metricsAddr, metadata)\n\t}\n\tjfs, err := fs.NewFileSystem(conf, m, store, registry)\n\tif err != nil {\n\t\tlogger.Fatalf(\"initialize failed: %s\", err)\n\t}\n\tjfs.InitMetrics(registerer)\n\treturn jfs\n}\n\nfunc mdtest(c *cli.Context) error {\n\tsetup(c, 2)\n\tmetaUrl := c.Args().Get(0)\n\trootDir := c.Args().Get(1)\n\tremovePassword(metaUrl)\n\tjfs := initForMdtest(c, \"mdtest\", metaUrl)\n\trunTest(jfs, rootDir, c.Int(\"threads\"), c.Int(\"dirs\"), c.Int(\"depth\"), c.Int(\"files\"), c.Int(\"write\"))\n\treturn jfs.Meta().CloseSession()\n}\n" }, { "path": "cmd/mount.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t_ \"net/http/pprof\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/collectors\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n\t\"github.com/urfave/cli/v2\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/metric\"\n\t\"github.com/juicedata/juicefs/pkg/usage\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n)\n\nfunc cmdMount() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"mount\",\n\t\tAction: mount,\n\t\tCategory: \"SERVICE\",\n\t\tUsage: \"Mount a volume\",\n\t\tArgsUsage: \"META-URL MOUNTPOINT\",\n\t\tDescription: `\nMount the target volume at the mount point.\n\nExamples:\n# Mount in foreground\n$ juicefs mount redis://localhost /mnt/jfs\n\n# Mount in background with password protected Redis\n$ juicefs mount redis://:mypassword@localhost /mnt/jfs -d\n# A safer alternative\n$ META_PASSWORD=mypassword juicefs mount redis://localhost /mnt/jfs -d\n\n# Mount with a sub-directory as root\n$ juicefs mount redis://localhost /mnt/jfs --subdir /dir/in/jfs\n\n# Enable \"writeback\" mode, which improves performance at the risk of losing objects\n$ juicefs mount redis://localhost /mnt/jfs -d --writeback\n\n# Enable \"read-only\" mode\n$ juicefs mount redis://localhost /mnt/jfs -d --read-only\n\n# Disable metadata backup\n$ juicefs mount redis://localhost /mnt/jfs --backup-meta 0`,\n\t\tFlags: expandFlags(mountFlags(), clientFlags(1.0), shareInfoFlags()),\n\t}\n}\n\nfunc exposeMetrics(c *cli.Context, registerer prometheus.Registerer, registry *prometheus.Registry) string {\n\tvar ip, port string\n\t// default set\n\tip, port, err := net.SplitHostPort(c.String(\"metrics\"))\n\tif err != nil {\n\t\tlogger.Fatalf(\"metrics format error: %v\", err)\n\t}\n\tgo metric.UpdateMetrics(registerer)\n\thttp.Handle(\"/metrics\", promhttp.HandlerFor(\n\t\tregistry,\n\t\tpromhttp.HandlerOpts{\n\t\t\t// Opt into OpenMetrics to support exemplars.\n\t\t\tEnableOpenMetrics: true,\n\t\t},\n\t))\n\tregisterer.MustRegister(collectors.NewBuildInfoCollector())\n\n\t// If not set metrics addr,the port will be auto set\n\tif !c.IsSet(\"metrics\") {\n\t\t// If only set consul, ip will auto set\n\t\tif c.IsSet(\"consul\") {\n\t\t\tip, err = utils.GetLocalIp(c.String(\"consul\"))\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"Get local ip failed: %v\", err)\n\t\t\t\treturn \"\"\n\t\t\t}\n\t\t}\n\t}\n\n\tln, err := net.Listen(\"tcp\", net.JoinHostPort(ip, port))\n\tif err != nil {\n\t\t// Don't try other ports on metrics set but listen failed\n\t\tif c.IsSet(\"metrics\") {\n\t\t\tlogger.Errorf(\"listen on %s:%s failed: %v\", ip, port, err)\n\t\t\treturn \"\"\n\t\t}\n\t\t// Listen port on 0 will auto listen on a free port\n\t\tln, err = net.Listen(\"tcp\", net.JoinHostPort(ip, \"0\"))\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Listen failed: %v\", err)\n\t\t\treturn \"\"\n\t\t}\n\t}\n\n\tgo func() {\n\t\tif err := http.Serve(ln, nil); err != nil {\n\t\t\tlogger.Errorf(\"Serve for metrics: %s\", err)\n\t\t}\n\t}()\n\n\tmetricsAddr := ln.Addr().String()\n\tlogger.Infof(\"Prometheus metrics listening on %s\", metricsAddr)\n\treturn metricsAddr\n}\n\nfunc wrapRegister(c *cli.Context, mp, name string) (prometheus.Registerer, *prometheus.Registry) {\n\tcommonLabels := prometheus.Labels{\"mp\": mp, \"vol_name\": name, \"juicefs_version\": version.Version()}\n\tif h, err := os.Hostname(); err == nil {\n\t\tcommonLabels[\"instance\"] = h\n\t} else {\n\t\tlogger.Warnf(\"cannot get hostname: %s\", err)\n\t}\n\tif c.IsSet(\"custom-labels\") {\n\t\tfor _, kv := range strings.Split(c.String(\"custom-labels\"), \";\") {\n\t\t\tsplited := strings.Split(kv, \":\")\n\t\t\tif len(splited) != 2 {\n\t\t\t\tlogger.Fatalf(\"invalid label format: %s\", kv)\n\t\t\t}\n\t\t\tif utils.StringContains([]string{\"mp\", \"vol_name\", \"instance\"}, splited[0]) {\n\t\t\t\tlogger.Warnf(\"overriding reserved label: %s\", splited[0])\n\t\t\t}\n\t\t\tcommonLabels[splited[0]] = splited[1]\n\t\t}\n\t}\n\tregistry := prometheus.NewRegistry() // replace default so only JuiceFS metrics are exposed\n\tregisterer := prometheus.WrapRegistererWithPrefix(\"juicefs_\",\n\t\tprometheus.WrapRegistererWith(commonLabels, registry))\n\n\tregisterer.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))\n\tregisterer.MustRegister(collectors.NewGoCollector())\n\treturn registerer, registry\n}\n\nfunc updateFormat(c *cli.Context) func(*meta.Format) {\n\treturn func(format *meta.Format) {\n\t\tif c.IsSet(\"bucket\") {\n\t\t\tformat.Bucket = c.String(\"bucket\")\n\t\t}\n\t\tif c.IsSet(\"storage\") {\n\t\t\tformat.Storage = c.String(\"storage\")\n\t\t}\n\t\tif c.IsSet(\"storage-class\") {\n\t\t\tformat.StorageClass = c.String(\"storage-class\")\n\t\t}\n\t\tif c.IsSet(\"upload-limit\") {\n\t\t\tformat.UploadLimit = utils.ParseMbps(c, \"upload-limit\")\n\t\t}\n\t\tif c.IsSet(\"download-limit\") {\n\t\t\tformat.DownloadLimit = utils.ParseMbps(c, \"download-limit\")\n\t\t}\n\t}\n}\n\nfunc relPathToAbs(ss []string) []string {\n\tfor i, d := range ss {\n\t\tif strings.HasPrefix(d, \"/\") {\n\t\t\tcontinue\n\t\t} else if strings.HasPrefix(d, \"~/\") {\n\t\t\tif h, err := os.UserHomeDir(); err == nil {\n\t\t\t\tss[i] = filepath.Join(h, d[1:])\n\t\t\t} else {\n\t\t\t\tlogger.Fatalf(\"Expand user home dir of %s: %s\", d, err)\n\t\t\t}\n\t\t} else {\n\t\t\tif ad, err := filepath.Abs(d); err == nil {\n\t\t\t\tss[i] = ad\n\t\t\t} else {\n\t\t\t\tlogger.Fatalf(\"Find absolute path of %s: %s\", d, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn ss\n}\n\nfunc cacheDirPathToAbs(c *cli.Context) {\n\tif runtime.GOOS != \"windows\" {\n\t\tif cd := c.String(\"cache-dir\"); cd != \"memory\" {\n\t\t\tds := utils.SplitDir(cd)\n\t\t\tds = relPathToAbs(ds)\n\t\t\tfor i, a := range os.Args {\n\t\t\t\tif a == cd || a == \"--cache-dir=\"+cd {\n\t\t\t\t\tos.Args[i] = a[:len(a)-len(cd)] + strings.Join(ds, string(os.PathListSeparator))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif rpAcLog := c.String(\"access-log\"); rpAcLog != \"\" {\n\t\tap, err := filepath.Abs(rpAcLog)\n\t\tif err == nil && ap != rpAcLog {\n\t\t\tfor i, a := range os.Args {\n\t\t\t\tif a == rpAcLog || a == \"--access-log=\"+rpAcLog {\n\t\t\t\t\tos.Args[i] = a[:len(a)-len(rpAcLog)] + ap\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc daemonRun(c *cli.Context, addr string, vfsConf *vfs.Config) {\n\tcacheDirPathToAbs(c)\n\t_ = expandPathForEmbedded(addr)\n\t// The default log to syslog is only in daemon mode.\n\tutils.InitLoggers(!c.Bool(\"no-syslog\"))\n\terr := makeDaemon(c, vfsConf)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Failed to make daemon: %s\", err)\n\t}\n\tif runtime.GOOS == \"linux\" {\n\t\tlog.SetOutput(os.Stderr)\n\t}\n}\n\nfunc expandPathForEmbedded(addr string) string {\n\tembeddedSchemes := []string{\"sqlite3://\", \"badger://\"}\n\tfor _, es := range embeddedSchemes {\n\t\tif strings.HasPrefix(addr, es) {\n\t\t\tpath := addr[len(es):]\n\t\t\tabsPath, err := filepath.Abs(path)\n\t\t\tif err == nil && absPath != path {\n\t\t\t\tfor i, a := range os.Args {\n\t\t\t\t\tif a == addr {\n\t\t\t\t\t\texpanded := es + absPath\n\t\t\t\t\t\tos.Args[i] = expanded\n\t\t\t\t\t\treturn expanded\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn addr\n}\n\nfunc getVfsConf(c *cli.Context, metaConf *meta.Config, format *meta.Format, chunkConf *chunk.Config) *vfs.Config {\n\tcfg := &vfs.Config{\n\t\tMeta: metaConf,\n\t\tFormat: *format,\n\t\tSecurity: &vfs.SecurityConfig{\n\t\t\tEnableCap: c.Bool(\"enable-cap\"),\n\t\t\tEnableSELinux: c.Bool(\"enable-selinux\"),\n\t\t},\n\t\tVersion: version.Version(),\n\t\tChunk: chunkConf,\n\t\tBackupMeta: utils.Duration(c.String(\"backup-meta\")),\n\t\tBackupSkipTrash: c.Bool(\"backup-skip-trash\"),\n\t\tPort: &vfs.Port{DebugAgent: debugAgent, PyroscopeAddr: c.String(\"pyroscope\")},\n\t\tPrefixInternal: c.Bool(\"prefix-internal\"),\n\t\tPid: os.Getpid(),\n\t\tPPid: os.Getppid(),\n\t\tUMask: 0xFFFF,\n\t\tHideInternal: c.Bool(\"hide-internal\"),\n\t}\n\n\tif c.IsSet(\"umask\") {\n\t\tumask, err := strconv.ParseUint(c.String(\"umask\"), 8, 16)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"invalid umask %s: %s\", c.String(\"umask\"), err)\n\t\t}\n\t\tcfg.UMask = uint16(umask)\n\t}\n\n\tskip_check := os.Getenv(\"SKIP_BACKUP_META_CHECK\") == \"true\"\n\tif !skip_check && cfg.BackupMeta > 0 && cfg.BackupMeta < time.Minute*5 {\n\t\tlogger.Fatalf(\"backup-meta should not be less than 5 minutes: %s\", cfg.BackupMeta)\n\t}\n\treturn cfg\n}\n\nfunc registerMetaMsg(m meta.Meta, store chunk.ChunkStore, chunkConf *chunk.Config) {\n\tm.OnMsg(meta.DeleteSlice, func(args ...interface{}) error {\n\t\treturn store.Remove(args[0].(uint64), int(args[1].(uint32)))\n\t})\n\tm.OnMsg(meta.CompactChunk, func(args ...interface{}) error {\n\t\treturn vfs.Compact(*chunkConf, store, args[0].([]meta.Slice), args[1].(uint64))\n\t})\n}\n\nfunc readConfig(mp string) ([]byte, error) {\n\tcontents, err := os.ReadFile(filepath.Join(mp, \".jfs.config\"))\n\tif os.IsNotExist(err) {\n\t\tcontents, err = os.ReadFile(filepath.Join(mp, \".config\"))\n\t}\n\treturn contents, err\n}\n\nfunc getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config {\n\tconf := meta.DefaultConf()\n\tconf.Retries = c.Int(\"io-retries\")\n\tconf.MaxDeletes = c.Int(\"max-deletes\")\n\tconf.SkipDirNlink = c.Int(\"skip-dir-nlink\")\n\tconf.ReadOnly = readOnly\n\tconf.NoBGJob = c.Bool(\"no-bgjob\")\n\tconf.OpenCache = utils.Duration(c.String(\"open-cache\"))\n\tconf.OpenCacheLimit = c.Uint64(\"open-cache-limit\")\n\tconf.Heartbeat = utils.Duration(c.String(\"heartbeat\"))\n\tconf.MountPoint = mp\n\tconf.Subdir = c.String(\"subdir\")\n\tconf.SkipDirMtime = utils.Duration(c.String(\"skip-dir-mtime\"))\n\tconf.Sid, _ = strconv.ParseUint(os.Getenv(\"_JFS_META_SID\"), 10, 64)\n\tconf.SortDir = c.Bool(\"sort-dir\")\n\tconf.FastStatfs = c.Bool(\"fast-statfs\")\n\n\tatimeMode := c.String(\"atime-mode\")\n\tif atimeMode != meta.RelAtime && atimeMode != meta.StrictAtime && atimeMode != meta.NoAtime {\n\t\tlogger.Warnf(\"unknown atime-mode \\\"%s\\\", changed to %s\", atimeMode, meta.NoAtime)\n\t\tatimeMode = meta.NoAtime\n\t}\n\tconf.AtimeMode = atimeMode\n\n\t// Parse network interfaces\n\tif ifaces := c.String(\"network-interfaces\"); ifaces != \"\" {\n\t\tconf.NetworkInterfaces = strings.Split(ifaces, \",\")\n\t\t// Trim whitespace from each interface name\n\t\tfor i := range conf.NetworkInterfaces {\n\t\t\tconf.NetworkInterfaces[i] = strings.TrimSpace(conf.NetworkInterfaces[i])\n\t\t}\n\t}\n\n\treturn conf\n}\n\nfunc getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {\n\tcm, err := strconv.ParseUint(c.String(\"cache-mode\"), 8, 32)\n\tif err != nil {\n\t\tlogger.Warnf(\"Invalid cache-mode %s, using default value 0600\", c.String(\"cache-mode\"))\n\t\tcm = 0600\n\t}\n\tchunkConf := &chunk.Config{\n\t\tBlockSize: format.BlockSize * 1024,\n\t\tCompress: format.Compression,\n\t\tHashPrefix: format.HashPrefix,\n\n\t\tGetTimeout: utils.Duration(c.String(\"get-timeout\")),\n\t\tPutTimeout: utils.Duration(c.String(\"put-timeout\")),\n\t\tMaxUpload: c.Int(\"max-uploads\"),\n\t\tMaxDownload: c.Int(\"max-downloads\"),\n\t\tMaxStageWrite: c.Int(\"max-stage-write\"),\n\t\tMaxRetries: c.Int(\"io-retries\"),\n\t\tWriteback: c.Bool(\"writeback\"),\n\t\tWritebackThresholdSize: int(utils.ParseBytes(c, \"writeback-threshold-size\", 'B')),\n\t\tPrefetch: c.Int(\"prefetch\"),\n\t\tBufferSize: utils.ParseBytes(c, \"buffer-size\", 'M'),\n\t\tUploadLimit: utils.ParseMbps(c, \"upload-limit\") * 1e6 / 8,\n\t\tDownloadLimit: utils.ParseMbps(c, \"download-limit\") * 1e6 / 8,\n\t\tUploadDelay: utils.Duration(c.String(\"upload-delay\")),\n\t\tUploadHours: c.String(\"upload-hours\"),\n\n\t\tCacheDir: c.String(\"cache-dir\"),\n\t\tCacheSize: utils.ParseBytes(c, \"cache-size\", 'M'),\n\t\tCacheItems: c.Int64(\"cache-items\"),\n\t\tFreeSpace: float32(c.Float64(\"free-space-ratio\")),\n\t\tCacheMode: os.FileMode(cm),\n\t\tCacheFullBlock: !c.Bool(\"cache-partial-only\"),\n\t\tCacheLargeWrite: c.Bool(\"cache-large-write\"),\n\t\tCacheChecksum: c.String(\"verify-cache-checksum\"),\n\t\tCacheEviction: c.String(\"cache-eviction\"),\n\t\tCacheScanInterval: utils.Duration(c.String(\"cache-scan-interval\")),\n\t\tCacheExpire: utils.Duration(c.String(\"cache-expire\")),\n\t\tOSCache: os.Getenv(\"JFS_DROP_OSCACHE\") == \"\",\n\t\tAutoCreate: true,\n\t}\n\tif c.IsSet(\"max-readahead\") {\n\t\tchunkConf.Readahead = int(utils.ParseBytes(c, \"max-readahead\", 'M'))\n\t} else {\n\t\tchunkConf.Readahead = 8 * chunkConf.BlockSize\n\t}\n\n\tif chunkConf.UploadLimit == 0 {\n\t\tchunkConf.UploadLimit = format.UploadLimit * 1e6 / 8\n\t}\n\tif chunkConf.DownloadLimit == 0 {\n\t\tchunkConf.DownloadLimit = format.DownloadLimit * 1e6 / 8\n\t}\n\tchunkConf.SelfCheck(format.UUID)\n\treturn chunkConf\n}\n\nfunc initBackgroundTasks(c *cli.Context, vfsConf *vfs.Config, metaConf *meta.Config, m meta.Meta, blob object.ObjectStorage, registerer prometheus.Registerer, registry *prometheus.Registry) {\n\tmetricsAddr := exposeMetrics(c, registerer, registry)\n\tm.InitMetrics(registerer)\n\tif !metaConf.NoBGJob {\n\t\tm.InitSharedMetrics(registerer)\n\t}\n\tvfs.InitMetrics(registerer)\n\tvfsConf.Port.PrometheusAgent = metricsAddr\n\tif c.IsSet(\"consul\") {\n\t\tmetadata := make(map[string]string)\n\t\tmetadata[\"mountPoint\"] = vfsConf.Meta.MountPoint\n\t\tmetric.RegisterToConsul(c.String(\"consul\"), metricsAddr, metadata)\n\t\tvfsConf.Port.ConsulAddr = c.String(\"consul\")\n\t}\n\tif !metaConf.ReadOnly && !metaConf.NoBGJob && vfsConf.BackupMeta > 0 {\n\t\tregisterer.MustRegister(vfs.LastBackupTimeG)\n\t\tregisterer.MustRegister(vfs.LastBackupDurationG)\n\t\tgo vfs.Backup(m, blob, vfsConf.BackupMeta, vfsConf.BackupSkipTrash)\n\t} else {\n\t\tlogger.Warnf(\"Metadata backup is disabled\")\n\t}\n\tif !c.Bool(\"no-usage-report\") {\n\t\tgo usage.ReportUsage(m, version.Version())\n\t}\n}\n\ntype storageHolder struct {\n\tobject.ObjectStorage\n\tfmt meta.Format\n}\n\nfunc (h *storageHolder) Shutdown() {\n\tobject.Shutdown(h.ObjectStorage)\n}\n\nfunc NewReloadableStorage(format *meta.Format, cli meta.Meta, patch func(*meta.Format)) (object.ObjectStorage, error) {\n\tif patch != nil {\n\t\tpatch(format)\n\t}\n\tblob, err := createStorage(*format)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tholder := &storageHolder{\n\t\tObjectStorage: blob,\n\t\tfmt: *format, // keep a copy to find the change\n\t}\n\tcli.OnReload(func(new *meta.Format) {\n\t\tif patch != nil {\n\t\t\tpatch(new)\n\t\t}\n\t\told := &holder.fmt\n\t\tif new.Storage != old.Storage || new.Bucket != old.Bucket || new.AccessKey != old.AccessKey || new.SecretKey != old.SecretKey || new.SessionToken != old.SessionToken || new.StorageClass != old.StorageClass {\n\t\t\tlogger.Infof(\"found new configuration: storage=%s bucket=%s ak=%s storageClass=%s\", new.Storage, new.Bucket, new.AccessKey, new.StorageClass)\n\n\t\t\tnewBlob, err := createStorage(*new)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"object storage: %s\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tholder.ObjectStorage = newBlob\n\t\t\tholder.fmt = *new\n\t\t}\n\t})\n\treturn holder, nil\n}\n\nfunc insideContainer() bool {\n\tif _, err := os.Stat(\"/.dockerenv\"); err == nil {\n\t\treturn true\n\t}\n\tmountinfo, err := os.Open(\"/proc/1/mountinfo\")\n\tif err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false\n\t\t} else {\n\t\t\tlogger.Warnf(\"Open /proc/1/mountinfo: %s\", err)\n\t\t\treturn false\n\t\t}\n\t}\n\tdefer mountinfo.Close()\n\tscanner := bufio.NewScanner(mountinfo)\n\tfor scanner.Scan() {\n\t\tline := scanner.Text()\n\t\tfields := strings.Fields(line)\n\t\tif len(fields) > 8 && fields[4] == \"/\" {\n\t\t\tfstype := fields[8]\n\t\t\treturn strings.Contains(fstype, \"overlay\") || strings.Contains(fstype, \"aufs\")\n\t\t}\n\t}\n\tif err = scanner.Err(); err != nil {\n\t\tlogger.Warnf(\"scan /proc/1/mountinfo: %s\", err)\n\t}\n\treturn false\n}\n\nfunc getDefaultLogDir() string {\n\tvar defaultLogDir = \"/var/log\"\n\tswitch runtime.GOOS {\n\tcase \"linux\":\n\t\tif os.Getuid() == 0 {\n\t\t\tbreak\n\t\t}\n\t\tfallthrough\n\tcase \"darwin\":\n\t\thomeDir, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tlogger.Warn(err)\n\t\t\thomeDir = defaultLogDir\n\t\t}\n\t\tdefaultLogDir = path.Join(homeDir, \".juicefs\")\n\tcase \"windows\":\n\t\thomeDir, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"%v\", err)\n\t\t}\n\t\tdefaultLogDir = path.Join(homeDir, \".juicefs\")\n\t}\n\treturn defaultLogDir\n}\n\nfunc mount(c *cli.Context) error {\n\tsetup(c, 2)\n\taddr := c.Args().Get(0)\n\tremovePassword(addr)\n\tmp := c.Args().Get(1)\n\n\tstage := getDaemonStage()\n\tif stage < 0 || stage > 2 {\n\t\tlogger.Fatalf(\"Invalid daemon stage: %d\", stage)\n\t}\n\tsupervisor := os.Getenv(\"JFS_SUPERVISOR\")\n\tif supervisor != \"\" || runtime.GOOS == \"windows\" {\n\t\tstage = 3\n\t}\n\n\tvar err error\n\tif stage == 0 || supervisor == \"test\" {\n\t\terr = utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\t\tmp, err = filepath.Abs(mp)\n\t\t\treturn err\n\t\t}, time.Second*3)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"abs %s: %s\", mp, err)\n\t\t}\n\t\tif mp == \"/\" {\n\t\t\tlogger.Fatalf(\"should not mount on the root directory\")\n\t\t}\n\t\tprepareMp(mp)\n\t\tif runtime.GOOS == \"linux\" && c.Bool(\"update-fstab\") && !calledViaMount(os.Args) && !insideContainer() {\n\t\t\tif os.Getuid() != 0 {\n\t\t\t\tlogger.Warnf(\"--update-fstab should be used with root\")\n\t\t\t} else {\n\t\t\t\tvar e1, e2 error\n\t\t\t\tif e1 = tryToInstallMountExec(); e1 != nil {\n\t\t\t\t\tlogger.Warnf(\"failed to create /sbin/mount.juicefs: %s\", e1)\n\t\t\t\t}\n\t\t\t\tif e2 = updateFstab(c); e2 != nil {\n\t\t\t\t\tlogger.Warnf(\"failed to update fstab: %s\", e2)\n\t\t\t\t}\n\t\t\t\tif e1 == nil && e2 == nil {\n\t\t\t\t\tlogger.Infof(\"Successfully updated fstab, now you can mount with `mount %s`\", mp)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tvar format = &meta.Format{}\n\tvar metaCli meta.Meta\n\tvar blob object.ObjectStorage\n\tmetaConf := getMetaConf(c, mp, c.Bool(\"read-only\") || utils.StringContains(strings.Split(c.String(\"o\"), \",\"), \"ro\"))\n\tif runtime.GOOS == \"windows\" {\n\t\tmetaConf.CaseInsensi = !c.Bool(\"case-sensitive\")\n\t}\n\t// stage 0: check the connection to fail fast\n\t// stage 2: need the volume name to check if it's already mounted\n\t// stage 3: the real service process\n\tif stage != 1 {\n\t\tmetaCli = meta.NewClient(addr, metaConf)\n\t\tformat, err = metaCli.Load(true)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tchunkConf := getChunkConf(c, format)\n\tvfsConf := getVfsConf(c, metaConf, format, chunkConf)\n\tsetFuseOption(c, format, vfsConf)\n\tif stage == 0 || stage == 3 {\n\t\tblob, err = NewReloadableStorage(format, metaCli, updateFormat(c))\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"object storage: %s\", err)\n\t\t}\n\t\tlogger.Infof(\"Data use %s\", blob)\n\n\t}\n\n\tif stage < 3 {\n\t\t// supervisor serves no user request\n\t\tif metaCli != nil {\n\t\t\tif err = metaCli.Shutdown(); err != nil {\n\t\t\t\tlogger.Errorf(\"[pid=%d] meta shutdown: %s\", os.Getpid(), err)\n\t\t\t}\n\t\t}\n\t\tif blob != nil {\n\t\t\t// test storage at startup to fail fast instead of throwing EIO in the middle of user's workload\n\t\t\tif c.Bool(\"check-storage\") {\n\t\t\t\tstart := time.Now()\n\t\t\t\tif err = test(blob); err != nil {\n\t\t\t\t\tlogger.Errorf(\"Object storage test failed: %s\", err)\n\t\t\t\t\treturn err\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Infof(\"Object storage test passed in %s\", time.Since(start))\n\t\t\t\t}\n\t\t\t}\n\t\t\tobject.Shutdown(blob)\n\t\t}\n\t\tvar foreground bool\n\t\tif runtime.GOOS == \"windows\" || !c.Bool(\"background\") || os.Getenv(\"JFS_FOREGROUND\") != \"\" {\n\t\t\tforeground = true\n\t\t} else if c.Bool(\"background\") || os.Getenv(\"__DAEMON_STAGE\") != \"\" {\n\t\t\tforeground = false\n\t\t} else {\n\t\t\tforeground = os.Getppid() == 1 && !insideContainer()\n\t\t}\n\t\tif foreground {\n\t\t\tgo checkMountpoint(format.Name, mp, c.String(\"log\"), false)\n\t\t} else {\n\t\t\tdaemonRun(c, addr, vfsConf) // only stage 0 needs the vfsConf\n\t\t}\n\t\tos.Setenv(\"JFS_SUPERVISOR\", strconv.Itoa(os.Getppid()))\n\t\treturn launchMount(c, mp, vfsConf)\n\t} else if runtime.GOOS == \"windows\" && c.Bool(\"background\") {\n\t\tdaemonRun(c, addr, vfsConf)\n\t\treturn nil\n\t}\n\tlogger.Infof(\"JuiceFS version %s\", version.Version())\n\n\tif commPath := os.Getenv(\"_FUSE_FD_COMM\"); commPath != \"\" {\n\t\tvfsConf.CommPath = commPath\n\t\tvfsConf.StatePath = fmt.Sprintf(\"/tmp/state%d.json\", os.Getppid())\n\t}\n\n\tif st := metaCli.Chroot(meta.Background(), metaConf.Subdir); st != 0 {\n\t\treturn st\n\t}\n\t// Wrap the default registry, all prometheus.MustRegister() calls should be afterwards\n\tregisterer, registry := wrapRegister(c, mp, format.Name)\n\n\tstore := chunk.NewCachedStore(blob, *chunkConf, registerer)\n\tregisterMetaMsg(metaCli, store, chunkConf)\n\n\terr = metaCli.NewSession(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"new session: %s\", err)\n\t}\n\n\tmetaCli.OnReload(func(fmt *meta.Format) {\n\t\tupdateFormat(c)(fmt)\n\t\tstore.UpdateLimit(fmt.UploadLimit, fmt.DownloadLimit)\n\t})\n\tv := vfs.NewVFS(vfsConf, metaCli, store, registerer, registry)\n\tinstallHandler(metaCli, mp, v, blob)\n\tv.UpdateFormat = updateFormat(c)\n\tinitBackgroundTasks(c, vfsConf, metaConf, metaCli, blob, registerer, registry)\n\tmountMain(v, c)\n\tif err := v.FlushAll(\"\"); err != nil {\n\t\tlogger.Errorf(\"flush all delayed data: %s\", err)\n\t}\n\terr = metaCli.CloseSession()\n\tobject.Shutdown(blob)\n\tlogger.Infof(\"The juicefs mount process exit successfully, mountpoint: %s\", metaConf.MountPoint)\n\treturn err\n}\n" }, { "path": "cmd/mount_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/agiledragon/gomonkey/v2\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nconst testMeta = \"redis://127.0.0.1:6379/11\"\nconst testMountPoint = \"/tmp/jfs-unit-test\"\nconst testVolume = \"test\"\n\n// gomonkey may encounter the problem of insufficient permissions under mac, please solve it by viewing this link https://github.com/agiledragon/gomonkey/issues/70\nfunc Test_exposeMetrics(t *testing.T) {\n\taddr := \"redis://127.0.0.1:6379/12\"\n\tclient := meta.NewClient(addr, nil)\n\tformat := &meta.Format{\n\t\tName: \"test\",\n\t\tBlockSize: 4096,\n\t\tCapacity: 1 << 30,\n\t\tDirStats: true,\n\t}\n\t_ = client.Init(format, true)\n\tvar appCtx *cli.Context\n\tstringPatches := gomonkey.ApplyMethod(reflect.TypeOf(appCtx), \"String\", func(_ *cli.Context, arg string) string {\n\t\tswitch arg {\n\t\tcase \"metrics\":\n\t\t\treturn \"127.0.0.1:9567\"\n\t\tcase \"consul\":\n\t\t\treturn \"127.0.0.1:8500\"\n\t\tcase \"custom-labels\":\n\t\t\treturn \"key1:value1\"\n\t\tdefault:\n\t\t\treturn \"\"\n\t\t}\n\t})\n\tisSetPatches := gomonkey.ApplyMethod(reflect.TypeOf(appCtx), \"IsSet\", func(_ *cli.Context, arg string) bool {\n\t\tswitch arg {\n\t\tcase \"custom-labels\":\n\t\t\treturn true\n\t\tdefault:\n\t\t\treturn false\n\t\t}\n\t})\n\tdefer stringPatches.Reset()\n\tdefer isSetPatches.Reset()\n\tResetHttp()\n\tregisterer, registry := wrapRegister(appCtx, \"test\", \"test\")\n\tmetricsAddr := exposeMetrics(appCtx, registerer, registry)\n\tclient.InitMetrics(registerer)\n\tvfs.InitMetrics(registerer)\n\tu := url.URL{Scheme: \"http\", Host: metricsAddr, Path: \"/metrics\"}\n\tresp, err := http.Get(u.String())\n\trequire.Nil(t, err)\n\tall, err := io.ReadAll(resp.Body)\n\trequire.Nil(t, err)\n\trequire.NotEmpty(t, all)\n\trequire.Contains(t, string(all), `key1=\"value1\"`)\n}\n\nfunc ResetHttp() {\n\thttp.DefaultServeMux = http.NewServeMux()\n}\n\nfunc resetTestMeta() *redis.Client { // using Redis\n\topt, _ := redis.ParseURL(testMeta)\n\trdb := redis.NewClient(opt)\n\t_ = rdb.FlushDB(context.Background())\n\treturn rdb\n}\n\nvar mountLock sync.Mutex\n\nfunc mountTemp(t *testing.T, bucket *string, extraFormatOpts []string, extraMountOpts []string) {\n\t// wait for last mount exit\n\tfor !mountLock.TryLock() {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\n\t_ = resetTestMeta()\n\ttestDir := t.TempDir()\n\tif bucket != nil {\n\t\t*bucket = testDir\n\t}\n\tformatArgs := []string{\"\", \"format\", \"--bucket\", testDir, testMeta, testVolume}\n\tif extraFormatOpts != nil {\n\t\tformatArgs = append(formatArgs, extraFormatOpts...)\n\t}\n\tif err := Main(formatArgs); err != nil {\n\t\tt.Fatalf(\"format failed: %s\", err)\n\t}\n\n\t// must do reset, otherwise will panic\n\tResetHttp()\n\n\tos.Setenv(\"JFS_SUPERVISOR\", \"test\")\n\tmountArgs := []string{\"\", \"mount\", \"--enable-xattr\", testMeta, testMountPoint, \"--attr-cache\", \"0\", \"--entry-cache\", \"0\", \"--dir-entry-cache\", \"0\", \"--no-usage-report\"}\n\tif extraMountOpts != nil {\n\t\tmountArgs = append(mountArgs, extraMountOpts...)\n\t}\n\tgo func() {\n\t\tdefer mountLock.Unlock()\n\t\tif err := Main(mountArgs); err != nil {\n\t\t\tt.Errorf(\"mount failed: %s\", err)\n\t\t}\n\t}()\n\ttime.Sleep(3 * time.Second)\n\tinode, err := utils.GetFileInode(testMountPoint)\n\tif err != nil {\n\t\tt.Fatalf(\"get file inode failed: %s\", err)\n\t}\n\tif inode != 1 {\n\t\tt.Fatalf(\"mount failed: inode of %s got %d, expect 1\", testMountPoint, inode)\n\t} else {\n\t\tt.Logf(\"mount %s success\", testMountPoint)\n\t}\n}\n\nfunc umountTemp(t *testing.T) {\n\tif err := Main([]string{\"\", \"umount\", testMountPoint}); err != nil {\n\t\tt.Fatalf(\"umount failed: %s\", err)\n\t}\n}\n\nfunc TestMount(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tif err := os.WriteFile(fmt.Sprintf(\"%s/f1.txt\", testMountPoint), []byte(\"test\"), 0644); err != nil {\n\t\tt.Fatalf(\"write file failed: %s\", err)\n\t}\n}\n\nfunc TestFtruncate(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tfpath := fmt.Sprintf(\"%s/f1.txt\", testMountPoint)\n\tif err := os.WriteFile(fpath, []byte(\"test\"), 0644); err != nil {\n\t\tt.Fatalf(\"write file failed: %s\", err)\n\t}\n\tfile, err := os.OpenFile(fpath, os.O_RDWR, 0644)\n\tif err != nil {\n\t\tt.Fatalf(\"open file failed: %s\", err)\n\t}\n\tif err = syscall.Ftruncate(int(file.Fd()), 1024); err != nil {\n\t\tt.Fatalf(\"ftruncate failed: %s\", err)\n\t}\n\tfileInfo, err := os.Stat(fpath)\n\tif err != nil {\n\t\tt.Fatalf(\"stat file failed: %s\", err)\n\t}\n\tif fileInfo.Size() != 1024 {\n\t\tt.Fatalf(\"ftruncate failed: file size is %d, expect 1024\", fileInfo.Size())\n\t}\n\tif err = os.Remove(fpath); err != nil {\n\t\tt.Fatalf(\"remove file failed: %s\", err)\n\t}\n\tif _, err = os.Stat(fpath); !errors.Is(err, syscall.ENOENT) {\n\t\tt.Fatalf(\"file still exists after delete: %s\", err)\n\t}\n\terr = syscall.Ftruncate(int(file.Fd()), 2048)\n\tif err != nil {\n\t\tt.Fatalf(\"ftruncate failed: %s\", err)\n\t}\n\tfile.Close()\n\t_, err = os.Stat(fpath)\n\tif !errors.Is(err, syscall.ENOENT) {\n\t\tt.Fatalf(\"file still exists after close: %s\", err)\n\t}\n}\nfunc TestUpdateFstab(t *testing.T) {\n\tif runtime.GOOS != \"linux\" {\n\t\tt.SkipNow()\n\t}\n\tmockFstab, err := os.CreateTemp(\"/tmp\", \"fstab\")\n\tif err != nil {\n\t\tt.Fatalf(\"cannot make temp file: %s\", err)\n\t}\n\tdefer os.Remove(mockFstab.Name())\n\n\tpatches := gomonkey.ApplyFunc(os.Rename, func(src, dest string) error {\n\t\tcontent, err := os.ReadFile(mockFstab.Name())\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error reading mocked fstab: %s\", err)\n\t\t}\n\t\trv := \"redis://127.0.0.1:6379/11 /tmp/jfs-unit-test juicefs _netdev,enable-xattr,entry-cache=2,max-uploads=3,max_read=99,no-usage-report,writeback 0 0\"\n\t\tlv := strings.TrimSpace(string(content))\n\t\tif lv != rv {\n\t\t\tt.Fatalf(\"incorrect fstab entry: %s\", content)\n\t\t}\n\t\treturn os.Rename(src, dest)\n\t})\n\tdefer patches.Reset()\n\tmountArgs := []string{\"juicefs\", \"mount\", \"--enable-xattr\", testMeta, testMountPoint, \"--no-usage-report\"}\n\tmountOpts := []string{\"--update-fstab\", \"--writeback\", \"--entry-cache=2\", \"--max-uploads\", \"3\", \"-o\", \"max_read=99\"}\n\tpatches = gomonkey.ApplyGlobalVar(&os.Args, append(mountArgs, mountOpts...))\n\tdefer patches.Reset()\n\tmountTemp(t, nil, nil, mountOpts)\n\tdefer umountTemp(t)\n}\n\nfunc TestUmount(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tumountTemp(t)\n\n\tinode, err := utils.GetFileInode(testMountPoint)\n\tif err != nil {\n\t\tt.Fatalf(\"get file inode failed: %s\", err)\n\t}\n\tif inode == 1 {\n\t\tt.Fatalf(\"umount failed: inode of %s is 1\", testMountPoint)\n\t}\n}\n\nfunc tryMountTemp(t *testing.T, bucket *string, extraFormatOpts []string, extraMountOpts []string) error {\n\t// wait for last mount exit\n\tfor !mountLock.TryLock() {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\n\t_ = resetTestMeta()\n\ttestDir := t.TempDir()\n\tif bucket != nil {\n\t\t*bucket = testDir\n\t}\n\tformatArgs := []string{\"\", \"format\", \"--bucket\", testDir, testMeta, testVolume}\n\tif extraFormatOpts != nil {\n\t\tformatArgs = append(formatArgs, extraFormatOpts...)\n\t}\n\tif err := Main(formatArgs); err != nil {\n\t\treturn fmt.Errorf(\"format failed: %w\", err)\n\t}\n\n\t// must do reset, otherwise will panic\n\tResetHttp()\n\n\tmountArgs := []string{\"\", \"mount\", \"--enable-xattr\", testMeta, testMountPoint, \"--attr-cache\", \"0\", \"--entry-cache\", \"0\", \"--dir-entry-cache\", \"0\", \"--no-usage-report\"}\n\tif extraMountOpts != nil {\n\t\tmountArgs = append(mountArgs, extraMountOpts...)\n\t}\n\n\tos.Setenv(\"JFS_SUPERVISOR\", \"test\")\n\terrChan := make(chan error, 1)\n\tgo func() {\n\t\tdefer mountLock.Unlock()\n\t\terrChan <- Main(mountArgs)\n\t}()\n\n\tselect {\n\tcase err := <-errChan:\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"mount failed: %w\", err)\n\t\t}\n\tcase <-time.After(3 * time.Second):\n\t}\n\n\tinode, err := utils.GetFileInode(testMountPoint)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"get file inode failed: %w\", err)\n\t}\n\tif inode != 1 {\n\t\treturn fmt.Errorf(\"mount failed: inode of %s is %d, expect 1\", testMountPoint, inode)\n\t}\n\tt.Logf(\"mount %s success\", testMountPoint)\n\treturn nil\n}\n\nfunc TestMountVersionMatch(t *testing.T) {\n\toriVersion := version.Version()\n\tversion.SetVersion(\"1.1.0\")\n\tdefer version.SetVersion(oriVersion)\n\n\terr := tryMountTemp(t, nil, nil, nil)\n\tassert.Nil(t, err)\n\tumountTemp(t)\n\n\terr = tryMountTemp(t, nil, []string{\"--enable-acl=true\"}, nil)\n\tassert.Contains(t, err.Error(), \"check version\")\n}\n\nfunc TestParseUIDGID(t *testing.T) {\n\ttests := []struct {\n\t\tinput string\n\t\tdefaultUid uint32\n\t\tdefaultGid uint32\n\t\texpectedUid uint32\n\t\texpectedGid uint32\n\t}{\n\t\t{\"1000:1000\", 65534, 65534, 1000, 1000},\n\t\t{\"1000:\", 65534, 65534, 1000, 65534},\n\t\t{\":1000\", 65534, 65534, 65534, 1000},\n\t\t{\"\", 65534, 65534, 65534, 65534},\n\t\t{\"0:1000\", 65534, 65534, 65534, 1000},\n\t\t{\"1000:0\", 65534, 65534, 1000, 65534},\n\t}\n\n\tfor _, tt := range tests {\n\t\tuid, gid := parseUIDGID(tt.input, tt.defaultUid, tt.defaultGid)\n\t\tif uid != tt.expectedUid || gid != tt.expectedGid {\n\t\t\tt.Errorf(\"parseUIDGID(%q) = (%d, %d), want (%d, %d)\", tt.input, uid, gid, tt.expectedUid, tt.expectedGid)\n\t\t}\n\t}\n}\n" }, { "path": "cmd/mount_unix.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"os/user\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/godaemon\"\n\t\"github.com/urfave/cli/v2\"\n\n\t\"github.com/juicedata/juicefs/pkg/fuse\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n)\n\nvar mountPid int\n\nfunc showThreadStack(agentAddr string) {\n\tif agentAddr == \"\" {\n\t\treturn\n\t}\n\tclient := http.Client{\n\t\tTimeout: 10 * time.Second,\n\t}\n\tresp, err := client.Get(fmt.Sprintf(\"http://%s/debug/pprof/goroutine?debug=2\", agentAddr))\n\tif err != nil {\n\t\tlogger.Warnf(\"list goroutine from %s: %s\", agentAddr, err)\n\t} else {\n\t\tgrs, _ := io.ReadAll(resp.Body)\n\t\tlogger.Infof(\"list goroutines from %s:\\n%s\", agentAddr, string(grs))\n\t\t_ = resp.Body.Close()\n\t}\n}\n\n// devMinor returns the minor component of a Linux device number.\nfunc devMinor(dev uint64) uint32 {\n\tminor := dev & 0xff\n\tminor |= (dev >> 12) & 0xffffff00\n\treturn uint32(minor)\n}\n\nfunc killMountProcess(pid int, dev uint64, lastActive *int64) {\n\tif pid > 0 {\n\t\tlogger.Infof(\"watchdog: kill %d\", pid)\n\t\terr := syscall.Kill(pid, syscall.SIGABRT)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"kill %d: %s\", pid, err)\n\t\t\t_ = syscall.Kill(pid, syscall.SIGKILL)\n\t\t}\n\t\t// double check\n\t\ttime.Sleep(time.Second * 10)\n\t\tif atomic.LoadInt64(lastActive)+30 > time.Now().Unix() {\n\t\t\treturn\n\t\t}\n\t}\n\tif runtime.GOOS == \"linux\" && dev > 0 {\n\t\ttids, _ := os.ReadDir(fmt.Sprintf(\"/proc/%d/task\", pid))\n\t\tfor _, tid := range tids {\n\t\t\tstack, err := os.ReadFile(fmt.Sprintf(\"/proc/%d/task/%s/stack\", pid, tid))\n\t\t\tif err == nil && bytes.Contains(stack, []byte(\"fuse_simple_request\")) {\n\t\t\t\tlogger.Errorf(\"find deadlock in mount process, abort it: %s\", string(stack))\n\t\t\t\tif fuseFd > 0 {\n\t\t\t\t\t_ = syscall.Close(fuseFd)\n\t\t\t\t\tfuseFd = 0\n\t\t\t\t}\n\t\t\t\tf, err := os.OpenFile(fmt.Sprintf(\"/sys/fs/fuse/connections/%d/abort\", devMinor(dev)), os.O_WRONLY, 0777)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Warn(err)\n\t\t\t\t} else {\n\t\t\t\t\t_, _ = f.WriteString(\"1\")\n\t\t\t\t\t_ = f.Close()\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc loadConfig(path string) (string, *vfs.Config, error) {\n\tfor d := path; d != \"/\"; d = filepath.Dir(d) {\n\t\tdata, err := readConfig(d)\n\t\tif err == nil {\n\t\t\tvar conf vfs.Config\n\t\t\terr = json.Unmarshal(data, &conf)\n\t\t\treturn d, &conf, err\n\t\t}\n\t\tif !os.IsNotExist(err) {\n\t\t\treturn \"\", nil, fmt.Errorf(\"read %s: %w\", d, err)\n\t\t}\n\t}\n\treturn \"\", nil, fmt.Errorf(\"%s is not inside JuiceFS\", path)\n}\n\nfunc watchdog(ctx context.Context, mp string) {\n\tvar lastActive int64\n\tvar pid int\n\tvar agentAddr string\n\tvar dev uint64\n\tgo func() {\n\t\ttime.Sleep(time.Millisecond * 100) // wait for child process\n\t\tatomic.StoreInt64(&lastActive, time.Now().Unix())\n\t\tfor ctx.Err() == nil {\n\t\t\tvar confName = \".config\"\n\t\t\tif !vfs.IsSpecialName(confName) {\n\t\t\t\tconfName = \".jfs\" + confName\n\t\t\t}\n\t\t\tvar confStat syscall.Stat_t\n\t\t\terr := syscall.Stat(filepath.Join(mp, confName), &confStat)\n\t\t\tino, _ := vfs.GetInternalNodeByName(confName)\n\t\t\tif err == nil && confStat.Ino == uint64(ino) {\n\t\t\t\tif dev == 0 && runtime.GOOS == \"linux\" {\n\t\t\t\t\tvar st syscall.Stat_t\n\t\t\t\t\tif err := syscall.Stat(mp, &st); err == nil && st.Ino == 1 {\n\t\t\t\t\t\tdev = uint64(st.Dev)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif pid == 0 {\n\t\t\t\t\t_, conf, err := loadConfig(mp)\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tlogger.Infof(\"watching %s, pid %d\", mp, conf.Pid)\n\t\t\t\t\t\tpid = conf.Pid\n\t\t\t\t\t\tagentAddr = conf.Port.DebugAgent\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlogger.Warnf(\"load config: %s\", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tatomic.StoreInt64(&lastActive, time.Now().Unix())\n\t\t\ttime.Sleep(time.Second * 5)\n\t\t}\n\t}()\n\tfor ctx.Err() == nil {\n\t\tnow := time.Now().Unix()\n\t\tif atomic.LoadInt64(&lastActive)+30 < now {\n\t\t\tshowThreadStack(agentAddr)\n\t\t\ttime.Sleep(time.Second * 30)\n\t\t\t// double check\n\t\t\tif atomic.LoadInt64(&lastActive)+60 < time.Now().Unix() && ctx.Err() == nil {\n\t\t\t\tlogger.Infof(\"mount point %s is not active for %s\", mp, time.Since(time.Unix(atomic.LoadInt64(&lastActive), 0)))\n\t\t\t\tshowThreadStack(agentAddr)\n\t\t\t\tkillMountProcess(pid, dev, &lastActive)\n\t\t\t\tatomic.StoreInt64(&lastActive, time.Now().Unix())\n\t\t\t\tpid = 0\n\t\t\t\tdev = 0\n\t\t\t}\n\t\t}\n\t\ttime.Sleep(time.Second * 10)\n\t}\n}\n\n// parseFuseFd checks if `mountPoint` is the special form /dev/fd/N (with N >= 0),\n// and returns N in this case. Returns -1 otherwise.\nfunc parseFuseFd(mountPoint string) (fd int) {\n\tdir, file := path.Split(mountPoint)\n\tif dir != \"/dev/fd/\" {\n\t\treturn -1\n\t}\n\tfd, err := strconv.Atoi(file)\n\tif err != nil || fd <= 0 {\n\t\treturn -1\n\t}\n\treturn fd\n}\n\nfunc checkMountpoint(name, mp, logPath string, background bool) {\n\tif parseFuseFd(mp) > 0 {\n\t\tlogger.Infof(\"\\033[92mOK\\033[0m, %s with special mount point %s\", name, mp)\n\t\treturn\n\t}\n\t_, oldConf, _ := loadConfig(mp)\n\tmountTimeOut := 10 // default 10 seconds\n\tinterval := 500 // check every 500 Millisecond\n\tif tStr, ok := os.LookupEnv(\"JFS_MOUNT_TIMEOUT\"); ok {\n\t\tif t, err := strconv.ParseInt(tStr, 10, 64); err == nil {\n\t\t\tmountTimeOut = int(t)\n\t\t} else {\n\t\t\tlogger.Errorf(\"invalid env JFS_MOUNT_TIMEOUT: %s %s\", tStr, err)\n\t\t}\n\t}\n\tfor i := 0; i < mountTimeOut*1000/interval; i++ {\n\t\ttime.Sleep(time.Duration(interval) * time.Millisecond)\n\t\tst, err := os.Stat(mp)\n\t\tif err == nil {\n\t\t\tif sys, ok := st.Sys().(*syscall.Stat_t); ok && sys.Ino == uint64(meta.RootInode) {\n\t\t\t\t// in pod, pid probably the same\n\t\t\t\tif csiCommPath == \"\" && oldConf != nil {\n\t\t\t\t\t_, newConf, _ := loadConfig(mp)\n\t\t\t\t\tif newConf == nil || newConf.Pid == oldConf.Pid {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tlogger.Infof(\"\\033[92mOK\\033[0m, %s is ready at %s\", name, mp)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\t_, _ = os.Stdout.WriteString(\".\")\n\t\t_ = os.Stdout.Sync()\n\t}\n\t_, _ = os.Stdout.WriteString(\"\\n\")\n\tmountDesc := \"mount process is not started yet\"\n\tif mountPid != 0 {\n\t\tmountDesc = fmt.Sprintf(\"tried to kill mount process %d\", mountPid)\n\t\t_ = syscall.Kill(mountPid, syscall.SIGABRT) // Kill and show stack trace\n\t}\n\tif background {\n\t\tlogger.Fatalf(\"The mount point is not ready in %d seconds (%s), please check the log (%s) or re-mount in foreground\", mountTimeOut, mountDesc, logPath)\n\t} else {\n\t\tlogger.Fatalf(\"The mount point is not ready in %d seconds (%s), exit it\", mountTimeOut, mountDesc)\n\t}\n}\n\nfunc checkSvcPort(address string) {\n\tmountTimeOut := 10\n\tinterval := 500\n\tfor i := 0; i < mountTimeOut*1000/interval; i++ {\n\t\ttime.Sleep(time.Duration(interval) * time.Millisecond)\n\t\tconn, err := net.DialTimeout(\"tcp\", address, 500*time.Millisecond)\n\t\tif err == nil {\n\t\t\t_ = conn.Close()\n\t\t\tlogger.Infof(\"\\033[92mOK\\033[0m, service is ready on %s\", address)\n\t\t\treturn\n\t\t}\n\t\t_, _ = os.Stdout.WriteString(\".\")\n\t\t_ = os.Stdout.Sync()\n\t}\n\t_, _ = os.Stdout.WriteString(\"\\n\")\n\tlogger.Fatalf(\"The service is not ready in %d seconds, please check the log or restart in foreground\", mountTimeOut)\n}\n\nfunc makeDaemonForSvc(c *cli.Context, m meta.Meta, metaUrl, listenAddr string) error {\n\tcacheDirPathToAbs(c)\n\t_ = expandPathForEmbedded(metaUrl)\n\n\tvar attrs godaemon.DaemonAttr\n\tlogfile := c.String(\"log\")\n\tattrs.OnExit = func(stage int) error {\n\t\tif stage == 0 {\n\t\t\tcheckSvcPort(listenAddr)\n\t\t}\n\t\treturn nil\n\t}\n\n\tif godaemon.Stage() == 0 {\n\t\tvar err error\n\t\tattrs.Stdout, err = os.OpenFile(logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"open log file %s: %s\", logfile, err)\n\t\t} else {\n\t\t\tlogger.Infof(\"open log file %s\", logfile)\n\t\t}\n\n\t\tconn, err := net.DialTimeout(\"tcp\", listenAddr, 500*time.Millisecond)\n\t\tif err == nil {\n\t\t\t_ = conn.Close()\n\t\t\tlogger.Fatalf(\"unable to start the server: %s is already in use\", listenAddr)\n\t\t}\n\t}\n\tif godaemon.Stage() <= 1 {\n\t\terr := m.Shutdown()\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"shutdown: %s\", err)\n\t\t}\n\t}\n\t_, _, err := godaemon.MakeDaemon(&attrs)\n\treturn err\n}\n\nfunc getDaemonStage() int {\n\treturn int(godaemon.Stage())\n}\n\nfunc fuseFlags() []cli.Flag {\n\treturn addCategories(\"FUSE\", []cli.Flag{\n\t\t&cli.BoolFlag{\n\t\t\tName: \"enable-xattr\",\n\t\t\tUsage: \"enable extended attributes (xattr)\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"enable-cap\",\n\t\t\tUsage: \"enable security.capability xattr\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"enable-selinux\",\n\t\t\tUsage: \"enable security.selinux xattr\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"enable-ioctl\",\n\t\t\tUsage: \"enable ioctl (support GETFLAGS/SETFLAGS only)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"root-squash\",\n\t\t\tUsage: \"mapping local root user (uid = 0) to another one specified as :\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"all-squash\",\n\t\t\tUsage: \"mapping all users to another one specified as :\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"prefix-internal\",\n\t\t\tUsage: \"add '.jfs' prefix to all internal files\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"non-default-permission\",\n\t\t\tUsage: \"disable `default_permissions` option, only for testing\",\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"max-fuse-io\",\n\t\t\tUsage: \"maximum size for fuse request\",\n\t\t\tValue: \"128K\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"umask\",\n\t\t\tUsage: \"umask for new files and directories in octal (overwrite the one from app)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"o\",\n\t\t\tUsage: \"other FUSE options\",\n\t\t},\n\t})\n}\n\nfunc mountFlags() []cli.Flag {\n\tselfFlags := []cli.Flag{\n\t\t&cli.BoolFlag{\n\t\t\tName: \"f\",\n\t\t\tAliases: []string{\"foreground\"},\n\t\t\tHidden: true,\n\t\t\tUsage: \"run in foreground\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"d\",\n\t\t\tAliases: []string{\"background\"},\n\t\t\tUsage: \"run in background\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"no-syslog\",\n\t\t\tUsage: \"disable syslog\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"log\",\n\t\t\tValue: path.Join(getDefaultLogDir(), \"juicefs.log\"),\n\t\t\tUsage: \"path of log file when running in background\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"force\",\n\t\t\tUsage: \"force to mount even if the mount point is already mounted by the same filesystem\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"hide-internal\",\n\t\t\tUsage: \"hide all internal files (.accesslog, .stats, etc.)\",\n\t\t},\n\t}\n\tif runtime.GOOS == \"linux\" {\n\t\tselfFlags = append(selfFlags, &cli.BoolFlag{\n\t\t\tName: \"update-fstab\",\n\t\t\tUsage: \"add / update entry in /etc/fstab, will create a symlink from /sbin/mount.juicefs to JuiceFS executable if not existing\",\n\t\t})\n\t\tselfFlags = append(selfFlags, &cli.BoolFlag{\n\t\t\tName: \"disable-transparent-hugepage\",\n\t\t\tUsage: \"disable transparent huge page to avoid latency spikes caused by kernel's memory compaction\",\n\t\t})\n\t}\n\treturn append(selfFlags, fuseFlags()...)\n}\n\nfunc disableUpdatedb() {\n\tpath := \"/etc/updatedb.conf\"\n\tfile, err := os.Open(path)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer file.Close()\n\n\t// obtain exclusive and not block flock\n\tif err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {\n\t\tif err == syscall.EAGAIN {\n\t\t\treturn\n\t\t}\n\t} else {\n\t\tdefer func() {\n\t\t\t// release flock\n\t\t\t_ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN)\n\t\t}()\n\t}\n\n\tdata, err := io.ReadAll(file)\n\tif err != nil {\n\t\treturn\n\t}\n\tfstype := \"fuse.juicefs\"\n\tif bytes.Contains(data, []byte(fstype)) {\n\t\treturn\n\t}\n\t// assume that fuse.sshfs is already in PRUNEFS\n\tknownFS := \"fuse.sshfs\"\n\tp1 := bytes.Index(data, []byte(\"PRUNEFS\"))\n\tp2 := bytes.Index(data, []byte(knownFS))\n\tif p1 > 0 && p2 > p1 {\n\t\tvar nd []byte\n\t\tnd = append(nd, data[:p2]...)\n\t\tnd = append(nd, fstype...)\n\t\tnd = append(nd, ' ')\n\t\tnd = append(nd, data[p2:]...)\n\t\terr = os.WriteFile(path, nd, 0644)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"update %s: %s\", path, err)\n\t\t} else {\n\t\t\tlogger.Infof(\"Add %s into PRUNEFS of %s\", fstype, path)\n\t\t}\n\t}\n}\n\nfunc getFuserMountVersion() string {\n\tvar version = \"0.0.0\"\n\tout, _ := exec.Command(\"fusermount\", \"-V\").CombinedOutput()\n\tps := strings.Split(string(out), \":\")\n\tif len(ps) > 1 {\n\t\treturn strings.TrimSpace(ps[1])\n\t}\n\treturn version\n}\n\nfunc setFuseOption(c *cli.Context, format *meta.Format, vfsConf *vfs.Config) {\n\trawOpts, mt, noxattr, noacl, maxWrite := genFuseOptExt(c, format)\n\toptions := vfs.FuseOptions(fuse.GenFuseOpt(vfsConf, rawOpts, mt, noxattr, noacl, maxWrite))\n\tvfsConf.FuseOpts = &options\n}\n\nfunc genFuseOpt(c *cli.Context, name string) string {\n\tfuseOpt := c.String(\"o\")\n\t// todo: remove ?\n\tprefix := os.Getenv(\"FSTAB_NAME_PREFIX\")\n\tif prefix == \"\" {\n\t\tprefix = \"JuiceFS:\"\n\t}\n\tfuseOpt += \",fsname=\" + prefix + name\n\tif c.Bool(\"allow-other\") || os.Getuid() == 0 && !strings.Contains(fuseOpt, \"allow_other\") {\n\t\tfuseOpt += \",allow_other\"\n\t}\n\tswitch runtime.GOOS {\n\tcase \"darwin\":\n\t\tfuseOpt += \",allow_recursion\"\n\tcase \"linux\":\n\t\t// nonempty has been removed since 3.0.0\n\t\tif getFuserMountVersion() < \"3.0.0\" {\n\t\t\tfuseOpt += \",nonempty\"\n\t\t}\n\t}\n\tfuseOpt = strings.TrimLeft(fuseOpt, \",\")\n\treturn fuseOpt\n}\n\nfunc prepareMp(mp string) {\n\tif csiCommPath != \"\" {\n\t\treturn\n\t}\n\tvar fi os.FileInfo\n\tvar ino uint64\n\terr := utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\tvar err error\n\t\tfi, err = os.Stat(mp)\n\t\treturn err\n\t}, time.Second*3)\n\tif !strings.Contains(mp, \":\") && err != nil {\n\t\terr2 := utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\t\treturn os.MkdirAll(mp, 0777)\n\t\t}, time.Second*3)\n\t\tif err2 != nil {\n\t\t\tif os.IsExist(err2) || strings.Contains(err2.Error(), \"timeout after 3s\") {\n\t\t\t\t// a broken mount point, umount it\n\t\t\t\tlogger.Infof(\"mountpoint %s is broken: %s, umount it\", mp, err)\n\t\t\t\t_ = doUmount(mp, true)\n\t\t\t} else {\n\t\t\t\tlogger.Fatalf(\"create %s: %s\", mp, err2)\n\t\t\t}\n\t\t}\n\t} else if err == nil {\n\t\tino, _ = utils.GetFileInode(mp)\n\t\tif ino <= uint64(meta.RootInode) && fi.Size() == 0 {\n\t\t\t// a broken mount point, umount it\n\t\t\tlogger.Infof(\"mountpoint %s is broken (ino=%d, size=%d), umount it\", mp, ino, fi.Size())\n\t\t\t_ = doUmount(mp, true)\n\t\t}\n\t}\n\n\tif os.Getuid() == 0 {\n\t\treturn\n\t}\n\tif ino == uint64(meta.RootInode) {\n\t\treturn\n\t}\n\tswitch runtime.GOOS {\n\tcase \"darwin\":\n\t\tif fi, err := os.Stat(mp); err == nil {\n\t\t\tif st, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\t\t\tif st.Uid != uint32(os.Getuid()) {\n\t\t\t\t\tlogger.Fatalf(\"current user should own %s\", mp)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\tcase \"linux\":\n\t\tf, err := os.CreateTemp(mp, \".test\")\n\t\tif err != nil && (os.IsPermission(err) || errors.Is(err, syscall.EPERM) || errors.Is(err, syscall.EROFS)) {\n\t\t\tlogger.Fatalf(\"Do not have write permission on %s\", mp)\n\t\t} else if f != nil {\n\t\t\t_ = f.Close()\n\t\t\t_ = os.Remove(f.Name())\n\t\t}\n\t}\n}\n\nfunc genFuseOptExt(c *cli.Context, format *meta.Format) (fuseOpt string, mt int, noxattr, noacl bool, maxWrite int) {\n\tenableXattr := c.Bool(\"enable-xattr\")\n\tif format.EnableACL {\n\t\tenableXattr = true\n\t}\n\treturn genFuseOpt(c, format.Name), 1, !enableXattr, !format.EnableACL, int(utils.ParseBytes(c, \"max-fuse-io\", 'B'))\n}\n\nfunc shutdownGraceful(mp string) {\n\t_, conf, err := loadConfig(mp)\n\tif err != nil {\n\t\tlogger.Warnf(\"load config from %s: %s\", mp, err)\n\t\treturn\n\t}\n\tfuseFd, fuseSetting = getFuseFd(conf.CommPath)\n\tfor i := 0; i < 100 && fuseFd == 0; i++ {\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\tfuseFd, fuseSetting = getFuseFd(conf.CommPath)\n\t}\n\tif fuseFd == 0 {\n\t\tlogger.Warnf(\"fail to recv FUSE fd from %s\", conf.CommPath)\n\t\treturn\n\t}\n\tfor i := 0; i < 600; i++ {\n\t\tif err := syscall.Kill(conf.Pid, syscall.SIGHUP); err != nil {\n\t\t\tos.Setenv(\"_FUSE_STATE_PATH\", conf.StatePath)\n\t\t\tos.Setenv(\"_JFS_META_SID\", strconv.Itoa(int(conf.Meta.Sid)))\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(time.Millisecond * 100)\n\t}\n\tlogger.Infof(\"mount point %s is busy, stop upgrade, mount on top of it\", mp)\n\terr = sendFuseFd(conf.CommPath, fuseSetting, fuseFd)\n\tif err != nil {\n\t\tlogger.Warnf(\"send FUSE fd: %s\", err)\n\t}\n\t_ = syscall.Close(fuseFd)\n\tfuseFd = 0\n\tfuseSetting = []byte(\"FUSE\")\n}\n\nfunc canShutdownGracefully(mp string, newConf *vfs.Config) bool {\n\tif csiCommPath != \"\" {\n\t\treturn false\n\t}\n\tvar ino uint64\n\tvar err error\n\terr = utils.WithTimeout(context.TODO(), func(context.Context) error {\n\t\tino, err = utils.GetFileInode(mp)\n\t\treturn err\n\t}, time.Second*3)\n\tif err != nil {\n\t\tlogger.Warnf(\"get inode of %s: %s\", mp, err)\n\t\t_ = doUmount(mp, true)\n\t\treturn false\n\t} else if ino != 1 {\n\t\treturn false\n\t}\n\t_, oldConf, err := loadConfig(mp)\n\tif err != nil {\n\t\tif !os.IsNotExist(err) {\n\t\t\tlogger.Warnf(\"load config: %s\", err)\n\t\t}\n\t\treturn false\n\t}\n\tif oldConf.Pid == 0 || oldConf.CommPath == \"\" {\n\t\tlogger.Infof(\"mount point %s is not ready for upgrade, mount on top of it\", mp)\n\t\treturn false\n\t}\n\tif oldConf.Format.Name != newConf.Format.Name {\n\t\tlogger.Infof(\"different volume %s != %s, mount on top of it\", oldConf.Format.Name, newConf.Format.Name)\n\t\treturn false\n\t}\n\toldVersion := version.Parse(oldConf.Version)\n\tif ret, _ := version.CompareVersions(oldVersion, version.Parse(\"1.2.0\")); ret <= 0 {\n\t\toldConf.FuseOpts.MaxWrite = 128 * 1024\n\t}\n\tif oldConf.FuseOpts != nil && !reflect.DeepEqual(oldConf.FuseOpts.StripOptions(), newConf.FuseOpts.StripOptions()) {\n\t\tlogger.Infof(\"different options, mount on top of it: %v != %v\", oldConf.FuseOpts.StripOptions(), newConf.FuseOpts.StripOptions())\n\t\treturn false\n\t}\n\tif oldConf.FuseOpts.DisableXAttrs && !newConf.FuseOpts.DisableXAttrs {\n\t\tlogger.Infof(\"Xattr is enabled, mount on top of it\")\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc absPath(d string) string {\n\tif strings.HasPrefix(d, \"/\") {\n\t\treturn d\n\t}\n\tif strings.HasPrefix(d, \"~/\") {\n\t\tif h, err := os.UserHomeDir(); err == nil {\n\t\t\treturn filepath.Join(h, d[1:])\n\t\t} else {\n\t\t\tlogger.Fatalf(\"Expand user home dir of %s: %s\", d, err)\n\t\t}\n\t}\n\td, err := filepath.Abs(d)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Expand %s: %s\", d, err)\n\t}\n\treturn d\n}\n\nfunc buildBoolFlagsMap(c *cli.Context) map[string]bool {\n\tboolFlags := make(map[string]bool)\n\taddBoolFlags := func(flags []cli.Flag) {\n\t\tfor _, flag := range flags {\n\t\t\tif _, ok := flag.(*cli.BoolFlag); ok {\n\t\t\t\tfor _, name := range flag.Names() {\n\t\t\t\t\tboolFlags[name] = true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif c.App != nil {\n\t\taddBoolFlags(c.App.Flags)\n\t}\n\tif c.Command != nil {\n\t\taddBoolFlags(c.Command.Flags)\n\t}\n\treturn boolFlags\n}\n\nfunc tellFstabOptions(c *cli.Context) string {\n\topts := []string{\"_netdev,nofail\"}\n\tboolFlags := buildBoolFlagsMap(c)\n\tfor _, s := range os.Args[2:] {\n\t\tif !strings.HasPrefix(s, \"-\") {\n\t\t\tcontinue\n\t\t}\n\t\ts = strings.TrimLeft(s, \"-\")\n\t\ts = strings.Split(s, \"=\")[0]\n\t\tif !c.IsSet(s) || s == \"update-fstab\" || s == \"background\" || s == \"d\" {\n\t\t\tcontinue\n\t\t}\n\t\tif s == \"o\" {\n\t\t\topts = append(opts, c.String(s))\n\t\t} else if boolFlags[s] && c.Bool(s) {\n\t\t\topts = append(opts, s)\n\t\t} else if s == \"cache-dir\" {\n\t\t\tvar dirString string\n\t\t\tif c.String(s) == \"memory\" {\n\t\t\t\tdirString = \"memory\"\n\t\t\t} else {\n\t\t\t\tdirs := utils.SplitDir(c.String(s))\n\t\t\t\tdirString = strings.Join(relPathToAbs(dirs), string(os.PathListSeparator))\n\t\t\t}\n\t\t\topts = append(opts, fmt.Sprintf(\"%s=%s\", s, dirString))\n\t\t} else {\n\t\t\topts = append(opts, fmt.Sprintf(\"%s=%s\", s, c.Generic(s)))\n\t\t}\n\t}\n\tsort.Strings(opts)\n\treturn strings.Join(opts, \",\")\n}\n\nfunc updateFstab(c *cli.Context) error {\n\taddr := expandPathForEmbedded(c.Args().Get(0))\n\tmp := absPath(c.Args().Get(1))\n\tvar fstab = \"/etc/fstab\"\n\n\tf, err := os.Open(fstab)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer f.Close()\n\tentryIndex := -1\n\tvar lines []string\n\tscanner := bufio.NewScanner(f)\n\tfor scanner.Scan() {\n\t\tline := scanner.Text()\n\t\tfields := strings.Fields(line)\n\t\tif len(fields) >= 6 && fields[2] == \"juicefs\" && fields[0] == addr && fields[1] == mp {\n\t\t\tentryIndex = len(lines)\n\t\t}\n\t\tlines = append(lines, line)\n\t}\n\tif err = scanner.Err(); err != nil {\n\t\treturn err\n\t}\n\topts := tellFstabOptions(c)\n\tentry := fmt.Sprintf(\"%s %s juicefs %s 0 0\", addr, mp, opts)\n\tif entryIndex >= 0 {\n\t\tif entry == lines[entryIndex] {\n\t\t\treturn nil\n\t\t}\n\t\tlines[entryIndex] = entry\n\t} else {\n\t\tlines = append(lines, entry)\n\t}\n\ttempFstab := fstab + \".tmp\"\n\ttmpf, err := os.OpenFile(tempFstab, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer tmpf.Close()\n\tif _, err := tmpf.WriteString(strings.Join(lines, \"\\n\") + \"\\n\"); err != nil {\n\t\t_ = os.Remove(tempFstab)\n\t\treturn err\n\t}\n\treturn os.Rename(tempFstab, fstab)\n}\n\nfunc tryToInstallMountExec() error {\n\tif _, err := os.Stat(\"/sbin/mount.juicefs\"); err == nil {\n\t\treturn nil\n\t}\n\tsrc, err := os.Executable()\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn os.Symlink(src, \"/sbin/mount.juicefs\")\n}\n\nfunc fixCacheDirs(c *cli.Context) {\n\tcd := c.String(\"cache-dir\")\n\tif cd == \"memory\" || strings.HasPrefix(cd, \"/\") {\n\t\treturn\n\t}\n\tds := utils.SplitDir(cd)\n\tfor i, d := range ds {\n\t\tds[i] = absPath(d)\n\t}\n\tfor i, a := range os.Args {\n\t\tif i > 0 && os.Args[i-1] == \"--cache-dir\" && a == cd || a == \"--cache-dir=\"+cd {\n\t\t\tos.Args[i] = a[:len(a)-len(cd)] + strings.Join(ds, string(os.PathListSeparator))\n\t\t}\n\t}\n}\n\nfunc makeDaemon(c *cli.Context, conf *vfs.Config) error {\n\tvar attrs godaemon.DaemonAttr\n\tlogfile := c.String(\"log\")\n\tmp := conf.Meta.MountPoint\n\tattrs.OnExit = func(stage int) error {\n\t\tif stage == 0 {\n\t\t\tcheckMountpoint(conf.Format.Name, mp, logfile, true)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// the current dir will be changed to root in daemon,\n\t// so the mount point has to be an absolute path.\n\tif godaemon.Stage() == 0 {\n\t\tmp := c.Args().Get(1)\n\t\tamp, err := filepath.Abs(mp)\n\t\tif err == nil && amp != mp {\n\t\t\tfor i := len(os.Args) - 1; i > 2; i-- {\n\t\t\t\tif os.Args[i] == mp {\n\t\t\t\t\t// FIXME: it could be other options\n\t\t\t\t\tos.Args[i] = amp\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfixCacheDirs(c)\n\n\t\t_ = os.MkdirAll(filepath.Dir(logfile), 0755)\n\t\tattrs.Stdout, err = os.OpenFile(logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"open log file %s: %s\", logfile, err)\n\t\t}\n\t}\n\t_, _, err := godaemon.MakeDaemon(&attrs)\n\treturn err\n}\n\nfunc increaseRlimit() {\n\tvar n uint64 = 100000\n\terr := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &syscall.Rlimit{Max: n, Cur: n})\n\tfor err != nil && n > 1024 {\n\t\tn = n * 2 / 3\n\t\terr = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &syscall.Rlimit{Max: n, Cur: n})\n\t}\n\tif err != nil {\n\t\tlogger.Warnf(\"setrlimit to %d: %s\", n, err)\n\t}\n}\n\nfunc installHandler(m meta.Meta, mp string, v *vfs.VFS, blob object.ObjectStorage) {\n\t// Go will catch all the signals\n\tsignal.Ignore(syscall.SIGPIPE)\n\tsignalChan := make(chan os.Signal, 10)\n\tsignal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)\n\tgo func() {\n\t\tfor {\n\t\t\tsig := <-signalChan\n\t\t\tlogger.Infof(\"Received signal %s, exiting...\", sig.String())\n\t\t\tif sig == syscall.SIGHUP {\n\t\t\t\tpath := fmt.Sprintf(\"/tmp/state%d.json\", os.Getppid())\n\t\t\t\tif err := v.FlushAll(\"\"); err == nil {\n\t\t\t\t\tfuse.Shutdown()\n\t\t\t\t\terr = v.FlushAll(path)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlogger.Fatalf(\"flush buffered data failed: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t\tm.FlushSession()\n\t\t\t\t\tobject.Shutdown(blob)\n\t\t\t\t\tlogger.Warnf(\"exit with code 1\")\n\t\t\t\t\tos.Exit(1)\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Warnf(\"flush buffered data failed: %s, don't restart\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tgo func() {\n\t\t\t\ttime.Sleep(time.Second * 30)\n\t\t\t\tif err := v.FlushAll(\"\"); err != nil {\n\t\t\t\t\tlogger.Errorf(\"flush all: %s\", err)\n\t\t\t\t}\n\t\t\t\tlogger.Errorf(\"exit after receiving signal %s, but umount does not finish in 30 seconds, force exit\", sig)\n\t\t\t\tos.Exit(meta.UmountCode)\n\t\t\t}()\n\t\t\tgo func() { _ = doUmount(mp, true) }()\n\t\t}\n\t}()\n}\nfunc launchMount(c *cli.Context, mp string, conf *vfs.Config) error {\n\tincreaseRlimit()\n\tutils.AdjustOOMKiller(-1000)\n\tutils.SetIOFlusher()\n\n\tif c.Bool(\"disable-transparent-hugepage\") {\n\t\tutils.DisableTHP()\n\t}\n\n\tif canShutdownGracefully(mp, conf) {\n\t\tshutdownGraceful(mp)\n\t}\n\tos.Setenv(\"_FUSE_FD_COMM\", serverAddress)\n\tserveFuseFD(serverAddress)\n\tdefer os.Remove(serverAddress)\n\n\tpath, err := os.Executable()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"find executable: %s\", err)\n\t}\n\tstart := time.Now()\n\tfor attempt := 0; ; attempt++ {\n\t\tif attempt == 3 && time.Since(start) < time.Second*10 {\n\t\t\treturn fmt.Errorf(\"fail 3 times in %s, give up\", time.Since(start))\n\t\t}\n\t\t// For volcengine VKE serverless container, no umount before mount when\n\t\t// `JFS_NO_UMOUNT` environment provided\n\t\tnoUmount := os.Getenv(\"JFS_NO_UMOUNT\")\n\t\tif fuseFd == 0 && (attempt > 0 || noUmount == \"0\") {\n\t\t\t_ = doUmount(mp, true)\n\t\t}\n\t\tif runtime.GOOS == \"linux\" {\n\t\t\tif !utils.Exists(serverAddress) {\n\t\t\t\tserveFuseFD(serverAddress)\n\t\t\t}\n\t\t}\n\n\t\tmountPid = 0\n\t\tcmd := exec.Command(path, os.Args[1:]...)\n\t\tcmd.Stdin = os.Stdin\n\t\tcmd.Stdout = os.Stdout\n\t\tcmd.Stderr = os.Stderr\n\t\terr = cmd.Start()\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"start process %s: %s\", path, err)\n\t\t\ttime.Sleep(time.Second)\n\t\t\tcontinue\n\t\t}\n\t\tos.Unsetenv(\"_FUSE_STATE_PATH\")\n\t\tmountPid = cmd.Process.Pid\n\n\t\tnotInCSI := os.Getenv(\"JFS_SUPER_COMM\") == \"\"\n\t\tsignalChan := make(chan os.Signal, 10)\n\t\tif notInCSI {\n\t\t\tsignal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)\n\t\t\tgo func() {\n\t\t\t\tfor {\n\t\t\t\t\tsig := <-signalChan\n\t\t\t\t\tif sig == nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tlogger.Infof(\"received signal %s, propagating to child process %d...\", sig.String(), mountPid)\n\t\t\t\t\tif err := cmd.Process.Signal(sig); err != nil && !errors.Is(err, os.ErrProcessDone) {\n\t\t\t\t\t\tlogger.Errorf(\"send signal %s to %d: %s\", sig.String(), mountPid, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\n\t\tctx, cancel := context.WithCancel(context.TODO())\n\t\tgo watchdog(ctx, mp)\n\t\terr = cmd.Wait()\n\t\tcancel()\n\t\tif notInCSI {\n\t\t\tsignal.Stop(signalChan)\n\t\t}\n\t\tclose(signalChan)\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t} else {\n\t\t\tvar exitError *exec.ExitError\n\t\t\tif ok := errors.As(err, &exitError); ok {\n\t\t\t\tif waitStatus, ok := exitError.Sys().(syscall.WaitStatus); ok && waitStatus.ExitStatus() == meta.UmountCode {\n\t\t\t\t\tlogger.Errorf(\"received umount exit code\")\n\t\t\t\t\t_ = doUmount(mp, true)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\tif fuseFd < 0 {\n\t\t\t\tlogger.Info(\"transfer FUSE session to others\")\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tlogger.Errorf(\"mount process %d: %s, will restart in 1 second\", mountPid, err)\n\t\t\ttime.Sleep(time.Second)\n\t\t}\n\t}\n}\n\nfunc getNobodyUIDGID() (uint32, uint32) {\n\tvar uid, gid uint32 = 65534, 65534\n\tif u, err := user.Lookup(\"nobody\"); err == nil {\n\t\tnobody, err := strconv.ParseUint(u.Uid, 10, 32)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"invalid uid: %s\", u.Uid)\n\t\t}\n\t\tuid = uint32(nobody)\n\t}\n\tif g, err := user.LookupGroup(\"nogroup\"); err == nil {\n\t\tnogroup, err := strconv.ParseUint(g.Gid, 10, 32)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"invalid gid: %s\", g.Gid)\n\t\t}\n\t\tgid = uint32(nogroup)\n\t}\n\treturn uid, gid\n}\n\nfunc parseUIDGID(input string, defaultUid uint32, defaultGid uint32) (uint32, uint32) {\n\tss := strings.SplitN(strings.TrimSpace(input), \":\", 2)\n\tuid, gid := defaultUid, defaultGid\n\tif ss[0] != \"\" {\n\t\tu, err := strconv.ParseUint(ss[0], 10, 32)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"invalid uid: %s\", ss[0])\n\t\t}\n\t\tuid = uint32(u)\n\t\tif uid == 0 {\n\t\t\tlogger.Warnf(\"Can't map uid as 0, use %d instead\", defaultUid)\n\t\t\tuid = defaultUid\n\t\t}\n\t}\n\tif len(ss) == 2 && ss[1] != \"\" {\n\t\tg, err := strconv.ParseUint(ss[1], 10, 32)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"invalid gid: %s\", ss[1])\n\t\t}\n\t\tgid = uint32(g)\n\t\tif gid == 0 {\n\t\t\tlogger.Warnf(\"Can't map gid as 0, use %d instead\", defaultGid)\n\t\t\tgid = defaultGid\n\t\t}\n\t}\n\treturn uid, gid\n}\n\nfunc mountMain(v *vfs.VFS, c *cli.Context) {\n\tif os.Getuid() == 0 {\n\t\tdisableUpdatedb()\n\t}\n\tconf := v.Conf\n\tconf.AttrTimeout = utils.Duration(c.String(\"attr-cache\"))\n\tconf.EntryTimeout = utils.Duration(c.String(\"entry-cache\"))\n\tconf.DirEntryTimeout = utils.Duration(c.String(\"dir-entry-cache\"))\n\tconf.NegEntryTimeout = utils.Duration(c.String(\"negative-entry-cache\"))\n\tconf.ReaddirCache = c.Bool(\"readdir-cache\")\n\tmajor, minor := utils.GetKernelVersion()\n\tif conf.ReaddirCache {\n\t\tif conf.AttrTimeout == 0 {\n\t\t\tlogger.Warnf(\"readdir-cache is enabled without attr-cache, it's performance may be affected\")\n\t\t}\n\t\tif major < 4 || (major == 4 && minor < 20) {\n\t\t\tlogger.Warnf(\"readdir-cache requires kernel version 4.20 or higher, current version: %d.%d\", major, minor)\n\t\t}\n\t\tif conf.Meta.SkipDirMtime > 0 {\n\t\t\tlogger.Warnf(\"When both readdir-cache and skip-dir-mtime are enabled, ignoring mtime may disable readdir refreshes on other nodes\")\n\t\t}\n\t}\n\tif conf.NegEntryTimeout > 0 && (major < 5 || (major == 5 && minor < 11)) {\n\t\tlogger.Warnf(\"On kernel versions below 5.11 (current: %d.%d), negative-entry-cache may cause concurrent check-then-create operations (e.g. mkdir -p) to fail in a distributed environment\", major, minor)\n\t}\n\tconf.NonDefaultPermission = c.Bool(\"non-default-permission\")\n\trootSquash := c.String(\"root-squash\")\n\tallSquash := c.String(\"all-squash\")\n\tif allSquash != \"\" || rootSquash != \"\" {\n\t\tnobodyUid, nobodyGid := getNobodyUIDGID()\n\t\t// all-squash takes precedence over root-squash\n\t\tif allSquash != \"\" {\n\t\t\tconf.NonDefaultPermission = true // disable kernel permission check\n\t\t\tuid, gid := parseUIDGID(allSquash, nobodyUid, nobodyGid)\n\t\t\tconf.AllSquash = &vfs.AnonymousAccount{Uid: uid, Gid: gid}\n\t\t\tlogger.Infof(\"Map all uid/gid to %d/%d by setting all-squash\", uid, gid)\n\t\t} else { // rootSquash != \"\"\n\t\t\tuid, gid := parseUIDGID(rootSquash, nobodyUid, nobodyGid)\n\t\t\tconf.RootSquash = &vfs.AnonymousAccount{Uid: uid, Gid: gid}\n\t\t\tlogger.Infof(\"Map root uid/gid 0 to %d/%d by setting root-squash\", uid, gid)\n\t\t}\n\t}\n\tlogger.Infof(\"Mounting volume %s at %s ...\", conf.Format.Name, conf.Meta.MountPoint)\n\terr := fuse.Serve(v, c.String(\"o\"), c.Bool(\"enable-xattr\"), c.Bool(\"enable-ioctl\"))\n\tif err != nil {\n\t\tlogger.Fatalf(\"fuse: %s\", err)\n\t}\n}\n\n" }, { "path": "cmd/mount_windows.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strconv\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/juicedata/juicefs/pkg/winfsp\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc mountFlags() []cli.Flag {\n\treturn []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"o\",\n\t\t\tUsage: \"other FUSE options\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"log\",\n\t\t\tValue: filepath.Join(getDefaultLogDir(), \"juicefs.log\"),\n\t\t\tUsage: \"path of log file when running in background\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"fuse-access-log\",\n\t\t\tAliases: []string{\"fuse-trace-log\"},\n\t\t\tUsage: \"Fuse Layer access log file\",\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"fuse-access-log-rotate-count\",\n\t\t\tUsage: \"Fuse Layer access log file rotate count\",\n\t\t\tValue: 7,\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"readdir-batch-size\",\n\t\t\tUsage: \"readdir batch size\",\n\t\t\tValue: 1000,\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"alias\",\n\t\t\tUsage: \"volume alias, useful for mounting a volume multiple times on the same machine\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"winfsp-dbg-log\",\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"as-local-volume\",\n\t\t\tUsage: \"If mount as a local volume, supports mounting to a path.\",\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"flush-on-cleanup\",\n\t\t\tUsage: \"When enabled, Will instruct the WinFsp to call Flush() when a file handle is closing (MJ_IRP_CLEANUP). Requires the dev branch of WinFsp or version that GREATER than 2.1.25156.\",\n\t\t\tValue: true,\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"as-root\",\n\t\t\tUsage: \"Access files as administrator\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"delay-close\",\n\t\t\tUsage: \"delay file closing duration\",\n\t\t\tValue: \"0s\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"d\",\n\t\t\tAliases: []string{\"background\"},\n\t\t\tUsage: \"run in background(Windows: as a system service. support ONLY 1 volume mounting at the same time)\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"show-dot-files\",\n\t\t\tUsage: \"If set, dot files will not be treated as hidden files\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"winfsp-threads\",\n\t\t\tUsage: \"WinFsp threads count option, Default is min(cpu core * 2, 16)\",\n\t\t\tValue: min(runtime.NumCPU()*2, 16),\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"case-sensitive\",\n\t\t\tUsage: \"If set, the file system will be case sensitive\",\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"report-case\",\n\t\t\tUsage: \"If set, juicefs will report the correct case of a file path for a case-insensitive filesystem. (May incur a performance lost)\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"admin-as-root\",\n\t\t\tUsage: \"If we treat the Windows build-in user 'Administrator' as the root user on Linux. Default true.\",\n\t\t\tValue: true,\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"create-perm\",\n\t\t\tUsage: \"When creating files or directories, this will overwrite the permission parameters if set. example: 0755. Default is empty.\",\n\t\t\tValue: \"\",\n\t\t\tAction: func(c *cli.Context, v string) error {\n\t\t\t\tif v != \"\" {\n\t\t\t\t\tif p, err := strconv.ParseUint(v, 8, 32); err != nil || p > 0o777 {\n\t\t\t\t\t\treturn cli.Exit(\"create-perm must be a valid octal number between 0000 and 0777\", 1)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc makeDaemon(c *cli.Context, conf *vfs.Config) error {\n\tlogPath := c.String(\"log\")\n\tif logPath != \"\" {\n\t\tif !filepath.IsAbs(logPath) {\n\t\t\treturn cli.Exit(\"log path must be an absolute path\", 1)\n\t\t}\n\t\tif err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil {\n\t\t\treturn cli.Exit(err, 1)\n\t\t}\n\t}\n\n\tdefaultCacheDir := getDefaultCacheDir()\n\n\treturn winfsp.RunAsSystemService(conf.Format.Name, c.Args().Get(1), logPath, defaultCacheDir, c)\n}\n\nfunc makeDaemonForSvc(c *cli.Context, m meta.Meta, metaUrl, listenAddr string) error {\n\tlogger.Warnf(\"Cannot run in background in Windows.\")\n\treturn nil\n}\n\nfunc getDaemonStage() int {\n\treturn 0\n}\n\nfunc mountMain(v *vfs.VFS, c *cli.Context) {\n\tv.Conf.AccessLog = c.String(\"access-log\")\n\tv.Conf.AttrTimeout = utils.Duration(c.String(\"attr-cache\"))\n\tv.Conf.EntryTimeout = utils.Duration(c.String(\"entry-cache\"))\n\tv.Conf.DirEntryTimeout = utils.Duration(c.String(\"dir-entry-cache\"))\n\tv.Conf.Mountpoint = c.Args().Get(1)\n\n\tdelayCloseTime := utils.Duration(c.String(\"delay-close\"))\n\n\terr := winfsp.Serve(v, c.String(\"o\"),\n\t\tc.Bool(\"as-root\"), int(delayCloseTime.Seconds()), c.Bool(\"show-dot-files\"),\n\t\tc.Int(\"winfsp-threads\"), c.Bool(\"case-sensitive\"), c.Bool(\"report-case\"), c)\n\n\tif err != nil {\n\t\tlogger.Errorf(\"Failed to mount volume %s: %s\", v.Conf.Format.Name, err)\n\t}\n}\n\nfunc checkMountpoint(name, mp, logPath string, background bool) {}\n\nfunc prepareMp(mp string) {}\n\nfunc setFuseOption(c *cli.Context, format *meta.Format, vfsConf *vfs.Config) {}\n\nfunc launchMount(c *cli.Context, mp string, conf *vfs.Config) error { return nil }\n\nfunc installHandler(m meta.Meta, mp string, v *vfs.VFS, blob object.ObjectStorage) {}\n\nfunc tryToInstallMountExec() error { return nil }\n\nfunc updateFstab(c *cli.Context) error { return nil }\n" }, { "path": "cmd/objbench.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"os/user\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\tosync \"github.com/juicedata/juicefs/pkg/sync\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n\t\"golang.org/x/sync/errgroup\"\n)\n\nfunc cmdObjbench() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"objbench\",\n\t\tAction: objbench,\n\t\tCategory: \"TOOL\",\n\t\tUsage: \"Run benchmarks on an object storage\",\n\t\tArgsUsage: \"ENDPOINT\",\n\t\tDescription: `\nRun basic benchmarks on the target object storage to test if it works as expected.\n\nExamples:\n# Run benchmarks on S3\n$ ACCESS_KEY=myAccessKey SECRET_KEY=mySecretKey juicefs objbench --storage s3 https://mybucket.s3.us-east-2.amazonaws.com -p 6\n# Run benchmakks on JuiceFS\n$ juicefs objbench --storage jfs redis://localhost/1\n\nDetails: https://juicefs.com/docs/community/performance_evaluation_guide#juicefs-objbench`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"storage\",\n\t\t\t\tValue: \"file\",\n\t\t\t\tUsage: \"object storage type (e.g. s3, gs, oss, cos)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"access-key\",\n\t\t\t\tUsage: \"access key for object storage (env ACCESS_KEY)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"secret-key\",\n\t\t\t\tUsage: \"secret key for object storage (env SECRET_KEY)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"session-token\",\n\t\t\t\tUsage: \"session token for object storage\",\n\t\t\t},\n\t\t\t&cli.IntFlag{\n\t\t\t\tName: \"shards\",\n\t\t\t\tUsage: \"store the blocks into N buckets by hash of key\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"block-size\",\n\t\t\t\tValue: \"4M\",\n\t\t\t\tUsage: \"size of each IO block in KiB\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"big-object-size\",\n\t\t\t\tValue: \"1G\",\n\t\t\t\tUsage: \"size of each big object in MiB\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"small-object-size\",\n\t\t\t\tValue: \"128K\",\n\t\t\t\tUsage: \"size of each small object in KiB\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"small-objects\",\n\t\t\t\tValue: 100,\n\t\t\t\tUsage: \"number of small object\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"skip-functional-tests\",\n\t\t\t\tUsage: \"skip functional tests\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tValue: 4,\n\t\t\t\tUsage: \"number of concurrent threads\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"storage-class\",\n\t\t\t\tAliases: []string{\"sc\"},\n\t\t\t\tUsage: \"storage class for object storage, e.g. Standard, IA\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nvar (\n\tnspt = \"not support\"\n\tpass = \"pass\"\n\tskipped = \"skipped\"\n\tfailed = \"failed\"\n)\n\ntype warning error\n\nvar groupName string\nvar listCount, bCount, sCount int\n\nfunc objbench(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tfor _, name := range []string{\"small-objects\", \"threads\"} {\n\t\tif ctx.Uint(name) == 0 {\n\t\t\tlogger.Fatalf(\"%s should not be set to zero\", name)\n\t\t}\n\t}\n\tbSize := int(utils.ParseBytes(ctx, \"block-size\", 'K'))\n\tfsize := int(utils.ParseBytes(ctx, \"big-object-size\", 'M'))\n\tsmallBSize := int(utils.ParseBytes(ctx, \"small-object-size\", 'K'))\n\tif bSize == 0 || fsize == 0 || smallBSize == 0 {\n\t\tlogger.Fatalf(\"block-size, big-object-size and small-object-size should not be zero\")\n\t}\n\tak, sk, token := ctx.String(\"access-key\"), ctx.String(\"secret-key\"), ctx.String(\"session-token\")\n\tif ak == \"\" {\n\t\tak = os.Getenv(\"ACCESS_KEY\")\n\t}\n\tif sk == \"\" {\n\t\tsk = os.Getenv(\"SECRET_KEY\")\n\t}\n\tif token == \"\" {\n\t\ttoken = os.Getenv(\"SESSION_TOKEN\")\n\t}\n\tendpoint := ctx.Args().First()\n\tstorageType := strings.ToLower(ctx.String(\"storage\"))\n\tif storageType == \"file\" {\n\t\tif strings.Contains(endpoint, \"://\") {\n\t\t\twarn(\"The bucket \\\"%s\\\" doesn't look like a file path.\", endpoint)\n\t\t\twarn(\"Did you forget to specify the `--storage `?\")\n\t\t\tif !userConfirmed() {\n\t\t\t\treturn errors.New(\"Aborted\")\n\t\t\t}\n\t\t}\n\t\tvar err error\n\t\tif endpoint, err = filepath.Abs(endpoint); err != nil {\n\t\t\tlogger.Fatalf(\"invalid path: %s\", err)\n\t\t}\n\t}\n\tvar blobOrigin object.ObjectStorage\n\tvar err error\n\tshards := ctx.Int(\"shards\")\n\tif shards > 1 {\n\t\tblobOrigin, err = object.NewSharded(storageType, endpoint, ak, sk, token, shards)\n\t} else {\n\t\tblobOrigin, err = object.CreateStorage(storageType, endpoint, ak, sk, token)\n\t}\n\tif err != nil {\n\t\tlogger.Fatalf(\"create storage failed: %v\", err)\n\t}\n\n\tprefix := fmt.Sprintf(\"__juicefs_benchmark_%d__/\", time.Now().UnixNano())\n\tblob := object.WithPrefix(blobOrigin, prefix)\n\tstorageClass := ctx.String(\"storage-class\")\n\tif os, ok := blob.(object.SupportStorageClass); ok && storageClass != \"\" {\n\t\tif err := os.SetStorageClass(storageClass); err != nil {\n\t\t\tlogger.Fatalf(\"set storageClass %s failed: %v\", storageClass, err)\n\t\t}\n\t}\n\tdefer func() {\n\t\t_ = blobOrigin.Delete(ctx.Context, prefix)\n\t}()\n\tbCount = int(math.Ceil(float64(fsize) / float64(bSize)))\n\tsCount = int(ctx.Uint(\"small-objects\"))\n\tlistCount = sCount + bCount\n\tif listCount > 1000 {\n\t\tlistCount = 1000\n\t}\n\tthreads := int(ctx.Uint(\"threads\"))\n\tif threads > bCount || threads > sCount {\n\t\tthreads = bCount\n\t\tif threads > sCount {\n\t\t\tthreads = sCount\n\t\t}\n\t\tlogger.Warnf(\"The number of threads was set too large and has been reduced to %d\", threads)\n\t}\n\tcolorful := utils.SupportANSIColor(os.Stdout.Fd())\n\tprogress := utils.NewProgress(false)\n\tif colorful {\n\t\tnspt = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, YELLOW, nspt, RESET_SEQ)\n\t\tskipped = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, YELLOW, skipped, RESET_SEQ)\n\t\tpass = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, GREEN, pass, RESET_SEQ)\n\t\tfailed = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, RED, failed, RESET_SEQ)\n\t}\n\tif runtime.GOOS != \"windows\" {\n\t\tnobody, err := user.Lookup(\"nobody\")\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"lookup nobody user failed: %v\", err)\n\t\t} else {\n\t\t\tgroup, err := user.LookupGroupId(nobody.Gid)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Fatalf(\"lookup nobody's group failed: %v\", err)\n\t\t\t}\n\t\t\tgroupName = group.Name\n\t\t}\n\t}\n\tif ctx.Bool(\"skip-functional-tests\") {\n\t\tif err := blob.Create(ctx.Context); err != nil {\n\t\t\treturn fmt.Errorf(\"can't create bucket: %s\", err)\n\t\t}\n\t} else {\n\t\tvar result [][]string\n\t\tresult = append(result, []string{\"CATEGORY\", \"TEST\", \"RESULT\"})\n\t\tfmt.Println(\"Start Functional Testing ...\")\n\t\tfunctionalTesting(ctx.Context, blob, &result, colorful)\n\t\tprintResult(result, -1, colorful)\n\t\tfmt.Println()\n\t}\n\tfmt.Println(\"Start Performance Testing ...\")\n\tvar pResult [][]string\n\tpResult = append(pResult, []string{\"ITEM\", \"VALUE\", \"COST\"})\n\n\tapis := []apiInfo{\n\t\t{\n\t\t\tname: \"smallput\",\n\t\t\tcount: sCount,\n\t\t\ttitle: \"put small objects\",\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"smallput\", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"smallget\",\n\t\t\tcount: sCount,\n\t\t\ttitle: \"get small objects\",\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"smallget\", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"put\",\n\t\t\tcount: bCount,\n\t\t\ttitle: \"upload objects\",\n\t\t\tstartKey: sCount,\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"put\", float64(bSize)/1024/1024*float64(bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful)\n\t\t\t\t\tline[1] += \" MiB/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"get\",\n\t\t\tcount: bCount,\n\t\t\ttitle: \"download objects\",\n\t\t\tstartKey: sCount,\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"get\", float64(bSize)/1024/1024*float64(bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful)\n\t\t\t\t\tline[1] += \" MiB/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"list\",\n\t\t\ttitle: \"list objects\",\n\t\t\tcount: threads,\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"list\", float64(listCount)*float64(threads)/cost, cost*1000, 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += fmt.Sprintf(\" ms/ %d objects\", listCount)\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"head\",\n\t\t\tcount: sCount + bCount,\n\t\t\ttitle: \"head objects\",\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"head\", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"chtimes\",\n\t\t\tcount: sCount + bCount,\n\t\t\ttitle: \"update mtime\",\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"chtimes\", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"chmod\",\n\t\t\tcount: sCount + bCount,\n\t\t\ttitle: \"change permissions\",\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"chmod\", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"chown\",\n\t\t\tcount: sCount + bCount,\n\t\t\ttitle: \"change owner/group\",\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"chown\", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t}, {\n\t\t\tname: \"delete\",\n\t\t\tcount: sCount + bCount,\n\t\t\ttitle: \"delete objects\",\n\t\t\tgetResult: func(cost float64) []string {\n\t\t\t\tline := []string{\"\", nspt, nspt}\n\t\t\t\tif cost > 0 {\n\t\t\t\t\tline[1], line[2] = colorize(\"delete\", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 2, colorful)\n\t\t\t\t\tline[1] += \" objects/s\"\n\t\t\t\t\tline[2] += \" ms/object\"\n\t\t\t\t}\n\t\t\t\treturn line\n\t\t\t},\n\t\t},\n\t}\n\n\tbm := &benchMarkObj{\n\t\tblob: blob,\n\t\tprogressBar: progress,\n\t\tthreads: threads,\n\t\tseed: make([]byte, bSize),\n\t\tsmallSeed: make([]byte, smallBSize),\n\t\tbuffPool: &sync.Pool{New: func() interface{} {\n\t\t\tbuff := make([]byte, bSize)\n\t\t\treturn &buff\n\t\t}},\n\t\tsmallBuffPool: &sync.Pool{New: func() interface{} {\n\t\t\tbuff := make([]byte, smallBSize)\n\t\t\treturn &buff\n\t\t}},\n\t}\n\tutils.RandRead(bm.seed)\n\tutils.RandRead(bm.smallSeed)\n\n\tfor _, api := range apis {\n\t\tpResult = append(pResult, bm.run(ctx.Context, api))\n\t}\n\tprogress.Done()\n\n\tfmt.Printf(\"Benchmark finished! block-size: %s, big-object-size: %s, small-object-size: %s, small-objects: %d, NumThreads: %d\\n\",\n\t\thumanize.IBytes(uint64(bSize)), humanize.IBytes(uint64(fsize)), humanize.IBytes(uint64(smallBSize)), sCount, threads)\n\n\t// adjust the print order\n\tpResult[1], pResult[3] = pResult[3], pResult[1]\n\tpResult[2], pResult[4] = pResult[4], pResult[2]\n\tpResult[7], pResult[10] = pResult[10], pResult[7]\n\tprintResult(pResult, -1, colorful)\n\treturn nil\n}\n\nvar resultRangeForObj = map[string][4]float64{\n\t\"put\": {100, 150, 50, 150},\n\t\"get\": {100, 150, 50, 150},\n\t\"smallput\": {10, 30, 30, 100},\n\t\"smallget\": {10, 30, 30, 100},\n\t\"multi-upload\": {100, 150, 20, 50},\n\t\"list\": {1000, 10000, 100, 200},\n\t\"head\": {10, 30, 30, 100},\n\t\"delete\": {10, 30, 30, 100},\n\t\"chmod\": {10, 30, 30, 100},\n\t\"chown\": {10, 30, 30, 100},\n\t\"chtimes\": {10, 30, 30, 100},\n}\n\nfunc colorize(item string, value, cost float64, prec int, colorful bool) (string, string) {\n\tsvalue := strconv.FormatFloat(value, 'f', prec, 64)\n\tvar fmtMode byte = 'f'\n\tif cost < 0.01 {\n\t\t// For 'g' and 'G' it is the maximum number of significant digits\n\t\tfmtMode = 'g'\n\t}\n\tscost := strconv.FormatFloat(cost, byte(fmtMode), 2, 64)\n\tif colorful {\n\t\tr, ok := resultRangeForObj[item]\n\t\tif !ok {\n\t\t\tlogger.Fatalf(\"Invalid item: %s\", item)\n\t\t}\n\t\tvar color int\n\t\tif value > r[1] { // max\n\t\t\tcolor = GREEN\n\t\t} else if value > r[0] { // min\n\t\t\tcolor = YELLOW\n\t\t} else {\n\t\t\tcolor = RED\n\t\t}\n\t\tsvalue = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, color, svalue, RESET_SEQ)\n\t\tif cost < r[2] { // min\n\t\t\tcolor = GREEN\n\t\t} else if cost < r[3] { // max\n\t\t\tcolor = YELLOW\n\t\t} else {\n\t\t\tcolor = RED\n\t\t}\n\t\tscost = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, color, scost, RESET_SEQ)\n\t}\n\treturn svalue, scost\n}\n\ntype apiInfo struct {\n\tname string\n\ttitle string\n\tcount int\n\tstartKey int\n\tgetResult func(cost float64) []string\n}\n\ntype benchMarkObj struct {\n\tprogressBar *utils.Progress\n\tblob object.ObjectStorage\n\tthreads int\n\tseed, smallSeed []byte\n\tbuffPool, smallBuffPool *sync.Pool\n}\n\nfunc (bm *benchMarkObj) run(ctx context.Context, api apiInfo) []string {\n\tif api.name == \"chown\" || api.name == \"chmod\" || api.name == \"chtimes\" {\n\t\tif err := bm.chmod(ctx, \"not_exists\", 0); err == utils.ENOTSUP {\n\t\t\tline := api.getResult(-1)\n\t\t\tline[0] = api.title\n\t\t\treturn line\n\t\t}\n\t\tif api.name == \"chown\" && (strings.HasPrefix(bm.blob.String(), \"file://\") || strings.HasPrefix(bm.blob.String(), \"jfs://\")) && os.Getuid() != 0 {\n\t\t\tlogger.Warnf(\"chown test should be run by root\")\n\t\t\treturn []string{api.title, skipped, skipped}\n\t\t}\n\t}\n\tvar fn func(ctx context.Context, key string, startKey int) error\n\tswitch api.name {\n\tcase \"put\":\n\t\tfn = bm.put\n\tcase \"get\":\n\t\tfn = bm.get\n\tcase \"smallput\":\n\t\tfn = bm.smallPut\n\tcase \"smallget\":\n\t\tfn = bm.smallGet\n\tcase \"delete\":\n\t\tfn = bm.delete\n\tcase \"head\":\n\t\tfn = bm.head\n\tcase \"list\":\n\t\tfn = bm.list\n\tcase \"chown\":\n\t\tfn = bm.chown\n\tcase \"chmod\":\n\t\tfn = bm.chmod\n\tcase \"chtimes\":\n\t\tfn = bm.chtimes\n\t}\n\n\tvar wg sync.WaitGroup\n\tpool := make(chan struct{}, bm.threads)\n\tcount := api.count\n\tvar bar *utils.Bar\n\tif api.name == \"list\" {\n\t\tbar = bm.progressBar.AddCountBar(api.title, int64(listCount)*int64(count))\n\t} else {\n\t\tbar = bm.progressBar.AddCountBar(api.title, int64(count))\n\t}\n\tvar err error\n\tstart := time.Now()\n\tfor i := api.startKey; i < api.startKey+count; i++ {\n\t\tpool <- struct{}{}\n\t\twg.Add(1)\n\t\tgo func(key int) {\n\t\t\tdefer func() {\n\t\t\t\t<-pool\n\t\t\t\twg.Done()\n\t\t\t}()\n\t\t\tif e := fn(ctx, strconv.Itoa(key), api.startKey); e != nil {\n\t\t\t\terr = e\n\t\t\t}\n\t\t\tif api.name == \"list\" {\n\t\t\t\tbar.IncrInt64(int64(listCount))\n\t\t\t} else {\n\t\t\t\tbar.Increment()\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n\tbar.Done()\n\tline := api.getResult(time.Since(start).Seconds())\n\tif err != nil {\n\t\tlogger.Errorf(\"%s test failed: %s\", api.name, err)\n\t\treturn []string{api.title, failed, failed}\n\t}\n\tline[0] = api.title\n\treturn line\n}\n\nfunc getMockData(seed []byte, idx int, result *[]byte) {\n\tsize := len(seed)\n\trSize := len(*result)\n\tif size == 0 || rSize == 0 {\n\t\treturn\n\t}\n\ti := idx % size\n\tif size-i > rSize {\n\t\tcopy(*result, seed[i:i+rSize])\n\t} else {\n\t\tcopy((*result)[:size-i], seed[i:size])\n\t\tcopy((*result)[size-i:rSize], seed[:rSize-(size-i)])\n\t}\n\n}\n\nfunc (bm *benchMarkObj) put(ctx context.Context, key string, startKey int) error {\n\tidx, _ := strconv.Atoi(key)\n\tif idx-startKey == 0 {\n\t\treturn bm.blob.Put(ctx, key, bytes.NewReader(bm.seed))\n\t}\n\tbuff := bm.buffPool.Get().(*[]byte)\n\tdefer bm.buffPool.Put(buff)\n\tgetMockData(bm.seed, idx-startKey, buff)\n\treturn bm.blob.Put(ctx, key, bytes.NewReader(*buff))\n}\n\nfunc (bm *benchMarkObj) smallPut(ctx context.Context, key string, startKey int) error {\n\tidx, _ := strconv.Atoi(key)\n\tif idx == 0 {\n\t\treturn bm.blob.Put(ctx, key, bytes.NewReader(bm.smallSeed))\n\t}\n\n\tbuff := bm.smallBuffPool.Get().(*[]byte)\n\tdefer bm.smallBuffPool.Put(buff)\n\tgetMockData(bm.smallSeed, idx-startKey, buff)\n\treturn bm.blob.Put(ctx, key, bytes.NewReader(*buff))\n}\n\nfunc getAndCheckN(ctx context.Context, blob object.ObjectStorage, key string, seed []byte, pool *sync.Pool, getOrgIdx func(idx int) int) error {\n\tidx, _ := strconv.Atoi(key)\n\tr, err := blob.Get(ctx, key, 0, -1)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer r.Close()\n\tcontent := pool.Get().(*[]byte)\n\tdefer pool.Put(content)\n\n\tvar n int\n\tn, err = io.ReadFull(r, *content)\n\tif err != nil {\n\t\treturn err\n\t}\n\torgIdx := getOrgIdx(idx)\n\tcheckN := 10\n\tl := len(seed)\n\tif l < checkN {\n\t\tcheckN = l\n\t}\n\n\t// if orgIdx is 0, mockdata is the same as the seed\n\tvar preNMockData []byte\n\tif orgIdx == 0 {\n\t\tpreNMockData = seed[:checkN]\n\t} else {\n\t\tmockResult := pool.Get().(*[]byte)\n\t\tdefer pool.Put(mockResult)\n\t\tpreNMockData = (*mockResult)[:checkN]\n\t\tgetMockData(seed, orgIdx, &preNMockData)\n\t}\n\n\tif n != len(seed) || !bytes.Equal((*content)[:checkN], preNMockData) {\n\t\treturn fmt.Errorf(\"the downloaded content is incorrect\")\n\t}\n\treturn nil\n}\n\nfunc (bm *benchMarkObj) get(ctx context.Context, key string, startKey int) error {\n\treturn getAndCheckN(ctx, bm.blob, key, bm.seed, bm.buffPool, func(idx int) int {\n\t\treturn idx - startKey\n\t})\n}\n\nfunc (bm *benchMarkObj) smallGet(ctx context.Context, key string, startKey int) error {\n\treturn getAndCheckN(ctx, bm.blob, key, bm.smallSeed, bm.smallBuffPool, func(idx int) int {\n\t\treturn idx\n\t})\n}\n\nfunc (bm *benchMarkObj) delete(ctx context.Context, key string, startKey int) error {\n\treturn bm.blob.Delete(ctx, key)\n}\n\nfunc (bm *benchMarkObj) head(ctx context.Context, key string, startKey int) error {\n\t_, err := bm.blob.Head(ctx, key)\n\treturn err\n}\n\nfunc (bm *benchMarkObj) list(ctx context.Context, key string, startKey int) error {\n\tresult, err := osync.ListAll(bm.blob, \"\", \"0\", \"999\", true)\n\tfor range result {\n\t}\n\treturn err\n}\n\nfunc (bm *benchMarkObj) chown(ctx context.Context, key string, startKey int) error {\n\treturn bm.blob.(object.FileSystem).Chown(key, \"nobody\", groupName)\n}\n\nfunc (bm *benchMarkObj) chmod(ctx context.Context, key string, startKey int) error {\n\treturn bm.blob.(object.FileSystem).Chmod(key, 0755)\n}\n\nfunc (bm *benchMarkObj) chtimes(ctx context.Context, key string, startKey int) error {\n\treturn bm.blob.(object.FileSystem).Chtimes(key, time.Now())\n}\n\nfunc listAll(ctx context.Context, s object.ObjectStorage, prefix, marker string, limit int64) ([]object.Object, error) {\n\tch, err := object.ListAll(ctx, s, prefix, marker, true, true)\n\tif err == nil {\n\t\tobjs := make([]object.Object, 0)\n\t\tfor obj := range ch {\n\t\t\tif len(objs) < int(limit) {\n\t\t\t\tobjs = append(objs, obj)\n\t\t\t}\n\t\t}\n\t\treturn objs, nil\n\t}\n\treturn nil, err\n}\n\nvar syncTests = map[string]bool{\n\t\"special key\": true,\n\t\"put a big object\": true,\n\t\"put an empty object\": true,\n\t\"multipart upload\": true,\n}\n\nfunc functionalTesting(ctx context.Context, blob object.ObjectStorage, result *[][]string, colorful bool) {\n\trunCase := func(title string, fn func(blob object.ObjectStorage) error) {\n\t\tr := pass\n\t\tif err := fn(blob); err == utils.ENOTSUP {\n\t\t\tr = nspt\n\t\t} else if err != nil {\n\t\t\tcolor := RED\n\t\t\tif _, ok := err.(warning); ok {\n\t\t\t\tcolor = YELLOW\n\t\t\t}\n\t\t\tr = err.Error()\n\t\t\tif len(r) > 45 {\n\t\t\t\tr = r[:45] + \"...\"\n\t\t\t}\n\t\t\tif colorful {\n\t\t\t\tr = fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, color, r, RESET_SEQ)\n\t\t\t}\n\t\t\tlogger.Debug(err.Error())\n\t\t}\n\n\t\tcategory := \"basic\"\n\t\tif syncTests[title] || strings.HasPrefix(title, \"change\") {\n\t\t\tcategory = \"sync\"\n\t\t}\n\n\t\tif colorful {\n\t\t\ttitle = fmt.Sprintf(\"%s%sm%s%s\", COLOR_SEQ, DEFAULT, title, RESET_SEQ)\n\t\t}\n\n\t\t*result = append(*result, []string{category, title, r})\n\t}\n\tisFileSystem := true\n\tfi, ok := blob.(object.FileSystem)\n\tif ok {\n\t\tif err := fi.Chmod(\"not_exists_file\", 0755); err == utils.ENOTSUP {\n\t\t\tisFileSystem = false\n\t\t}\n\t}\n\n\tget := func(s object.ObjectStorage, k string, off, limit int64) (string, error) {\n\t\tr, err := s.Get(ctx, k, off, limit)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tdefer r.Close()\n\t\tdata, err := io.ReadAll(r)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\treturn string(data), nil\n\t}\n\tkey := \"put_test_file\"\n\n\tfunFSCase := func(name string, fn func() error) {\n\t\trunCase(name, func(blob object.ObjectStorage) error {\n\t\t\tif !isFileSystem {\n\t\t\t\treturn utils.ENOTSUP\n\t\t\t}\n\t\t\tbr := []byte(\"hello\")\n\t\t\tif err := blob.Put(ctx, key, bytes.NewReader(br)); err != nil {\n\t\t\t\treturn fmt.Errorf(\"put object failed: %s\", err)\n\t\t\t}\n\t\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\t\treturn warning(fn())\n\t\t})\n\t}\n\n\trunCase(\"create a bucket\", func(blob object.ObjectStorage) error {\n\t\tcreated := true\n\t\tif err := blob.Put(ctx, key, bytes.NewReader([]byte(\"1\"))); err != nil {\n\t\t\tcreated = false\n\t\t}\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\n\t\tif !created {\n\t\t\tif err := blob.Create(ctx); err != nil {\n\t\t\t\treturn fmt.Errorf(\"can't create bucket: %s\", err)\n\t\t\t}\n\t\t}\n\t\tif err := blob.Create(ctx); err != nil {\n\t\t\treturn fmt.Errorf(\"creating a bucket that already exists returns an error\")\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"put an object\", func(blob object.ObjectStorage) error {\n\t\tbr := []byte(\"hello\")\n\t\tif err := blob.Put(ctx, key, bytes.NewReader(br)); err != nil {\n\t\t\treturn fmt.Errorf(\"put object failed: %s\", err)\n\t\t}\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\treturn nil\n\t})\n\n\trunCase(\"get an object\", func(blob object.ObjectStorage) error {\n\t\tbr := []byte(\"hello\")\n\t\tif err := blob.Put(ctx, key, bytes.NewReader(br)); err != nil {\n\t\t\treturn fmt.Errorf(\"put object failed: %s\", err)\n\t\t}\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\tif d, e := get(blob, key, 0, -1); e != nil || d != string(br) {\n\t\t\treturn fmt.Errorf(`failed to get an object: expect \"hello\", but got %v, error: %s`, d, e)\n\t\t}\n\t\tif d, e := get(blob, key, 0, 5); e != nil || d != string(br) {\n\t\t\treturn fmt.Errorf(`failed to get an object: expect \"hello\", but got %v, error: %s`, d, e)\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"get non-exist\", func(blob object.ObjectStorage) error {\n\t\tif _, err := blob.Get(ctx, \"not_exists_file\", 0, -1); err == nil {\n\t\t\treturn fmt.Errorf(\"get not existed object should failed: %s\", err)\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"get partial object\", func(blob object.ObjectStorage) error {\n\t\tbr := []byte(\"hello\")\n\t\tif err := blob.Put(ctx, key, bytes.NewReader(br)); err != nil {\n\t\t\treturn fmt.Errorf(\"put object failed: %s\", err)\n\t\t}\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\n\t\t// get first\n\t\tif d, e := get(blob, key, 0, 1); e != nil || d != \"h\" {\n\t\t\treturn fmt.Errorf(`failed to get the first byte:, expect \"h\", but got %q, error: %s`, d, e)\n\t\t}\n\t\t// get last\n\t\tif d, e := get(blob, key, 4, 1); e != nil || d != \"o\" {\n\t\t\treturn fmt.Errorf(`failed to get the last byte: expect \"o\", but got %q, error: %s`, d, e)\n\t\t}\n\t\t// get last 3\n\t\tif d, e := get(blob, key, 2, 3); e != nil || d != \"llo\" {\n\t\t\treturn fmt.Errorf(`failed to get the last three bytes: expect \"llo\", but got %q, error: %s`, d, e)\n\t\t}\n\t\t// get middle\n\t\tif d, e := get(blob, key, 2, 2); e != nil || d != \"ll\" {\n\t\t\treturn fmt.Errorf(`failed to get two bytes: expect \"ll\", but got %q, error: %s`, d, e)\n\t\t}\n\t\t// get the end out of range\n\t\tif d, e := get(blob, key, 4, 2); e != nil || d != \"o\" {\n\t\t\treturn warning(fmt.Errorf(`failed to get object with the end out of range, expect \"o\", but got %q, error: %s`, d, e))\n\t\t}\n\t\t// get the off out of range\n\t\tif d, e := get(blob, key, 6, 2); e != nil || d != \"\" {\n\t\t\treturn warning(fmt.Errorf(`failed to get object with the offset out of range, expect \"\", but got %q, error: %s`, d, e))\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"head an object\", func(blob object.ObjectStorage) error {\n\t\tbr := []byte(\"hello\")\n\t\tif err := blob.Put(ctx, key, bytes.NewReader(br)); err != nil {\n\t\t\treturn fmt.Errorf(\"put object failed: %s\", err)\n\t\t}\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\tif h, err := blob.Head(ctx, key); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to head object %s\", err)\n\t\t} else {\n\t\t\tif h.Key() != key {\n\t\t\t\treturn fmt.Errorf(\"expected key 'test' but got %s\", h.Key())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"delete an object\", func(blob object.ObjectStorage) error {\n\t\tbr := []byte(\"hello\")\n\t\tif err := blob.Put(ctx, key, bytes.NewReader(br)); err != nil {\n\t\t\treturn fmt.Errorf(\"put object failed: %s\", err)\n\t\t}\n\t\tif err := blob.Delete(ctx, key); err != nil {\n\t\t\treturn fmt.Errorf(\"delete failed: %s\", err)\n\t\t}\n\t\tif _, err := blob.Head(ctx, key); err == nil {\n\t\t\treturn fmt.Errorf(\"expect err is not nil\")\n\t\t}\n\n\t\tif err := blob.Delete(ctx, key); err != nil {\n\t\t\treturn fmt.Errorf(\"delete not existed: %v\", err)\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"delete non-exist\", func(blob object.ObjectStorage) error {\n\t\tif err := blob.Delete(ctx, key); err != nil {\n\t\t\treturn fmt.Errorf(\"deleting a non-existent object returns an error %v\", err)\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"list objects\", func(blob object.ObjectStorage) error {\n\t\tbr := []byte(\"hello\")\n\t\tif err := blob.Put(ctx, key, bytes.NewReader(br)); err != nil {\n\t\t\treturn fmt.Errorf(\"put object failed: %s\", err)\n\t\t}\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\tif isFileSystem {\n\t\t\tobjs, err := listAll(ctx, blob, \"\", \"\", 2)\n\t\t\tif err == nil {\n\t\t\t\tif len(objs) != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"list should return 2 keys, but got %d\", len(objs))\n\t\t\t\t}\n\t\t\t\tif objs[0].Key() != \"\" {\n\t\t\t\t\treturn fmt.Errorf(\"first key should be empty string, but got %s\", objs[0].Key())\n\t\t\t\t}\n\t\t\t\tif objs[0].Size() != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"first object size should be 0, but got %d\", objs[0].Size())\n\t\t\t\t}\n\t\t\t\tif objs[1].Key() != key {\n\t\t\t\t\treturn fmt.Errorf(\"first key should be test, but got %s\", objs[1].Key())\n\t\t\t\t}\n\t\t\t\tif objs[1].Size() != 5 {\n\t\t\t\t\treturn fmt.Errorf(\"size of first key shold be 5, but got %v\", objs[1].Size())\n\t\t\t\t}\n\t\t\t\tnow := time.Now()\n\t\t\t\tif objs[1].Mtime().Before(now.Add(-30*time.Second)) || objs[1].Mtime().After(now.Add(time.Second*30)) {\n\t\t\t\t\treturn fmt.Errorf(\"mtime of key should be within 30 seconds, but got %s\", objs[1].Mtime().Sub(now))\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn fmt.Errorf(\"list failed: %s\", err)\n\t\t\t}\n\n\t\t\tobjs, err = listAll(ctx, blob, \"\", \"test2\", 1)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"list failed: %s\", err)\n\t\t\t} else if len(objs) != 0 {\n\t\t\t\treturn fmt.Errorf(\"list should not return anything, but got %d\", len(objs))\n\t\t\t}\n\t\t} else {\n\t\t\tobjs, err2 := listAll(ctx, blob, \"\", \"\", 1)\n\t\t\tif err2 == nil {\n\t\t\t\tif len(objs) != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"list should return 1 keys, but got %d\", len(objs))\n\t\t\t\t}\n\t\t\t\tif objs[0].Key() != key {\n\t\t\t\t\treturn fmt.Errorf(\"first key should be test, but got %s\", objs[0].Key())\n\t\t\t\t}\n\t\t\t\tif objs[0].Size() != 5 {\n\t\t\t\t\treturn fmt.Errorf(\"size of first key shold be 5, but got %v\", objs[0].Size())\n\t\t\t\t}\n\t\t\t\tnow := time.Now()\n\t\t\t\tif objs[0].Mtime().Before(now.Add(-30*time.Second)) || objs[0].Mtime().After(now.Add(time.Second*30)) {\n\t\t\t\t\treturn fmt.Errorf(\"mtime of key should be within 30 seconds, but got %s\", objs[0].Mtime().Sub(now))\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn fmt.Errorf(\"list failed: %s\", err2)\n\t\t\t}\n\n\t\t\tobjs, err2 = listAll(ctx, blob, \"\", \"test2\", 1)\n\t\t\tif err2 != nil {\n\t\t\t\treturn fmt.Errorf(\"list failed: %s\", err2)\n\t\t\t} else if len(objs) != 0 {\n\t\t\t\treturn fmt.Errorf(\"list should not return anything, but got %d\", len(objs))\n\t\t\t}\n\t\t}\n\t\tkeyTotal := 100\n\t\tvar sortedKeys []string\n\t\tfor i := 0; i < keyTotal; i++ {\n\t\t\tk := fmt.Sprintf(\"hashKey%d\", i)\n\t\t\tsortedKeys = append(sortedKeys, k)\n\t\t\tif err := blob.Put(ctx, fmt.Sprintf(\"hashKey%d\", i), bytes.NewReader(br)); err != nil {\n\t\t\t\treturn fmt.Errorf(\"put object failed: %s\", err.Error())\n\t\t\t}\n\t\t}\n\t\tsort.Strings(sortedKeys)\n\t\tdefer func() {\n\t\t\tfor i := 0; i < keyTotal; i++ {\n\t\t\t\t_ = blob.Delete(ctx, fmt.Sprintf(\"hashKey%d\", i))\n\t\t\t}\n\t\t}()\n\n\t\tif objs, err := listAll(ctx, blob, \"hashKey\", \"\", int64(keyTotal)); err != nil {\n\t\t\treturn fmt.Errorf(\"list failed: %s\", err)\n\t\t} else {\n\t\t\tfor i := 0; i < keyTotal; i++ {\n\t\t\t\tif objs[i].Key() != sortedKeys[i] {\n\t\t\t\t\treturn fmt.Errorf(\"the result for list is incorrect\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"special key\", func(blob object.ObjectStorage) error {\n\t\tkey := \"测试编码文件\" + `{\"name\":\"juicefs\"}` + string('\\u001F') + \"%uFF081%uFF09.jpg\"\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\tif err := blob.Put(ctx, key, bytes.NewReader([]byte(\"1\"))); err != nil {\n\t\t\treturn fmt.Errorf(\"put encode file failed: %s\", err)\n\t\t} else {\n\t\t\tif resp, _, _, err := blob.List(ctx, \"\", \"测试编码文件\", \"\", \"\", 1, true); err != nil && err != utils.ENOTSUP {\n\t\t\t\treturn fmt.Errorf(\"list encode file failed %s\", err)\n\t\t\t} else if len(resp) == 1 && resp[0].Key() != key {\n\t\t\t\treturn fmt.Errorf(\"list encode file failed: expect key %s, but got %s\", key, resp[0].Key())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"put a big object\", func(blob object.ObjectStorage) error {\n\t\tfsize := 256 << 20\n\t\tbuffL := 4 << 20\n\t\tbuff := make([]byte, buffL)\n\t\tutils.RandRead(buff)\n\t\tcount := int(math.Floor(float64(fsize) / float64(buffL)))\n\t\tcontent := make([]byte, fsize)\n\t\tfor i := 0; i < count; i++ {\n\t\t\tcopy(content[i*buffL:(i+1)*buffL], buff)\n\t\t}\n\t\tif err := blob.Put(ctx, key, bytes.NewReader(content)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\treturn nil\n\t})\n\n\trunCase(\"put an empty object\", func(blob object.ObjectStorage) error {\n\t\t// Copy empty objects\n\t\tdefer blob.Delete(ctx, \"empty_test_file\") //nolint:errcheck\n\t\tif err := blob.Put(ctx, \"empty_test_file\", bytes.NewReader([]byte{})); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Copy `/` suffixed object\n\t\tdefer blob.Delete(ctx, \"slash_test_file/\") //nolint:errcheck\n\t\tif err := blob.Put(ctx, \"slash_test_file/\", bytes.NewReader([]byte(\"1\"))); err != nil {\n\t\t\treturn fmt.Errorf(\"put `/` suffixed object failed: %s\", err)\n\t\t}\n\t\treturn nil\n\t})\n\n\trunCase(\"multipart upload\", func(blob object.ObjectStorage) (err error) {\n\t\tdefer func() {\n\t\t\terr = warning(err)\n\t\t}()\n\n\t\tkey := \"multi_test_file\"\n\t\tif err = blob.CompleteUpload(ctx, key, \"notExistsUploadId\", []*object.Part{}); err != utils.ENOTSUP {\n\t\t\tdefer blob.Delete(ctx, key) //nolint:errcheck\n\t\t\tupload, err := blob.CreateMultipartUpload(ctx, key)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"create multipart upload failed: %s\", err)\n\t\t\t}\n\t\t\ttotal := 3\n\t\t\tseed := make([]byte, upload.MinPartSize)\n\t\t\tutils.RandRead(seed)\n\t\t\tparts := make([]*object.Part, total)\n\t\t\tcontent := make([][]byte, total)\n\t\t\tfor i := 0; i < total; i++ {\n\t\t\t\tcontent[i] = make([]byte, upload.MinPartSize)\n\t\t\t\tgetMockData(seed, i, &content[i])\n\t\t\t}\n\t\t\tvar eg errgroup.Group\n\t\t\teg.SetLimit(4)\n\t\t\tfor i := 1; i <= total; i++ {\n\t\t\t\tnum := i\n\t\t\t\teg.Go(func() error {\n\t\t\t\t\tvar err error\n\t\t\t\t\tparts[num-1], err = blob.UploadPart(ctx, key, upload.UploadID, num, content[num-1])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\terr = fmt.Errorf(\"multipart upload error: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t\treturn err\n\t\t\t\t})\n\t\t\t}\n\t\t\terr = eg.Wait()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t// overwrite the first part\n\t\t\tfirstPartContent := append(seed, seed...)\n\t\t\tif parts[0], err = blob.UploadPart(ctx, key, upload.UploadID, 1, firstPartContent); err != nil {\n\t\t\t\treturn fmt.Errorf(\"multipart upload error: %v\", err)\n\t\t\t}\n\t\t\tcontent[0] = firstPartContent\n\n\t\t\t// overwrite the last part\n\t\t\tlastPartContent := []byte(\"hello\")\n\t\t\tif parts[total-1], err = blob.UploadPart(ctx, key, upload.UploadID, total, lastPartContent); err != nil {\n\t\t\t\treturn fmt.Errorf(\"multipart upload error: %v\", err)\n\t\t\t}\n\t\t\tcontent[total-1] = lastPartContent\n\n\t\t\tif err = blob.CompleteUpload(ctx, key, upload.UploadID, parts); err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to complete multipart upload: %v\", err)\n\t\t\t}\n\t\t\tr, err := blob.Get(ctx, key, 0, -1)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to get multipart upload file: %v\", err)\n\t\t\t}\n\t\t\tcnt, err := io.ReadAll(r)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to get multipart upload file: %v\", err)\n\t\t\t}\n\t\t\tif !bytes.Equal(cnt, bytes.Join(content, nil)) {\n\t\t\t\treturn fmt.Errorf(\"the content of the multipart upload file is incorrect\")\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\treturn utils.ENOTSUP\n\t})\n\n\tfunFSCase(\"change owner/group\", func() error {\n\t\tif (strings.HasPrefix(blob.String(), \"file://\") || strings.HasPrefix(blob.String(), \"jfs://\")) && os.Getuid() != 0 {\n\t\t\treturn errors.New(\"root required\")\n\t\t}\n\t\tif err := fi.Chown(key, \"nobody\", groupName); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to chown object %s\", err)\n\t\t}\n\t\tif objInfo, err := blob.Head(ctx, key); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to head object %s\", err)\n\t\t} else if info, ok := objInfo.(object.File); ok {\n\t\t\tif info.Owner() != \"nobody\" {\n\t\t\t\treturn fmt.Errorf(\"expect owner nobody but got %s\", info.Owner())\n\t\t\t}\n\t\t\tif info.Group() != groupName {\n\t\t\t\treturn fmt.Errorf(\"expect group %s but got %s\", groupName, info.Group())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tfunFSCase(\"change permission\", func() error {\n\t\tif err := fi.Chmod(key, 0777); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif objInfo, err := blob.Head(ctx, key); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to head object %s\", err)\n\t\t} else if info, ok := objInfo.(object.File); ok {\n\t\t\tif info.Mode()&0xFFF != 0777 {\n\t\t\t\treturn fmt.Errorf(\"expect mode %o but got %o\", 0777, info.Mode())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tfunFSCase(\"change mtime\", func() error {\n\t\tmtime := time.Now().Add(-10 * time.Minute)\n\t\tif err := fi.Chtimes(key, mtime); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to chtimes %s\", err)\n\t\t}\n\t\tif objInfo, err := blob.Head(ctx, key); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to head object %s\", err)\n\t\t} else {\n\t\t\tif objInfo.Mtime().Before(mtime.Add(-2*time.Second)) || objInfo.Mtime().After(mtime.Add(2*time.Second)) {\n\t\t\t\treturn fmt.Errorf(\"mtime deviation is too large, the actual mtime is %s but got %s\", mtime.Format(time.RFC3339), objInfo.Mtime().Format(time.RFC3339))\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n" }, { "path": "cmd/object.go", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nvar (\n\tdirSuffix = \"/\"\n\tcliCtx *cli.Context\n\tpid, uid, gid uint32\n)\n\nfunc toError(eno syscall.Errno) error {\n\tif eno == 0 {\n\t\treturn nil\n\t}\n\treturn eno\n}\n\ntype juiceFS struct {\n\tobject.DefaultObjectStorage\n\tname string\n\tumask uint16\n\tjfs *fs.FileSystem\n}\n\nfunc (j *juiceFS) String() string {\n\treturn fmt.Sprintf(\"jfs://%s/\", j.name)\n}\n\nfunc (j *juiceFS) path(key string) string {\n\treturn dirSuffix + key\n}\n\ntype jFile struct {\n\tf *fs.File\n\tlimit int64\n}\n\nfunc (f *jFile) Read(buf []byte) (int, error) {\n\tif len(buf) == 0 {\n\t\treturn 0, nil\n\t}\n\tif f.limit <= 0 {\n\t\treturn 0, io.EOF\n\t}\n\tif len(buf) > int(f.limit) {\n\t\tbuf = buf[:f.limit]\n\t}\n\tn, err := f.f.Read(ctx, buf)\n\tf.limit -= int64(n)\n\treturn n, err\n}\n\nfunc (f *jFile) Write(buf []byte) (int, error) {\n\tn, eno := f.f.Write(ctx, buf)\n\treturn n, toError(eno)\n}\n\nfunc (f *jFile) Close() error {\n\treturn toError(f.f.Close(ctx))\n}\n\nfunc (j *juiceFS) Get(rCtx context.Context, key string, off, limit int64, getters ...object.AttrGetter) (io.ReadCloser, error) {\n\tctx := meta.WrapWithoutCancel(rCtx, pid, uid, []uint32{gid})\n\tf, err := j.jfs.Open(ctx, j.path(key), vfs.MODE_MASK_R)\n\tif err != 0 {\n\t\treturn nil, err\n\t}\n\tif off > 0 {\n\t\t_, _ = f.Seek(ctx, off, io.SeekStart)\n\t}\n\tif limit <= 0 {\n\t\tlimit = 1 << 62\n\t}\n\treturn &jFile{f, limit}, nil\n}\n\nvar bufPool = sync.Pool{\n\tNew: func() interface{} {\n\t\tbuf := make([]byte, 128<<10)\n\t\treturn &buf\n\t},\n}\n\nfunc (j *juiceFS) Put(rCtx context.Context, key string, in io.Reader, getters ...object.AttrGetter) (err error) {\n\tctx := meta.WrapWithoutCancel(rCtx, pid, uid, []uint32{gid})\n\tif vfs.IsSpecialName(key) {\n\t\treturn fmt.Errorf(\"skip special file %s for jfs: %w\", key, utils.ErrSkipped)\n\t}\n\tp := j.path(key)\n\tif strings.HasSuffix(p, \"/\") {\n\t\teno := j.jfs.MkdirAll(ctx, p, 0777, j.umask)\n\t\treturn toError(eno)\n\t}\n\tvar tmp string\n\tif object.PutInplace {\n\t\ttmp = p\n\t} else {\n\t\tname := path.Base(p)\n\t\tif len(name) > 200 {\n\t\t\tname = name[:200]\n\t\t}\n\t\ttmp = object.TmpFilePath(p, name)\n\t\tdefer func() {\n\t\t\tif err != nil {\n\t\t\t\tif e := j.jfs.Delete(ctx, tmp); e != 0 {\n\t\t\t\t\tlogger.Warnf(\"Failed to delete %s: %s\", tmp, e)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\tf, eno := j.jfs.Create(ctx, tmp, 0666, j.umask)\n\tif eno == syscall.ENOENT {\n\t\tif eno = j.jfs.MkdirAll(ctx, path.Dir(tmp), 0777, j.umask); eno != 0 {\n\t\t\treturn toError(eno)\n\t\t}\n\t\tf, eno = j.jfs.Create(ctx, tmp, 0666, j.umask)\n\t}\n\n\tif eno == syscall.EEXIST {\n\t\tif eno = j.jfs.Delete(ctx, tmp); eno != 0 {\n\t\t\treturn toError(eno)\n\t\t}\n\t\tf, eno = j.jfs.Create(ctx, tmp, 0666, j.umask)\n\t}\n\n\tif eno != 0 {\n\t\treturn toError(eno)\n\t}\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\t_, err = io.CopyBuffer(&jFile{f, 0}, in, *buf)\n\tif err != nil {\n\t\treturn\n\t}\n\teno = f.Close(ctx)\n\tif eno != 0 {\n\t\treturn toError(eno)\n\t}\n\tif !object.PutInplace {\n\t\tif eno = j.jfs.Rename(ctx, tmp, p, 0); eno != 0 {\n\t\t\treturn toError(eno)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (j *juiceFS) Delete(rCtx context.Context, key string, getters ...object.AttrGetter) error {\n\tctx := meta.WrapWithoutCancel(rCtx, pid, uid, []uint32{gid})\n\tif key == \"\" {\n\t\treturn nil\n\t}\n\tp := strings.TrimSuffix(j.path(key), dirSuffix)\n\teno := j.jfs.Delete(ctx, p)\n\tif eno == syscall.ENOENT {\n\t\teno = 0\n\t}\n\treturn toError(eno)\n}\n\ntype jObj struct {\n\tkey string\n\tfi *fs.FileStat\n\tisSymlink bool\n}\n\nfunc (o *jObj) Key() string { return o.key }\nfunc (o *jObj) Size() int64 {\n\tif o.fi.IsDir() {\n\t\treturn 0\n\t}\n\treturn o.fi.Size()\n}\nfunc (o *jObj) Mtime() time.Time { return o.fi.ModTime() }\nfunc (o *jObj) IsDir() bool { return o.fi.IsDir() }\nfunc (o *jObj) IsSymlink() bool { return o.isSymlink }\nfunc (o *jObj) Owner() string { return utils.UserName(o.fi.Uid()) }\nfunc (o *jObj) Group() string { return utils.GroupName(o.fi.Gid()) }\nfunc (o *jObj) Mode() os.FileMode { return o.fi.Mode() }\nfunc (o *jObj) StorageClass() string { return \"\" }\n\nfunc (j *juiceFS) Head(rCtx context.Context, key string) (object.Object, error) {\n\tctx := meta.WrapWithoutCancel(rCtx, pid, uid, []uint32{gid})\n\terrConv := func(eno syscall.Errno) error {\n\t\tif errors.Is(eno, syscall.ENOENT) {\n\t\t\treturn os.ErrNotExist\n\t\t} else {\n\t\t\treturn eno\n\t\t}\n\t}\n\tfi, eno := j.jfs.Lstat(ctx, j.path(key))\n\tif eno != 0 {\n\t\treturn nil, errConv(eno)\n\t}\n\tisSymlink := fi.IsSymlink()\n\tif isSymlink {\n\t\tfi, eno = j.jfs.Stat(ctx, j.path(key))\n\t\tif eno != 0 {\n\t\t\treturn nil, errConv(eno)\n\t\t}\n\t}\n\treturn &jObj{key, fi, isSymlink}, nil\n}\n\nfunc (j *juiceFS) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]object.Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", utils.ENOTSUP\n\t}\n\tdir := j.path(prefix)\n\tvar objs []object.Object\n\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\tdir = path.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t} else if marker == \"\" {\n\t\tobj, err := j.Head(ctx, prefix)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil, false, \"\", nil\n\t\t\t}\n\t\t\treturn nil, false, \"\", err\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\tentries, err := j.readDirSorted(dir, followLink)\n\tif err != 0 {\n\t\tif err == syscall.ENOENT {\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\tfor _, e := range entries {\n\t\tkey := dir[1:] + e.name\n\t\tif !strings.HasPrefix(key, prefix) || (marker != \"\" && key <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\tf := &jObj{key, e.fi, e.fi.IsSymlink()}\n\t\tobjs = append(objs, f)\n\t\tif len(objs) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\tvar nextMarker string\n\tif len(objs) > 0 {\n\t\tnextMarker = objs[len(objs)-1].Key()\n\t}\n\treturn objs, len(objs) == int(limit), nextMarker, nil\n}\n\ntype mEntry struct {\n\tfi *fs.FileStat\n\tname string\n\tisSymlink bool\n}\n\n// readDirSorted reads the directory named by dirname and returns\n// a sorted list of directory entries.\nfunc (j *juiceFS) readDirSorted(dirname string, followLink bool) ([]*mEntry, syscall.Errno) {\n\tf, err := j.jfs.Open(ctx, dirname, 0)\n\tif err != 0 {\n\t\treturn nil, err\n\t}\n\tdefer f.Close(ctx)\n\tentries, err := f.ReaddirPlus(ctx, 0)\n\tif err != 0 {\n\t\treturn nil, err\n\t}\n\tmEntries := make([]*mEntry, len(entries))\n\tfor i, e := range entries {\n\t\tfi := fs.AttrToFileInfo(e.Inode, e.Attr)\n\t\tif fi.IsDir() {\n\t\t\tmEntries[i] = &mEntry{fi, string(e.Name) + dirSuffix, false}\n\t\t} else if fi.IsSymlink() && followLink {\n\t\t\tfi2, err := j.jfs.Stat(ctx, path.Join(dirname, string(e.Name)))\n\t\t\tif err != 0 {\n\t\t\t\tmEntries[i] = &mEntry{fi, string(e.Name), true}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tname := string(e.Name)\n\t\t\tif fi2.IsDir() {\n\t\t\t\tname += dirSuffix\n\t\t\t}\n\t\t\tmEntries[i] = &mEntry{fi2, name, false}\n\t\t} else {\n\t\t\tmEntries[i] = &mEntry{fi, string(e.Name), fi.IsSymlink()}\n\t\t}\n\t}\n\tsort.Slice(mEntries, func(i, j int) bool { return mEntries[i].name < mEntries[j].name })\n\treturn mEntries, err\n}\n\nfunc (j *juiceFS) Chtimes(key string, mtime time.Time) error {\n\tf, err := j.jfs.Lopen(ctx, j.path(key), 0)\n\tif err != 0 {\n\t\treturn err\n\t}\n\tdefer f.Close(ctx)\n\treturn toError(f.Utime(ctx, -1, mtime.UnixNano()/1e6))\n}\n\n// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.\nfunc syscallMode(i os.FileMode) (o uint32) {\n\to |= uint32(i.Perm())\n\tif i&os.ModeSetuid != 0 {\n\t\to |= syscall.S_ISUID\n\t}\n\tif i&os.ModeSetgid != 0 {\n\t\to |= syscall.S_ISGID\n\t}\n\tif i&os.ModeSticky != 0 {\n\t\to |= syscall.S_ISVTX\n\t}\n\t// No mapping for Go's ModeTemporary (plan9 only).\n\treturn\n}\n\nfunc (j *juiceFS) Chmod(key string, mode os.FileMode) error {\n\tf, err := j.jfs.Open(ctx, j.path(key), 0)\n\tif err != 0 {\n\t\treturn err\n\t}\n\tdefer f.Close(ctx)\n\treturn toError(f.Chmod(ctx, uint16(syscallMode(mode))))\n}\n\nfunc (j *juiceFS) Chown(key string, owner, group string) error {\n\tuid := utils.LookupUser(owner)\n\tgid := utils.LookupGroup(group)\n\tif uid == -1 || gid == -1 {\n\t\treturn fmt.Errorf(\"user(%s):group(%s) not found\", owner, group)\n\t}\n\tf, err := j.jfs.Lopen(ctx, j.path(key), 0)\n\tif err != 0 {\n\t\treturn err\n\t}\n\tdefer f.Close(ctx)\n\treturn toError(f.Chown(ctx, uint32(uid), uint32(gid)))\n}\n\nfunc (j *juiceFS) Symlink(oldName, newName string) error {\n\tp := j.path(newName)\n\terr := j.jfs.Symlink(ctx, oldName, p)\n\tif err == syscall.ENOENT {\n\t\tif err = j.jfs.MkdirAll(ctx, path.Dir(p), 0777, j.umask); err != 0 {\n\t\t\treturn toError(err)\n\t\t}\n\t\terr = j.jfs.Symlink(ctx, oldName, p)\n\t}\n\treturn toError(err)\n}\n\nfunc (j *juiceFS) Readlink(name string) (string, error) {\n\ttarget, err := j.jfs.Readlink(ctx, j.path(name))\n\treturn string(target), toError(err)\n}\n\nfunc getDefaultChunkConf(format *meta.Format) *chunk.Config {\n\tchunkConf := &chunk.Config{\n\t\tBlockSize: format.BlockSize * 1024,\n\t\tCompress: format.Compression,\n\t\tHashPrefix: format.HashPrefix,\n\t\tGetTimeout: time.Minute,\n\t\tPutTimeout: time.Minute,\n\t\tMaxUpload: 50,\n\t\tMaxDownload: 200,\n\t\tMaxRetries: 10,\n\t\tBufferSize: 300 << 20,\n\t}\n\tchunkConf.SelfCheck(format.UUID)\n\treturn chunkConf\n}\n\nfunc (j *juiceFS) Shutdown() {\n\t_ = j.jfs.Meta().CloseSession()\n}\n\nfunc newJFS(endpoint, accessKey, secretKey, token string) (object.ObjectStorage, error) {\n\tpid, uid, gid = uint32(os.Getpid()), uint32(utils.GetCurrentUID()), uint32(utils.GetCurrentGID())\n\tif runtime.GOOS == \"windows\" && utils.IsWinAdminOrElevatedPrivilege() {\n\t\tuid = 0\n\t\tgid = 0\n\t}\n\tmetaUrl := os.Getenv(endpoint)\n\tif metaUrl == \"\" {\n\t\tmetaUrl = endpoint\n\t}\n\tmetaConf := meta.DefaultConf()\n\tmetaConf.MaxDeletes = 10\n\tmetaConf.NoBGJob = true\n\tmetaCli := meta.NewClient(metaUrl, metaConf)\n\tformat, err := metaCli.Load(true)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"load setting: %s\", err)\n\t}\n\tblob, err := NewReloadableStorage(format, metaCli, nil)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"object storage: %s\", err)\n\t}\n\tchunkConf := getDefaultChunkConf(format)\n\tstore := chunk.NewCachedStore(blob, *chunkConf, nil)\n\tregisterMetaMsg(metaCli, store, chunkConf)\n\terr = metaCli.NewSession(false)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"new session: %s\", err)\n\t}\n\tmetaCli.OnReload(func(fmt *meta.Format) {\n\t\tstore.UpdateLimit(fmt.UploadLimit, fmt.DownloadLimit)\n\t})\n\n\tvfsConf := &vfs.Config{\n\t\tMeta: metaConf,\n\t\tFormat: *format,\n\t\tVersion: version.Version(),\n\t\tChunk: chunkConf,\n\t\tAttrTimeout: time.Second,\n\t\tDirEntryTimeout: time.Second,\n\t\tMountpoint: cliCtx.String(\"mountpoint\"),\n\t}\n\n\tvfsConf.Format.RemoveSecret()\n\td, _ := json.MarshalIndent(vfsConf, \" \", \"\")\n\tlogger.Debugf(\"Config: %s\", string(d))\n\n\tjfs, err := fs.NewFileSystem(vfsConf, metaCli, store, nil)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Initialize: %s\", err)\n\t}\n\treturn &juiceFS{object.DefaultObjectStorage{}, format.Name, uint16(utils.GetUmask()), jfs}, nil\n}\n\nfunc init() {\n\tobject.Register(\"jfs\", newJFS)\n}\n" }, { "path": "cmd/object_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage cmd\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n)\n\nfunc testKeysEqual(objs []object.Object, expectedKeys []string) error {\n\tgottenKeys := make([]string, len(objs))\n\tfor idx, obj := range objs {\n\t\tgottenKeys[idx] = obj.Key()\n\t}\n\tif len(gottenKeys) != len(expectedKeys) {\n\t\treturn fmt.Errorf(\"Expected {%s}, got {%s}\", strings.Join(expectedKeys, \", \"),\n\t\t\tstrings.Join(gottenKeys, \", \"))\n\t}\n\n\tfor idx, key := range gottenKeys {\n\t\tif key != expectedKeys[idx] {\n\t\t\treturn fmt.Errorf(\"Expected {%s}, got {%s}\", strings.Join(expectedKeys, \", \"),\n\t\t\t\tstrings.Join(gottenKeys, \", \"))\n\t\t}\n\t}\n\treturn nil\n}\n\n// copied from pkg/object/filesystem_test.go\nfunc testFileSystem(t *testing.T, s object.ObjectStorage) {\n\tctx := context.Background()\n\tkeys := []string{\n\t\t\"x/\",\n\t\t\"x/x.txt\",\n\t\t\"xy.txt\",\n\t\t\"xyz/\",\n\t\t\"xyz/xyz.txt\",\n\t}\n\t// initialize directory tree\n\tfor _, key := range keys {\n\t\tif err := s.Put(ctx, key, bytes.NewReader([]byte{})); err != nil {\n\t\t\tt.Fatalf(\"PUT object `%s` failed: %q\", key, err)\n\t\t}\n\t}\n\tif o, err := s.Head(ctx, \"x/\"); err != nil {\n\t\tt.Fatalf(\"Head x/: %s\", err)\n\t} else if f, ok := o.(object.File); !ok {\n\t\tt.Fatalf(\"Head should return File\")\n\t} else if !f.IsDir() {\n\t\tt.Fatalf(\"x/ should be a dir\")\n\t}\n\t// cleanup\n\tdefer func() {\n\t\t// delete reversely, directory only can be deleted when it's empty\n\t\tobjs, err := listAll(ctx, s, \"\", \"\", 100)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"listall failed: %s\", err)\n\t\t}\n\t\tgottenKeys := make([]string, len(objs))\n\t\tfor idx, obj := range objs {\n\t\t\tgottenKeys[idx] = obj.Key()\n\t\t}\n\t\tidx := len(gottenKeys) - 1\n\t\tfor ; idx >= 0; idx-- {\n\t\t\tif err := s.Delete(ctx, gottenKeys[idx]); err != nil {\n\t\t\t\tt.Fatalf(\"DELETE object `%s` failed: %q\", gottenKeys[idx], err)\n\t\t\t}\n\t\t}\n\t}()\n\tobjs, err := listAll(ctx, s, \"x/\", \"\", 100)\n\tif err != nil {\n\t\tt.Fatalf(\"list failed: %s\", err)\n\t}\n\texpectedKeys := []string{\"x/\", \"x/x.txt\"}\n\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t}\n\n\tobjs, err = listAll(ctx, s, \"x\", \"\", 100)\n\tif err != nil {\n\t\tt.Fatalf(\"list failed: %s\", err)\n\t}\n\texpectedKeys = []string{\"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t}\n\n\tobjs, err = listAll(ctx, s, \"xy\", \"\", 100)\n\tif err != nil {\n\t\tt.Fatalf(\"list failed: %s\", err)\n\t}\n\texpectedKeys = []string{\"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t}\n\n\tif ss, ok := s.(object.SupportSymlink); ok {\n\t\t// a< a- < a/ < a0 < b< b- < b/ < b0\n\t\t_ = s.Put(ctx, \"a-\", bytes.NewReader([]byte{}))\n\t\t_ = s.Put(ctx, \"a0\", bytes.NewReader([]byte{}))\n\t\t_ = s.Put(ctx, \"b-\", bytes.NewReader([]byte{}))\n\t\t_ = s.Put(ctx, \"b0\", bytes.NewReader([]byte{}))\n\t\t_ = s.Put(ctx, \"xyz/ol1/p.txt\", bytes.NewReader([]byte{}))\n\t\tif err = ss.Symlink(\"./xyz/ol1/\", \"a\"); err != nil {\n\t\t\tt.Fatalf(\"symlink a %s\", err)\n\t\t}\n\t\tif target, err := ss.Readlink(\"a\"); err != nil || target != \"./xyz/ol1/\" {\n\t\t\tt.Fatalf(\"readlink a %s %s\", target, err)\n\t\t}\n\t\tif err = ss.Symlink(\"/xyz/notExist/\", \"b\"); err != nil {\n\t\t\tt.Fatalf(\"symlink b %s\", err)\n\t\t}\n\t\tif target, err := ss.Readlink(\"b\"); err != nil || target != \"/xyz/notExist/\" {\n\t\t\tt.Fatalf(\"readlink b %s %s\", target, err)\n\t\t}\n\t\thead, err := s.Head(ctx, \"a\")\n\t\tif err != nil || !head.IsSymlink() {\n\t\t\tt.Fatalf(\"head a %s %s\", head, err)\n\t\t}\n\t\tss.Symlink(\"notExit\", \"brokenLink\")\n\t\t_, err = s.Head(ctx, \"brokenLink\")\n\t\tif !errors.Is(err, os.ErrNotExist) {\n\t\t\tt.Fatalf(\"head b %s %s\", head, err)\n\t\t}\n\t\ts.Delete(ctx, \"brokenLink\")\n\t\tobjs, err = listAll(ctx, s, \"\", \"\", 100)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"listall failed: %s\", err)\n\t\t}\n\t\texpectedKeys = []string{\"\", \"a-\", \"a/\", \"a/p.txt\", \"a0\", \"b\", \"b-\", \"b0\", \"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/ol1/\", \"xyz/ol1/p.txt\", \"xyz/xyz.txt\"}\n\t\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t\t}\n\t}\n\n\t// put a file with very long name\n\tlongName := strings.Repeat(\"a\", 255)\n\tif err := s.Put(ctx, \"dir/\"+longName, bytes.NewReader([]byte{0})); err != nil {\n\t\tt.Fatalf(\"PUT a file with long name `%s` failed: %q\", longName, err)\n\t}\n}\n\nfunc TestJFS(t *testing.T) {\n\tm := meta.NewClient(\"memkv://\", nil)\n\tformat := &meta.Format{\n\t\tName: \"test\",\n\t\tBlockSize: 4096,\n\t\tCapacity: 1 << 30,\n\t\tDirStats: true,\n\t}\n\t_ = m.Init(format, true)\n\tvar conf = vfs.Config{\n\t\tMeta: meta.DefaultConf(),\n\t\tChunk: &chunk.Config{\n\t\t\tBlockSize: format.BlockSize << 10,\n\t\t\tMaxUpload: 1,\n\t\t\tMaxDownload: 200,\n\t\t\tBufferSize: 100 << 20,\n\t\t},\n\t\tDirEntryTimeout: time.Millisecond * 100,\n\t\tEntryTimeout: time.Millisecond * 100,\n\t\tAttrTimeout: time.Millisecond * 100,\n\t\tAccessLog: \"/tmp/juicefs.access.log\",\n\t}\n\tobjStore, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tstore := chunk.NewCachedStore(objStore, *conf.Chunk, nil)\n\tjfs, err := fs.NewFileSystem(&conf, m, store, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"initialize failed: %s\", err)\n\t}\n\n\tjstore := &juiceFS{object.DefaultObjectStorage{}, \"test\", uint16(utils.GetUmask()), jfs}\n\ttestFileSystem(t, jstore)\n\ttestFileSystem(t, object.WithPrefix(jstore, \"unittest/\"))\n}\n" }, { "path": "cmd/passfd.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"sync\"\n\t\"syscall\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\n// Get receives file descriptors from a Unix domain socket.\n//\n// Num specifies the expected number of file descriptors in one message.\n// Internal files' names to be assigned are specified via optional filenames\n// argument.\n//\n// You need to close all files in the returned slice. The slice can be\n// non-empty even if this function returns an error.\nfunc getFd(via *net.UnixConn, num int) ([]byte, []int, error) {\n\tif num < 1 {\n\t\treturn nil, nil, nil\n\t}\n\n\t// get the underlying socket\n\tviaf, err := via.File()\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tdefer viaf.Close()\n\tsocket := int(viaf.Fd())\n\n\t// recvmsg\n\tmsg := make([]byte, syscall.CmsgSpace(100))\n\toob := make([]byte, syscall.CmsgSpace(num*4))\n\tn, oobn, _, _, err := syscall.Recvmsg(socket, msg, oob, 0)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// parse control msgs\n\tmsgs, err := syscall.ParseSocketControlMessage(oob[:oobn])\n\n\t// convert fds to files\n\tfds := make([]int, 0, len(msgs))\n\tfor _, msg := range msgs {\n\t\tvar rights []int\n\t\trights, err = syscall.ParseUnixRights(&msg)\n\t\tfds = append(fds, rights...)\n\t\tif err != nil {\n\t\t\tfor i := range fds {\n\t\t\t\tsyscall.Close(fds[i])\n\t\t\t}\n\t\t\tfds = nil\n\t\t\tbreak\n\t\t}\n\t}\n\treturn msg[:n], fds, err\n}\n\n// putFd sends file descriptors to Unix domain socket.\n//\n// Please note that the number of descriptors in one message is limited\n// and is rather small.\nfunc putFd(via *net.UnixConn, msg []byte, fds ...int) error {\n\tif len(fds) == 0 {\n\t\treturn nil\n\t}\n\tviaf, err := via.File()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer viaf.Close()\n\tsocket := int(viaf.Fd())\n\trights := syscall.UnixRights(fds...)\n\treturn syscall.Sendmsg(socket, msg, rights, nil, 0)\n}\n\nvar fuseMu sync.Mutex\nvar fuseFd int = 0\nvar fuseSetting = []byte(\"FUSE\")\nvar serverAddress string = fmt.Sprintf(\"/tmp/fuse_fd_comm.%d\", os.Getpid())\nvar csiCommPath = os.Getenv(\"JFS_SUPER_COMM\")\n\nfunc handleFDRequest(conn *net.UnixConn) {\n\tdefer conn.Close()\n\tvar fds = []int{0}\n\tfuseMu.Lock()\n\tif fuseFd > 0 {\n\t\tfds = append(fds, fuseFd)\n\t\tlogger.Debugf(\"send FUSE fd: %d\", fuseFd)\n\t}\n\terr := putFd(conn, fuseSetting, fds...)\n\tif err != nil {\n\t\tfuseMu.Unlock()\n\t\tlogger.Errorf(\"send fuse fds: %s\", err)\n\t\treturn\n\t}\n\tif fuseFd > 0 {\n\t\t_ = syscall.Close(fuseFd)\n\t\tfuseFd = -1\n\t}\n\tfuseMu.Unlock()\n\n\tvar msg []byte\n\tmsg, fds, err = getFd(conn, 1)\n\tif err != nil {\n\t\tlogger.Debugf(\"recv fuse fds: %s\", err)\n\t\treturn\n\t}\n\tfuseMu.Lock()\n\tif string(msg) != \"CLOSE\" && fuseFd <= 0 && len(fds) == 1 {\n\t\tlogger.Debugf(\"recv FUSE fd: %d\", fds[0])\n\t\tfuseFd = fds[0]\n\t\tfuseSetting = msg\n\t\tif csiCommPath != \"\" {\n\t\t\terr = sendFuseFd(csiCommPath, fuseSetting, fuseFd)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"send fd to %s: %v\", csiCommPath, err)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tfor _, fd := range fds {\n\t\t\t_ = syscall.Close(fd)\n\t\t}\n\t\tlogger.Debugf(\"msg: %s fds: %+v\", string(msg), fds)\n\t}\n\tfuseMu.Unlock()\n}\n\nfunc serveFuseFD(path string) {\n\tif csiCommPath != \"\" {\n\t\tfd, fSetting := getFuseFd(csiCommPath)\n\t\tif fd > 0 {\n\t\t\tfuseFd, fuseSetting = fd, fSetting\n\t\t}\n\t}\n\t_ = os.Remove(path)\n\tsock, err := net.Listen(\"unix\", path)\n\tif err != nil {\n\t\tlogger.Error(err)\n\t\treturn\n\t}\n\tgo func() {\n\t\tdefer sock.Close()\n\t\tfor {\n\t\t\tconn, err := sock.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"accept : %s\", err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tgo handleFDRequest(conn.(*net.UnixConn))\n\t\t}\n\t}()\n}\n\nfunc getFuseFd(path string) (int, []byte) {\n\tif !utils.Exists(path) {\n\t\treturn -1, nil\n\t}\n\tconn, err := net.Dial(\"unix\", path)\n\tif err != nil {\n\t\tlogger.Warnf(\"dial %s: %s\", path, err)\n\t\treturn -1, nil\n\t}\n\tdefer conn.Close()\n\tmsg, fds, err := getFd(conn.(*net.UnixConn), 2)\n\tif err != nil {\n\t\tlogger.Warnf(\"recv fds: %s\", err)\n\t\treturn -1, nil\n\t}\n\t_ = syscall.Close(fds[0])\n\tif len(fds) > 1 {\n\t\t// for old version\n\t\t_ = putFd(conn.(*net.UnixConn), []byte(\"CLOSE\"), 0) // close it\n\t\tlogger.Debugf(\"recv FUSE fd: %d\", fds[1])\n\t\treturn fds[1], msg\n\t}\n\treturn 0, nil\n}\n\nfunc sendFuseFd(path string, msg []byte, fd int) error {\n\tconn, err := net.Dial(\"unix\", path)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer conn.Close()\n\t_, fds, err := getFd(conn.(*net.UnixConn), 2)\n\tif err != nil {\n\t\tlogger.Warnf(\"recv fds: %s\", err)\n\t\treturn err\n\t}\n\tfor _, fd := range fds {\n\t\t_ = syscall.Close(fd)\n\t}\n\tlogger.Debugf(\"send FUSE fd: %d\", fd)\n\treturn putFd(conn.(*net.UnixConn), msg, fd)\n}\n" }, { "path": "cmd/printsid.go", "content": "package cmd\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdPrintSID() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"printsid\",\n\t\tCategory: \"TOOL\",\n\t\tAction: printSID,\n\t\tUsage: \"Show SID info and the convected UID/GID for the current user.\",\n\t\tHidden: true,\n\t}\n}\n\nfunc printSID(ctx *cli.Context) error {\n\tif runtime.GOOS != \"windows\" {\n\t\treturn fmt.Errorf(\"printsid command is only supported on Windows\")\n\t}\n\n\tuserSid := utils.GetCurrentUserSIDStr()\n\tgroupSid := utils.GetCurrentUserGroupSIDStr()\n\tfmt.Printf(\"Current User SID: %s, UID: %d\\n\", userSid, utils.GetCurrentUID())\n\tfmt.Printf(\"Current Group SID: %s, GID: %d\\n\", groupSid, utils.GetCurrentGID())\n\n\treturn nil\n}\n" }, { "path": "cmd/profile.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdProfile() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"profile\",\n\t\tAction: profile,\n\t\tCategory: \"INSPECTOR\",\n\t\tUsage: \"Show profiling of operations completed in JuiceFS\",\n\t\tArgsUsage: \"MOUNTPOINT/LOGFILE\",\n\t\tDescription: `\nThis is a tool that analyzes access log of JuiceFS and shows an overview of recently completed operations.\n\nExamples:\n# Monitor real time operations\n$ juicefs profile /mnt/jfs\n\n# Replay an access log\n$ cat /mnt/jfs/.accesslog > /tmp/juicefs.accesslog\n# Press Ctrl-C to stop the \"cat\" command after some time\n$ juicefs profile /tmp/juicefs.accesslog\n\n# Analyze an access log and print the total statistics immediately\n$ juicefs profile /tmp/juicefs.accesslog --interval 0\n\nDetails: https://juicefs.com/docs/community/fault_diagnosis_and_analysis#profile`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"uid\",\n\t\t\t\tAliases: []string{\"u\"},\n\t\t\t\tUsage: \"track only specified UIDs(separated by comma ,)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"gid\",\n\t\t\t\tAliases: []string{\"g\"},\n\t\t\t\tUsage: \"track only specified GIDs(separated by comma ,)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"pid\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tUsage: \"track only specified PIDs(separated by comma ,)\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"paths\",\n\t\t\t\tAliases: []string{\"filter-by-path\"},\n\t\t\t\tUsage: \"track only specified paths (separated by comma , Only for Windows FUSE log)\",\n\t\t\t\tHidden: true,\n\t\t\t},\n\t\t\t&cli.Int64Flag{\n\t\t\t\tName: \"interval\",\n\t\t\t\tValue: 2,\n\t\t\t\tUsage: \"flush interval in seconds; set it to 0 when replaying a log file to get an immediate result\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nvar findDigits = regexp.MustCompile(`\\d+`)\n\ntype profiler struct {\n\tfile *os.File\n\treplay bool\n\tcolorful bool\n\tinterval time.Duration\n\tuids []string\n\tgids []string\n\tpids []string\n\tpaths []string\n\tentryChan chan *logEntry // one line\n\tstatsChan chan map[string]*stat\n\tpause chan bool\n\t/* --- for replay --- */\n\tprintTime chan time.Time\n\tdone chan bool\n}\n\ntype stat struct {\n\tcount int\n\ttotal int // total latency in 'us'\n}\n\ntype keyStat struct {\n\tkey string\n\tsPtr *stat\n}\n\ntype logEntry struct {\n\tts time.Time\n\tuid, gid, pid string\n\top string\n\tlatency int // us\n\tpath string // only for Windows FUSE log\n}\n\nfunc parseLine(line string, winFuseLog bool) *logEntry {\n\tif len(line) < 3 { // dummy line: \"#\"\n\t\treturn nil\n\t}\n\tfields := strings.Fields(line)\n\tif len(fields) < 5 {\n\t\tlogger.Warnf(\"Log line is invalid: %s\", line)\n\t\treturn nil\n\t}\n\tts, err := time.Parse(\"2006.01.02 15:04:05.000000\", strings.Join([]string{fields[0], fields[1]}, \" \"))\n\tif err != nil {\n\t\tlogger.Warnf(\"Failed to parse log line: %s: %s\", line, err)\n\t\treturn nil\n\t}\n\tids := findDigits.FindAllString(fields[2], 3) // e.g: [uid:0,gid:0,pid:36674]\n\tif len(ids) != 3 {\n\t\tlogger.Warnf(\"Log line is invalid: %s\", line)\n\t\treturn nil\n\t}\n\tlatStr := fields[len(fields)-1] // e.g: <0.000003>\n\tlatFloat, err := strconv.ParseFloat(latStr[1:len(latStr)-1], 64)\n\tif err != nil {\n\t\tlogger.Warnf(\"Failed to parse log line: %s: %s\", line, err)\n\t\treturn nil\n\t}\n\n\tfilePath := \"\"\n\tif winFuseLog {\n\t\t// Find the path in Windows log, should after the \"{op} (/xxxx/bb bb/cc cc.*)\"\n\t\t// the windows path may contain space or \"(\", \")\"\n\t\trestPart := strings.Join(fields[4:len(fields)-1], \" \")\n\t\tif strings.HasPrefix(restPart, \"(\") && strings.Contains(restPart, \")\") {\n\t\t\tlastIndex := strings.LastIndex(restPart, \")\")\n\t\t\tif lastIndex > 1 {\n\t\t\t\tpaths := strings.SplitN(restPart[1:lastIndex], \",\", 2)\n\t\t\t\tif len(paths) > 0 {\n\t\t\t\t\tfilePath = paths[0]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif filePath == \"\" {\n\t\t\tlogger.Warnf(\"log line is invalid, cannot find path: %s\", line)\n\t\t}\n\t}\n\n\treturn &logEntry{\n\t\tts: ts,\n\t\tuid: ids[0],\n\t\tgid: ids[1],\n\t\tpid: ids[2],\n\t\top: fields[3],\n\t\tlatency: int(latFloat * 1000000.0),\n\t\tpath: filePath,\n\t}\n}\n\nfunc (p *profiler) reader() {\n\tscanner := bufio.NewScanner(p.file)\n\tfor scanner.Scan() {\n\t\tp.entryChan <- parseLine(scanner.Text(), p.isWinFuseLog())\n\t}\n\tif err := scanner.Err(); err != nil {\n\t\tlogger.Fatalf(\"Reading log file failed with error: %s\", err)\n\t}\n\tclose(p.entryChan)\n\tif p.replay {\n\t\tp.done <- true\n\t}\n}\n\nfunc (p *profiler) isWinFuseLog() bool {\n\treturn len(p.paths) > 0\n}\n\nfunc (p *profiler) isValid(entry *logEntry) bool {\n\tvalid := func(f []string, e string) bool {\n\t\tif len(f) == 1 && f[0] == \"\" {\n\t\t\treturn true\n\t\t}\n\t\tfor _, v := range f {\n\t\t\tif v == e {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n\treturn valid(p.uids, entry.uid) && valid(p.gids, entry.gid) && valid(p.pids, entry.pid) && valid(p.paths, entry.path)\n}\n\nfunc (p *profiler) counter() {\n\tvar edge time.Time\n\tstats := make(map[string]*stat)\n\tfor {\n\t\tselect {\n\t\tcase entry := <-p.entryChan:\n\t\t\tif entry == nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif !p.isValid(entry) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif p.replay {\n\t\t\t\tif edge.IsZero() {\n\t\t\t\t\tedge = entry.ts.Add(p.interval)\n\t\t\t\t}\n\t\t\t\tfor ; entry.ts.After(edge); edge = edge.Add(p.interval) {\n\t\t\t\t\tp.statsChan <- stats\n\t\t\t\t\tp.printTime <- edge\n\t\t\t\t\tstats = make(map[string]*stat)\n\t\t\t\t}\n\t\t\t}\n\t\t\tvalue, ok := stats[entry.op]\n\t\t\tif !ok {\n\t\t\t\tvalue = &stat{}\n\t\t\t\tstats[entry.op] = value\n\t\t\t}\n\t\t\tvalue.count++\n\t\t\tvalue.total += entry.latency\n\t\tcase p.statsChan <- stats:\n\t\t\tif p.replay {\n\t\t\t\tp.printTime <- edge\n\t\t\t\tedge = edge.Add(p.interval)\n\t\t\t}\n\t\t\tstats = make(map[string]*stat)\n\t\t}\n\t}\n}\n\nfunc (p *profiler) fastCounter() {\n\tvar start, last time.Time\n\tstats := make(map[string]*stat)\n\tfor entry := range p.entryChan {\n\t\tif entry == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif !p.isValid(entry) {\n\t\t\tcontinue\n\t\t}\n\t\tif start.IsZero() {\n\t\t\tstart = entry.ts\n\t\t}\n\t\tlast = entry.ts\n\t\tvalue, ok := stats[entry.op]\n\t\tif !ok {\n\t\t\tvalue = &stat{}\n\t\t\tstats[entry.op] = value\n\t\t}\n\t\tvalue.count++\n\t\tvalue.total += entry.latency\n\t}\n\tp.statsChan <- stats\n\tp.printTime <- start\n\tp.printTime <- last\n}\n\nfunc colorize1(msg string, color int) string {\n\treturn fmt.Sprintf(\"%s%dm%s%s\", COLOR_SEQ, color, msg, RESET_SEQ)\n}\n\nfunc printLines(lines []string, colorful bool) {\n\tif colorful {\n\t\tfmt.Print(CLEAR_SCREEM)\n\t\tfmt.Println(colorize1(lines[0], GREEN))\n\t\tfmt.Println(colorize1(lines[1], YELLOW))\n\t\tfmt.Println(colorize1(lines[2], BLUE))\n\t\tif len(lines) > 3 {\n\t\t\tfor _, l := range lines[3:] {\n\t\t\t\tfmt.Println(colorize1(l, BLACK))\n\t\t\t}\n\t\t}\n\t} else {\n\t\tfmt.Println(lines[0])\n\t\tfor _, l := range lines[2:] {\n\t\t\tfmt.Println(l)\n\t\t}\n\t\tfmt.Println()\n\t}\n}\n\nfunc (p *profiler) flush(timeStamp time.Time, keyStats []keyStat, done bool) {\n\tvar head string\n\tif p.replay {\n\t\tif done {\n\t\t\thead = \"(replay done)\"\n\t\t} else {\n\t\t\thead = \"(replaying)\"\n\t\t}\n\t}\n\toutput := make([]string, 3)\n\toutput[0] = fmt.Sprintf(\"> JuiceFS Profiling %13s Refresh: %.0f seconds %20s\",\n\t\thead, p.interval.Seconds(), timeStamp.Format(\"2006-01-02T15:04:05\"))\n\toutput[2] = fmt.Sprintf(\"%-14s %10s %15s %18s %14s\", \"Operation\", \"Count\", \"Average(us)\", \"Total(us)\", \"Percent(%)\")\n\tfor _, s := range keyStats {\n\t\toutput = append(output, fmt.Sprintf(\"%-14s %10d %15.0f %18d %14.1f\",\n\t\t\ts.key, s.sPtr.count, float64(s.sPtr.total)/float64(s.sPtr.count), s.sPtr.total, float64(s.sPtr.total)/float64(p.interval.Microseconds())*100.0))\n\t}\n\tif p.replay {\n\t\toutput[1] = fmt.Sprintln(\"\\n[enter]Pause/Continue\")\n\t}\n\tprintLines(output, p.colorful)\n}\n\nfunc (p *profiler) flusher() {\n\tvar paused, done bool\n\tticker := time.NewTicker(p.interval)\n\tts := time.Now()\n\tp.flush(ts, nil, false)\n\tfor {\n\t\tselect {\n\t\tcase t := <-ticker.C:\n\t\t\tstats := <-p.statsChan\n\t\t\tif paused { // ticker event might be passed long ago\n\t\t\t\tpaused = false\n\t\t\t\tticker.Stop()\n\t\t\t\tticker = time.NewTicker(p.interval)\n\t\t\t\tt = time.Now()\n\t\t\t}\n\t\t\tif done {\n\t\t\t\tticker.Stop()\n\t\t\t}\n\t\t\tif p.replay {\n\t\t\t\tts = <-p.printTime\n\t\t\t} else {\n\t\t\t\tts = t\n\t\t\t}\n\t\t\tkeyStats := make([]keyStat, 0, len(stats))\n\t\t\tfor k, s := range stats {\n\t\t\t\tkeyStats = append(keyStats, keyStat{k, s})\n\t\t\t}\n\t\t\tsort.Slice(keyStats, func(i, j int) bool { // reversed\n\t\t\t\treturn keyStats[i].sPtr.total > keyStats[j].sPtr.total\n\t\t\t})\n\t\t\tp.flush(ts, keyStats, done)\n\t\t\tif done {\n\t\t\t\tos.Exit(0)\n\t\t\t}\n\t\tcase paused = <-p.pause:\n\t\t\tfmt.Printf(\"\\n\\033[97mPaused. Press [enter] to continue.\\n\\033[0m\")\n\t\t\t<-p.pause\n\t\tcase done = <-p.done:\n\t\t}\n\t}\n}\n\nfunc profile(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tlogPath := ctx.Args().First()\n\tst, err := os.Stat(logPath)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Failed to stat path %s: %s\", logPath, err)\n\t}\n\tvar replay bool\n\tif st.IsDir() { // mount point\n\t\tinode, err := utils.GetFileInode(logPath)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"Failed to lookup inode for %s: %s\", logPath, err)\n\t\t}\n\t\tif inode != uint64(meta.RootInode) {\n\t\t\tlogger.Fatalf(\"Path %s is not a mount point!\", logPath)\n\t\t}\n\t\tif p := filepath.Join(logPath, \".jfs.accesslog\"); utils.Exists(p) {\n\t\t\tlogPath = p\n\t\t} else {\n\t\t\tlogPath = filepath.Join(logPath, \".accesslog\")\n\t\t}\n\t} else { // log file to be replayed\n\t\treplay = true\n\t}\n\tnodelay := ctx.Int64(\"interval\") == 0\n\tif nodelay && !replay {\n\t\tlogger.Fatalf(\"Interval must be > 0 for real time mode!\")\n\t}\n\tfile, err := os.Open(logPath)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Failed to open log file %s: %s\", logPath, err)\n\t}\n\tdefer file.Close()\n\n\tprof := profiler{\n\t\tfile: file,\n\t\treplay: replay,\n\t\tcolorful: utils.SupportANSIColor(os.Stdout.Fd()),\n\t\tinterval: time.Second * time.Duration(ctx.Int64(\"interval\")),\n\t\tuids: strings.Split(ctx.String(\"uid\"), \",\"),\n\t\tgids: strings.Split(ctx.String(\"gid\"), \",\"),\n\t\tpids: strings.Split(ctx.String(\"pid\"), \",\"),\n\t\tpaths: strings.Split(ctx.String(\"paths\"), \",\"),\n\t\tentryChan: make(chan *logEntry, 16),\n\t\tstatsChan: make(chan map[string]*stat),\n\t\tpause: make(chan bool),\n\t}\n\tif prof.replay {\n\t\tprof.printTime = make(chan time.Time)\n\t\tprof.done = make(chan bool)\n\t}\n\n\tgo prof.reader()\n\tif nodelay {\n\t\tgo prof.fastCounter()\n\t\tstats := <-prof.statsChan\n\t\tstart := <-prof.printTime\n\t\tlast := <-prof.printTime\n\t\tkeyStats := make([]keyStat, 0, len(stats))\n\t\tfor k, s := range stats {\n\t\t\tkeyStats = append(keyStats, keyStat{k, s})\n\t\t}\n\t\tsort.Slice(keyStats, func(i, j int) bool { // reversed\n\t\t\treturn keyStats[i].sPtr.total > keyStats[j].sPtr.total\n\t\t})\n\t\tprof.replay = false\n\t\tprof.interval = last.Sub(start)\n\t\tprof.flush(last, keyStats, <-prof.done)\n\t\treturn nil\n\t}\n\n\tgo prof.counter()\n\tgo prof.flusher()\n\tvar input string\n\tfor {\n\t\t_, _ = fmt.Scanln(&input)\n\t\tif prof.colorful {\n\t\t\tfmt.Print(\"\\033[1A\\033[K\") // move cursor back\n\t\t}\n\t\tif prof.replay {\n\t\t\tprof.pause <- true // pause/continue\n\t\t}\n\t}\n}\n" }, { "path": "cmd/quota.go", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdQuota() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"quota\",\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"Manage directory quotas\",\n\t\tArgsUsage: \"META-URL\",\n\t\tHideHelpCommand: true,\n\t\tDescription: `\nExamples:\n$ juicefs quota set redis://localhost --path /dir1 --capacity 1 --inodes 100\n$ juicefs quota get redis://localhost --path /dir1\n$ juicefs quota list redis://localhost\n$ juicefs quota delete redis://localhost --path /dir1\n$ juicefs quota check redis://localhost --path /dir1 --repair\n$ juicefs quota set redis://localhost --uid 1000 --capacity 2 --inodes 200\n$ juicefs quota get redis://localhost --uid 1000\n$ juicefs quota delete redis://localhost --uid 1000\n$ juicefs quota set redis://localhost --gid 100 --capacity 5 --inodes 500\n$ juicefs quota get redis://localhost --gid 100\n$ juicefs quota delete redis://localhost --gid 100`,\n\t\tSubcommands: []*cli.Command{\n\t\t\t{\n\t\t\t\tName: \"set\",\n\t\t\t\tUsage: \"Set quota to a directory, user, or group\",\n\t\t\t\tArgsUsage: \"META-URL\",\n\t\t\t\tAction: quota,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"get\",\n\t\t\t\tUsage: \"Get quota of a directory, user, or group\",\n\t\t\t\tArgsUsage: \"META-URL\",\n\t\t\t\tAction: quota,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"delete\",\n\t\t\t\tAliases: []string{\"del\"},\n\t\t\t\tUsage: \"Delete quota of a directory, user, or group\",\n\t\t\t\tArgsUsage: \"META-URL\",\n\t\t\t\tAction: quota,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"list\",\n\t\t\t\tAliases: []string{\"ls\"},\n\t\t\t\tUsage: \"List all quotas (directory, user, and group)\",\n\t\t\t\tArgsUsage: \"META-URL\",\n\t\t\t\tAction: quota,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"check\",\n\t\t\t\tUsage: \"Check quota consistency of a directory, user, or group\",\n\t\t\t\tArgsUsage: \"META-URL\",\n\t\t\t\tAction: quota,\n\t\t\t},\n\t\t},\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"path\",\n\t\t\t\tUsage: \"full path of the directory within the volume\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"create\",\n\t\t\t\tUsage: \"create the directory if not exists\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"capacity\",\n\t\t\t\tUsage: \"hard quota of the directory limiting its usage of space in GiB\",\n\t\t\t},\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"inodes\",\n\t\t\t\tUsage: \"hard quota of the directory limiting its number of inodes\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"repair\",\n\t\t\t\tUsage: \"repair inconsistent quota\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"strict\",\n\t\t\t\tUsage: \"calculate total usage of directory in strict mode (NOTE: may be slow for huge directory)\",\n\t\t\t},\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"uid\",\n\t\t\t\tUsage: \"user ID for user quota management\",\n\t\t\t},\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"gid\",\n\t\t\t\tUsage: \"group ID for group quota management\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc quota(c *cli.Context) error {\n\tsetup(c, 1)\n\tvar cmd uint8\n\tswitch c.Command.Name {\n\tcase \"set\":\n\t\tcmd = meta.QuotaSet\n\tcase \"get\":\n\t\tcmd = meta.QuotaGet\n\tcase \"delete\":\n\t\tcmd = meta.QuotaDel\n\tcase \"list\":\n\t\tcmd = meta.QuotaList\n\tcase \"check\":\n\t\tcmd = meta.QuotaCheck\n\tdefault:\n\t\tlogger.Fatalf(\"Invalid quota command: %s\", c.Command.Name)\n\t}\n\n\tvar uid, gid uint32\n\tvar quotaKey string\n\tvar quotaType string\n\tvalidateID := func(name string) uint32 {\n\t\tid := c.Uint64(name)\n\t\tif id == 0 {\n\t\t\tlogger.Fatalf(\"Invalid --%s: 0 is not allowed\", name)\n\t\t}\n\t\tif id > math.MaxUint32 {\n\t\t\tlogger.Fatalf(\"Invalid --%s: %d exceeds maximum value %d\", name, id, math.MaxUint32)\n\t\t}\n\t\treturn uint32(id)\n\t}\n\tif c.IsSet(\"uid\") {\n\t\tuid = validateID(\"uid\")\n\t\tquotaKey = fmt.Sprintf(\"uid:%d\", uid)\n\t\tquotaType = \"user\"\n\t\tif c.IsSet(\"gid\") {\n\t\t\tlogger.Fatalf(\"Cannot specify both --uid and --gid at the same time\")\n\t\t}\n\t\tif c.IsSet(\"path\") {\n\t\t\tlogger.Fatalf(\"Cannot specify both --uid and --path at the same time\")\n\t\t}\n\t} else if c.IsSet(\"gid\") {\n\t\tgid = validateID(\"gid\")\n\t\tquotaKey = fmt.Sprintf(\"gid:%d\", gid)\n\t\tquotaType = \"group\"\n\t\tif c.IsSet(\"path\") {\n\t\t\tlogger.Fatalf(\"Cannot specify both --gid and --path at the same time\")\n\t\t}\n\t} else {\n\t\tdpath := c.String(\"path\")\n\t\tif dpath == \"\" && cmd != meta.QuotaList {\n\t\t\tlogger.Fatalf(\"Please specify the directory with `--path ` option\")\n\t\t}\n\t\tquotaKey = dpath\n\t\tquotaType = \"directory\"\n\t}\n\n\tremovePassword(c.Args().Get(0))\n\n\tm := meta.NewClient(c.Args().Get(0), nil)\n\t_, err := m.Load(true)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Load setting: %s\", err)\n\t}\n\tqs := make(map[string]*meta.Quota)\n\tvar strict, repair bool\n\tif cmd == meta.QuotaSet {\n\t\tstrict = c.Bool(\"strict\")\n\t\tq := &meta.Quota{MaxSpace: -1, MaxInodes: -1} // negative means no change\n\t\tif c.IsSet(\"capacity\") {\n\t\t\tq.MaxSpace = int64(utils.ParseBytes(c, \"capacity\", 'G'))\n\t\t}\n\t\tif c.IsSet(\"inodes\") {\n\t\t\tq.MaxInodes = int64(c.Uint64(\"inodes\"))\n\t\t}\n\t\tqs[quotaKey] = q\n\t} else if cmd == meta.QuotaCheck {\n\t\tstrict = c.Bool(\"strict\")\n\t\trepair = c.Bool(\"repair\")\n\t}\n\n\tif err := m.HandleQuota(meta.Background(), cmd, quotaKey, uid, gid, qs, strict, repair, c.Bool(\"create\")); err != nil {\n\t\treturn err\n\t} else if len(qs) == 0 {\n\t\treturn nil\n\t}\n\n\tresult := make([][]string, 1, len(qs)+1)\n\n\tif quotaType == \"user\" {\n\t\tresult[0] = []string{\"User ID\", \"Size\", \"Used\", \"Use%\", \"Inodes\", \"IUsed\", \"IUse%\"}\n\t} else if quotaType == \"group\" {\n\t\tresult[0] = []string{\"Group ID\", \"Size\", \"Used\", \"Use%\", \"Inodes\", \"IUsed\", \"IUse%\"}\n\t} else {\n\t\tresult[0] = []string{\"Path\", \"Size\", \"Used\", \"Use%\", \"Inodes\", \"IUsed\", \"IUse%\"}\n\t}\n\n\tpaths := make([]string, 0, len(qs))\n\tfor p := range qs {\n\t\tpaths = append(paths, p)\n\t}\n\tsort.Strings(paths)\n\tfor _, p := range paths {\n\t\tq := qs[p]\n\t\tif q.UsedSpace < 0 {\n\t\t\tlogger.Warnf(\"Used space of %s is negative (%d), please run `juicefs quota check` to fix it\", p, q.UsedSpace)\n\t\t\tq.UsedSpace = 0\n\t\t}\n\t\tif q.UsedInodes < 0 {\n\t\t\tlogger.Warnf(\"Used inodes of %s is negative (%d), please run `juicefs quota check` to fix it\", p, q.UsedInodes)\n\t\t\tq.UsedInodes = 0\n\t\t}\n\t\tused := humanize.IBytes(uint64(q.UsedSpace))\n\t\tvar size, usedR string\n\t\tif q.MaxSpace > 0 {\n\t\t\tsize = humanize.IBytes(uint64(q.MaxSpace))\n\t\t\tusedR = fmt.Sprintf(\"%d%%\", q.UsedSpace*100/q.MaxSpace)\n\t\t} else {\n\t\t\tsize = \"unchanged\"\n\t\t}\n\t\tiused := humanize.Comma(q.UsedInodes)\n\t\tvar itotal, iusedR string\n\t\tif q.MaxInodes > 0 {\n\t\t\titotal = humanize.Comma(q.MaxInodes)\n\t\t\tiusedR = fmt.Sprintf(\"%d%%\", q.UsedInodes*100/q.MaxInodes)\n\t\t} else {\n\t\t\titotal = \"unchanged\"\n\t\t}\n\n\t\tvar identifier string\n\t\tif strings.HasPrefix(p, \"uid:\") {\n\t\t\tidentifier = fmt.Sprintf(\"UID:%s\", strings.TrimPrefix(p, \"uid:\"))\n\t\t} else if strings.HasPrefix(p, \"gid:\") {\n\t\t\tidentifier = fmt.Sprintf(\"GID:%s\", strings.TrimPrefix(p, \"gid:\"))\n\t\t} else {\n\t\t\tidentifier = p\n\t\t}\n\t\tresult = append(result, []string{identifier, size, used, usedR, itotal, iused, iusedR})\n\t}\n\tprintResult(result, 0, false)\n\treturn nil\n}\n" }, { "path": "cmd/restore.go", "content": "package cmd\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"os\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdRestore() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"restore\",\n\t\tAction: restore,\n\t\tCategory: \"ADMIN\",\n\t\tUsage: \"restore files from trash\",\n\t\tArgsUsage: \"META HOUR ...\",\n\t\tDescription: `\nRebuild the tree structure for trash files, and put them back to original directories.\n\nExamples:\n$ juicefs restore redis://localhost/1 2023-05-10-01`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"put-back\",\n\t\t\t\tUsage: \"move the recovered files into original directory\",\n\t\t\t},\n\t\t\t&cli.IntFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"number of threads\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc restore(ctx *cli.Context) error {\n\tsetup0(ctx, 2, 0)\n\tif runtime.GOOS == \"windows\" && !utils.IsWinAdminOrElevatedPrivilege() {\n\t\treturn fmt.Errorf(\"restore command requires Administrator or elevated privilege on Windows\")\n\t}\n\tif os.Getuid() != 0 && runtime.GOOS != \"windows\" {\n\t\treturn fmt.Errorf(\"only root can restore files from trash\")\n\t}\n\tremovePassword(ctx.Args().Get(0))\n\tm := meta.NewClient(ctx.Args().Get(0), nil)\n\t_, err := m.Load(true)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor i := 1; i < ctx.NArg(); i++ {\n\t\thour := ctx.Args().Get(i)\n\t\tdoRestore(m, hour, ctx.Bool(\"put-back\"), ctx.Int(\"threads\"))\n\t}\n\treturn nil\n}\n\nfunc doRestore(m meta.Meta, hour string, putBack bool, threads int) {\n\tif err := m.NewSession(false); err != nil {\n\t\tlogger.Warningf(\"running without sessions because fail to new session: %s\", err)\n\t} else {\n\t\tdefer func() {\n\t\t\t_ = m.CloseSession()\n\t\t}()\n\t}\n\tlogger.Infof(\"restore files in %s ...\", hour)\n\tctx := meta.Background()\n\tvar parent meta.Ino\n\tvar attr meta.Attr\n\terr := m.Lookup(ctx, meta.TrashInode, hour, &parent, &attr, false)\n\tif err != 0 {\n\t\tlogger.Errorf(\"lookup %s: %s\", hour, err)\n\t\treturn\n\t}\n\tvar entries []*meta.Entry\n\terr = m.Readdir(meta.Background(), parent, 0, &entries)\n\tif err != 0 {\n\t\tlogger.Errorf(\"list %s: %s\", hour, err)\n\t\treturn\n\t}\n\tentries = entries[2:]\n\t// to avoid conflict\n\trand.Shuffle(len(entries), func(i, j int) {\n\t\tentries[i], entries[j] = entries[j], entries[i]\n\t})\n\n\tvar parents = make(map[meta.Ino]bool)\n\tif !putBack {\n\t\tfor _, e := range entries {\n\t\t\tif e.Attr.Typ == meta.TypeDirectory {\n\t\t\t\tparents[e.Inode] = true\n\t\t\t}\n\t\t}\n\t}\n\n\ttodo := make(chan *meta.Entry, 1000)\n\tp := utils.NewProgress(false)\n\trestored := p.AddCountBar(\"restored\", int64(len(entries)))\n\tskipped := p.AddCountSpinner(\"skipped\")\n\tfailed := p.AddCountSpinner(\"failed\")\n\tvar mu sync.Mutex\n\trestoredTo := make(map[meta.Ino]int)\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < threads; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor e := range todo {\n\t\t\t\tps := bytes.SplitN(e.Name, []byte(\"-\"), 3)\n\t\t\t\tdst, _ := strconv.Atoi(string(ps[0]))\n\t\t\t\tif putBack || parents[meta.Ino(dst)] {\n\t\t\t\t\terr = m.Rename(ctx, parent, string(e.Name), meta.Ino(dst), string(ps[2]), meta.RenameNoReplace|meta.RenameRestore, nil, nil)\n\t\t\t\t\tif err != 0 {\n\t\t\t\t\t\tlogger.Warnf(\"restore %s: %s\", string(e.Name), err)\n\t\t\t\t\t\tfailed.Increment()\n\t\t\t\t\t} else {\n\t\t\t\t\t\trestored.Increment()\n\t\t\t\t\t\tmu.Lock()\n\t\t\t\t\t\trestoredTo[meta.Ino(dst)] += 1\n\t\t\t\t\t\tmu.Unlock()\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tskipped.Increment()\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tfor _, e := range entries {\n\t\ttodo <- e\n\t}\n\tclose(todo)\n\twg.Wait()\n\tfailed.Done()\n\tskipped.Done()\n\trestored.Done()\n\tp.Done()\n\tlogger.Infof(\"restored %d files in %s\", restored.Current(), hour)\n\tfor dst, count := range restoredTo {\n\t\tlogger.Infof(\"restored %d files to %q\", count, strings.Join(m.GetPaths(ctx, dst), \", \"))\n\t}\n}\n" }, { "path": "cmd/restore_test.go", "content": "package cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestRestore(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tpaths := []string{\"/jfs-dir\", \"/jfs-dir/a\"}\n\tif err := os.Mkdir(fmt.Sprintf(\"%s%s\", testMountPoint, \"/jfs-dir\"), 0777); err != nil {\n\t\tt.Fatalf(\"mkdirAll err: %s\", err)\n\t}\n\n\tfilename := fmt.Sprintf(\"%s%s\", testMountPoint, \"/jfs-dir/a\")\n\tif err := os.WriteFile(filename, []byte(\"test\"), 0644); err != nil {\n\t\tt.Fatalf(\"write file failed: %s\", err)\n\t}\n\n\tfor i := len(paths) - 1; i >= 0; i-- {\n\t\tpath := paths[i]\n\t\tif err := os.Remove(fmt.Sprintf(\"%s%s\", testMountPoint, path)); err != nil {\n\t\t\tt.Fatalf(\"removeAll err: %s\", err)\n\t\t}\n\t}\n\n\thour := time.Now().UTC().Format(\"2006-01-02-15\")\n\trestoreArgs := []string{\"\", \"restore\", testMeta, hour}\n\tif err := Main(restoreArgs); err != nil {\n\t\tt.Fatalf(\"restore failed: %s\", err)\n\t}\n\n\thourDir := fmt.Sprintf(\"%s/%s/%s\", testMountPoint, \".trash\", hour)\n\tchild, err := os.ReadDir(hourDir)\n\tif err != nil {\n\t\tt.Fatalf(\"read dir failed: %s\", err)\n\t}\n\tfor _, entry := range child {\n\t\tif strings.Contains(entry.Name(), \"jfs-dir\") {\n\t\t\tfileInfo, err := os.Stat(fmt.Sprintf(\"%s/%s/%s\", hourDir, entry.Name(), \"a\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"stat failed: %s\", err)\n\t\t\t}\n\t\t\tif fileInfo.IsDir() {\n\t\t\t\tt.Fatalf(\"restore failed, file: %v is dir\", fileInfo)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatalf(\"restore failed, cannot find file: %s in trash\", \"jfs-dir\")\n}\n\nfunc TestRestorePutBack(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tpaths := []string{\"/jfs-dir1\", \"/jfs-dir1/a\"}\n\tif err := os.Mkdir(fmt.Sprintf(\"%s%s\", testMountPoint, \"/jfs-dir1\"), 0777); err != nil {\n\t\tt.Fatalf(\"mkdirAll err: %s\", err)\n\t}\n\n\tfilename := fmt.Sprintf(\"%s%s\", testMountPoint, \"/jfs-dir1/a\")\n\tif err := os.WriteFile(filename, []byte(\"test\"), 0644); err != nil {\n\t\tt.Fatalf(\"write file failed: %s\", err)\n\t}\n\n\tfor i := len(paths) - 1; i >= 0; i-- {\n\t\tpath := paths[i]\n\t\tif err := os.Remove(fmt.Sprintf(\"%s%s\", testMountPoint, path)); err != nil {\n\t\t\tt.Fatalf(\"removeAll err: %s\", err)\n\t\t}\n\t}\n\n\thour := time.Now().UTC().Format(\"2006-01-02-15\")\n\trestoreArgs := []string{\"\", \"restore\", testMeta, hour, \"--put-back=true\"}\n\tif err := Main(restoreArgs); err != nil {\n\t\tt.Fatalf(\"restore failed: %s\", err)\n\t}\n\n\tfileInfo, err := os.Stat(fmt.Sprintf(\"%s%s\", testMountPoint, \"/jfs-dir1/a\"))\n\tif err != nil {\n\t\tt.Fatalf(\"stat failed: %s\", err)\n\t}\n\tif fileInfo.IsDir() {\n\t\tt.Fatalf(\"restore failed, file: %v is dir\", fileInfo)\n\t}\n}\n" }, { "path": "cmd/rmr.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdRmr() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"rmr\",\n\t\tAction: rmr,\n\t\tCategory: \"TOOL\",\n\t\tUsage: \"Remove directories recursively\",\n\t\tArgsUsage: \"PATH ...\",\n\t\tDescription: `\nThis command provides a faster way to remove huge directories in JuiceFS.\n\nExamples:\n$ juicefs rmr /mnt/jfs/foo`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"skip-trash\",\n\t\t\t\tUsage: \"skip trash and delete files directly (requires root)\",\n\t\t\t},\n\t\t\t&cli.IntFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tValue: 50,\n\t\t\t\tUsage: \"number of threads for delete jobs (max 255)\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc openController(dpath string) (*os.File, error) {\n\tst, err := os.Stat(dpath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif !st.IsDir() {\n\t\tdpath = filepath.Dir(dpath)\n\t}\n\tfp, err := os.OpenFile(filepath.Join(dpath, \".jfs.control\"), os.O_RDWR, 0)\n\tif os.IsNotExist(err) {\n\t\tfp, err = os.OpenFile(filepath.Join(dpath, \".control\"), os.O_RDWR, 0)\n\t}\n\treturn fp, err\n}\n\nfunc rmr(ctx *cli.Context) error {\n\tsetup0(ctx, 1, 0)\n\tvar flag uint8\n\tvar numThreads int\n\n\tnumThreads = ctx.Int(\"threads\")\n\tif numThreads <= 0 {\n\t\tnumThreads = meta.RmrDefaultThreads\n\t}\n\tif numThreads > 255 {\n\t\tnumThreads = 255\n\t}\n\tif ctx.Bool(\"skip-trash\") {\n\t\tif runtime.GOOS != \"windows\" && os.Getuid() != 0 {\n\t\t\tlogger.Fatalf(\"Only root can remove files directly\")\n\t\t} else if runtime.GOOS == \"windows\" && !utils.IsWinAdminOrElevatedPrivilege() {\n\t\t\tlogger.Fatalf(\"Removing files directly requires Administrator or elevated privilege on Windows\")\n\t\t}\n\t\tflag = 1\n\t}\n\tprogress := utils.NewProgress(false)\n\tspin := progress.AddCountSpinner(\"Removing entries\")\n\tfor i := 0; i < ctx.Args().Len(); i++ {\n\t\tpath := ctx.Args().Get(i)\n\t\tp, err := filepath.Abs(path)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"abs of %s: %s\", path, err)\n\t\t\tcontinue\n\t\t}\n\t\td := filepath.Dir(p)\n\t\tname := filepath.Base(p)\n\t\tinode, err := utils.GetFileInode(d)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"lookup inode for %s: %s\", d, err)\n\t\t}\n\t\tf, err := openController(d)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Open control file for %s: %s\", d, err)\n\t\t\tcontinue\n\t\t}\n\t\twb := utils.NewBuffer(8 + 8 + 1 + uint32(len(name)) + 1 + 1)\n\t\twb.Put32(meta.Rmr)\n\t\twb.Put32(8 + 1 + uint32(len(name)) + 1 + 1)\n\t\twb.Put64(inode)\n\t\twb.Put8(uint8(len(name)))\n\t\twb.Put([]byte(name))\n\t\twb.Put8(flag)\n\t\twb.Put8(uint8(numThreads))\n\t\t_, err = f.Write(wb.Bytes())\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"write message: %s\", err)\n\t\t}\n\t\tif _, errno := readProgress(f, func(count, bytes uint64) {\n\t\t\tspin.SetCurrent(int64(count))\n\t\t}); errno != 0 {\n\t\t\tlogger.Fatalf(\"RMR %s: %s\", path, errno)\n\t\t}\n\t\t_ = f.Close()\n\t}\n\tprogress.Done()\n\treturn nil\n}\n" }, { "path": "cmd/rmr_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestRmr(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tpaths := []string{\"/dir1\", \"/dir2\", \"/dir3/dir2\"}\n\tfor _, path := range paths {\n\t\tif err := os.MkdirAll(fmt.Sprintf(\"%s%s/dir2/dir3/dir4/dir5\", testMountPoint, path), 0777); err != nil {\n\t\t\tt.Fatalf(\"mkdirAll err: %s\", err)\n\t\t}\n\t}\n\tfor i := 0; i < 5; i++ {\n\t\tfilename := fmt.Sprintf(\"%s/dir1/f%d.txt\", testMountPoint, i)\n\t\tif err := os.WriteFile(filename, []byte(\"test\"), 0644); err != nil {\n\t\t\tt.Fatalf(\"write file failed: %s\", err)\n\t\t}\n\t}\n\n\trmrArgs := []string{\"\", \"rmr\", testMountPoint + paths[0], testMountPoint + paths[1], testMountPoint + paths[2]}\n\tif err := Main(rmrArgs); err != nil {\n\t\tt.Fatalf(\"rmr failed: %s\", err)\n\t}\n\n\tfor _, path := range paths {\n\t\tif dir, err := os.ReadDir(testMountPoint + path); !os.IsNotExist(err) {\n\t\t\tt.Fatalf(\"test rmr error: %s len(dir): %d\", err, len(dir))\n\t\t}\n\t}\n}\n" }, { "path": "cmd/stats.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdStats() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"stats\",\n\t\tAction: stats,\n\t\tCategory: \"INSPECTOR\",\n\t\tUsage: \"Show real time performance statistics of JuiceFS\",\n\t\tArgsUsage: \"MOUNTPOINT\",\n\t\tDescription: `\nThis is a tool that reads Prometheus metrics and shows real time statistics of the target mount point.\n\nExamples:\n$ juicefs stats /mnt/jfs\n\n# More metrics\n$ juicefs stats /mnt/jfs -l 1\n\nDetails: https://juicefs.com/docs/community/fault_diagnosis_and_analysis#stats`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"schema\",\n\t\t\t\tValue: \"ufmco\",\n\t\t\t\tUsage: \"schema string of output sections (t:time, u: usage, f: fuse, m: meta, c: blockcache, o: object, g: go)\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"interval\",\n\t\t\t\tValue: 1,\n\t\t\t\tUsage: \"interval in seconds between each update\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"verbosity\",\n\t\t\t\tAliases: []string{\"l\"},\n\t\t\t\tUsage: \"verbosity level, 0 or 1 is enough for most cases\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"count\",\n\t\t\t\tAliases: []string{\"c\"},\n\t\t\t\tUsage: \"number of updates to display before exiting\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nconst (\n\tBLACK = 30 + iota\n\tRED\n\tGREEN\n\tYELLOW\n\tBLUE\n\tMAGENTA\n\tCYAN\n\tWHITE\n\tDEFAULT = \"00\"\n)\n\nconst (\n\tRESET_SEQ = \"\\033[0m\"\n\tCOLOR_SEQ = \"\\033[1;\" // %dm\n\tCOLOR_DARK_SEQ = \"\\033[0;\" // %dm\n\tUNDERLINE_SEQ = \"\\033[4m\"\n\tCLEAR_SCREEM = \"\\033[2J\\033[1;1H\"\n\tUNIXTIME_FMT = \"01-02 15:04:05\"\n\t// BOLD_SEQ = \"\\033[1m\"\n)\n\ntype statsWatcher struct {\n\tcolorful bool\n\tinterval uint\n\tmp string\n\theader string\n\tsections []*section\n}\n\nfunc (w *statsWatcher) colorize(msg string, color int, dark bool, underline bool) string {\n\tif !w.colorful || msg == \"\" || msg == \" \" {\n\t\treturn msg\n\t}\n\tvar cseq, useq string\n\tif dark {\n\t\tcseq = COLOR_DARK_SEQ\n\t} else {\n\t\tcseq = COLOR_SEQ\n\t}\n\tif underline {\n\t\tuseq = UNDERLINE_SEQ\n\t}\n\treturn fmt.Sprintf(\"%s%s%dm%s%s\", useq, cseq, color, msg, RESET_SEQ)\n}\n\nconst (\n\tmetricByte = 1 << iota\n\tmetricCount\n\tmetricTime\n\tmetricCPU\n\tmetricGauge\n\tmetricCounter\n\tmetricHist\n\tmetricUnixtime\n)\n\ntype item struct {\n\tnick string // must be size <= 5\n\tname string\n\ttyp uint8\n}\n\ntype section struct {\n\tname string\n\titems []*item\n}\n\nfunc (w *statsWatcher) buildSchema(schema string, verbosity uint) {\n\tfor _, r := range schema {\n\t\tvar s section\n\t\tswitch r {\n\t\tcase 't':\n\t\t\ts.name = \"system\"\n\t\t\ts.items = append(s.items, &item{\"time\", \"juicefs_timestamp\", metricUnixtime})\n\t\tcase 'u':\n\t\t\ts.name = \"usage\"\n\t\t\ts.items = append(s.items, &item{\"cpu\", \"juicefs_cpu_usage\", metricCPU | metricCounter})\n\t\t\ts.items = append(s.items, &item{\"mem\", \"juicefs_memory\", metricGauge})\n\t\t\ts.items = append(s.items, &item{\"buf\", \"juicefs_used_buffer_size_bytes\", metricGauge})\n\t\t\tif verbosity > 0 {\n\t\t\t\ts.items = append(s.items, &item{\"cache\", \"juicefs_store_cache_size_bytes\", metricGauge})\n\t\t\t}\n\t\tcase 'f':\n\t\t\ts.name = \"fuse\"\n\t\t\ts.items = append(s.items, &item{\"ops\", \"juicefs_fuse_ops_durations_histogram_seconds\", metricTime | metricHist})\n\t\t\ts.items = append(s.items, &item{\"read\", \"juicefs_fuse_read_size_bytes_sum\", metricByte | metricCounter})\n\t\t\ts.items = append(s.items, &item{\"write\", \"juicefs_fuse_written_size_bytes_sum\", metricByte | metricCounter})\n\t\tcase 'm':\n\t\t\ts.name = \"meta\"\n\t\t\ts.items = append(s.items, &item{\"ops\", \"juicefs_meta_ops_durations_histogram_seconds\", metricTime | metricHist})\n\t\t\tif verbosity > 0 {\n\t\t\t\ts.items = append(s.items, &item{\"txn\", \"juicefs_transaction_durations_histogram_seconds\", metricTime | metricHist})\n\t\t\t\ts.items = append(s.items, &item{\"retry\", \"juicefs_transaction_restart\", metricCount | metricCounter})\n\t\t\t}\n\t\tcase 'c':\n\t\t\ts.name = \"blockcache\"\n\t\t\ts.items = append(s.items, &item{\"read\", \"juicefs_blockcache_hit_bytes\", metricByte | metricCounter})\n\t\t\ts.items = append(s.items, &item{\"write\", \"juicefs_blockcache_write_bytes\", metricByte | metricCounter})\n\t\tcase 'o':\n\t\t\ts.name = \"object\"\n\t\t\ts.items = append(s.items, &item{\"get\", \"juicefs_object_request_data_bytes_GET\", metricByte | metricCounter})\n\t\t\tif verbosity > 0 {\n\t\t\t\ts.items = append(s.items, &item{\"get_c\", \"juicefs_object_request_durations_histogram_seconds_GET\", metricTime | metricHist})\n\t\t\t}\n\t\t\ts.items = append(s.items, &item{\"put\", \"juicefs_object_request_data_bytes_PUT\", metricByte | metricCounter})\n\t\t\tif verbosity > 0 {\n\t\t\t\ts.items = append(s.items, &item{\"put_c\", \"juicefs_object_request_durations_histogram_seconds_PUT\", metricTime | metricHist})\n\t\t\t\ts.items = append(s.items, &item{\"del_c\", \"juicefs_object_request_durations_histogram_seconds_DELETE\", metricTime | metricHist})\n\t\t\t}\n\t\tcase 'g':\n\t\t\ts.name = \"go\"\n\t\t\ts.items = append(s.items, &item{\"alloc\", \"juicefs_go_memstats_alloc_bytes\", metricGauge})\n\t\t\ts.items = append(s.items, &item{\"sys\", \"juicefs_go_memstats_sys_bytes\", metricGauge})\n\t\tdefault:\n\t\t\tfmt.Printf(\"Warning: no item defined for %c\\n\", r)\n\t\t\tcontinue\n\t\t}\n\t\tw.sections = append(w.sections, &s)\n\t}\n\tif len(w.sections) == 0 {\n\t\tlogger.Fatalln(\"no section to watch, please check the schema string\")\n\t}\n}\n\nfunc padding(name string, width int, char byte) string {\n\tpad := width - len(name)\n\tif pad < 0 {\n\t\tpad = 0\n\t\tname = name[0:width]\n\t}\n\tprefix := (pad + 1) / 2\n\tbuf := make([]byte, width)\n\tfor i := 0; i < prefix; i++ {\n\t\tbuf[i] = char\n\t}\n\tcopy(buf[prefix:], name)\n\tfor i := prefix + len(name); i < width; i++ {\n\t\tbuf[i] = char\n\t}\n\treturn string(buf)\n}\n\nfunc (w *statsWatcher) formatHeader() {\n\theaders := make([]string, len(w.sections))\n\tsubHeaders := make([]string, len(w.sections))\n\tfor i, s := range w.sections {\n\t\tsubs := make([]string, 0, len(s.items))\n\t\tfor _, it := range s.items {\n\t\t\tif (it.typ & 0xF0) == metricUnixtime {\n\t\t\t\tsubs = append(subs, w.colorize(padding(it.nick, len(UNIXTIME_FMT), ' '), BLUE, false, true))\n\t\t\t} else {\n\t\t\t\tsubs = append(subs, w.colorize(padding(it.nick, 5, ' '), BLUE, false, true))\n\t\t\t}\n\t\t\tif it.typ&metricHist != 0 {\n\t\t\t\tif it.typ&metricTime != 0 {\n\t\t\t\t\tsubs = append(subs, w.colorize(\" lat \", BLUE, false, true))\n\t\t\t\t} else {\n\t\t\t\t\tsubs = append(subs, w.colorize(\" avg \", BLUE, false, true))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\twidth := 6*len(subs) - 1 // nick(5) + space(1)\n\t\tif s.name == \"system\" {\n\t\t\twidth = len(UNIXTIME_FMT)\n\t\t}\n\t\tsubHeaders[i] = strings.Join(subs, \" \")\n\t\theaders[i] = w.colorize(padding(s.name, width, '-'), BLUE, true, false)\n\t}\n\tw.header = fmt.Sprintf(\"%s\\n%s\", strings.Join(headers, \" \"),\n\t\tstrings.Join(subHeaders, w.colorize(\"|\", BLUE, true, false)))\n}\n\nfunc (w *statsWatcher) formatU64(v float64, dark, isByte bool) string {\n\tif v <= 0.0 {\n\t\treturn w.colorize(\" 0 \", BLACK, false, false)\n\t}\n\tvar vi uint64\n\tvar unit string\n\tvar color int\n\tswitch vi = uint64(v); {\n\tcase vi < 10000:\n\t\tif isByte {\n\t\t\tunit = \"B\"\n\t\t} else {\n\t\t\tunit = \" \"\n\t\t}\n\t\tcolor = RED\n\tcase vi>>10 < 10000:\n\t\tvi, unit, color = vi>>10, \"K\", YELLOW\n\tcase vi>>20 < 10000:\n\t\tvi, unit, color = vi>>20, \"M\", GREEN\n\tcase vi>>30 < 10000:\n\t\tvi, unit, color = vi>>30, \"G\", BLUE\n\tcase vi>>40 < 10000:\n\t\tvi, unit, color = vi>>40, \"T\", MAGENTA\n\tdefault:\n\t\tvi, unit, color = vi>>50, \"P\", CYAN\n\t}\n\treturn w.colorize(fmt.Sprintf(\"%4d\", vi), color, dark, false) +\n\t\tw.colorize(unit, BLACK, false, false)\n}\n\nfunc (w *statsWatcher) formatTime(v float64, dark bool) string {\n\tvar ret string\n\tvar color int\n\tswitch {\n\tcase v <= 0.0:\n\t\tret, color, dark = \" 0 \", BLACK, false\n\tcase v < 10.0:\n\t\tret, color = fmt.Sprintf(\"%4.2f \", v), GREEN\n\tcase v < 100.0:\n\t\tret, color = fmt.Sprintf(\"%4.1f \", v), YELLOW\n\tcase v < 10000.0:\n\t\tret, color = fmt.Sprintf(\"%4.f \", v), RED\n\tdefault:\n\t\tret, color = fmt.Sprintf(\"%1.e\", v), MAGENTA\n\t}\n\treturn w.colorize(ret, color, dark, false)\n}\n\nfunc (w *statsWatcher) formatCPU(v float64, dark bool) string {\n\tvar ret string\n\tvar color int\n\tswitch v = v * 100.0; {\n\tcase v <= 0.0:\n\t\tret, color = \" 0.0\", WHITE\n\tcase v < 30.0:\n\t\tret, color = fmt.Sprintf(\"%4.1f\", v), GREEN\n\tcase v < 100.0:\n\t\tret, color = fmt.Sprintf(\"%4.1f\", v), YELLOW\n\tdefault:\n\t\tret, color = fmt.Sprintf(\"%4.f\", v), RED\n\t}\n\treturn w.colorize(ret, color, dark, false) +\n\t\tw.colorize(\"%\", BLACK, false, false)\n}\n\nfunc (w *statsWatcher) printDiff(left, right map[string]float64, dark bool) {\n\tif !w.colorful && dark {\n\t\treturn\n\t}\n\tvalues := make([]string, len(w.sections))\n\tfor i, s := range w.sections {\n\t\tvals := make([]string, 0, len(s.items))\n\t\tfor _, it := range s.items {\n\t\t\tswitch it.typ & 0xF0 {\n\t\t\tcase metricUnixtime: // current timestamp\n\t\t\t\tif dark {\n\t\t\t\t\tvals = append(vals, w.colorize(time.Now().Format(UNIXTIME_FMT), BLACK, false, false))\n\t\t\t\t} else {\n\t\t\t\t\tvals = append(vals, w.colorize(time.Now().Format(UNIXTIME_FMT), WHITE, true, false))\n\t\t\t\t}\n\t\t\tcase metricGauge: // currently must be metricByte\n\t\t\t\tvals = append(vals, w.formatU64(right[it.name], dark, true))\n\t\t\tcase metricCounter:\n\t\t\t\tv := (right[it.name] - left[it.name])\n\t\t\t\tif !dark {\n\t\t\t\t\tv /= float64(w.interval)\n\t\t\t\t}\n\t\t\t\tif it.typ&metricByte != 0 {\n\t\t\t\t\tvals = append(vals, w.formatU64(v, dark, true))\n\t\t\t\t} else if it.typ&metricCPU != 0 {\n\t\t\t\t\tvals = append(vals, w.formatCPU(v, dark))\n\t\t\t\t} else { // metricCount\n\t\t\t\t\tvals = append(vals, w.formatU64(v, dark, false))\n\t\t\t\t}\n\t\t\tcase metricHist: // metricTime\n\t\t\t\tcount := right[it.name+\"_total\"] - left[it.name+\"_total\"]\n\t\t\t\tvar avg float64\n\t\t\t\tif count > 0.0 {\n\t\t\t\t\tcost := right[it.name+\"_sum\"] - left[it.name+\"_sum\"]\n\t\t\t\t\tif it.typ&metricTime != 0 {\n\t\t\t\t\t\tcost *= 1000 // s -> ms\n\t\t\t\t\t}\n\t\t\t\t\tavg = cost / count\n\t\t\t\t}\n\t\t\t\tif !dark {\n\t\t\t\t\tcount /= float64(w.interval)\n\t\t\t\t}\n\t\t\t\tvals = append(vals, w.formatU64(count, dark, false), w.formatTime(avg, dark))\n\t\t\t}\n\t\t}\n\t\tvalues[i] = strings.Join(vals, \" \")\n\t}\n\tif w.colorful && dark {\n\t\tfmt.Printf(\"%s\\r\", strings.Join(values, w.colorize(\"|\", BLUE, true, false)))\n\t} else {\n\t\tfmt.Printf(\"%s\\n\", strings.Join(values, w.colorize(\"|\", BLUE, true, false)))\n\t}\n}\n\nfunc readStats(mp string) map[string]float64 {\n\tf, err := os.Open(filepath.Join(mp, \".jfs.stats\"))\n\tif os.IsNotExist(err) {\n\t\tf, err = os.Open(filepath.Join(mp, \".stats\"))\n\t}\n\tif err != nil {\n\t\tlogger.Warnf(\"open stats file under mount point %s: %s\", mp, err)\n\t\treturn nil\n\t}\n\tdefer f.Close()\n\td, err := io.ReadAll(f)\n\tif err != nil {\n\t\tlogger.Warnf(\"read stats file under mount point %s: %s\", mp, err)\n\t\treturn nil\n\t}\n\tstats := make(map[string]float64)\n\tlines := strings.Split(string(d), \"\\n\")\n\tfor _, line := range lines {\n\t\tfields := strings.Fields(line)\n\t\tif len(fields) == 2 {\n\t\t\tv, err := strconv.ParseFloat(fields[1], 64)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"parse %s: %s\", fields[1], err)\n\t\t\t}\n\t\t\tstats[fields[0]] += v\n\t\t}\n\t}\n\treturn stats\n}\n\nfunc stats(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tmp := ctx.Args().First()\n\tinode, err := utils.GetFileInode(mp)\n\tif err != nil {\n\t\tlogger.Fatalf(\"lookup inode for %s: %s\", mp, err)\n\t}\n\tif inode != 1 {\n\t\tlogger.Fatalf(\"path %s is not a mount point\", mp)\n\t}\n\n\twatcher := &statsWatcher{\n\t\tcolorful: !ctx.Bool(\"no-color\") && utils.SupportANSIColor(os.Stdout.Fd()),\n\t\tinterval: ctx.Uint(\"interval\"),\n\t\tmp: mp,\n\t}\n\twatcher.buildSchema(ctx.String(\"schema\"), ctx.Uint(\"verbosity\"))\n\twatcher.formatHeader()\n\tcount := ctx.Uint(\"count\")\n\n\tvar tick uint\n\tvar start, last, current map[string]float64\n\tticker := time.NewTicker(time.Second)\n\tdefer ticker.Stop()\n\tcurrent = readStats(watcher.mp)\n\tstart = current\n\tlast = current\n\tfor {\n\t\tif tick%(watcher.interval*30) == 0 {\n\t\t\tfmt.Println(watcher.header)\n\t\t}\n\t\tif tick%watcher.interval == 0 {\n\t\t\twatcher.printDiff(start, current, false)\n\t\t\tstart = current\n\t\t} else {\n\t\t\twatcher.printDiff(last, current, true)\n\t\t}\n\t\tif count > 0 && tick >= watcher.interval*(count-1) {\n\t\t\tbreak\n\t\t}\n\t\tlast = current\n\t\ttick++\n\t\t<-ticker.C\n\t\tcurrent = readStats(watcher.mp)\n\t}\n\treturn nil\n}\n" }, { "path": "cmd/status.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdStatus() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"status\",\n\t\tAction: status,\n\t\tCategory: \"INSPECTOR\",\n\t\tUsage: \"Show status of a volume\",\n\t\tArgsUsage: \"META-URL\",\n\t\tDescription: `\nIt shows basic setting of the target volume, and a list of active sessions (including mount, SDK,\nS3-gateway and WebDAV) that are connected with the metadata engine.\n\nNOTE: Read-only session is not listed since it cannot register itself in the metadata.\n\nExamples:\n$ juicefs status redis://localhost`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.Uint64Flag{\n\t\t\t\tName: \"session\",\n\t\t\t\tAliases: []string{\"s\"},\n\t\t\t\tUsage: \"show detailed information (sustained inodes, locks) of the specified session (sid)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"more\",\n\t\t\t\tAliases: []string{\"m\"},\n\t\t\t\tUsage: \"show more statistic information, may take a long time\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc printJson(v interface{}) {\n\toutput, err := json.MarshalIndent(v, \"\", \" \")\n\tif err != nil {\n\t\tlogger.Fatalf(\"json: %s\", err)\n\t}\n\tfmt.Println(string(output))\n}\n\nfunc status(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tmetaUrl := ctx.Args().Get(0)\n\tremovePassword(metaUrl)\n\tm := meta.NewClient(metaUrl, nil)\n\n\tif sid := ctx.Uint64(\"session\"); sid != 0 {\n\t\ts, err := m.GetSession(sid, true)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"get session: %s\", err)\n\t\t}\n\t\tprintJson(s)\n\t\treturn nil\n\t}\n\n\tsections := &meta.Sections{}\n\terr := meta.Status(ctx.Context, m, ctx.Bool(\"more\"), sections)\n\tif err != nil {\n\t\tlogger.Fatalf(\"get status: %s\", err)\n\t}\n\tprintJson(sections)\n\treturn nil\n}\n" }, { "path": "cmd/status_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/agiledragon/gomonkey/v2\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\nfunc TestStatus(t *testing.T) {\n\ttmpFile, err := os.CreateTemp(\"/tmp\", \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create temporary file: %s\", err)\n\t}\n\tdefer tmpFile.Close()\n\tdefer os.Remove(tmpFile.Name())\n\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\t// mock os.Stdout\n\tpatches := gomonkey.ApplyGlobalVar(os.Stdout, *tmpFile)\n\tdefer patches.Reset()\n\n\tif err = Main([]string{\"\", \"status\", testMeta}); err != nil {\n\t\tt.Fatalf(\"status failed: %s\", err)\n\t}\n\tcontent, err := os.ReadFile(tmpFile.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"read file failed: %s\", err)\n\t}\n\ts := meta.Sections{}\n\tif err = json.Unmarshal(content, &s); err != nil {\n\t\tt.Fatalf(\"json unmarshal failed: %s\", err)\n\t}\n\tif s.Setting.Name != testVolume || s.Setting.Storage != \"file\" {\n\t\tt.Fatalf(\"setting is not as expected: %+v\", s.Setting)\n\t}\n}\n" }, { "path": "cmd/summary.go", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"encoding/csv\"\n\t\"encoding/json\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdSummary() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"summary\",\n\t\tAction: summary,\n\t\tCategory: \"INSPECTOR\",\n\t\tUsage: \"Show tree summary of a directory\",\n\t\tArgsUsage: \"PATH\",\n\t\tDescription: `\n It is used to show tree summary of target directory.\n \n Examples:\n # Show with path\n $ juicefs summary /mnt/jfs/foo\n \n # Show max depth of 5\n $ juicefs summary --depth 5 /mnt/jfs/foo\n\n # Show top 20 entries\n $ juicefs summary --entries 20 /mnt/jfs/foo\n\n # Show accurate result\n $ juicefs summary --strict /mnt/jfs/foo\n `,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"depth\",\n\t\t\t\tAliases: []string{\"d\"},\n\t\t\t\tValue: 2,\n\t\t\t\tUsage: \"depth of tree to show (zero means only show root)\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"entries\",\n\t\t\t\tAliases: []string{\"e\"},\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"show top N entries (sort by size)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"strict\",\n\t\t\t\tUsage: \"show accurate summary, including directories and files (may be slow)\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"csv\",\n\t\t\t\tUsage: \"print summary in csv format\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc summary(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tvar strict uint8\n\tif ctx.Bool(\"strict\") {\n\t\tstrict = 1\n\t}\n\tdepth := ctx.Uint(\"depth\")\n\tif depth > 10 {\n\t\tlogger.Warn(\"depth should be less than 11\")\n\t\tdepth = 10\n\t}\n\ttopN := ctx.Uint(\"entries\")\n\tif topN > 100 {\n\t\tlogger.Warn(\"entries should be less than 101\")\n\t\ttopN = 100\n\t}\n\n\tcsv := ctx.Bool(\"csv\")\n\tprogress := utils.NewProgress(csv)\n\tpath := ctx.Args().Get(0)\n\tdspin := progress.AddDoubleSpinner(path)\n\tdpath, err := filepath.Abs(path)\n\tif err != nil {\n\t\tlogger.Fatalf(\"abs of %s: %s\", path, err)\n\t}\n\tinode, err := utils.GetFileInode(dpath)\n\tif err != nil {\n\t\tlogger.Fatalf(\"lookup inode for %s: %s\", path, err)\n\t}\n\tif inode < uint64(meta.RootInode) {\n\t\tlogger.Fatalf(\"inode number shouldn't be less than %d\", meta.RootInode)\n\t}\n\tf, err := openController(dpath)\n\tif err != nil {\n\t\tlogger.Fatalf(\"open controller: %s\", err)\n\t}\n\tdefer f.Close()\n\theaderLen := uint32(8)\n\tcontentLen := uint32(8 + 1 + 1 + 1)\n\twb := utils.NewBuffer(headerLen + contentLen)\n\twb.Put32(meta.OpSummary)\n\twb.Put32(contentLen)\n\twb.Put64(inode)\n\twb.Put8(uint8(depth))\n\twb.Put8(uint8(topN))\n\twb.Put8(strict)\n\t_, err = f.Write(wb.Bytes())\n\tif err != nil {\n\t\tlogger.Fatalf(\"write message: %s\", err)\n\t}\n\tdata, errno := readProgress(f, func(count, size uint64) {\n\t\tdspin.SetCurrent(int64(count), int64(size))\n\t})\n\tif errno == syscall.EINVAL {\n\t\tlogger.Fatalf(\"summary is not supported, please upgrade and mount again\")\n\t}\n\tif errno != 0 {\n\t\tlogger.Errorf(\"failed to get info: %s\", syscall.Errno(errno))\n\t}\n\tdspin.Done()\n\tprogress.Done()\n\n\tvar resp vfs.SummaryReponse\n\terr = json.Unmarshal(data, &resp)\n\tif err == nil && resp.Errno != 0 {\n\t\terr = resp.Errno\n\t}\n\tif err != nil {\n\t\tlogger.Fatalf(\"summary: %s\", err)\n\t}\n\tresults := [][]string{{\"PATH\", \"SIZE\", \"DIRS\", \"FILES\"}}\n\trenderTree(&results, &resp.Tree, csv)\n\tif csv {\n\t\tprintCSVResult(results)\n\t} else {\n\t\tprintResult(results, 0, false)\n\t}\n\treturn nil\n}\n\nfunc printCSVResult(results [][]string) {\n\tw := csv.NewWriter(os.Stdout)\n\tfor _, r := range results {\n\t\tif err := w.Write(r); err != nil {\n\t\t\tlogger.Fatalln(\"error writing record to csv:\", err)\n\t\t}\n\t}\n\tw.Flush()\n\tif err := w.Error(); err != nil {\n\t\tlogger.Fatal(err)\n\t}\n}\n\nfunc renderTree(results *[][]string, tree *meta.TreeSummary, csv bool) {\n\tif tree == nil {\n\t\treturn\n\t}\n\tvar size string\n\tif csv {\n\t\tsize = strconv.FormatUint(tree.Size, 10)\n\t} else {\n\t\tsize = humanize.IBytes(uint64(tree.Size))\n\t}\n\n\tpath := tree.Path\n\tif tree.Type == meta.TypeDirectory && !strings.HasSuffix(path, \"/\") {\n\t\tpath += \"/\"\n\t}\n\n\tresult := []string{\n\t\tpath,\n\t\tsize,\n\t\tstrconv.FormatUint(tree.Dirs, 10),\n\t\tstrconv.FormatUint(tree.Files, 10),\n\t}\n\t*results = append(*results, result)\n\tfor _, child := range tree.Children {\n\t\trenderTree(results, child, csv)\n\t}\n}\n" }, { "path": "cmd/sync.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t_ \"net/http/pprof\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/juicedata/juicefs/pkg/metric\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/sync\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/collectors\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdSync() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"sync\",\n\t\tAction: doSync,\n\t\tCategory: \"TOOL\",\n\t\tUsage: \"Sync between two storages\",\n\t\tArgsUsage: \"SRC DST\",\n\t\tDescription: `\nThis tool spawns multiple threads to concurrently syncs objects of two data storages.\nSRC and DST should be [NAME://][ACCESS_KEY:SECRET_KEY[:TOKEN]@]BUCKET[.ENDPOINT][/PREFIX].\n\nInclude/exclude pattern rules:\nThe include/exclude rules each specify a pattern that is matched against the names of the files that are going to be transferred. These patterns can take several forms:\n\n- if the pattern ends with a / then it will only match a directory, not a file, link, or device.\n- it chooses between doing a simple string match and wildcard matching by checking if the pattern contains one of these three wildcard characters: '*', '?', and '[' .\n- a '*' matches any non-empty path component (it stops at slashes).\n- a '?' matches any character except a slash (/).\n- a '[' introduces a character class, such as [a-z] or [[:alpha:]].\n- in a wildcard pattern, a backslash can be used to escape a wildcard character, but it is matched literally when no wildcards are present.\n- it does a prefix match of pattern, i.e. always recursive\n\nExamples:\n# Sync object from OSS to S3\n$ juicefs sync oss://mybucket.oss-cn-shanghai.aliyuncs.com s3://mybucket.s3.us-east-2.amazonaws.com\n\n# Sync objects from S3 to JuiceFS\n$ myfs=redis://localhost juicefs sync s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://myfs/ -p 50\n\n# SRC: a1/b1,a2/b2,aaa/b1 DST: empty sync result: aaa/b1\n$ juicefs sync --exclude='a?/b*' s3://mybucket.s3.us-east-2.amazonaws.com/ /mnt/jfs/\n\n# SRC: a1/b1,a2/b2,aaa/b1 DST: empty sync result: a1/b1,aaa/b1\n$ juicefs sync --include='a1/b1' --exclude='a[1-9]/b*' s3://mybucket.s3.us-east-2.amazonaws.com/ /mnt/jfs/\n\n# SRC: a1/b1,a2/b2,aaa/b1,b1,b2 DST: empty sync result: b2\n$ juicefs sync --include='a1/b1' --exclude='a*' --include='b2' --exclude='b?' s3://mybucket.s3.us-east-2.amazonaws.com/ /mnt/jfs/\n\nDetails: https://juicefs.com/docs/community/administration/sync\nSupported storage systems: https://juicefs.com/docs/community/how_to_setup_object_storage#supported-object-storage`,\n\n\t\tFlags: expandFlags(\n\t\t\tselectionFlags(),\n\t\t\tsyncActionFlags(),\n\t\t\tsyncStorageFlags(),\n\t\t\tclusterFlags(),\n\t\t\taddCategories(\"METRICS\", []cli.Flag{\n\t\t\t\t&cli.StringFlag{\n\t\t\t\t\tName: \"metrics\",\n\t\t\t\t\tValue: \"127.0.0.1:9567\",\n\t\t\t\t\tUsage: \"address to export metrics\",\n\t\t\t\t},\n\t\t\t\t&cli.StringFlag{\n\t\t\t\t\tName: \"consul\",\n\t\t\t\t\tValue: \"127.0.0.1:8500\",\n\t\t\t\t\tUsage: \"consul address to register\",\n\t\t\t\t},\n\t\t\t}),\n\t\t),\n\t}\n}\n\nfunc selectionFlags() []cli.Flag {\n\treturn addCategories(\"SELECTION\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"start\",\n\t\t\tAliases: []string{\"s\"},\n\t\t\tUsage: \"the first `KEY` to sync\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"end\",\n\t\t\tAliases: []string{\"e\"},\n\t\t\tUsage: \"the last `KEY` to sync\",\n\t\t},\n\t\t&cli.StringSliceFlag{\n\t\t\tName: \"exclude\",\n\t\t\tUsage: \"exclude Key matching PATTERN\",\n\t\t},\n\t\t&cli.StringSliceFlag{\n\t\t\tName: \"include\",\n\t\t\tUsage: \"don't exclude Key matching PATTERN, need to be used with \\\"--exclude\\\" option\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"match-full-path\",\n\t\t\tUsage: \"match filters again the full path\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"max-size\",\n\t\t\tUsage: \"skip files larger than `SIZE`\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"min-size\",\n\t\t\tUsage: \"skip files smaller than `SIZE`\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"max-age\",\n\t\t\tUsage: \"skip files older than `DURATION`\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"min-age\",\n\t\t\tUsage: \"skip files newer than `DURATION`\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"start-time\",\n\t\t\tUsage: \"skip files modified before start-time. example: 2006-01-02 15:04:05\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"end-time\",\n\t\t\tUsage: \"skip files modified after end-time. example: 2006-01-02 15:04:05\",\n\t\t},\n\t\t&cli.Int64Flag{\n\t\t\tName: \"limit\",\n\t\t\tUsage: \"limit the number of objects that will be processed (-1 is unlimited, 0 is to process nothing)\",\n\t\t\tValue: -1,\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"update\",\n\t\t\tAliases: []string{\"u\"},\n\t\t\tUsage: \"skip files if the destination is newer\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"force-update\",\n\t\t\tAliases: []string{\"f\"},\n\t\t\tUsage: \"always update existing files\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"existing\",\n\t\t\tAliases: []string{\"ignore-non-existing\"},\n\t\t\tUsage: \"skip creating new files on destination\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"ignore-existing\",\n\t\t\tUsage: \"skip updating files that already exist on destination\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"files-from\",\n\t\t\tUsage: \"read list of files or dirs to sync from FILE\",\n\t\t},\n\t})\n}\n\nfunc syncActionFlags() []cli.Flag {\n\treturn addCategories(\"ACTION\", []cli.Flag{\n\t\t&cli.BoolFlag{\n\t\t\tName: \"dirs\",\n\t\t\tUsage: \"sync directories or holders\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"perms\",\n\t\t\tUsage: \"preserve permissions\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"links\",\n\t\t\tAliases: []string{\"l\"},\n\t\t\tUsage: \"copy symlinks as symlinks\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"inplace\",\n\t\t\tUsage: \"put directly to destination file instead of atomic download to temp/rename\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"delete-src\",\n\t\t\tAliases: []string{\"deleteSrc\"},\n\t\t\tUsage: \"delete objects from source those already exist in destination\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"delete-dst\",\n\t\t\tAliases: []string{\"deleteDst\"},\n\t\t\tUsage: \"delete extraneous objects from destination\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"check-all\",\n\t\t\tUsage: \"verify integrity of all files in source and destination\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"check-new\",\n\t\t\tUsage: \"verify integrity of newly copied files\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"check-change\",\n\t\t\tUsage: \"check if source file changes after sync\",\n\t\t},\n\t\t&cli.Int64Flag{\n\t\t\tName: \"max-failure\",\n\t\t\tValue: -1,\n\t\t\tUsage: \"max number of allowed failed files (-1 for unlimited)\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"dry\",\n\t\t\tUsage: \"don't copy file\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"mountpoint\",\n\t\t\tUsage: \"the mount point for current volume (to follow symlink)\",\n\t\t},\n\t})\n}\n\nfunc syncStorageFlags() []cli.Flag {\n\treturn addCategories(\"STORAGE\", []cli.Flag{\n\t\t&cli.IntFlag{\n\t\t\tName: \"threads\",\n\t\t\tAliases: []string{\"p\"},\n\t\t\tValue: 10,\n\t\t\tUsage: \"number of concurrent threads\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"list-threads\",\n\t\t\tValue: 1,\n\t\t\tUsage: \"number of threads to list objects\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"list-depth\",\n\t\t\tValue: 1,\n\t\t\tUsage: \"list the top N level of directories in parallel\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"no-https\",\n\t\t\tUsage: \"donot use HTTPS\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"storage-class\",\n\t\t\tUsage: \"the storage class for destination\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"bwlimit\",\n\t\t\tUsage: \"limit bandwidth in Mbps (0 means unlimited)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"traffic-control-url\",\n\t\t\tUsage: \"the url of the traffic control\",\n\t\t},\n\t})\n}\n\nfunc clusterFlags() []cli.Flag {\n\treturn addCategories(\"CLUSTER\", []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"manager\",\n\t\t\tUsage: \"the manager address used only by the worker node\",\n\t\t\tHidden: true,\n\t\t},\n\t\t&cli.StringSliceFlag{\n\t\t\tName: \"worker\",\n\t\t\tUsage: \"hosts (separated by comma) to launch worker\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"manager-addr\",\n\t\t\tUsage: \"the IP address to communicate with workers\",\n\t\t},\n\t})\n}\n\nfunc supportHTTPS(name, endpoint string) bool {\n\tswitch name {\n\tcase \"ufile\":\n\t\treturn !(strings.Contains(endpoint, \".internal-\") || strings.HasSuffix(endpoint, \".ucloud.cn\"))\n\tcase \"oss\":\n\t\treturn !(strings.Contains(endpoint, \".vpc100-oss\") || strings.Contains(endpoint, \"internal.aliyuncs.com\"))\n\tcase \"s3\":\n\t\tps := strings.SplitN(strings.Split(endpoint, \":\")[0], \".\", 2)\n\t\tif len(ps) > 1 && net.ParseIP(ps[1]) != nil {\n\t\t\treturn false\n\t\t}\n\tcase \"minio\":\n\t\treturn false\n\t}\n\treturn true\n}\n\n// Check if uri is local file path\nfunc isFilePath(uri string) bool {\n\t// check drive pattern when running on Windows\n\tif runtime.GOOS == \"windows\" &&\n\t\tlen(uri) > 1 && (('a' <= uri[0] && uri[0] <= 'z') ||\n\t\t('A' <= uri[0] && uri[0] <= 'Z')) && uri[1] == ':' {\n\t\treturn true\n\t}\n\treturn !strings.Contains(uri, \":\")\n}\n\nfunc extractToken(uri string) (string, string) {\n\tif submatch := regexp.MustCompile(`^.*:.*:.*(:.*)@.*$`).FindStringSubmatch(uri); len(submatch) == 2 {\n\t\treturn strings.ReplaceAll(uri, submatch[1], \"\"), strings.TrimLeft(submatch[1], \":\")\n\t}\n\treturn uri, \"\"\n}\n\nfunc createSyncStorage(uri string, conf *sync.Config) (object.ObjectStorage, error) {\n\t// nolint:staticcheck\n\turi = strings.TrimPrefix(uri, \"sftp://\")\n\tif !strings.Contains(uri, \"://\") {\n\t\tif isFilePath(uri) {\n\t\t\tabsPath, err := filepath.Abs(uri)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Fatalf(\"invalid path: %s\", err.Error())\n\t\t\t}\n\t\t\tif !strings.HasPrefix(absPath, \"/\") { // Windows path\n\t\t\t\tabsPath = \"/\" + strings.Replace(absPath, \"\\\\\", \"/\", -1)\n\t\t\t}\n\t\t\tif strings.HasSuffix(uri, \"/\") {\n\t\t\t\tabsPath += \"/\"\n\t\t\t}\n\n\t\t\t// Windows: file:///C:/a/b/c, Unix: file:///a/b/c\n\t\t\turi = \"file://\" + absPath\n\t\t} else { // sftp\n\t\t\tvar user string\n\t\t\tif strings.Contains(uri, \"@\") {\n\t\t\t\tparts := strings.Split(uri, \"@\")\n\t\t\t\tuser = parts[0]\n\t\t\t\turi = parts[1]\n\t\t\t}\n\t\t\tvar pass string\n\t\t\tif strings.Contains(user, \":\") {\n\t\t\t\tparts := strings.Split(user, \":\")\n\t\t\t\tuser = parts[0]\n\t\t\t\tpass = parts[1]\n\t\t\t}\n\t\t\treturn object.CreateStorage(\"sftp\", uri, user, pass, \"\")\n\t\t}\n\t}\n\turi, token := extractToken(uri)\n\tu, err := url.Parse(uri)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Can't parse %s: %s\", uri, err.Error())\n\t}\n\tuser := u.User\n\tvar accessKey, secretKey string\n\tif user != nil {\n\t\taccessKey = user.Username()\n\t\tsecretKey, _ = user.Password()\n\t}\n\tname := strings.ToLower(u.Scheme)\n\n\tvar endpoint string\n\tif name == \"file\" {\n\t\tendpoint = u.Path\n\t} else if name == \"hdfs\" {\n\t\tendpoint = u.Host\n\t} else if name == \"jfs\" {\n\t\tendpoint, err = url.PathUnescape(u.Host)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unescape %s: %s\", u.Host, err)\n\t\t}\n\t\tif os.Getenv(endpoint) != \"\" {\n\t\t\tconf.Env[endpoint] = os.Getenv(endpoint)\n\t\t}\n\t} else if name == \"nfs\" {\n\t\tendpoint = u.Host + u.Path\n\t} else if !conf.NoHTTPS && supportHTTPS(name, u.Host) {\n\t\tendpoint = \"https://\" + u.Host\n\t} else {\n\t\tendpoint = \"http://\" + u.Host\n\t}\n\n\tisS3PathTypeUrl := isS3PathType(u.Host)\n\tif name == \"minio\" || name == \"s3\" && isS3PathTypeUrl {\n\t\t// bucket name is part of path\n\t\tendpoint += u.Path\n\t}\n\n\tstore, err := object.CreateStorage(name, endpoint, accessKey, secretKey, token)\n\tif name == \"nfs\" && err != nil {\n\t\tp := u.Path\n\t\tfor err != nil && strings.Contains(err.Error(), \"MNT3ERR_NOENT\") {\n\t\t\tp = filepath.Dir(p)\n\t\t\tstore, err = object.CreateStorage(name, u.Host+p, accessKey, secretKey, token)\n\t\t}\n\t\tif err == nil {\n\t\t\tstore = object.WithPrefix(store, u.Path[len(p):])\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"create %s %s: %s\", name, endpoint, err)\n\t}\n\n\tif conf.Links {\n\t\tif _, ok := store.(object.SupportSymlink); !ok {\n\t\t\tlogger.Warnf(\"storage %s does not support symlink, ignore it\", uri)\n\t\t\tconf.Links = false\n\t\t}\n\t}\n\n\tif conf.Perms {\n\t\tif _, ok := store.(object.FileSystem); !ok {\n\t\t\tlogger.Warnf(\"%s is not a file system, can not preserve permissions\", store)\n\t\t\tconf.Perms = false\n\t\t}\n\t}\n\tswitch name {\n\tcase \"file\", \"nfs\":\n\tcase \"minio\":\n\t\tif strings.Count(u.Path, \"/\") > 1 {\n\t\t\t// skip bucket name\n\t\t\tstore = object.WithPrefix(store, strings.SplitN(u.Path[1:], \"/\", 2)[1])\n\t\t}\n\tcase \"s3\":\n\t\tif isS3PathTypeUrl && strings.Count(u.Path, \"/\") > 1 {\n\t\t\tstore = object.WithPrefix(store, strings.SplitN(u.Path[1:], \"/\", 2)[1])\n\t\t} else if len(u.Path) > 1 {\n\t\t\tstore = object.WithPrefix(store, u.Path[1:])\n\t\t}\n\tdefault:\n\t\tif len(u.Path) > 1 {\n\t\t\tstore = object.WithPrefix(store, u.Path[1:])\n\t\t}\n\t}\n\n\treturn store, nil\n}\n\nfunc isS3PathType(endpoint string) bool {\n\t//localhost[:8080] 127.0.0.1[:8080] s3.ap-southeast-1.amazonaws.com[:8080] s3-ap-southeast-1.amazonaws.com[:8080]\n\tpattern := `^((localhost)|(s3[.-].*\\.amazonaws\\.com)|((1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[1-9])\\.((1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|\\d)\\.){2}(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|\\d)))?(:\\d*)?$`\n\treturn regexp.MustCompile(pattern).MatchString(endpoint)\n}\n\nfunc doSync(c *cli.Context) error {\n\tsetup(c, 2)\n\tif c.IsSet(\"include\") && !c.IsSet(\"exclude\") {\n\t\tlogger.Warnf(\"The include option needs to be used with the exclude option, otherwise the result of the current sync may not match your expectations\")\n\t}\n\tconfig := sync.NewConfigFromCli(c)\n\tcliCtx = c\n\tif config.Manager != \"\" {\n\t\tlogger.Debugf(\"worker process start\")\n\t}\n\t// Windows support `\\` and `/` as its separator, Unix only use `/`\n\tsrcURL := c.Args().Get(0)\n\tdstURL := c.Args().Get(1)\n\tremovePassword(srcURL, dstURL)\n\tif runtime.GOOS == \"windows\" {\n\t\tif !strings.Contains(srcURL, \"://\") {\n\t\t\tsrcURL = strings.Replace(srcURL, \"\\\\\", \"/\", -1)\n\t\t}\n\t\tif !strings.Contains(dstURL, \"://\") {\n\t\t\tdstURL = strings.Replace(dstURL, \"\\\\\", \"/\", -1)\n\t\t}\n\t}\n\tif strings.HasSuffix(srcURL, \"/\") != strings.HasSuffix(dstURL, \"/\") {\n\t\tlogger.Fatalf(\"SRC and DST should both end with path separator or not!\")\n\t}\n\tsrc, err := createSyncStorage(srcURL, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdst, err := createSyncStorage(dstURL, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tobject.Shutdown(src)\n\t\tobject.Shutdown(dst)\n\t}()\n\tif config.StorageClass != \"\" {\n\t\tif os, ok := dst.(object.SupportStorageClass); ok {\n\t\t\terr := os.SetStorageClass(config.StorageClass)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"set storage class %s: %s\", config.StorageClass, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tif config.Manager == \"\" && !config.Dry {\n\t\tvar srcPath, dstPath string\n\t\tif strings.HasPrefix(src.String(), \"file://\") {\n\t\t\tsrcPath = src.String()\n\t\t}\n\t\tif strings.HasPrefix(dst.String(), \"file://\") {\n\t\t\tdstPath = dst.String()\n\t\t}\n\t\tsrcPath = utils.RemovePassword(srcPath)\n\t\tdstPath = utils.RemovePassword(dstPath)\n\t\tregistry := prometheus.NewRegistry()\n\t\tconfig.Registerer = prometheus.WrapRegistererWithPrefix(\"juicefs_sync_\",\n\t\t\tprometheus.WrapRegistererWith(prometheus.Labels{\"cmd\": \"sync\", \"pid\": strconv.Itoa(os.Getpid())}, registry))\n\t\tconfig.Registerer.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))\n\t\tconfig.Registerer.MustRegister(collectors.NewGoCollector())\n\t\tmetricsAddr := exposeMetrics(c, config.Registerer, registry)\n\t\tif c.IsSet(\"consul\") {\n\t\t\tmetadata := make(map[string]string)\n\t\t\tmetadata[\"src\"] = srcPath\n\t\t\tmetadata[\"dst\"] = dstPath\n\t\t\tmetadata[\"pid\"] = strconv.Itoa(os.Getpid())\n\t\t\tmetric.RegisterToConsul(c.String(\"consul\"), metricsAddr, metadata)\n\t\t}\n\t}\n\treturn sync.Sync(src, dst, config)\n}\n" }, { "path": "cmd/sync_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n)\n\nfunc TestSync(t *testing.T) {\n\tif os.Getenv(\"MINIO_TEST_BUCKET\") == \"\" {\n\t\tt.Skip()\n\t}\n\tminioDir := \"synctest\"\n\tlocalDir := \"/tmp/synctest\"\n\tdefer os.RemoveAll(localDir)\n\tstorage, err := object.CreateStorage(\"minio\", os.Getenv(\"MINIO_TEST_BUCKET\"), os.Getenv(\"MINIO_ACCESS_KEY\"), os.Getenv(\"MINIO_SECRET_KEY\"), \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create storage failed: %v\", err)\n\t}\n\n\ttestInstances := []struct{ path, content string }{\n\t\t{\"t1.txt\", \"content1\"},\n\t\t{\"testDir1/t2.txt\", \"content2\"},\n\t\t{\"testDir1/testDir3/t3.txt\", \"content3\"},\n\t}\n\n\tfor _, instance := range testInstances {\n\t\terr = storage.Put(context.Background(), fmt.Sprintf(\"/%s/%s\", minioDir, instance.path), bytes.NewReader([]byte(instance.content)))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"storage put failed: %v\", err)\n\t\t}\n\t}\n\tsyncArgs := []string{\"\", \"sync\", fmt.Sprintf(\"minio://%s/%s\", os.Getenv(\"MINIO_TEST_BUCKET\"), minioDir), fmt.Sprintf(\"file://%s\", localDir)}\n\terr = Main(syncArgs)\n\tif err != nil {\n\t\tt.Fatalf(\"sync failed: %v\", err)\n\t}\n\n\tfor _, instance := range testInstances {\n\t\tc, err := os.ReadFile(fmt.Sprintf(\"%s/%s\", localDir, instance.path))\n\t\tif err != nil || string(c) != instance.content {\n\t\t\tt.Fatalf(\"sync failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc Test_isS3PathType(t *testing.T) {\n\n\ttests := []struct {\n\t\tendpoint string\n\t\twant bool\n\t}{\n\t\t{\"localhost\", true},\n\t\t{\"localhost:8080\", true},\n\t\t{\"127.0.0.1\", true},\n\t\t{\"127.0.0.1:8080\", true},\n\t\t{\"s3.ap-southeast-1.amazonaws.com\", true},\n\t\t{\"s3.ap-southeast-1.amazonaws.com:8080\", true},\n\t\t{\"s3-ap-southeast-1.amazonaws.com\", true},\n\t\t{\"s3-ap-southeast-1.amazonaws.com:8080\", true},\n\t\t{\"s3-ap-southeast-1.amazonaws..com:8080\", false},\n\t\t{\"ap-southeast-1.amazonaws.com\", false},\n\t\t{\"s3-ap-southeast-1amazonaws.com:8080\", false},\n\t\t{\"s3-ap-southeast-1\", false},\n\t\t{\"s3-ap-southeast-1:8080\", false},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(\"Test host\", func(t *testing.T) {\n\t\t\tif got := isS3PathType(tt.endpoint); got != tt.want {\n\t\t\t\tt.Errorf(\"isS3PathType() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc Test_extractToken(t *testing.T) {\n\t// [NAME://][ACCESS_KEY:SECRET_KEY[:TOKEN]@]BUCKET[.ENDPOINT][/PREFIX]\n\ttests := []struct {\n\t\turi, removedTokenUri, token string\n\t}{\n\t\t{\"NAME://ACCESS_KEY:SECRET_KEY@BUCKET.ENDPOINT/PREFIX\", \"NAME://ACCESS_KEY:SECRET_KEY@BUCKET.ENDPOINT/PREFIX\", \"\"},\n\t\t{\"NAME://:@BUCKET.ENDPOINT/PREFIX\", \"NAME://:@BUCKET.ENDPOINT/PREFIX\", \"\"},\n\t\t{\"NAME://ACCESS_KEY:SECRET_KEY:TOKEN@BUCKET.ENDPOINT/PREFIX\", \"NAME://ACCESS_KEY:SECRET_KEY@BUCKET.ENDPOINT/PREFIX\", \"TOKEN\"},\n\t\t{\"NAME://:@BUCKET.ENDPOINT/PREFIX\", \"NAME://:@BUCKET.ENDPOINT/PREFIX\", \"\"},\n\t\t{\"NAME://::TOKEN@BUCKET.ENDPOINT/PREFIX\", \"NAME://:@BUCKET.ENDPOINT/PREFIX\", \"TOKEN\"},\n\t\t{\"NAME://BUCKET.ENDPOINT/PREFIX\", \"NAME://BUCKET.ENDPOINT/PREFIX\", \"\"},\n\t\t{\"file:///tmp/testbucket\", \"file:///tmp/testbucket\", \"\"},\n\t\t{\"/tmp/testbucket\", \"/tmp/testbucket\", \"\"},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tremovedTokenUri, token := extractToken(tt.uri)\n\t\t\tif removedTokenUri != tt.removedTokenUri {\n\t\t\t\tt.Errorf(\"extractToken() removedTokenUri = %v, want %v\", removedTokenUri, tt.removedTokenUri)\n\t\t\t}\n\t\t\tif token != tt.token {\n\t\t\t\tt.Errorf(\"extractToken() token = %v, want %v\", token, tt.token)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "cmd/umount.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdUmount() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"umount\",\n\t\tAction: umount,\n\t\tCategory: \"SERVICE\",\n\t\tUsage: \"Unmount a volume\",\n\t\tArgsUsage: \"MOUNTPOINT\",\n\t\tDescription: `\nExamples:\n$ juicefs umount /mnt/jfs`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"force\",\n\t\t\t\tAliases: []string{\"f\"},\n\t\t\t\tUsage: \"unmount a busy mount point by force\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"flush\",\n\t\t\t\tUsage: \"wait for all staging chunks to be flushed\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc doUmount(mp string, force bool) error {\n\tvar cmd *exec.Cmd\n\tswitch runtime.GOOS {\n\tcase \"darwin\":\n\t\tif force {\n\t\t\tcmd = exec.Command(\"umount\", \"-f\", mp)\n\t\t} else {\n\t\t\tcmd = exec.Command(\"umount\", mp)\n\t\t}\n\tcase \"linux\":\n\t\tif _, err := exec.LookPath(\"fusermount\"); err == nil {\n\t\t\tif force {\n\t\t\t\tcmd = exec.Command(\"fusermount\", \"-uz\", mp)\n\t\t\t} else {\n\t\t\t\tcmd = exec.Command(\"fusermount\", \"-u\", mp)\n\t\t\t}\n\t\t} else {\n\t\t\tif force {\n\t\t\t\tcmd = exec.Command(\"umount\", \"-l\", mp)\n\t\t\t} else {\n\t\t\t\tcmd = exec.Command(\"umount\", mp)\n\t\t\t}\n\t\t}\n\tcase \"windows\":\n\t\tif !force {\n\t\t\t_ = os.Mkdir(filepath.Join(mp, \".UMOUNTIT\"), 0777)\n\t\t\treturn nil\n\t\t} else {\n\t\t\tcmd = exec.Command(\"taskkill\", \"/IM\", \"juicefs.exe\", \"/F\")\n\t\t}\n\tdefault:\n\t\treturn fmt.Errorf(\"OS %s is not supported\", runtime.GOOS)\n\t}\n\tout, err := cmd.CombinedOutput()\n\tif err != nil && len(out) != 0 {\n\t\terr = errors.New(string(out))\n\t}\n\treturn err\n}\n\nfunc umount(ctx *cli.Context) error {\n\tsetup(ctx, 1)\n\tmp := ctx.Args().Get(0)\n\tif ctx.Bool(\"flush\") {\n\t\traw, err := readConfig(mp)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn fmt.Errorf(\"not a JuiceFS mount point\")\n\t\t\t}\n\t\t\treturn errors.Wrap(err, \"failed to read config\")\n\t\t}\n\n\t\tvar conf vfs.Config\n\t\tif err = json.Unmarshal(raw, &conf); err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to parse config\")\n\t\t}\n\t\tif conf.Chunk.Writeback {\n\t\t\tstagingDir := path.Join(conf.Chunk.CacheDir, \"rawstaging\")\n\t\t\tif err := waitWritebackComplete(stagingDir); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdefer func() {\n\t\t\t\tsize, _ := fileSizeInDir(stagingDir)\n\t\t\t\tclearLastLine()\n\t\t\t\tif size == 0 {\n\t\t\t\t\tfmt.Println(\"\\rAll staging chunks are flushed\")\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Printf(\"\\r%s staging chunks are not flushed\\n\", humanize.IBytes(size))\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t}\n\treturn doUmount(mp, ctx.Bool(\"force\"))\n}\n\nfunc waitWritebackComplete(stagingDir string) error {\n\tlastLeft := uint64(0)\n\tfor {\n\t\t_, err := os.Stat(stagingDir)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn errors.Wrap(err, \"failed to read staging directory\")\n\t\t}\n\t\tstart := time.Now()\n\t\tsize, err := fileSizeInDir(stagingDir)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn errors.Wrap(err, \"failed to read staging directory\")\n\t\t}\n\t\tif lastLeft == 0 {\n\t\t\tlastLeft = size\n\t\t}\n\n\t\tif size == 0 && lastLeft == 0 {\n\t\t\treturn nil\n\t\t}\n\n\t\tspeed := uint64(0)\n\t\tif lastLeft > size {\n\t\t\tspeed = lastLeft - size\n\t\t}\n\n\t\tleftTime := 720 * time.Hour\n\t\tif speed != 0 {\n\t\t\tleftTime = time.Duration(size/speed) * time.Second\n\t\t}\n\t\tclearLastLine()\n\t\tfmt.Printf(\"\\r%s staging chunks are being flushed... %s/s, left %s\", humanize.IBytes(size), humanize.IBytes(speed), leftTime)\n\t\tlastLeft = size\n\t\ttime.Sleep(time.Second - time.Since(start))\n\t}\n}\n\nfunc fileSizeInDir(dir string) (uint64, error) {\n\tvar size uint64\n\terr := filepath.WalkDir(dir, func(name string, d fs.DirEntry, err error) error {\n\t\tif d != nil && !d.IsDir() {\n\t\t\tfi, _ := d.Info()\n\t\t\tif fi != nil {\n\t\t\t\tsize += uint64(fi.Size())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\treturn size, err\n}\n\nfunc clearLastLine() {\n\tfmt.Printf(\"\\r \")\n}\n" }, { "path": "cmd/version.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdVersion() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"version\",\n\t\tCategory: \"ADMIN\",\n\t\tAction: func(c *cli.Context) error {\n\t\t\tfmt.Printf(\"%s version %s\\n\", c.App.Name, c.App.Version)\n\t\t\treturn nil\n\t\t},\n\t\tUsage: \"Show version\",\n\t}\n}\n" }, { "path": "cmd/warmup.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"bufio\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdWarmup() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"warmup\",\n\t\tAction: warmup,\n\t\tCategory: \"TOOL\",\n\t\tUsage: \"Build cache for target directories/files\",\n\t\tArgsUsage: \"[PATH ...]\",\n\t\tDescription: `\nThis command provides a faster way to actively build cache for the target files. It reads all objects\nof the files and then write them into local cache directory.\n\nExamples:\n# Warm all files in datadir\n$ juicefs warmup /mnt/jfs/datadir\n\n# Warm only three files in datadir\n$ cat /tmp/filelist\n/mnt/jfs/datadir/f1\n/mnt/jfs/datadir/f2\n/mnt/jfs/datadir/f3\n$ juicefs warmup -f /tmp/filelist`,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"file\",\n\t\t\t\tAliases: []string{\"f\"},\n\t\t\t\tUsage: \"file containing a list of paths\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"threads\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tValue: 50,\n\t\t\t\tUsage: \"number of concurrent workers\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"background\",\n\t\t\t\tAliases: []string{\"b\"},\n\t\t\t\tUsage: \"run in background\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"evict\",\n\t\t\t\tUsage: \"evict cached blocks\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"check\",\n\t\t\t\tUsage: \"check whether the data blocks are cached or not\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nconst batchMax = 10240\n\nconst maxInterval = 300\nconst minInterval = 1\n\nvar interval int\n\nfunc readControl(cf *os.File, resp []byte) int {\n\tif interval <= 0 {\n\t\tinterval = 10\n\t}\n\tfor {\n\t\tif n, err := cf.Read(resp); err == nil {\n\t\t\tinterval = max(interval/2, minInterval)\n\t\t\treturn n\n\t\t} else if err == io.EOF {\n\t\t\tinterval = min(interval*2, maxInterval)\n\t\t\ttime.Sleep(time.Millisecond * time.Duration(interval))\n\t\t} else if errors.Is(err, syscall.EBADF) {\n\t\t\tlogger.Fatalf(\"JuiceFS client was restarted\")\n\t\t} else {\n\t\t\tlogger.Fatalf(\"Read message: %d %s\", n, err)\n\t\t}\n\t}\n}\n\nfunc readProgress(cf *os.File, showProgress func(uint64, uint64)) (data []byte, errno syscall.Errno) {\n\tvar resp = make([]byte, 2<<16)\nEND:\n\tfor {\n\t\tn := readControl(cf, resp)\n\t\tfor off := 0; off < n; {\n\t\t\tif off+1 == n {\n\t\t\t\terrno = syscall.Errno(resp[off])\n\t\t\t\tbreak END\n\t\t\t} else if off+17 <= n && resp[off] == meta.CPROGRESS {\n\t\t\t\tshowProgress(binary.BigEndian.Uint64(resp[off+1:off+9]), binary.BigEndian.Uint64(resp[off+9:off+17]))\n\t\t\t\toff += 17\n\t\t\t} else if off+5 < n && resp[off] == meta.CDATA {\n\t\t\t\tsize := binary.BigEndian.Uint32(resp[off+1 : off+5])\n\t\t\t\tdata = resp[off+5:]\n\t\t\t\tif size > uint32(len(resp[off+5:])) {\n\t\t\t\t\ttailData, err := io.ReadAll(cf)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlogger.Errorf(\"Read data error: %v\", err)\n\t\t\t\t\t\tbreak END\n\t\t\t\t\t}\n\t\t\t\t\tdata = append(data, tailData...)\n\t\t\t\t} else {\n\t\t\t\t\tdata = data[:size]\n\t\t\t\t}\n\t\t\t\tbreak END\n\t\t\t} else {\n\t\t\t\tlogger.Errorf(\"Bad response off %d n %d: %v\", off, n, resp)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tif errno != 0 && runtime.GOOS == \"windows\" {\n\t\terrno += 0x20000000\n\t}\n\treturn\n}\n\n// send fill-cache command to controller file\nfunc sendCommand(cf *os.File, action vfs.CacheAction, batch []string, threads uint, background bool, dspin *utils.DoubleSpinner) *vfs.CacheResponse {\n\tpaths := strings.Join(batch, \"\\n\")\n\tvar back uint8\n\tif background {\n\t\tback = 1\n\t}\n\theaderLen, bodyLen := uint32(8), uint32(4+len(paths)+2+1+1)\n\twb := utils.NewBuffer(headerLen + bodyLen)\n\twb.Put32(meta.FillCache)\n\twb.Put32(bodyLen)\n\n\twb.Put32(uint32(len(paths)))\n\twb.Put([]byte(paths))\n\twb.Put16(uint16(threads))\n\twb.Put8(back)\n\twb.Put8(uint8(action))\n\n\tif _, err := cf.Write(wb.Bytes()); err != nil {\n\t\tlogger.Fatalf(\"Write message: %s\", err)\n\t}\n\n\tresp := &vfs.CacheResponse{}\n\tif background {\n\t\tlogger.Infof(\"%s for %d paths in background\", action, len(batch))\n\t\treturn resp\n\t}\n\n\tlastCnt, lastBytes := dspin.Current()\n\tdata, errno := readProgress(cf, func(fileCount, totalBytes uint64) {\n\t\tdspin.SetCurrent(lastCnt+int64(fileCount), lastBytes+int64(totalBytes))\n\t})\n\n\tif errno != 0 {\n\t\tlogger.Fatalf(\"%s failed: %s\", action, errno)\n\t}\n\n\terr := json.Unmarshal(data, resp)\n\tif err != nil {\n\t\tlogger.Fatalf(\"unmarshal error: %s\", err)\n\t}\n\n\treturn resp\n}\n\nfunc warmup(ctx *cli.Context) error {\n\tsetup0(ctx, 0, 0)\n\n\tevict, check := ctx.Bool(\"evict\"), ctx.Bool(\"check\")\n\tif evict && check {\n\t\tlogger.Fatalf(\"--check and --evict can't be used together\")\n\t}\n\n\tvar paths []string\n\tfor _, p := range ctx.Args().Slice() {\n\t\tif abs, err := filepath.Abs(p); err == nil {\n\t\t\tpaths = append(paths, abs)\n\t\t} else {\n\t\t\tlogger.Fatalf(\"Failed to get absolute path of %s: %s\", p, err)\n\t\t}\n\t}\n\tif fname := ctx.String(\"file\"); fname != \"\" {\n\t\tfd, err := os.Open(fname)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"Failed to open file %s: %s\", fname, err)\n\t\t}\n\t\tdefer fd.Close()\n\t\tscanner := bufio.NewScanner(fd)\n\t\tfor scanner.Scan() {\n\t\t\tif p := strings.TrimSpace(scanner.Text()); p != \"\" {\n\t\t\t\tif abs, e := filepath.Abs(p); e == nil {\n\t\t\t\t\tpaths = append(paths, abs)\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Warnf(\"Skipped path %s because it fails to get absolute path: %s\", p, e)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif err = scanner.Err(); err != nil {\n\t\t\tlogger.Fatalf(\"Reading file %s failed with error: %s\", fname, err)\n\t\t}\n\t}\n\tif len(paths) == 0 {\n\t\tlogger.Infof(\"no path\")\n\t\treturn nil\n\t}\n\n\t// find mount point\n\tfirst := paths[0]\n\tcontroller, err := openController(first)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"open control file for %s: %s\", first, err)\n\t}\n\tdefer controller.Close()\n\n\tmp := first\n\tfor ; mp != \"/\"; mp = filepath.Dir(mp) {\n\t\tinode, err := utils.GetFileInode(mp)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"lookup inode for %s: %s\", mp, err)\n\t\t}\n\t\tif inode == uint64(meta.RootInode) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tthreads := ctx.Uint(\"threads\")\n\tif threads == 0 {\n\t\tlogger.Warnf(\"threads should be larger than 0, reset it to 1\")\n\t\tthreads = 1\n\t}\n\n\taction := vfs.WarmupCache\n\tif evict {\n\t\taction = vfs.EvictCache\n\t} else if check {\n\t\taction = vfs.CheckCache\n\t}\n\n\tbackground := ctx.Bool(\"background\")\n\tstart := len(mp)\n\tbatch := make([]string, 0, batchMax)\n\tprogress := utils.NewProgress(background)\n\tdspin := progress.AddDoubleSpinnerTwo(fmt.Sprintf(\"%s file\", action), fmt.Sprintf(\"%s size\", action))\n\ttotal := &vfs.CacheResponse{Locations: make(map[string]uint64)}\n\tfor _, path := range paths {\n\t\tif mp == \"/\" {\n\t\t\tinode, err := utils.GetFileInode(path)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"lookup inode for %s: %s\", mp, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbatch = append(batch, fmt.Sprintf(\"inode:%d\", inode))\n\t\t} else if strings.HasPrefix(path, mp) {\n\t\t\tbatch = append(batch, path[start:])\n\t\t} else {\n\t\t\tlogger.Errorf(\"Path %s is not under mount point %s\", path, mp)\n\t\t\tcontinue\n\t\t}\n\t\tif len(batch) >= batchMax {\n\t\t\tresp := sendCommand(controller, action, batch, threads, background, dspin)\n\t\t\ttotal.Add(resp)\n\t\t\tbatch = batch[:0]\n\t\t}\n\t}\n\tif len(batch) > 0 {\n\t\tresp := sendCommand(controller, action, batch, threads, background, dspin)\n\t\ttotal.Add(resp)\n\t}\n\tprogress.Done()\n\n\tif !background {\n\t\tcount, bytes := dspin.Current()\n\t\tswitch action {\n\t\tcase vfs.WarmupCache:\n\t\t\tlogger.Infof(\"%s: %d files (%s bytes)\", action, count, humanize.IBytes(uint64(bytes)))\n\t\tcase vfs.EvictCache:\n\t\t\tlogger.Infof(\"%s: %d files (%s bytes)\", action, count, humanize.IBytes(uint64(bytes)))\n\t\tcase vfs.CheckCache:\n\t\t\tif len(total.Locations) > 0 {\n\t\t\t\tvar result = [][]string{\n\t\t\t\t\t{\"Location\", \"Size\", \"Percentage\"},\n\t\t\t\t}\n\t\t\t\tvar locs []string\n\t\t\t\tfor loc := range total.Locations {\n\t\t\t\t\tlocs = append(locs, loc)\n\t\t\t\t}\n\t\t\t\tsort.Strings(locs)\n\t\t\t\tfor _, loc := range locs {\n\t\t\t\t\tsize := total.Locations[loc]\n\t\t\t\t\tresult = append(result, []string{loc, humanize.IBytes(size), fmt.Sprintf(\"%.1f%%\", float64(size)*100/float64(bytes))})\n\t\t\t\t}\n\t\t\t\tprintResult(result, 0, false)\n\t\t\t}\n\t\t\tpct := 0.0\n\t\t\tif bytes != 0 {\n\t\t\t\tpct = float64(uint64(bytes)-total.MissBytes) * 100 / float64(bytes)\n\t\t\t}\n\t\t\tlogger.Infof(\"%s: %d files checked, %s of %s (%2.1f%%) cached\", action, count,\n\t\t\t\thumanize.IBytes(uint64(bytes)-total.MissBytes),\n\t\t\t\thumanize.IBytes(uint64(bytes)),\n\t\t\t\tpct)\n\t\t}\n\t}\n\treturn nil\n}\n" }, { "path": "cmd/warmup_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\nfunc TestWarmup(t *testing.T) {\n\tmountTemp(t, nil, nil, nil)\n\tdefer umountTemp(t)\n\n\tif err := os.WriteFile(fmt.Sprintf(\"%s/f1.txt\", testMountPoint), []byte(\"test\"), 0644); err != nil {\n\t\tt.Fatalf(\"write file failed: %s\", err)\n\t}\n\tm := meta.NewClient(testMeta, nil)\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\tt.Fatalf(\"load setting err: %s\", err)\n\t}\n\tuuid := format.UUID\n\tvar cacheDir = \"/var/jfsCache\"\n\tvar filePath string\n\tswitch runtime.GOOS {\n\tcase \"linux\":\n\t\tif os.Getuid() == 0 {\n\t\t\tbreak\n\t\t}\n\t\tfallthrough\n\tcase \"darwin\", \"windows\":\n\t\thomeDir, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"%v\", err)\n\t\t}\n\t\tcacheDir = fmt.Sprintf(\"%s/.juicefs/cache\", homeDir)\n\t}\n\n\tos.RemoveAll(fmt.Sprintf(\"%s/%s\", cacheDir, uuid))\n\tdefer os.RemoveAll(fmt.Sprintf(\"%s/%s\", cacheDir, uuid))\n\n\tif err = Main([]string{\"\", \"warmup\", testMountPoint}); err != nil {\n\t\tt.Fatalf(\"warmup: %s\", err)\n\t}\n\n\ttime.Sleep(2 * time.Second)\n\tfilePath = fmt.Sprintf(\"%s/%s/raw/chunks/0/0/1_0_4\", cacheDir, uuid)\n\tcontent, err := os.ReadFile(filePath)\n\tif err != nil || len(content) < 4 || string(content[:4]) != \"test\" {\n\t\tt.Fatalf(\"warmup: %s; got content %s\", err, content)\n\t}\n}\n" }, { "path": "cmd/webdav.go", "content": "//go:build !nowebdav\n// +build !nowebdav\n\n/*\n * * JuiceFS, Copyright 2022 Juicedata, Inc.\n * *\n * * Licensed under the Apache License, Version 2.0 (the \"License\");\n * * you may not use this file except in compliance with the License.\n * * You may obtain a copy of the License at\n * *\n * * http://www.apache.org/licenses/LICENSE-2.0\n * *\n * * Unless required by applicable law or agreed to in writing, software\n * * distributed under the License is distributed on an \"AS IS\" BASIS,\n * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * * See the License for the specific language governing permissions and\n * * limitations under the License.\n *\n */\n\npackage cmd\n\nimport (\n\t\"os\"\n\t\"path\"\n\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdWebDav() *cli.Command {\n\tselfFlags := []cli.Flag{\n\t\t&cli.StringFlag{\n\t\t\tName: \"cert-file\",\n\t\t\tUsage: \"certificate file for https\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"key-file\",\n\t\t\tUsage: \"key file for https\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"gzip\",\n\t\t\tUsage: \"compress served files via gzip\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"disallowList\",\n\t\t\tUsage: \"disallow list a directory\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"enable-proppatch\",\n\t\t\tUsage: \"enable proppatch method support\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"log\",\n\t\t\tUsage: \"path for WebDAV log\",\n\t\t\tValue: path.Join(getDefaultLogDir(), \"juicefs-webdav.log\"), //nolint:typecheck\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"access-log\",\n\t\t\tUsage: \"path for JuiceFS access log\",\n\t\t},\n\t\t&cli.BoolFlag{\n\t\t\tName: \"background\",\n\t\t\tAliases: []string{\"d\"},\n\t\t\tUsage: \"run in background\",\n\t\t},\n\t\t&cli.IntFlag{\n\t\t\tName: \"threads\",\n\t\t\tAliases: []string{\"p\"},\n\t\t\tValue: 50,\n\t\t\tUsage: \"number of threads for delete jobs (max 255)\",\n\t\t},\n\t\t&cli.StringFlag{\n\t\t\tName: \"mountpoint\",\n\t\t\tValue: \"webdav\",\n\t\t\tUsage: \"the mount point for current volume (to follow symlink)\",\n\t\t},\n\t}\n\n\treturn &cli.Command{\n\t\tName: \"webdav\",\n\t\tAction: webdav,\n\t\tCategory: \"SERVICE\",\n\t\tUsage: \"Start a WebDAV server\",\n\t\tArgsUsage: \"META-URL ADDRESS\",\n\t\tDescription: `\nExamples:\n$ export WEBDAV_USER=root\n$ export WEBDAV_PASSWORD=1234\n$ juicefs webdav redis://localhost localhost:9007`,\n\t\tFlags: expandFlags(selfFlags, clientFlags(0), shareInfoFlags()),\n\t}\n}\n\nfunc webdav(c *cli.Context) error {\n\tsetup(c, 2)\n\tmetaUrl := c.Args().Get(0)\n\tlistenAddr := c.Args().Get(1)\n\t_, jfs := initForSvc(c, c.String(\"mountpoint\"), \"webdav\", metaUrl, listenAddr)\n\tfs.StartHTTPServer(jfs, fs.WebdavConfig{\n\t\tAddr: listenAddr,\n\t\tDisallowList: c.Bool(\"disallowList\"),\n\t\tEnableGzip: c.Bool(\"gzip\"),\n\t\tUsername: os.Getenv(\"WEBDAV_USER\"),\n\t\tPassword: os.Getenv(\"WEBDAV_PASSWORD\"),\n\t\tCertFile: c.String(\"cert-file\"),\n\t\tKeyFile: c.String(\"key-file\"),\n\t\tEnableProppatch: c.Bool(\"enable-proppatch\"),\n\t\tMaxDeletes: c.Int(\"threads\"),\n\t})\n\treturn jfs.Meta().CloseSession()\n}\n" }, { "path": "cmd/webdav_noop.go", "content": "//go:build nowebdav\n// +build nowebdav\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage cmd\n\nimport (\n\t\"errors\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc cmdWebDav() *cli.Command {\n\treturn &cli.Command{\n\t\tName: \"webdav\",\n\t\tCategory: \"SERVICE\",\n\t\tUsage: \"Start a WebDAV server (not included)\",\n\t\tDescription: `This feature is not included. If you want it, recompile juicefs without \"nowebdav\" flag`,\n\t\tAction: func(*cli.Context) error {\n\t\t\treturn errors.New(\"not supported\")\n\t\t},\n\t}\n}\n" }, { "path": "codecov.yml", "content": "github_checks: false\ncoverage:\n status:\n project: false\n patch: false\n" }, { "path": "deploy/juicefs-s3-gateway.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: juicefs-s3-gateway\n namespace: kube-system\nspec:\n replicas: 1\n selector:\n matchLabels:\n app.kubernetes.io/name: juicefs-s3-gateway\n template:\n metadata:\n labels:\n app.kubernetes.io/name: juicefs-s3-gateway\n spec:\n initContainers:\n - name: format\n image: juicedata/mount:latest\n command:\n - sh\n - -c\n - juicefs format --storage=${storage} --bucket=${bucket} --access-key=${accesskey} --secret-key=${secretkey} ${metaurl} ${name}\n envFrom:\n - secretRef:\n name: juicefs-secret\n env:\n - name: accesskey\n valueFrom:\n secretKeyRef:\n name: juicefs-secret\n key: access-key\n - name: secretkey\n valueFrom:\n secretKeyRef:\n name: juicefs-secret\n key: secret-key\n containers:\n - name: gateway\n image: juicedata/mount:latest\n command:\n - sh\n - -c\n - juicefs gateway ${METAURL} ${NODE_IP}:9000 --metrics=${NODE_IP}:9567\n env:\n - name: NODE_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: METAURL\n valueFrom:\n secretKeyRef:\n name: juicefs-secret\n key: metaurl\n - name: MINIO_ROOT_USER\n valueFrom:\n secretKeyRef:\n name: juicefs-secret\n key: access-key\n - name: MINIO_ROOT_PASSWORD\n valueFrom:\n secretKeyRef:\n name: juicefs-secret\n key: secret-key\n ports:\n - containerPort: 9000\n - containerPort: 9567\n resources:\n limits:\n cpu: 5000m\n memory: 5Gi\n requests:\n cpu: 1000m\n memory: 1Gi\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: juicefs-s3-gateway\n namespace: kube-system\n labels:\n app.kubernetes.io/name: juicefs-s3-gateway\nspec:\n selector:\n app.kubernetes.io/name: juicefs-s3-gateway\n ports:\n - name: http\n port: 9000\n targetPort: 9000\n - name: metrics\n port: 9567\n targetPort: 9567\n" }, { "path": "docs/README.md", "content": "# JuiceFS User Manual\n\nPlease visit JuiceFS Documentation Center for more information:\n\n- [🇬🇧 English](https://juicefs.com/docs/community/introduction)\n- [🇨🇳 简体中文](https://juicefs.com/docs/zh/community/introduction)\n" }, { "path": "docs/en/administration/destroy.md", "content": "---\ntitle: How to destroy a file system\nsidebar_position: 8\n---\n\nJuiceFS client provides the `destroy` command to completely destroy a file system, which will result in\n\n- Deletion of all metadata entries of this file system\n- Deletion of all data blocks of this file system\n\nUse this command in the following format.\n\n```shell\njuicefs destroy \n```\n\n- ``: The URL address of the metadata engine\n- ``: The UUID of the file system\n\n## Find the UUID of the file system\n\nJuiceFS client provides a `status` command to view detailed information about a file system by simply specifying the file system's metadata engine URL, e.g.\n\n```shell {8}\n$ juicefs status redis://127.0.0.1:6379\n\n2022/01/26 21:41:37.577645 juicefs[31181] : Meta address: redis://127.0.0.1:6379\n2022/01/26 21:41:37.578238 juicefs[31181] : Ping redis: 55.041µs\n{\n \"Setting\": {\n \"Name\": \"macjfs\",\n \"UUID\": \"eabb96d5-7228-461e-9240-fddbf2b576d8\",\n \"Storage\": \"file\",\n \"Bucket\": \"jfs/\",\n \"AccessKey\": \"\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0,\n \"TrashDays\": 1\n },\n ...\n}\n```\n\n## Destroy a file system\n\n:::danger\nThe destroy operation will cause all the data in the database and the object storage associated with the file system to be deleted. Please make sure to back up the important data before operating!\n:::\n\n```shell {1}\n$ juicefs destroy redis://127.0.0.1:6379 eabb96d5-7228-461e-9240-fddbf2b576d8\n\n2022/01/26 21:52:17.488987 juicefs[31518] : Meta address: redis://127.0.0.1:6379\n2022/01/26 21:52:17.489668 juicefs[31518] : Ping redis: 55.542µs\n volume name: macjfs\n volume UUID: eabb96d5-7228-461e-9240-fddbf2b576d8\ndata storage: file://jfs/\n used bytes: 18620416\n used inodes: 23\nWARNING: The target volume will be destroyed permanently, including:\nWARNING: 1. objects in the data storage\nWARNING: 2. entries in the metadata engine\nProceed anyway? [y/N]: y\ndeleting objects: 68\nThe volume has been destroyed! You may need to delete cache directory manually.\n```\n\nWhen destroying a file system, the client will issue a confirmation prompt. Please make sure to check the file system information carefully and enter `y` after confirming it is correct.\n\n## FAQ\n\n```shell\n2022/01/26 21:47:30.949149 juicefs[31483] : 1 sessions are active, please disconnect them first\n```\n\nIf you receive an error like the one above, which indicates that the file system has not been properly unmounted, please check and confirm that all mount points are unmounted before proceeding.\n" }, { "path": "docs/en/administration/fault_diagnosis_and_analysis.md", "content": "---\ntitle: Troubleshooting Methods\nsidebar_position: 5\nslug: /fault_diagnosis_and_analysis\ndescription: This article introduces troubleshooting methods for JuiceFS mount point, CSI Driver, Hadoop Java SDK, S3 Gateway, and other clients.\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n## Client log {#client-log}\n\nJuiceFS client will output logs for troubleshooting while running. The level of logs in terms of fatality follows DEBUG < INFO < WARNING < ERROR < FATAL. Since DEBUG logs are not printed by default, you need to explicitly enable it if needed, e.g. by adding the `--debug` option when running the JuiceFS client.\n\nDifferent JuiceFS clients print logs in different ways, which are described as follows.\n\n### Mount point\n\nWhen a JuiceFS file system is mounted with the [`-d` option](../reference/command_reference.mdx#mount) (indicating running in the background), it will print logs to the system log file and local log file simultaneously. Depending on which user is running when mounting the file system, the paths of the local log files are slightly different. For root, the local log file locates at `/var/log/juicefs.log`, while it locates at `$HOME/.juicefs/juicefs.log` for non-root users. Please refer to [`--log` option](../reference/command_reference.mdx#mount) for details.\n\nDepending on the operating system, there are different commands to retrieve system logs or read local log files directly.\n\n\n \n\n```bash\ntail -n 100 /var/log/juicefs.log\n```\n\n \n \n\n```bash\nsyslog | grep 'juicefs'\n```\n\n \n \n\n```bash\ncat /var/log/syslog | grep 'juicefs'\n```\n\n \n \n\n```bash\ncat /var/log/messages | grep 'juicefs'\n```\n\n \n\n\nYou can use the `grep` command to filter different levels of logs for performance analysis or troubleshooting:\n\n```shell\ncat /var/log/syslog | grep 'juicefs' | grep ''\n```\n\n### Kubernetes CSI Driver\n\nDepending on the version of the JuiceFS CSI Driver, there are different ways to retrieve logs. Please refer to [CSI Driver documentation](https://juicefs.com/docs/csi/troubleshooting) for details.\n\n### S3 Gateway\n\nThe S3 gateway can only run in the foreground, so client logs are output directly to the terminal. If you deploys the S3 gateway in Kubernetes, you can get logs from the corresponding pods.\n\n### Hadoop Java SDK\n\nThe JuiceFS client logs will be mixed into the logs of processes using JuiceFS Hadoop Java SDK, e.g. Spark executor. Thus, you need to use keywords, e.g. `juicefs` (case-insensitive), to filter out the logs you do not want.\n\n## Access log {#access-log}\n\nEach JuiceFS client has an access log that records all operations on the file system in detail, such as operation type, user ID, group ID, file inodes and time cost. Access logs can be used for various purposes such as performance analysis, auditing, and troubleshooting.\n\n### Access log format\n\nAn example format of an access log is as follows:\n\n```\n2021.01.15 08:26:11.003330 [uid:0,gid:0,pid:4403] write (17669,8666,4993160): OK <0.000010>\n```\n\nThe meaning of each column is:\n\n- `2021.01.15 08:26:11.003330`: The time of the current operation\n- `[uid:0,gid:0,pid:4403]`: User ID, group ID, process ID of the current operation\n- `write`: Operation type\n- `(17669,8666,4993160)`: The input parameters of the current operation type. For example, the input parameters of the `write` operation in the example are the inode of the written file, the size of the written data, and the offset of the written file. Different operation types have different parameters. For details, please refer to the [`vfs.go`](https://github.com/juicedata/juicefs/blob/main/pkg/vfs/vfs.go) file.\n- `OK`: Indicate the current operation is successful or not. If it is unsuccessful, specific failure information will be output.\n- `<0.000010>`: The time (in seconds) that the current operation takes.\n\nAccess logs tend to get very large and difficult for human to process directly, use [`juicefs profile`](#profile) to quickly visualize performance data based on these logs.\n\nDifferent JuiceFS clients obtain access log in different ways, which are described below.\n\n### Mount point\n\nThere is a virtual file named `.accesslog` in the root directory of the JuiceFS file system mount point, the contents of which can be viewed by the `cat` command (the command will not exit), for example (assuming the root directory of the mount point is `/jfs`):\n\n```bash\ncat /jfs/.accesslog\n```\n\n```output\n2021.01.15 08:26:11.003330 [uid:0,gid:0,pid:4403] write (17669,8666,4993160): OK <0.000010>\n2021.01.15 08:26:11.003473 [uid:0,gid:0,pid:4403] write (17675,198,997439): OK <0.000014>\n2021.01.15 08:26:11.003616 [uid:0,gid:0,pid:4403] write (17666,390,951582): OK <0.000006>\n```\n\n### Kubernetes CSI Driver\n\nPlease refer to [CSI Driver documentation](https://juicefs.com/docs/csi/troubleshooting) to find the mount pod or CSI Driver pod depending on the version of JuiceFS CSI Driver you are using, and the `.accesslog` file can be viewed in the root directory of the JuiceFS file system mount point in the pod. The mount point path in the pod is `/jfs/`. Assuming there is a mount pod named as `juicefs-1.2.3.4-pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373`, in which `pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373` is ``, you can then use the following command to view the `.accesslog` file:\n\n```bash\nkubectl -n kube-system exec juicefs-chaos-k8s-002-pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373 -- cat /jfs/pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373/.accesslog\n```\n\n### S3 Gateway\n\nYou need to add the [`--access-log` option](../reference/command_reference.mdx#gateway) when starting the S3 gateway to specify the path to output the access log. By default, the S3 gateway does not output the access log.\n\n### Hadoop Java SDK\n\nYou need to add the `juicefs.access-log` configuration item in the [client configurations](../deployment/hadoop_java_sdk.md#other-configurations) of the JuiceFS Hadoop Java SDK to specify the path of the access log output, and the access log is not output by default.\n\n## Collect Various Information Using the `debug` Subcommand {#debug}\n\nThe `juicefs debug` subcommand can help you automatically collect various information about a specified mount point, facilitating troubleshooting and diagnosis.\n\n```shell\njuicefs debug \n```\n\nThis command collects the following information:\n\n1. JuiceFS version\n2. Operating system version and kernel version\n3. Contents of the JuiceFS `.config` internal file\n4. Contents of the `.stat` internal file in JuiceFS and recorded again after 5 seconds\n5. Command-line parameters used for mounting\n6. Go pprof information\n7. JuiceFS logs (defaulting to the last 5000 lines)\n\nBy default, a `debug` directory is created in the current directory, and the collected information is saved in that directory. Here's an example:\n\n```shell\n$ juicefs debug /tmp/mountpoint\n\n$ tree ./debug\n./debug\n├── tmp-test1-20230609104324\n│ ├── config.txt\n│ ├── juicefs.log\n│ ├── pprof\n│ │ ├── juicefs.allocs.pb.gz\n│ │ ├── juicefs.block.pb.gz\n│ │ ├── juicefs.cmdline.txt\n│ │ ├── juicefs.goroutine.pb.gz\n│ │ ├── juicefs.goroutine.stack.txt\n│ │ ├── juicefs.heap.pb.gz\n│ │ ├── juicefs.mutex.pb.gz\n│ │ ├── juicefs.profile.30s.pb.gz\n│ │ ├── juicefs.threadcreate.pb.gz\n│ │ └── juicefs.trace.5s.pb.gz\n│ ├── stats.5s.txt\n│ ├── stats.txt\n│ └── system-info.log\n└── tmp-test1-20230609104324.zip\n```\n\n## Real-time performance monitoring {#performance-monitor}\n\nJuiceFS provides the `profile` and `stats` subcommands to visualize real-time performance data, the `profile` command is based on the [file system access log](#access-log), while the `stats` command uses [Real-time statistics](../administration/monitoring.md).\n\n### `juicefs profile` {#profile}\n\n[`juicefs profile`](../reference/command_reference.mdx#profile) will collect data from [file system access log](#access-log), run the `juicefs profile MOUNTPOINT` command, you can see the real-time statistics of each file system operation based on the latest access log:\n\n![JuiceFS-profiling](../images/juicefs-profiling.gif)\n\nApart from real-time mode, this command also provides a play-back mode, which performs the same visualization on existing access log files:\n\n```shell\n# Collect access logs in advance\ncat /jfs/.accesslog > /tmp/juicefs.accesslog\n\n# After performance issue is reproduced, re-play this log file to find system bottleneck\njuicefs profile -f /tmp/juicefs.accesslog\n```\n\nIf the replay speed is too fast, pause anytime using Enter/Return, and continue by pressing it again. If too slow, use `--interval 0` and it will replay the whole log file as fast as possible, and directly show the final result.\n\nIf you're only interested in a certain user or process, you can set filters:\n\n```bash\njuicefs profile /tmp/juicefs.accesslog --uid 12345\n```\n\n### `juicefs stats` {#stats}\n\nThe [`juicefs stats`](../reference/command_reference.mdx#stats) command reads JuiceFS Client internal metrics data, and output performance data in a format similar to `dstat`:\n\n![juicefs_stats_watcher](../images/juicefs_stats_watcher.png)\n\nMetrics description:\n\n#### `usage`\n\n- `cpu`: CPU usage of the process.\n- `mem`: Physical memory used by the process.\n- `buf`: Current [buffer size](../guide/cache.md#buffer-size), if this value is constantly close to (or even exceeds) the configured [`--buffer-size`](../reference/command_reference.mdx#mount-data-cache-options), you should increase buffer size or decrease application workload.\n- `cache`: Internal metric, ignore this.\n\n#### `fuse`\n\n- `ops`/`lat`: Operations processed by FUSE per second, and their average latency (in milliseconds).\n- `read`/`write`: Read/write bandwidth usage of FUSE.\n\n#### `meta`\n\n- `ops`/`lat`: Metadata operations processed per second, and their average latency (in milliseconds). Please note that, operations returned directly from cache are not counted in, in order to show a more accurate latency of clients actually interacting with metadata engine.\n- `txn`/`lat`: Write transactions per second processed by the metadata engine and their average latency (in milliseconds). Read-only requests such as `getattr` are only counted as `ops` but not `txn`.\n- `retry`: Write transactions per second that the metadata engine retries.\n\n#### `blockcache`\n\nThe `blockcache` stands for local cache data, if read requests are already handled by kernel page cache, they won't be counted into the `blockcache` read metric. If there's consistent `blockcache` read traffic while you are conducting repeated read on a fixed file, this means read requests never enter page cache, and you should probably troubleshoot in this direction (e.g. not enough memory).\n\n- `read`/`write`: Read/write bandwidth of client local data cache\n\n#### `object`\n\nThe `object` stands for object storage related metrics, when cache is enabled, penetration to object storage will significantly hinder read performance, use these metrics to check if data has been fully cached. On the other hand, you can also compare `object.get` and `fuse.read` traffic to get a rough idea of the current [read amplification](./troubleshooting.md#read-amplification) status.\n\n- `get`/`get_c`/`lat`: Bandwidth, requests per second, and their average latency (in milliseconds) for object storage processing read requests.\n- `put`/`put_c`/`lat`: Bandwidth, requests per second, and their average latency (in milliseconds) for object storage processing write requests.\n- `del_c`/`lat`: Delete requests per second the object storage can process, and the average latency (in milliseconds).\n\n## Get runtime information using pprof {#runtime-information}\n\nBy default, JuiceFS clients will listen to a TCP port locally via [pprof](https://pkg.go.dev/net/http/pprof) to get runtime information such as Goroutine stack information, CPU performance statistics, memory allocation statistics. You can view the specific port number that the current JuiceFS client is listening to through the `.config` file under the mount point:\n\n```bash\n# Assume the mount point is /jfs\n$ cat /jfs/.config | grep 'DebugAgent'\n \"DebugAgent\": \"127.0.0.1:6064\",\n```\n\nThe default port number range that pprof listens to starts from 6060 and ends at 6099. From the above example, you can see that the actual port number is 6064. Once you get the listening port number, you can view all the available runtime information by accessing `http://localhost:/debug/pprof`, and some important runtime information will be shown as follows:\n\n- Goroutine stack information: `http://localhost:/debug/pprof/goroutine?debug=1`\n- CPU performance statistics: `http://localhost:/debug/pprof/profile?seconds=30`\n- Memory allocation statistics: `http://localhost:/debug/pprof/heap`\n\nTo make it easier to analyze this runtime information, you can save it locally, e.g.:\n\n```bash\ncurl 'http://localhost:/debug/pprof/goroutine?debug=1' > juicefs.goroutine.txt\n```\n\n```bash\ncurl 'http://localhost:/debug/pprof/profile?seconds=30' > juicefs.cpu.pb.gz\n```\n\n```bash\ncurl 'http://localhost:/debug/pprof/heap' > juicefs.heap.pb.gz\n```\n\n:::tip\nYou can also use the `juicefs debug` command to automatically collect these runtime information and save it locally. By default, it is saved to the `debug` directory under the current directory, for example:\n\n```bash\njuicefs debug /mnt/jfs\n```\n\nFor more information about the `juicefs debug` command, see [command reference](../reference/command_reference.mdx#debug).\n:::\n\nIf you have the `go` command installed, you can analyze it directly with the `go tool pprof` command. For example to analyze CPU performance statistics:\n\n```bash\n$ go tool pprof 'http://localhost:/debug/pprof/profile?seconds=30'\nFetching profile over HTTP from http://localhost:/debug/pprof/profile?seconds=30\nSaved profile in /Users/xxx/pprof/pprof.samples.cpu.001.pb.gz\nType: cpu\nTime: Dec 17, 2021 at 1:41pm (CST)\nDuration: 30.12s, Total samples = 32.06s (106.42%)\nEntering interactive mode (type \"help\" for commands, \"o\" for options)\n(pprof) top\nShowing nodes accounting for 30.57s, 95.35% of 32.06s total\nDropped 285 nodes (cum <= 0.16s)\nShowing top 10 nodes out of 192\n flat flat% sum% cum cum%\n 14.73s 45.95% 45.95% 14.74s 45.98% runtime.cgocall\n 7.39s 23.05% 69.00% 7.41s 23.11% syscall.syscall\n 2.92s 9.11% 78.10% 2.92s 9.11% runtime.pthread_cond_wait\n 2.35s 7.33% 85.43% 2.35s 7.33% runtime.pthread_cond_signal\n 1.13s 3.52% 88.96% 1.14s 3.56% runtime.nanotime1\n 0.77s 2.40% 91.36% 0.77s 2.40% syscall.Syscall\n 0.49s 1.53% 92.89% 0.49s 1.53% runtime.memmove\n 0.31s 0.97% 93.86% 0.31s 0.97% runtime.kevent\n 0.27s 0.84% 94.70% 0.27s 0.84% runtime.usleep\n 0.21s 0.66% 95.35% 0.21s 0.66% runtime.madvise\n```\n\nRuntime information can also be exported to visual charts for a more intuitive analysis. The visual charts can be exported to various formats such as HTML, PDF, SVG, PNG, etc. For example, the command to export memory allocation statistics as a PDF file is as follows:\n\n:::note\nThe export to visual chart function relies on [Graphviz](https://graphviz.org), so please install it first.\n:::\n\n```bash\ngo tool pprof -pdf 'http://localhost:/debug/pprof/heap' > juicefs.heap.pdf\n```\n\nFor more information about pprof, please see the [official documentation](https://github.com/google/pprof/blob/main/doc/README.md).\n\n### Profiling with the Pyroscope {#use-pyroscope}\n\n![Pyroscope](../images/pyroscope.png)\n\n[Pyroscope](https://github.com/pyroscope-io/pyroscope) is an open source continuous profiling platform. It will help you:\n\n+ Find performance issues and bottlenecks in your code\n+ Resolve issues of high CPU utilization\n+ Understand the call tree of your application\n+ Track changes over time\n\nJuiceFS supports using the `--pyroscope` option to pass in the pyroscope server address, and metrics are pushed to the server every 10 seconds. If permission verification is enabled on the server, the verification information API Key can be passed in by the environment variable `PYROSCOPE_AUTH_TOKEN`:\n\n```bash\nexport PYROSCOPE_AUTH_TOKEN=xxxxxxxxxxxxxxxx\njuicefs mount --pyroscope http://localhost:4040 redis://localhost /mnt/jfs\njuicefs dump --pyroscope http://localhost:4040 redis://localhost dump.json\n```\n" }, { "path": "docs/en/administration/metadata/_category_.yml", "content": "label: \"Metadata Engine Best Practices\"\nposition: 1" }, { "path": "docs/en/administration/metadata/etcd_best_practices.md", "content": "---\nsidebar_label: etcd\nsidebar_position: 4\nslug: /etcd_best_practices\n---\n\n# etcd Best Practices\n\n## Data size\n\nBy default, etcd sets a [space quota](https://etcd.io/docs/latest/op-guide/maintenance/#space-quota) of 2GB, which can support storing metadata of two million files. Adjusted via the `--quota-backend-bytes` option, [official suggestion](https://etcd.io/docs/latest/dev-guide/limit) do not exceed 8GB.\n\nBy default, etcd will keep the modification history of all data until the amount of data exceeds the space quota and the service cannot be provided. It is recommended to add the following options to enable [automatic compaction](https://etcd.io/docs/latest/op-guide/maintenance/#auto-compaction):\n\n````\n--auto-compaction-mode revision --auto-compaction-retention 1000000\n````\n\nWhen the amount of data reaches the quota and cannot be written, the capacity can be reduced by manual compaction (`etcdctl compact`) and defragmentation (`etcdctl defrag`). **It is strongly recommended to perform these operations on the nodes of the etcd cluster one by one, otherwise the entire etcd cluster may become unavailable.**\n\n## Performance\n\netcd provides strongly consistent read and write access, and all operations involve multi-machine transactions and disk data persistence. **It is recommended to use high-performance SSD for deployment**, otherwise it will affect the performance of the file system. For more hardware configuration suggestions, please refer to [official documentation](https://etcd.io/docs/latest/op-guide/hardware).\n\nIf the etcd cluster has power-down protection, or other measures that can ensure that all nodes will not go down at the same time, you can also disable data synchronization and disk storage through the `--unsafe-no-fsync` option to reduce access latency and improve files system performance. **At this time, if two nodes are down at the same time, there is a risk of data loss.**\n\n## Kubernetes\n\nIt is recommended to build an independent etcd service in the Kubernetes environment for JuiceFS to use, instead of using the default etcd service in the cluster, to avoid affecting the stability of the Kubernetes cluster when the file system access pressure is high.\n" }, { "path": "docs/en/administration/metadata/fdb_best_practices.md", "content": "---\nsidebar_label: FoundationDB\nsidebar_position: 6\nslug: /fdb_best_practices\n---\n# FoundationDB Best Practices\n\nThis document is currently only available in chinese, translation is in progress...\n" }, { "path": "docs/en/administration/metadata/mysql_best_practices.md", "content": "---\nsidebar_label: MySQL\nsidebar_position: 2\n---\n# MySQL Best Practices\n\nFor distributed file systems where data and metadata are stored separately, the read and write performance and security of metadata directly affects the efficiency and data security of the whole system, respectively.\n\nIn the production environment, it is recommended to select hosted cloud databases provided by cloud computing platforms first, and comebine it with appropriate high availability architecture to use.\n\nPlease always pay attention to the integrity and security of metadata when using JuiceFS no matter whether databases is build on your own or in the cloud.\n\n## Passing sensitive information via environment variables\n\nDatabase password can be set directly through the metadata URL. Although it is easy and convenient, the password may leak during logging and process outputing processes. For the sake of security, it's better to pass the database password through an environment variable.\n\n`META_PASSWORD` is a predefined environment variable for the database password:\n\n```shell\nexport META_PASSWORD=mypassword\njuicefs mount -d \"mysql://user:@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\nSimilarly, `META_PASSWORD_FILE` can be used to provide the database password as a file:\n\n```shell\nexport META_PASSWORD_FILE=/secret/mypassword.txt\njuicefs mount -d \"mysql://user:@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\n## Database connection control\n\nMySQL is a multiple threads database, every client connection need a dedicate server thread, limition of total connections and new connects are prefered. JuiceFS now provides the following options for better control of the connections:\n\n- max_open_conns: The maximim database connections allowed for this mount point, default value is 0 which means ulimited connections. If a non-zero values is provided, lower limit may cause current requests have to wait for other reqeusts to free the database connections under high concurrency, while higher value may waste the server side resources. Dynamicly adjusting is prefered based on real business trafics.\n- max_idle_conns: The minimum database connections allowed for this mount point, default values is double of logical CPU cores. Lower value will bring new database connetions under peak time, while higher value may waste some server side resource and get other mount points lack of database connections in peak time.\n- max_idle_time: The maximum idle time allowed for a database connection, default value is 300 seconds. If a connection has no request to database for a given time, it will be closed to free the server side resource. Lower value will bring new database connetions under peak time.\n- max_life_time: The maximum life time allowed for a database connection, default value is 0 which means unlimited. As database connections are shared with different business requests, some resources (such as memory) may not be freed cleanly or be fragmented. Provide a non-zero value (such as 3600 seconds) will let the connection to be destroyed at given time to fully release the resource associated.\n\nWe can pass the above options in metadata URL :\n\n```shell\nexport META_PASSWORD=mypassword\njuicefs mount -d \"mysql://user:@(192.168.1.6:3306)/juicefs?max_open_conns=30&max_life_time=3600\" /mnt/jfs\n```\n\nPlase refer Go official module manual [Database/SQL](https://pkg.go.dev/database/sql#SetConnMaxIdleTime) for more information.\n\n## Periodic backups\n\nPlease refer to the official manual [Chapter 9. Backup and Recovery](https://dev.mysql.com/doc/refman/8.0/en/backup-and-recovery.html) to learn how to back up and restore databases.\n\nIt is recommended to make a plan for regularly backing up your database, and at the same time, do some tests to restore the data in an experimental environment to confirm that the backup is valid.\n\n## High Availability\n\nThe official MySQL document [Chapter 19. Replication](https://dev.mysql.com/doc/refman/8.0/en/replication.html) and [Chapter 20. Group Replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication.html) are prefered high availability solutions. Please choose the appropriate ones according to your needs.\n\n:::note\nJuiceFS uses [transactions] to ensure atomicity of metadata operations, so a transactional storage engine such as [InnoDB](https://dev.mysql.com/doc/refman/8.0/en/backup-and-recovery.html) is required. Some MySQL based distributed (Multi-Shards) databases may not fully compatiable with MySQL both in SQL syntax or transactions, we do not have any testing or certificating works on them.\n:::\n" }, { "path": "docs/en/administration/metadata/postgresql_best_practices.md", "content": "---\nsidebar_label: PostgreSQL\nsidebar_position: 3\nslug: /postgresql_best_practices\n---\n# PostgreSQL Best Practices\n\nFor distributed file systems where data and metadata are stored separately, the read and write performance and security of metadata directly affects the efficiency and data security of the whole system, respectively.\n\nIn the production environment, it is recommended to select hosted cloud databases provided by cloud computing platforms first, and comebine it with appropriate high availability architecture to use.\n\nPlease always pay attention to the integrity and security of metadata when using JuiceFS no matter whether databases is build on your own or in the cloud.\n\n## Communication Security\n\nBy default, JuiceFS clients will use SSL encryption to connect to PostgreSQL. If SSL encryption is not enabled on the database, you need to append the `sslmode=disable` parameter to the metadata URL.\n\nIt is recommended to configure and keep SSL encryption enabled on the database server side all the time.\n\n## Passing sensitive information via environment variables\n\nDatabase password can be set directly through the metadata URL. Although it is easy and convenient, the password may leak during logging and process outputing processes. For the sake of security, it's better to pass the database password through an environment variable.\n\n`META_PASSWORD` is a predefined environment variable for the database password:\n\n```shell\nexport META_PASSWORD=mypassword\njuicefs mount -d \"postgres://user@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\nSimilarly, `META_PASSWORD_FILE` can be used to provide the database password as a file:\n\n```shell\nexport META_PASSWORD_FILE=/secret/mypassword.txt\njuicefs mount -d \"postgres://user@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\nPostgreSQL is a multiple process database, every client connection need a dedicate server process, limition of total connections and new connects are prefered. JuiceFS now provides the following options for better control of the connections:\n\n- max_open_conns: The maximim database connections allowed for this mount point, default value is 0 which means ulimited connections. If a non-zero values is provided, lower limit may cause current requests have to wait for other reqeusts to free the database connections under high concurrency, while higher value may waste the server side resources. Dynamicly adjusting is prefered based on real business trafics.\n- max_idle_conns: The minimum database connections allowed for this mount point, default values is double of logical CPU cores. Lower value will bring new database connetions under peak time, while higher value may waste some server side resource and get other mount points lack of database connections in peak time. \n- max_idle_time: The maximum idle time allowed for a database connection, default value is 300 seconds. If a connection has no request to database for a given time, it will be closed to free the server side resource. Lower value will bring new database connetions under peak time.\n- max_life_time: The maximum life time allowed for a database connection, default value is 0 which means unlimited. As database connections are shared with different business requests, some resources (such as memory) may not be freed cleanly or be fragmented. Provide a non-zero value (such as 3600 seconds) will let the connection to be destroyed at given time to fully release the resource associated.\n\nWe can pass the above options in metadata URL :\n\n```shell\nexport META_PASSWORD=mypassword\njuicefs mount -d \"postgres://user@192.168.1.6:5432/juicefs?max_open_conns=30&max_life_time=3600\" /mnt/jfs\n```\n\nPlase refer Go official module manual [Datatabase/SQL](https://pkg.go.dev/database/sql#SetConnMaxIdleTime) for more information.\n\n## Authentication methods\n\nPostgreSQL supports the md5 authentication method. The following section can be adapted in the pg_hba.conf of your PostgreSQL instance.\n\n```\n# TYPE DATABASE USER ADDRESS METHOD\nhost juicefs juicefsuser 192.168.1.0/24 md5\n```\n\n## Periodic backups\n\nPlease refer to the official manual [Chapter 26. Backup and Restore](https://www.postgresql.org/docs/current/backup.html) to learn how to back up and restore databases.\n\nIt is recommended to make a plan for regularly backing up your database, and at the same time, do some tests to restore the data in an experimental environment to confirm that the backup is valid.\n\n## Using connection pooler\n\nConnection pooler is a middleware that works between client and database and reuses the earlier connection from the pool, which improve connection efficiency and reduce the loss of short connections. Commonly used connection poolers are [PgBouncer](https://www.pgbouncer.org) and [Pgpool-II](https://www.pgpool.net).\n\n## High Availability\n\nThe official PostgreSQL document [High Availability, Load Balancing, and Replication](https://www.postgresql.org/docs/current/different-replication-solutions.html) compares several common databases in terms of high availability solutions. Please choose the appropriate ones according to your needs.\n\n:::note\nJuiceFS uses [transactions](https://www.postgresql.org/docs/current/tutorial-transactions.html) to ensure atomicity of metadata operations. Since PostgreSQL does not yet support Multi-Shard (Distributed) transactions, do not use a multi-server distributed architecture for the JuiceFS metadata.\n:::\n" }, { "path": "docs/en/administration/metadata/redis_best_practices.md", "content": "---\nsidebar_label: Redis\nsidebar_position: 1\nslug: /redis_best_practices\n---\n\n# Redis Best Practices\n\nTo ensure metadata service performance, we recommend use Redis service managed by public cloud provider, see [Recommended Managed Redis Service](#recommended-managed-redis-service).\n\n## Memory usage\n\nThe space used by the JuiceFS metadata engine is mainly related to the number of files in the file system. According to our experience, the metadata of each file occupies approximately 300 bytes of memory. Therefore, if you want to store 100 million files, approximately 30 GiB of memory is required.\n\nYou can check the specific memory usage through Redis' [`INFO memory`](https://redis.io/commands/info) command, for example:\n\n```\n> INFO memory\nused_memory: 19167628056\nused_memory_human: 17.85G\nused_memory_rss: 20684886016\nused_memory_rss_human: 19.26G\n...\nused_memory_overhead: 5727954464\n...\nused_memory_dataset: 13439673592\nused_memory_dataset_perc: 70.12%\n```\n\nAmong them, `used_memory_rss` is the total memory size actually used by Redis, which includes not only the size of data stored in Redis (that is, `used_memory_dataset` above) but also some Redis [system overhead](https://redis.io/commands/memory-stats) (that is, `used_memory_overhead` above). As mentioned earlier that the metadata of each file occupies about 300 bytes, this is actually calculated by `used_memory_dataset`. If you find that the metadata of a single file in your JuiceFS file system occupies much more than 300 bytes, you can try to run [`juicefs gc`](../../reference/command_reference.mdx#gc) command to clean up possible redundant data.\n\n## High availability\n\n### Sentinel mode {#sentinel-mode}\n\n[Redis Sentinel](https://redis.io/docs/manual/sentinel) is the official solution to high availability for Redis. It provides following capabilities:\n\n- **Monitoring**. Sentinel constantly checks if your master and replica instances are working as expected.\n- **Notification**. Sentinel can notify the system administrator, or other computer programs, via an API, that something is wrong with one of the monitored Redis instances.\n- **Automatic failover**. If a master is not working as expected, Sentinel can start a failover process where a replica is promoted to master, the other additional replicas are reconfigured to use the new master, and the applications using the Redis server are informed about the new address to use when connecting.\n- **Configuration provider**. Sentinel acts as a source of authority for clients service discovery: clients connect to Sentinels in order to ask for the address of the current Redis master responsible for a given service. If a failover occurs, Sentinels will report the new address.\n\n**A stable release of Redis Sentinel is shipped since Redis 2.8**. Redis Sentinel version 1, shipped with Redis 2.6, is deprecated and should not be used.\n\nBefore start using Redis sentinel, learn the [fundamentals](https://redis.io/docs/manual/sentinel#fundamental-things-to-know-about-sentinel-before-deploying):\n\n1. You need at least three Sentinel instances for a robust deployment.\n2. The three Sentinel instances should be placed into computers or virtual machines that are believed to fail in an independent way. So for example different physical servers or Virtual Machines executed on different availability zones.\n3. **Sentinel + Redis distributed system does not guarantee that acknowledged writes are retained during failures, since Redis uses asynchronous replication.** However there are ways to deploy Sentinel that make the window to lose writes limited to certain moments, while there are other less secure ways to deploy it.\n4. There is no HA setup which is safe if you don't test from time to time in development environments, or even better if you can, in production environments, if they work. You may have a misconfiguration that will become apparent only when it's too late (at 3am when your master stops working).\n5. **Sentinel, Docker, or other forms of Network Address Translation or Port Mapping should be mixed with care**: Docker performs port remapping, breaking Sentinel auto discovery of other Sentinel processes and the list of replicas for a master.\n\nRead the [official documentation](https://redis.io/docs/manual/sentinel) for more information.\n\nOnce Redis servers and Sentinels are deployed, `META-URL` can be specified as `redis[s]://[[USER]:PASSWORD@]MASTER_NAME,SENTINEL_ADDR[,SENTINEL_ADDR]:SENTINEL_PORT[/DB]`, for example:\n\n```shell\n./juicefs mount redis://:password@masterName,1.2.3.4,1.2.5.6:26379/2 ~/jfs\n```\n\n:::tip\nFor JuiceFS v0.16+, the `PASSWORD` in the URL will be used to connect Redis server, and the password for Sentinel should be provided using the environment variable `SENTINEL_PASSWORD`. For early versions of JuiceFS, the `PASSWORD` is used for both Redis server and Sentinel, which can be overwritten by the environment variables `SENTINEL_PASSWORD` and `REDIS_PASSWORD`.\n:::\n\nSince JuiceFS v1.0.0, it is supported to use Redis replica when mounting file systems, to reduce the load on Redis master. In order to achieve this, you must mount the JuiceFS file system in read-only mode (that is, set the `--read-only` mount option), and connect to the metadata engine through Redis Sentinel. Finally, you need to add `?route-read=replica` to the end of the metadata URL. For example: `redis://:password@masterName,1.2.3.4,1.2.5.6:26379/2?route-read=replica`.\n\nIt should be noted that since the data of the Redis master node is asynchronously replicated to the replica nodes, the read metadata may not be the latest.\n\n### Cluster mode {#cluster-mode}\n\n:::note\nThis feature requires JuiceFS v1.0.0 or higher\n:::\n\nJuiceFS also supports Redis Cluster as a metadata engine, the `META-URL` format is `redis[s]://[[USER]:PASSWORD@]ADDR:PORT,[ADDR:PORT],[ADDR:PORT][/DB]`. For example:\n\n```shell\njuicefs format redis://127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002/1 myjfs\n```\n\n:::tip\nRedis Cluster does not support multiple databases. However, it splits the key space into 16384 hash slots, and distributes the slots to several nodes. Based on Redis Cluster's [Hash Tag](https://redis.io/docs/reference/cluster-spec/#hash-tags) feature, JuiceFS adds `{DB}` before all file system keys to ensure they will be hashed to the same hash slot, assuring that transactions can still work. Besides, one Redis Cluster can serve for multiple JuiceFS file systems as long as they use different db numbers.\n:::\n\n## Data durability\n\nRedis provides various options for [persistence](https://redis.io/docs/manual/persistence) in different ranges:\n\n- **RDB**: The RDB persistence performs point-in-time snapshots of your dataset at specified intervals.\n- **AOF**: The AOF persistence logs every write operation received by the server, which will be played again at server startup, meaning that the original dataset will be reconstructed each time server is restarted. Commands are logged using the same format as the Redis protocol in an append-only fashion. Redis is able to rewrite logs in the background when it gets too big.\n- **RDB+AOF** Recommended: It is possible to combine AOF and RDB in the same instance. Notice that, in this case, when Redis restarts the AOF file will be used to reconstruct the original dataset since it is guaranteed to be the most complete.\n\nWhen using AOF, you can have different fsync policies:\n\n1. No fsync\n2. fsync every second Default\n3. fsync at every query\n\nWith the default policy of fsync every second write performance is good enough (fsync is performed using a background thread and the main thread will try hard to perform writes when no fsync is in progress.), **but you may lose the writes from the last second**.\n\nIn addition, be aware that, even if the RBD+AOF mode is adopted, the disk may be damaged and the virtual machine may disappear. Thus, **Redis data needs to be backed up regularly**.\n\nRedis is very data backup friendly since you can copy RDB files while the database is running. The RDB is never modified once produced: while RDB is produced, a temporary name is assigned to it and will be renamed into its final destination atomically using `rename` only when the new snapshot is complete. You can also copy the AOF file to create backups.\n\nPlease read the [official documentation](https://redis.io/docs/manual/persistence) for more information.\n\n## Backing up Redis data\n\n**Make Sure to Back up Your Database.** as Disks break, instances in the cloud disappear, and so forth.\n\nBy default Redis saves snapshots of the dataset on disk as a binary file called `dump.rdb`. You can configure Redis to save the dataset every N seconds if there are at least M changes in the dataset, or manually call the [`SAVE`](https://redis.io/commands/save) or [`BGSAVE`](https://redis.io/commands/bgsave) commands as needed.\n\nAs we mentioned above, Redis is very data backup friendly. This means that copying the RDB file is completely safe while the server is running. The following are our suggestions:\n\n- Create a cron job in your server, and create hourly snapshots of the RDB file in one directory, and daily snapshots in a different directory.\n- Every time running the cron script, call the `find` command to check if old snapshots have been deleted: for instance you can take hourly snapshots for the latest 48 hours, and daily snapshots for one or two months. Make sure to name the snapshots with data and time information.\n- Make sure to transfer an RDB snapshot _outside your data center_ or at least _outside the physical machine_ running your Redis instance at least one time every day.\n\nPlease read the [official documentation](https://redis.io/docs/manual/persistence) for more information.\n\n## Restore Redis data\n\nAfter generating the AOF or RDB backup file, you can restore the data by copying the backup file to the path corresponding to the `dir` configuration of the new Redis instance. The instance configuration information can be obtained by the [`CONFIG GET dir`](https://redis.io/commands/config-get) command.\n\nIf both AOF and RDB persistence are enabled, Redis will use the AOF file first on starting to recover the data because AOF is guaranteed to be the most complete data.\n\nAfter recovering Redis data, you can continue to use the JuiceFS file system via the new Redis address. It is recommended to run [`juicefs fsck`](../../reference/command_reference.mdx#fsck) command to check the integrity of the file system data.\n\n## Recommended Managed Redis Service\n\n### Amazon MemoryDB for Redis\n\n[Amazon MemoryDB for Redis](https://aws.amazon.com/memorydb) is a durable, in-memory database service that delivers ultra-fast performance. MemoryDB is compatible with Redis, with MemoryDB, all of your data is stored in memory, which enables you to achieve microsecond read and single-digit millisecond write latency and high throughput. MemoryDB also stores data durably across multiple Availability Zones (AZs) using a Multi-AZ transactional log to enable fast failover, database recovery, and node restarts.\n\n### Google Cloud Memorystore for Redis\n\n[Google Cloud Memorystore for Redis](https://cloud.google.com/memorystore/docs/redis) is a fully managed Redis service for the Google Cloud. Applications running on Google Cloud can achieve extreme performance by leveraging the highly scalable, available, secure Redis service without the burden of managing complex Redis deployments.\n\n### Azure Cache for Redis\n\n[Azure Cache for Redis](https://azure.microsoft.com/en-us/services/cache) is a fully managed, in-memory cache that enables high-performance and scalable architectures. It is used to create cloud or hybrid deployments that handle millions of requests per second at sub-millisecond latency, with the advantages of configuration, security, and availability of a managed service.\n\n### Alibaba Cloud ApsaraDB for Redis\n\n[Alibaba Cloud ApsaraDB for Redis](https://www.alibabacloud.com/product/apsaradb-for-redis) is a database service compatible with native Redis protocols. It supports hybrid of memory and hard disks for data persistence. ApsaraDB for Redis provides a highly available hot standby architecture and are scalable to meet requirements for high-performance and low-latency read/write operations.\n\n### Tencent Cloud TencentDB for Redis\n\n[Tencent Cloud TencentDB for Redis](https://intl.cloud.tencent.com/product/crs) is a caching and storage service compatible with the Redis protocol. It features a rich variety of data structure options to help you develop different types of business scenarios, and offers a complete set of database services such as primary-secondary hot backup, automatic switchover for disaster recovery, data backup, failover, instance monitoring, online scaling and data rollback.\n\n## Use Redis compatible product as metadata engine\n\nIf you want to use a Redis compatible product as the metadata engine, you need to confirm whether the following Redis data types and commands required by JuiceFS are fully supported.\n\n### Redis data types used by JuiceFS\n\n+ [String](https://redis.io/docs/data-types/strings)\n+ [Set](https://redis.io/docs/data-types/sets)\n+ [Sorted Set](https://redis.io/docs/data-types/sorted-sets)\n+ [Hash](https://redis.io/docs/data-types/hashes)\n+ [List](https://redis.io/docs/data-types/lists)\n\n### Redis features used by JuiceFS\n\n+ [Pipelining](https://redis.io/docs/manual/pipelining)\n\n### Redis commands used by JuiceFS\n\n#### String\n\n+ [DECRBY](https://redis.io/commands/decrby)\n+ [DEL](https://redis.io/commands/del)\n+ [GET](https://redis.io/commands/get)\n+ [INCR](https://redis.io/commands/incr)\n+ [INCRBY](https://redis.io/commands/incrby)\n+ [DECR](https://redis.io/commands/decr)\n+ [MGET](https://redis.io/commands/mget)\n+ [MSET](https://redis.io/commands/mset)\n+ [SETNX](https://redis.io/commands/setnx)\n+ [SET](https://redis.io/commands/set)\n\n#### Set\n\n+ [SADD](https://redis.io/commands/sadd)\n+ [SMEMBERS](https://redis.io/commands/smembers)\n+ [SREM](https://redis.io/commands/srem)\n\n#### Sorted Set\n\n+ [ZADD](https://redis.io/commands/zadd)\n+ [ZRANGEBYSCORE](https://redis.io/commands/zrangebyscore)\n+ [ZRANGE](https://redis.io/commands/zrange)\n+ [ZREM](https://redis.io/commands/zrem)\n+ [ZSCORE](https://redis.io/commands/zscore)\n\n#### Hash\n\n+ [HDEL](https://redis.io/commands/hdel)\n+ [HEXISTS](https://redis.io/commands/hexists)\n+ [HGETALL](https://redis.io/commands/hgetall)\n+ [HGET](https://redis.io/commands/hget)\n+ [HINCRBY](https://redis.io/commands/hincrby)\n+ [HINCRBY](https://redis.io/commands/hincrby)\n+ [HKEYS](https://redis.io/commands/hkeys)\n+ [HSCAN](https://redis.io/commands/hscan)\n+ [HSETNX](https://redis.io/commands/hsetnx)\n+ [HSET](https://redis.io/commands/hset) (need to support setting multiple fields and values)\n\n#### List\n\n+ [LLEN](https://redis.io/commands/llen)\n+ [LPUSH](https://redis.io/commands/lpush)\n+ [LRANGE](https://redis.io/commands/lrange)\n+ [LTRIM](https://redis.io/commands/ltrim)\n+ [RPUSHX](https://redis.io/commands/rpushx)\n+ [RPUSH](https://redis.io/commands/rpush)\n+ [SCAN](https://redis.io/commands/scan)\n\n#### Transaction\n\n+ [EXEC](https://redis.io/commands/exec)\n+ [MULTI](https://redis.io/commands/multi)\n+ [WATCH](https://redis.io/commands/watch)\n+ [UNWATCH](https://redis.io/commands/unwatch)\n\n#### Connection management\n\n+ [PING](https://redis.io/commands/ping)\n\n#### Server management\n\n+ [CONFIG GET](https://redis.io/commands/config-get)\n+ [CONFIG SET](https://redis.io/commands/config-set)\n+ [DBSIZE](https://redis.io/commands/dbsize)\n+ [FLUSHDB](https://redis.io/commands/flushdb) (optional)\n+ [INFO](https://redis.io/commands/info)\n\n#### Cluster management\n\n+ [CLUSTER INFO](https://redis.io/commands/cluster-info)\n\n#### Scripting (optional)\n\n+ [EVALSHA](https://redis.io/commands/evalsha)\n+ [SCRIPT LOAD](https://redis.io/commands/script-load)\n" }, { "path": "docs/en/administration/metadata/tikv_best_practices.md", "content": "---\nsidebar_label: TiKV\nsidebar_position: 5\nslug: /tikv_best_practices\n---\n# TiKV Best Practices\n\nThis document is currently only available in chinese, translation is in progress...\n" }, { "path": "docs/en/administration/metadata_dump_load.md", "content": "---\ntitle: Metadata Backup & Recovery\nsidebar_position: 2\nslug: /metadata_dump_load\n---\n\n:::tip\n\n- JuiceFS v1.0.0 starts to support automatic metadata backup.\n- JuiceFS v1.0.4 starts to support importing an encrypted backup.\n- JuiceFS v1.3.0 starts to support binary format metadata backup and recovery.\n\n:::\n\nJuiceFS supports [multiple metadata engines](../reference/how_to_set_up_metadata_engine.md), and each engine stores and manages data in a different format internally. JuiceFS provides the [`dump`](../reference/command_reference.mdx#dump) command to export metadata in a uniform JSON format, also there's the [`load`](../reference/command_reference.mdx#load) command to restore or migrate backups to any metadata storage engine. This dump / load process can also be used to migrate a community edition file system to enterprise edition (read [enterprise docs](https://juicefs.com/docs/cloud/administration/metadata_dump_load) for more), and vice versa.\n\n## Metadata backup {#backup}\n\n:::note\n\n* `juicefs dump` does not provide snapshot consistency. If files are modified during the export, the final backup file will contain information from different points in time, which might prove unusable for some applications (like databases). If you have higher standards for consistency, you should suspend all writes to the system before exporting.\n* For large scale file systems, dumping directly from online database may prove risks to system reliability, use with caution.\n\n:::\n\n## File format\n\nJuiceFS supports two formats for metadata backup: JSON and binary. The binary format was introduced in v1.3.0, mainly for large-scale import/export and migration scenarios. The binary format backup is smaller, uses less memory, and supports concurrent import/export.\n\n| Format Type | Structure & Features | Use Case | Size | Memory Usage | Version |\n|------------------|-----------------------------|---------------------------|-------------------|---------------------|------------|\n| **JSON** | Complete directory tree, human-readable | Small/medium FS; troubleshooting | Larger | Higher | All versions |\n| **Binary** | Flattened, efficient, compact | Large-scale import/export/migration | ~1/3 of JSON | < 1GiB (100M files) | v1.3.0+ |\n\n### Manual backup {#backup-manually}\n\nUsing the `dump` command provided by JuiceFS client, you can export metadata to a file, for example:\n\n```shell\n# Export as JSON format\njuicefs dump redis://192.168.1.6:6379 meta-dump\n\n# Export as binary format\njuicefs dump redis://192.168.1.6:6379 meta-dump --binary\n```\n\nThe JSON or binary file exported by using the `dump` command provided by the JuiceFS client can have any filename and extension that you prefer, as shown in the example above. In particular, if the file extension is `.gz` (e.g. `meta-dump.gz`), the exported data will be compressed using the Gzip algorithm. Starting from version 1.3, the Zstandard compression algorithm is also supported, using .zstd as the file extension.\n\nBy default, the `dump` command starts from the root directory `/` and iterates recursively through all the files in the directory tree, and writes the metadata of each file to a JSON output. The object storage credentials will be omitted for data security, but it can be preserved using the `--keep-secret-key` option.\n\nThe value of `juicefs dump` is that it can export complete metadata information in a uniform JSON format for easy management and preservation, and it can be recognized and imported by different metadata storage engines.\n\nIn practice, the `dump` command should be used in conjunction with the backup tool that comes with the database to complement each other, such as [Redis RDB](https://redis.io/topics/persistence#backing-up-redis-data) and [`mysqldump`](https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/mysqldump-sql-format.html), etc.\n\n### Automatic backup {#backup-automatically}\n\nStarting with JuiceFS v1.0.0, the client automatically backs up metadata and copies it to the object storage every hour, regardless of whether the file system is mounted via the `mount` command or accessed via the JuiceFS S3 gateway and Hadoop Java SDK.\n\nThe backup files are stored in the `meta` directory of the object storage. It is a separate directory from the data store and not visible in the mount point and does not interact with the data store, and the directory can be viewed and managed using the file browser of the object storage.\n\n![meta-auto-backup-list](../images/meta-auto-backup-list.png)\n\nBy default, the JuiceFS client backs up metadata once an hour. The frequency of automatic backups can be adjusted by the `--backup-meta` option when mounting the filesystem, for example, to set the auto-backup to be performed every 8 hours.\n\n```shell\njuicefs mount -d --backup-meta 8h redis://127.0.0.1:6379/1 /mnt\n```\n\nThe backup frequency can be accurate to the second and it supports the following units.\n\n- `h`: accurate to the hour, e.g. `1h`.\n- `m`: accurate to the minute, e.g. `30m`, `1h30m`.\n- `s`: accurate to the second, such as `50s`, `30m50s`, `1h30m50s`;\n\nIt is worth mentioning that the time cost of backup will increase with the number of files in the filesystem. Hence, when the number is too large (by default 1 million) with the automatic backup frequency 1 hour (by default), JuiceFS will automatically skip backup and print the corresponding warning log. At this point you may mount a new client with a bigger `--backup-meta` option value to re-enable automatic backups.\n\nFor reference, when using Redis as the metadata engine, backing up the metadata for one million files takes about 1 minute and consumes about 1GB of memory.\n\n:::caution\n When using `--read-only` mount, metadata will not be automatically backed up.\n:::\n\n#### Automatic backup policy\n\nAlthough automatic metadata backup becomes a default action for clients, backup conflicts do not occur when multiple hosts share the same file system mount.\n\nJuiceFS maintains a global timestamp to ensure that only one client performs the backup operation at the same time. When different backup periods are set between clients, then it will back up based on the shortest period setting.\n\n#### Backup cleanup policy\n\nJuiceFS periodically cleans up backups according to the following rules.\n\n- Keep all backups up to 2 days.\n- For backups older than 2 days and less than 2 weeks, keep 1 backup for each day.\n- For backups older than 2 weeks and less than 2 months, keep 1 backup for each week.\n- For backups older than 2 months, keep 1 backup for each month.\n\n## Metadata recovery and migration {#recovery-and-migration}\n\nUse the [`load`](../reference/command_reference.mdx#load) command to restore the metadata dump file into an empty database, for example:\n\n```shell\n# Import from JSON file\njuicefs load redis://192.168.1.6:6379 meta-dump\n\n# Import from binary backup\njuicefs load redis://192.168.1.6:6379 meta-dump --binary\n```\n\nOnce imported, JuiceFS will recalculate the file system statistics including space usage, inode counters, and eventually generates a globally consistent metadata in the database. If you have a deep understanding of the metadata design of JuiceFS, you can also modify the metadata backup file before restoring to debug.\n\nThe dump file is written in an uniform format, which can be recognized and imported by all metadata engines, making it easy to migrate to other types of metadata engines.\n\nFor instance, to migrate from a Redis database to MySQL:\n\n1. Exporting metadata backup from Redis:\n\n ```shell\n juicefs dump redis://192.168.1.6:6379 meta-dump.json\n ```\n\n1. Restoring metadata to a new MySQL database:\n\n ```shell\n juicefs load mysql://user:password@(192.168.1.6:3306)/juicefs meta-dump.json\n ```\n\nIt is also possible to migrate directly through the system's pipe:\n\n```shell\njuicefs dump redis://192.168.1.6:6379 | juicefs load mysql://user:password@(192.168.1.6:3306)/juicefs\n```\n\nNote that since the API access key for object storage is excluded by default from the backup, when loading metadata, you need to use the [`juicefs config`](../reference/command_reference.mdx#config) command to reconfigure the object storage credentials. For example:\n\n```shell\njuicefs config --secret-key xxxxx mysql://user:password@(192.168.1.6:3306)/juicefs\n```\n\n### Encrypted file system {#encrypted-file-system}\n\nFor [encrypted file system](../security/encryption.md), all data is encrypted before uploading to the object storage, including automatic metadata backups. This is different from the `dump` command, which only output metadata in plain text.\n\nFor an encrypted file system, it is necessary to additionally set the `JFS_RSA_PASSPHRASE` environment variable and specify the RSA private key and encryption algorithm when restoring the automatically backed-up metadata:\n\n```shell\nexport JFS_RSA_PASSPHRASE=xxxxxx\njuicefs load \\\n --encrypt-rsa-key my-private.pem \\\n --encrypt-algo aes256gcm-rsa \\\n redis://192.168.1.6:6379/1 \\\n dump-2023-03-16-090750.json.gz\n```\n\n## Metadata inspection {#inspection}\n\nIn addition to completely exporting metadata, you can also export specific subdirectories. You can intuitively inspect the metadata in the directory tree.\n\n```shell\njuicefs dump redis://192.168.1.6:6379 meta-dump.json --subdir /path/in/juicefs\n```\n\nUsing tools like `jq` to analyze the exported file is also an option.\n\n### Binary backup content analysis and troubleshooting\n\nBinary backup also supports direct inspection of type statistics and segment information:\n\n```shell\n# View backup metadata type statistics\njuicefs load meta-dump --binary --stat\n\n# View backup metadata Segments info (get offset)\njuicefs load meta-dump --binary --stat --offset=-1\n\n# View backup metadata for a specific Segment (by offset)\njuicefs load meta-dump --binary --stat --offset=123416309\n```\n\nExample output:\n\n```\nBackup Version: 1\n-----------------------\nName | Num\n-----------------------\nacl | 0\nchunk | 1111179\ncounter | 6\ndelFile | 0\nedge | 1112124\nformat | 1\n…\nSegment: format\nValue: {\n\"Name\": \"test2\",\n\"UUID\": \"15b92123-1395-40e4-a5aa-edb38918985a\",\n\"Storage\": \"file\",\n\"Bucket\": \"/home/hjf/.juicefs/local/\",\n\"BlockSize\": 4096,\n\"Compression\": \"none\",\n\"EncryptAlgo\": \"aes256gcm-rsa\",\n\"TrashDays\": 1,\n\"MetaVersion\": 1,\n\"MinClientVersion\": \"1.1.0-A\",\n\"DirStats\": true,\n\"EnableACL\": false\n}\n```\n\n> The binary backup is in PB format, and you can also use custom tools to verify and inspect the backup.\n" }, { "path": "docs/en/administration/monitoring.md", "content": "---\ntitle: Monitoring and Data Visualization\nsidebar_position: 3\ndescription: This guide will help you understand the monitoring metrics provided by JuiceFS, and how to visualize these metrics using Prometheus and Grafana.\n---\n\nJuiceFS offers a suite of monitoring metrics, and this document outlines how to collect these metrics and visualize them with a monitoring system similar to the one depicted in the following image using Prometheus and Grafana.\n\n![Monitoring Dashboard](../images/grafana_dashboard.png)\n\nThe setup process is as follows:\n\n1. Configure Prometheus to scrape JuiceFS monitoring metrics.\n2. Configure Grafana to read the monitoring data from Prometheus.\n3. Use the official JuiceFS Grafana dashboard template to display the monitoring metrics.\n\n:::tip\nThis document uses open-source versions of Grafana and Prometheus for examples.\n:::\n\n## 1. Configuring Prometheus to Scrape JuiceFS Monitoring Metrics {#add-scrape-config}\n\nAfter mounting JuiceFS, it will automatically expose Prometheus-formatted metrics at `http://localhost:9567/metrics`. To observe the state changes of various metrics over a time range, you'll need to set up Prometheus and configure it to periodically scrape and save these metrics.\n\n![Prometheus Client Data](../images/prometheus-client-data.jpg)\n\nThe process for collecting metrics may vary slightly depending on the mount method or access type (such as FUSE mount, CSI Driver, S3 Gateway, Hadoop SDK, etc.). For detailed instructions, see [Collecting Monitoring metrics data](#collect-metrics).\n\nFor example, here's how you might configure Prometheus for a common FUSE mount: If you haven't already set up Prometheus, follow the [official documentation](https://prometheus.io/docs/prometheus/latest/installation).\n\nEdit your `prometheus.yml` configuration file and add a new scrape configuration under `scrape_configs`. Define the JuiceFS client metrics address:\n\n```yaml {20-22}\nglobal:\n scrape_interval: 15s\n evaluation_interval: 15s\n\nalerting:\n alertmanagers:\n - static_configs:\n - targets:\n # - alertmanager:9093\n\nrule_files:\n # - \"rules.yml\"\n\nscrape_configs:\n - job_name: \"prometheus\"\n static_configs:\n - targets: [\"localhost:9090\"]\n\n - job_name: \"juicefs\"\n static_configs:\n - targets: [\"localhost:9567\"]\n```\n\nStart the Prometheus service:\n\n```shell\n./prometheus --config.file=prometheus.yml\n```\n\nVisit `http://localhost:9090` to see the Prometheus interface.\n\n## 2. Configuring Grafana to Read from Prometheus {#grafana}\n\nOnce Prometheus begins scraping JuiceFS metrics, the next step is to set up Grafana to read from Prometheus.\n\nIf you haven't yet installed Grafana, follow the [official documentation](https://grafana.com/docs/grafana/latest/installation).\n\nIn Grafana, create a new data source of type Prometheus:\n\n- **Name**: A name that helps you identify the data source, such as the name of the file system.\n- **URL**: The Prometheus data API endpoint, typically `http://localhost:9090`.\n\n![Grafana Data Source](../images/grafana-data-source.jpg)\n\n## 3. Using the Official JuiceFS Grafana Dashboard Template {#grafana-dashboard}\n\nJuiceFS's official Grafana dashboard templates can be found in the Grafana Dashboard repository and can be imported directly into Grafana via the URL `https://grafana.com/grafana/dashboards/20794/` or by using the ID `20794`.\n\nHere's what the official JuiceFS Grafana dashboard might look like:\n\n![Grafana Monitoring Dashboard](../images/grafana_dashboard.png)\n\n## Collecting metrics data {#collect-metrics}\n\nFor different types of JuiceFS Client, metrics data is handled slightly differently.\n\n### Mount point {#mount-point}\n\nWhen the JuiceFS file system is mounted via the [`juicefs mount`](../reference/command_reference.mdx#mount) command, you can collect monitoring metrics via the address `http://localhost:9567/metrics`, or you can customize it via the `--metrics` option. For example:\n\n```shell\njuicefs mount --metrics localhost:9567 ...\n```\n\nYou can view these monitoring metrics using the command line tool:\n\n```shell\ncurl http://localhost:9567/metrics\n```\n\nIn addition, the root directory of each JuiceFS file system has a hidden file called `.stats`, through which you can also view monitoring metrics. For example (assuming here that the path to the mount point is `/jfs`):\n\n```shell\ncat /jfs/.stats\n```\n\n:::tip\nIf you want to view the metrics in real-time, you can use the [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) command.\n:::\n\n### Kubernetes {#kubernetes}\n\nSee [CSI Driver documentation](https://juicefs.com/docs/csi/administration/going-production#monitoring).\n\n### S3 Gateway {#s3-gateway}\n\n:::note\nThis feature needs to run JuiceFS client version 0.17.1 and above.\n:::\n\nThe [JuiceFS S3 Gateway](../guide/gateway.md) will provide monitoring metrics at the address `http://localhost:9567/metrics` by default, or you can customize it with the `-metrics` option. For example:\n\n```shell\njuicefs gateway --metrics localhost:9567 ...\n```\n\nIf you are deploying JuiceFS S3 Gateway [in Kubernetes](../guide/gateway.md#deploy-in-kubernetes), you can refer to the Prometheus configuration in the [Kubernetes](#kubernetes) section to collect monitoring metrics (the difference is mainly in the regular expression for the label `__meta_kubernetes_pod_label_app_kubernetes_io_name`), e.g.:\n\n```yaml {6-8}\nscrape_configs:\n - job_name: 'juicefs-s3-gateway'\n kubernetes_sd_configs:\n - role: pod\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]\n action: keep\n regex: juicefs-s3-gateway\n - source_labels: [__address__]\n action: replace\n regex: ([^:]+)(:\\d+)?\n replacement: $1:9567\n target_label: __address__\n - source_labels: [__meta_kubernetes_pod_node_name]\n target_label: node\n action: replace\n```\n\n#### Collected via Prometheus Operator {#prometheus-operator}\n\n[Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) enables users to quickly deploy and manage Prometheus in Kubernetes. With the `ServiceMonitor` CRD provided by Prometheus Operator, scrape configuration can be automatically generated. For example (assuming that the `Service` of the JuiceFS S3 Gateway is deployed in the `kube-system` namespace):\n\n```yaml\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n name: juicefs-s3-gateway\nspec:\n namespaceSelector:\n matchNames:\n - kube-system\n selector:\n matchLabels:\n app.kubernetes.io/name: juicefs-s3-gateway\n endpoints:\n - port: metrics\n```\n\nFor more information on Prometheus Operator, please refer to the [official documentation](https://prometheus-operator.dev/docs/user-guides/getting-started).\n\n### Hadoop Java SDK {#hadoop}\n\n[JuiceFS Hadoop Java SDK](../deployment/hadoop_java_sdk.md) supports reporting monitoring metrics to [Pushgateway](https://github.com/prometheus/pushgateway), [Graphite](https://graphiteapp.org), and [Prometheus remote write](https://prometheus.io/docs/specs/prw/remote_write_spec) endpoints.\n\n#### Pushgateway\n\nReport metrics to Pushgateway:\n\n```xml\n\n juicefs.push-gateway\n host:port\n\n```\n\nAt the same time, the frequency of reporting metrics can be modified through the `juicefs.push-interval` configuration. The default is to report once every 10 seconds.\n\n:::info\nAccording to the suggestion of [Pushgateway official document](https://github.com/prometheus/pushgateway/blob/master/README.md#configure-the-pushgateway-as-a-target-to-scrape), it is required to set `honor_labels: true` in the Prometheus's [scrape configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).\n\nIt is important to note that the timestamp of the metrics scraped by Prometheus from Pushgateway is not the time when the JuiceFS Hadoop Java SDK reported it, but the time when it scraped. For details, please refer to [Pushgateway official document](https://github.com/prometheus/pushgateway/blob/master/README.md#about-timestamps).\n\nBy default, Pushgateway will only save metrics in memory. If you need to persist metrics to disk, you can specify the file path for saving by the `--persistence.file` option and the frequency of saving to the file with the `--persistence.interval` option (by default, the metrics will be saved every 5 minutes).\n:::\n\n:::note\nEach process using JuiceFS Hadoop Java SDK will have a unique metric, and Pushgateway will always remember all the collected metrics. This may cause the continuous accumulation of metrics and taking up too much memory, and it will also make Prometheus scraping metrics slow. Therefore, it is recommended to clean up metrics on Pushgateway regularly.\n\nFor this, the following command can help. Clearing the metrics will not affect the running JuiceFS Hadoop Java SDK to continuously report data. **Note that the `--web.enable-admin-api` option must be specified when Pushgateway is started, and the following command will clear all monitoring metrics in Pushgateway.**\n\n```bash\ncurl -X PUT http://host:9091/api/v1/admin/wipe\n```\n\n:::\n\nFor more information about Pushgateway, please check [official document](https://github.com/prometheus/pushgateway/blob/master/README.md).\n\n#### Graphite\n\nReport metrics to Graphite:\n\n```xml\n\n juicefs.push-graphite\n host:port\n\n```\n\nAt the same time, the frequency of reporting metrics can be modified through the `juicefs.push-interval` configuration. The default is to report every 10 seconds.\n\n#### Remote Write\n\nReport metrics to Prometheus remote write endpoint:\n\n```xml\n\n juicefs.push-remote-write\n http://host:port/api/v1/write\n\n```\n\nAt the same time, the frequency of reporting metrics can be modified through the `juicefs.push-interval` configuration. The default is to report every 10 seconds.\n\n:::info\nThe remote write feature supports various Prometheus-compatible endpoints including:\n\n- [Prometheus with remote write enabled](https://prometheus.io/docs/prometheus/latest/querying/api/#remote-write-receiver)\n- [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/vmagent)\n- [Cortex](https://cortexmetrics.io/docs/architecture)\n- [Grafana Mimir](https://grafana.com/docs/mimir/latest/send)\n- ETC\n\n:::\n\nFor all configurations supported by JuiceFS Hadoop Java SDK, please refer to [documentation](../deployment/hadoop_java_sdk.md#client-configurations).\n\n### Use Consul as registration center {#use-consul}\n\n:::note\nThis feature needs to run JuiceFS client version 1.0.0 and above.\n:::\n\nJuiceFS support to use Consul as registration center for metrics API. The default Consul address is `127.0.0.1:8500`. You could customize the address through `--consul` option, e.g.:\n\n```shell\njuicefs mount --consul 1.2.3.4:8500 ...\n```\n\nWhen the Consul address is configured, the configuration of the `--metrics` option is not needed, and JuiceFS will automatically configure metrics URL according to its own network and port conditions. If `--metrics` is set at the same time, it will first try to listen on the configured metrics URL.\n\nFor each service registered to Consul, the [service name](https://developer.hashicorp.com/consul/docs/services/configuration/services-configuration-reference#name) is always `juicefs`, and the format of [service ID](https://developer.hashicorp.com/consul/docs/services/configuration/services-configuration-reference#id) is `:`, for example: `127.0.0.1:/tmp/jfs`.\n\nThe [`meta`](https://developer.hashicorp.com/consul/docs/services/configuration/services-configuration-reference#meta) of each service contains two keys `hostname` and `mountpoint`, the corresponding values ​​represent the host name and path of the mount point respectively. In particular, the `mountpoint` value for the S3 Gateway is always `s3gateway`.\n\nAfter successfully registering with Consul, you need to add a new [`consul_sd_config`](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) configuration to `prometheus.yml` and fill in the `services` with `juicefs`.\n\n## Monitoring metrics reference {#metrics-reference}\n\nRefer to [JuiceFS Metrics](../reference/p8s_metrics.md).\n" }, { "path": "docs/en/administration/mount_at_boot.md", "content": "---\ntitle: Mount JuiceFS at Boot Time\nsidebar_position: 3\nslug: /mount_juicefs_at_boot_time\n---\n\nAfter JuiceFS has been successfully mounted, follow this guide to set up auto-mount on boot.\n\n## Linux\n\nStarting with JuiceFS v1.1.0, the `--update-fstab` option of the mount command will automatically help you set up mount at boot:\n\n```bash\n$ sudo juicefs mount --update-fstab --max-uploads=50 --writeback --cache-size 204800 \n$ grep /etc/fstab\n juicefs _netdev,max-uploads=50,writeback,cache-size=204800 0 0\n$ ls -l /sbin/mount.juicefs\nlrwxrwxrwx 1 root root 29 Aug 11 16:43 /sbin/mount.juicefs -> /usr/local/bin/juicefs\n```\n\nIf you'd like to control this process by hand, note that:\n\n* A symlink needs to be created from `/sbin/mount.juicefs` to the JuiceFS executable, e.g. `ln -s /usr/local/bin/juicefs /sbin/mount.juicefs`.\n* All mount options must also be included in the fstab options to take effect. Remember to remove the prefixing hyphen(s), and add their values with `=`, for example:\n\n ```bash\n $ sudo juicefs mount --update-fstab --max-uploads=50 --writeback --cache-size 204800 -o max_read=99 /jfs\n # -o stands for FUSE options, and is handled differently\n $ grep jfs /etc/fstab\n redis://localhost:6379/1 /jfs juicefs _netdev,max-uploads=50,max_read=99,writeback,cache-size=204800 0 0\n ```\n\n:::tip\nBy default, CentOS 6 will NOT mount network file system after boot, run following command to enable it:\n\n```bash\nsudo chkconfig --add netfs\n```\n\n:::\n\n### Automating Mounting with systemd.mount\n\nIf you're using JuiceFS and need to apply settings like database access password, S3 access key, and secret key, which are hidden from the command line using environment variables for security reason, it may not be easy to configure them in the `/etc/fstab` file. In such cases, you can utilize systemd to mount your JuiceFS instance.\n\nHere's how you can set up your systemd configuration file:\n\n1. Create the file `/etc/systemd/system/juicefs.mount` and add the following content:\n\n ```conf\n [Unit]\n Description=Juicefs\n Before=docker.service\n\n [Mount]\n Environment=\"ALICLOUD_ACCESS_KEY_ID=mykey\" \"ALICLOUD_ACCESS_KEY_SECRET=mysecret\" \"META_PASSWORD=mypassword\"\n What=mysql://juicefs@(mysql.host:3306)/juicefs\n Where=/juicefs\n Type=juicefs\n Options=_netdev,allow_other,writeback_cache\n\n [Install]\n WantedBy=remote-fs.target\n WantedBy=multi-user.target\n ```\n\n Feel free to modify the options and environments according to your needs.\n\n2. Enable and start the JuiceFS mount using the following commands:\n\n ```sh\n ln -s /usr/local/bin/juicefs /sbin/mount.juicefs\n systemctl enable juicefs.mount\n systemctl start juicefs.mount\n ```\n\nAfter completing these steps, you will be able to access `/juicefs` and store your files there.\n\n## macOS\n\nCreate a file named `io.juicefs..plist` under `~/Library/LaunchAgents`. Replace `` with JuiceFS file system name. Add following contents to the file (again, replace `NAME`, `PATH-TO-JUICEFS`, `META-URL`, `MOUNTPOINT` and `MOUNT-OPTIONS` with appropriate value):\n\n```xml\n\n\n\n\n Label\n io.juicefs.NAME\n ProgramArguments\n \n PATH-TO-JUICEFS\n mount\n META-URL\n MOUNTPOINT\n MOUNT-OPTIONS\n \n RunAtLoad\n \n\n\n```\n\n:::tip\nIf there are multiple mount options, they can be set in multiple lines, for example:\n\n```xml\n --max-uploads\n 50\n --cache-size\n 204800\n```\n\n:::\n\nUse following commands to load the file created in the previous step and test whether the loading is successful. **Please make sure the metadata engine is running properly.**\n\n```bash\nlaunchctl load ~/Library/LaunchAgents/io.juicefs..plist\nlaunchctl start ~/Library/LaunchAgents/io.juicefs.\nls \n```\n\nIf mount failed, you can add following configuration to `io.juicefs..plist` file for debug purpose:\n\n```xml\n StandardOutPath\n /tmp/juicefs.out\n StandardErrorPath\n /tmp/juicefs.err\n```\n\nUse following commands to reload the latest configuration and inspect the output:\n\n```bash\nlaunchctl unload ~/Library/LaunchAgents/io.juicefs..plist\nlaunchctl load ~/Library/LaunchAgents/io.juicefs..plist\ncat /tmp/juicefs.out\ncat /tmp/juicefs.err\n```\n\nIf you install Redis server by Homebrew, you could use following command to start it at boot:\n\n```bash\nbrew services start redis\n```\n\nThen add following configuration to `io.juicefs..plist` file for ensure Redis server is loaded:\n\n```xml\n KeepAlive\n \n OtherJobEnabled\n homebrew.mxcl.redis\n \n```\n" }, { "path": "docs/en/administration/status_check_and_maintenance.md", "content": "---\ntitle: Status Check & Maintenance\nsidebar_position: 4\ndescription: This document introduces JuiceFS' status check and maintenance tools to help you ensure file system reliability and integrity.\n---\n\nAny storage system needs regular checks and maintenance after it is put into use to promptly identify and address potential issues, ensuring the reliability of the file system and the integrity and consistency of stored data.\n\nJuiceFS provides a series of tools to check and maintain the file system. These tools not only help you understand the basic information of the file system and its operational status, but also help you detect and fix potential problems more easily.\n\n## status\n\nThe `juicefs status` command reviews basic information about a JuiceFS file system and the status of all active sessions, including mounts, SDK accesses, S3 Gateway, and WebDAV connections.\n\nThe basic information of the file system includes name, UUID, storage type, bucket, and Trash status.\n\n```shell\njuicefs status redis://xxx.cache.amazonaws.com:6379/1\n```\n\n```json\n{\n \"Setting\": {\n \"Name\": \"myjfs\",\n \"UUID\": \"6b0452fc-0502-404c-b163-c9ab577ec766\",\n \"Storage\": \"s3\",\n \"Bucket\": \"https://xxx.s3.amazonaws.com\",\n \"AccessKey\": \"xxx\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"TrashDays\": 1,\n \"MetaVersion\": 1\n },\n \"Sessions\": [\n {\n \"Sid\": 2,\n \"Heartbeat\": \"2021-08-23T16:47:59+08:00\",\n \"Version\": \"1.0.0+2022-08-08.cf0c269\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869146\n }\n ]\n}\n```\n\nSpecifying the `Sid` of a session with the `--session, -s` option allows you to provide more information about the session.\n\n```shell\njuicefs status --session 2 redis://xxx.cache.amazonaws.com:6379/1\n```\n\n```json\n{\n \"Sid\": 2,\n \"Heartbeat\": \"2021-08-23T16:47:59+08:00\",\n \"Version\": \"1.0.0+2022-08-08.cf0c269\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869146\n}\n```\n\nDepending on the status of the session, the message may also include:\n\n- Sustained inodes: These are files that have been deleted but remain open in the current session, temporarily retained until they are closed.\n- Flocks: BSD lock information about the file locked by this session.\n- Plocks: POSIX lock information about the file locked by this session.\n\n## info\n\nThe `juicefs info` command checks the metadata information of the specified file or directory, including the object path on the object storage for each block corresponding to that file.\n\n### Check file metadata\n\nThis command checks the metadata of a file:\n\n```shell\n$ juicefs info mnt/luggage-6255515.jpg\n\nmnt/luggage-6255515.jpg :\n inode: 36\n files: 1\n dirs: 0\n length: 789.02 KiB (807955 Bytes)\n size: 792.00 KiB (811008 Bytes)\n path: /luggage-6255515.jpg\nobjects:\n+------------+------------------------------+--------+--------+--------+\n| chunkIndex | objectName | size | offset | length |\n+------------+------------------------------+--------+--------+--------+\n| 0 | myjfs/chunks/0/0/80_0_807955 | 807955 | 0 | 807955 |\n+------------+------------------------------+--------+--------+--------+\n```\n\n### Check directory metadata\n\nThis command checks only one level of directories by default:\n\n```shell\n$ juicefs info ./mnt\n\nmnt :\n inode: 1\n files: 9\n dirs: 4\n length: 2.41 MiB (2532102 Bytes)\n size: 2.44 MiB (2555904 Bytes)\n path: /\n```\n\nIf you want to recursively check all subdirectories, you need to specify the `--recursive, -r` option:\n\n```shell\n$ juicefs info -r ./mnt\n\n./mnt :\n inode: 1\n files: 33\n dirs: 4\n length: 80.29 MiB (84191037 Bytes)\n size: 80.34 MiB (84242432 Bytes)\n path: /\n```\n\n### Check metadata with inodes\n\nYou can also perform reverse lookup on the file path and data block information via inodes, but you need to enter the mount point directory.\n\n```shell\n~ $ cd mnt\n~/mnt $ juicefs info -i 36\n\n36 :\n inode: 36\n files: 1\n dirs: 0\n length: 789.02 KiB (807955 Bytes)\n size: 792.00 KiB (811008 Bytes)\n path: /luggage-6255515.jpg\nobjects:\n+------------+------------------------------+--------+--------+--------+\n| chunkIndex | objectName | size | offset | length |\n+------------+------------------------------+--------+--------+--------+\n| 0 | myjfs/chunks/0/0/80_0_807955 | 807955 | 0 | 807955 |\n+------------+------------------------------+--------+--------+--------+\n```\n\n## gc\n\nThe `juicefs gc` command handles \"object leaks\" and runs compaction on data fragments created by file overwrites. It scans metadata and compares it with object storage to find or clean up any object storage blocks that need processing.\n\n:::info\nAn **object leak** is a situation where a block of data is in the object storage, but there is no corresponding record in the metadata engine. Object leaks are rare and can be caused by program bugs, unanticipated problems with the metadata engine or object storage, power outages, and network disconnections.\n:::\n\n:::tip\nTemporary intermediate files may be produced when files are uploaded to the object storage. After the writing is complete, they will be cleaned up. To avoid intermediate files being misclassified as leaked objects, `juicefs gc` skips files uploaded in the last 1 hour by default. The skipped time range (in seconds) can be adjusted via the `JFS_GC_SKIPPEDTIME` environment variable. For example, to set skip the last 2 hours of files: `export JFS_GC_SKIPPEDTIME=7200`.\n:::\n\n:::tip\nBecause the `juicefs gc` command scans all objects in the object storage, there is some overhead in executing this command for file systems with large amounts of data.\n:::\n\n### Scan for leaked objects\n\nAlthough object leaks almost never occur, you can still perform the appropriate routine checks as needed. By default, `juicefs gc` only performs scans:\n\n```shell\n$ juicefs gc sqlite3://myjfs.db\n\n2022/11/10 11:35:53.662024 juicefs[24404] : Meta address: sqlite3://myjfs.db [interface.go:402]\n2022/11/10 11:35:53.662759 juicefs[24404] : Data use file:///Users/herald/.juicefs/local/myjfs/ [gc.go:108]\n Listed slices count: 92\nScanned objects count: 91 / 91 [======================================] done\n Valid objects count: 91\n Valid objects bytes: 7.67 MiB (8040969 Bytes)\n Leaked objects count: 0\n Leaked objects bytes: 0.00 b (0 Bytes)\nSkipped objects count: 0\nSkipped objects bytes: 0.00 b (0 Bytes)\n2022/11/10 11:35:53.665015 juicefs[24404] : scanned 91 objects, 91 valid, 0 leaked (0 bytes), 0 skipped (0 bytes) [gc.go:306]\n```\n\n### Purge leaked objects\n\nWhen the `juicefs gc` command scans for \"leaked objects\", you can purge them with the `--delete` option. The client starts 10 threads by default to perform the purge operation. You can adjust the number of threads with the `--threads, -p` option.\n\n```shell\n$ juicefs gc sqlite3://myjfs.db --delete\n\n2022/11/10 10:49:31.490016 juicefs[24086] : Meta address: sqlite3://myjfs.db [interface.go:402]\n2022/11/10 10:49:31.490831 juicefs[24086] : Data use file:///Users/herald/.juicefs/local/myjfs/ [gc.go:108]\n Listed slices count: 92\nDeleted pending count: 0\nScanned objects count: 103 / 103 [====================================] done\n Valid objects count: 92\n Valid objects bytes: 7.67 MiB (8045065 Bytes)\n Leaked objects count: 11\n Leaked objects bytes: 12.87 MiB (13494874 Bytes)\nSkipped objects count: 0\nSkipped objects bytes: 0.00 b (0 Bytes)\n2022/11/10 10:49:31.493682 juicefs[24086] : scanned 103 objects, 92 valid, 11 leaked (13494874 bytes), 0 skipped (0 bytes) [gc.go:306]\n```\n\nThen, you can run `juicefs gc` again to check if the purge was successful.\n\n## fsck\n\nThe `juicefs fsck` tool performs block-by-block comparison with metadata, mainly to fix various problems that may occur and can be fixed within the file system. It can help you find cases where records exist in the metadata engine but there is no corresponding data block in the object storage. It can also check if the file attribute information exists.\n\n```shell {5}\n$ juicefs fsck sqlite3://myjfs2.db\n\n2022/11/10 17:31:19.062348 juicefs[26158] : Meta address: sqlite3://myjfs2.db [interface.go:402]\n2022/11/10 17:31:19.063132 juicefs[26158] : Data use file:///Users/herald/.juicefs/local/myjfs/ [fsck.go:73]\n2022/11/10 17:31:19.065857 juicefs[26158] : can't find block 0/1/1063_0_2693747 for file /david-bruno-silva-Z19vToWBDIc-unsplash.jpg: stat /Users/herald/.juicefs/local/myjfs/chunks/0/1/1063_0_2693747: no such file or directory [fsck.go:146]\n Found blocks count: 68\n Found blocks bytes: 34.24 MiB (35904042 Bytes)\n Listed slices count: 65\nScanned slices count: 65 / 65 [=======================================] done\nScanned slices bytes: 36.81 MiB (38597789 Bytes)\n Lost blocks count: 1\n Lost blocks bytes: 2.57 MiB (2693747 Bytes)\n2022/11/10 17:31:19.066243 juicefs[26158] : 1 objects are lost (2693747 bytes), 1 broken files:\n INODE: PATH\n 57: /david-bruno-silva-Z19vToWBDIc-unsplash.jpg [fsck.go:168]\n```\n\nAs you can see from the results, the `juicefs fsck` scan found a file corruption in the file system due to a missing data block.\n\nAlthough the result indicates that the file in the backend storage is corrupted, it is still necessary to check if the file is accessible at the mount point. This is because JuiceFS caches the recently accessed file data locally, and the version of the file before the corruption can be re-uploaded with the cached file data block to avoid losing data if it is already cached locally. You can look for cached data in the cache directory (the path corresponding to the `--cache-dir` option) based on the path of the block output from the `juicefs fsck` command. For example, the path of the missing block in the above example is `0/1/1063_0_2693747`.\n\n## compact {#compact}\n\nThe `juicefs compact` command is a new feature introduced in version v1.2. It is a tool used to handle the fragmented data caused by overwrite operations. This tool merges or cleans up the large amounts of non-contiguous slices created by random writes, thereby improving the read performance of the file system.\n\nUnlike `juicefs gc`, which performs garbage collection and fragment cleaning for the entire file system, `juicefs compact` only handles the fragmented data caused by overwrite operations and does not handle object leaks or pending cleanup objects. Additionally, `juicefs compact` only handles the fragmented data within a specified directory and does not handle the entire file system.\n\nYou can use the following command to execute `juicefs compact`:\n\n```shell\njuicefs compact /mnt/jfs/foo\n```\n\nYou can also specify the number of concurrent threads using the `-p` or `--threads` option to speed up processing. The default value is 10, but you can adjust it based on your actual situation.\n\n```shell\njuicefs compact /mnt/jfs/foo -p 20\n```\n" }, { "path": "docs/en/administration/sync_accounts_between_multiple_hosts.md", "content": "---\ntitle: Sync Accounts between Multiple Hosts\nsidebar_position: 7\nslug: /sync_accounts_between_multiple_hosts\n---\n\nJuiceFS supports Unix file permission, you can manage permissions by directory or file granularity, just like a local file system.\n\nTo provide users with an intuitive and consistent permission management experience (e.g. the files accessible by user A on host X should be accessible by the same user on host Y), the same user who wants to access JuiceFS should have the same UID and GID on all hosts.\n\nHere we provide a simple [Ansible](https://www.ansible.com/community) playbook to demonstrate how to ensure an account with same UID and GID on multiple hosts.\n\n:::note\nIf you are using JuiceFS in Hadoop environment, besides sync accounts between multiple hosts, you can also specify a global user list and user group file. Please refer to [here](../deployment/hadoop_java_sdk.md#other-configurations) for more information.\n:::\n\n## Install Ansible\n\nSelect a host as a [control node](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements) which can access all hosts using `ssh` with the same privileged account like `root` or other sudo account. Then, install Ansible on this host. Refer to [Installing Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#installing-ansible) for details.\n\n## Ensure the same account on all hosts\n\nCreate `account-sync/play.yaml` as follows:\n\n```yaml\n---\n- hosts: all\n tasks:\n - name: \"Ensure group {{ group }} with gid {{ gid }} exists\"\n group:\n name: \"{{ group }}\"\n gid: \"{{ gid }}\"\n state: present\n\n - name: \"Ensure user {{ user }} with uid {{ uid }} exists\"\n user:\n name: \"{{ user }}\"\n uid: \"{{ uid }}\"\n group: \"{{ gid }}\"\n state: present\n```\n\nCreate the Ansible inventory `hosts`, which contains IP addresses of all hosts that need to create account.\n\nHere we ensure an account `alice` with UID 1200 and group `staff` with GID 500 on 2 hosts:\n\n```shell\n~/account-sync$ cat hosts\n172.16.255.163\n172.16.255.180\n~/account-sync$ ansible-playbook -i hosts -u root --ssh-extra-args \"-o StrictHostKeyChecking=no\" \\\n--extra-vars \"group=staff gid=500 user=alice uid=1200\" play.yaml\n\nPLAY [all] ************************************************************************************************\n\nTASK [Gathering Facts] ************************************************************************************\nok: [172.16.255.180]\nok: [172.16.255.163]\n\nTASK [Ensure group staff with gid 500 exists] *************************************************************\nok: [172.16.255.163]\nok: [172.16.255.180]\n\nTASK [Ensure user alice with uid 1200 exists] *************************************************************\nchanged: [172.16.255.180]\nchanged: [172.16.255.163]\n\nPLAY RECAP ************************************************************************************************\n172.16.255.163 : ok=3 changed=1 unreachable=0 failed=0\n172.16.255.180 : ok=3 changed=1 unreachable=0 failed=0\n```\n\nNow the new account `alice:staff` has been created on these 2 hosts.\n\nIf the specified UID or GID has been allocated to another user or group on some hosts, the creation would fail.\n\n```shell\n~/account-sync$ ansible-playbook -i hosts -u root --ssh-extra-args \"-o StrictHostKeyChecking=no\" \\\n--extra-vars \"group=ubuntu gid=1000 user=ubuntu uid=1000\" play.yaml\n\nPLAY [all] ************************************************************************************************\n\nTASK [Gathering Facts] ************************************************************************************\nok: [172.16.255.180]\nok: [172.16.255.163]\n\nTASK [Ensure group ubuntu with gid 1000 exists] ***********************************************************\nok: [172.16.255.163]\nfatal: [172.16.255.180]: FAILED! => {\"changed\": false, \"msg\": \"groupmod: GID '1000' already exists\\n\", \"name\": \"ubuntu\"}\n\nTASK [Ensure user ubuntu with uid 1000 exists] ************************************************************\nok: [172.16.255.163]\n to retry, use: --limit @/home/ubuntu/account-sync/play.retry\n\nPLAY RECAP ************************************************************************************************\n172.16.255.163 : ok=3 changed=0 unreachable=0 failed=0\n172.16.255.180 : ok=1 changed=0 unreachable=0 failed=1\n```\n\nIn the above example, the group ID 1000 has been allocated to another group on host `172.16.255.180`. So we should **change the GID** or **delete the group with GID 1000** on host `172.16.255.180`, and then run the playbook again.\n\n:::caution\nIf the UID / GID of an existing user is changed, the user may lose permissions to previously accessible files. For example:\n\n```shell\n$ ls -l /tmp/hello.txt\n-rw-r--r-- 1 alice staff 6 Apr 26 21:43 /tmp/hello.txt\n$ id alice\nuid=1200(alice) gid=500(staff) groups=500(staff)\n```\n\nWe change the UID of alice from 1200 to 1201\n\n```shell\n~/account-sync$ ansible-playbook -i hosts -u root --ssh-extra-args \"-o StrictHostKeyChecking=no\" \\\n--extra-vars \"group=staff gid=500 user=alice uid=1201\" play.yaml\n```\n\nNow we have no permission to remove this file as its owner is not alice:\n\n```shell\n$ ls -l /tmp/hello.txt\n-rw-r--r-- 1 1200 staff 6 Apr 26 21:43 /tmp/hello.txt\n$ rm /tmp/hello.txt\nrm: remove write-protected regular file '/tmp/hello.txt'? y\nrm: cannot remove '/tmp/hello.txt': Operation not permitted\n```\n\n:::\n" }, { "path": "docs/en/administration/troubleshooting.md", "content": "---\ntitle: Troubleshooting Cases\nsidebar_position: 6\n---\n\nDebugging process for some frequently encountered JuiceFS problems.\n\n## Volume format error {#format-error}\n\n### Error creating an already formatted volume {#create-file-system-repeatedly}\n\nIf `juicefs format` has been run on the metadata engine, executing `juicefs format` command again might result in the following error:\n\n```\ncannot update volume XXX from XXX to XXX\n```\n\nIn this case, clean up the metadata engine, and try again.\n\n### Invalid Redis URL {#invalid-redis-url}\n\nWhen using Redis below 6.0.0, `juicefs format` will fail when `username` is specified:\n\n```\nformat: ERR wrong number of arguments for 'auth' command\n```\n\nUsername is supported in Redis 6.0.0 and above, you'll need to omit the `username` from the Redis URL, e.g. `redis://:password@host:6379/1`.\n\n### Redis Sentinel mode NOAUTH error {#redis-sentinel-noauth-error}\n\nIf you encounter the following error when using [Redis Sentinel mode](../administration/metadata/redis_best_practices.md#sentinel-mode):\n\n```\nsentinel: GetMasterAddrByName master=\"xxx\" failed: NOAUTH Authentication required.\n```\n\nPlease confirm whether [the password is set](https://redis.io/docs/management/sentinel/#configuring-sentinel-instances-with-authentication) for the Redis Sentinel instance, if it is set, then you need to pass the `SENTINEL_PASSWORD` environment variable configures the password to connect to the Sentinel instance separately, and the password in the metadata engine URL will only be used to connect to the Redis server.\n\n## Mount errors due to permission issue {#mount-permission-error}\n\nWhen using [Docker bind mounts](https://docs.docker.com/storage/bind-mounts) to mount a directory on the host machine into a container, you may encounter the following error:\n\n```\ndocker: Error response from daemon: error while creating mount source path 'XXX': mkdir XXX: file exists.\n```\n\nThis is usually due to the `juicefs mount` command being executed with a non-root user, thus Docker daemon doesn't have permission to access this directory. You can deal with this using one of below methods:\n\n* Execute `juicefs mount` command with root user\n* Add [`allow_other`](../reference/fuse_mount_options.md#allow_other) option to both FUSE config file, and mount command.\n\nWhen executing `juicefs mount` command with a non-root user, you may see:\n\n```\nfuse: fuse: exec: \"/bin/fusermount\": stat /bin/fusermount: no such file or directory\n```\n\nThis only occurs when a non-root user is trying to mount file system, meaning `fusermount` is not found, there are two solutions to this problem:\n\n* Execute `juicefs mount` command with root user\n* Install `fuse` package (e.g. `apt-get install fuse`, `yum install fuse`)\n\nIf current user doesn't have permission to execute `fusermount` command, you'll see:\n\n```\nfuse: fuse: fork/exec /usr/bin/fusermount: permission denied\n```\n\nWhen this happens, check `fusermount` permission:\n\n```shell\n# Only root user and fuse group user have executable permission\n$ ls -l /usr/bin/fusermount\n-rwsr-x---. 1 root fuse 27968 Dec 7 2011 /usr/bin/fusermount\n\n# All users have executable permission\n$ ls -l /usr/bin/fusermount\n-rwsr-xr-x 1 root root 32096 Oct 30 2018 /usr/bin/fusermount\n```\n\n## Read write slow & read write error {#read-write-error}\n\n### Connection problems with object storage (slow internet speed) {#io-error-object-storage}\n\nIf JuiceFS Client cannot connect to object storage, or the bandwidth is simply not enough, JuiceFS will complain in logs:\n\n```text\n# upload speed is slow\n: slow request: PUT chunks/0/0/1_0_4194304 (%!s(), 20.512s)\n\n# flush timeouts usually means failure to upload data to object storage\n: flush 9902558 timeout after waited 8m0s\n: pending slice 9902558-80: ...\n```\n\nIf the problem is a network connection issue, or the object storage has service issue, troubleshooting is relatively simple. But if the error was caused by low bandwidth, there's some more to consider.\n\nThe first issue with slow connection is upload / download timeouts (demonstrated in the above error logs), to tackle this problem:\n\n* Reduce upload concurrency, e.g. [`--max-uploads=1`](../reference/command_reference.mdx#mount-data-storage-options), to avoid upload timeouts.\n* Reduce buffer size, e.g. [`--buffer-size=64`](../reference/command_reference.mdx#mount-data-cache-options) or even lower. In a large bandwidth condition, increasing buffer size improves parallel performance. But in a low speed environment, this only makes `flush` operations slow and prone to timeouts.\n* Default timeout for GET / PUT requests are 60 seconds, increasing `--get-timeout` and `--put-timeout` may help with read / write timeouts.\n\nIn addition, the [\"Client Write Cache\"](../guide/cache.md#client-write-cache) feature needs to be used with caution in low bandwidth environment. Let's briefly go over the JuiceFS Client background job design: every JuiceFS Client runs background jobs by default, one of which is data compaction, and if the client has poor internet speed, it'll drag down performance for the whole system. A worse case is when client write cache is also enabled, compaction results are uploaded too slowly, forcing other clients into a read hang when accessing the affected files:\n\n```text\n# While compaction results are slowly being uploaded in low speed clients, read from other clients will hang and eventually fail\n: read file 14029704: input/output error\n: slow operation: read (14029704,131072,0): input/output error (0) <74.147891>\n: fail to read sliceId 1771585458 (off:4194304, size:4194304, clen: 37746372): get chunks/0/0/1_0_4194304: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=\"The specified key does not exist.\", RequestId=62E8FB058C0B5C3134CB80B6\n```\n\nTo avoid this type of issue, we recommend disabling background jobs on low-bandwidth clients, i.e. adding [`--no-bgjob`](../reference/command_reference.mdx#mount-metadata-options) option to the mount command.\n\n### WARNING log: block not found in object storage {#warning-log-block-not-found-in-object-storage}\n\nWhen using JuiceFS at scale, there will be some warnings in client logs:\n\n```\n: fail to read sliceId 1771585458 (off:4194304, size:4194304, clen: 37746372): get chunks/0/0/1_0_4194304: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=\"The specified key does not exist.\", RequestId=62E8FB058C0B5C3134CB80B6\n```\n\nWhen this type of warning occurs, but not accompanied by I/O errors (indicated by `input/output error` in client logs), you can safely ignore them and continue normal use, client will retry automatically and resolves this issue.\n\nThis warning means that JuiceFS Client cannot read a particular slice, because a block does not exist, and object storage has to return a `NoSuchKey` error. Usually this is caused by:\n\n* Clients carry out compaction asynchronously, which upon completion, will change the relationship between file and its corresponding blocks, causing problems for other clients that's already reading this file, hence the warning.\n* Some clients enabled [\"Client Write Cache\"](../guide/cache.md#client-write-cache), they write a file, commit to the Metadata Service, but the corresponding blocks are still pending to upload (caused by for example, [slow internet speed](#io-error-object-storage)). Meanwhile, other clients that are already accessing this file will meet this warning.\n\nAgain, if no errors occur, just safely ignore this warning.\n\n## Read amplification\n\nIn JuiceFS, a typical read amplification manifests as object storage traffic being much larger than JuiceFS Client read speed. For example, JuiceFS Client is reading at 200MiB/s, while S3 traffic grows up to 2GiB/s.\n\nJuiceFS is equipped with the [prefetch mechanism](../guide/cache.md#client-read-cache): when reading a block at arbitrary position, the whole block is asynchronously scheduled for download. This is a read optimization enabled by default, but in some cases, this brings read amplification. Once we know this, we can start the diagnose.\n\nWe'll collect JuiceFS access log (see [Access log](./fault_diagnosis_and_analysis.md#access-log)) to determine the file system access patterns of our application, and adjust JuiceFS configuration accordingly. Below is a diagnose process in an actual production environment:\n\n```shell\n# Collect access log for a period of time, like 30 seconds:\ncat /jfs/.accesslog | grep -v \"^#$\" >> access.log\n\n# Simple analysis using wc / grep finds out that most operations are read:\nwc -l access.log\ngrep \"read (\" access.log | wc -l\n\n# Pick a file and track operation history using its inode (first argument of read):\ngrep \"read (148153116,\" access.log\n```\n\nAccess log looks like:\n\n```\n2022.09.22 08:55:21.013121 [uid:0,gid:0,pid:0] read (148153116,131072,28668010496): OK (131072) <1.309992>\n2022.09.22 08:55:21.577944 [uid:0,gid:0,pid:0] read (148153116,131072,14342746112): OK (131072) <1.385073>\n2022.09.22 08:55:22.098133 [uid:0,gid:0,pid:0] read (148153116,131072,35781816320): OK (131072) <1.301371>\n2022.09.22 08:55:22.883285 [uid:0,gid:0,pid:0] read (148153116,131072,3570397184): OK (131072) <1.305064>\n2022.09.22 08:55:23.362654 [uid:0,gid:0,pid:0] read (148153116,131072,100420673536): OK (131072) <1.264290>\n2022.09.22 08:55:24.068733 [uid:0,gid:0,pid:0] read (148153116,131072,48602152960): OK (131072) <1.185206>\n2022.09.22 08:55:25.351035 [uid:0,gid:0,pid:0] read (148153116,131072,60529270784): OK (131072) <1.282066>\n2022.09.22 08:55:26.631518 [uid:0,gid:0,pid:0] read (148153116,131072,4255297536): OK (131072) <1.280236>\n2022.09.22 08:55:27.724882 [uid:0,gid:0,pid:0] read (148153116,131072,715698176): OK (131072) <1.093108>\n2022.09.22 08:55:31.049944 [uid:0,gid:0,pid:0] read (148153116,131072,8233349120): OK (131072) <1.020763>\n2022.09.22 08:55:32.055613 [uid:0,gid:0,pid:0] read (148153116,131072,119523176448): OK (131072) <1.005430>\n2022.09.22 08:55:32.056935 [uid:0,gid:0,pid:0] read (148153116,131072,44287774720): OK (131072) <0.001099>\n2022.09.22 08:55:33.045164 [uid:0,gid:0,pid:0] read (148153116,131072,1323794432): OK (131072) <0.988074>\n2022.09.22 08:55:36.502687 [uid:0,gid:0,pid:0] read (148153116,131072,47760637952): OK (131072) <1.184290>\n2022.09.22 08:55:38.525879 [uid:0,gid:0,pid:0] read (148153116,131072,53434183680): OK (131072) <0.096732>\n```\n\nStudying the access log, it's easy to conclude that our application performs frequent random small reads on a very large file, notice how the offset (the third argument of `read`) jumps significantly between each read, this means consecutive reads are accessing very different parts of the large file, thus prefetched data blocks is not being effectively utilized (a block is 4MiB by default, an offset of 4194304 bytes), only causing read amplifications. In this situation, we can safely set `--prefetch` to 0, so that prefetch concurrency is zero, which is essentially disabled. Re-mount and our problem is solved.\n\n## High memory usage {#memory-optimization}\n\nIf JuiceFS Client takes up too much memory, you may choose to optimize memory usage using below methods, but note that memory optimization is not free, and each setting adjustment will bring corresponding overhead, please do sufficient testing and verification before adjustment.\n\n* Read/Write buffer size (`--buffer-size`) directly correlate to JuiceFS Client memory usage, using a lower `--buffer-size` will effectively decrease memory usage, but please note that the reduction may also affect the read and write performance. Read more at [Read/Write Buffer](../guide/cache.md#buffer-size).\n* JuiceFS mount client is an Go program, which means you can decrease `GOGC` (default to 100, in percentage) to adopt a more active garbage collection. This inevitably increase CPU usage and may even directly hinder performance. Read more at [Go Runtime](https://pkg.go.dev/runtime#hdr-Environment_Variables).\n* If you use self-hosted Ceph RADOS as the data storage of JuiceFS, consider replacing glibc with [TCMalloc](https://google.github.io/tcmalloc), the latter comes with more efficient memory management and may decrease off-heap memory footprint in this scenario.\n\n## Unmount error {#unmount-error}\n\nIf a file or directory are opened when you unmount JuiceFS, you'll see below errors, assuming JuiceFS is mounted on `/jfs`:\n\n```shell\n# Linux\numount: /jfs: target is busy.\n (In some cases useful info about processes that use\n the device is found by lsof(8) or fuser(1))\n\n# macOS\nResource busy -- try 'diskutil unmount'\n```\n\nIn such case:\n\n* Locate the files being opened using commands like `lsof /jfs`, deal with these processes (like force quit), and retry.\n* Force close the FUSE connection by `echo 1 > /sys/fs/fuse/connections/[device-number]/abort`, and then retry. You might need to find out the `[device-number]` using `lsof /jfs`, but if JuiceFS is the only FUSE mount point in the system, then `/sys/fs/fuse/connections` will contain only a single directory, no need to check further.\n* If you just want to unmount ASAP, and do not care what happens to opened files, run `juicefs umount --force` to forcibly umount, note that behavior is different between Linux and macOS:\n * For Linux, `juicefs umount --force` is translated to `umount --lazy`, file system will be detached, but opened files remain, FUSE client will exit when file descriptors are released.\n * For macOS, `juicefs umount --force` is translated to `umount -f`, file system will be forcibly unmounted and opened files will be closed immediately.\n\n## Fail to mount jfs after system reboot {#netmount}\n\nMinimized Linux distribution, such as Alpine, may lack the 'netmount' package within their base image. The absence of the 'netmount' package can lead to failure in automatically mounting network file system like JuiceFS defined in '/etc/fstab' post-rebooting. To rectify this problem, following is the recommended method to install the 'netmount' package, using Alpine as an example:\n\n```bash\n# use --update-fstab to add juicefs mount to /etc/fstab\n\n# install and enable netmount service\napk add openrc\n\nrc-update add netmount boot\n# * service netmount added to runlevel boot\n\n rc-service netmount start\n# / # rc-service netmount start\n# * Mounting network filesystems ...\n```\n\n## Development related issues {#development-related-issues}\n\nCompiling JuiceFS requires GCC 5.4 and above, this error may occur when using lower versions:\n\n```\n/go/pkg/tool/linux_amd64/link: running gcc failed: exit status 1\n/go/pkg/tool/linux_amd64/compile: signal: killed\n```\n\nIf glibc version is different between build environment and runtime, you may see below error:\n\n```\n$ juicefs\njuicefs: /lib/aarch64-linux-gnu/libc.so.6: version 'GLIBC_2.28' not found (required by juicefs)\n```\n\nThis requires you to re-compile JuiceFS Client in your runtime host environment. Most Linux distributions comes with glibc by default, you can check its version with `ldd --version`.\n" }, { "path": "docs/en/administration/upgrade.md", "content": "---\nsidebar_position: 9\n---\n\n# Upgrade\n\nUpgrade methods vary with different JuiceFS clients.\n\n## Mount point\n\n### Normal upgrade\n\nThe JuiceFS client only has one binary file. So to upgrade the new version, you only need to replace the old one with the new one.\n\n- **Use pre-compiled client**: Refer to [Install the pre-compiled client](../getting-started/installation.md#install-the-pre-compiled-client) for details.\n- **Manually compile client**: You can pull the latest source code and recompile it to overwrite the old version of the client. Please refer to [\"Installation\"](../getting-started/installation.md#manually-compiling) for details.\n\n:::caution\nFor the file system that has been mounted using the old version of JuiceFS client, you need to [unmount file system](../getting-started/for_distributed.md#7-unmount-the-file-system), and then re-mount it with the new version of JuiceFS client.\n\nWhen unmounting the file system, make sure that no application is accessing it. Otherwise the unmount will fail. Do not forcibly unmount the file system, as it may cause the application unable to continue to access it as expected.\n:::\n\n### Smooth upgrade\n\nStarting from version v1.2, JuiceFS supports the smooth upgrade feature, which allows you to mount JuiceFS again at the same mount point to achieve a seamless client upgrade. In addition, this feature can also be used to dynamically adjust mount parameters.\n\nHere are two common scenarios for illustration:\n\n- Client upgrade\n For example, if you have a `juicefs mount` process like `juicefs mount redis://127.0.0.1:6379/0 /mnt/jfs -d` and want to upgrade to a new JuiceFS client without unmounting, perform the following steps:\n\n ```shell\n # 1. Backup the current binary\n cp juicefs juicefs.bak\n \n # 2. Download the new binary to overwrite the current juicefs binary\n \n # 3. Execute the juicefs mount command again to complete the smooth upgrade\n juicefs mount redis://127.0.0.1:6379/0 /mnt/jfs -d\n ```\n\n- Dynamically adjusting mount parameters\n\n For example, if you have a `juicefs mount` process like `juicefs mount redis://127.0.0.1:6379/0 /mnt/jfs -d` and want to adjust the log level to debug without unmounting, execute the following command:\n\n```shell\n# Adjust the log level\njuicefs mount redis://127.0.0.1:6379/0 /mnt/jfs --debug -d\n ```\n\nNotes:\n\n- Smooth upgrades require both old and new JuiceFS client versions to be v1.2 or higher.\n\n- The FUSE parameters in the new mount parameters should be consistent with the old mount parameters, otherwise the smooth upgrade will overwrite the mount at the current mount point.\n\n- When `enable-xattr` is enabled, smooth upgrade will overwrite the mount at the current mount point.\n\n## Kubernetes CSI Driver\n\nPlease refer to [official documentation](https://juicefs.com/docs/csi/upgrade-csi-driver) to learn how to upgrade JuiceFS CSI Driver.\n\n## S3 Gateway\n\nLike [mount point](#mount-point), upgrading S3 Gateway is to replace the old version with the new version.\n\nIf it is [deployed through Kubernetes](../guide/gateway.md#deploy-in-kubernetes), you need to upgrade according to the specific deployment method, which is described in detail below.\n\n### Upgrade via kubectl\n\nDownload and modify the `juicedata/juicefs-csi-driver` image tag in S3 Gateway [deploy YAML](https://github.com/juicedata/juicefs/blob/main/deploy/juicefs-s3-gateway.yaml) to the version you want to upgrade (see [here](https://github.com/juicedata/juicefs-csi-driver/releases) for a detailed description of all versions), and then run the following command:\n\n```shell\nkubectl apply -f ./juicefs-s3-gateway.yaml\n```\n\n### Upgrade via Helm\n\nPlease run the following commands in sequence to upgrade the S3 Gateway:\n\n```shell\nhelm repo update\nhelm upgrade juicefs-s3-gateway juicefs-s3-gateway/juicefs-s3-gateway -n kube-system -f ./values.yaml\n```\n\n## Hadoop Java SDK\n\nPlease refer to [Install and compile the client](../deployment/hadoop_java_sdk.md#install-and-compile-the-client) to learn how to install the new version of the Hadoop Java SDK, and then follow steps in [Deploy the client](../deployment/hadoop_java_sdk.md#deploy-the-client) to redeploy the new version of the client to complete the upgrade.\n\n:::note\nSome components must be restarted to use the new version of the Hadoop Java SDK. Please refer to the [\"Restart Services\"](../deployment/hadoop_java_sdk.md#restart-services) for details.\n:::\n" }, { "path": "docs/en/benchmark/benchmark.md", "content": "---\ntitle: Performance Benchmark\nsidebar_position: 1\nslug: .\ndescription: This article describes benchmarking the file system using FIO, mdtest, and the bench command that comes with JuiceFS.\n---\n\nRedis is used as Metadata Engine in this benchmark. Under this test condition, JuiceFS performs 10x better than [Amazon EFS](https://aws.amazon.com/efs) and [S3FS](https://github.com/s3fs-fuse/s3fs-fuse).\n\n## Basic benchmark\n\nJuiceFS provides a subcommand `bench` to run a few basic benchmarks to evaluate how it works in your environment:\n\n![JuiceFS Bench](../images/juicefs-bench.png)\n\n## Throughput\n\nPerformed sequential read/write benchmarks on JuiceFS, [EFS](https://aws.amazon.com/efs) and [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) by [fio](https://github.com/axboe/fio). Here is the result:\n\n[![Sequential Read Write Benchmark](../images/sequential-read-write-benchmark.svg)](../images/sequential-read-write-benchmark.svg)\n\nIt shows JuiceFS can provide 10X more throughput than the other two. Read [more details](fio.md).\n\n## Metadata IOPS\n\nPerformed a simple [mdtest](https://github.com/hpc/ior) benchmark on JuiceFS, [EFS](https://aws.amazon.com/efs) and [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) by [mdtest](https://github.com/hpc/ior). Here is the result:\n\n[![Metadata Benchmark](../images/metadata-benchmark.svg)](../images/metadata-benchmark.svg)\n\nIt shows JuiceFS can provide significantly more metadata IOPS than the other two. Read [more details](mdtest.md).\n\n## Analyze performance\n\nSee [Real-Time Performance Monitoring](../administration/fault_diagnosis_and_analysis.md#performance-monitor) if you encounter performance issues.\n" }, { "path": "docs/en/benchmark/fio.md", "content": "---\ntitle: Benchmark with fio\nsidebar_position: 7\nslug: /fio\n---\n\n:::tip\nTrash is enabled in JuiceFS v1.0+ by default. As a result, temporary files are created and deleted in the file system during the benchmark, and these files will be eventually dumped into a directory named `.trash`. To avoid storage space being occupied by `.trash`, you can run command `juicefs config META-URL --trash-days 0` to disable Trash before benchmark. See [trash](../security/trash.md) for details.\n:::\n\n## Testing Approach\n\nPerform a sequential read/write benchmark on JuiceFS, [EFS](https://aws.amazon.com/efs) and [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) with [fio](https://github.com/axboe/fio).\n\n## Testing Tool\n\nThe following tests are performed with `fio` 3.1.\n\nSequential read test (numjobs: 1):\n\n```\nfio --name=sequential-read --directory=/s3fs --rw=read --refill_buffers --bs=4M --size=4G\nfio --name=sequential-read --directory=/efs --rw=read --refill_buffers --bs=4M --size=4G\nfio --name=sequential-read --directory=/jfs --rw=read --refill_buffers --bs=4M --size=4G\n```\n\nSequential write test (numjobs: 1):\n\n```\nfio --name=sequential-write --directory=/s3fs --rw=write --refill_buffers --bs=4M --size=4G --end_fsync=1\nfio --name=sequential-write --directory=/efs --rw=write --refill_buffers --bs=4M --size=4G --end_fsync=1\nfio --name=sequential-write --directory=/jfs --rw=write --refill_buffers --bs=4M --size=4G --end_fsync=1\n```\n\nSequential read test (numjobs: 16):\n\n```\nfio --name=big-file-multi-read --directory=/s3fs --rw=read --refill_buffers --bs=4M --size=4G --numjobs=16\nfio --name=big-file-multi-read --directory=/efs --rw=read --refill_buffers --bs=4M --size=4G --numjobs=16\nfio --name=big-file-multi-read --directory=/jfs --rw=read --refill_buffers --bs=4M --size=4G --numjobs=16\n```\n\nSequential write test (numjobs: 16):\n\n```\nfio --name=big-file-multi-write --directory=/s3fs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=16 --end_fsync=1\nfio --name=big-file-multi-write --directory=/efs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=16 --end_fsync=1\nfio --name=big-file-multi-write --directory=/jfs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=16 --end_fsync=1\n```\n\n## Testing Environment\n\nAll the following tests are all performed using `fio` on a c5d.18xlarge EC2 instance (72 CPU, 144G RAM) with Ubuntu 18.04 LTS (Kernel 5.4.0) operating system. JuiceFS uses a local Redis instance (version 4.0.9) to store metadata.\n\nJuiceFS mount command:\n\n```\n./juicefs format --storage=s3 --bucket=https://.s3..amazonaws.com localhost benchmark\n./juicefs mount --max-uploads=150 --io-retries=20 localhost /jfs\n```\n\nEFS mount command (the same as the configuration page):\n\n```\nmount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport, .efs..amazonaws.com:/ /efs\n```\n\nS3FS (version 1.82) mount command:\n\n```\ns3fs :/s3fs /s3fs -o host=https://s3..amazonaws.com,endpoint=,passwd_file=${HOME}/.passwd-s3fs\n```\n\n## Testing Result\n\n![Sequential Read Write Benchmark](../images/sequential-read-write-benchmark.svg)\n" }, { "path": "docs/en/benchmark/mdtest.md", "content": "---\ntitle: Benchmark with mdtest\nsidebar_position: 8\nslug: /mdtest\n---\n\n:::tip\nTrash is enabled in JuiceFS v1.0+ by default. As a result, temporary files are created and deleted in the file system during the benchmark, and these files will be eventually dumped into a directory named `.trash`. To avoid storage space being occupied by `.trash`, you can run command `juicefs config META-URL --trash-days 0` to disable Trash before benchmark. See [trash](../security/trash.md) for details.\n:::\n\n## Testing Approach\n\nPerform a metadata test on JuiceFS, [EFS](https://aws.amazon.com/efs) and [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) with [mdtest](https://github.com/hpc/ior).\n\n## Testing Tool\n\nThe following tests are performed with `mdtest` 3.4.\nThe arguments of `mdtest` are tuned to ensure that the command will finish within 5 minutes.\n\n```\n./mdtest -d /s3fs/mdtest -b 6 -I 8 -z 2\n./mdtest -d /efs/mdtest -b 6 -I 8 -z 4\n./mdtest -d /jfs/mdtest -b 6 -I 8 -z 4\n```\n\n## Testing Environment\n\nAll the following tests are performed using `mdtest` on a c5.large EC2 instance (2 CPU, 4G RAM) with Ubuntu 18.04 LTS (Kernel 5.4.0) operating system. The Redis (version 4.0.9) which JuiceFS uses runs on a c5.large EC2 instance in the same available zone to store metadata.\n\nJuiceFS mount command:\n\n```\n./juicefs format --storage=s3 --bucket=https://.s3..amazonaws.com localhost benchmark\nnohup ./juicefs mount localhost /jfs &\n```\n\nEFS mount command (the same as the configuration page):\n\n```\nmount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport, .efs..amazonaws.com:/ /efs\n```\n\nS3FS (version 1.82) mount command:\n\n```\ns3fs :/s3fs /s3fs -o host=https://s3..amazonaws.com,endpoint=,passwd_file=${HOME}/.passwd-s3fs\n```\n\n## Testing Result\n\n![Metadata Benchmark](../images/metadata-benchmark.svg)\n\n### S3FS\n\n```\nmdtest-3.4.0+dev was launched with 1 total task(s) on 1 node(s)\nCommand line used: ./mdtest '-d' '/s3fs/mdtest' '-b' '6' '-I' '8' '-z' '2'\nWARNING: Read bytes is 0, thus, a read test will actually just open/close.\nPath : /s3fs/mdtest\nFS : 256.0 TiB Used FS: 0.0% Inodes: 0.0 Mi Used Inodes: -nan%\nNodemap: 1\n1 tasks, 344 files/directories\n\nSUMMARY rate: (of 1 iterations)\n Operation Max Min Mean Std Dev\n --------- --- --- ---- -------\n Directory creation : 5.977 5.977 5.977 0.000\n Directory stat : 435.898 435.898 435.898 0.000\n Directory removal : 8.969 8.969 8.969 0.000\n File creation : 5.696 5.696 5.696 0.000\n File stat : 68.692 68.692 68.692 0.000\n File read : 33.931 33.931 33.931 0.000\n File removal : 23.658 23.658 23.658 0.000\n Tree creation : 5.951 5.951 5.951 0.000\n Tree removal : 9.889 9.889 9.889 0.000\n```\n\n### EFS\n\n```\nmdtest-3.4.0+dev was launched with 1 total task(s) on 1 node(s)\nCommand line used: ./mdtest '-d' '/efs/mdtest' '-b' '6' '-I' '8' '-z' '4'\nWARNING: Read bytes is 0, thus, a read test will actually just open/close.\nPath : /efs/mdtest\nFS : 8388608.0 TiB Used FS: 0.0% Inodes: 0.0 Mi Used Inodes: -nan%\nNodemap: 1\n1 tasks, 12440 files/directories\n\nSUMMARY rate: (of 1 iterations)\n Operation Max Min Mean Std Dev\n --------- --- --- ---- -------\n Directory creation : 192.301 192.301 192.301 0.000\n Directory stat : 1311.166 1311.166 1311.166 0.000\n Directory removal : 213.132 213.132 213.132 0.000\n File creation : 179.293 179.293 179.293 0.000\n File stat : 915.230 915.230 915.230 0.000\n File read : 371.012 371.012 371.012 0.000\n File removal : 217.498 217.498 217.498 0.000\n Tree creation : 187.906 187.906 187.906 0.000\n Tree removal : 218.357 218.357 218.357 0.000\n```\n\n### JuiceFS\n\n```\nmdtest-3.4.0+dev was launched with 1 total task(s) on 1 node(s)\nCommand line used: ./mdtest '-d' '/jfs/mdtest' '-b' '6' '-I' '8' '-z' '4'\nWARNING: Read bytes is 0, thus, a read test will actually just open/close.\nPath : /jfs/mdtest\nFS : 1024.0 TiB Used FS: 0.0% Inodes: 10.0 Mi Used Inodes: 0.0%\nNodemap: 1\n1 tasks, 12440 files/directories\n\nSUMMARY rate: (of 1 iterations)\n Operation Max Min Mean Std Dev\n --------- --- --- ---- -------\n Directory creation : 1416.582 1416.582 1416.582 0.000\n Directory stat : 3810.083 3810.083 3810.083 0.000\n Directory removal : 1115.108 1115.108 1115.108 0.000\n File creation : 1410.288 1410.288 1410.288 0.000\n File stat : 5023.227 5023.227 5023.227 0.000\n File read : 3487.947 3487.947 3487.947 0.000\n File removal : 1163.371 1163.371 1163.371 0.000\n Tree creation : 1503.004 1503.004 1503.004 0.000\n Tree removal : 1119.806 1119.806 1119.806 0.000\n```\n" }, { "path": "docs/en/benchmark/metadata_engines_benchmark.md", "content": "---\ntitle: Metadata Engines Benchmark\nsidebar_position: 6\nslug: /metadata_engines_benchmark\ndescription: This article describes how to test and evaluate the performance of various metadata engines for JuiceFS using a real-world environment.\n---\n\nConclusion first:\n\n- For pure metadata operations, MySQL costs about 2~4x times of Redis; TiKV has similar performance to MySQL, and in most cases it costs a bit less; etcd costs about 1.5x times of TiKV.\n- For small I/O (~100 KiB) workloads, total time costs with MySQL are about 1~3x of those with Redis; TiKV and etcd performs similarly to MySQL.\n- For large I/O (~4 MiB) workloads, total time costs with different metadata engines show no significant difference (object storage becomes the bottleneck).\n\n:::note\n\n1. By changing `appendfsync` from `always` to `everysec`, Redis gains performance boost but loses a bit of data reliability. More information can be found [here](https://redis.io/docs/manual/persistence).\n2. Both Redis and MySQL store only one replica locally, while TiKV and etcd stores three replicas on three different hosts using Raft protocol.\n\n:::\n\nDetails are provided below. Please note all the tests are run with the same object storage (to save data), clients and metadata hosts, only metadata engines differ.\n\n## Environment\n\n### JuiceFS Version\n\n1.1.0-beta1+2023-06-08.5ef17ba0\n\n### Object Storage\n\nAmazon S3\n\n### Client Hosts\n\n- Amazon c5.xlarge: 4 vCPUs, 8 GiB Memory, Up to 10 Gigabit Network\n- Ubuntu 20.04.1 LTS\n\n### Metadata Hosts\n\n- Amazon c5d.xlarge: 4 vCPUs, 8 GiB Memory, Up to 10 Gigabit Network, 100 GB SSD (local storage for metadata engines)\n- Ubuntu 20.04.1 LTS\n- SSD is formatted as ext4 and mounted on `/data`\n\n### Metadata Engines\n\n#### Redis\n\n- Version: [7.0.9](https://download.redis.io/releases/redis-7.0.9.tar.gz)\n- Configuration:\n - `appendonly`: `yes`\n - `appendfsync`: `always` or `everysec`\n - `dir`: `/data/redis`\n\n#### MySQL\n\n- Version: 8.0.25\n- `/var/lib/mysql` is bind mounted on `/data/mysql`\n\n#### PostgreSQL\n\n- Version: 15.3\n- The data directory was changed to `/data/pgdata`\n\n#### TiKV\n\n- Version: 6.5.3\n- Configuration:\n - `deploy_dir`: `/data/tikv-deploy`\n - `data_dir`: `/data/tikv-data`\n\n#### etcd\n\n- Version: 3.3.25\n- Configuration:\n - `data-dir`: `/data/etcd`\n\n#### FoundationDB\n\n- Version: 6.3.23\n- Configuration：\n - `data-dir`：`/data/fdb`\n\n## Tools\n\nAll the following tests are run for each metadata engine.\n\n### Golang Benchmark\n\nSimple benchmarks within the source code: [`pkg/meta/benchmarks_test.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/benchmarks_test.go)\n\n### JuiceFS Bench\n\nJuiceFS provides a basic benchmark command:\n\n```bash\n./juicefs bench /mnt/jfs -p 4\n```\n\n### mdtest\n\n- Version: mdtest-3.3.0\n\nRun parallel tests on 3 client nodes:\n\n```bash\n$ cat myhost\nclient1 slots=4\nclient2 slots=4\nclient3 slots=4\n```\n\nTest commands:\n\n```bash\n# metadata only\nmpirun --use-hwthread-cpus --allow-run-as-root -np 12 --hostfile myhost --map-by slot /root/mdtest -b 3 -z 1 -I 100 -u -d /mnt/jfs\n\n# 12000 * 100KiB files\nmpirun --use-hwthread-cpus --allow-run-as-root -np 12 --hostfile myhost --map-by slot /root/mdtest -F -w 102400 -I 1000 -z 0 -u -d /mnt/jfs\n```\n\n### fio\n\n- Version: fio-3.28\n\n```bash\nfio --name=big-write --directory=/mnt/jfs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=4 --end_fsync=1 --group_reporting\n```\n\n## Results\n\n### Golang Benchmark\n\n- Shows time cost (us/op). Smaller is better.\n- Number in parentheses is the multiple of Redis-Always cost (`always` and `everysec` are candidates for Redis configuration `appendfsync`).\n- Because of enabling metadata cache, the results of `read` are all less than 1us, which are not comparable for now.\n\n| | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n|--------------|--------------|----------------|--------------|--------------|------------|--------------|--------------|\n| mkdir | 558 | 468 (0.8) | 2042 (3.7) | 1076 (1.9) | 1237 (2.2) | 1916 (3.4) | 1842 (3.3) |\n| mvdir | 693 | 621 (0.9) | 2693 (3.9) | 1459 (2.1) | 1414 (2.0) | 2486 (3.6) | 1895 (2.7) |\n| rmdir | 717 | 648 (0.9) | 3050 (4.3) | 1697 (2.4) | 1641 (2.3) | 2980 (4.2) | 2088 (2.9) |\n| readdir_10 | 280 | 288 (1.0) | 1350 (4.8) | 1098 (3.9) | 995 (3.6) | 1757 (6.3) | 1744 (6.2) |\n| readdir_1k | 1490 | 1547 (1.0) | 18779 (12.6) | 18414 (12.4) | 5834 (3.9) | 15809 (10.6) | 15276 (10.3) |\n| mknod | 562 | 464 (0.8) | 1547 (2.8) | 849 (1.5) | 1211 (2.2) | 1838 (3.3) | 1763 (3.1) |\n| create | 570 | 455 (0.8) | 1570 (2.8) | 844 (1.5) | 1209 (2.1) | 1849 (3.2) | 1761 (3.1) |\n| rename | 728 | 627 (0.9) | 2735 (3.8) | 1478 (2.0) | 1419 (1.9) | 2445 (3.4) | 1911 (2.6) |\n| unlink | 658 | 567 (0.9) | 2365 (3.6) | 1280 (1.9) | 1443 (2.2) | 2461 (3.7) | 1940 (2.9) |\n| lookup | 173 | 178 (1.0) | 557 (3.2) | 375 (2.2) | 608 (3.5) | 1054 (6.1) | 1029 (5.9) |\n| getattr | 87 | 86 (1.0) | 530 (6.1) | 350 (4.0) | 306 (3.5) | 536 (6.2) | 504 (5.8) |\n| setattr | 471 | 345 (0.7) | 1029 (2.2) | 571 (1.2) | 1001 (2.1) | 1279 (2.7) | 1596 (3.4) |\n| access | 87 | 89 (1.0) | 518 (6.0) | 356 (4.1) | 307 (3.5) | 534 (6.1) | 526 (6.0) |\n| setxattr | 393 | 262 (0.7) | 992 (2.5) | 534 (1.4) | 800 (2.0) | 717 (1.8) | 1300 (3.3) |\n| getxattr | 84 | 87 (1.0) | 494 (5.9) | 333 (4.0) | 303 (3.6) | 529 (6.3) | 511 (6.1) |\n| removexattr | 215 | 96 (0.4) | 697 (3.2) | 385 (1.8) | 1007 (4.7) | 1336 (6.2) | 1597 (7.4) |\n| listxattr_1 | 85 | 87 (1.0) | 516 (6.1) | 342 (4.0) | 303 (3.6) | 531 (6.2) | 515 (6.1) |\n| listxattr_10 | 87 | 91 (1.0) | 561 (6.4) | 383 (4.4) | 322 (3.7) | 565 (6.5) | 529 (6.1) |\n| link | 680 | 545 (0.8) | 2435 (3.6) | 1375 (2.0) | 1732 (2.5) | 3058 (4.5) | 2402 (3.5) |\n| symlink | 580 | 448 (0.8) | 1785 (3.1) | 954 (1.6) | 1224 (2.1) | 1897 (3.3) | 1764 (3.0) |\n| newchunk | 0 | 0 (0.0) | 1 (0.0) | 1 (0.0) | 1 (0.0) | 1 (0.0) | 2 (0.0) |\n| write | 553 | 369 (0.7) | 2352 (4.3) | 1183 (2.1) | 1573 (2.8) | 1788 (3.2) | 1747 (3.2) |\n| read_1 | 0 | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) |\n| read_10 | 0 | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) |\n\n### JuiceFS Bench\n\n| | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n|------------------|------------------|------------------|-----------------|-----------------|-----------------|-----------------|-----------------|\n| Write big file | 730.84 MiB/s | 731.93 MiB/s | 729.00 MiB/s | 744.47 MiB/s | 730.01 MiB/s | 746.07 MiB/s | 744.70 MiB/s |\n| Read big file | 923.98 MiB/s | 892.99 MiB/s | 905.93 MiB/s | 895.88 MiB/s | 918.19 MiB/s | 939.63 MiB/s | 948.81 MiB/s |\n| Write small file | 95.20 files/s | 109.10 files/s | 82.30 files/s | 86.40 files/s | 101.20 files/s | 95.80 files/s | 94.60 files/s |\n| Read small file | 1242.80 files/s | 937.30 files/s | 752.40 files/s | 1857.90 files/s | 681.50 files/s | 1229.10 files/s | 1301.40 files/s |\n| Stat file | 12313.80 files/s | 11989.50 files/s | 3583.10 files/s | 7845.80 files/s | 4211.20 files/s | 2836.60 files/s | 3400.00 files/s |\n| FUSE operation | 0.41 ms/op | 0.40 ms/op | 0.46 ms/op | 0.44 ms/op | 0.41 ms/op | 0.41 ms/op | 0.44 ms/op |\n| Update meta | 2.45 ms/op | 1.76 ms/op | 2.46 ms/op | 1.78 ms/op | 3.76 ms/op | 3.40 ms/op | 2.87 ms/op |\n\n### mdtest\n\n- Shows rate (ops/sec). Bigger is better.\n\n| | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n|--------------------|--------------|----------------|----------|------------|-----------|----------|--------------|\n| **EMPTY FILES** | | | | | | | |\n| Directory creation | 4901.342 | 9990.029 | 1252.421 | 4091.934 | 4041.304 | 1910.768 | 3065.578 |\n| Directory stat | 289992.466 | 379692.576 | 9359.278 | 69384.097 | 49465.223 | 6500.178 | 17746.670 |\n| Directory removal | 5131.614 | 10356.293 | 902.077 | 1254.890 | 3210.518 | 1450.842 | 2460.604 |\n| File creation | 5472.628 | 9984.824 | 1326.613 | 4726.582 | 4053.610 | 1801.956 | 2908.526 |\n| File stat | 288951.216 | 253218.558 | 9135.571 | 233148.252 | 50432.658 | 6276.787 | 14939.411 |\n| File read | 64560.148 | 60861.397 | 8445.953 | 20013.027 | 18411.280 | 9094.627 | 11087.931 |\n| File removal | 6084.791 | 12221.083 | 1073.063 | 3961.855 | 3742.269 | 1648.734 | 2214.311 |\n| Tree creation | 80.121 | 83.546 | 34.420 | 61.937 | 77.875 | 56.299 | 74.982 |\n| Tree removal | 218.535 | 95.599 | 42.330 | 44.696 | 114.414 | 76.002 | 64.036 |\n| **SMALL FILES** | | | | | | | |\n| File creation | 295.067 | 312.182 | 275.588 | 289.627 | 307.121 | 275.578 | 263.487 |\n| File stat | 54069.827 | 52800.108 | 8760.709 | 19841.728 | 14076.214 | 8214.318 | 10009.670 |\n| File read | 62341.568 | 57998.398 | 4639.571 | 19244.678 | 23376.733 | 5477.754 | 6533.787 |\n| File removal | 5615.018 | 11573.415 | 1061.600 | 3907.740 | 3411.663 | 1024.421 | 1750.613 |\n| Tree creation | 57.860 | 57.080 | 23.723 | 52.621 | 44.590 | 19.998 | 11.243 |\n| Tree removal | 96.756 | 65.279 | 23.227 | 19.511 | 27.616 | 17.868 | 10.571 |\n\n### fio\n\n| | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n|-----------------|--------------|----------------|-----------|------------|-----------|-----------|--------------|\n| Write bandwidth | 729 MiB/s | 737 MiB/s | 736 MiB/s | 768 MiB/s | 731 MiB/s | 738 MiB/s | 745 MiB/s |\n" }, { "path": "docs/en/benchmark/performance_evaluation_guide.md", "content": "---\ntitle: Performance Evaluation Guide\nsidebar_position: 2\nslug: /performance_evaluation_guide\n---\n\nBefore starting performance testing, it is a good idea to write down a general description of usage scenario, including:\n\n1. What is the application for? For example, Apache Spark, PyTorch, or a program you developed yourself\n2. The requisite resource for running the application, including CPU, memory, network, and node size\n3. The estimated data size, including the number of files and their volume\n4. The file size and access mode (large or small files, sequential or random reads and writes)\n5. Performance requirements, such as the amount of data to be written or read per second, QPS, operation latency, etc.\n\nThe clearer and more detailed the above description is, the easier it will be to prepare a suitable test plan and find the performance indicators that need to be focused on. Clear plans and good performance indicators are helpful for evaluating the application requirements from various aspects of the storage system, including JuiceFS metadata configuration, network bandwidth requirements, configuration parameters, etc. It is surely not easy to have all details in mind at the beginning, and some of the content can be clarified gradually during the testing process. Still, **it is essential to make the usage scenario descriptions mentioned above and the corresponding test methods, test data, and test results complete at the end of a full test**.\n\nEven if the above is not yet clear, it does not matter. JuiceFS built-in test tools can get the core indicators of benchmark performance of the standalone machine just by a one-line command. This article also introduces two more JuiceFS built-in performance analysis tools, which provide a simple and clear way for more complex tests.\n\n## Performance Testing Quick Start\n\nAn example of the basic usage of the JuiceFS built-in `bench` tool is shown below.\n\n### Working Environment\n\n- Host: Amazon EC2 c5.xlarge one\n- OS: Ubuntu 20.04.1 LTS (Kernel `5.4.0-1029-aws`)\n- Metadata Engine: Redis 6.2.3, storage (dir) configured on system disk\n- Object Storage: Amazon S3\n- JuiceFS Version: 0.17-dev (2021-09-23 2ec2badf)\n\n### Attention\n\nJuiceFS v1.0+ has Trash enabled by default, which means the benchmark tools will create and delete temporary files in the file system. These files will eventually be dumped to the `.trash` folder which consumes storage space. To avoid this, you can disable the Trash before benchmarking by running `juicefs config META-URL --trash-days 0`. See [trash](../security/trash.md) for details.\n\n### `juicefs bench`\n\nThe [`juicefs bench`](../reference/command_reference.mdx#bench) command can help you do a quick performance test on a standalone machine. With the test results, it is easy to evaluate if your environment configuration and JuiceFS performance are normal. Assuming you have mounted JuiceFS to `/mnt/jfs` on your server, execute the following command for this test (the `-p` option is recommended to set to the number of CPU cores on the server). If you need help with initializing or mounting JuiceFS, please refer to [Create a File System](../getting-started/standalone.md#juicefs-format).\n\n```bash\njuicefs bench /mnt/jfs -p 4\n```\n\nThe test results are presented in a table format, where `ITEM` represents the tested item, `VALUE` represents the processing capacity per second (throughput, number of files, number of operations, etc.), and `COST` represents the time required for each file or operation.\n\nThe results will be displayed in green, yellow, or red to differentiate performance. If there are red indicators in your results, please check the relevant configurations first. Feel free to post any problems you encountered in detail on [GitHub Discussions](https://github.com/juicedata/juicefs/discussions).\n\n![bench](../images/bench-guide-bench.png)\n\nThe detailed `juicefs bench` performance test flows are shown below (The logic behind is very simple. Please take a look at the [source code](https://github.com/juicedata/juicefs/blob/main/cmd/bench.go) if you are interested).\n\n1. N concurrent `write`, each to a large file of 1 GiB with IO size of 1 MiB\n2. N concurrent `read`, each from the large file of 1 GiB previously written, with IO size of 1 MiB\n3. N concurrent `write`, each to 100 small files of 128 KiB, with IO size of 128 KiB\n4. N concurrent `read`, each from the 100 small files of 128 KiB previously written, with IO size of 128 KiB\n5. N concurrent `stat`, each on the 100 small files of 128 KiB previously written\n6. clean up the temporary directory for testing\n\nThe concurrency scale N could be provided through the `-p` option of the `bench` command.\n\nHere's a performance comparison using a few common storage types provided by AWS.\n\n- EFS with 1TiB capacity performs 150MiB/s of `read` and 50MiB/s of `write` at a cost of $0.08/GB-month.\n- EBS st1 is a throughput-optimized HDD with a maximum throughput of 500MiB/s, a maximum IOPS (1MiB I/O) of 500, and a maximum capacity of 16TiB, priced at $0.045/GB-month.\n- EBS gp2 is a universal SSD with a maximum throughput of 250MiB/s, maximum IOPS (16KiB I/O) of 16,000, and a maximum capacity of 16TiB, priced at $0.10/GB-month.\n\nThe above tests clearly show that JuiceFS performs much better than AWS EFS in terms of sequential read and write capabilities and than the commonly used EBS regarding throughput. However, the JuiceFS performance is not that outstanding when writing small files because each file written needs to be persisted to S3 and there is typically a fixed overhead of 10-30ms on calling the object storage API.\n\n:::note\nThe performance of Amazon EFS is linearly related to capacity ([refer to the official documentation](https://docs.aws.amazon.com/efs/latest/ug/performance.html#performancemodes)), which makes it unsuitable for being used in high throughput scenarios with small data sizes.\n:::\n\n:::note\nPrices refer to [AWS US East, Ohio Region](https://aws.amazon.com/ebs/pricing/?nc1=h_ls), differing slightly among regions.\n:::\n\n:::note\nThe data above is from [AWS official documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html), and the performance metrics are their maximum values. The actual performance of EBS is related to its volume capacity and instance type of mounted EC2. In general, the larger the volume and the higher the specification of EC2, the better the EBS performance will be, but not exceeding the maximum value mentioned above.\n:::\n\n### `juicefs objbench`\n\nThe [`juicefs objbench`](../reference/command_reference.mdx#objbench) command can run some tests on object storage to evaluate how well it performs as a backend storage for JuiceFS. Take testing Amazon S3 as an example:\n\n```bash\njuicefs objbench \\\n --storage s3 \\\n --access-key myAccessKey \\\n --secret-key mySecretKey \\\n https://mybucket.s3.us-east-2.amazonaws.com\n```\n\nThe test results are shown in the figure below:\n\n![JuiceFS Bench](../images/objbench.png)\n\nAmong them, the result `not support` indicates that the tested object storage does not support this feature.\n\n#### Test flow\n\nFirst perform object storage function test, the following are test cases:\n\n1. Create bucket\n2. Upload an object\n3. Download an object\n4. Download non-existent object\n5. Get object part content\n6. Get an object metadata\n7. Delete an object\n8. Delete non-existent object\n9. List objects\n10. Upload a large object\n11. Upload a empty object\n12. Multipart upload\n13. Change the owner/group of a file (requires `root` permission)\n14. Change permission\n15. Change mtime (last modified time)\n\nAnd then perform performance testing:\n\n1. Upload `--small-objects` objects of `--small-object-size` size with `--threads` concurrency\n2. Download the objects uploaded in step 1 and check the contents\n3. Split the `--big-object-size` object of size according to the size of `--block-size` and upload it concurrently with `--threads`\n4. Download the objects uploaded in step 3 and check the content, then clean up all objects uploaded to the object store in step 3\n5. List all objects in the object store 100 times with `--threads` concurrency\n6. Get meta information of all objects uploaded in step 1 with `--threads` concurrency\n7. Change mtime (last modified time) of all objects uploaded in step 1 by `--threads` concurrency\n8. Change permission of all objects uploaded in step 1 by `--threads` concurrency\n9. Change owner/group of all objects uploaded in step 1 by `--threads` concurrency (requires `root` permission)\n10. Remove all objects uploaded in step 1 with `--threads` concurrency\n\nFinally clean up the test files.\n\n## Performance Observation and Analysis Tools\n\nThe next two performance observation and analysis tools are essential tools for testing, using, and tuning JuiceFS.\n\n### `juicefs stats`\n\nThe [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) command is a tool for real-time statistics of JuiceFS performance metrics, similar to the `dstat` command on Linux systems. It can display changes of metrics for JuiceFS clients in real-time. For this, create a new session and execute the following command when the `juicefs bench` is running:\n\n```bash\njuicefs stats /mnt/jfs --verbosity 1\n```\n\nThe results are shown below, which would be easier to understand when combing with the `bench` performance test flows described above.\n\n![bench-guide-stats](../images/bench-guide-stats.png)\n\nLearn the meaning of indicators in [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats).\n\n### `juicefs profile`\n\nThe [`juicefs profile`](../administration/fault_diagnosis_and_analysis.md#profile) command is used to output all [access logs](../administration/fault_diagnosis_and_analysis.md#access-log) of the JuiceFS client in real time, including information about each request. It can also be used to play back and count JuiceFS access logs, and visualize the JuiceFS running status. To run the JuiceFS profile, execute the following command in another session while the `juicefs bench` command is running.\n\n```bash\ncat /mnt/jfs/.accesslog > juicefs.accesslog\n```\n\nThe `.accessslog` is a virtual file for JuiceFS access logs. It does not produce any data until it is read (e.g. by executing `cat`). Press Ctrl + C to terminate the `cat` command and run the following one.\n\n```bash\njuicefs profile juicefs.accesslog --interval 0\n```\n\nThe `---interval` parameter sets the sampling interval for accessing the log. 0 means quickly replay the log file to generate statistics, as shown in the following figure.\n\n![profile](../images/bench-guide-profile.png)\n\nBased on the bench performance test flows as described above, a total of `(1 + 100) * 4 = 404` files were created during this test, and each file went through the process of \"Create → Write → Close → Open → Read → Close → Delete\". So there are a total of:\n\n- 404 `create`, `open` and `unlink` requests\n- 808 `flush` requests: `flush` is automatically invoked whenever a file is closed\n- 33168 `write`/`read` requests: each large file takes 1024 1 MiB IOs on write, while the maximum size of a request at the FUSE level is 128 KiB by default. It means that each application IO is split into 8 FUSE requests, so there are `(1024 * 8 + 100) * 4 = 33168` requests. The read IOs work in a similar way, and so does its counting.\n\nAll these values correspond exactly to the results of `profile`. In addition, the test result shows that the average latency for the `write` operations is extremely low (45 μs). This is because JuiceFS `write` writes to a memory buffer first by default and then calls `flush` to upload data to the object storage when the file is closed, as expected.\n\n## Other Test Tool Configuration Examples\n\n:::tip\nJuiceFS v1.0+ has Trash enabled by default. The benchmark process will create and delete temporary files in the file system, and these files will eventually be dumped to the `.trash` folder which consumes storage space. To avoid this, you can disable Trash before benchmarking by running `juicefs config META-URL --trash-days 0`. See [trash](../security/trash.md) for details.\n:::\n\n### Fio Standalone Performance Test\n\nFio is a common performance testing tool that can be used to do more complex performance tests after completing the JuiceFS bench.\n\n#### Working Environment\n\nConsistent with the JuiceFS Bench test environment described above.\n\n#### Testing tasks\n\nPerform the following 4 Fio tasks for sequential write, sequential read, random write, and random read tests.\n\nSequential write\n\n```shell\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=write --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\nSequential read\n\n```bash\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=read --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\nRandom write\n\n```shell\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=randwrite --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\nRandom read\n\n```shell\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=randread --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\nOptions explanation:\n\n- `--name`: user-specified test name, which affects the test file name\n- `--directory`: test directory\n- `--ioengine`: the way to send IO when testing; usually `libaio` is used\n- `--rw`: commonly used options are read, write, randread and randwrite, which stand for sequential read/write and random read/write, respectively\n- `--bs`: the size of each IO\n- `--size`: the total size of IO per thread; usually equal to the size of the test file\n- `--numjobs`: number of concurrent test threads; each thread runs with an individual test file by default\n- `--direct`: add the `O_DIRECT` flag bit on opening a file to disable system buffering, which can make the test results more stable and accurate\n\nThe results are as follows:\n\n```bash\n# Sequential\nWRITE: bw=703MiB/s (737MB/s), 703MiB/s-703MiB/s (737MB/s-737MB/s), io=4096MiB (4295MB), run=5825-5825msec\nREAD: bw=817MiB/s (856MB/s), 817MiB/s-817MiB/s (856MB/s-856MB/s), io=4096MiB (4295MB), run=5015-5015msec\n\n# Random\nWRITE: bw=285MiB/s (298MB/s), 285MiB/s-285MiB/s (298MB/s-298MB/s), io=4096MiB (4295MB), run=14395-14395msec\nREAD: bw=93.6MiB/s (98.1MB/s), 93.6MiB/s-93.6MiB/s (98.1MB/s-98.1MB/s), io=4096MiB (4295MB), run=43773-43773msec\n```\n\n### Vdbench Multi-machine Performance Test\n\nVdbench is a commonly used file system evaluation tool, and supports multi-machine concurrent testing well.\n\n#### Working Environment\n\nSimilar to the JuiceFS Bench test environment, but with two more hosts (three in total) with the same hardware specifications.\n\n#### Preparation\n\nvdbench needs to be installed under the same path on each node:\n\n1. Download version 50406 from the [Official Website](https://www.oracle.com/downloads/server-storage/vdbench-downloads.html)\n2. Install Java: `apt-get install openjdk-8-jre`\n3. Verify that vdbench is installed successfully: `./vdbench -t`\n\nAssuming the names of the three nodes are `node0`, `node1` and `node2`, you need to create a configuration file on `node0` as follows (to test reading and writing a large number of small files):\n\n```bash\n$ cat jfs-test\nhd=default,vdbench=/root/vdbench50406,user=root\nhd=h0,system=node0\nhd=h1,system=node1\nhd=h2,system=node2\n\nfsd=fsd1,anchor=/mnt/jfs/vdbench,depth=1,width=100,files=3000,size=128k,shared=yes\n\nfwd=default,fsd=fsd1,operation=read,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd1,host=h0\nfwd=fwd2,host=h1\nfwd=fwd3,host=h2\n\nrd=rd1,fwd=fwd*,fwdrate=max,format=yes,elapsed=300,interval=1\n```\n\nParameters description:\n\n- `vdbench=/root/vdbench50406`: specifies the path where the vdbench tool is installed\n- `anchor=/mnt/jfs/vdbench`: specifies the path to run test tasks on each node\n- `depth=1,width=100,files=3000,size=128k`: defines the file tree structure of the test task, creating 100 more directories under the test directory, each contains 3000 files of 128 KiB, 300,000 files in total\n- `operation=read,xfersize=128k,fileio=random,fileselect=random`: defines the actual test task, which randomly selects files to send 128 KiB size read requests\n\nThe results are as follows:\n\n```\nFILE_CREATES Files created: 300,000 498/sec\nREAD_OPENS Files opened for read activity: 188,317 627/sec\n```\n\nThe overall rate of 128 KiB file creating is 498 (files/s), while file reading rate is 627.\n\n#### More References\n\nHere are some profiles available for simple local evaluation of file system performance. The specific test set size and number of concurrencies can be adjusted according to the actual situation.\n\n##### Sequential reading and writing of large files\n\nAll files are 1GiB in size, where `fwd1` is a large file for sequential writing, and `fwd2` is a large file for sequential reading.\n\n```bash\n$ cat local-big\nfsd=fsd1,anchor=/mnt/jfs/local-big,depth=1,width=1,files=4,size=1g,openflags=o_direct\n\nfwd=fwd1,fsd=fsd1,operation=write,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4\nfwd=fwd2,fsd=fsd1,operation=read,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4\n\nrd=rd1,fwd=fwd1,fwdrate=max,format=restart,elapsed=120,interval=1\nrd=rd2,fwd=fwd2,fwdrate=max,format=restart,elapsed=120,interval=1\n```\n\n##### Random reading and writing of small files\n\nAll files are 128KiB in size, where `fwd1` is a small file for random writing, `fwd2` is a small file for random reading, and `fwd3` is a small file for random mixed reading/writing (ratio read/write = 7:3).\n\n```bash\n$ cat local-small\nfsd=fsd1,anchor=/mnt/jfs/local-small,depth=1,width=20,files=2000,size=128k,openflags=o_direct\n\nfwd=fwd1,fsd=fsd1,operation=write,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd2,fsd=fsd1,operation=read,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd3,fsd=fsd1,rdpct=70,xfersize=128k,fileio=random,fileselect=random,threads=4\n\nrd=rd1,fwd=fwd1,fwdrate=max,format=restart,elapsed=120,interval=1\nrd=rd2,fwd=fwd2,fwdrate=max,format=restart,elapsed=120,interval=1\nrd=rd3,fwd=fwd3,fwdrate=max,format=restart,elapsed=120,interval=1\n```\n" }, { "path": "docs/en/community/_roadmap.md", "content": "---\ntitle: Roadmap\nsidebar_position: 3\n---\n" }, { "path": "docs/en/community/adopters.md", "content": "---\ntitle: Adopters\nsidebar_position: 1\nslug: /adopters\n---\n\n| Company/Team | Industry & Use Cases | User Story |\n|--------------|----------------------|------------|\n| [NAVER](https://www.naver.com) | Search engine, Training, Inference | [NAVER, Korea's No.1 Search Engine, Chose JuiceFS over Alluxio for AI Storage](https://juicefs.com/en/blog/user-stories/juicefs-vs-alluxio-ai-storage-naver) |\n| [Character.AI](https://character.ai) | GenAI, Training | |\n| [Fal](https://fal.ai) | GenAI, Inference | |\n| [BentoML](https://bentoml.com) | GenAI, Inference | [BentoML Reduced LLM Loading Time from 20+ to a Few Minutes with JuiceF](https://juicefs.com/en/blog/user-stories/accelerate-large-language-model-loading) |\n| [Lepton AI](https://www.lepton.ai) | GenAI, Training, Inference | [How Lepton AI Cut Cloud Storage Costs by 98% for AI Workflows with JuiceFS](https://juicefs.com/en/blog/user-stories/cloud-storage-artificial-intelligence-juicefs-vs-efs) |\n| [Graviti Diffus](https://www.diffus.graviti.com) | GenAI, Inference | |\n| [Plus.AI](https://plus.ai) | Autonomous driving, AI pipeline | |\n| [Jerry](https://getjerry.com) | Car insurance, Data platform | [Low-Cost Read/Write Separation: Jerry Builds a Primary-Replica ClickHouse Architecture](https://juicefs.com/en/blog/user-stories/read-write-separation) |\n| [DJI](https://www.dji.com) | Drone & Autonomous driving, AI pipeline| |\n| [Clobotics](https://clobotics.com) | Drone, AI pipeline | [How Clobotics Overcame Multi-Cloud and Massive File Storage Challenges](https://juicefs.com/en/blog/user-stories/multi-cloud-storage-posix-compatible) |\n| [TP-LINK](https://www.tp-link.com) | AI | |\n| [MemVerge](https://memverge.com) | BioTech, High performance file store | |\n| [MDI Biological Laboratory](https://mdibl.org) | BioTech, High performance file store | |\n| [Lawrence Berkeley Lab](https://www.lbl.gov) | BioTech, High performance file store | |\n| [American Museum of Natural History](https://www.amnh.org) | Non-profit, HPC, File sharing | |\n| [Argonne National Laboratory](https://www.anl.gov) | Non-profit | |\n| [Texas A&M Unversity](https://www.tamu.edu) | Education | |\n| [Simon Fraser University](https://www.sfu.ca) | Education | |\n| [University of Canberra](https://www.canberra.edu.au) | Education | |\n| [PITS Globale Datenrettungsdienste](https://www.pitsdatenrettung.de) | Data recovery service, File Sharing | |\n| [ExaLeap Semiconductor](https://exaleapsemi.com) | Semiconductor | |\n| [Cherry Digital](https://cherrydigital.com) | Digital media | |\n| [Shopee](https://shopee.com) | E-commerce, Data platform | [Shopee x JuiceFS: ClickHouse Cold and Hot Data Separation Storage Architecture and Practice](https://juicefs.com/en/blog/shopee-clickhouse-with-juicefs) |\n| [Zhihu](https://www.zhihu.com) | Internet service, Training, Inference | [How Zhihu Ensures Stable Storage for LLM Training in Multi-Cloud Architecture](https://juicefs.com/en/blog/user-stories/ai-storage-llm-training-multi-cloud) |\n| [Grab](https://grab.com/sg) | Data platform | |\n| [CCB Fintech](https://www.ccbft.com) | Fintech, AI, File sharing | |\n| [Pingan Bank](https://pingan.com) | Fintech, Data platform | |\n| [Tongdun](https://tongdun.cn) | Fintech, Data platform | |\n| [Yaoxin Financing Re-Guarantee](https://www.yaoxinhd.com) | Data platform, File sharing | |\n| [China Telecom](https://www.chinatelecomglobal.com) | Telecom, Data platform | [Scaling Hadoop on cloud: Managing PB-Level Data through Separation of Compute and Storage with JuiceFS](https://juicefs.com/en/blog/user-stories/applicatio-of-juicefs-in-china-telecoms-daily-average-pb-data-scenario) |\n| [China Mobile Cloud](https://ecloud.he.chinamobile.com) | Public cloud, Data platform | [Improving Apache HBase Performance on Cloud with JuiceFS](https://juicefs.com/en/blog/user-stories/juicefs-support-hbase-at-chinamobile-cloud) |\n| [Volcano Engine](https://www.volcengine.com) | Public cloud, File sharing, VFX rendering | [How JuiceFS Accelerates Edge Rendering Performance in Volcengine](https://juicefs.com/en/blog/user-stories/how-juicefs-accelerates-edge-rendering-performance-in-volcengine) |\n| [Kingsoft Cloud](https://en.ksyun.com) | Public cloud, Data platform | [Storing Elasticsearch Warm/Cold Data on Object Storage with JuiceFS: A Guide by Kingsoft Cloud](https://juicefs.com/en/blog/user-stories/kingsoft-cloud-how-to-store-elasticsearch-data-in-objective-storage-with-juicefs) |\n| [Piesat Information Technology Co., Ltd.](https://www.piesat.cn) | GIS, File sharing | |\n| [National Supercomputing Center in JiNan](https://www.nsccjn.cn) | HPC, AI | |\n| [Xiaomi](https://www.mi.com/global) | Consumer electronics, Training, Inference | [Xiaomi: Building a Cloud-Native File Storage Platform to Host 5B+ Files in AI Training & More](https://juicefs.com/en/blog/user-stories/cloud-native-file-storage-platform-ai-training) |\n| [vivo](https://www.vivo.com) | Consumer electronics, Training, Inference | |\n| [SF Express](https://www.sf-express.com) | Logistics, AI pipeline, File sharing | |\n| [Unisound](https://www.unisound.com) | AI, Training, Inference | [Unisound’s HPC Platform accelerates AI model training and development with JuiceFS](https://juicefs.com/en/blog/unisounds-hpc-platform-accelerates-ai-model-training-and-development-with-juicefs) |\n| [Yimian](https://www.yimian.io) | Consulting, Data platform | [Yimian Migrated Hadoop to the Cloud: 2x Storage Capacity & Fewer Ops Costs](https://juicefs.com/en/blog/user-stories/migrating-hadoop-to-cloud-2x-storage-capacity-fewer-ops-costs) |\n| [Trip.com](https://www.trip.com) | Internet service, Data platform, File sharing | [Trip.com’s practice of massive cold data migrating to object storage with JuiceFS](https://juicefs.com/en/blog/user-stories/a-practice-of-massive-cold-data-migrating-to-oss-with-juicefs), [JuiceFS at Trip.com: Managing 10 PB of Data for Stable and Cost-Effective LLM Storage](https://juicefs.com/en/blog/user-stories/large-language-model-artificial-intelligence-storage-cost-effective) |\n| [Beike](https://ke.com) | Internet service, AI pipline | [Beike Loads AI Models 20x Faster with Hybrid Cloud Storage](https://juicefs.com/en/blog/user-stories/ai-model-accelerate) |\n| [Baidu](https://ir.baidu.com/company-overview) | Internet Service | |\n| [Tongcheng Travel](https://www.tongchengir.com) | Internet service, File sharing | [Tongcheng Travel Chose JuiceFS over CephFS to Manage Hundreds of Millions of Files](https://juicefs.com/en/blog/user-stories/juicefs-vs-cephfs-distributed-file-system-artificial-intelligence-storage) |\n| [Skyplatanus](https://www.kuaidianyuedu.com) | Internet service, AI, File sharing | |\n| [NetEase Games](https://www.neteasegames.com) | Gaming, Data platform, File sharing | [50%+ Cut in Both Storage & Compute Costs: Designing NetEase Games' Cloud Big Data Platform](https://juicefs.com/en/blog/user-stories/cut-storage-compute-costs-cloud-big-data-platform) |\n| [Joyient](http://www.joyient.com) | Gaming, File sharing, VFX rendering | |\n| [CVTE](http://www.cvte.com/en) | Education, File sharing | |\n| [Ricequant](https://www.ricequant.com) | Quantitative trading, AI, File sharing | |\n| [Dmall](https://www.dmall.com/en) | SaaS, Data platform, File sharing | [Why DMALL Switched to a Big Data Storage-Compute Decoupled Architecture](https://juicefs.com/en/blog/user-stories/storage-compute-decoupled-architecture-cloud-native-big-data) |\n| [Horizon Robotics](https://horizon.ai) | Autonomous driving, AI pipeline | |\n| [Li Auto](https://www.lixiang.com/en) | Automotive, Big Data, AI | [Migrating on-Premises Hadoop to Cloud with JuiceFS: A Case Study from Li Auto](https://juicefs.com/en/blog/user-stories/li-autos-practice-of-migrating-data-from-hdfs-to-juicefs) |\n| [NIO Auto](https://www.nio.com) | Automotive, AI, File sharing | |\n| [SAIC Motor](https://www.saicmotor.com/english) | Automotive, AI | |\n| [Wuling Auto](https://wuling.com) | Automotive, Data platform | |\n| [coSence](https://www.coscene.io) | Robotics, AI pipeline | [coScene Chose JuiceFS over Alluxio to Tackle Object Storage Drawbacks](https://juicefs.com/en/blog/user-stories/juicefs-vs-alluxio-ai-robot-storage) |\n| [DP Technology](https://www.dp.tech) | BioTech, AI pipeline | [AI & HPC Workloads on Hybrid Cloud: Storage Challenges and Solutions](https://juicefs.com/en/blog/user-stories/storage-architectures-for-ai-hpc-in-hybridmulti-cloud) |\n| [Gene Way](https://www.geneway.cn) | BioTech, File sharing | |\n| [CoCalc](https://doc.cocalc.com/cloud_file_system.html) | Data Science, AI, Education, Cloud GPU's | |\n\nYou are welcome to share your experience after using JuiceFS, either by submitting a Pull Request directly to this list, or by contacting us at [`hello@juicedata.io`](mailto:hello@juicedata.io).\n" }, { "path": "docs/en/community/articles.md", "content": "---\ntitle: JuiceFS Article Collection\nsidebar_position: 2\nslug: /articles\ndescription: Explore JuiceFS' collection of technical articles and real-world case studies in AI, machine learning, deep learning, big data, data sharing, backup, and recovery scenarios.\n---\n\nJuiceFS is widely applicable to various data storage and sharing scenarios. This page compiles its technical articles and real-world case studies. Explore valuable insights and practical examples to deepen your understanding of JuiceFS and related applications. We encourage all community users to contribute and maintain this list.\n\n## Articles sorted in categories\n\n### AI, machine learning, and deep learning\n\n- [How D-Robotics Manages Massive Small Files in a Multi-Cloud Environment with JuiceFS](https://juicefs.com/en/blog/user-stories/multi-cloud-store-massive-small-files), 2026-03-05, Han Zhao @ D-Robotics\n- [From GlusterFS to JuiceFS: Lightillusions Achieved 2.5x Faster 3D AIGC Data Processing](https://juicefs.com/en/blog/user-stories/aigc-storage-glusterfs-cephfs-vs-juicefs), 2026-01-08, Weiyu Li @ Lightillusions\n- [AI Data Storage: Challenges, Capabilities, and Comparative Analysis](https://juicefs.com/en/blog/solutions/ai-data-storage-challenges-capabilities-solution-comparison), 2025-12-18, Rui Su\n- [JuiceFS+MinIO: Ariste AI Achieved 3x Faster I/O and Cut Storage Costs by 40%+](https://juicefs.com/en/blog/user-stories/quantitative-storage-artificial-intelligence-solution), 2025-12-11, Yutang Gao @ Ariste AI\n- [NAS vs. Object Storage vs. JuiceFS: Storage Selection of Billion-Dollar Quantitative Firms](https://juicefs.com/en/blog/solutions/quant-research-storage-selection-nas-object-storage-juicefs), 2025-11-27, Jerry Cai\n- [Building AI Inference with JuiceFS: Supporting Multi-Modal Complex I/O, Cross-Cloud, and Multi-Tenancy](https://juicefs.com/en/blog/solutions/ai-inference-multi-cloud-storage-multi-tenancy), 2025-10-23, Shaojie Li\n- [Zelos Tech Manages Hundreds of Millions of Files for Autonomous Driving with JuiceFS](https://juicefs.com/en/blog/user-stories/multi-cloud-storage-autonomous-driving), 2025-10-09, Junyu Deng @ Zelos Tech\n- [Why Gaoding Technology Chose JuiceFS for AI Storage in a Multi-Cloud Architecture](https://juicefs.com/en/blog/user-stories/multi-cloud-storage-artificial-intelligence-training), 2025-09-03, Jia Ke @ Gaoding Technology\n- [StepFun Built an Efficient and Cost-Effective LLM Storage Platform with JuiceFS](https://juicefs.com/en/blog/user-stories/artificial-intelligence-storage-large-language-model-multimodal), 2025-07-31, Changxin Miao @ StepFun\n- [INTSIG Built Unified Storage Based on JuiceFS to Support Petabyte-Scale AI Training](https://juicefs.com/en/blog/user-stories/artificial-intelligence-model-training-unified-storage-solution), 2025-07-24, Yifan Tang @ INTSIG\n- [vivo Migrated from GlusterFS to a Distributed File System Built on JuiceFS](https://juicefs.com/en/blog/user-stories/glusterfs-vs-juicefs-ai-computing), 2025-07-17, Xiangyang Yu @ vivo\n- [NFS to JuiceFS: Building a Scalable Storage Platform for LLM Training & Inference](https://juicefs.com/en/blog/user-stories/ai-storage-platform-large-language-model-training-inference), 2025-06-11, Wei Sun\n- [BioMap Cut AI Model Storage Costs by 90% Using JuiceFS](https://juicefs.com/en/blog/user-stories/ai-storage-life-sciences-solution-juicefs-vs-lustre-alluxio), 2025-05-15, Zedong​​ ​​Zheng @ BioMap\n- [JuiceFS at Trip.com: Managing 10 PB of Data for Stable and Cost-Effective LLM Storage](https://juicefs.com/en/blog/user-stories/large-language-model-artificial-intelligence-storage-cost-effective), 2025-03-13, Songlin Wu @ Trip.com\n- [How Lepton AI Cut Cloud Storage Costs by 98% for AI Workflows with JuiceFS](https://juicefs.com/en/blog/user-stories/cloud-storage-artificial-intelligence-juicefs-vs-efs), 2025-02-07, Cong Ding @ Lepton AI\n- [Tongcheng Travel Chose JuiceFS over CephFS to Manage Hundreds of Millions of Files](https://juicefs.com/en/blog/user-stories/juicefs-vs-cephfs-distributed-file-system-artificial-intelligence-storage), 2025-01-08, Chuanhai Wei @ Tongcheng Travel\n- [LLM Storage Selection & Detailed Performance Analysis of JuiceFS](https://juicefs.com/en/blog/solutions/llm-storage-selection), 2024-10-23, Shaojie Li\n- [MiniMax Built a Cost-Effective, High-Performance AI Platform with JuiceFS](https://juicefs.com/en/blog/user-stories/minimax-foundation-model-ai-storage), 2024-09-02\n- [How JuiceFS Boosts Foundation Model Inference in Multi-Cloud Architectures](https://juicefs.com/en/blog/solutions/boost-foundation-model-inference-multi-cloud), 2024-08-29, Changjian Gao\n- [Enhancing AI Training Workflows with JuiceFS](https://juicefs.com/en/blog/solutions/enhance-ai-training-workflow), 2024-08-27\n- [vivo Migrated from GlusterFS to a Distributed File System for AI Training](https://juicefs.com/en/blog/user-stories/improve-ai-training), 2024-07-18, Yige Peng @ vivo\n- [iSEE Lab Stores 500M+ Files on JuiceFS Replacing NFS](https://juicefs.com/en/blog/user-stories/deep-learning-ai-storage), 2024-07-03, Guohao Xu @ Sun Yat-sen University\n- [Beike Loads AI Models 20x Faster with Hybrid Cloud Storage](https://juicefs.com/en/blog/user-stories/ai-model-accelerate), 2024-06-26, Tianqing Wang @ Beike\n- [Low-Cost Read/Write Separation: Jerry Builds a Primary-Replica ClickHouse Architecture](https://juicefs.com/en/blog/user-stories/read-write-separation), 2024-05-29, Tao Ma @ Jerry\n- [LLM Storage: Performance, Cost, and Multi-Cloud Architecture](https://juicefs.com/en/blog/solutions/llm-storage-performance-cost-multi-cloud), 2024-04-09, Sui Su\n- [How Zhihu Ensures Stable Storage for LLM Training in Multi-Cloud Architecture](https://juicefs.com/en/blog/user-stories/ai-storage-llm-training-multi-cloud), 2024-04-03, Xin Wang @ Zhihu\n- [BentoML Reduced LLM Loading Time from 20+ to a Few Minutes with JuiceFS](https://juicefs.com/en/blog/user-stories/accelerate-large-language-model-loading), 2024-02-29, Xipeng Guan @ BentoML\n- [coScene Chose JuiceFS over Alluxio to Tackle Object Storage Drawbacks](https://juicefs.com/en/blog/user-stories/juicefs-vs-alluxio-ai-robot-storage), 2024-01-24, Juchao Song @ coScene\n- [NAVER, Korea's No.1 Search Engine, Chose JuiceFS over Alluxio for AI Storage](https://juicefs.com/en/blog/user-stories/juicefs-vs-alluxio-ai-storage-naver), 2024-01-17, Nam Kyung-wan @ NAVER\n- [Building an Easy-to-Operate AI Training Platform: SmartMore's Storage Selection & Best Practices](https://juicefs.com/en/blog/user-stories/ai-training-storage-selection-seaweedfs-juicefs), 2023-12-14, Jichuan Sun @ SmartMore\n- [A Leading Self-Driving Company Chose JuiceFS over Amazon S3 and Alluxio in the Multi-Cloud Architecture](https://juicefs.com/en/blog/user-stories/data-storage-multi-cloud-autonomous-driving-juicefs), 2023-11-09\n- [Choosing JuiceFS over s3fs and Alluxio for Our Ultra-Heterogeneous Computing Cluster](https://juicefs.com/en/blog/user-stories/high-performance-scale-out-heterogeneous-computing-power-cluster-storage), 2023-06-09, Chen Hong @ Zhejiang Lab\n- [Achieving Elastic Throughput in the Cloud with a Distributed File System to Boost AI Training](https://juicefs.com/en/blog/solutions/accelerate-ai-training-flexible-elastic-throughput-cloud), 2023-05-06, Sui Su\n- [Improving Read Performance by ~30% in AI Speech and Text Processing by a Distributed Storage System](https://juicefs.com/en/blog/user-stories/unisounds-hpc-platform-accelerates-ai-model-training-and-development-with-juicefs), 2022-09-06, Dongdong Lv @ Unisound\n\n### Big data\n\n- [From Object Storage to K8s+JuiceFS: 85% Storage Cost Cut, HDFS-Level Performance](https://juicefs.com/en/blog/user-stories/object-storage-kubernetes-hdfs), 2024-02-07, Experienced JuiceFS user\n- [From Hadoop to Cloud: Why and How to Decouple Storage and Compute in Big Data Platforms](https://juicefs.com/en/blog/solutions/hadoop-cloud-decouple-storage-compute-big-data), 2023-11-01\n- [Costs Cut & Ops Efficiency Boosted: Switching to a Big Data Storage-Compute Decoupled Architecture](https://juicefs.com/en/blog/user-stories/storage-compute-decoupled-architecture-cloud-native-big-data), 2023-09-28, Ming Li @ DMALL\n- [50%+ Cut in Both Storage & Compute Costs: Designing NetEase Games' Cloud Big Data Platform](https://juicefs.com/en/blog/user-stories/cut-storage-compute-costs-cloud-big-data-platform), 2023-09-14, Weihong Ke @ NetEase Games\n- [Migrating Hadoop to the Cloud: 2x Storage Capacity & Fewer Ops Costs](https://juicefs.com/en/blog/user-stories/migrating-hadoop-to-cloud-2x-storage-capacity-fewer-ops-costs), 2023-08-09, Chang Liu & Yangliang Li @ Yimian\n- [Gaoding Technology Saves 60% Of Storage Cost Used By Elasticsearch](https://juicefs.com/en/blog/user-stories/gaoding-with-juicefs), 2021-10-09, Gaoding SRE Team\n- [Shopee x JuiceFS: ClickHouse Cold and Hot Data Separation Storage Architecture and Practice](https://juicefs.com/en/blog/user-stories/shopee-clickhouse-with-juicefs), 2021-10-09, Teng @ Shopee\n- [How to effectively reduce the load of HDFS cluster for Qutoutiao(NASDAQ:QTT)](https://juicefs.com/blog/en/posts/qutoutiao-big-data-platform-user-case)\n- [How does the Globalegrow data platform achieve both speed and money savings?](https://juicefs.com/blog/en/posts/globalegrow-big-data-platform-user-case)\n- [How to make HBase faster, more stable, and cheaper](https://juicefs.com/blog/en/posts/how-to-make-hbase-faster-more-stable-and-cheaper)\n- [Exploring storage and computing separation for ClickHouse](https://juicefs.com/blog/en/posts/clickhouse-disaggregated-storage-and-compute-practice)\n\n### Cloud-native & Kubernetes\n\n- [Hai Robotics Achieved High Availability & Easy Operations in a Hybrid Cloud Architecture with JuiceFS](https://juicefs.com/en/blog/user-stories/high-availability-easy-operations-hybrid-cloud-ai-storage), 2024-11-27, Sendong Wu @ Hai Robotics\n- [TAL: Building a Low-Operation Model Repository Based on JuiceFS in a Multi-Cloud Environment](https://juicefs.com/en/blog/user-stories/multi-cloud-llm-model-repository-storage), 2024-11-21, Longhua He @ TAL\n- [Training LLMs: Best Practices for Storing Thousands of Nodes in K8s](https://juicefs.com/en/blog/usage-tips/train-large-language-model-kubernetes-storage), 2024-10-09, Weiwei Zhu\n- [How Clobotics Overcame Multi-Cloud and Massive File Storage Challenges](https://juicefs.com/en/blog/user-stories/multi-cloud-storage-posix-compatible), 2024-09-11, Jonnas @ Clobotics\n- [K8s Data Persistence: Getting Started with JuiceFS CSI Driver](https://juicefs.com/en/blog/usage-tips/kubernetes-data-persistence-juicefs), 2023-12-28, Herald Yu\n- [Building a Cloud-Native File Storage Platform to Host 5B+ Files in AI Training & More](https://juicefs.com/en/blog/user-stories/cloud-native-file-storage-platform-ai-training), 2023-10-12, Jiapeng Sun @ Xiaomi\n- [An Elastic Platform & Simplified Storage Achieved by Migrating to Spark+K8s+JuiceFS](https://juicefs.com/en/blog/user-stories/scalable-computing-unified-data-storage-ops-cloud-spark-k8s-juicefs), 2023-05-10, Fengyu Cao @ Douban\n\n### Data sharing\n\n- [Conda + JuiceFS: Enhancing AI Development Environment Sharing](https://juicefs.com/en/blog/usage-tips/improve-artificial-intelligence-development-environment-sharing), 2024-12-18, Herald Yu\n- [Hugging Face + JuiceFS: Simplifying Model Sharing Across Multiple Users and Nodes](https://juicefs.com/en/blog/usage-tips/ai-model-storage-share-multi-users-nodes), 2024-10-17, Herald Yu\n- [Ollama + JuiceFS: Pull Once, Run Anywhere](https://juicefs.com/en/blog/usage-tips/ollama-large-language-model), 2024-09-25, Weiwei Zhu\n- [Building a Milvus Cluster Based on JuiceFS](https://juicefs.com/blog/en/posts/build-milvus-distributed-cluster-based-on-juicefs)\n\n### Data backup and recovery\n\n- [How JuiceFS 1.3 Backs Up 100 Million Files in Just Minutes](https://juicefs.com/en/blog/release-notes/juicefs-1-3-binary-backup), 2025-05-29, Jiefeng Huang\n- [Trip.com’s practice of massive cold data migrating to object storage with JuiceFS](https://juicefs.com/en/blog/user-stories/a-practice-of-massive-cold-data-migrating-to-oss-with-juicefs), 2022-09-19, Miaocheng & Xiaofeng @ Trip.com\n- [JuiceFS for archive NGINX logs](https://juicefs.com/docs/en/archive_nginx_log_in_juicefs.html)\n- [JuiceFS for MySQL backup, verification and recovery](https://juicefs.com/docs/en/backup_mysql_in_juicefs.html)\n- [Customer Stories: Xiachufang MySQL backup practice on JuiceFS](https://juicefs.com/blog/en/posts/xiachufang-mysql-backup-practice-on-juicefs)\n\n### Engineering insights\n\n- [The Design Journey of FUSE: From Kernel-Space to User-Space File Systems](https://juicefs.com/en/blog/engineering/design-fuse-kernel-user-space), 2026-02-14, Yuchao Xu\n- [Design and Performance Optimization of juice sync for Enterprise Data Synchronization](https://juicefs.com/en/blog/engineering/design-performance-optimization-juice-sync), 2025-12-08, Jian Zhi\n- [Deep Dive into the JuiceFS Garbage Collection Mechanism](https://juicefs.com/en/blog/engineering/juicefs-garbage-collection), 2025-11-06, Yuchao Xu\n- [MLPerf Storage v2.0: JuiceFS Leads in Bandwidth Utilization and Scalability for AI Training](https://juicefs.com/en/blog/engineering/mlperf-storage-v2-ai-training-storage-performance), 2025-09-25, Feihu Mo\n- [Achieving TB-Level Aggregate Bandwidth: How JuiceFS Optimized Distributed Cache Network](https://juicefs.com/en/blog/engineering/terabyte-aggregate-bandwidth-distributed-cache-network), 2025-09-18, Feihu Mo\n- [JuiceFS on Windows: Challenges in the Beta Release](https://juicefs.com/en/blog/engineering/optimize-juicefs-on-windows), 2025-08-20, Ethan Chen\n- [Deep Dive into JuiceFS Permission Management: Full Compatibility with Linux Security Mechanisms](https://juicefs.com/en/blog/engineering/linux-file-system-juicefs-access-management), 2025-06-26, Jiefeng Huang\n- [Code-Level Analysis: Design Principles of JuiceFS Metadata and Data Storage](https://juicefs.com/en/blog/engineering/design-metadata-data-storage), 2024-12-12, Arthur\n- [Deep Dive into JuiceFS Data Synchronization and Consistency in Multi-Cloud Architectures](https://juicefs.com/en/blog/engineering/data-synchronization-consistency-multi-cloud-storage), 2024-11-06\n- [Optimizing JuiceFS Read Performance: Readahead, Prefetch, and Cache](https://juicefs.com/en/blog/engineering/optimize-read-performance), 2024-08-06, Feihu Mo\n- [Smooth Upgrade: Implementation and Usage](https://juicefs.com/en/blog/engineering/smooth-upgrade), 2024-05-08, Jian Zhi\n- [How We Optimized ACL Implementation for Minimal Performance Impact](https://juicefs.com/en/blog/engineering/access-control-list), 2024-04-30, Jiefeng Huang\n- [98% GPU Utilization Achieved in 1k GPU-Scale AI Training Using Distributed Cache](https://juicefs.com/en/blog/engineering/ai-gpu-utilization-mlperf-benchmark), 2024-03-07, Feihu Mo\n- [How a Distributed File System in Go Reduced Memory Usage by 90%](https://juicefs.com/en/blog/engineering/reduce-metadata-memory-usage), 2024-02-22, Sandy\n- [How We Achieved a 40x Performance Boost in Metadata Backup and Recovery](https://juicefs.com/en/blog/engineering/increase-performance-metadata-backup-recovery), 2023-12-20, Jian Zhi\n- [A Deep Dive into the Design of Directory Quotas in JuiceFS](https://juicefs.com/en/blog/engineering/design-juicefs-directory-quotas), 2023-10-26, Sandy\n\n### Tutorial, guide, and best practice\n\n- [JuiceFS Enterprise 5.3: 500B+ Files per File System & RDMA Support](https://juicefs.com/en/blog/release-notes/juicefs-enterprise-5-3-rdma-support), 2026-02-04, Sandy\n- [How Just Two Cache Nodes Achieved 1.45 TB/s Throughput](https://juicefs.com/en/blog/solutions/cache-nodes-support-high-throughput), 2026-01-29, Jerry Cai\n- [JuiceFS Writeback: The Write Acceleration Mechanism and Its Applicable Scenarios](https://juicefs.com/en/blog/solutions/juicefs-write-acceleration), 2025-09-11, Jerry Cai\n- [JuiceFS Community 1.3: Python SDK, Faster Backup, SQL & Windows Optimizations](https://juicefs.com/en/blog/release-notes/juicefs-1-3-python-sdk-backup-sql-windows-optimization), 2025-07-09\n- [JuiceFS 1.3 Beta 2 Integrates Apache Ranger for Fine-Grained Access Control](https://juicefs.com/en/blog/release-notes/juicefs-1-3-integrates-apache-ranger-access-control), 2025-06-18, Youpeng Tang\n- [JuiceFS Enterprise Edition 5.2: Supporting Hundreds of Billions of Files and Windows Clients](https://juicefs.com/en/blog/release-notes/juicefs-5-2-windows-client), 2025-06-05\n- [​​JuiceFS 1.3 Beta: Enhanced Support for SQL Databases, a New Option for Billion-Scale Metadata Management](https://juicefs.com/en/blog/release-notes/juicefs-1-3-support-sql-database), 2025-04-28, Fangxin Lou\n- [Automated Cache Management: JuiceFS Enterprise Edition Introduces Cache Group Operator](https://juicefs.com/en/blog/usage-tips/automated-cache-management-cache-group-operator), 2025-01-16, Xuhui Zhang\n- [Database Release and End-to-End Testing: Bringing Modern Software Development Best Practices to the Data World](https://juicefs.com/en/blog/user-stories/end-to-end-test-clickhouse-database-clone), 2024-12-04, Tao Ma @ Jerry\n- [JuiceFS CSI: Smooth Upgrades of Mount Pods and Implementation Details](https://juicefs.com/en/blog/usage-tips/mount-pod-smooth-upgrade), 2024-11-13, Weiwei Zhu\n- [Getting Started with the JuiceFS Python SDK](https://juicefs.com/en/blog/usage-tips/use-python-sdk), 2024-10-30, Herald Yu\n- [JuiceFS CSI Workflow: K8s Pod Creation with PVs](https://juicefs.com/en/blog/usage-tips/csi-workflow-kubernetes-pod), 2024-09-30, Arthur\n- [JuiceFS Enterprise 5.1: Write Support for Mirrors, Python SDK, and AI Enhancements](https://juicefs.com/en/blog/release-notes/uicefs-enterprise-5-1-artificial-intelligence), 2024-09-19\n- [How to Check If a Database or Object Storage Is Used by JuiceFS](https://juicefs.com/en/blog/usage-tips/check-database-object-storage-in-use), 2024-08-22, Herald Yu\n- [Metabit Trading Built a Cloud-Based Quantitative Research Platform with JuiceFS](https://juicefs.com/en/blog/user-stories/build-cloud-quantitative-platform-posix-compatible-storage), 2024-08-14, Jianhong Li @ Metabit Trading\n- [Empowering NAS for AI Training with JuiceFS Direct-Mode NFS](https://juicefs.com/en/blog/usage-tips/direct-nfs)，2024-07-25，Herald Yu\n- [How to Deploy SeaweedFS+TiKV for Using JuiceFS](https://juicefs.com/en/blog/usage-tips/seaweedfs-tikv), 2024-07-11, Jinhao Yang @ SmartMore\n- [JuiceFS 1.2: Introducing Enterprise-Grade Permission Management and Smooth Upgrades](https://juicefs.com/en/blog/release-notes/juicefs-12), 2024-06-20\n- [JuiceFS S3 Gateway: IAM and Bucket Event Notifications](https://juicefs.com/en/blog/usage-tips/s3-gateway), 2024-06-13, Herald Yu\n- [Managing POSIX ACL Permissions in JuiceFS](https://juicefs.com/en/blog/usage-tips/manage-acl), 2024-06-06, Herald Yu\n- [Data Sync in JuiceFS 1.2: Enhanced Selective Sync and Performance Optimizations](https://juicefs.com/en/blog/usage-tips/data-sync), 2024-05-16, Jian Zhi\n- [JuiceFS 1.2: Gateway Upgrade, Enhanced Multi-User Permission Management](https://juicefs.com/en/blog/release-notes/juicefs-12-beta-1), 2024-04-22, Jian Zhi\n- [How to Monitor the JuiceFS File System with Grafana Cloud](https://juicefs.com/en/blog/usage-tips/monitor-file-system-grafana-cloud), 2024-04-18, Herald Yu\n- [How to Persist Data in Google Colab Using JuiceFS](https://juicefs.com/en/blog/usage-tips/colab-persist-data), 2024-03-27, Jet\n- [How to Build a Ceph Cluster and Integrate with the JuiceFS File System](https://juicefs.com/en/blog/usage-tips/build-ceph-cluster-integrate-juicefs-file-system), 2023-12-07, Yifu Liu\n- [6 Essential Tips for JuiceFS Users](https://juicefs.com/en/blog/usage-tips/juicefs-user-tips-distributed-file-storage-system), 2023-11-23, Herald Yu\n- [What's New in JuiceFS Enterprise Edition 5.0](https://juicefs.com/en/blog/release-notes/juicefs-enterprise-edition-v5), 2023-11-20\n- [Configuring Samba and NFS on JuiceFS to Unlock Unlimited Cloud Storage](https://juicefs.com/en/blog/usage-tips/scalable-cloud-storage-samba-nfs-shares-juicefs), 2023-08-29, Herald Yu\n- [How to Store and Share AI Models for Stable Diffusion in the Cloud](https://juicefs.com/en/blog/usage-tips/share-store-model-data-stable-diffusion-cloud), 2023-07-19, Herald Yu\n- [JuiceFS Enterprise Edition: Architecture, Features, and Community Edition Comparison](https://juicefs.com/en/blog/solutions/juicefs-enterprise-edition-features-vs-community-edition), 2023-06-06, Changjian Gao\n- [How to Boost AI Model Training with a Distributed Storage System](https://juicefs.com/en/blog/usage-tips/how-to-use-juicefs-to-speed-up-ai-model-training), 2023-04-25, Changjian Gao\n- [How To Use JuiceFS To Store Data On DigitalOcean](https://www.youtube.com/watch?v=pdFzyflcRGA&t=75s), Youtube video, by Education Ecosystem\n- [Guidance on selecting metadata engine in JuiceFS](https://juicefs.com/en/blog/usage-tips/juicefs-metadata-engine-selection-guide), 2022-10-14, Sandy\n- [The strengths and weaknesses of using Redis as the JuiceFS metadata engine](https://juicefs.com/en/blog/usage-tips/introduce-redis-as-juicefs-metadata-engine), 2022-07-22, Changjian Gao\n- [How JuiceFS uses Redis as a Metastore](https://www.youtube.com/watch?v=P7H1H-Zj5oU&t=757s) on Redis Monthly Live with Davies Liu and Mikhail Volkov, YouTube video\n- [Tutorial, how to use JuiceFS with Cloudflare R2](https://github.com/centminmod/centminmod-juicefs), George Liu (eva2000)\n- [JuiceFS Source Code Analysis](https://github.com/dollarkillerx/juicefs-source-analysis), Dollarkillerx\n\n### Others\n\n- [3,000 Concurrent Renders: The JuiceFS Client for Windows Averages 22m 22s](https://juicefs.com/en/blog/solutions/juicefs-windows-performance-test), 2025-08-28, Jerry Cai\n- [LanceDB Query Performance: NVMe vs. EBS vs. JuiceFS vs. EFS vs. FSx for Lustre](https://juicefs.com/en/blog/solutions/lancedb-query-performance-benchmark-storage-solutions), 2025-08-13, Brent Bai\n- [How JuiceFS Transformed Idle Resources into a 70 GB/s Cache Pool](https://juicefs.com/en/blog/solutions/idle-resources-elastic-high-throughput-storage-cache-pool), 2025-08-07, Jerry Cai\n- [Lustre vs. JuiceFS: A Comparative Analysis of Architecture, File Distribution, and Features](https://juicefs.com/en/blog/engineering/lustre-vs-juicefs-architecture-file-distribution-feature), 2025-07-02, Qing Liu\n- [Introducing JuiceFS Python SDK: 3x Faster than FUSE for Data Loading](https://juicefs.com/en/blog/release-notes/juicefs-1-3-python-sdk), 2025-05-22, Feihu Mo\n- [DeepSeek 3FS vs. JuiceFS: Architectures, Features, and Innovations in AI Storage](https://juicefs.com/en/blog/engineering/deepseek-3fs-vs-juicefs-architecture-feature), 2025-04-02, Qing Liu\n- [How JuiceFS Achieves Consistency and Low-Latency Data Distribution in Multi-Cloud Architectures](https://juicefs.com/en/blog/solutions/consistency-low-latency-data-distribution-multi-cloud-storage), 2025-01-22, Jerry Cai\n- [JuiceFS Evaluation with AWS EFS and FSx for Lustre](https://juicefs.com/en/blog/engineering/juicefs-vs-efs-fsx-for-lustre), 2024-08-07, Brent Bai\n- [MemVerge Chose JuiceFS: Small File Writes 5x Faster than s3fs](https://juicefs.com/en/blog/user-stories/vs-s3fs-memverge), 2024-07-31, Jon Jiang @ MemVerge\n- [From HPC to AI: Evolution and Performance Evaluation of File Systems](https://juicefs.com/en/blog/user-stories/hpc-ai-file-system), 2024-05-23, Weizheng Lu @ Renmin University of China\n- [Is POSIX Really Unsuitable for Object Stores? A Data-Backed Answer](https://juicefs.com/en/blog/community/posix-object-store-suitable-file-system), 2023-11-16, Herald Yu\n- [Comparative Analysis of Major Distributed File System Architectures: GFS vs. Tectonic vs. JuiceFS](https://juicefs.com/en/blog/engineering/compare-distributed-file-system-architectures-gfs-tectonic-juicefs), 2023-10-20, Changjian Gao\n- [JuiceFS vs. SeaweedFS](https://juicefs.com/docs/community/comparison/juicefs_vs_seaweedfs), 2023-09-31, Yifu Liu\n- [GlusterFS vs. JuiceFS](https://juicefs.com/en/blog/engineering/glusterfs-vs-juicefs-distributed-storage), 2023-09-21, Sandy\n\n## Contribution\n\nIf you want to add JuiceFS application cases to this list, you can do so through the following methods:\n\n### GitHub contribution\n\nFeel free to contribute by creating a branch in this repository on GitHub. Add the title and URL of your case page to the appropriate category, and then submit a pull request for review. Our team will review the submission and merge the branch if approved.\n\n### Social media\n\nYou can join the official JuiceFS [Slack channel](https://go.juicefs.com/slack). There, you can get in touch with any staff member to discuss your contribution.\n" }, { "path": "docs/en/community/integrations.md", "content": "---\nsidebar_label: Integrations\nsidebar_position: 2\nslug: /integrations\n---\n\n# Community Integrations\n\n## SDK\n\n- [Megvii](https://en.megvii.com) team contributed [Python SDK](https://github.com/megvii-research/juicefs-python).\n\n## AI\n\n- [UniSound](https://www.unisound.com) team participated in the development of [Fluid](https://github.com/fluid-cloudnative/fluid) JuiceFSRuntime cache engine, please refer to [this document](https://github.com/fluid-cloudnative/fluid/blob/master/docs/en/samples/juicefs_runtime.md).\n- [PaddlePaddle](https://github.com/paddlepaddle/paddle) team has integrated JuiceFS into [Paddle Operator](https://github.com/PaddleFlow/paddle-operator), please refer to [the document](https://github.com/PaddleFlow/paddle-operator/blob/sampleset/docs/en/ext-overview.md).\n- Build a distributed [Milvus](https://milvus.io) cluster based on JuiceFS, the Milvus team wrote a [case sharing](https://zilliz.com/blog/building-a-milvus-cluster-based-on-juicefs) and [tutorial](https://tutorials.milvus.io/en-juicefs/index.html?index=..%2F..index#0).\n\n## Big data\n\n- [Apache Kylin 4.0](http://kylin.apache.org) that is a OLAP engine could deploy with the JuiceFS in dissaggregated storage and compute architecture on every public cloud platform, there is [the video sharing](https://www.bilibili.com/video/BV1c54y1W72S) (in Chinese) and [the post](https://juicefs.com/en/blog/optimize-kylin-on-juicefs) for this use case.\n- [Apache Hudi](https://hudi.apache.org) supports JuiceFS since v0.10.0, you can refer to [official documentation](https://hudi.apache.org/docs/jfs_hoodie) to learn how to configure JuiceFS.\n\n## DevOps\n\n- [Terraform Provider for JuiceFS](https://github.com/toowoxx/terraform-provider-juicefs) by Toowoxx IT GmbH, an IT service company from Germany\n\n## Alfred\n\nJuiceFS documents offers an Alfred workflow to search documents of JuiceFS with instant results\n\n![JuiceFS Alfred Workflow](../images/workflow-root.png)\n\nSimply type your keyword into Alfred (default: jfs) and provide a query to see instant search results from JuiceFS documents.\n\n### Install\n\nWorkflow of Alfred 5 version: [Latest Download](https://github.com/zwwhdls/juicefs-alfred-workflow/releases/download/v0.2.0/JuiceFS.Search.alfredworkflow)\n\n### Usage\n\nSearch all documents of JuiceFS, including community, enterprise and CSI:\n\n```\n# JuiceFS community documents\njfs ce \n# JuiceFS enterprise documents\njfs ee \n# JuiceFS csi documents\njfs csi \n```\n\n![JuiceFS Alfred Workflow demo](../images/workflow-demo.gif)\n\n### Workflow Variables\n\n- `API_KEY`: API key for algolia which JuiceFS documents uses. Default value is ok.\n- `LANGUAGE`: Language of JuiceFS documents to search. Default is `en`.\n- `HITS_PER_PAGE`: Hits of each search. Default is `10`.\n\n![JuiceFS Alfred Workflow configuration](../images/configuration.png)\n" }, { "path": "docs/en/community/usage_tracking.md", "content": "---\ntitle: Usage Tracking\nsidebar_position: 4\n---\n\nJuiceFS by default collects and reports **anonymous** usage data. It only collects core metrics (e.g. version number, file system size), no user or any sensitive data will be collected. You could review related code [here](https://github.com/juicedata/juicefs/blob/main/pkg/usage/usage.go).\n\nThese data help us understand how the community is using this project. You could disable reporting easily by command line option `--no-usage-report`:\n\n```\njuicefs mount --no-usage-report\n```\n" }, { "path": "docs/en/deployment/_share_via_nfs.md", "content": "---\ntitle: Deploy JuiceFS with NFS\nsidebar_position: 5\n---\n" }, { "path": "docs/en/deployment/_share_via_smb.md", "content": "---\ntitle: Deploy JuiceFS with SMB\nsidebar_position: 6\n---\n" }, { "path": "docs/en/deployment/automation.md", "content": "---\ntitle: Automated Deployment\nsidebar_position: 7\n---\n\nAutomated deployment is recommended when JuiceFS Client is to be installed on a large number of hosts.\n\nBelow examples only demonstrate the mount process, you should [Create a file system](../getting-started/standalone.md#juicefs-format) before getting started.\n\n## Ansible\n\nBelow is the [Ansible](https://ansible.com) example to install and mount JuiceFS in localhost:\n\n```yaml\n- hosts: localhost\n tasks:\n - set_fact:\n # Change accordingly\n meta_url: sqlite3:///tmp/myjfs.db\n jfs_path: /jfs\n jfs_pkg: /tmp/juicefs-ce.tar.gz\n jfs_bin_dir: /usr/local/bin\n\n - get_url:\n # Change download URL accordingly\n url: https://d.juicefs.com/juicefs/releases/download/v1.0.2/juicefs-1.0.2-linux-amd64.tar.gz\n dest: \"{{jfs_pkg}}\"\n\n - ansible.builtin.unarchive:\n src: \"{{jfs_pkg}}\"\n dest: \"{{jfs_bin_dir}}\"\n include:\n - juicefs\n\n - name: Create symbolic for fstab\n ansible.builtin.file:\n src: \"{{jfs_bin_dir}}/juicefs\"\n dest: \"/sbin/mount.juicefs\"\n state: link\n\n - name: Mount JuiceFS and create fstab entry\n mount:\n path: \"{{jfs_path}}\"\n src: \"{{meta_url}}\"\n fstype: juicefs\n opts: _netdev\n state: mounted\n```\n" }, { "path": "docs/en/deployment/hadoop_java_sdk.md", "content": "---\ntitle: Use JuiceFS on Hadoop Ecosystem\nsidebar_position: 3\nslug: /hadoop_java_sdk\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\nJuiceFS provides [Hadoop-compatible File System](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/filesystem/introduction.html) by Hadoop Java SDK. Various applications in the Hadoop ecosystem can smoothly use JuiceFS to store data without changing the code.\n\n## Requirements\n\n### 1. Hadoop and related components\n\nJuiceFS Hadoop Java SDK is compatible with Hadoop 2.x and Hadoop 3.x. As well as variety of components in Hadoop ecosystem.\n\n### 2. User permissions\n\nJuiceFS uses local \"User/UID\" and \"Group/GID\" mappings by default, and when used in a distributed environment, to avoid permission issues, please refer to [documentation](../administration/sync_accounts_between_multiple_hosts.md) synchronizes the \"User/UID\" and \"Group/GID\" that needs to be used to all Hadoop nodes. It is also possible to define a global user and group file to make all nodes in the cluster share the permission configuration. Please see [here](#other-configurations) for related configurations.\n\n### 3. File system\n\nYou should first create at least one JuiceFS file system to provide storage for components related to the Hadoop ecosystem through the JuiceFS Java SDK. When deploying the Java SDK, specify the metadata engine address of the created file system in the configuration file.\n\nTo create a file system, please refer to [our quick start](../getting-started/standalone.md).\n\n:::note\nIf you want to use JuiceFS in a distributed environment, when creating a file system, please plan the object storage and database to be used reasonably to ensure that they can be accessed by each node in the cluster.\n:::\n\n### 4. Memory\n\nDepending on the read and write load of computing tasks (such as Spark executor), JuiceFS Hadoop Java SDK may require an additional 4 * [`juicefs.memory-size`](#io-configurations) off-heap memory to speed up read and write performance. By default, it is recommended to configure at least 1.2GB of off-heap memory for compute tasks.\n\n### 5. Java runtime version\n\nJuiceFS Hadoop Java SDK is compiled with JDK 8 by default. If it needs to be used in a higher version of Java runtime (such as Java 17), the following options need to be added to the JVM parameters to allow the use of reflection API:\n\n```shell\n--add-exports=java.base/sun.nio.ch=ALL-UNNAMED\n```\n\nFor more information on the above option, please refer to [official documentation](https://docs.oracle.com/en/java/javase/17/migrate/migrating-jdk-8-later-jdk-releases.html#GUID-7BB28E4D-99B3-4078-BDC4-FC24180CE82B).\n\n## Install and compile the client\n\n### Install the pre-compiled client\n\nPlease refer to the [\"Installation\"](../getting-started/installation.md#install-the-pre-compiled-client) document to learn how to download the precompiled JuiceFS Hadoop Java SDK.\n\n### Compile the client manually\n\n:::note\nNo matter which system environment the client is compiled for, the compiled JAR file has the same name and can only be deployed in the matching system environment. For example, when compiled in Linux, it can only be used in the Linux environment. In addition, since the compiled package depends on glibc, it is recommended to compile with a lower version system to ensure better compatibility.\n:::\n\nCompilation depends on the following tools:\n\n- [Go](https://golang.org) 1.20+\n- JDK 8+\n- [Maven](https://maven.apache.org) 3.3+\n- Git\n- make\n- GCC 5.4+\n\n#### Linux and macOS\n\nClone the repository:\n\n```shell\ngit clone https://github.com/juicedata/juicefs.git\n```\n\nEnter the directory and compile:\n\n```shell\ncd juicefs/sdk/java\nmake\n```\n\n:::note\nIf Ceph RADOS is used to store data, you need to install `librados-dev` first and [build `libjfs.so`]`.\n:::\n\n```shell\ncd juicefs/sdk/java\nmake ceph\n```\n\nAfter the compilation, you can find the compiled `JAR` file in the `sdk/java/target` directory, including two versions:\n\n- Contains third-party dependent packages: `juicefs-hadoop-X.Y.Z.jar`\n- Does not include third-party dependent packages: `original-juicefs-hadoop-X.Y.Z.jar`\n\nIt is recommended to use a version that includes third-party dependencies.\n\n#### Windows\n\nThe client used in the Windows environment needs to be obtained through cross-compilation on Linux or macOS. The compilation depends on [mingw-w64](https://www.mingw-w64.org), which needs to be installed first.\n\nThe steps are the same as compiling on Linux or macOS. For example, on the Ubuntu system, install the `mingw-w64` package first to solve the dependency problem:\n\n```shell\nsudo apt install mingw-w64\n```\n\nClone and enter the JuiceFS source code directory, execute the following code to compile:\n\n```shell\ncd juicefs/sdk/java\n```\n\n```shell\nmake win\n```\n\n## Deploy the client\n\nTo enable each component of the Hadoop ecosystem to correctly identify JuiceFS, the following configurations are required:\n\n1. Place the compiled JAR file and `$JAVA_HOME/lib/tools.jar` into the `classpath` of the component. The installation paths of common big data platforms and components are shown in the table below.\n2. Put JuiceFS configurations into the configuration file of each Hadoop ecosystem component (usually `core-site.xml`), see [Client Configurations](#client-configurations) for details.\n\nIt is recommended to place the JAR file in a fixed location, and the other locations are called it through symbolic links.\n\n### Big Data Platforms\n\n| Name | Installing Paths |\n|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| CDH | `/opt/cloudera/parcels/CDH/lib/hadoop/lib`

`/opt/cloudera/parcels/CDH/spark/jars`

`/var/lib/impala` |\n| HDP | `/usr/hdp/current/hadoop-client/lib`

`/usr/hdp/current/hive-client/auxlib`

`/usr/hdp/current/spark2-client/jars` |\n| Amazon EMR | `/usr/lib/hadoop/lib`

`/usr/lib/spark/jars`

`/usr/lib/hive/auxlib` |\n| Alibaba Cloud EMR | `/opt/apps/ecm/service/hadoop/*/package/hadoop*/share/hadoop/common/lib`

`/opt/apps/ecm/service/spark/*/package/spark*/jars`

`/opt/apps/ecm/service/presto/*/package/presto*/plugin/hive-hadoop2`

`/opt/apps/ecm/service/hive/*/package/apache-hive*/lib`

`/opt/apps/ecm/service/impala/*/package/impala*/lib` |\n| Tencent Cloud EMR | `/usr/local/service/hadoop/share/hadoop/common/lib`

`/usr/local/service/presto/plugin/hive-hadoop2`

`/usr/local/service/spark/jars`

`/usr/local/service/hive/auxlib` |\n| UCloud UHadoop | `/home/hadoop/share/hadoop/common/lib`

`/home/hadoop/hive/auxlib`

`/home/hadoop/spark/jars`

`/home/hadoop/presto/plugin/hive-hadoop2` |\n| Baidu Cloud EMR | `/opt/bmr/hadoop/share/hadoop/common/lib`

`/opt/bmr/hive/auxlib`

`/opt/bmr/spark2/jars` |\n\n### Community Components\n\n| Name | Installing Paths |\n|-----------|-----------------------------------------------------------------------------------------|\n| Hadoop | `${HADOOP_HOME}/share/hadoop/common/lib/`, `${HADOOP_HOME}/share/hadoop/mapreduce/lib/` |\n| Spark | `${SPARK_HOME}/jars` |\n| Presto | `${PRESTO_HOME}/plugin/hive-hadoop2` |\n| Trino | `${TRINO_HOME}/plugin/hive` |\n| Flink | `${FLINK_HOME}/lib` |\n| StarRocks | `${StarRocks_HOME}/fe/lib/`, `${StarRocks_HOME}/be/lib/hadoop/common/lib` |\n\n### Client Configurations\n\nPlease refer to the following table to set the relevant parameters of the JuiceFS file system and write it into the configuration file, which is generally `core-site.xml`.\n\n#### Core Configurations\n\n| Configuration | Default Value | Description |\n|----------------------------------|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `fs.jfs.impl` | `io.juicefs.JuiceFileSystem` | Specify the storage implementation to be used. By default, `jfs://` scheme is used. If you want to use different scheme (e.g. `cfs://`), just modify it to `fs.cfs.impl`. No matter what scheme you use, it is always access the data in JuiceFS. |\n| `fs.AbstractFileSystem.jfs.impl` | `io.juicefs.JuiceFS` | Specify the storage implementation to be used. By default, `jfs://` scheme is used. If you want to use different scheme (e.g. `cfs://`), just modify it to `fs.AbstractFileSystem.cfs.impl`. No matter what scheme you use, it is always access the data in JuiceFS. |\n| `juicefs.meta` | | Specify the metadata engine address of the pre-created JuiceFS file system. You can configure multiple file systems for the client at the same time through the format of `juicefs.{vol_name}.meta`. Refer to [\"Multiple file systems configuration\"](#multiple-file-systems-configuration). |\n\n#### Cache Configurations\n\n| Configuration | Default Value | Description |\n|------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `juicefs.cache-dir` | `memory` | Directory paths of local cache. Use colon to separate multiple paths. Also support wildcard in path. **It's recommended create these directories manually and set `0777` permission so that different applications could share the cache data.** If not specified, default to process memory. |\n| `juicefs.cache-size` | 100 | Maximum size of local cache in MiB. Default size is small because Hadoop SDK uses memory as default cache location. It's the total size when set multiple cache directories. |\n| `juicefs.cache-full-block` | `true` | Whether cache every read blocks, `false` means only cache random/small read blocks. |\n| `juicefs.free-space` | 0.1 | Min free space ratio of cache directory |\n| `juicefs.open-cache` | 0 | Open files cache timeout in seconds (0 means disable this feature) |\n| `juicefs.attr-cache` | 0 | Expire of attributes cache in seconds |\n| `juicefs.entry-cache` | 0 | Expire of file entry cache in seconds |\n| `juicefs.dir-entry-cache` | 0 | Expire of directory entry cache in seconds |\n| `juicefs.discover-nodes-url` | | Specify the node discovery API, the node list will be refreshed every 10 minutes.

  • YARN: `yarn`
  • Spark Standalone: `http://spark-master:web-ui-port/json/`
  • Spark ThriftServer: `http://thrift-server:4040/api/v1/applications/`
  • Presto: `http://coordinator:discovery-uri-port/v1/service/presto/`
  • File system: `jfs://{VOLUME}/etc/nodes`, you need to create this file manually, and write the hostname of the node into this file line by line
|\n\n#### I/O Configurations\n\n| Configuration | Default Value | Description |\n|--------------------------|---------------|-------------------------------------------------|\n| `juicefs.max-uploads` | 20 | The max number of connections to upload |\n| `juicefs.max-downloads` | 200 | The max number of connections to download |\n| `juicefs.max-deletes` | 10 | The max number of connections to delete |\n| `juicefs.get-timeout` | 5 | The max number of seconds to download an object |\n| `juicefs.put-timeout` | 60 | The max number of seconds to upload an object |\n| `juicefs.memory-size` | 300 | Total read/write buffering in MiB |\n| `juicefs.prefetch` | 1 | Prefetch N blocks in parallel |\n| `juicefs.upload-limit` | 0 | Bandwidth limit for upload in Mbps |\n| `juicefs.download-limit` | 0 | Bandwidth limit for download in Mbps |\n| `juicefs.io-retries` | 10 | Number of retries after network failure |\n| `juicefs.writeback` | `false` | Upload objects in background |\n\n#### Other Configurations\n\n| Configuration | Default Value | Description |\n|-------------------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `juicefs.bucket` | | Specify a different endpoint for object storage |\n| `juicefs.debug` | `false` | Whether enable debug log |\n| `juicefs.access-log` | | Access log path. Ensure Hadoop application has write permission, e.g. `/tmp/juicefs.access.log`. The log file will rotate automatically to keep at most 7 files. |\n| `juicefs.superuser` | `hdfs` | The super user |\n| `juicefs.supergroup` | `supergroup` | The super user group |\n| `juicefs.users` | `null` | The path of username and UID list file, e.g. `jfs://name/etc/users`. The file format is `:`, one user per line. |\n| `juicefs.groups` | `null` | The path of group name, GID and group members list file, e.g. `jfs://name/etc/groups`. The file format is `::,`, one group per line. |\n| `juicefs.umask` | `null` | The umask used when creating files and directories (e.g. `0022`), default value is `fs.permissions.umask-mode`. |\n| `juicefs.push-gateway` | | [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) address, format is `:`. |\n| `juicefs.push-auth` | | [Prometheus basic auth](https://prometheus.io/docs/guides/basic-auth) information, format is `:`. |\n| `juicefs.push-graphite` | | [Graphite](https://graphiteapp.org) address, format is `:`. |\n| `juicefs.push-remote-write` | | [Prometheus remote write](https://prometheus.io/docs/specs/prw/remote_write_spec) endpoint, format is `http://:`. |\n| `juicefs.push-remote-write-auth` | | Authentication for remote write endpoint, format is `:`. |\n| `juicefs.push-interval` | 10 | Metric push interval (in seconds) |\n| `juicefs.push-labels` | | Metric labels, format is `key1:value1;key2:value2`. |\n| `juicefs.fast-resolve` | `true` | Whether enable faster metadata lookup using Redis Lua script |\n| `juicefs.no-usage-report` | `false` | Whether disable usage reporting. JuiceFS only collects anonymous usage data (e.g. version number), no user or any sensitive data will be collected. |\n| `juicefs.no-bgjob` | `false` | Disable background jobs (clean-up, backup, etc.) |\n| `juicefs.backup-meta` | 3600 | Interval (in seconds) to automatically backup metadata in the object storage (0 means disable backup) |\n| `juicefs.backup-skip-trash` | `false` | Skip files and directories in trash when backup metadata. |\n| `juicefs.heartbeat` | 12 | Heartbeat interval (in seconds) between client and metadata engine. It's recommended that all clients use the same value. |\n| `juicefs.skip-dir-mtime` | 100ms | Minimal duration to modify parent dir mtime. |\n| `juicefs.subdir` | | Allow access only to the subpaths of this directory. Multiple paths can be specified, separated by commas. All other paths, including the root or sibling directories, will be denied access. |\n\n#### Multiple file systems configuration\n\nWhen multiple JuiceFS file systems need to be used at the same time, all the above configuration items can be specified for a specific file system. You only need to put the file system name in the middle of the configuration item, such as `jfs1` and `jfs2` in the following example:\n\n```xml\n\n juicefs.jfs1.meta\n redis://jfs1.host:port/1\n\n\n juicefs.jfs2.meta\n redis://jfs2.host:port/1\n\n```\n\n#### Configuration Example\n\nThe following is a commonly used configuration example. Please replace the `{HOST}`, `{PORT}` and `{DB}` variables in the `juicefs.meta` configuration with actual values.\n\n```xml\n\n fs.jfs.impl\n io.juicefs.JuiceFileSystem\n\n\n fs.AbstractFileSystem.jfs.impl\n io.juicefs.JuiceFS\n\n\n juicefs.meta\n redis://{HOST}:{PORT}/{DB}\n\n\n juicefs.cache-dir\n /data*/jfs\n\n\n juicefs.cache-size\n 1024\n\n\n juicefs.access-log\n /tmp/juicefs.access.log\n\n```\n\n## Configuration in Hadoop\n\nPlease refer to the aforementioned configuration tables and add configuration parameters to the Hadoop configuration file `core-site.xml`.\n\n### CDH6\n\nIf you are using CDH 6, in addition to modifying `core-site`, you also need to modify `mapreduce.application.classpath` through the YARN service interface, adding:\n\n```shell\n$HADOOP_COMMON_HOME/lib/juicefs-hadoop.jar\n```\n\n### HDP\n\nIn addition to modifying `core-site`, you also need to modify the configuration `mapreduce.application.classpath` through the MapReduce2 service interface and add it at the end (variables do not need to be replaced):\n\n```shell\n/usr/hdp/${hdp.version}/hadoop/lib/juicefs-hadoop.jar\n```\n\n### Flink\n\nAdd configuration parameters to `conf/flink-conf.yaml`. If you only use JuiceFS in Flink, you don't need to configure JuiceFS in the Hadoop environment, you only need to configure the Flink client.\n\n### Hudi\n\n:::note\nHudi supports JuiceFS since v0.10.0, please make sure you are using the correct version.\n:::\n\nPlease refer to [\"Hudi Official Documentation\"](https://hudi.apache.org/docs/jfs_hoodie) to learn how to configure JuiceFS.\n\n### Kafka Connect\n\nIt is possible to use Kafka Connect and HDFS Sink Connector（[HDFS 2](https://docs.confluent.io/kafka-connect-hdfs/current/overview.html) and [HDFS 3](https://docs.confluent.io/kafka-connect-hdfs3-sink/current/overview.html)）to store data on JuiceFS.\n\nFirst you need to add JuiceFS SDK to `classpath` in Kafka Connect, e.g., `/usr/share/java/confluentinc-kafka-connect-hdfs/lib`.\n\nWhile creating a Connect Sink task, configuration needs to be set up as follows:\n\n- Specify `hadoop.conf.dir` as the directory that contains the configuration file `core-site.xml`. If it is not running in Hadoop environment, you can create a separate directory such as `/usr/local/juicefs/hadoop`, and then add the JuiceFS related configurations to `core-site.xml`.\n- Specify `store.url` as a path starting with `jfs://`.\n\nFor example:\n\n```ini\n# Other configuration items are omitted.\nhadoop.conf.dir=/path/to/hadoop-conf\nstore.url=jfs://path/to/store\n```\n\n### HBase\n\nJuiceFS can be used by HBase for HFile, but is not fast (low latency) enough for Write Ahead Log (WAL), because it take much longer time to persist data into object storage than memory of DataNode.\n\nIt is recommended to deploy a small HDFS cluster to store WAL and HFile files to be stored on JuiceFS.\n\n#### Create a new HBase cluster\n\nModify `hbase-site.xml`:\n\n```xml title=\"hbase-site.xml\"\n\n hbase.rootdir\n jfs://{vol_name}/hbase\n\n\n hbase.wal.dir\n hdfs://{ns}/hbase-wal\n\n```\n\n#### Modify existing HBase cluster\n\nIn addition to modifying the above configurations, since the HBase cluster has already stored some data in ZooKeeper, in order to avoid conflicts, there are two solutions:\n\n1. Delete the old cluster\n\n Delete the znode (default `/hbase`) configured by `zookeeper.znode.parent` via the ZooKeeper client.\n\n :::note\n This operation will delete all data on this HBase cluster.\n :::\n\n2. Use a new znode\n\n Keep the znode of the original HBase cluster so that it can be recovered later. Then configure a new value for `zookeeper.znode.parent`:\n\n ```xml title=\"hbase-site.xml\"\n \n zookeeper.znode.parent\n /hbase-jfs\n \n ```\n\n### Restart Services\n\nWhen the following components need to access JuiceFS, they should be restarted.\n\n:::note\nBefore restart, you need to confirm JuiceFS related configuration has been written to the configuration file of each component, usually you can find them in `core-site.xml` on the machine where the service of the component was deployed.\n:::\n\n| Components | Services |\n| ---------- | -------------------------- |\n| Hive | HiveServer
Metastore |\n| Spark | ThriftServer |\n| Presto | Coordinator
Worker |\n| Impala | Catalog Server
Daemon |\n| HBase | Master
RegionServer |\n\nHDFS, Hue, ZooKeeper and other services don't need to be restarted.\n\nWhen `Class io.juicefs.JuiceFileSystem not found` or `No FilesSystem for scheme: jfs` exceptions was occurred after restart, reference [FAQ](#faq).\n\n### Trash\n\nJuiceFS Hadoop Java SDK also has the same trash function as HDFS, which needs to be enabled by setting `fs.trash.interval` and `fs.trash.checkpoint.interval`, please refer to [HDFS documentation](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html#File_Deletes_and_Undeletes) for more information.\n\n## Environmental Verification\n\nAfter the deployment of the JuiceFS Java SDK, the following methods can be used to verify the success of the deployment.\n\n### Hadoop CLI\n\n```bash\nhadoop fs -ls jfs://{JFS_NAME}/\n```\n\n:::info\nThe `JFS_NAME` is the volume name when you format JuiceFS file system.\n:::\n\n### Hive\n\n```sql\nCREATE TABLE IF NOT EXISTS person\n(\n name STRING,\n age INT\n) LOCATION 'jfs://{JFS_NAME}/tmp/person';\n```\n\n### Java/Scala project\n\n1. Add Maven or Gradle dependencies:\n\n \n \n\n ```xml\n \n org.apache.hadoop\n hadoop-common\n {HADOOP_VERSION}\n provided\n \n \n io.juicefs\n juicefs-hadoop\n {JUICEFS_HADOOP_VERSION}\n provided\n \n ```\n\n \n \n\n ```groovy\n dependencies {\n implementation 'org.apache.hadoop:hadoop-common:${hadoopVersion}'\n implementation 'io.juicefs:juicefs-hadoop:${juicefsHadoopVersion}'\n }\n ```\n\n \n \n\n2. Use the following sample code to verify:\n\n\n ```java\n package demo;\n\n import org.apache.hadoop.conf.Configuration;\n import org.apache.hadoop.fs.FileStatus;\n import org.apache.hadoop.fs.FileSystem;\n import org.apache.hadoop.fs.Path;\n\n public class JuiceFSDemo {\n public static void main(String[] args) throws Exception {\n Configuration conf = new Configuration();\n conf.set(\"fs.jfs.impl\", \"io.juicefs.JuiceFileSystem\");\n conf.set(\"juicefs.meta\", \"redis://127.0.0.1:6379/0\"); // JuiceFS metadata engine URL\n Path p = new Path(\"jfs://{JFS_NAME}/\"); // Please replace \"{JFS_NAME}\" with the correct value\n FileSystem jfs = p.getFileSystem(conf);\n FileStatus[] fileStatuses = jfs.listStatus(p);\n // Traverse JuiceFS file system and print file paths\n for (FileStatus status : fileStatuses) {\n System.out.println(status.getPath());\n }\n }\n }\n ```\n\n\n## Monitoring metrics collection\n\nPlease see the [\"Monitoring\"](../administration/monitoring.md) documentation to learn how to collect and display JuiceFS monitoring metrics.\n\n## Benchmark\n\nHere are a series of methods to use the built-in stress testing tool of the JuiceFS client to test the performance of the client environment that has been successfully deployed.\n\n### 1. Local Benchmark\n\n#### Metadata\n\n- **create**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench create -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n This command will create 10000 empty files\n\n- **open**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench open -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n This command will open 10000 files without reading data\n\n- **rename**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench rename -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n- **delete**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench delete -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n- **For reference**\n\n | Operation | TPS | Latency (ms) |\n | --------- | ---- | ------------ |\n | create | 644 | 1.55 |\n | open | 3467 | 0.29 |\n | rename | 483 | 2.07 |\n | delete | 506 | 1.97 |\n\n#### I/O Performance\n\n- **sequential write**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -write -size 20000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO -local\n ```\n\n- **sequential read**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -read -size 20000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO -local\n ```\n\n When run the cmd for the second time, the result may be much better than the first run. It's because the data was cached in memory, just clean the local disk cache.\n\n- **For reference**\n\n | Operation | Throughput (MB/s) |\n | --------- | ----------------- |\n | write | 647 |\n | read | 111 |\n\nIf the network bandwidth of the machine is relatively low, it can generally reach the network bandwidth bottleneck.\n\n### 2. Distributed Benchmark\n\nThe following command will start the MapReduce distributed task to test the metadata and IO performance. During the test, it is necessary to ensure that the cluster has sufficient resources to start the required map tasks.\n\nComputing resources used in this test:\n\n- **Server**: 4 cores and 32 GB memory, burst bandwidth 5Gbit/s x 3\n- **Database**: Alibaba Cloud Redis 5.0 Community 4G Master-Slave Edition\n\n#### Metadata\n\n- **create**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench create -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 10 map task, each has 10 threads, each thread create 1000 empty file. 100000 files in total\n\n- **open**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench open -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 10 map task, each has 10 threads, each thread open 1000 file. 100000 files in total\n\n- **rename**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench rename -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 10 map task, each has 10 threads, each thread rename 1000 file. 100000 files in total\n\n- **delete**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench delete -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 10 map task, each has 10 threads, each thread delete 1000 file. 100000 files in total\n\n- **For reference**\n\n - 10 threads\n\n | Operation | IOPS | Latency (ms) |\n | --------- | ---- | ------------ |\n | create | 4178 | 2.2 |\n | open | 9407 | 0.8 |\n | rename | 3197 | 2.9 |\n | delete | 3060 | 3.0 |\n\n - 100 threads\n\n | Operation | IOPS | Latency (ms) |\n | --------- | ---- | ------------ |\n | create | 11773 | 7.9 |\n | open | 34083 | 2.4 |\n | rename | 8995 | 10.8 |\n | delete | 7191 | 13.6 |\n\n#### I/O Performance\n\n- **sequential write**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -write -maps 10 -size 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO\n ```\n\n 10 map task, each task write 10000MB random data sequentially\n\n- **sequential read**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -read -maps 10 -size 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO\n ```\n\n 10 map task, each task read 10000MB random data sequentially\n\n- **For reference**\n\n | Operation | Average throughput (MB/s) | Total Throughput (MB/s) |\n | --------- | ------------------------- | ----------------------- |\n | write | 198 | 1835 |\n | read | 124 | 1234 |\n\n### 3. TPC-DS\n\nThe test dataset is 100GB in size, and both Parquet and ORC file formats are tested.\n\nThis test only tests the first 10 queries.\n\nSpark Thrift JDBC/ODBC Server is used to start the Spark resident process and then submit the task via Beeline connection.\n\n#### Test Hardware\n\n| Node Category | Instance Type | CPU | Memory | Disk | Number |\n| ------------- | ------------- | --- | ------ | ---- | ------ |\n| Master | Alibaba Cloud ecs.r6.xlarge | 4 | 32GiB | System Disk: 100GiB | 1 |\n| Core | Alibaba Cloud ecs.r6.xlarge | 4 | 32GiB | System Disk: 100GiB
Data Disk: 500GiB Ultra Disk x 2 | 3 |\n\n#### Software Configuration\n\n##### Spark Thrift JDBC/ODBC Server\n\n```shell\n${SPARK_HOME}/sbin/start-thriftserver.sh \\\n --master yarn \\\n --driver-memory 8g \\\n --executor-memory 10g \\\n --executor-cores 3 \\\n --num-executors 3 \\\n --conf spark.locality.wait=100 \\\n --conf spark.sql.crossJoin.enabled=true \\\n --hiveconf hive.server2.thrift.port=10001\n```\n\n##### JuiceFS Cache Configurations\n\nThe 2 data disks of Core node are mounted in the `/data01` and `/data02` directories, and `core-site.xml` is configured as follows:\n\n```xml\n\n juicefs.cache-size\n 200000\n\n\n juicefs.cache-dir\n /data*/jfscache\n\n\n juicefs.cache-full-block\n false\n\n\n juicefs.discover-nodes-url\n yarn\n\n\n juicefs.attr-cache\n 3\n\n\n juicefs.entry-cache\n 3\n\n\n juicefs.dir-entry-cache\n 3\n\n```\n\n#### Test\n\nThe task submission command is as follows:\n\n```shell\n${SPARK_HOME}/bin/beeline -u jdbc:hive2://localhost:10001/${DATABASE} \\\n -n hadoop \\\n -f query{i}.sql\n```\n\n#### Results\n\nJuiceFS can use local disk as a cache to accelerate data access, the following data is the result (in seconds) after 4 runs using Redis and TiKV as the metadata engine of JuiceFS respectively.\n\n##### ORC\n\n| Queries | JuiceFS (Redis) | JuiceFS (TiKV) | HDFS |\n| ------- | --------------- | -------------- | ---- |\n| q1 | 20 | 20 | 20 |\n| q2 | 28 | 33 | 26 |\n| q3 | 24 | 27 | 28 |\n| q4 | 300 | 309 | 290 |\n| q5 | 116 | 117 | 91 |\n| q6 | 37 | 42 | 41 |\n| q7 | 24 | 28 | 23 |\n| q8 | 13 | 15 | 16 |\n| q9 | 87 | 112 | 89 |\n| q10 | 23 | 24 | 22 |\n\n![orc](../images/spark_ql_orc.png)\n\n##### Parquet\n\n| Queries | JuiceFS (Redis) | JuiceFS (TiKV) | HDFS |\n| ------- | --------------- | -------------- | ---- |\n| q1 | 33 | 35 | 39 |\n| q2 | 28 | 32 | 31 |\n| q3 | 23 | 25 | 24 |\n| q4 | 273 | 284 | 266 |\n| q5 | 96 | 107 | 94 |\n| q6 | 36 | 35 | 42 |\n| q7 | 28 | 30 | 24 |\n| q8 | 11 | 12 | 14 |\n| q9 | 85 | 97 | 77 |\n| q10 | 24 | 28 | 38 |\n\n![parquet](../images/spark_sql_parquet.png)\n\n## Permission control by Apache Ranger(from v1.3)\n\nJuiceFS currently supports path permission control by integrating with Apache Ranger's HDFS module. Only supported in Hadoop Java SDK.\n\n### 1. Configurations\n\nThe config for Apache Ranger is sotred in the metadata database. You can enable Ranger permission control by the following methods:\n\n```shell\n# configure with format\njuicefs format META-URL NAME --ranger-rest-url http://localhost:6080 --ranger-service jfs\n\n# or configure with config\njuicefs config META-URL --ranger-rest-url http://localhost:6080 --ranger-service jfs\n\n# disable ranger\njuicefs config META-URL --ranger-rest-url \"\" --ranger-service jfs \"\"\n```\n\n### 2. Dependencies\n\nConsidering the convenience of use, JuiceFS packages all Ranger dependencies into the JuiceFS SDK. If you encounter version conflicts with Apache Ranger, you may need to modify the version and recompile.\n\n### 3. Tips\n\n#### 3.1 Ranger version\n\nThe code is tested on `Ranger2.3` and `Ranger2.4`. As no other features are used except for `HDFS` module authentication, theoretically all other versions are applicable.\n\n#### 3.2 Ranger Audit\n\nCurrently, only support authentication function, and the `Ranger Audit` is disabled.\n\n#### 3.3 Ranger's other parameters\n\nTo improve usage efficiency, currently only support some **CORE** parameters of Ranger.\n\n#### 3.4 Security tips\n\nDue to the complete open source of the project, it is unavoidable for users to disrupt permission control by replacing parameters such as `ranger-rest-url`. If stricter control is required, it is recommended to compile the code independently and solve the problem by encrypting relevant security parameters.\n\n## FAQ\n\n### 1. `Class io.juicefs.JuiceFileSystem not found` exception\n\nIt means JAR file was not loaded, you can verify it by `lsof -p {pid} | grep juicefs`.\n\nYou should check whether the JAR file was located properly, or other users have the read permission.\n\nSome Hadoop distribution also need to modify `mapred-site.xml` and put the JAR file location path to the end of the parameter `mapreduce.application.classpath`.\n\n### 2. `No FilesSystem for scheme: jfs` exception\n\nIt means JuiceFS Hadoop Java SDK was not configured properly, you need to check whether there is JuiceFS related configuration in the `core-site.xml` of the component configuration.\n\n### 3. What are the similarities and differences between user permission management in JuiceFS and HDFS?\n\nJuiceFS also uses the \"User/Group\" method to manage file permissions, using local users and groups by default. In order to ensure the unified permissions of different nodes during distributed computing, you can configure global \"User/UID\" and \"Group/GID\" mappings through `juicefs.users` and `juicefs.groups` configurations.\n\n### 4. After the data is deleted, it is directly stored in the `.trash` directory of JuiceFS. Although the files are all there, it is difficult to restore the data through the `mv` command as easily as HDFS. Is there any way to achieve a similar effect of HDFS trash?\n\nIn the Hadoop application scenario, the functions similar to the HDFS trash are still retained. It needs to be explicitly enabled by `fs.trash.interval` and `fs.trash.checkpoint.interval` configurations, please refer to [document](#trash) for more information.\n\n### 5. What are the benefits of setting the `juicefs.discover-nodes-url` configuration?\n\nIn HDFS, each data block will have [`BlockLocation`](https://hadoop.apache.org/docs/current/api/org/apache/hadoop/fs/BlockLocation.html) information, which the computing engine uses to schedule the computing tasks as much as possible to the nodes where the data is stored. JuiceFS will calculate the corresponding `BlockLocation` for each data block through the consistent hashing algorithm, so that when the same data is read for the second time, the computing engine may schedule the computing task to the same node, and the data cached on the local disk during the first computing can be used to accelerate data access.\n\nThis algorithm needs to know all the computing node information in advance. The `juicefs.discover-nodes-url` configuration is used to obtain these computing node information.\n\n### 6. Does the community version of JuiceFS currently support a Kerberos-authenticated CDH cluster?\n\nNot supported. JuiceFS does not verify the validity of Kerberos users, but can use Kerberos-authenticated username.\n" }, { "path": "docs/en/deployment/how_to_use_on_kubernetes.md", "content": "---\ntitle: Use JuiceFS on Kubernetes\nsidebar_position: 2\nslug: /how_to_use_on_kubernetes\n---\n\nJuiceFS is an ideal storage layer for Kubernetes, read this chapter to learn how to use JuiceFS in Kubernetes.\n\n## Use JuiceFS via `hostPath`\n\nIf you simply need to use JuiceFS inside Kubernetes pods, without any special requirements (e.g. isolation, permission control), then [`hostPath`](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) can be a good practice, which is also really easy to setup:\n\n1. Install and mount JuiceFS on all Kubernetes worker nodes, [Automated Deployment](./automation.md) is recommended for this type of work.\n1. Use `hostPath` volume inside pod definition, and mount a JuiceFS sub-directory to container:\n\n ```yaml {8-16}\n apiVersion: v1\n kind: Pod\n metadata:\n name: juicefs-app\n spec:\n containers:\n - ...\n volumeMounts:\n - name: jfs-data\n mountPath: /opt/app-data\n volumes:\n - name: jfs-data\n hostPath:\n # Assuming JuiceFS is mounted on /jfs\n path: \"/jfs/myapp/\"\n type: Directory\n ```\n\nIn comparison to using JuiceFS CSI Driver, `hostPath` is a much more simple practice, and easier to debug when things go wrong, but notice that:\n\n* For ease of management, generally all pods use the same host mount point. Lack of isolation may lead to data security issues, and obviously, you won't be able to adjust JuiceFS mount parameters separately for each application. Please evaluate carefully.\n* All worker nodes should mount JuiceFS in advance, so when adding a new node to the cluster, JuiceFS needs to be installed and mounted during the initialization process, otherwise the new node does not have a JuiceFS mount point, and the container will not be created.\n* The system resources (such as CPU, memory, etc.) occupied by the JuiceFS mounting process on the host are not controlled by Kubernetes, and may occupy too many host resources. You can consider using [`system-reserved`](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#system-reserved) to properly adjust the system resource reservation of Kubernetes, to reserve more resources for the JuiceFS mount process.\n* If the JuiceFS mount process on the host exits unexpectedly, the application pod will not be able to access the mount point normally. In this case, the JuiceFS file system needs to be remounted and the application pod must be rebuilt. However, JuiceFS CSI Driver solves this problem well by providing the [Automatic Mount Point Recovery](https://juicefs.com/docs/csi/recover-failed-mountpoint) mechanism.\n* If you're using Docker as Kubernetes container runtime, it's best to start JuiceFS mount prior to Docker in startup order, to avoid containers being created before JuiceFS is properly mounted. For systemd, you can use below unit file to manually control startup order:\n\n ```systemd title=\"/etc/systemd/system/docker.service.d/override.conf\"\n [Unit]\n # Use below command to obtain JuiceFS mount service name\n # systemctl list-units | grep \"\\.mount\"\n After=network-online.target firewalld.service containerd.service jfs.mount\n ```\n\n## JuiceFS CSI Driver\n\nTo use JuiceFS in Kubernetes, refer to [JuiceFS CSI Driver Documentation](https://juicefs.com/docs/csi/introduction).\n\n## Mount JuiceFS in the container\n\nIn some cases, you may need to mount JuiceFS volume directly in the container, which requires the use of the JuiceFS client in the container. You can refer to the following `Dockerfile` example to integrate the JuiceFS client into your application image:\n\n```dockerfile title=\"Dockerfile\"\nFROM alpine:latest\nLABEL maintainer=\"Juicedata \"\n\n# Install JuiceFS client\nRUN apk add --no-cache curl && \\\n JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '\"' -f 4 | tr -d 'v') && \\\n wget \"https://github.com/juicedata/juicefs/releases/download/v${JFS_LATEST_TAG}/juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" && \\\n tar -zxf \"juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" && \\\n install juicefs /usr/bin && \\\n rm juicefs \"juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" && \\\n rm -rf /var/cache/apk/* && \\\n apk del curl\n\nENTRYPOINT [\"/usr/bin/juicefs\", \"mount\"]\n```\n\nSince JuiceFS needs to use the FUSE device to mount the file system, it is necessary to allow the container to run in privileged mode when creating a Pod:\n\n```yaml {19-20}\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-run\nspec:\n selector:\n matchLabels:\n app: nginx\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: linuxserver/nginx\n ports:\n - containerPort: 80\n securityContext:\n privileged: true\n```\n\n:::caution\nWith the privileged mode being enabled by `privileged: true`, the container has access to all devices of the host, that is, it has full control of the host's kernel. Improper uses will bring serious safety hazards. Please conduct a thorough safety assessment before using it.\n:::\n" }, { "path": "docs/en/deployment/juicefs_on_docker.md", "content": "---\ntitle: Using JuiceFS in Docker\nsidebar_position: 6\nslug: /juicefs_on_docker\ndescription: Using JuiceFS in Docker in different ways, including volume mapping, volume plugin, and mounting in containers.\n---\n\nYou can use the JuiceFS file system in Docker by running the client directly in the container or using a volume plugin.\n\n## Using a volume plugin {#volume-plugin}\n\nIf you have specific requirements for mount management, such as managing mount points through Docker to facilitate different application containers using different JuiceFS file systems, you can use a [Docker volume plugin](https://github.com/juicedata/docker-volume-juicefs).\n\nDocker plugins are usually provided in the form of images. The [JuiceFS volume plugin image](https://hub.docker.com/r/juicedata/juicefs) contains the [JuiceFS Community Edition](../introduction/README.md) and [JuiceFS Cloud Service](https://juicefs.com/docs/cloud) clients. After installation, you can run the volume plugin to create JuiceFS volumes in Docker.\n\nInstall the plugin using the following command and provide the necessary permissions for FUSE as prompted:\n\n```shell\ndocker plugin install juicedata/juicefs\n```\n\nYou can use the following commands to manage the volume plugin:\n\n```shell\n# Disable the plugin\ndocker plugin disable juicedata/juicefs\n\n# Upgrade the plugin (must be disabled first)\ndocker plugin upgrade juicedata/juicefs\ndocker plugin enable juicedata/juicefs\n\n# Remove the plugin\ndocker plugin rm juicedata/juicefs\n```\n\n### Create a storage volume {#create-volume}\n\nReplace ``, ``, ``, ``, ``, and `` in the following command with your own file system configuration:\n\n```shell\ndocker volume create -d juicedata/juicefs \\\n -o name= \\\n -o metaurl= \\\n -o storage= \\\n -o bucket= \\\n -o access-key= \\\n -o secret-key= \\\n jfsvolume\n```\n\nFor pre-created file systems, you only need to specify the file system name and database address when creating the volume plugin, for example:\n\n```shell\ndocker volume create -d juicedata/juicefs \\\n -o name= \\\n -o metaurl= \\\n jfsvolume\n```\n\nIf you need to pass additional environment variables when mounting the file system, such as in [Google Cloud](../reference/how_to_set_up_object_storage.md#google-cloud), you can append parameters similar to `-o env=FOO=bar,SPAM=egg` to the above command.\n\n### Usage and management {#usage-and-management}\n\n```shell\n# Mount the volume when creating a container\ndocker run -it -v jfsvolume:/opt busybox ls /opt\n\n# After unmounting, you can delete the storage volume. Note that this only deletes the corresponding resources in Docker and does not affect the data stored in JuiceFS.\ndocker volume rm jfsvolume\n```\n\n### Using the plugin in Docker Compose {#using-plugin-in-docker-compose}\n\nHere is an example of using the JuiceFS volume plugin in `docker-compose`:\n\n```yaml\nversion: '3'\nservices:\n busybox:\n image: busybox\n command: \"ls /jfs\"\n volumes:\n - jfsvolume:/jfs\n \nvolumes:\n jfsvolume:\n driver: juicedata/juicefs\n driver_opts:\n name: ${VOL_NAME}\n # SQLite creates the database file in the plugin container's local path,\n # and sqlite:// will fail when the service is restarted.\n # (See https://github.com/juicedata/docker-volume-juicefs/issues/37 for details)\n metaurl: ${META_URL}\n storage: ${STORAGE_TYPE}\n bucket: ${BUCKET}\n access-key: ${ACCESS_KEY}\n secret-key: ${SECRET_KEY}\n # If necessary, you can pass additional environment variables using env\n # env: FOO=bar,SPAM=egg\n```\n\nUsage and management:\n\n```shell\n# Start the service\ndocker-compose up\n\n# Stop the service and unmount the JuiceFS file system from Docker\ndocker-compose down --volumes\n```\n\n### Troubleshooting the volume plugin {#troubleshooting}\n\nIf it is not working properly, it is recommended to first [upgrade the volume plugin](#volume-plugin), and then check the logs based on the problem.\n\n* Collect JuiceFS client logs. The logs are located inside the Docker volume plugin container and need to be accessed by entering the container:\n\n ```shell\n # Confirm the docker plugins runtime directory, which may be different from the example below depending on the actual situation\n # The directory printed by ls is the container directory, and the name is the container ID\n ls /run/docker/plugins/runtime-root/plugins.moby\n\n # Print plugin container information\n # If the printed container list is empty, it means that the plugin container failed to be created\n # Read the plugin startup log below to continue troubleshooting\n runc --root /run/docker/plugins/runtime-root/plugins.moby list\n\n # Enter the container and print the log\n runc --root /run/docker/plugins/runtime-root/plugins.moby exec 452d2c0cf3fd45e73a93a2f2b00d03ed28dd2bc0c58669cca9d4039e8866f99f cat /var/log/juicefs.log\n ```\n\n If the container does not exist (`ls` finds an empty directory) or the `juicefs.log` does not exist in the final log printing stage, it is likely that the mount itself failed. Continue to check the plugin's own logs to find the cause.\n\n* Collect plugin logs, using systemd as an example:\n\n ```shell\n journalctl -f -u docker | grep \"plugin=\"\n ```\n\n If there is an error when the plugin calls `juicefs` or if the plugin itself reports an error, it will be reflected in the logs.\n\n## Using the JuiceFS client in containers {#mount-juicefs-in-docker}\n\nCompared to the volume plugin, using the JuiceFS client directly in the container is more flexible. You can directly mount the JuiceFS file system in the container or access it through S3 Gateway or WebDAV.\n\n### Method 1: Build your own image\n\nThe JuiceFS client is a standalone binary program that provides versions for both AMD64 and ARM64 architectures. You can define the command to download and install the JuiceFS client in the Dockerfile, for example:\n\n```Dockerfile\nFROM ubuntu:22.04\n...\n# Use the official one-click installation script\nRUN curl -sSL https://d.juicefs.com/install | sh - \n```\n\nFor more information, see [Customizing Container Images](https://juicefs.com/docs/csi/guide/custom-image).\n\n### Method 2: Use the officially maintained image\n\nThe JuiceFS officially maintained image [`juicedata/mount`](https://hub.docker.com/r/juicedata/mount) is tagged to specify the desired version. **The community edition tags include `latest` and `ce`**, such as `ce-v1.1.2` and `ce-nightly`. The `latest` tag represents the latest community edition, and the `nightly` tag points to the latest development version. For details, see the [tags page](https://hub.docker.com/r/juicedata/mount/tags) on Docker Hub.\n\nBefore you start, you need to prepare [object storage](../reference/how_to_set_up_object_storage.md) and [metadata engine](../reference/how_to_set_up_metadata_engine.md).\n\n#### Create a file system\n\nCreate a file system through a temporary container, for example:\n\n```sh\ndocker run --rm \\\n juicedata/mount:ce-v1.1.2 juicefs format \\\n --storage s3 \\\n --bucket https://xxx.your-s3-endpoint.com \\\n --access-key=ACCESSKEY \\\n --secret-key=SECRETKEY \\\n rediss://user:password@xxx.your-redis-server.com:6379/1 myjfs\n```\n\nReplace `--storage`, `--bucket`, `--access-key`, `--secret-key`, and the metadata engine URL with your own configuration.\n\n#### Mount the file system directly in the container\n\nCreate a container and mount the JuiceFS file system in the container, for example:\n\n```sh\ndocker run --privileged --name myjfs \\\n juicedata/mount:ce-v1.1.2 juicefs mount \\\n rediss://user:password@xxx.your-redis-server.com:6379/1 /mnt\n```\n\nReplace the metadata engine URL with your own configuration. `/mnt` is the mount point and can be modified as needed. Since FUSE is used, `--privileged` permission is also required.\n\n#### Mount the file system through Docker Compose\n\nHere is an example using Docker Compose. Replace the metadata engine URL and mount point with your own configuration.\n\n```yaml\nversion: \"3\"\nservices:\n busybox:\n image: busybox\n command: \"ls /jfs\"\n volumes:\n - ./mnt:/jfs\n depends_on:\n juicefs:\n condition: service_healthy\n\n juicefs:\n image: juicedata/mount:ce-v1.1.2\n container_name: myjfs\n volumes:\n - ./mnt:/mnt:rw,rshared\n cap_add:\n - SYS_ADMIN\n devices:\n - /dev/fuse\n security_opt: \n - apparmor:unconfined\n command: [\"juicefs\", \"mount\", \"rediss://user:password@xxx.your-redis-server.com:6379/1\", \"/mnt\"]\n restart: unless-stopped\n healthcheck:\n test: [\"CMD-SHELL\", \"cat /mnt/.control\"]\n interval: 60s\n retries: 5\n start_period: 30s\n timeout: 10s\n```\n\nIn the container, the JuiceFS file system is mounted to the `/mnt` directory, and the volumes section in the configuration file maps the `/mnt` in the container to the `./mnt` directory on the host, allowing direct access to the JuiceFS file system mounted in the container from the host. At the same time, by combining `depends_on` and `volumes`, the directory mapped to the host machine can be remounted into the container for use.\n\n#### Access the file system through S3 Gateway\n\nHere is an example of exposing JuiceFS for access through S3 Gateway. Replace `MINIO_ROOT_USER`, `MINIO_ROOT_PASSWORD`, the metadata engine URL, and the address and port number to listen on with your own configuration.\n\n```yaml\nversion: \"3\"\nservices:\n s3-gateway:\n image: juicedata/mount:ce-v1.1.2\n container_name: juicefs-s3-gateway\n environment:\n - MINIO_ROOT_USER=your-username\n - MINIO_ROOT_PASSWORD=your-password\n ports:\n - \"9090:9090\"\n command: [\"juicefs\", \"gateway\", \"rediss://user:password@xxx.your-redis-server.com:6379/1\", \"0.0.0.0:9090\"]\n restart: unless-stopped\n```\n\nUse port `9090` on the host to access the S3 Gateway console, and use the same address to read and write the JuiceFS file system through the S3 client or SDK.\n" }, { "path": "docs/en/deployment/nfs.md", "content": "---\ntitle: Create NFS Shares\nsidebar_position: 9\ndescription: Learn how to use the NFS protocol to share directories within the JuiceFS file system.\n---\n\nNFS (Network File System) is a network file-sharing protocol that allows different computers to share files and directories over a network. It was originally developed by Sun Microsystems and is a standard way of file sharing between Unix and Unix-like systems. The NFS protocol enables clients to access remote file systems as if they were local, achieving transparent remote file access.\n\nWhen you need to share directories from the JuiceFS file system through NFS, you can simply use the `juicefs mount` command to mount the file system. Then, you can create NFS shares with the JuiceFS mount point or subdirectories.\n\n:::note\n`juicefs mount` mounts the file system as a local user-space file system through the FUSE interface, making it identical to the local file system in terms of appearance and usage. Hence, it can be directly used to create NFS shares.\n:::\n\n## Step 1. Install NFS\n\nTo configure NFS shares, you need to install the relevant software packages on both the server and client sides. Let's take Ubuntu/Debian systems as an example:\n\n### 1. Server-side installation\n\nCreate a host for NFS sharing (with the JuiceFS file system also mounted on this server).\n\n```shell\nsudo apt install nfs-kernel-server\n```\n\n### 2. Client-side installation\n\nAll Linux hosts that need to access NFS shares should install the client software.\n\n```shell\nsudo apt install nfs-common\n```\n\n## Step 2. Create shares\n\nAssuming the JuiceFS is mounted on the server system at the path `/mnt/myjfs`, if you want to set the `media` subdirectory as an NFS share, you can add the following configuration to the `/etc/exports` file on the server system:\n\n```\n\"/mnt/myjfs/media\" *(rw,sync,no_subtree_check,fsid=1)\n```\n\nThe syntax for NFS share configuration is as follows:\n\n```\n (options)\n```\n\nFor example, if you want to restrict the mounting of this share to hosts in the `192.168.1.0/24` IP range and avoid squashing root privileges, you can modify it as follows:\n\n```\n\"/mnt/myjfs/media\" 192.168.1.0/24(rw,async,no_subtree_check,no_root_squash,fsid=1)\n```\n\n### Share option description\n\n**Explanation of the share options:**\n\n- `rw`: Represents read and write permissions. If read-only access is desired, use `ro`.\n- `sync` and `async`: `sync` enables synchronous writes, meaning that when writing to the NFS share, the client waits for the server's confirmation of successful data write before proceeding with subsequent operations. `async`, on the other hand, allows asynchronous writes. In this mode, the client does not wait for the server's confirmation of successful write before proceeding with subsequent operations.\n- `no_subtree_check`: Disables subtree checking, allowing clients to mount both the parent and child directories of the NFS share. This can reduce some security but improve NFS compatibility. Setting it to `subtree_check` enables subtree checking, allowing clients to only mount the NFS share and its subdirectories.\n- `no_root_squash`: Controls the mapping behavior of the client's root user when accessing the NFS share. By default, when the client mounts the NFS share as root, the server maps it to a non-privileged user (usually nobody or nfsnobody), which is known as root squashing. Enabling this option cancels the root squashing, giving the client the same root user privileges as the server. This option comes with certain security risks and should be used with caution.\n- `fsid`: A file system identifier used to identify different file systems on NFS. In NFSv4, the root directory of NFS is defined as fsid=0, and other file systems need to be numbered uniquely under it. Here, JuiceFS is an externally mounted FUSE file system, so it needs to be assigned a unique identifier.\n\n### Choosing between async and sync modes\n\nFor NFS shares, the sync (synchronous writes) mode can improve data reliability but always requires waiting for the server's confirmation before proceeding with the next operation. This may result in lower write performance. For JuiceFS, which is a cloud-based distributed file system, network latency also needs to be considered. Using the sync mode can often lead to lower write performance due to network latency.\n\nIn most cases, when creating NFS shares with JuiceFS, it is recommended to set the write mode to async (asynchronous writes) to avoid sacrificing write performance. If data reliability must be prioritized and sync mode is necessary, it is recommended to configure JuiceFS with a high-performance SSD as a local cache with sufficient capacity and enable the writeback cache mode.\n" }, { "path": "docs/en/deployment/production_deployment_recommendations.md", "content": "---\nsidebar_position: 1\nslug: /production_deployment_recommendations\ndescription: This article is intended as a reference for users who are about to deploy JuiceFS to a production environment and provides a series of environment configuration recommendations.\n---\n\n# Production Deployment Recommendations\n\nThis document provides deployment recommendations for JuiceFS Community Edition in production environments. It focuses on monitoring metric collection, automatic metadata backup, trash configuration, background tasks of clients, client log rolling, and command-line auto-completion to ensure the stability and reliability of the file system.\n\n## Metrics collection and visualization\n\nIt is necessary to collect monitoring metrics from JuiceFS clients and visualize them using Grafana. This allows for real-time monitoring of file system performance and health status. For detailed instructions, see this [document](../administration/monitoring.md).\n\n## Automatic metadata backup\n\n:::tip\nAutomatic metadata backup is a feature that has been added since JuiceFS v1.0.0.\n:::\n\nMetadata is critical to the JuiceFS file system, and any loss or corruption of metadata may affect a large number of files or even the entire file system. Therefore, metadata must be backed up regularly.\n\nThis feature is enabled by default and the backup interval is 1 hour. The backed-up metadata is compressed and stored in the corresponding object storage, separate from file system data. Backups are performed by JuiceFS clients, which may increase CPU and memory usage during the process. By default, one client is randomly selected for backup operations.\n\nIt is important to note that this feature is disabled when the number of files reaches **one million**. To re-enable it, set a larger backup interval (the `--backup-meta` option). The interval is configured independently for each client. You can use `--backup-meta 0` to disable automatic backup.\n\n:::note\nThe time required for metadata backup depends on the specific metadata engine. Different metadata engines have different performance.\n:::\n\nFor detailed information on automatic metadata backups, see this [document](../administration/metadata_dump_load.md#backup-automatically). Alternatively, you can back up metadata manually. In addition, follow the operational and maintenance recommendations of the metadata engine you are using to back up your data regularly.\n\n## Trash\n\n:::tip\nThe Trash feature has been available since JuiceFS v1.0.0.\n:::\n\nTrash is enabled by default. The retention time for deleted files defaults to 1 day to mitigate the risk of accidental data loss.\n\nHowever, enabling Trash may have side effects. If the application needs to frequently delete files or overwrite them, it will cause the object storage usage to be much larger than the file system. This is because the JuiceFS client retain deleted files and overwritten blocks on the object storage for a certain period. Therefore, it is highly recommended to evaluate workload requirements before deploying JuiceFS in a production environment to configure Trash appropriately. You can configure the retention time as follows (`--trash-days 0` disables Trash):\n\n- For new file systems: set via the `--trash-days ` option of `juicefs format`\n- For existing file systems: modify with the `--trash-days ` option of `juicefs config`\n\nFor more information on Trash, see this [document](../security/trash.md).\n\n## Client background tasks\n\nThe JuiceFS file system maintains background tasks through clients, which can automatically execute cleaning tasks such as deleting pending files and objects, purging expired files and fragments from Trash, and terminating long-stalled client sessions.\n\nAll clients of the same JuiceFS volume share a set of background tasks during runtime. Each task is executed at regular intervals, with the client chosen randomly. Background tasks include:\n\n- Cleaning up files and objects to be deleted\n- Clearing out-of-date files and fragments in Trash\n- Cleaning up stale client sessions\n- Automatic backup of metadata\n\nSince these tasks take up some resources when executed, you can set the `--no-bgjob` option to disable them for clients with heavy workload.\n\n:::note\nMake sure that at least one JuiceFS client can execute background tasks.\n:::\n\n## Client log rotation\n\nWhen running a JuiceFS mount point in the background, the client outputs logs to a local file by default. The path to the local log file is slightly different depending on the user running the process:\n\n- For the root user, the path is `/var/log/juicefs.log`.\n- For others, the path is `$HOME/.juicefs/juicefs.log`.\n\nThe local log file is not rotated by default and needs to be configured manually in production to prevent excessive disk space usage. The following is a configuration example for log rotation:\n\n```text title=\"/etc/logrotate.d/juicefs\"\n/var/log/juicefs.log {\n daily\n rotate 7\n compress\n delaycompress\n missingok\n notifempty\n copytruncate\n}\n```\n\nYou can check the correctness of the configuration file with the `logrotate -d` command:\n\n```shell\nlogrotate -d /etc/logrotate.d/juicefs\n```\n\nFor details about the logrotate configuration, see this [link](https://linux.die.net/man/8/logrotate).\n\n## Command line auto-completion\n\nJuiceFS provides command line auto-completion scripts for Bash and Zsh to facilitate the use of `juicefs` commands. For details, see this [document](../reference/command_reference.mdx#auto-completion) for details.\n" }, { "path": "docs/en/deployment/python_sdk.md", "content": "---\ntitle: Python SDK\nsidebar_position: 6\n---\n\nThe JuiceFS Community Edition introduced the Python SDK in v1.3.0, making it suitable for containerized or virtualized environments where FUSE mounting is not available. The Python SDK also implements the `fsspec` interface, enabling easy integration with frameworks such as Ray.\n\n## Compilation\n\nYou can compile the Python SDK directly in your current working environment or use a Docker container. Both methods require you to first clone the repository and navigate to the SDK directory.\n\n```bash\n# Clone JuiceFS repository\ngit clone https://github.com/juicedata/juicefs.git\n# Enter JuiceFS directory\ncd juicefs/sdk/python\n```\n\n### Direct Compilation\n\nDirect compilation requires `go1.20+` and `python3` environments.\n\n#### Step 1: Compile libjfs.so\n\n```bash\ngo build -buildmode c-shared -ldflags=\"-s -w\" -o juicefs/juicefs/libjfs.so ../java/libjfs\n```\n\nThe compiled `libjfs.so` and `libjfs.h` files will be in the `sdk/python/juicefs/juicefs` directory.\n\n#### Step 2: Compile Python SDK\n\n```bash\ncd juicefs && python3 -m build -w\n```\n\nThe compiled Python SDK will be in the `juicefs/sdk/python/dist` directory, named `juicefs-1.3.0-py3-none-any.whl`.\n\n### Docker Compilation\n\nUsing Docker containers for compilation requires `Docker`, `make`, and `go1.20+` installed on your system.\n\n#### Step 1: Build Docker image\n\n```bash\n# For arm64\nmake arm-builder\n\n# For amd64\nmake builder\n```\n\n#### Step 2: Compile Python SDK\n\n```bash\nmake juicefs\n```\n\nThe compiled Python SDK will be in the `juicefs/sdk/python/dist` directory, named `juicefs-1.3.0-py3-none-any.whl`.\n\n### Compilation Error Handling\n\nIf you encounter an error like `sed: 1: \"juicefs/setup.py\": invalid command code j` during compilation, you can try commenting out the `sed`-related commands in the `Makefile`.\n\n## Installation and Usage\n\n### Installing the SDK\n\nCopy the compiled `juicefs-1.3.0-py3-none-any.whl` file to the target machine and install it using `pip`:\n\n```bash\npip install juicefs-1.3.0-py3-none-any.whl\n```\n\n### Preparing the File System\n\n:::tip\nJuiceFS Python SDK currently does not support formatting a file system, so please ensure you have already created a JuiceFS file system before use.\n:::\n\nLet's assume there is a pre-created file system named `myfs` with metadata engine URL `redis://192.168.1.8/0`.\n\n### Using the Client\n\nThe `Client` class implementation is similar to Python's io module.\n\nYou can instantiate a JuiceFS client with the following code, where the `name` parameter is the file system name and the `meta` parameter is the URL of the metadata engine. The `name` parameter must exist but can be an empty string or `None`.\n\n```python\nfrom juicefs import Client\n\n# Create JuiceFS client\njfs = Client(name='', meta='redis://192.168.1.8/0')\n\n# List files in a directory\njfs.listdir('/')\n```\n\n### Using fsspec\n\nJuiceFS Python SDK also supports the `fsspec` interface to operate the JuiceFS file system.\n\n```bash\n# Install fsspec\npip install fsspec\n```\n\nUsing `fsspec` is similar to using the `Client` class, but you need to specify `jfs` or `juicefs` as the file system type.\n\n```python\nimport fsspec\nfrom juicefs.spec import JuiceFS\n\njfs = fsspec.filesystem('jfs', name='', meta='redis://192.168.1.8/0')\n\n# List files in a directory\njfs.ls('/')\n```\n\n### Getting Help Information\n\nYou can use the `help()` function to get help information for classes and methods.\n\n```python\nimport juicefs\n\nhelp(juicefs.Client)\n```\n\nYou can also use the `dir()` function to get a list of classes and methods.\n\n```python\nimport juicefs\n\ndir(juicefs.Client)\n```\n" }, { "path": "docs/en/deployment/samba.md", "content": "---\ntitle: Create Samba Shares\nsidebar_position: 8\ndescription: Learn how to share directories in the JuiceFS file system through Samba.\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\nSamba is an open-source software suite that implements the SMB/CIFS (Server Message Block / Common Internet File System) protocol, which is a commonly used file-sharing protocol in Windows systems. With Samba, you can create shared directories on Linux/Unix servers, allowing Windows computers to access and use these shared resources over the network.\n\nTo create a shared folder on a Linux system with Samba installed, you can edit the `smb.conf` configuration file. Once configured, Windows and macOS systems can access and read/write the shared folder using their file managers. Linux needs to install the Samba client for access.\n\nWhen you need to share directories from the JuiceFS file system through Samba, you can simply use the `juicefs mount` command to mount the file system. Then, you can create Samba shares with the JuiceFS mount point or subdirectories.\n\n:::note\n`juicefs mount` mounts the file system as a local user-space file system through the FUSE interface, making it identical to the local file system in terms of appearance and usage. Hence, it can be directly used to create Samba shares.\n:::\n\n## Step 1: Install Samba\n\nMost Linux distributions provide Samba through their package managers.\n\n\n\n\n```shell\nsudo apt install samba\n```\n\n\n \n\n```shell\nsudo dnf install samba\n```\n\n\n\n\nIf you need to configure AD/DC (Active Directory / Domain Controller), additional software packages need to be installed. For more details, refer to the [Samba Official Installation Guide](https://wiki.samba.org/index.php/Distribution-specific_Package_Installation).\n\n## Step 2: Enable JuiceFS extended attribute (xattr) support\n\nAccording to the [Samba official documentation](https://wiki.samba.org/index.php/File_System_Support#File_systems_without_xattr_support), it is recommended to use file systems that support extended attributes (xattr). To enable extended attribute support for JuiceFS during the mount process, use the `--enable-xattr` option. For example:\n\n```shell\nsudo juicefs mount -d --enable-xattr sqlite3://myjfs.db /mnt/myjfs\n```\n\nFor cases where you configure automatic mounting through `/etc/fstab`, you can add the `enable-xattr` option to the mount options section. For example:\n\n```ini\n# \nredis://127.0.0.1:6379/0 /mnt/myjfs juicefs _netdev,max-uploads=50,writeback,cache-size=1024000,enable-xattr 0 0\n```\n\n### Knowledge extension: why Samba requires file system support for extended attributes\n\nSamba is software designed for Linux/Unix systems, serving file sharing to Windows systems. In Windows systems, many files and directories have additional metadata, for example, file authors, keywords, and icon positions. This information is typically stored outside the POSIX file system and requires xattr format for storage in Windows. To ensure that these files can be correctly stored in Linux systems, Samba recommends using file systems that support extended attributes when creating shares.\n\n## Step 3: Create a Samba share\n\nAssuming the JuiceFS mount point is `/mnt/myjfs`, if you want to create a Samba share for the `media` directory within it, you can configure it as follows:\n\n```ini\n[Media]\n path = /mnt/myjfs/media\n guest ok = no\n read only = no\n browseable = yes\n```\n\n## Share for macOS\n\nApple macOS systems support direct access to Samba shares. Similar to Windows, macOS also has additional metadata (e.g., icon positions, Spotlight search) that needs to be saved using xattr. Samba version 4.9 and above have the support for macOS extended attributes enabled by default.\n\nIf your Samba version is lower than 4.9, you need to add the `ea support = yes` option to the [global] section of the Samba configuration to enable extended attribute support for macOS. Edit the configuration file `/etc/samba/smb.conf`, for example:\n\n```ini\n[global]\n workgroup = SAMBA\n security = user\n passdb backend = tdbsam\n ea support = yes\n```\n\n## User management in Samba\n\nSamba has its own user database, independent of the operating system users. However, since Samba shares directories from the system, appropriate user permissions are required to read and write files.\n\n### Create Samba users\n\nWhen creating users for Samba, it is required that the user already exists in the system, as Samba will automatically map the Samba user to the same-named system user with corresponding permissions.\n\n- If the user already exists in the system, assuming the system account is \"herald,\" you can create a Samba account for it as follows:\n\n ```shell\n sudo smbpasswd -a herald\n ```\n\n Follow the on-screen prompts to set the password. The Samba account can have a different password than the system user.\n\n- If you need to create a new user, taking the example of creating a user named \"abc\":\n\n 1. Create a user:\n\n ```shell\n sudo adduser abc\n ```\n\n 2. Create a corresponding Samba user with the same name:\n\n ```shell\n sudo smbpasswd -a abc\n ```\n\n### View created Samba users\n\n`pdbedit` is a built-in tool in Samba used to manage the Samba user database. You can use this tool to list all the created Samba users:\n\n```shell\nsudo pdbedit -L\n```\n\nIt will display a list of all created Samba users, including their usernames, security identifiers (SIDs), group membership, and other related information.\n" }, { "path": "docs/en/deployment/webdav.md", "content": "---\ntitle: Deploy WebDAV Server\nsidebar_position: 5\n---\n\nWebDAV is an extension of the HTTP protocol, a sharing protocol that facilitates collaborative editing and management of documents on a network between multiple users. WebDAV client support is built into many tools involved in file editing and synchronization, macOS Finder, and the file managers of some Linux distributions.\n\nJuiceFS supports accessing through the WebDAV protocol, which is very convenient for macOS and other operating systems that do not have native FUSE support.\n\n## Pre-requisites\n\nBefore you can configure a WebDAV server, you need to [create a JuiceFS file system](../getting-started/standalone.md#juicefs-format).\n\n## Anonymous WebDAV\n\nFor security insensitive environments such as standalone or intranet, anonymous WebDAV without authentication can be configured with the following command format.\n\n```shell\njuicefs webdav META-URL LISTENING-ADDRESS:PORT\n```\n\nFor example, enable the WebDAV access protocol for a JuiceFS file system:\n\n```shell\nsudo juicefs webdav sqlite3://myjfs.db 192.168.1.8:80\n```\n\nWebDAV server needs to be accessed through the set listening address and port, such as the above example uses the IP address `192.168.1.8` of the intranet, and the standard Web port number `80`, when accessing without specifying the port, directly access `http://192.168.1.8`.\n\nIf you use another port number, you need to specify it explicitly in the address, for example, if you listen to port `9007`, the access address should be `http://192.168.1.8:9007`.\n\n:::tip\nDo not use \"Guest\" identity when accessing anonymous WebDAV using macOS's Finder. Please use \"Registered User\" identity, user name can enter any character, password can be empty, and then connect directly.\n:::\n\n## WebDAV with authentication\n\n:::info\nJuiceFS v1.0.3 and previous versions do not support authentication features.\n:::\n\nThe WebDAV authentication feature of JuiceFS requires setting the user name (`WEBDAV_USER`) and password (`WEBDAV_PASSWORD`) through environment variables, e.g.:\n\n```shell\nexport WEBDAV_USER=user\nexport WEBDAV_PASSWORD=mypassword\nsudo juicefs webdav sqlite3://myjfs.db 192.168.1.8:80\n```\n\n## Enable HTTPS support\n\nJuiceFS supports configuring WebDAV server protected by the HTTPS protocol, specifying certificates and private keys through `--cert-file` and `--key-file` options, either using a certificate issued by a trusted digital certificate authority CA or using OpenSSL to create self-signed certificate.\n\n### Self-signed certificate\n\nTo create a private key and certificate using OpenSSL.\n\n1. Generate server private key\n\n ```shell\n openssl genrsa -out client.key 4096\n ```\n\n2. Generate Certificate Signing Request (CSR)\n\n ```shell\n openssl req -new -key client.key -out client.csr\n ```\n\n3. Issuing certificates using CSR\n\n ```shell\n openssl x509 -req -days 365 -in client.csr -signkey client.key -out client.crt\n ```\n\nThe above command will produce the following files in the current directory:\n\n- `client.key`: Server private Key\n- `client.csr`: Certificate Signing Request file\n- `client.crt`: Self-signed certificate\n\nTo create a WebDAV server you need to use `client.key` and `client.crt`, e.g.\n\n```shell\nsudo juicefs webdav \\\n --cert-file ./client.crt \\\n --key-file ./client.key \\\n sqlite3://myjfs.db 192.168.1.8:443\n```\n\nWith HTTPS support enabled, the listening port number can be changed to the standard HTTPS port number `443`, and then the `https://` protocol is used instead, so that the port number does not need to be specified when accessing, for example: `https://192.168.1.8`.\n\nLikewise, if a non-HTTPS standard port number is set, it should be explicitly specified in the access address, e.g., if you set a port to listen on `9999`, the access address should be `https://192.168.1.8:9999`.\n" }, { "path": "docs/en/development/contributing_guide.md", "content": "---\ntitle: Contributing Guide\nsidebar_position: 1\ndescription: JuiceFS is open source software and the code is contributed and maintained by developers worldwide. Learn how to participate in this article.\n---\n\n## Guidelines\n\n- Before starting work on a feature or bug fix, search GitHub or reach out to us via GitHub or Slack, make sure no one else is already working on it and we'll ask you to open a GitHub issue if necessary.\n- Before contributing, use the GitHub issue to discuss the feature and reach an agreement with the core developers.\n- For major feature updates, write a design document to help the community understand your motivation and solution.\n- Find issues with the label [\"kind/good-first-issue\"](https://github.com/juicedata/juicefs/labels/kind%2Fgood-first-issue) or [\"kind/help-wanted\"](https://github.com/juicedata/juicefs/labels/kind%2Fhelp-wanted).\n\nRead [internals](./internals.md) for important data structure references.\n\n## Coding style\n\n- We're following [\"Effective Go\"](https://go.dev/doc/effective_go) and [\"Go Code Review Comments\"](https://github.com/golang/go/wiki/CodeReviewComments).\n- Use `go fmt` to format your code before committing. You can find information in editor support for Go tools in [\"IDEs and Plugins for Go\"](https://github.com/golang/go/wiki/IDEsAndTextEditorPlugins).\n- Every new source file must begin with a license header.\n- Install [pre-commit](https://pre-commit.com) and use it to set up a pre-commit hook for static analysis. Just run `pre-commit install` in the root of the repo.\n\n## Sign the CLA\n\nBefore you can contribute to JuiceFS, you will need to sign the [Contributor License Agreement](https://cla-assistant.io/juicedata/juicefs). There're a CLA assistant to guide you when you first time submit a pull request.\n\n## What is a good PR\n\n- Presence of unit tests\n- Adherence to the coding style\n- Adequate in-line comments\n- Explanatory commit message\n\n## Contribution flow\n\n1. Create a topic branch from where to base the contribution. This is usually `main`.\n1. Make commits of logical units.\n1. Make sure commit messages are in the proper format.\n1. Push changes in a topic branch to a personal fork of the repository.\n1. Submit a pull request to [`juicedata/juicefs`](https://github.com/juicedata/juicefs/compare). The PR should link to one issue which either created by you or others.\n1. The PR must receive approval from at least one maintainer before it be merged.\n" }, { "path": "docs/en/development/internals.md", "content": "---\ntitle: Internals\nsidebar_position: 4\nslug: /internals\n---\n\nThis article introduces implementation details of JuiceFS, use this as a reference if you'd like to contribute. The content below is based on JuiceFS v1.0.0, metadata version v1.\n\nBefore digging into source code, you should read [Data Processing Workflow](../introduction/io_processing.md).\n\n## Keyword Definition\n\nHigh level concepts:\n\n- File system: i.e. JuiceFS Volume, represents a separate namespace. Files can be moved freely within the same file system, while data copies are required between different file systems.\n- Metadata engine: A supported database instance of your choice, that stores and manages file system metadata. There are three categories of metadata engines currently supported by JuiceFS.\n - Redis: Redis and various protocol-compatible services\n - SQL: MySQL, PostgreSQL, SQLite, etc.\n - TKV: TiKV, BadgerDB, etc.\n- Datastore: Object storage service that stores and manages file system data, such as Amazon S3, Aliyun OSS, etc. It can also be served by other storage systems that are compatible with object storage semantics, such as local file systems, Ceph RADOS, TiKV, etc.\n- Client: can be in various forms, such as mount process, S3 gateway, WebDAV server, Java SDK, etc.\n- File: refers to all types of files in general in this documentation, including regular files, directory files, link files, device files, etc.\n- Directory: is a special kind of file used to organize the tree structure, and its contents are an index to a set of other files.\n\nLow level concepts (learn more at [Data Processing Workflow](../introduction/io_processing.md)):\n\n- Chunk: Logical concept, file is split into 64MiB chunks, allowing fast lookups during file reads;\n- Slice: Logical concept, basic unit for file writes. Block's purpose is to improve read speed, and slice exists to improve file edits and random writes. All file writes are assigned a new or existing slice, and when file is read, what application sees is the consolidated view of all slices.\n- Block: A chunk contains one or more blocks (4MiB by default), block is the basic storage unit in object storage. JuiceFS Client reads multiple blocks concurrently which greatly improves read performance. Apart from this, block is also the basic storage unit on disk cache, so this design improves cache eviction efficiency. Apart from this, block is immutable, all file edits is achieved through new blocks: after file edit, new blocks are uploaded to object storage, and new slices are appended to the slice list in the corresponding file metadata;\n\n## Learn source code {#source-code-structure}\n\nAssuming you're already familiar with Go, as well as [JuiceFS architecture](https://juicefs.com/docs/community/architecture), this is the overall code structure:\n\n* [`cmd`](https://github.com/juicedata/juicefs/tree/main/cmd) is the top-level entrance, all JuiceFS functionalities is rooted here, e.g. the `juicefs format` command resides in `cmd/format.go`；\n* [`pkg`](https://github.com/juicedata/juicefs/tree/main/pkg) is actual implementation:\n * `pkg/fuse/fuse.go` provides abstract FUSE API;\n * `pkg/vfs` contains actual FUSE implementation, Metadata requests are handled in `pkg/meta`, read requests are handled in `pkg/vfs/reader.go` and write requests are handled by `pkg/vfs/writer.go`;\n * `pkg/meta` directory is the implementation of all metadata engines, where:\n * `pkg/meta/interface.go` is the interface definition for all types of metadata engines\n * `pkg/meta/redis.go` is the interface implementation of Redis database\n * `pkg/meta/sql.go` is the interface definition and general interface implementation of relational database, and the implementation of specific databases is in a separate file (for example, the implementation of MySQL is in `pkg/meta/sql_mysql.go`)\n * `pkg/meta/tkv.go` is the interface definition and general interface implementation of the KV database, and the implementation of a specific database is in a separate file (for example, the implementation of TiKV is in `pkg/meta/tkv_tikv.go`)\n * `pkg/object` contains all object storage integration code;\n* [`sdk/java`](https://github.com/juicedata/juicefs/tree/main/sdk/java) is the Hadoop Java SDK, it uses `sdk/java/libjfs` through JNI.\n\n## FUSE interface implementation {#fuse-interface-implementation}\n\nJuiceFS implements a userspace file system based on [FUSE](https://en.wikipedia.org/wiki/Filesystem_in_Userspace) (Filesystem in Userspace), and the implementation library [`libfuse`](https://github.com/libfuse/libfuse) provides two APIs: high-level API and low-level API, where the high-level API is based on file name and path, and the low-level API is based on inode.\n\nJuiceFS is implemented based on low-level API (in fact JuiceFS does not depend on `libfuse`, but [`go-fuse`](https://github.com/hanwen/go-fuse)), because this is the same set of APIs used by kernel VFS when interacting with FUSE. If JuiceFS were to use high level API, it'll have to implement the VFS tree within `libfuse`, and then expose path based API. This method works better for systems that already expose path based APIs (e.g. HDFS, S3). If metadata itself implements file / directory tree based on inode, the inode → path → inode conversions will have an impact on performance (this is the reason why FUSE API for HDFS doesn't perform well). JuiceFS Metadata directly implements file tree and API based on inode, so naturally it uses FUSE low level API.\n\n## Metadata Structure\n\nFile systems are usually organized in a tree structure, where nodes represent files and edges represent directory containment relationships. There are more than ten metadata structures in JuiceFS. Most of them are used to maintain the organization of file tree and properties of individual nodes, while the rest are used to manage system configuration, client sessions, asynchronous tasks, etc. All metadata structures are described below.\n\n### General Structure\n\n#### Setting\n\nIt is created when the `juicefs format` command is executed, and some of its fields can be modified later by the `juicefs config` command. The structure is specified as follows.\n\n```go\ntype Format struct {\n Name string\n UUID string\n Storage string\n Bucket string\n AccessKey string `json:\",omitempty\"`\n SecretKey string `json:\",omitempty\"`\n SessionToken string `json:\",omitempty\"`\n BlockSize int\n Compression string `json:\",omitempty\"`\n Shards int `json:\",omitempty\"`\n HashPrefix bool `json:\",omitempty\"`\n Capacity uint64 `json:\",omitempty\"`\n Inodes uint64 `json:\",omitempty\"`\n EncryptKey string `json:\",omitempty\"`\n KeyEncrypted bool `json:\",omitempty\"`\n TrashDays int `json:\",omitempty\"`\n MetaVersion int `json:\",omitempty\"`\n MinClientVersion string `json:\",omitempty\"`\n MaxClientVersion string `json:\",omitempty\"`\n EnableACL bool\n}\n```\n\n- Name: name of the file system, specified by the user when formatting\n- UUID: unique ID of the file system, automatically generated by the system when formatting\n- Storage: short name of the object storage used to store data, such as `s3`, `oss`, etc.\n- Bucket: the bucket path of the object storage\n- AccessKey: access key used to access the object storage\n- SecretKey: secret key used to access the object storage\n- SessionToken: session token used to access the object storage, as some object storage supports the use of temporary token to obtain permission for a limited time\n- BlockSize: size of the data block when splitting the file (the default is 4 MiB)\n- Compression: compression algorithm that is executed before uploading data blocks to the object storage (the default is no compression)\n- Shards: number of buckets in the object storage, only one bucket by default; when Shards > 1, data objects will be randomly hashed into Shards buckets\n- HashPrefix: whether to set a hash prefix for the object name, false by default\n- Capacity: quota limit for the total capacity of the file system\n- Inodes: quota limit for the total number of files in the file system\n- EncryptKey: the encrypted private key of the data object, which can be used only if the data encryption function is enabled\n- KeyEncrypted: whether the saved key is encrypted or not, by default the SecretKey, EncryptKey and SessionToken will be encrypted\n- TrashDays: number of days the deleted files are kept in trash, the default is 1 day\n- MetaVersion: the version of the metadata structure, currently V1 (V0 and V1 are the same)\n- MinClientVersion: the minimum client version allowed to connect, clients earlier than this version will be denied\n- MaxClientVersion: the maximum client version allowed to connect\n- EnableACL: enable ACL or not\n\nThis structure is serialized into JSON format and stored in the metadata engine.\n\n#### Counter\n\nMaintains the value of each counter in the system and the start timestamps of some background tasks, specifically\n\n- usedSpace: used capacity of the file system\n- totalInodes: number of used files in the file system\n- nextInode: the next available inode number (in Redis, the maximum inode number currently in use)\n- nextChunk: the next available sliceId (in Redis, the largest sliceId currently in use)\n- nextSession: the maximum SID (sessionID) currently in use\n- nextTrash: the maximum trash inode number currently in use\n- nextCleanupSlices: timestamp of the last check on the cleanup of residual slices\n- lastCleanupSessions: timestamp of the last check on the cleanup of residual stale sessions\n- lastCleanupFiles: timestamp of the last check on the cleanup of residual files\n- lastCleanupTrash: timestamp of the last check on the cleanup of trash\n\n#### Session\n\nRecords the session IDs of clients connected to this file system and their timeouts. Each client sends a heartbeat message to update the timeout, and those who have not updated for a long time will be automatically cleaned up by other clients.\n\n:::tip\nRead-only clients cannot write to the metadata engine, so their sessions **will not** be recorded.\n:::\n\n#### SessionInfo\n\nRecords specific metadata of the client session so that it can be viewed with the `juicefs status` command. This is specified as\n\n```go\ntype SessionInfo struct {\n Version string // JuiceFS version\n HostName string // Host name\n MountPoint string // path to mount point. S3 gateway and WebDAV server are \"s3gateway\" and \"webdav\" respectively\n ProcessID int // Process ID\n}\n```\n\nThis structure is serialized into JSON format and stored in the metadata engine.\n\n#### Node\n\nRecords attribute information of each file, as follows\n\n```go\ntype Attr struct {\n Flags uint8 // reserved flags\n Typ uint8 // type of a node\n Mode uint16 // permission mode\n Uid uint32 // owner id\n Gid uint32 // group id of owner\n Rdev uint32 // device number\n Atime int64 // last access time\n Mtime int64 // last modified time\n Ctime int64 // last change time for meta\n Atimensec uint32 // nanosecond part of atime\n Mtimensec uint32 // nanosecond part of mtime\n Ctimensec uint32 // nanosecond part of ctime\n Nlink uint32 // number of links (sub-directories or hardlinks)\n Length uint64 // length of regular file\n\n Parent Ino // inode of parent; 0 means tracked by parentKey (for hardlinks)\n Full bool // the attributes are completed or not\n KeepCache bool // whether to keep the cached page or not\n\n AccessACL uint32 // access ACL id (identical ACL rules share the same access ACL ID.)\n DefaultACL uint32 // default ACL id (default ACL and the access ACL share the same cache and store)\n}\n```\n\nThere are a few fields that need clarification.\n\n- Atime/Atimensec: See [`--atime-mode`](../reference/command_reference.mdx#mount-metadata-options)\n- Nlink\n - Directory file: initial value is 2 ('.' and '..'), add 1 for each subdirectory\n - Other files: initial value is 1, add 1 for each hard link created\n- Length\n - Directory file: fixed at 4096\n - Soft link (symbolic link) file: the string length of the path to which the link points\n - Other files: the length of the actual content of the file\n\nThis structure is usually encoded in binary format and stored in the metadata engine.\n\n#### Edges\n\nRecords information on each edge in the file tree, as follows\n\n```\nparentInode, name -> type, inode\n```\n\nwhere parentInode is the inode number of the parent directory, and the others are the name, type, and inode number of the child files, respectively.\n\n#### LinkParent\n\nRecords the parent directory of some files. The parent directory of most files is recorded in the Parent field of the attribute; however, for files that have been created with hard links, there may be more than one parent directory, so the Parent field is set to 0, and all parent inodes are recorded independently, as follows\n\n```\ninode -> parentInode, links\n```\n\nwhere links is the count of the parentInode, because multiple hard links can be created in the same directory, and these hard links share one inode.\n\n#### Chunk\n\nRecords information on each Chunk, as follows\n\n```\ninode, index -> []Slices\n```\n\nwhere inode is the inode number of the file to which the Chunk belongs, and index is the number of all Chunks in the file, starting from 0. The Chunk value is an array of Slices. Each Slice represents a piece of data written by the client, and is appended to this array in the order of writing time. When there is an overlap between different Slices, the later Slice is used.\n\n```go\ntype Slice struct {\n Pos uint32 // offset of the Slice in the Chunk\n ID uint64 // ID of the Slice, globally unique\n Size uint32 // size of the Slice\n Off uint32 // offset of valid data in this Slice\n Len uint32 // size of valid data in this Slice\n}\n```\n\nThis structure is encoded and saved in binary format, taking up 24 bytes.\n\n#### SliceRef {#sliceref}\n\nRecords the reference count of a Slice, as follows\n\n```\nsliceId, size -> refs\n```\n\nSince the reference count of most Slices is 1, to reduce the number of related entries in the database, the actual value minus 1 is used as the stored count value in Redis and TKV. In this way, most of the Slices have a refs value of 0, and there is no need to create related entries in the database.\n\n#### Symlink\n\nRecords the location of the softlink file, as follows\n\n```\ninode -> target\n```\n\n#### Xattr\n\nRecords extended attributes (Key-Value pairs) of a file, as follows\n\n```\ninode, key -> value\n```\n\n#### Flock\n\nRecords BSD locks (flock) of a file, specifically.\n\n```\ninode, sid, owner -> ltype\n```\n\nwhere `sid` is the client session ID, `owner` is a string of numbers, usually associated with a process, and `ltype` is the lock type, which can be 'R' or 'W'.\n\n#### Plock\n\nRecord POSIX record locks (fcntl) of a file, specifically\n\n```\ninode, sid, owner -> []plockRecord\n```\n\nHere plock is a more fine-grained lock that can only lock a certain segment of the file.\n\n```go\ntype plockRecord struct {\n ltype uint32 // lock type\n pid uint32 // process ID\n start uint64 // start position of the lock\n end uint64 // end position of the lock\n}\n```\n\nThis structure is encoded and stored in binary format, taking up 24 bytes.\n\n#### DelFiles\n\nRecords the list of files to be cleaned. It is needed as data cleanup of files is an asynchronous and potentially time-consuming operation that can be interrupted by other factors.\n\n```\ninode, length -> expire\n```\n\nwhere length is the length of the file and expire is the time when the file was deleted.\n\n#### DelSlices\n\nRecords delayed deleted Slices. When the Trash feature is enabled, old Slices deleted by the Slice Compaction will be kept for the same amount of time as the Trash configuration, to be available for data recovery if necessary.\n\n```\nsliceId, deleted -> []slice\n```\n\nwhere sliceId is the ID of the new slice after compaction, deleted is the timestamp of the compaction, and the mapped value is the list of all old slices that were compacted. Each slice only encodes its ID and size.\n\n```go\ntype slice struct {\n ID uint64\n Size uint32\n}\n```\n\nThis structure is encoded and stored in binary format, taking up 12 bytes.\n\n#### Sustained\n\nRecords the list of files that need to be kept temporarily during the session. If a file is still open when it is deleted, the data cannot be cleaned up immediately, but needs to be held temporarily until the file is closed.\n\n```\nsid -> []inode\n```\n\nwhere `sid` is the session ID and the mapped value is the list of temporarily undeleted file inodes.\n\n### Redis\n\nThe common format of keys in Redis is `${prefix}${JFSKey}`, where\n\n- In standalone mode the prefix is an empty string, while in cluster mode it is a database number enclosed in curly braces, e.g. \"{10}\"\n- JFSKey is the Key of different data structures in JuiceFS, which are listed in the subsequent subsections\n\nIn Redis Keys, integers (including inode numbers) are represented as decimal strings if not otherwise specified.\n\n#### Setting {#redis-setting}\n\n- Key: `setting`\n- Value Type: String\n- Value: file system formatting information in JSON format\n\n#### Counter\n\n- Key: counter name\n- Value Type: String\n- Value: value of the counter, which is actually an integer\n\n#### Session\n\n- Key: `allSessions`\n- Value Type: Sorted Set\n- Value: all non-read-only sessions connected to this file system. In Set,\n - Member: session ID\n - Score: timeout point of this session\n\n#### SessionInfo\n\n- Key: `sessionInfos`\n- Value Type: Hash\n- Value: basic meta-information on all non-read-only sessions. In Hash,\n - Key: session ID\n - Value: session information in JSON format\n\n#### Node {#redis-node}\n\n- Key: `i${inode}`\n- Value Type: String\n- Value: binary encoded file attribute\n\n#### Edge {#redis-edge}\n\n- Key: `d${inode}`\n- Value Type: Hash\n- Value: all directory entries in this directory. In Hash,\n - Key: file name\n - Value: binary encoded file type and inode number\n\n#### LinkParent\n\n- Key: `p${inode}`\n- Value Type: Hash\n- Value: all parent inodes of this file. in Hash.\n - Key: parent inode\n - Value: count of this parent inode\n\n#### Chunk {#redis-chunk}\n\n- Key: `c${inode}_${index}`\n- Value Type: List\n- Value: list of Slices, each Slice is binary encoded with 24 bytes\n\n#### SliceRef\n\n- Key: `sliceRef`\n- Value Type: Hash\n- Value: the count value of all Slices to be recorded. In Hash,\n - Key: `k${sliceId}_${size}`\n - Value: reference count of this Slice minus 1 (if the reference count is 1, the corresponding entry is generally not created)\n\n#### Symlink\n\n- Key: `s${inode}`\n- Value Type: String\n- Value: path that the symbolic link points to\n\n#### Xattr\n\n- Key: `x${inode}`\n- Value Type: Hash\n- Value: all extended attributes of this file. In Hash,\n - Key: name of the extended attribute\n - Value: value of the extended attribute\n\n#### Flock\n\n- Key: `lockf${inode}`\n- Value Type: Hash\n- Value: all flocks of this file. In Hash,\n - Key: `${sid}_${owner}`, owner in hexadecimal\n - Value: lock type, can be 'R' or 'W'\n\n#### Plock {#redis-plock}\n\n- Key: `lockp${inode}`\n- Value Type: Hash\n- Value: all plocks of this file. In Hash,\n - Key: `${sid}_${owner}`, owner in hexadecimal\n - Value: array of bytes, where every 24 bytes corresponds to a [plockRecord](#plock)\n\n#### DelFiles\n\n- Key：`delfiles`\n- Value Type: Sorted Set\n- Value: list of all files to be cleaned. In Set,\n - Member: `${inode}:${length}`\n - Score: the timestamp when this file was added to the set\n\n#### DelSlices {#redis-delslices}\n\n- Key: `delSlices`\n- Value Type: Hash\n- Value: all Slices to be cleaned. In Hash,\n - Key: `${sliceId}_${deleted}`\n - Value: array of bytes, where every 12 bytes corresponds to a [slice](#delslices)\n\n#### Sustained\n\n- Key: `session${sid}`\n- Value Type: List\n- Value: list of files temporarily reserved in this session. In List,\n - Member: inode number of the file\n\n### SQL\n\nMetadata is stored in different tables by type, and each table is named with `jfs_` followed by its specific structure name to form the table name, e.g. `jfs_node`. Some tables use `Id` with the `bigserial` type as primary keys to ensure that each table has a primary key, and the `Id` columns do not contain actual information.\n\n#### Setting {#sql-setting}\n\n```go\ntype setting struct {\n Name string `xorm:\"pk\"`\n Value string `xorm:\"varchar(4096) notnull\"`\n}\n```\n\nThere is only one entry in this table with \"format\" as Name and file system formatting information in JSON as Value.\n\n#### Counter\n\n```go\ntype counter struct {\n Name string `xorm:\"pk\"`\n Value int64 `xorm:\"notnull\"`\n}\n```\n\n#### Session\n\n```go\ntype session2 struct {\n Sid uint64 `xorm:\"pk\"`\n Expire int64 `xorm:\"notnull\"`\n Info []byte `xorm:\"blob\"`\n}\n```\n\n#### SessionInfo\n\nThere is no separate table for this, but it is recorded in the `Info` column of `session2`.\n\n#### Node {#sql-node}\n\n```go\ntype node struct {\n Inode Ino `xorm:\"pk\"`\n Type uint8 `xorm:\"notnull\"`\n Flags uint8 `xorm:\"notnull\"`\n Mode uint16 `xorm:\"notnull\"`\n Uid uint32 `xorm:\"notnull\"`\n Gid uint32 `xorm:\"notnull\"`\n Atime int64 `xorm:\"notnull\"`\n Mtime int64 `xorm:\"notnull\"`\n Ctime int64 `xorm:\"notnull\"`\n Nlink uint32 `xorm:\"notnull\"`\n Length uint64 `xorm:\"notnull\"`\n Rdev uint32\n Parent Ino\n AccessACLId uint32 `xorm:\"'access_acl_id'\"`\n DefaultACLId uint32 `xorm:\"'default_acl_id'\"`\n}\n```\n\nMost of the fields are the same as [Attr](#node), but the timestamp precision is lower, i.e., Atime/Mtime/Ctime are in microseconds.\n\n#### Edge {#sql-edge}\n\n```go\ntype edge struct {\n Id int64 `xorm:\"pk bigserial\"`\n Parent Ino `xorm:\"unique(edge) notnull\"`\n Name []byte `xorm:\"unique(edge) varbinary(255) notnull\"`\n Inode Ino `xorm:\"index notnull\"`\n Type uint8 `xorm:\"notnull\"`\n}\n```\n\n#### LinkParent\n\nThere is no separate table for this. All `Parent`s are found based on the `Inode` index in `edge`.\n\n#### Chunk {#sql-chunk}\n\n```go\ntype chunk struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"unique(chunk) notnull\"`\n Indx uint32 `xorm:\"unique(chunk) notnull\"`\n Slices []byte `xorm:\"blob notnull\"`\n}\n```\n\nSlices are an array of bytes, and each [Slice](#chunk) corresponds to 24 bytes.\n\n#### SliceRef\n\n```go\ntype sliceRef struct {\n Id uint64 `xorm:\"pk chunkid\"`\n Size uint32 `xorm:\"notnull\"`\n Refs int `xorm:\"notnull\"`\n}\n```\n\n#### Symlink\n\n```go\ntype symlink struct {\n Inode Ino `xorm:\"pk\"`\n Target []byte `xorm:\"varbinary(4096) notnull\"`\n}\n```\n\n#### Xattr\n\n```go\ntype xattr struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"unique(name) notnull\"`\n Name string `xorm:\"unique(name) notnull\"`\n Value []byte `xorm:\"blob notnull\"`\n}\n```\n\n#### Flock\n\n```go\ntype flock struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"notnull unique(flock)\"`\n Sid uint64 `xorm:\"notnull unique(flock)\"`\n Owner int64 `xorm:\"notnull unique(flock)\"`\n Ltype byte `xorm:\"notnull\"`\n}\n```\n\n#### Plock {#sql-plock}\n\n```go\ntype plock struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"notnull unique(plock)\"`\n Sid uint64 `xorm:\"notnull unique(plock)\"`\n Owner int64 `xorm:\"notnull unique(plock)\"`\n Records []byte `xorm:\"blob notnull\"`\n}\n```\n\nRecords is an array of bytes, and each [plockRecord](#plock) corresponds to 24 bytes.\n\n#### DelFiles\n\n```go\ntype delfile struct {\n Inode Ino `xorm:\"pk notnull\"`\n Length uint64 `xorm:\"notnull\"`\n Expire int64 `xorm:\"notnull\"`\n}\n```\n\n#### DelSlices {#sql-delslices}\n\n```go\ntype delslices struct {\n Id uint64 `xorm:\"pk chunkid\"`\n Deleted int64 `xorm:\"notnull\"`\n Slices []byte `xorm:\"blob notnull\"`\n}\n```\n\nSlices is an array of bytes, and each [slice](#delslices) corresponds to 12 bytes.\n\n#### Sustained\n\n```go\ntype sustained struct {\n Id int64 `xorm:\"pk bigserial\"`\n Sid uint64 `xorm:\"unique(sustained) notnull\"`\n Inode Ino `xorm:\"unique(sustained) notnull\"`\n}\n```\n\n### TKV\n\nThe common format of keys in TKV (Transactional Key-Value Database) is `${prefix}${JFSKey}`, where\n\n- prefix is used to distinguish between different file systems, usually `${VolumeName}0xFD`, where `0xFD` is used as a special byte to handle cases when there is an inclusion relationship between different file system names. In addition, for databases that are not shareable (e.g. BadgerDB), the empty string is used as prefix.\n- JFSKey is the JuiceFS Key for different data types, which is listed in the following subsections.\n\nIn TKV's Keys, all integers are stored in encoded binary form.\n\n- inode and counter value occupy 8 bytes and are encoded with **small endian**.\n- SID, sliceId and timestamp occupy 8 bytes and are encoded with **big endian**.\n\n#### Setting {#tkv-setting}\n\n```\nsetting -> file system formatting information in JSON format\n```\n\n#### Counter\n\n```\nC${name} -> counter value\n```\n\n#### Session\n\n```\nSE${sid} -> timestamp\n```\n\n#### SessionInfo\n\n```\nSI${sid} -> session information in JSON format\n```\n\n#### Node {#tkv-node}\n\n```\nA${inode}I -> encoded Attr\n```\n\n#### Edge {#tkv-edge}\n\n```\nA${inode}D${name} -> encoded {type, inode}\n```\n\n#### LinkParent\n\n```\nA${inode}P${parentInode} -> counter value\n```\n\n#### Chunk {#tkv-chunk}\n\n```\nA${inode}C${index} -> Slices\n```\n\nwhere index takes up 4 bytes and is encoded with **big endian**. Slices is an array of bytes, one [Slice](#chunk) per 24 bytes.\n\n#### SliceRef\n\n```\nK${sliceId}${size} -> counter value\n```\n\nwhere size takes up 4 bytes and is encoded with **big endian**.\n\n#### Symlink\n\n```\nA${inode}S -> target\n```\n\n#### Xattr\n\n```\nA${inode}X${name} -> xattr value\n```\n\n#### Flock\n\n```\nF${inode} -> flocks\n```\n\nwhere flocks is an array of bytes, one flock per 17 bytes.\n\n```go\ntype flock struct {\n sid uint64\n owner uint64\n ltype uint8\n}\n```\n\n#### Plock {#tkv-plock}\n\n```\nP${inode} -> plocks\n```\n\nwhere plocks is an array of bytes and the corresponding plock is variable-length.\n\n```go\ntype plock struct {\n sid uint64\n owner uint64\n size uint32\n records []byte\n}\n```\n\nwhere size is the length of the records array and every 24 bytes in records corresponds to one [plockRecord](#plock).\n\n#### DelFiles\n\n```\nD${inode}${length} -> timestamp\n```\n\nwhere length takes up 8 bytes and is encoded with **big endian**.\n\n#### DelSlices {#tkv-delslices}\n\n```\nL${timestamp}${sliceId} -> slices\n```\n\nwhere slices is an array of bytes, and one [slice](#delslices) corresponds to 12 bytes.\n\n#### Sustained\n\n```\nSS${sid}${inode} -> 1\n```\n\nHere the Value value is only used as a placeholder.\n\n## File Data Format\n\n### Finding files by path\n\nAccording to the design of [Edge](#edges), only the direct children of each directory are recorded in the metadata engine. When an application provides a path to access a file, JuiceFS needs to look it up level by level. Now suppose the application wants to open the file `/dir1/dir2/testfile`, then it needs to\n\n1. search for the entry with name \"dir1\" in the Edge structure of the root directory (inode number is fixed to 1) and get its inode number N1\n2. search for the entry with the name \"dir2\" in the Edge structure of N1 and get its inode number N2\n3. search for the entry with the name \"testfile\" in the Edge structure of N2, and get its inode number N3\n4. search for the [Node](#node) structure corresponding to N3 to get the attributes of the file\n\nFailure in any of the above steps will result in the file pointed to by that path not being found.\n\n### File data splitting\n\nFrom the previous section, we know how to find the file based on its path and get its attributes. The metadata related to the contents of the file can be found based on the inode and size fields in the file properties. Now suppose a file has an inode of 100 and a size of 160 MiB, then the file has `(size-1) / 64 MiB + 1 = 3` Chunks, as follows.\n\n```\n File: |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _|\nChunk: |<--- Chunk 0 --->|<--- Chunk 1 --->|<-- Chunk 2 -->|\n```\n\nIn standalone Redis, this means that there are 3 [Chunk Keys](#chunk), i.e.,`c100_0`, `c100_1` and `c100_2`, each corresponding to a list of Slices. These Slices are mainly generated when the data is written and may overwrite each other or may not fill the Chunk completely, so you need to traverse this list of Slices sequentially and reconstruct the latest version of the data distribution before using it, so that\n\n1. the part covered by more than one Slice is based on the last added Slice\n2. the part that is not covered by Slice is automatically zeroed, and is represented by sliceId = 0\n3. truncate Chunk according to file size\n\nNow suppose there are 3 Slices in Chunk 0\n\n```go\nSlice{pos: 10M, id: 10, size: 30M, off: 0, len: 30M}\nSlice{pos: 20M, id: 11, size: 16M, off: 0, len: 16M}\nSlice{pos: 16M, id: 12, size: 10M, off: 0, len: 10M}\n```\n\nIt can be illustrated as follows (each '_' denotes 2 MiB)\n\n```\n Chunk: |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|\nSlice 10: |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _|\nSlice 11: |_ _ _ _ _ _ _ _|\nSlice 12: |_ _ _ _ _|\n\nNew List: |_ _ _ _ _|_ _ _|_ _ _ _ _|_ _ _ _ _|_ _|_ _ _ _ _ _ _ _ _ _ _ _|\n 0 10 12 11 10 0\n```\n\nThe reconstructed new list contains and only contains the latest data distribution for this Chunk as follows\n\n```go\nSlice{pos: 0, id: 0, size: 10M, off: 0, len: 10M}\nSlice{pos: 10M, id: 10, size: 30M, off: 0, len: 6M}\nSlice{pos: 16M, id: 12, size: 10M, off: 0, len: 10M}\nSlice{pos: 26M, id: 11, size: 16M, off: 6M, len: 10M}\nSlice{pos: 36M, id: 10, size: 30M, off: 26M, len: 4M}\nSlice{pos: 40M, id: 0, size: 24M, off: 0, len: 24M} // can be omitted\n```\n\n### Data objects\n\n#### Object naming {#object-storage-naming-format}\n\nBlock is the basic unit for JuiceFS to manage data. Its size is 4 MiB by default, and can be changed only when formatting a file system, within the interval [64 KiB, 16 MiB]. Each Block is an object in the object storage after upload, and is named in the format `${fsname}/chunks/${hash}/${basename}`, where\n\n- fsname is the file system name\n- \"chunks\" is a fixed string representing the data object of JuiceFS\n- hash is the hash value calculated from basename, which plays a role in isolation management\n- basename is the valid name of the object in the format of `${sliceId}_${index}_${size}`, where\n - sliceId is the ID of the Slice to which the object belongs, and each Slice in JuiceFS has a globally unique ID\n - index is the index of the object in the Slice it belongs to, by default a Slice can be split into at most 16 Blocks, so its value range is [0, 16)\n - size is the size of the Block, and by default it takes the value of (0, 4 MiB]\n\nCurrently there are two hash algorithms, and both use the sliceId in basename as the parameter. Which algorithm will be chosen to use follows the [HashPrefix](#setting) of the file system.\n\n```go\nfunc hash(sliceId int) string {\n if HashPrefix {\n return fmt.Sprintf(\"%02X/%d\", sliceId%256, sliceId/1000/1000)\n }\n return fmt.Sprintf(\"%d/%d\", sliceId/1000/1000, sliceId/1000)\n}\n```\n\nSuppose a file system named `jfstest` is written with a continuous 10 MiB of data and internally given a SliceID of 1 with HashPrefix disabled, then the following three objects will be generated in the object storage.\n\n```\njfstest/chunks/0/0/1_0_4194304\njfstest/chunks/0/0/1_1_4194304\njfstest/chunks/0/0/1_2_2097152\n```\n\nSimilarly, now taking the 64 MiB chunk in the previous section as an example, its actual data distribution is as follows\n\n```\n 0 ~ 10M: Zero\n10 ~ 16M: 10_0_4194304, 10_1_4194304(0 ~ 2M)\n16 ~ 26M: 12_0_4194304, 12_1_4194304, 12_2_2097152\n26 ~ 36M: 11_1_4194304(2 ~ 4M), 11_2_4194304, 11_3_4194304\n36 ~ 40M: 10_6_4194304(2 ~ 4M), 10_7_2097152\n40 ~ 64M: Zero\n```\n\nAccording to this, the client can quickly find the data needed for the application. For example, reading 8 MiB data at offset 10 MiB location will involve 3 objects, as follows\n\n- Read the entire object from `10_0_4194304`, corresponding to 0 to 4 MiB of the read data\n- Read 0 to 2 MiB from `10_1_4194304`, corresponding to 4 to 6 MiB of the read data\n- Read 0 to 2 MiB from `12_0_4194304`, corresponding to 6 to 8 MiB of the read data\n\nTo facilitate obtaining the list of objects of a certain file, JuiceFS provides the `info` command, e.g. `juicefs info /mnt/jfs/test.tmp`.\n\n```bash\nobjects:\n+------------+---------------------------------+----------+---------+----------+\n| chunkIndex | objectName | size | offset | length |\n+------------+---------------------------------+----------+---------+----------+\n| 0 | | 10485760 | 0 | 10485760 |\n| 0 | jfstest/chunks/0/0/10_0_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/10_1_4194304 | 4194304 | 0 | 2097152 |\n| 0 | jfstest/chunks/0/0/12_0_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/12_1_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/12_2_2097152 | 2097152 | 0 | 2097152 |\n| 0 | jfstest/chunks/0/0/11_1_4194304 | 4194304 | 2097152 | 2097152 |\n| 0 | jfstest/chunks/0/0/11_2_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/11_3_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/10_6_4194304 | 4194304 | 2097152 | 2097152 |\n| 0 | jfstest/chunks/0/0/10_7_2097152 | 2097152 | 0 | 2097152 |\n| ... | ... | ... | ... | ... |\n+------------+---------------------------------+----------+---------+----------+\n```\n\nThe empty objectName in the table means a file hole and is read as 0. As you can see, the output is consistent with the previous analysis.\n\nIt is worth mentioning that the 'size' here is size of the original data in the Block, rather than that of the actual object in object storage. The original data is written directly to object storage by default, so the 'size' is equal to object size. However, when data compression or data encryption is enabled, the size of the actual object will change and may no longer be the same as the 'size'.\n\n#### Data compression\n\nYou can configure the compression algorithm (supporting `lz4` and `zstd`) with the `--compress ` parameter when formatting a file system, so that all data blocks of this file system will be compressed before uploading to object storage. The object name remains the same as default, and the content is the result of the compression algorithm, without any other meta information. Therefore, the compression algorithm in the [file system formatting Information](#setting) is not allowed to be modified, otherwise it will cause the failure of reading existing data.\n\n#### Data encryption\n\nThe RSA private key can be configured to enable [static data encryption](../security/encryption.md) when formatting a file system with the `--encrypt-rsa-key ` parameter, which allows all data blocks of this file system to be encrypted before uploading to the object storage. The object name is still the same as default, while its content becomes a header plus the result of the data encryption algorithm. The header contains a random seed and the symmetric key used for decryption, and the symmetric key itself is encrypted with the RSA private key. Therefore, it is not allowed to modify the RSA private key in the [file system formatting Information](#setting), otherwise reading existing data will fail.\n\n:::note\nIf both compression and encryption are enabled, the original data will be compressed and then encrypted before uploading to the object storage.\n:::\n" }, { "path": "docs/en/faq.md", "content": "---\ntitle: FAQ\nslug: /faq\n---\n\n## My question is not answered in the documentation\n\nIf you can't find an answer in the documentation, please try using the \"Ask AI\" feature (in the bottom right corner). If the AI assistant's answer helps you or provides a wrong answer, feel free to leave feedback on the response. Alternatively, use the document search feature (in the top right corner) and try searching with different keywords.\n\nIf these methods still do not resolve your question, you can join the [JuiceFS Community](https://juicefs.com/en/community) for further assistance.\n\n## General Questions\n\n### What's the difference between JuiceFS and XXX?\n\nSee [\"Comparison with Others\"](introduction/comparison/juicefs_vs_alluxio.md) for more information.\n\n### How to upgrade JuiceFS client?\n\nFirst unmount JuiceFS volume, then re-mount the volume with newer version client.\n\n### Where is the JuiceFS log?\n\nDifferent types of JuiceFS clients have different ways to obtain logs. For details, please refer to [\"Client log\"](administration/fault_diagnosis_and_analysis.md#client-log) document.\n\n### Can JuiceFS directly read files that already exist in object storage?\n\nJuiceFS cannot directly read files that already exist in object storage. Although JuiceFS typically uses object storage as the data storage layer, it is not a tool for accessing object storage in the traditional sense. You can refer to the [technical architecture](introduction/architecture.md) documentation for more details.\n\nIf you want to migrate existing data in an object storage bucket to JuiceFS, you can use [`JuiceFS Sync`](guide/sync.md).\n\n### How can I combine multiple servers into a single JuiceFS file system for use?\n\nNo, while JuiceFS supports using local disks or SFTP as the underlying storage, it does not interfere with the logical structure management of the underlying storage. If you wish to consolidate storage space from multiple servers, you may consider using MinIO or Ceph to create an object storage cluster, and then create a JuiceFS file system on top of it.\n\n## Metadata Related Questions\n\n### Does support Redis in Sentinel or Cluster-mode as the metadata engine for JuiceFS?\n\nYes, There is also a [best practice document](administration/metadata/redis_best_practices.md) for Redis as the JuiceFS metadata engine for reference.\n\n## Object Storage Related Questions\n\n### Why doesn't JuiceFS support XXX object storage?\n\nJuiceFS already supported many object storage, please check [the list](reference/how_to_set_up_object_storage.md#supported-object-storage) first. If this object storage is compatible with S3, you could treat it as S3. Otherwise, try reporting issue.\n\n### Why do I delete files at the mount point, but there is no change or very little change in object storage footprint?\n\nThe first reason is that you may have enabled the trash feature. In order to ensure data security, the trash is enabled by default. The deleted files are actually placed in the trash and are not actually deleted, so the size of the object storage will not change. trash retention time can be specified with `juicefs format` or modified with `juicefs config`. Please refer to the [\"Trash\"](security/trash.md) documentation for more information.\n\nThe second reason is that JuiceFS deletes the data in the object storage asynchronously, so the space change of the object storage will be slower. If you need to immediately clean up the data in the object store that needs to be deleted, you can try running the [`juicefs gc`](reference/command_reference.mdx#gc) command.\n\n### How Does JuiceFS Asynchronous Deletion Work?\n\n* ​**When trash is disabled:**\n - The system checks whether the file is being opened by other processes:\n * If the file is in use, it is marked as ​**\"deferred deletion (`sustained`)\"** and will be processed after the program closes the file\n * If the file is not in use, it is marked as ​**pending deletion (`delfile`)** and attempts to place it into the ​**deletion queue (`maxDeleting`)**\n \n* ​**When trash is enabled:**\n - The system creates subdirectories in the trash based on ​**current time (accurate to the hour)** (e.g., `2024-01-15-14`)\n - Files pending deletion are moved to the corresponding time-stamped directory:\n * ​**All chunks and slices of data remain intact**\n * Only the ​**parent directory pointer** in metadata changes\n * Filenames are ​**re-encoded** to prevent conflicts\n - A background task cleans expired files based on retention period:\n * Starts cleaning from the ​**oldest directory**\n * Method: Marked as ​**pending deletion (`delfile`)**, placed into the ​**deletion queue (`maxDeleting`)**\n \n* ​**Deletion queue processing (asynchronous cleanup):**\n 1. ​**Find all chunks corresponding to the file and delete them**\n 2. Deleting chunks will ​**decrement the reference count of their slices**\n 3. When a slice's reference count drops to zero, it becomes ​**`Pending Deleted Slices`**\n 4. The background task cleans these data slices from object storage\n\n![JuiceFS-delete-file](./images/juicefs-delete-file-English.svg)\n\n* The deletion queue has a capacity limit. If too many files are deleted simultaneously, deletion requests will return immediately when the queue is full. Then a background cleanup task that runs hourly continues the cleanup. It finds all files marked as ​**pending deletion (`delfile`)** and cleans them using the same method as files in the deletion queue.\n* If NoBGJob is configured, the hourly scheduled background cleanup task and trash cleanup task are disabled. After deleting files, manual cleanup is required in the trash.\n* In a special scenario, when you manually delete files directly from the trash, it ensures synchronous insertion into the deletion queue, enabling relatively fast reclamation of object storage space. However, subsequent chunk cleanup remains asynchronous.\n* Regarding slice reference count: Deleting chunks and compaction (`compact`) will decrease the reference count of related slices, while `clone` and `copyFileRange` will increase the reference count of related slices.\n\n### Why is file system data size different from object storage usage? {#size-inconsistency}\n\n* [\"Random write in JuiceFS\"](#random-write) produces data fragments, causing higher storage usage for object storage, especially after a large number of overwrites in a short period of time, many fragments will be generated. These fragments continue to occupy space in object storage until they are compacted and released. You shouldn't worry about this because JuiceFS checks for file compaction with every read/write, and cleans up in the client background job. Alternatively, you can manually trigger merges and garbage collection with [`juicefs gc --compact --delete`](./reference/command_reference.mdx#gc).\n* If [Trash](./security/trash.md) is enabled, deleted files will be kept for a specified period of time, and then be garbage collected (all carried out in client background job).\n* After data fragments are compacted, stale slices will be kept inside Trash as well (not visible to user), following the same expiration settings. To delete this type of data, read [Trash and stale slices](./security/trash.md#gc).\n* If compression is enabled (the `--compress` parameter in the [`format`](./reference/command_reference.mdx#format) command, disabled by default), object storage usage may be smaller than the actual file size (depending on the compression ratio of different types of files).\n* Different [storage class](reference/how_to_set_up_object_storage.md#storage-class) of the object storage may calculate storage usage differently. The cloud service provider may set the minimum billable size for some storage classes. For example, the [minimum billable size](https://www.alibabacloud.com/help/en/object-storage-service/latest/storage-fees) for Alibaba Cloud OSS IA storage is 64KB. If a file is smaller than 64KB, it will be calculated as 64KB.\n* For self-hosted object storage services, for example MinIO, actual data usage is affected by [storage class settings](https://github.com/minio/minio/blob/master/docs/erasure/storage-class/README.md).\n\n### Does JuiceFS support using a directory in object storage as the value of the `--bucket` option?\n\nAs of the release of JuiceFS 1.0, this feature is not supported.\n\n### Does JuiceFS support accessing data that already exists in object storage?\n\nAs of the release of JuiceFS 1.0, this feature is not supported.\n\n### Is it possible to bind multiple different object storages to a single file system (e.g. one file system with Amazon S3, GCS and OSS at the same time)?\n\nNo. However, you can set up multiple buckets associated with the same object storage service when creating a file system, thus solving the problem of limiting the number of individual bucket objects, for example, multiple S3 Buckets can be associated with a single file system. Please refer to [`--shards`](./reference/command_reference.mdx#format) option for details.\n\n## Performance Related Questions\n\n### How is the performance of JuiceFS?\n\nJuiceFS is a distributed file system, the latency of metadata is determined by 1 (reading) or 2 (writing) round trip(s) between client and metadata service (usually 1-3 ms). The latency of first byte is determined by the performance of underlying object storage (usually 20-100 ms). Throughput of sequential read/write could be 50MB/s - 2800MiB/s (see [fio benchmark](benchmark/fio.md) for more information), depends on network bandwidth and how the data could be compressed.\n\nJuiceFS is built with multiple layers of caching (invalidated automatically), once the caching is warmed up, the latency and throughput of JuiceFS could be close to local file system (having the overhead of FUSE).\n\n### Does JuiceFS support random read/write? How? {#random-write}\n\nYes, including those issued using mmap. Currently JuiceFS is optimized for sequential reading/writing, and optimized for random reading/writing is work in progress. If you want better random read performance, it's best to turn off compression ([`--compress none`](reference/command_reference.mdx#format)).\n\nJuiceFS does not store the original file in the object storage, but splits it into data blocks using a fixed size (4MiB by default), then uploads it to the object storage, and stores the ID of the data block in the metadata engine. When random write happens, the original metadata is marked stale, and then JuiceFS Client uploads the **new data block** to the object storage, then update the metadata accordingly.\n\nWhen reading the data of the overwritten part, according to the **latest metadata**, it can be read from the **new data block** uploaded during random writing, and the **old data block** may be deleted by the background garbage collection tasks automatically clean up. This shifts complexity from random writes to reads.\n\nRead [JuiceFS Internals](development/internals.md) and [Data Processing Flow](introduction/io_processing.md) to learn more.\n\n### How to copy a large number of small files into JuiceFS quickly?\n\nUse the [`--writeback` option](reference/command_reference.mdx#mount-data-cache-options) to write data to the local cache and then asynchronously upload it to the object storage backend. This is significantly faster than writing directly to object storage. See [\"Write Cache in Client\"](guide/cache.md#client-write-cache) for more information.\n\n### Does JuiceFS support distributed cache?\n\n[Distributed cache](https://juicefs.com/docs/cloud/guide/distributed-cache) is supported in our enterprise edition.\n\n## Mount Related Questions\n\n### Can I mount JuiceFS without `root` privileges?\n\nYes, JuiceFS could be mounted using `juicefs` without root privileges. The default directory for caching is `$HOME/.juicefs/cache` (macOS) or `/var/jfsCache` (Linux), you should change that to a directory which you have write permission.\n\nSee [\"Read Cache in Client\"](guide/cache.md#client-read-cache) for more information.\n\n## Access Related Questions\n\n### What other ways does JuiceFS offer to access data?\n\nIn addition to mounting, the following methods are also supported:\n\n- Kubernetes CSI Driver: Use JuiceFS as the storage layer of Kubernetes cluster through the Kubernetes CSI Driver. For details, please refer to [\"Use JuiceFS on Kubernetes\"](deployment/how_to_use_on_kubernetes.md).\n- Hadoop Java SDK: It is convenient to use a Java client compatible with the HDFS interface to access JuiceFS in the Hadoop ecosystem. For details, please refer to [\"Use JuiceFS on Hadoop Ecosystem\"](deployment/hadoop_java_sdk.md).\n- S3 Gateway: Access JuiceFS through the S3 protocol. For details, please refer to [\"Deploy JuiceFS S3 Gateway\"](./guide/gateway.md).\n- Docker Volume Plugin: A convenient way to use JuiceFS in Docker, please refer to [\"Use JuiceFS on Docker\"](deployment/juicefs_on_docker.md).\n- WebDAV Gateway: Access JuiceFS via WebDAV protocol\n\n### Why does the same username have different permissions on different hosts when accessing JuiceFS files?\n\nAlthough a user has the same username on both hosts (for example, `alice` on host X and host Y), their user ID (UID) or group ID (GID) differs between them. File permissions in JuiceFS are based on these numeric IDs, not the username.\n\nTo confirm this, run the `id` command on each host and compare the output:\n\n```bash\n$ id alice\nuid=1201(alice) gid=500(staff) groups=500(staff)\n```\n\nSee [Sync Accounts between Multiple Hosts](administration/sync_accounts_between_multiple_hosts.md) to resolve this issue.\n\n### Does JuiceFS S3 Gateway support advanced features such as multi-user management?\n\nThe built-in `gateway` subcommand does not support functions including as multi-user management, and provides only basic S3 gateway functions. If you need to use these advanced features, please refer to the [documentation](guide/gateway.md).\n\n### Is there an SDK available for JuiceFS?\n\nAs of the release of JuiceFS 1.0, the community has two SDKs, one is the [Java SDK](deployment/hadoop_java_sdk.md) that is highly compatible with the HDFS interface officially maintained by Juicedata, and the other is the [Python SDK](https://github.com/megvii-research/juicefs-python) maintained by community users.\n" }, { "path": "docs/en/getting-started/for_distributed.md", "content": "---\nsidebar_position: 3\ndescription: This article will guide you through building a distributed, shared-access JuiceFS file system using cloud-based object storage and databases.\n---\n\n# Distributed Mode\n\n[The previous document](./standalone.md) introduces how to create a file system that can be mounted on any host using an *object storage* and an *SQLite* database. Since object storage is accessible by any computer with privileges on the network, you can also access the same JuiceFS file system on different computers by simply copying the SQLite database file to any computer that needs to access the storage.\n\nHowever, this approach does not guarantee real-time file availability when the file system is shared. Since SQLite is a single file database that cannot be accessed by multiple computers at the same time, a database that supports network access is needed, such as Redis, PostgreSQL, or MySQL. This allows a file system to be mounted and read by multiple computers in a distributed environment.\n\nIn this document, a multi-user *cloud database* will be used to replace the single-user *SQLite* database used in the previous document. This aims to implement a distributed file system that can be mounted on any computer on the network for reading and writing.\n\n## Network databases\n\nA *network database* is one that allows multiple users to access it simultaneously through a network. From this perspective, databases can generally be classified as:\n\n- **Standalone databases**: Single-file databases usually only accessed locally, such as SQLite and Microsoft Access.\n- **Network databases**: Databases usually with complex multi-file structures, providing network-based access interfaces and supporting simultaneous access by multiple users, such as Redis and PostgreSQL.\n\nJuiceFS currently supports the following network-based databases.\n\n- **Key-value databases**: Redis, TiKV, etcd, and FoundationDB\n- **Relational databases**: PostgreSQL, MySQL, and MariaDB\n\nDifferent databases have different performance and stability. For example, Redis is an in-memory key-value database with excellent performance but relatively weak reliability, while PostgreSQL is a more reliable relational database with lower performance than in-memory databases.\n\nA detailed guide on database selection will be available soon.\n\n## Cloud databases\n\nCloud computing platforms usually offer a wide variety of cloud databases, such as Amazon RDS for various relational database versions and Amazon ElastiCache for Redis-compatible in-memory database products. These services allow to create a multi-copy and highly available database cluster by a simple initial setup.\n\nAlternatively, you can also build your own database on the server.\n\nFor simplicity, we'll use Amazon ElastiCache for Redis as an example. The most basic information of a network database include:\n\n- **Database address**: The database's access address, with different links for internal and external networks.\n- **Username and password**: Authentication information used to access the database.\n\n## Hands-on practice\n\n### 1. Install the client\n\nInstall the JuiceFS client on all computers that need to mount the file system. Refer to the [Installation](installation.md) guide for details.\n\n### 2. Prepare object storage\n\nHere is a pseudo sample with Amazon S3 as an example. You can also use other object storage services. Refer to [JuiceFS Supported Storage](../reference/how_to_set_up_object_storage.md#supported-object-storage) for details.\n\n- **Bucket Endpoint**: `https://myjfs.s3.us-west-1.amazonaws.com`\n- **Access Key ID**: `ABCDEFGHIJKLMNopqXYZ`\n- **Access Key Secret**: `ZYXwvutsrqpoNMLkJiHgfeDCBA`\n\n### 3. Prepare the database\n\nHere is a pseudo sample with Amazon ElastiCache for Redis as an example. You can also use other types of databases. Refer to [JuiceFS Supported Databases](../reference/how_to_set_up_metadata_engine.md) for details.\n\n- **Database address**: `myjfs-sh-abc.apse1.cache.amazonaws.com:6379`\n- **Database username**: `tom`\n- **Database password**: `mypassword`\n\nThe format for using a Redis database in JuiceFS is as follows:\n\n```\nredis://:@:6379/1\n```\n\n:::tip\nRedis versions lower than 6.0 do not have a username, so omit the `` part in the URL. For example: `redis://:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1` (note that the colon before the password is a separator and must be included).\n:::\n\n### 4. Create a file system\n\nTo create a file system that supports cross-network, multi-server simultaneous mounts with shared read/write access using object storage and a Redis database, run:\n\n```shell\njuicefs format \\\n --storage s3 \\\n --bucket https://myjfs.s3.us-west-1.amazonaws.com \\\n --access-key ABCDEFGHIJKLMNopqXYZ \\\n --secret-key ZYXwvutsrqpoNMLkJiHgfeDCBA \\\n redis://tom:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1 \\\n myjfs\n```\n\nOnce the file system is created, you'll see an output similar to:\n\n```shell\n2021/12/16 16:37:14.264445 juicefs[22290] : Meta address: redis://@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1\n2021/12/16 16:37:14.277632 juicefs[22290] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/12/16 16:37:14.281432 juicefs[22290] : Ping redis: 3.609453ms\n2021/12/16 16:37:14.527879 juicefs[22290] : Data uses s3://myjfs/myjfs/\n2021/12/16 16:37:14.593450 juicefs[22290] : Volume is formatted as {Name:myjfs UUID:4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b Storage:s3 Bucket:https://myjfs AccessKey:ABCDEFGHIJKLMNopqXYZ SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n:::info\nOnce the file system is created, all relevant information, including its name, object storage details, and access keys, are stored in the database. In this example, the file system information is stored in Redis, so any computer with the database address, username, and password information can mount and read the file system.\n:::\n\n### 5. Mount the file system\n\nSince the file system's *data* and *metadata* are stored in cloud services, it can be mounted on any computer with a JuiceFS client installed for shared reads and writes at the same time. For example:\n\n```shell\njuicefs mount redis://tom:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1 ~/jfs\n```\n\n#### Strong data consistency guarantee\n\nJuiceFS ensures *close-to-open* consistency. When multiple clients are reading and writing to the same file, changes made by client A may not be immediately visible to client B. Once client A closes the file, any other client, no matter whether the file is on the same node with A, is guaranteed to see the latest data upon reopening the file.\n\n#### Increase cache size to improve performance\n\nSince object storage is a network-based service, access latency is inevitable. To mitigate this, JuiceFS offers a caching mechanism, enabled by default. This allocates a portion of local storage as a buffer layer between your data and the object storage, asynchronously caching data to local storage when files are accessed. For more details, refer to [Cache](../guide/cache.md).\n\nJuiceFS sets 100GiB cache in the `$HOME/.juicefs/cache` or `/var/jfsCache` directory by default. Setting a larger cache space on a faster SSD can effectively improve read and write performance of JuiceFS.\n\nYou can use `--cache-dir` to adjust the location of the cache directory and `--cache-size` to adjust the size of the cache space. For example:\n\n```shell\njuicefs mount\n --background \\\n --cache-dir /mycache \\\n --cache-size 512000 \\\n redis://tom:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1 \\\n ~/jfs\n```\n\n:::note\nThe JuiceFS process needs read and write permissions for the `--cache-dir` directory.\n:::\n\nThe above command sets the cache directory in the `/mycache` directory and specifies the cache space as 500GiB.\n\n#### Auto-mount on boot\n\nIn a Linux environment, you can set up automatic mounting when mounting a file system via the `--update-fstab` option, which adds the necessary options to mount JuiceFS to `/etc/fstab`. For example:\n\n:::note\nThis feature requires JuiceFS version 1.1.0 or above.\n:::\n\n```bash\n$ sudo juicefs mount --update-fstab --max-uploads=50 --writeback --cache-size 204800 redis://tom:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1 \n$ grep /etc/fstab\nredis://tom:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1 juicefs _netdev,max-uploads=50,writeback,cache-size=204800 0 0\n$ ls -l /sbin/mount.juicefs\nlrwxrwxrwx 1 root root 29 Aug 11 16:43 /sbin/mount.juicefs -> /usr/local/bin/juicefs\n```\n\nRefer to [Mount JuiceFS at Boot Time](../administration/mount_at_boot.md) for more details.\n\n### 6. Verify the file system\n\nAfter the file system is mounted, you can use the `juicefs bench` command to perform basic performance tests and functional verification of the file system to ensure that the JuiceFS file system can be accessed normally and its performance meets expectations.\n\n:::info\nThe `juicefs bench` command can only complete basic performance tests. If you need a more comprehensive evaluation of JuiceFS, refer to [JuiceFS Performance Evaluation Guide](../benchmark/performance_evaluation_guide.md).\n:::\n\n```shell\njuicefs bench ~/jfs\n```\n\nThis command writes and reads a specified number of large (1 by default) and small (100 by default) files to and from the JuiceFS file system according to the specified concurrency (1 by default). The command then measures the throughput and latency of read and write operations, as well as the latency of metadata engine access.\n\nIf you encounter any problems during the verification of the file system, refer to the [Fault Diagnosis and Analysis](../administration/fault_diagnosis_and_analysis.md) document for troubleshooting.\n\n### 7. Unmount the file system\n\nYou can unmount the JuiceFS file system (assuming the mount point path is `~/jfs`) by the command `juicefs umount`.\n\n```shell\njuicefs umount ~/jfs\n```\n\nIf the command fails to unmount the file system after execution, it will prompt `Device or resource busy`.\n\n```shell\n2021-05-09 22:42:55.757097 I | fusermount: failed to unmount ~/jfs: Device or resource busy\nexit status 1\n```\n\nThis failure happens probably because some programs are reading or writing files in the file system when executing the `unmount` command. To avoid data loss,first determine which processes are accessing files in the file system (for example, via the `lsof` command) and try to release the files before re-executing the `unmount` command.\n\n:::caution\nThe following command may result in file corruption and loss. Proceed with caution!\n:::\n\nYou can force unmounting by adding the `--force` or `-f` option if you're sure of the consequences:\n\n```shell\njuicefs umount --force ~/jfs\n```\n" }, { "path": "docs/en/getting-started/installation.md", "content": "---\ntitle: Installation\nsidebar_position: 1\ndescription: Learn how to install JuiceFS on Linux, macOS, and Windows, including one-click installation, pre-compiled installation, and containerized deployment methods.\n---\n\nJuiceFS has good cross-platform capability and supports various operating systems across almost all major architectures, including but not limited to Linux, macOS, and Windows.\n\nThe JuiceFS client has only one binary file. You can either download the pre-compiled version to unzip it and use it directly, or manually compile it from the provided source code.\n\n## One-click installation {#one-click-installation}\n\nThe one-click installation script is available for Linux and macOS systems. It automatically downloads and installs the latest version of the JuiceFS client based on your hardware architecture.\n\n**Option 1 (Recommended):** Install to the default location `/usr/local/bin`:\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n**Option 2:** If you need to install to a custom location, for example `/tmp` directory:\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -s /tmp\n```\n\n:::tip\nMost users should choose **Option 1** for the default installation. Only use **Option 2** if you have specific requirements for the installation directory.\n:::\n\n## Install the pre-compiled client {#install-the-pre-compiled-client}\n\nYou can download the latest version of the client at [GitHub](https://github.com/juicedata/juicefs/releases). Pre-compiled versions for different CPU architectures and operating systems are available in the download list of each client version. Please select the version that best suits your application. For example:\n\n| File Name | Description |\n|--------------------------------------|----------------------------------------------------------------------------------------------|\n| `juicefs-x.y.z-darwin-amd64.tar.gz` | For macOS systems with Intel chips |\n| `juicefs-x.y.z-darwin-arm64.tar.gz` | For macOS systems with M1 series chips |\n| `juicefs-x.y.z-linux-amd64.tar.gz` | For Linux distributions on x86 architecture |\n| `juicefs-x.y.z-linux-arm64.tar.gz` | For Linux distributions on ARM architecture |\n| `juicefs-x.y.z-windows-amd64.tar.gz` | For Windows on x86 architecture |\n| `juicefs-hadoop-x.y.z.jar` | Hadoop Java SDK on x86 and ARM architectures (supports Linux, macOS, and Windows systems) |\n\n### Linux {#linux}\n\nFor Linux systems with x86 architecture, download the file with the file name `linux-amd64` and execute the following commands in the terminal.\n\n1. Get the latest version number:\n\n ```shell\n JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '\"' -f 4 | tr -d 'v')\n ```\n\n2. Download the client to the current directory:\n\n ```shell\n wget \"https://github.com/juicedata/juicefs/releases/download/v${JFS_LATEST_TAG}/juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\"\n ```\n\n3. Unzip the installation package:\n\n ```shell\n tar -zxf \"juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\"\n ```\n\n4. Install the client:\n\n ```shell\n sudo install juicefs /usr/local/bin\n ```\n\nAfter completing the above 4 steps, execute the `juicefs` command in the terminal. If the client installation is successful, a help message will be returned.\n\n:::info\nIf the terminal prompts `command not found`, it is probably because `/usr/local/bin` is not in your system's `PATH` environment variable. You can execute `echo $PATH` to see which executable paths are set in your system. Based on the returned result, select an appropriate path, adjust, and re-execute the installation command in step 4.\n:::\n\n#### Ubuntu PPA\n\nJuiceFS also provides a [PPA](https://launchpad.net/~juicefs) repository, which makes it easy to install the latest version of the client on Ubuntu systems. Choose the corresponding PPA repository based on your CPU architecture:\n\n- **x86 architecture**：`ppa:juicefs/ppa`\n- **ARM architecture**：`ppa:juicefs/arm64`\n\nFor example, on a Ubuntu 22.04 system with x86 architecture, execute the following commands:\n\n1. Add the PPA repository:\n\n ```shell\n sudo add-apt-repository ppa:juicefs/ppa\n ```\n\n2. Update the package list:\n\n ```shell\n sudo apt-get update\n ```\n\n3. Install the JuiceFS client:\n\n ```shell\n sudo apt-get install juicefs\n ```\n\n#### Fedora Copr\n\nJuiceFS also provides a [Copr](https://copr.fedorainfracloud.org/coprs/juicedata/juicefs) repository, which allows for easy installation of the latest version of the client on Red Hat and its derivatives. The supported systems currently include:\n\n- **Amazonlinux 2023**\n- **CentOS 8, 9**\n- **Fedora 37, 38, 39, rawhide**\n- **RHEL 7, 8, 9**\n\nTaking Fedora 38 as an example, execute the following commands to install the client:\n\nEnable the Copr repository:\n\n```shell\nsudo dnf copr enable -y juicedata/juicefs\n```\n\nInstall the client:\n\n```shell\nsudo dnf install juicefs\n```\n\n#### Snapcraft\n\nWe have also packaged and released the [Snap version of the JuiceFS client](https://github.com/juicedata/juicefs-snapcraft) on the [Canonical Snapcraft](https://snapcraft.io) platform. For Ubuntu 16.04 and above and other operating systems that support Snap, you can install it using the following commands:\n\n```shell\nsudo snap install juicefs\n```\n\nSince Snap is a closed sandbox environment, it may affect the client's FUSE mount. You can remove the restriction by executing the following command. If you only need to use WebDAV and Gateway, there is no need to execute:\n\n```shell\nsudo ln -s -f /snap/juicefs/current/juicefs /snap/bin/juicefs\n```\n\nWhen there is a new version, execute the following command to update the client:\n\n```shell\nsudo snap refresh juicefs\n```\n\n#### AUR (Arch User Repository) {#aur}\n\nJuiceFS also provides an [AUR](https://aur.archlinux.org/packages/juicefs) repository, which makes it convenient to install the latest version of the client on Arch Linux and its derivatives.\n\nFor systems using the Yay package manager, execute the following command to install the client:\n\n```shell\nyay -S juicefs\n```\n\n:::info\nThere are multiple JuiceFS client packages available on AUR. The following are versions officially maintained by JuiceFS:\n\n- [`aur/juicefs`](https://aur.archlinux.org/packages/juicefs): A stable compiled version that fetches the latest stable source code and compiles it during installation.\n- [`aur/juicefs-bin`](https://aur.archlinux.org/packages/juicefs-bin): A stable pre-compiled version that directly downloads and installs the latest stable pre-compiled program.\n- [`aur/juicefs-git`](https://aur.archlinux.org/packages/juicefs-git): A development version that fetches the latest development source code and compiles it during installation.\n\n:::\n\nAdditionally, you can manually compile and install using `makepkg`, as shown for an Arch Linux system:\n\nInstall dependencies:\n\n```shell\nsudo pacman -S base-devel git go\n```\n\nClone the AUR repository to be packaged:\n\n```shell\ngit clone https://aur.archlinux.org/juicefs.git\n```\n\nNavigate to the repository directory:\n\n```shell\ncd juicefs\n```\n\nCompile and install:\n\n```shell\nmakepkg -si\n```\n\n### Windows {#windows}\n\nSince Windows does not natively support the FUSE interface, you need to download and install [WinFsp](https://winfsp.dev) first in order to implement FUSE support.\n\n :::tip\n **[WinFsp](https://github.com/winfsp/winfsp)** is an open source Windows file system agent. It provides a FUSE emulation layer that allows JuiceFS clients to mount file systems on Windows systems for use.\n :::\n\nThere are three ways to use JuiceFS on Windows systems.\n\n- [Using the pre-compiled Windows client](#using-the-pre-compiled-windows-client)\n- [Using Scoop](#scoop)\n- [Using the Linux client in WSL](#using-the-linux-client-in-wsl)\n\n#### Using the pre-compiled Windows client\n\nThe Windows client of JuiceFS is also a standalone binary. After you download and extract it, you can run it right away.\n\nTake the Windows 10 system as an example, download the file with the file name `windows-amd64`, unzip it, and get `juicefs.exe` which is the JuiceFS client binary.\n\nFor convenience, you can move `juicefs.exe` to `C:\\Windows\\System32`, so you can run the `juicefs` command directly from any directory in the command line.\n\nIf you prefer more flexible management of the JuiceFS client, you can create a `juicefs` folder under the `C:\\` drive, place `juicefs.exe` inside it, and add `C:\\juicefs` to your system's PATH environment variable. After restarting your system, you can use the `juicefs` command directly in terminals such as Command Prompt or PowerShell.\n\n![Windows ENV path](../images/windows-path-en.png)\n\n#### Using Scoop {#scoop}\n\nIf you have [Scoop](https://scoop.sh) installed in your Windows system, you can use the following command to install the latest version of the JuiceFS client:\n\n```shell\nscoop install juicefs\n```\n\n#### Using the Linux client in WSL\n\n[WSL](https://docs.microsoft.com/en-us/windows/wsl/about) is short for Windows Subsystem for Linux, which is supported from Windows 10 version 2004 onwards or Windows 11. It allows you to run most of the command-line tools, utilities, and applications of GNU/Linux natively on a Windows system without incurring the overhead of a traditional virtual machine or dual-boot setup.\n\nFor details, see [Using JuiceFS on WSL](../tutorials/juicefs_on_wsl.md).\n\n### macOS {#macos}\n\nSince macOS does not support the FUSE interface by default, you need to install [macFUSE](https://osxfuse.github.io) to enable FUSE mounting. If FUSE mounting is not your primary use case, installing macFUSE is not required. You can also conveniently read and write data using JuiceFS through [WebDAV](../deployment/webdav.md), [Gateway](../guide/gateway.md), or the [Python SDK](../deployment/python_sdk.md).\n\n:::tip\n[macFUSE](https://github.com/osxfuse/osxfuse) is an open source file system enhancement tool that allows macOS to mount third-party file systems. It enables JuiceFS clients to mount file systems on macOS systems.\n:::\n\n#### Homebrew\n\nIf you have the [Homebrew](https://brew.sh) package manager installed on your system, you can install the JuiceFS client by executing the following command:\n\n```shell\nbrew install juicefs\n```\n\n*For more information about this command, please refer to [Homebrew Formulae](https://formulae.brew.sh/formula/juicefs#default) page.*\n\n#### Pre-compiled binary\n\nYou can also download the binary with the file name `darwin-amd64`. After downloading, unzip the file and install the program to any executable path on your system using the `install` command, for example:\n\n```shell\nsudo install juicefs /usr/local/bin\n```\n\n### Docker {#docker}\n\nFor those interested in using JuiceFS in a Docker container, a `Dockerfile` for building a JuiceFS client image is provided below. It can be used as a base to build a JuiceFS client image alone or packaged together with other applications.\n\n```dockerfile\nFROM ubuntu:20.04\n\nRUN apt update && apt install -y curl fuse && \\\n apt-get autoremove && \\\n apt-get clean && \\\n rm -rf \\\n /tmp/* \\\n /var/lib/apt/lists/* \\\n /var/tmp/*\n\nRUN set -x && \\\n mkdir /juicefs && \\\n cd /juicefs && \\\n JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '\"' -f 4 | tr -d 'v') && \\\n curl -s -L \"https://github.com/juicedata/juicefs/releases/download/v${JFS_LATEST_TAG}/juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" \\\n | tar -zx && \\\n install juicefs /usr/bin && \\\n cd .. && \\\n rm -rf /juicefs\n\nCMD [ \"juicefs\" ]\n```\n\n## Manually compiling {#manually-compiling}\n\nIf there is no pre-compiled client versions that are suitable for your operating system, such as FreeBSD, you can manually compile the JuiceFS client.\n\nOne advantage of manual compilation is that you have priority access to various new features in JuiceFS development, but it requires some basic knowledge of software compilation.\n\n:::tip\nFor users in China, in order to speed up the acquisition of Go modules, it is recommended to set the `GOPROXY` environment variable to the domestic mirror server by executing `go env -w GOPROXY=https://goproxy.cn,direct`. For details, see [Goproxy China](https://github.com/goproxy/goproxy.cn).\n:::\n\n### Unix-like client\n\nCompiling clients for Linux, macOS, BSD and other Unix-like systems requires the following dependencies:\n\n- [Go](https://golang.org) 1.20+\n- GCC 5.4+\n\n1. Clone the source code:\n\n ```shell\n git clone https://github.com/juicedata/juicefs.git\n ```\n\n2. Enter the source code directory:\n\n ```shell\n cd juicefs\n ```\n\n3. Switch to the desired branch, such as release v1.0.0:\n\n The source code uses the `main` branch by default. You can switch to any official release, for example, to the release `v1.0.0`:\n\n ```shell\n git checkout v1.0.0\n ```\n\n :::caution\n The development branch often involves large changes, so do not use the clients compiled in the \"development branch\" for the production environment.\n :::\n\n4. Compile:\n\n ```shell\n make\n ```\n\n The compiled `juicefs` binary is located in the current directory.\n\n### Compiling on Windows\n\nTo compile the JuiceFS client on Windows systems, you need to install the following dependencies:\n\n- [WinFsp](https://github.com/winfsp/winfsp)\n- [Go](https://golang.org) 1.20+\n- GCC 5.4+\n\nAmong them, WinFsp and Go can be downloaded and installed directly. GCC needs to use a version provided by a third party, which can use [MinGW-w64](https://www.mingw-w64.org) or [Cygwin](https://www.cygwin.com). Here we take MinGW-w64 as an example.\n\nOn the [MinGW-w64 download page](https://www.mingw-w64.org/downloads), select a precompiled version for Windows, such as [mingw-builds-binaries](https://github.com/niXman/mingw-builds-binaries/releases). After downloading, extract it to the root directory of the `C` drive, then find PATH in the system environment variable settings and add the `C:\\mingw64\\bin` directory. After restarting the system, execute the `gcc -v` command in the command prompt or PowerShell. If you can see version information, it means that MingGW-w64 is successfully installed, and you can start compiling.\n\n1. Clone and enter the project directory:\n\n ```shell\n git clone https://github.com/juicedata/juicefs.git && cd juicefs\n ```\n\n2. Copy WinFsp headers:\n\n ```shell\n mkdir \"C:\\WinFsp\\inc\\fuse\"\n ```\n\n ```shell\n copy .\\hack\\winfsp_headers\\* C:\\WinFsp\\inc\\fuse\\\n ```\n\n ```shell\n dir \"C:\\WinFsp\\inc\\fuse\"\n ```\n\n ```shell\n set CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n ```\n\n ```shell\n go env -w CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n ```\n\n3. Compile the client:\n\n ```shell\n go build -ldflags=\"-s -w\" -o juicefs.exe .\n ```\n\nThe compiled `juicefs.exe` binary program is located in the current directory. For convenience, it can be moved to the `C:\\Windows\\System32` directory, so that the `juicefs.exe` command can be used directly anywhere.\n\n### Cross-compiling Windows clients on Linux\n\nCompiling a specific version of the client for Windows is essentially the same as [Unix-like Client](#unix-like-client) and can be done directly on a Linux system. However, in addition to `go` and `gcc`, you also need to install [MinGW-w64](https://www.mingw-w64.org/downloads).\n\nThe latest version can be installed from software repositories on many Linux distributions. For example, on Ubuntu 20.04+, you can install `mingw-w64` with the following command:\n\n```shell\nsudo apt install mingw-w64\n```\n\nCompile the Windows client:\n\n```shell\nmake juicefs.exe\n```\n\nThe compiled client is a binary file named `juicefs.exe`, located in the current directory.\n\n### Cross-compiling Linux clients on macOS\n\n1. Clone and enter the project directory:\n\n ```shell\n git clone https://github.com/juicedata/juicefs.git && cd juicefs\n ```\n\n2. Install dependencies:\n\n ```shell\n brew install FiloSottile/musl-cross/musl-cross\n ```\n\n3. Compile the client:\n\n ```shell\n make juicefs.linux\n ```\n\n## Uninstall {#uninstall}\n\nThe JuiceFS client has only one binary file, so it can be easily deleted once you find the location of the program. For example, to uninstall the client that is installed on the Linux system as described above, you only need to execute the following command:\n\n```shell\nsudo rm /usr/local/bin/juicefs\n```\n\nYou can also check where the program is located by using the `which` command:\n\n```shell\nwhich juicefs\n```\n\nThe path returned by the command is the location where the JuiceFS client is installed on your system. The uninstallation of the JuiceFS client on other operating systems follows the same way.\n" }, { "path": "docs/en/getting-started/standalone.md", "content": "---\nsidebar_position: 2\npagination_next: getting-started/for_distributed\ndescription: Learn how to use JuiceFS in standalone mode, combining object storage and databases for efficient file system management.\n---\n\n# Standalone Mode\n\nThe JuiceFS file system is driven by both [\"Object Storage\"](../reference/how_to_set_up_object_storage.md) and [\"Database\"](../reference/how_to_set_up_metadata_engine.md). In addition to object storage, it also supports using local disks, WebDAV, and HDFS as underlying storage options. Therefore, you can create a standalone file system using local disks and SQLite database to get a quick overview of how JuiceFS works.\n\n## Install the client\n\nFor Linux distributions and macOS users, you can quickly install the JuiceFS client using a one-click installation script:\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\nFor other operating systems and installation methods, refer to [Installation](installation.md).\n\nOnce installed successfully, executing the `juicefs` command in the terminal will return a help message regardless of the operating system.\n\n## Create a file system {#juicefs-format}\n\n### Basic concepts\n\nThe JuiceFS client provides the [`format`](../reference/command_reference.mdx#format) command to create a file system as follows:\n\n```shell\njuicefs format [command options] META-URL NAME\n```\n\nTo format a file system, three types of information are required:\n\n- **[command options]**: Specifies the storage medium for the file system. By default, the **local disk** is used with the path set to `\"$HOME/.juicefs/local\"`(on darwin/macOS), `\"/var/jfs\"`(on Linux), or `\"C:/jfs/local\"`(on Windows).\n- **META-URL**: Defines the metadata engine, typically a URL or the file path of a database.\n- **NAME**: The name of the file system.\n\n:::tip\nJuiceFS supports a wide range of storage media and metadata storage engines. For more details, see [JuiceFS supported storage media](../reference/how_to_set_up_object_storage.md) and [JuiceFS supported metadata storage engines](../reference/how_to_set_up_metadata_engine.md).\n:::\n\n### Hands-on practice\n\nFor example, on a Linux system, you can create a file system named `myjfs` with the following command:\n\n```shell\njuicefs format sqlite3://myjfs.db myjfs\n```\n\nAfter executing the command, you will receive an output similar to the following:\n\n```shell {1,4}\n2021/12/14 18:26:37.666618 juicefs[40362] : Meta address: sqlite3://myjfs.db\n[xorm] [info] 2021/12/14 18:26:37.667504 PING DATABASE sqlite3\n2021/12/14 18:26:37.674147 juicefs[40362] : The latency to database is too high: 7.257333ms\n2021/12/14 18:26:37.675713 juicefs[40362] : Data use file:///Users/herald/.juicefs/local/myjfs/\n2021/12/14 18:26:37.689683 juicefs[40362] : Volume is formatted as {Name:myjfs UUID:d5bdf7ea-472c-4640-98a6-6f56aea13982 Storage:file Bucket:/Users/herald/.juicefs/local/ AccessKey: SecretKey: BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\nThis output shows that SQLite is being used as the metadata storage engine. The database file named `myjfs.db` is located in the current directory. It creates a table to store all the metadata for the `myjfs` file system.\n\n![SQLite-info](../images/sqlite-info.png)\n\nSince no storage-related options were specified, the file system uses the local disk by default, with the storage path set to `file:///Users/herald/.juicefs/local/myjfs/`.\n\n## Mount the file system\n\n### Basic concepts\n\nThe JuiceFS client provides the [`mount`](../reference/command_reference.mdx#mount) command to mount file systems in the following format:\n\n```shell\njuicefs mount [command options] META-URL MOUNTPOINT\n```\n\nSimilar to the command of creating a file system, the following information is also required to mount a file system:\n\n- `[command options]`: Specifies file system-related options. For example, `-d` enables background mounting.\n- `META-URL`: Defines the metadata storage, typically a URL or file path of a database.\n- `MOUNTPOINT`: Specifies a mount point of the file system.\n\n:::tip\nThe mount point (`MOUNTPOINT`) on Windows systems should be an unused drive letter, such as `Z:` or `Y:`.\n:::\n\n### Hands-on practice\n\n:::note\nAs SQLite is a single-file database, please pay attention to the path of the database file when mounting it. JuiceFS supports both relative and absolute paths.\n:::\n\nTo mount the `myjfs` file system to the `~/jfs` folder, use the following command:\n\n```shell\njuicefs mount sqlite3://myjfs.db ~/jfs\n```\n\n![SQLite-mount-local](../images/sqlite-mount-local.png)\n\nThe client mounts the file system in the foreground by default. As you can see in the above image, the program keeps running in the current terminal. To unmount the file system, press Ctrl + C or close the terminal window.\n\nTo keep the file system mounted in the background, specify the `-d` or `--background` option when mounting. This allows you to mount the file system in the daemon:\n\n```shell\njuicefs mount sqlite3://myjfs.db ~/jfs -d\n```\n\nNext, any files stored in the `~/jfs` mount point will be split into specific blocks according to [How JuiceFS Stores Files](../introduction/architecture.md#how-juicefs-store-files) and stored in the `$HOME/.juicefs/local/myjfs` directory; the corresponding metadata will be stored in the `myjfs.db` database.\n\nTo unmount `~/jfs`, execute the following command:\n\n```shell\njuicefs umount ~/jfs\n```\n\n## Further exploration\n\nThe example above offers a quick local experience with JuiceFS and a basic understanding of its operation. For a more practical use case, you can use SQLite for metadata storage while replacing local storage with \"object storage.\"\n\n### Object storage\n\nObject storage is a web storage service based on the HTTP protocol that offers simple APIs for access. It has a flat structure and is easy to scale and cost-effective, particularly suitable for storing large amounts of unstructured data. Almost all mainstream cloud computing platforms provide object storage services, such as Amazon S3, Alibaba Cloud OSS, and Backblaze B2.\n\nJuiceFS supports almost all object storage services, see [JuiceFS supported storage media](../reference/how_to_set_up_object_storage.md).\n\nTo set up object storage:\n\n1. Create a **Bucket** and get the Endpoint address.\n2. Create the **Access Key ID** and **Access Key Secret**, which serve as the access keys for the Object Storage API.\n\nTaking AWS S3 as an example, the created resources would look like the following:\n\n- **Bucket Endpoint**: `https://myjfs.s3.us-west-1.amazonaws.com`\n- **Access Key ID**: `ABCDEFGHIJKLMNopqXYZ`\n- **Access Key Secret**: `ZYXwvutsrqpoNMLkJiHgfeDCBA`\n\n:::note\nThe process of creating object storage may vary slightly from platform to platform. It is recommended to check the help manual of the corresponding cloud platform. In addition, some platforms may provide different Endpoint addresses for internal and external networks. Please choose the external network access for your application. This document illustrates accessing object storage from a local environment.\n:::\n\n### Hands-on practice\n\nTo create a JuiceFS file system using SQLite and Amazon S3 object storage:\n\n:::note\nIf the `myjfs.db` file already exists, delete it first and then execute the following command.\n:::\n\n```shell\n# Replace relevant options with the actual object storage being used\njuicefs format --storage s3 \\\n --bucket https://myjfs.s3.us-west-1.amazonaws.com \\\n --access-key ABCDEFGHIJKLMNopqXYZ \\\n --secret-key ZYXwvutsrqpoNMLkJiHgfeDCBA \\\n sqlite3://myjfs.db myjfs\n```\n\nThe command above creates a file system using the same database name and file system name with the object storage options provided.\n\n- `--storage`: Specifies the storage type, such as `oss` or `s3`.\n- `--bucket`: Specifies the Endpoint address of the object storage.\n- `--access-key`: Specifies the Object Storage Access Key ID.\n- `--secret-key`: Specifies the Object Storage Access Key Secret.\n\nOnce created, you can mount the file system:\n\n```shell\njuicefs mount sqlite3://myjfs.db ~/jfs\n```\n\nThe mount command is exactly the same as using the local storage because JuiceFS has already written the metadata of the object storage to the `myjfs.db` database. Therefore, you do not need to provide it again when mounting.\n\nCompared with using local disks, the combination of SQLite and object storage is more practical. From an application perspective, this approach is equivalent to plugging an object storage with almost unlimited capacity into your local computer, allowing you to use cloud storage as a local disk.\n\nFurther, all the data of the file system is stored in the cloud-based object storage. Therefore, the `myjfs.db` database can be copied to other computers where JuiceFS clients are installed for mounting, reading, and writing. That is, any computer that can read the metadata database can mount and read/write the file system.\n\nObviously, it is difficult for a single file database like SQLite to be accessed by multiple computers at the same time. If SQLite is replaced by databases like Redis, PostgreSQL, and MySQL, which can be accessed by multiple computers at the same time through the network, it is possible to achieve distributed reads and writes on the JuiceFS file system.\n" }, { "path": "docs/en/grafana_template.json", "content": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"datasource\",\n \"uid\": \"grafana\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"gnetId\": 20794,\n \"graphTooltip\": 0,\n \"id\": 9,\n \"links\": [],\n \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${datasource}\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"custom\": {\n \"align\": \"auto\",\n \"cellOptions\": {\n \"type\": \"auto\"\n },\n \"inspect\": false\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": [\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"uptime\"\n },\n \"properties\": [\n {\n \"id\": \"custom.cellOptions\",\n \"value\": {\n \"mode\": \"gradient\",\n \"type\": \"color-background\"\n }\n },\n {\n \"id\": \"color\",\n \"value\": {\n \"mode\": \"continuous-YlBl\"\n }\n }\n ]\n }\n ]\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 31,\n \"options\": {\n \"cellHeight\": \"sm\",\n \"footer\": {\n \"countRows\": false,\n \"fields\": \"\",\n \"reducer\": [\n \"sum\"\n ],\n \"show\": false\n },\n \"showHeader\": true\n },\n \"pluginVersion\": \"10.2.3\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"P3DC81DD2E812B130\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"sort(max(juicefs_uptime{vol_name=\\\"$name\\\"}) by (${node_label}, juicefs_version))\",\n \"format\": \"table\",\n \"instant\": true,\n \"legendFormat\": \"__auto\",\n \"range\": false,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Uptime\",\n \"transformations\": [\n {\n \"id\": \"organize\",\n \"options\": {\n \"excludeByName\": {\n \"Time\": true\n },\n \"includeByName\": {},\n \"indexByName\": {\n \"Time\": 0,\n \"Value\": 3,\n \"${node_label}\": 1,\n \"juicefs_version\": 2\n },\n \"renameByName\": {\n \"Value\": \"uptime\",\n \"${node_label}\": \"\",\n \"juicefs_version\": \"version\"\n }\n }\n }\n ],\n \"type\": \"table\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"bytes\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 7\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"avg(juicefs_used_space{vol_name=\\\"$name\\\"})\",\n \"format\": \"time_series\",\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Data Size\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Data Size\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"none\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 7\n },\n \"id\": 4,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"avg(juicefs_used_inodes{vol_name=\\\"$name\\\"})\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Files\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Files\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"none\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 7\n },\n \"id\": 5,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"count by (juicefs_version) (juicefs_uptime{vol_name=\\\"$name\\\"})\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{juicefs_version}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Client Sessions\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 13\n },\n \"id\": 8,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_fuse_ops_durations_histogram_seconds_count{vol_name=\\\"$name\\\"}[$__rate_interval]) < 5000000000) by (${node_label})\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Ops {{${node_label}}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Operations\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"binBps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 13\n },\n \"id\": 7,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_fuse_written_size_bytes_sum{vol_name=\\\"$name\\\"}[$__rate_interval]) < 5000000000) by (${node_label})\",\n \"format\": \"time_series\",\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Write {{${node_label}}}\",\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(rate(juicefs_fuse_read_size_bytes_sum{vol_name=\\\"$name\\\"}[$__rate_interval]) < 5000000000) by (${node_label})\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Read {{${node_label}}}\",\n \"refId\": \"B\"\n }\n ],\n \"title\": \"IO Throughput\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"µs\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 13\n },\n \"id\": 18,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(rate(juicefs_fuse_ops_durations_histogram_seconds_sum{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) * 1000000 / sum(rate(juicefs_fuse_ops_durations_histogram_seconds_count{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"IO Latency\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 19\n },\n \"id\": 13,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(rate(juicefs_transaction_durations_histogram_seconds_count{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label})\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Transactions\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"µs\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 19\n },\n \"id\": 14,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(rate(juicefs_transaction_durations_histogram_seconds_sum{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) * 1000000 / sum(rate(juicefs_transaction_durations_histogram_seconds_count{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Transaction Latency\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 19\n },\n \"id\": 20,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(rate(juicefs_transaction_restart{vol_name=~\\\"$name\\\"}[$__rate_interval])) by (${node_label})\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Restarts {{${node_label}}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Transaction Restarts\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 25\n },\n \"id\": 15,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(rate(juicefs_object_request_durations_histogram_seconds_count{vol_name=\\\"$name\\\"}[$__rate_interval])) by (method)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{method}}\",\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_object_request_errors{vol_name=\\\"$name\\\"}[$__rate_interval])) \",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"errors\",\n \"refId\": \"B\"\n }\n ],\n \"title\": \"Objects Requests\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"Bps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 25\n },\n \"id\": 17,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_object_request_data_bytes{method=\\\"PUT\\\",vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},method)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{method}} {{${node_label}}}\",\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_object_request_data_bytes{method=\\\"GET\\\",vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},method)\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{method}} {{${node_label}}}\",\n \"refId\": \"B\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_object_request_data_bytes{method=\\\"GET\\\",vol_name=\\\"$name\\\"}[$__rate_interval]))\",\n \"hide\": false,\n \"interval\": \"\",\n \"legendFormat\": \"Total\",\n \"refId\": \"C\"\n }\n ],\n \"title\": \"Objects Throughput\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"µs\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 25\n },\n \"id\": 16,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(rate(juicefs_object_request_durations_histogram_seconds_sum{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label}) * 1000000 / sum(rate(juicefs_object_request_durations_histogram_seconds_count{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label})\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Objects Latency\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"percent\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 31\n },\n \"id\": 10,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_cpu_usage{vol_name=\\\"$name\\\"}[$__rate_interval])*100 < 1000) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Client CPU Usage\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"bytes\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 31\n },\n \"id\": 11,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"expr\": \"sum(juicefs_memory{vol_name=\\\"$name\\\"}) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Client Memory Usage\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"none\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 31\n },\n \"id\": 21,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"sum(juicefs_go_goroutines{vol_name=\\\"$name\\\"}) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Go threads\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"bytes\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 37\n },\n \"id\": 22,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(juicefs_blockcache_bytes{vol_name=\\\"$name\\\"}) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Block Cache Size\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"none\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 37\n },\n \"id\": 23,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(juicefs_blockcache_blocks{vol_name=\\\"$name\\\"}) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Block Cache Count\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"percent\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 37\n },\n \"id\": 24,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_blockcache_hits{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) *100 / (sum(rate(juicefs_blockcache_hits{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) + sum(rate(juicefs_blockcache_miss{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp))\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Hits {{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_blockcache_hit_bytes{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) *100 / (sum(rate(juicefs_blockcache_hit_bytes{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) + sum(rate(juicefs_blockcache_miss_bytes{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp))\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"HitBytes {{${node_label}}}:{{mp}}\",\n \"refId\": \"B\"\n }\n ],\n \"title\": \"Block Cache Hit Ratio\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 43\n },\n \"id\": 25,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_compact_size_histogram_bytes_count{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Compaction\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"binBps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 43\n },\n \"id\": 26,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_compact_size_histogram_bytes_sum{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Compacted Data\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"description\": \"\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 43\n },\n \"id\": 27,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"exemplar\": true,\n \"expr\": \"sum(juicefs_fuse_open_handlers{vol_name=\\\"$name\\\"}) by (${node_label},mp)\",\n \"format\": \"time_series\",\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Open File Handlers\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"normal\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 0,\n \"y\": 49\n },\n \"id\": 28,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"juicefs_staging_blocks{vol_name=\\\"$name\\\"}\",\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Juicefs Staging Blocks\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"normal\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"bytes\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 8,\n \"y\": 49\n },\n \"id\": 29,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"juicefs_staging_block_bytes{vol_name=\\\"$name\\\"}\",\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Juicefs Staging Block Usage\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 8,\n \"x\": 16,\n \"y\": 49\n },\n \"id\": 30,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"10.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$datasource\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": true,\n \"expr\": \"sum(rate(juicefs_staging_block_delay_seconds{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) / sum(rate(juicefs_object_request_durations_histogram_seconds_count{vol_name=\\\"$name\\\"}[$__rate_interval])) by (${node_label},mp) \",\n \"hide\": false,\n \"legendFormat\": \"{{${node_label}}}:{{mp}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Juicefs Staging Block Delay\",\n \"type\": \"timeseries\"\n }\n ],\n \"refresh\": \"\",\n \"schemaVersion\": 39,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"selected\": false,\n \"text\": \"prometheus\",\n \"value\": \"ef7836b8-b451-4d56-ad1e-1838d429b738\"\n },\n \"hide\": 0,\n \"includeAll\": false,\n \"multi\": false,\n \"name\": \"datasource\",\n \"options\": [],\n \"query\": \"prometheus\",\n \"queryValue\": \"\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"skipUrlSync\": false,\n \"type\": \"datasource\"\n },\n {\n \"current\": {\n \"selected\": false,\n \"text\": \"instance\",\n \"value\": \"instance\"\n },\n \"description\": \"Select the node label type based on deployment environment\",\n \"hide\": 0,\n \"includeAll\": false,\n \"multi\": false,\n \"label\": \"node label\",\n \"name\": \"node_label\",\n \"options\": [\n {\n \"selected\": true,\n \"text\": \"instance\",\n \"value\": \"instance\"\n },\n {\n \"selected\": false,\n \"text\": \"node\",\n \"value\": \"node\"\n }\n ],\n \"query\": \"instance,node\",\n \"queryValue\": \"\",\n \"skipUrlSync\": false,\n \"type\": \"custom\"\n },\n {\n \"current\": {\n \"selected\": true,\n \"text\": \"test1\",\n \"value\": \"test1\"\n },\n \"datasource\": {\n \"uid\": \"${datasource}\"\n },\n \"definition\": \"label_values(juicefs_uptime, vol_name)\",\n \"hide\": 0,\n \"includeAll\": false,\n \"multi\": false,\n \"name\": \"name\",\n \"options\": [],\n \"query\": {\n \"query\": \"label_values(juicefs_uptime, vol_name)\",\n \"refId\": \"StandardVariableQuery\"\n },\n \"refresh\": 2,\n \"regex\": \"\",\n \"skipUrlSync\": false,\n \"sort\": 0,\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-5m\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"\",\n \"title\": \"JuiceFS Dashboard\",\n \"uid\": \"-hm07csGk\",\n \"version\": 2,\n \"weekStart\": \"\"\n}\n" }, { "path": "docs/en/guide/cache.md", "content": "---\ntitle: Cache\nsidebar_position: 3\n---\n\nFor a file system driven by a combination of object storage and database, cache is an important medium for interacting efficiently between the local client and the remote service. Read and write data can be loaded into the cache in advance or asynchronously, and then the client uploads to or prefetches from the remote service in the background. The use of caching technology can significantly reduce the latency of storage operations and increase data throughput compared to interacting with remote services directly.\n\nJuiceFS provides various caching mechanisms including metadata caching, data read/write caching, etc.\n\n:::tip Does my application really need cache?\nData caching will effectively improve the performance of random reads. For applications that require high random read performance (e.g. Elasticsearch, ClickHouse), it is recommended to use faster storage medium and allocate more space for cache.\n\nMeanwhile, cache improve performance only when application needs to repeatedly read files, so if you know for sure you're in a scenario where data is only accessed once (e.g. data cleansing during ETL), you can safely turn off cache to prevent overhead.\n:::\n\n## Consistency {#consistency}\n\nDistributed systems often need to make tradeoffs between cache and consistency. Due to the decoupled architecture of JuiceFS, think consistency problems in terms of metadata, file data (in object storage), and file data local cache.\n\nFor [metadata](#metadata-cache), the default configuration offers a \"close-to-open\" consistency guarantee, i.e. after a client modified and closed a file, other clients will see the latest state when they open this file again. Also, default mount option uses 1s of kernel metadata cache which provides decent performance for the usual cases. If your scenario demands higher level of cache performance, learn how to tune cache settings in below sections. In particular, the client (mount point) initiating file modifications will enjoy a stronger consistency, read [consistency exceptions](#consistency-exceptions) for more.\n\nAs for object storage, JuiceFS clients split files into data blocks (default 4MiB), each is assigned an unique ID and uploaded to object storage. Subsequent modifications on the file are carried out on new data blocks, and the original blocks remain unchanged. This guarantees consistency of the object storage data, because once the file is modified, clients will then read from the new data blocks, while the stale ones which will be deleted through [Trash](../security/trash.md) or compaction.\n\n[Local file data cache](#client-read-cache) is object storage blocks downloaded into local disks. So consistency depends on the reliability of the disks, if data are tempered, clients will read bad data. To resolve this concern, choose an appropriate [`--verify-cache-checksum`](../reference/command_reference.mdx#mount-data-cache-options) strategy to ensure data integrity.\n\n## Metadata cache {#metadata-cache}\n\nAs a userspace filesystem, JuiceFS metadata cache is both managed as kernel cache (via FUSE API), and maintained in client memory space.\n\n### Metadata cache in kernel {#kernel-metadata-cache}\n\nJuiceFS Client controls these kinds of metadata as kernel cache: attribute (file name, size, permission, mtime, etc.), entry (inode, name, and type. The word \"entry\" and \"dir-entry\" is used in parameter names, to further distinguish between file and directory). Use the following parameters to control TTL through FUSE:\n\n```\n# File attribute cache TTL in seconds, default to 1, improves getattr performance\n--attr-cache=1\n\n# File entry cache TTL in seconds, default to 1, improves lookup performance\n--entry-cache=1\n\n# Directory entry cache TTL in seconds, default to 1, improves lookup performance\n--dir-entry-cache=1\n\n# Negative lookup (return ENOENT) cache TTL in seconds，default to 0，improves lookup performance for non-existent files or directories\n--negative-entry-cache=1\n```\n\nCaching these metadata in kernel for 1 second really speeds up `lookup` and `getattr` calls.\n\nDo note that `entry` cache is gradually built upon file access and may not contain a complete file list under directory, so `readdir` calls or `ls` command cannot utilize this cache, that's why `entry` cache only improves `lookup` performance. The meaning of `direntry` here is different from [kernel directory entry](https://www.kernel.org/doc/html/latest/filesystems/ext4/directory.html), `direntry` does not tell you the files under a directory, but rather, it's the same concept as `entry`, just distinguished based on whether it's a directory.\n\nReal world scenarios scarcely require setting different values for `--entry-cache` and `--dir-entry-cache`, these options exist for theoretical possibilities like when directories seldom change while files change a lot, in that situation, you can use a higher `--dir-entry-cache` than `--entry-cache`.\n\n### Metadata cache in client memory {#client-memory-metadata-cache}\n\nWhen JuiceFS Client `open` a file, its file attributes are cached in client memory, this attribute cache includes not only the kernel cached file attributes like size, mtime, but also information specific to JuiceFS like [the relationship between file and chunks and slices](../introduction/architecture.md#how-juicefs-store-files).\n\nTo maintain the default close-to-open consistency, `open` calls will always query metadata service, bypassing local cache, modifications done by client A isn't guaranteed available immediately for client B, but once A closes file, all other clients (across different nodes) will see the latest state. File attribute cache isn't necessarily obtained through `open`, for example `tail -f` will periodically query attributes, in this case, latest state is fetched without reopening the file.\n\nTo utilize the memory metadata cache, use [`--open-cache`](../reference/command_reference.mdx#mount-metadata-cache-options) to specify its TTL, so that before cache expiration, `getattr` and `open` calls directly uses the slice information in client memory. These cached information avoids the overhead of querying metadata service on every call.\n\nWith `--open-cache` enabled, JuiceFS no longer operates under close-to-open consistency, but similar to kernel metadata cache, the client initiating the modifications can also actively invalidate client memory metadata cache, while other clients can only wait for expiration. That's why in order to maintain semantics, `--open-cache` is disabled by default. For read intensive (or read-only) scenarios, such as AI model training, it is recommended to set `--open-cache` according to the situation to further improve the read performance.\n\nIn comparison, JuiceFS Enterprise Edition provides richer functionalities around memory metadata cache (supports active invalidation). Read [Enterprise Edition documentation](https://juicefs.com/docs/cloud/guide/cache/#client-memory-metadata-cache) for more.\n\n### Consistency exceptions {#consistency-exceptions}\n\nThe metadata cache in discussed above really only pertain to multi-client situations, which can be deemed as a \"minimum consistency guarantee\". But for the client initiating file changes, it's not hard to imagine that due to changes happening \"locally\", the client initiating changes will enjoy a higher level of consistency:\n\n* For the mount point initiating changes, kernel cache is automatically invalidated upon modification. But when different mount points access and modify the same file, active kernel cache invalidation is only effective on the client initiating the modifications, other clients can only wait for expiration.\n* When concurrent operations are performed on multiple mount points, if one client deletes and then recreates a file with the same name, other clients may still use the old file’s inode due to the kernel’s entry cache. As a result, they may fail to find the file or may read old content (if the trash feature is enabled), and must wait for the entry cache to expire. This situation does not fall under traditional “close-to-open” consistency semantics, because the file is no longer the same object.\n* When a `write` call completes, the mount point itself will immediately see the resulting file length change (e.g. use `ls -al` to verify that file size is growing)——but this doesn't mean the changes have been committed, before `flush` finishes, these modifications will not be reflected onto the object storage, and other mount points cannot see the latest writes. Using methods like `fsync, fdatasync, close` will all result in `flush` calls that will persist the file changes and make them visible to other clients.\n* As an extreme case of the previous situation, if you `write` successfully and have observed file size change in the current mount point, but the eventual `flush` fails for some reason, for example file system usage exceeds global quota, then the previously growing file size will suddenly be reduced, for example, dropped from 10M to 0, this often leads to misunderstanding that JuiceFS just emptied your files, while the reality is that the files haven't been successfully written from the beginning, the file size change that's only available to the current mount point is simply a preview of things, not an actual committed state.\n* The mount point initiating changes have access to file change events, and can use tools like [`fswatch`](https://emcrisostomo.github.io/fswatch/) or [`Watchdog`](https://python-watchdog.readthedocs.io/en/stable). But the scope is obviously limited to the files changed within the mount point itself, i.e. files modified by A cannot be monitored by mount point B.\n* Due to the fact that FUSE doesn't yet support inotify API, if you'd like to monitor file change events using libraries like [Watchdog](https://python-watchdog.readthedocs.io/en/stable), you can only achieve this via polling (e.g. [`PollingObserver`](https://python-watchdog.readthedocs.io/en/stable/_modules/watchdog/observers/polling.html#PollingObserver)).\n\n## Read/Write buffer {#buffer-size}\n\nThe Read/Write buffer is a memory space allocated to the JuiceFS Client, size controlled by [`--buffer-size`](../reference/command_reference.mdx#mount-data-cache-options) which defaults to 300 (in MiB). Read/Write data all pass through this buffer, making it crucial for all I/O operations, that's why under large scale scenarios, increase buffer size is often used as a first step of optimization.\n\n### Readahead and prefetch {#readahead-prefetch}\n\n:::tip\nTo accurately describe the internal mechanism of JuiceFS Client, we use the term \"readahead\" and \"prefetch\" to refer to the two different behaviors that both download data ahead of time to increase read performance.\n:::\n\nWhen a file is sequentially read, JuiceFS Client performs what's called \"readahead\", which involves downloading data ahead of the current read offset. In fact, the similar behavior is already being built into the [Linux Kernel](https://www.halolinux.us/kernel-architecture/page-cache-readahead.html): when reading files, kernel dynamically settles on a readahead window based on the actual read behavior, and load file into the page cache. With JuiceFS being a network file system, the classic kernel readahead mechanism is simply not enough to bring the desired performance increase, that's why JuiceFS performs its own readahead on top of kernel readahead, using algorithm to \"guess\" the size of the readahead window (more aggressive than kernel's), and then download the object storage data in advance. The maximum readahead window size can be controlled by the `--max-readahead` parameter. In random read scenarios, you may consider setting it to 0 to disable readahead.\n\n![readahead](../images/buffer-readahead.svg)\n\nApparently readahead is only good for sequential reads, that's why there's another similar mechanism called \"prefetch\": when a block is randomly read by a small offset range, the whole block is scheduled for download asynchronously.\n\n![prefetch](../images/buffer-prefetch.svg)\n\nThis mechanism assumes that if a file is randomly read at a given range, then its adjacent content is also more likely to get read momentarily. This isn't necessarily true for various different types of applications, for example, if an application decides to read read a huge file in a very sparse fashion, i.e. read offsets are far from each other. In such case, prefetch isn't really useful and can cause serious read amplification, so if you are already familiar with the file system access pattern of your application, and concluded that prefetch isn't really needed, you can disable by using [`--prefetch=0`](../reference/command_reference.mdx#mount-data-cache-options).\n\nReadahead and prefetch effectively increase sequential read and random read performance, but it also comes with read amplification, read [\"Read amplification\"](../administration/troubleshooting.md#read-amplification) for more information.\n\n### Write {#buffer-write}\n\nA successful `write` does not mean data is persisted: that's actually `flush`'s job. This is true for both local file systems, and JuiceFS file systems. In JuiceFS, `write` only commits changes to the buffer, from the writing mount point's POV, you may notice that file size is changing, but do not mistake this for persistence (this behavior is also covered in detail in [consistency exceptions](#consistency-exceptions)). To sum up, before `flush` actually finishes, changes are only kept inside the client buffer. Applications may explicitly invoke `flush`, but even without this, `flush` is automatically triggered when a pending slice's size exceed its chunk border, or have waited in the buffer for a certain amount of time.\n\nTogether with the previously introduced readahead mechanism, buffer function can be described in below diagram:\n\n![read write buffer](../images/buffer-read-write.svg)\n\nBuffer is shared by both read & write, obviously write is treated with higher priority, this implies the possibility of write getting in the way of read. For instance, if object storage bandwidth isn't enough to support write load, there'll be congestion:\n\n![buffer congestion](../images/buffer-congestion.svg)\n\nAs illustrated above, a high write load puts too much pending slices inside the buffer, leaving little buffer space for readahead, file read will hence slow down. Due to a low upload speed, write may also fail due to `flush` timeouts.\n\n### Observation and optimization {#buffer-observation}\n\nBuffer is crucial to both read & write, as is already introduced in above sections, making `--buffer-size` the first optimization target when faced with large scale scenarios. But simply increasing buffer size is not enough and might cause other problems (like buffer congestion, illustrated in the above section). The size of the buffer should be smartly decided along with other performance options.\n\nBefore making any adjustments, we recommend running a [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) command to check the current buffer usage, and read below content to guide your tuning.\n\nIf you wish to improve sequential read speed, use larger `--max-readahead` and `--buffer-size` to expand the readahead window, all data blocks within the window will be concurrently fetched from object storage. Also keep in mind that, reading a single large file will never consume the full buffer, the space reserved for readahead is between 1/4 to 1/2 of the total buffer size. So if you noticed that `juicefs stats` indicates `buf` is already half full, while performing sequential read on a single large file, then it's time to increase `--buffer-size` to set a larger readahead window.\n\nIf you wish to improve write speed, and have already increased [`--max-uploads`](../reference/command_reference.mdx#mount-data-storage-options) for more upload concurrency, with no noticeable increase in upload traffic, consider also increasing `--buffer-size` so that concurrent threads may easier allocate memory for data uploads. This also works in the opposite direction: if tuning up `--buffer-size` didn't bring out an increase in upload traffic, you should probably increase `--max-uploads` as well.\n\nThe `--buffer-size` also controls the data upload size for each `flush` operation, this means for clients working in a low bandwidth environment, you may need to use a lower `--buffer-size` to avoid `flush` timeouts. Refer to [\"Connection problems with object storage\"](../administration/troubleshooting.md#io-error-object-storage) for troubleshooting under low internet speed.\n\n## Data cache {#data-cache}\n\nTo improve performance, JuiceFS also provides various caching mechanisms for data, including page cache in the kernel, local file system cache in client host, and read/write buffer in client process itself. Read requests will try the kernel page cache, the client process buffer, and the local disk cache in turn. If the data requested is not found in any level of the cache, it will be read from the object storage, and also be written into every level of the cache asynchronously to improve the performance of the next access.\n\n![JuiceFS-cache](../images/juicefs-cache.png)\n\n### Kernel page cache {#kernel-data-cache}\n\nKernel will build page cache for opened files. If this file is not updated (i.e. `mtime` doesn't change) afterwards, it will be read directly from the page cache to achieve the best performance.\n\nJuiceFS Client tracks a list of recently opened files. If file is opened again, client will check if file has been modified to decide whether the kernel page cache is valid, if file is already modified, all relevant page cache is invalidated on the next open, this ensures that the client can always read the latest data.\n\nRepeated reads of the same file in JuiceFS can be extremely fast, with latencies as low as a few microseconds and throughput up to several GiBs per second.\n\n### Kernel writeback-cache mode {#fuse-writeback-cache}\n\nStarting from Linux kernel 3.15, FUSE supports [writeback-cache](https://www.kernel.org/doc/Documentation/filesystems/fuse-io.txt) mode, the kernel will consolidate high-frequency random small (10-100 bytes) write requests to significantly improve its performance, but this comes with a side effect: sequential writes are also turned into random writes, hence sequential write performance is hindered, so only use it on intensive random write scenarios.\n\nTo enable writeback-cache mode, use the [`-o writeback_cache`](../reference/fuse_mount_options.md#writeback_cache) option when you [mount JuiceFS](../reference/command_reference.mdx#mount). Note that writeback-cache mode is not the same as [Client write data cache](#client-write-cache), the former is a kernel implementation while the latter happens inside the JuiceFS Client, read the corresponding section to learn their intended scenarios.\n\n### Read cache in client {#client-read-cache}\n\nThe client will perform prefetch and cache automatically to improve sequence read performance according to the read mode in the application. Data will be cached in local file system, which can be any local storage device like HDD, SSD or even memory.\n\nData downloaded from object storage, as well as small data (smaller than a single block) uploaded to object storage will be cached by JuiceFS Client, without compression or encryption. To achieve better performance on application's first read, use [`juicefs warmup`](../reference/command_reference.mdx#warmup) to cache data in advance.\n\nWhen '--writeback' is not enabled, if the file system where the cache directory is located is not working properly, the JuiceFS client can return an error and downgrade to direct access to object storage. In the case of enable '--writeback', if the file system where the cache directory is located is abnormal and the read operation is stuck (such as some kernel-mode network file system), then JuiceFS will also get stuck together. This requires you to tune the underlying file system behavior of the cache directory to fail fast.\n\nBelow are some important options for cache configuration (see [`juicefs mount`](../reference/command_reference.mdx#mount) for complete reference):\n\n* `--prefetch`\n\n Concurrent prefetch of N (1 by default) blocks. Prefetching refers to randomly reading a segment of a file's block, and the client asynchronously downloads the entire object storage block. Prefetching can often improve the performance of random reads. However, if the file access pattern in your scenario cannot effectively utilize the prefetched data (for example, reading large files randomly and sparsely), prefetching may lead to noticeable read amplification. In such cases, you can set it to 0 to disable the prefetch feature.\n\n JuiceFS is equipped with another internal similar mechanism called \"readahead\": when doing sequential reads, client will download nearby blocks in advance, improving sequential performance. The concurrency of readahead is affected by the size of [\"Read/Write Buffer\"](#buffer-size), the larger the read-write buffer, the higher the concurrency.\n\n* `--cache-dir`\n\n Cache directory, default to `/var/jfsCache` or `$HOME/.juicefs/cache`. Please read [\"Cache directory\"](#cache-dir) for more information.\n\n If you are in urgent need to free up disk space, you can manually delete data under the cache directory, which is `//raw/`.\n\n* `--cache-size` and `--free-space-ratio`\n\n Cache size (in MiB, default to 102400) and minimum ratio of free size (default 0.1). Both parameters is able to control cache size, if any of the two criteria is met, JuiceFS Client will expire cache usage using an algorithm similar to LRU, i.e. remove older and less used blocks.\n\n Actual cache size may exceed configured value, because it is difficult to calculate the exact disk space taken by cache. Currently, JuiceFS takes the sum of all cached objects sizes using a minimum 4 KiB size, which is often different from the result of `du`.\n\n* `--cache-partial-only`\n\n Only cache small files and random small reads, do not cache whole block. This applies to conditions where object storage throughput is higher than the local cache device. Default value is false.\n\n There are two main read patterns, sequential read and random read. Sequential read usually demands higher throughput while random reads needs lower latency. When local disk throughput is lower than object storage, consider enable `--cache-partial-only` so that sequential reads do not cache the whole block, but rather, only small reads (like footer of Parquet / ORC file) are cached. This allows JuiceFS to take advantage of low latency provided by local disk, and high throughput provided by object storage, at the same time.\n\n### Client write data cache {#client-write-cache}\n\nEnabling client write cache can improve performance when writing large amount of small files. Read this section to learn about client write cache.\n\nClient write cache is disabled by default, data writes will be held in the [read/write buffer](#buffer-size) (in memory), and is uploaded to object storage when a chunk is filled full, or forced by application with `close()`/`fsync()` calls. To ensure data security, client will not commit file writes to the Metadata Service until data is uploaded to object storage.\n\nYou can see how the default \"upload first, then commit\" write process will not perform well when writing large amount of small files. After the client write cache is enabled, the write process becomes \"commit first, then upload asynchronously\", file writes will not be blocked by data uploads, instead it will be written to the local cache directory and committed to the metadata service, and then returned immediately. The file data in the cache directory will be asynchronously uploaded to the object storage.\n\nIf you need to use JuiceFS as a temporary storage, which doesn't require persistence and distributed access, use [`--upload-delay`](../reference/command_reference.mdx#mount-data-cache-options) to delay data upload, this saves the upload process if files are deleted during the delay. Meanwhile, compared with a local disk, JuiceFS uploads files automatically when the cache directory is running out of space, which keeps the applications away from unexpected failures.\n\nAdd `--writeback` to the mount command to enable client write cache, but this mode comes with some risks and caveats:\n\n* Disk reliability is crucial to data integrity, if write cache data suffers loss before upload is complete, file data is lost forever. Use with caution when data reliability is critical.\n* Write cache data by default is stored in `/var/jfsCache//rawstaging/`, do not delete files under this directory or data will be lost.\n* Write cache size is controlled by [`--free-space-ratio`](#client-read-cache). By default, if the write cache is not enabled, the JuiceFS client uses up to 90% of the disk space of the cache directory (the calculation rule is `(1 - ) * 100`). After the write cache is enabled, a certain percentage of disk space will be overused. The calculation rule is `(1 - ( / 2)) * 100`, that is, by default, up to 95% of the disk space of the cache directory will be used.\n* Write cache and read cache share cache disk space, so they affect each other. For example, if the write cache takes up too much disk space, the size of the read cache will be limited, and vice versa.\n* If local disk write speed is lower than object storage upload speed, enabling `--writeback` will only result in worse write performance.\n* If the file system of the cache directory raises error, client will fallback and write synchronously to object storage, which is the same behavior as [Read Cache in Client](#client-read-cache).\n* If object storage upload speed is too slow (low bandwidth), local write cache can take forever to upload, meanwhile reads from other nodes will result in timeout error (I/O error). See [Connection problems with object storage](../administration/troubleshooting.md#io-error-object-storage).\n\nImproper usage of client write cache can easily cause problems, that's why only recommend to temporarily enable this when writing large number of small files (e.g. extracting a compressed file containing a large number of small files).\n\nWhen `--writeback` is enabled, apart from checking `/var/jfsCache//rawstaging/` directly, you can also view upload progress using:\n\n```shell\n# Assuming mount point is /jfs\n$ cd /jfs\n$ cat .stats | grep \"staging\"\njuicefs_staging_block_bytes 1621127168 # The size of the data blocks to be uploaded\njuicefs_staging_block_delay_seconds 46116860185.95535\njuicefs_staging_blocks 394 # The number of data blocks to be uploaded\n```\n\n### Cache directory {#cache-dir}\n\nDepending on the operating system, the default cache path for JuiceFS is as follows:\n\n- **Linux**: `/var/jfsCache`\n- **macOS**: `$HOME/.juicefs/cache`\n- **Windows**: `%USERPROFILE%\\.juicefs\\cache`\n\nFor Linux, note that the default cache path requires administrator privileges and that normal users need to be granted to use `sudo` to set it up, e.g.:\n\n```shell\nsudo juicefs mount redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\nAlternatively, the `--cache-dir` option can be set to any storage path accessible to the current system when mounting the filesystem. For normal users who do not have permission to access the `/var` directory, the cache can be set in the user's `HOME` directory, e.g.:\n\n```shell\njuicefs mount --cache-dir ~/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\n:::tip\nIt is recommended to use a high performance dedicated disk as the cache directory, avoid using the system disk, and do not share it with other applications. Sharing not only affects the performance of each other, but may also cause errors in other applications (such as insufficient disk space left). If it is unavoidable to share, you must estimate the disk capacity required by other applications, limit the size of the cache space (see below for details), and avoid JuiceFS's read cache or write cache takes up too much space.\n:::\n\n#### RAM disk\n\nIf a higher file read performance is required, you can set up the cache into the RAM disk. For Linux systems, check the `tmpfs` file system with the `df` command.\n\n```shell\n$ df -Th | grep tmpfs\ntmpfs tmpfs 362M 2.0M 360M 1% /run\ntmpfs tmpfs 3.8G 0 3.8G 0% /dev/shm\ntmpfs tmpfs 5.0M 4.0K 5.0M 1% /run/lock\n```\n\nWhere `/dev/shm` is a typical memory disk that can be used as a cache path for JuiceFS, it is typically half the capacity of memory and can be manually adjusted as needed, for example, to 32GB.\n\n```shell\nsudo mount -o size=32000M -o remount /dev/shm\n```\n\nThen, using that path as a cache, mount the filesystem.\n\n```shell\njuicefs mount --cache-dir /dev/shm/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\nAnother way to use memory for cache is set `--cache-dir` option to `memory`, this puts cache directly in client process memory, which is simpler compared to `/dev/shm`, but obviously cache will be lost after process restart, use this for tests and evaluations.\n\n#### Shared folders\n\nShared directories created via SMB or NFS can also be used as cache for JuiceFS. For the case where multiple devices on the LAN mount the same JuiceFS file system, using shared directories on the LAN as cache paths can effectively relieve the bandwidth pressure of duplicate caches for multiple devices.\n\nBut special attention needs to be paid. Usually, when the file system where the cache directory is located fails to work properly, the JuiceFS client can immediately return an error and downgrade to direct access to object storage. If the abnormality of the shared directory shows that the read operation is stuck (such as some network file system in kernel mode), then JuiceFS will also be stuck together. This requires you to tune the underlying file system behavior of the shared directory to achieve rapid failure.\n\nUsing SMB/CIFS as an example, mount the shared directories on the LAN by using the tools provided by the `cifs-utils` package.\n\n```shell\nsudo mount.cifs //192.168.1.18/public /mnt/jfscache\n```\n\nUsing shared directories as JuiceFS caches:\n\n```shell\nsudo juicefs mount --cache-dir /mnt/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\n#### Multiple cache directories\n\nJuiceFS supports setting multiple cache directories at the same time, thus avoiding the problem of insufficient cache space by separating multiple paths using `:` (Linux, macOS) or `;` (Windows), e.g.:\n\n```shell\nsudo juicefs mount --cache-dir ~/jfscache:/mnt/jfscache:/dev/shm/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\nWhen multiple cache directories are set, or multiple devices are used as cache disks, the `--cache-size` option represents the total size of data in all cache directories. The client will use the hash strategy to evenly write data to each cache path, and cannot perform special tuning for multiple cache disks with different capacities or performances.\n\nTherefore, it is recommended that the available space of different cache directories/cache disks be consistent, otherwise it may cause the situation that the space of a certain cache directory cannot be fully utilized. For example, `--cache-dir` is `/data1:/data2`, where `/data1` has a free space of 1GiB, `/data2` has a free space of 2GiB, `--cache-size` is 3GiB, `--free-space-ratio` is 0.1. Because the cache write strategy is to write evenly, the maximum space allocated to each cache directory is `3GiB / 2 = 1.5GiB`, resulting in a maximum of 1.5GiB cache space in the `/data2` directory instead of `2GiB * 0.9 = 1.8GiB`.\n" }, { "path": "docs/en/guide/clone.md", "content": "---\ntitle: Clone Files or Directories\nsidebar_position: 6\ndescription: Learn how to use the juicefs clone command to efficiently clone files or directories by creating a metadata-only copy. \n---\n\nCloning specific data does not involve copying the actual object storage data but only copies metadata. Therefore, cloning is very fast regardless of the size of the file or directory. For JuiceFS, this command is a better alternative to `cp`. Moreover, for Linux clients using kernels with [`copy_file_range`](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) support, using `cp` effectively performs the same metadata copy and is very fast.\n\n![clone](../images/juicefs-clone.svg)\n\nThe clone result is a metadata copy only, where all the files still reference the same underlying object storage blocks. That is why a clone behaves the same in every way as its originals. Upon any file data modification, new data is written to new object storage blocks, with its associating metadata redirected to the new blocks as well (ROW, Redirect-on-Write), while the unchanged part of the files remains the same, still referencing the original blocks. And just like native JuiceFS files, random write on a clone can result in file fragmentations, you can merge them with `juicefs compact` to process the slice of the file,in order to improve read performance.\n\nNote that system tools like disk-free or disk-usage (`df`, `du`) show the space used by the cloned data, but the underlying object storage space remains unchanged since the blocks are not duplicated. Cloning replicates metadata, so it will occupy the same metadata engine storage space as the original.\n\n**Cloning impacts file system storage space, inodes, and metadata engine storage space.** Be cautious when cloning large directories.\n\n```shell\njuicefs clone SRC DST\n\n# Clone a file\njuicefs clone /mnt/jfs/file1 /mnt/jfs/file2\n\n# Clone a directory\njuicefs clone /mnt/jfs/dir1 /mnt/jfs/dir2\n```\n\n## Consistency {#consistency}\n\nIn terms of transaction consistency, cloning behaves as follows:\n\n- The destination file is not visible until the `clone` command completes.\n- For files: The `clone` command ensures atomicity, meaning that the cloned file will always be in a correct and consistent state.\n- For directories: The `clone` command does not guarantee atomicity for directories. In other words, if the source directory changes during the cloning process, the target directory may be different from the source directory.\n- Only one `clone` operation can succeed from the same source at the same time. Any failed clones will clean up the temporarily created directory tree.\n\nThe cloning operation is performed by the mount process. It will be interrupted, if the `clone` command is terminated. If a cloning operation fails or is interrupted, the `mount` process will clean up any created inodes. If this cleanup fails, it may lead to metadata leaks and potential object storage leaks, because the dangling tree continues to reference the underlying data blocks. They could be cleaned up by the [`juicefs gc --delete`](../reference/command_reference.mdx#gc) command.\n" }, { "path": "docs/en/guide/dir-stats.md", "content": "---\ntitle: Directory Statistics\nsidebar_position: 5\ndescription: Learn how to enable, check, and troubleshoot directory statistics in JuiceFS version 1.1.0 and later. \n---\n\nFrom JuiceFS v1.1.0, the directory statistics feature is enabled by default when formatting a new volume. For existing volumes, this feature is disabled by default and must be enabled manually. The directory statistics feature accelerates the `quota`, `info` and `summary` subcommands, but it comes with a minor performance cost.\n\n:::tip\nDirectory statistics rely on the mount process. Ensure all writable mount processes are upgraded to v1.1.0 before enabling this feature.\n:::\n\n## Enable directory statistics {#enable-directory-stats}\n\nRun `juicefs config $URL --dir-stats` to enable directory statistics. After that, run `juicefs config $URL` to confirm the change:\n\n```shell\n$ juicefs config redis://localhost\n2023/05/31 15:56:39.721188 juicefs[30626] : Meta address: redis://localhost [interface.go:494]\n2023/05/31 15:56:39.723284 juicefs[30626] : Ping redis latency: 159.226µs [redis.go:3566]\n{\n \"Name\": \"myjfs\",\n \"UUID\": \"82db28de-bf5f-43bf-bba3-eb3535a86c48\",\n \"Storage\": \"file\",\n \"Bucket\": \"/root/.juicefs/local/\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"EncryptAlgo\": \"aes256gcm-rsa\",\n \"TrashDays\": 1,\n \"MetaVersion\": 1,\n \"DirStats\": true\n}\n```\n\nIf `\"DirStats\": true` appears, the directory statistics feature is successfully enabled. To disable it:\n\n```shell\n$ juicefs config redis://localhost --dir-stats=false\n2023/05/31 15:59:39.046134 juicefs[30752] : Meta address: redis://localhost [interface.go:494]\n2023/05/31 15:59:39.048301 juicefs[30752] : Ping redis latency: 171.308µs [redis.go:3566]\n dir-stats: true -> false\n```\n\n:::tip\nThe [directory quota](./quota.md#directory-quota) functionality depends on directory statistics, so setting a quota automatically enables directory statistics. To disable directory statistics for such volumes, you need to remove all quotas.\n:::\n\n## Check directory statistics {#check-directory-stats}\n\nUse `juicefs info $PATH` to check statistics for a single directory:\n\n```shell\n$ juicefs info /mnt/jfs/pjdfstest/\n/mnt/jfs/pjdfstest/ :\n inode: 2\n files: 10\n dirs: 4\n length: 43.74 KiB (44794 Bytes)\n size: 92.00 KiB (94208 Bytes)\n path: /pjdfstest\n```\n\nRun `juicefs info -r $PATH` to recursively sum up:\n\n```shell\n/mnt/jfs/pjdfstest/: 278 921.0/s\n/mnt/jfs/pjdfstest/: 1.6 MiB (1642496 Bytes) 5.2 MiB/s\n/mnt/jfs/pjdfstest/ :\n inode: 2\n files: 278\n dirs: 37\n length: 592.42 KiB (606638 Bytes)\n size: 1.57 MiB (1642496 Bytes)\n path: /pjdfstest\n```\n\nYou can also use `juicefs summary $PATH` to list all directory statistics:\n\n```shell\n$ ./juicefs summary /mnt/jfs/pjdfstest/\n/mnt/jfs/pjdfstest/: 315 1044.4/s\n/mnt/jfs/pjdfstest/: 1.6 MiB (1642496 Bytes) 5.2 MiB/s\n+------------------+---------+------+-------+\n| PATH | SIZE | DIRS | FILES |\n+------------------+---------+------+-------+\n| / | 1.6 MiB | 37 | 278 |\n| tests/ | 1.1 MiB | 18 | 240 |\n| tests/open/ | 112 KiB | 1 | 26 |\n| tests/... | 328 KiB | 7 | 71 |\n| .git/ | 432 KiB | 17 | 26 |\n| .git/objects/ | 252 KiB | 3 | 2 |\n| ... | 12 KiB | 0 | 3 |\n+------------------+---------+------+-------+\n```\n\n:::note\nDirectory statistics only track usage for individual directories. To get a recursive sum, use `juicefs info -r`. This could be a costly operation for large directories. If you need to frequently get the total statistics for particular directories, consider [setting an empty quota](./quota.md#limit-capacity-and-inodes-of-directory) for such directories to achieve recursive statistics.\n\nUnlike the Community Edition, JuiceFS Enterprise Edition provides a [recursive sum](/docs/cloud/guide/quota#file-directory-size) in directory statistics. You can directly view the total usage by running `ls -lh`.\n:::\n\n## Troubleshoot {#troubleshooting}\n\nDirectory statistics are calculated asynchronously and can potentially produce inaccurate results when clients run into problems. `juicefs info`, `juicefs summary`, and `juicefs quota` all provide a `--strict` option to run in strict mode. This bypasses directory statistics, unlike the default fast mode.\n\nWhen strict mode and fast mode produce different results, use `juicefs fsck` to diagnose:\n\n```shell\n$ juicefs info -r /jfs/d\n/jfs/d: 1 3.3/s\n/jfs/d: 448.0 MiB (469766144 Bytes) 1.4 GiB/s\n/jfs/d :\n inode: 2\n files: 1\n dirs: 1\n length: 448.00 MiB (469762048 Bytes)\n size: 448.00 MiB (469766144 Bytes)\n path: /d\n\n$ juicefs info -r --strict /jfs/d\n/jfs/d: 1 3.3/s\n/jfs/d: 1.0 GiB (1073745920 Bytes) 3.3 GiB/s\n/jfs/d :\n inode: 2\n files: 1\n dirs: 1\n length: 1.00 GiB (1073741824 Bytes)\n size: 1.00 GiB (1073745920 Bytes)\n path: /d\n\n# Check directory statistics for /d\n$ juicefs fsck sqlite3://test.db --path /d --sync-dir-stat\n2023/05/31 17:14:34.700239 juicefs[32667] : Meta address: sqlite3://test.db [interface.go:494]\n[xorm] [info] 2023/05/31 17:14:34.700291 PING DATABASE sqlite3\n2023/05/31 17:14:34.701553 juicefs[32667] : usage stat of /d should be &{1073741824 1073741824 1}, but got &{469762048 469762048 1} [base.go:2010]\n2023/05/31 17:14:34.701577 juicefs[32667] : Stat of path /d (inode 2) should be synced, please re-run with '--path /d --repair --sync-dir-stat' to fix it [base.go:2025]\n2023/05/31 17:14:34.701615 juicefs[32667] : some errors occurred, please check the log of fsck [main.go:31]\n\n# Fix directory statistics for /d\n$ juicefs fsck -v sqlite3://test.db --path /d --sync-dir-stat --repair\n2023/05/31 17:14:43.445153 juicefs[32721] : maxprocs: Leaving GOMAXPROCS=8: CPU quota undefined [maxprocs.go:47]\n2023/05/31 17:14:43.445289 juicefs[32721] : Meta address: sqlite3://test.db [interface.go:494]\n[xorm] [info] 2023/05/31 17:14:43.445350 PING DATABASE sqlite3\n2023/05/31 17:14:43.462374 juicefs[32721] : Stat of path /d (inode 2) is successfully synced [base.go:2018]\n\n# Verify that statistics have been fixed\n$ juicefs info -r /jfs/d\n/jfs/d: 1 3.3/s\n/jfs/d: 1.0 GiB (1073745920 Bytes) 3.3 GiB/s\n/jfs/d :\n inode: 2\n files: 1\n dirs: 1\n length: 1.00 GiB (1073741824 Bytes)\n size: 1.00 GiB (1073745920 Bytes)\n path: /d\n```\n" }, { "path": "docs/en/guide/gateway.md", "content": "---\ntitle: JuiceFS S3 Gateway\nsidebar_position: 5\ndescription: JuiceFS S3 Gateway allows the JuiceFS file system to be accessed externally using the S3 protocol. This enables applications to access files stored on JuiceFS through Amazon S3 SDKs.\n---\n\nJuiceFS S3 Gateway is one of the various access methods supported by JuiceFS. It allows the JuiceFS file system to be accessed externally using the S3 protocol. This enables applications to access files stored on JuiceFS using Amazon S3 SDKs.\n\n## Architecture and principles\n\nIn JuiceFS, [files are stored as objects and distributed in chunks within the underlying object storage](../introduction/architecture.md#how-juicefs-store-files). JuiceFS provides multiple access methods, including the FUSE POSIX, WebDAV, S3 Gateway, and CSI Driver. Among these options, S3 Gateway is particularly popular. Below is the S3 Gateway architecture:\n\n![JuiceFS S3 Gateway architecture](../images/juicefs-s3-gateway-arch.png)\n\nJuiceFS S3 Gateway implements its functionality through [MinIO S3 Gateway](https://github.com/minio/minio/tree/ea1803417f80a743fc6c7bb261d864c38628cf8d/docs/gateway). Leveraging MinIO's [`object` interface](https://github.com/minio/minio/blob/d46386246fb6db5f823df54d932b6f7274d46059/cmd/object-api-interface.go#L88), we integrate the JuiceFS file system as the backend storage for MinIO servers. This provides a user experience close to that of native MinIO usage while inheriting many advanced features of MinIO. In this architecture, JuiceFS acts as a local disk for the MinIO instance, and the principle is similar to the `minio server /data1` command.\n\nCommon application scenarios for JuiceFS S3 Gateway include:\n\n- **Exposing the S3 API for JuiceFS:** Applications can access files stored on JuiceFS using S3 SDKs.\n- **Using S3 clients:** Using tools like S3cmd, AWS CLI, and MinIO clients to easily access and manage files stored on JuiceFS.\n- **Managing files in JuiceFS:** JuiceFS S3 Gateway provides a web-based file manager to manage files in JuiceFS directly from a browser.\n- **Cluster replication:** In scenarios requiring cross-cluster data replication, JuiceFS S3 Gateway serves as a unified data export for clusters. This avoids cross-region metadata access and enhances data transfer performance. For details, see [Sync across regions using JuiceFS S3 Gateway](../guide/sync.md#sync-across-region).\n\n## Quick start\n\nJuiceFS S3 Gateway enables access to an existing JuiceFS volume. If you do not have one, follow the steps in this [guide](../getting-started/standalone.md) to create a JuiceFS file system.\n\nThe gateway is built on MinIO, so you must set the `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD` environment variables. They serve as the access key and secret key for authentication when you access the S3 API. These credentials are administrator credentials with the highest privileges.\n\n```shell\nexport MINIO_ROOT_USER=admin\nexport MINIO_ROOT_PASSWORD=12345678\n\n# Use \"set\" on Windows\nset MINIO_ROOT_USER=admin\n```\n\nNote that `MINIO_ROOT_USER` must be at least 3 characters long, and `MINIO_ROOT_PASSWORD` must be at least 8 characters long. If these requirements are not met, the gateway service will display an error: `MINIO_ROOT_USER should be specified as an environment variable with at least 3 characters`.\n\nStart the gateway:\n\n```shell\n# The first argument is the metadata engine URL; the second argument is the address/port for JuiceFS S3 Gateway to listen on.\njuicefs gateway redis://localhost:6379/1 localhost:9000\n\n# Since v1.2, JuiceFS supports running services in the background, using --background or -d.\n# When running in background, use --log to specify the log path.\njuicefs gateway redis://localhost:6379 localhost:9000 -d --log=/var/log/juicefs-s3-gateway.log\n```\n\nBy default, [multi-bucket support](#multi-bucket-support) is not enabled. You can enable it by adding the `--multi-buckets` option. Additionally, you can add [other options](../reference/command_reference.mdx#gateway) to `gateway` subcommands as needed. For example, you can set the default local cache to 20 GiB.\n\n```shell\njuicefs gateway --cache-size 20480 redis://localhost:6379/1 localhost:9000\n```\n\nThis example assumes that the JuiceFS file system uses a local Redis database. When JuiceFS S3 Gateway is enabled, you can access the gateway's management interface at `http://localhost:9000` on the **current host**.\n\n![S3-gateway-file-manager](../images/s3-gateway-file-manager.jpg)\n\nTo allow access to JuiceFS S3 Gateway from other hosts on the local network or the internet, adjust the listen address. For example:\n\n```shell\njuicefs gateway redis://localhost:6379/1 0.0.0.0:9000\n```\n\nThis configuration makes JuiceFS S3 Gateway accept requests from all networks by default. Different S3 clients can access JuiceFS S3 Gateway using different addresses. For example:\n\n- Third-party clients on the same host as JuiceFS S3 Gateway can use `http://127.0.0.1:9000` or `http://localhost:9000` for access.\n- Third-party clients on the same local network as the JuiceFS S3 Gateway host can use `http://192.168.1.8:9000` for access (assuming the JuiceFS S3 Gateway host's internal IP address is `192.168.1.8`).\n- Using `http://110.220.110.220:9000` to access JuiceFS S3 Gateway over the internet (assuming the JuiceFS S3 Gateway host's public IP address is `110.220.110.220`).\n\n## Access JuiceFS S3 Gateway\n\nVarious S3 API-supported clients, desktop applications, and web applications can access JuiceFS S3 Gateway. Ensure you use the correct address and port for accessing JuiceFS S3 Gateway.\n\n:::tip Note\nThe following examples assume accessing JuiceFS S3 Gateway running on the local host with third-party clients. Adjust JuiceFS S3 Gateway's address according to your specific scenario.\n:::\n\n### Use the AWS CLI\n\nDownload and install the AWS Command Line Interface (AWS CLI) from [https://aws.amazon.com/cli](https://aws.amazon.com/cli).\n\nConfigure it:\n\n```bash\n$ aws configure\nAWS Access Key ID [None]: admin\nAWS Secret Access Key [None]: 12345678\nDefault region name [None]:\nDefault output format [None]:\n```\n\nThe program guides you interactively to add new configurations. Use the same values for `Access Key ID` as `MINIO_ROOT_USER` and `Secret Access Key` as `MINIO_ROOT_PASSWORD`. Leave the region name and output format blank.\n\nNow you can use the `aws s3` command to access JuiceFS storage, for example:\n\n```bash\n# List buckets\n$ aws --endpoint-url http://localhost:9000 s3 ls\n\n# List objects in bucket\n$ aws --endpoint-url http://localhost:9000 s3 ls s3://\n```\n\n### Use the MinIO Client\n\nTo avoid compatibility issues, we recommend using the `RELEASE.2021-04-22T17-40-00Z` version of the MinIO Client (`mc`). You can find historical versions with different architectures of `mc` at this [address](https://dl.min.io/client/mc/release). For example, for the amd64 architecture, you can download the `RELEASE.2021-04-22T17-40-00Z` version of `mc` from this [link](https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z).\n\nAfter installing `mc`, add a new alias:\n\n```bash\nmc alias set juicefs http://localhost:9000 admin 12345678\n```\n\nThen, you can freely copy, move, add, and delete files and folders between the local disk, JuiceFS storage, and other cloud storage services using the `mc` client.\n\n```shell\n$ mc ls juicefs/jfs\n[2021-10-20 11:59:00 CST] 130KiB avatar-2191932_1920.png\n[2021-10-20 11:59:00 CST] 4.9KiB box-1297327.svg\n[2021-10-20 11:59:00 CST] 21KiB cloud-4273197.svg\n[2021-10-20 11:59:05 CST] 17KiB hero.svg\n[2021-10-20 11:59:06 CST] 1.7MiB hugo-rocha-qFpnvZ_j9HU-unsplash.jpg\n[2021-10-20 11:59:06 CST] 16KiB man-1352025.svg\n[2021-10-20 11:59:06 CST] 1.3MiB man-1459246.ai\n[2021-10-20 11:59:08 CST] 19KiB sign-up-accent-left.07ab168.svg\n[2021-10-20 11:59:10 CST] 11MiB work-4997565.svg\n```\n\n## Common features\n\n### Multi-bucket support\n\nBy default, JuiceFS S3 Gateway only allows one bucket. The bucket name is the file system name. If you need multiple buckets, you can add `--multi-buckets` at startup to enable multi-bucket support. This parameter exports each subdirectory under the top-level directory of the JuiceFS file system as a separate bucket. Creating a bucket means creating a subdirectory with the same name at the top level of the file system.\n\n```shell\njuicefs gateway redis://localhost:6379/1 localhost:9000 --multi-buckets\n```\n\n### Retain ETags\n\nBy default, JuiceFS S3 Gateway does not save or return object ETag information. You can enable this with `--keep-etag`.\n\n```shell\njuicefs gateway myjfs localhost:9000 --keep-etag\n```\n\nThen if you upload the file through gateway into JuiceFS, you can get this etag by s3API `head-object`:\n\n```shell\naws s3api --endpoint=http://localhost:9000 head-object --bucket myjfs --key test123/test.etag\n{\n \"AcceptRanges\": \"bytes\",\n \"LastModified\": \"Wed, 23 Apr 2025 00:17:16 GMT\",\n \"ContentLength\": 7,\n \"ETag\": \"\\\"d2fde576f44a6601b73201234b491904\\\"\",\n \"ContentType\": \"application/octet-stream\",\n \"Metadata\": {}\n}\n```\n\nThis etag is calculated using the MD5 algorithm, and it's also `setXattr` to file with key `s3-tag`, if you mount the JuiceFS with `--enable-xattr` then you can use `getfattr` to get this etag:\n\n```shell\ngetfattr -n s3-etag test.etag\n# file: test.etag\ns3-etag=\"d2fde576f44a6601b73201234b491904\"\n```\n\n### Enable object tags\n\nObject tags are not supported by default, but you can use `--object-tag` to enable them.\n\n### Enable object metadata 1.3\n\nObject metadata is not supported by default, but you can use `--object-meta` to enable it. Refer to the [documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html) for usage.\n\n### Enable virtual host-style requests\n\nBy default, JuiceFS S3 Gateway supports path-style requests in the format of `http://mydomain.com/bucket/object`. The `MINIO_DOMAIN` environment variable is used to enable virtual host-style requests. If the request's `Host` header information matches `(.+).mydomain.com`, the matched pattern `$1` is used as the bucket, and the path is used as the object.\n\nFor example:\n\n```shell\nexport MINIO_DOMAIN=mydomain.com\n```\n\n### Adjust the IAM refresh interval\n\nThe default refresh interval for Identity and Access Management (IAM) caching is 5 minutes. You can adjust this using `--refresh-iam-interval`. The value of this parameter is a time string with a unit, such as \"300ms\", \"-1.5h\", or \"2h45m.\" Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", and \"h\".\n\nFor example, to set a refresh interval of 1 minute:\n\n```sh\njuicefs gateway xxxx xxxx --refresh-iam-interval 1m\n```\n\n### Multiple gateway instances\n\nThe distributed nature of JuiceFS allows multiple JuiceFS S3 gateway instances to be started on different nodes simultaneously. This can improve the availability and performance of S3 Gateway instances. In this scenario, each gateway instance independently handles requests, but all access the same JuiceFS file system. It is important to note the following:\n\n- Ensure that all instances are started with the same user at initialization; use the same UID and GID for all instances.\n- The IAM refresh time between nodes can vary, but it must be ensured that this interval is not too short to prevent excessive load on JuiceFS.\n- Each instance's listening address and port can be freely configured. If multiple instances are started on the same machine, ensure that there is no conflict in port numbers.\n\n### Run as a daemon service\n\nJuiceFS S3 Gateway can be configured as a systemd unit.\n\n```shell\ncat > /lib/systemd/system/juicefs-gateway.service< 9000/TCP 142m\n```\n\nThe deployment will launch a Deploy named `juicefs-s3-gateway`. Run this command to check the Pod status:\n\n```sh\n$ kubectl -n kube-system get po -l app.kubernetes.io/name=juicefs-s3-gateway\nNAME READY STATUS RESTARTS AGE\njuicefs-s3-gateway-5c69d574cc-t92b6 1/1 Running 0 136m\n```\n\n## Advanced features\n\nThe core feature of JuiceFS S3 Gateway is to provide the S3 API. Now, the support for the S3 protocol is comprehensive. Version 1.2 supports IAM and bucket event notifications.\n\nThese advanced features require the `RELEASE.2021-04-22T17-40-00Z` version of the `mc` client. For the usage of these advanced features, see the [MinIO documentation](https://github.com/minio/minio/tree/e0d3a8c1f4e52bb4a7d82f7f369b6796103740b3/docs) or the `mc` command-line help information.\n\nIf you are unsure about the available features or how to use a specific feature, you can append `-h` to a subcommand to view the help information.\n\n### Identity and access control\n\n#### Regular users\n\nBefore version 1.2, `juicefs gateway` only created a superuser when starting, and this superuser belonged only to that process. Even if multiple gateway processes shared the same file system, their users were isolated between processes. You could set different superusers for each gateway process, and they were independent and unaffected by each other.\n\nStarting from version 1.2, `juicefs gateway` still requires setting a superuser at startup, and this superuser remains isolated per process. However, it allows adding new users using `mc admin user add`. Newly added users are shared across the same file system. You can manage new users using `mc admin user`. This supports adding, disabling, enabling, and deleting users, as well as viewing all users and displaying user information and policies.\n\n```Shell\n$ mc admin user -h\nNAME:\n mc admin user - manage users\n\nUSAGE:\n mc admin user COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add a new user\n disable disable user\n enable enable user\n remove remove user\n list list all users\n info display info of a user\n policy export user policies in JSON format\n svcacct manage service accounts\n```\n\nAn example of adding a user:\n\n```Shell\n# Add a new user.\n$ mc admin user add myjfs user1 admin123\n\n# List current users.\n$ mc admin user list myjfs\nenabled user1\n\n# List current users in JSON format.\n$ mc admin user list myjfs --json\n{\n \"status\": \"success\",\n \"accessKey\": \"user1\",\n \"userStatus\": \"enabled\"\n}\n```\n\n### Service accounts\n\nService accounts are used to create a copy of an existing user with the same permissions, allowing different applications to use separate access keys. The privileges for service accounts inherit from their parent users. They can be managed using the command:\n\n```Shell\n$ mc admin user svcacct -h\nNAME:\n mc admin user svcacct - manage service accounts\n\nUSAGE:\n mc admin user svcacct COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add a new service account\n ls List services accounts\n rm Remove a service account\n info Get a service account info\n set edit an existing service account\n enable Enable a service account\n disable Disable a services account\n```\n\n:::tip\nService accounts inherit privileges from their parent users and cannot be directly attached with permission policies.\n:::\n\nFor example, consider a user named `user1`. You can create a service account named `svcacct1` for it using the following command:\n\n```Shell\nmc admin user svcacct add myjfs user1 --access-key svcacct1 --secret-key 123456abc\n```\n\nIf the parent user, `user1`, has read-only permissions, then so will `svcacct1`. To grant different permissions to `svcacct1`, you would need to adjust the privileges of the parent user.\n\n#### AssumeRole security token service\n\nThe S3 Gateway Security Token Service (STS) is a service that allows clients to request temporary credentials to access MinIO resources. The working principle of temporary credentials is almost the same as default administrator credentials but with some differences:\n\n- **Temporary credentials are short-lived.** They can be configured to last from minutes to hours. After expiration, the gateway no longer recognizes them and does not allow any form of API request access.\n- **Temporary credentials do not need to be stored with the application. They are dynamically generated and provided to the application when requested.** When temporary credentials expire, applications can request new credentials.\n\nThe `AssumeRole` operation returns a set of temporary security credentials. You can use them to access gateway resources. `AssumeRole` requires authorization credentials for an existing gateway user and returns temporary security credentials, including an access key, secret key, and security token. Applications can use these temporary security credentials to sign requests for gateway API operations. The policies applied to these temporary credentials inherit from gateway user credentials.\n\nBy default, `AssumeRole` creates temporary security credentials with a validity period of one hour. However, you can specify the duration of the credentials using the optional parameter `DurationSeconds`, which can range from 900 (15 minutes) to 604,800 (7 days).\n\n##### API request parameters\n\n- `Version`\n\n Indicates the STS API version information. The only supported value is '2011-06-15', borrowed from the AWS STS API documentation for compatibility.\n\n | Parameter | Value |\n | ---------- | ------ |\n | Type | String |\n | Require | Yes |\n\n- `AUTHPARAMS`\n\n Indicates the STS API authorization information. If you are familiar with AWS Signature V4 authorization headers, this STS API supports the signature V4 authorization as described [here](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html).\n\n- `DurationSeconds`\n\n Duration in seconds. This value can range from 900 seconds (15 minutes) to 7 days. If the value is higher than this setting, the operation fails. By default, this value is set to 3,600 seconds.\n\n | Parameter | Value |\n |-------------|---------------------|\n | *Type* | Integer |\n | Valid range | From 900 to 604,800 |\n | Required | No |\n\n- Policy\n\n A JSON-format IAM policy that you want to use as an inline session policy. This parameter is optional. Passing a policy to this operation returns new temporary credentials. The permissions of the generated session are the intersection of preset policy names and the policy set here. You cannot use this policy to grant more permissions than allowed by the assumed preset policy names.\n\n | Parameter | Value |\n |-------------|-------------------|\n | Type | String |\n | Valid range | From 1 to 2,048 |\n | Required | No |\n\n##### Response elements\n\nThe XML response of this API is similar to [AWS STS `AssumeRole`](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_ResponseElements).\n\n##### Errors\n\nThe XML error response of this API is similar to [AWS STS `AssumeRole`](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_Errors).\n\n##### A `POST` request example\n\n```\nhttp://minio:9000/?Action=AssumeRole&DurationSeconds=3600&Version=2011-06-15&Policy={\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Stmt1\",\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}]}&AUTHPARAMS\n```\n\n##### A response example\n\n```\n\n\n \n \n \n \n \n \n Y4RJU1RNFGK48LGO9I2S\n sYLRKS1Z7hSjluf6gEbb9066hnx315wHTiACPAjg\n 2019-08-08T20:26:12Z\n eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZNFJKVTFSTkZHSzQ4TEdPOUkyUyIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQxODExMDcxLCJpYXQiOjE1NDE4MDc0NzEsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiYTBiMjc2MjktZWUxYS00M2JmLTg3MzktZjMzNzRhNGNkYmMwIn0.ewHqKVFTaP-j_kgZrcOEKroNUjk10GEp8bqQjxBbYVovV0nHO985VnRESFbcT6XMDDKHZiWqN2vi_ETX_u3Q-w\n \n \n \n c6104cbe-af31-11e0-8154-cbc7ccf896c7\n \n\n```\n\n##### Use the AWS CLI with the AssumeRole API\n\n1. Start the gateway and create a user named `foobar`.\n\n2. Configure the AWS CLI:\n\n ```\n [foobar]\n region = us-east-1\n aws_access_key_id = foobar\n aws_secret_access_key = foo12345\n ```\n\n3. Use the AWS CLI to request the `AssumeRole` API.\n\n :::note Note\n In the command below, `--role-arn` and `--role-session-name` have no significance for the gateway. You can set them to any value that meets the command line requirements.\n :::\n\n ```sh\n $ aws --profile foobar --endpoint-url http://localhost:9000 sts assume-role --policy '{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Stmt1\",\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}]}' --role-arn arn:xxx:xxx:xxx:xxxx --role-session-name anything\n {\n \"AssumedRoleUser\": {\n \"Arn\": \"\"\n },\n \"Credentials\": {\n \"SecretAccessKey\": \"xbnWUoNKgFxi+uv3RI9UgqP3tULQMdI+Hj+4psd4\",\n \"SessionToken\": \"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJLOURUSU1VVlpYRVhKTDNBVFVPWSIsImV4cCI6MzYwMDAwMDAwMDAwMCwicG9saWN5IjoidGVzdCJ9.PetK5wWUcnCJkMYv6TEs7HqlA4x_vViykQ8b2T_6hapFGJTO34sfTwqBnHF6lAiWxRoZXco11B0R7y58WAsrQw\",\n \"Expiration\": \"2019-02-20T19:56:59-08:00\",\n \"AccessKeyId\": \"K9DTIMUVZXEXJL3ATUOY\"\n }\n }\n ```\n\n##### Access the AssumeRole API in Go applications\n\nSee the [MinIO official example program](https://github.com/minio/minio/blob/master/docs/sts/assume-role.go).\n\n:::note\nSuperusers defined by environment variables cannot use AssumeRole APIs; only users added by `mc admin user add` can use AssumeRole APIs.\n:::\n\n#### Permission management\n\nBy default, newly created users have no permissions and need to be granted permissions using `mc admin policy` before they can be used. This command supports adding, deleting, updating, and listing policies, as well as adding, deleting, and updating permissions for users.\n\n```Shell\n$ mc admin policy -h\nNAME:\n mc admin policy - manage policies defined in the MinIO server\n\nUSAGE:\n mc admin policy COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add new policy\n remove remove policy\n list list all policies\n info show info on a policy\n set set IAM policy on a user or group\n unset unset an IAM policy for a user or group\n update Attach new IAM policy to a user or group\n```\n\nThe gateway includes the following common policies:\n\n- `readonly`: Read-only users.\n- `readwrite`: Read-write users.\n- `writeonly`: Write-only users.\n- `consoleAdmin`: Read-write-admin users, where \"admin\" means the ability to use management APIs such as creating users.\n\nFor example, to set a user as a read-only user:\n\n```Shell\n# Set user1 as a read-only user.\n$ mc admin policy set myjfs readonly user=user1\n\n# Check user policy.\n$ mc admin user list myjfs\nenabled user1 readonly\n```\n\nFor custom policies, use `mc admin policy add`:\n\n```Shell\n$ mc admin policy add -h\nNAME:\n mc admin policy add - add new policy\n\nUSAGE:\n mc admin policy add TARGET POLICYNAME POLICYFILE\n\nPOLICYNAME:\n Name of the canned policy on the MinIO server.\n\nPOLICYFILE:\n Name of the policy file associated with the policy name.\n\nEXAMPLES:\n 1. Add a new canned policy 'writeonly'.\n $ mc admin policy add myjfs writeonly /tmp/writeonly.json\n```\n\nThe policy file to be added here must be in JSON format with [IAM-compatible](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies.html) syntax, limited to 2,048 characters. This syntax allows for more fine-grained access control. If you are unfamiliar with this, you can first use the following command to see the simple policies and then modify them accordingly.\n\n```Shell\n$ mc admin policy info myjfs readonly\n{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetBucketLocation\",\n \"s3:GetObject\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::*\"\n ]\n }\n ]\n}\n```\n\n#### User group management\n\nJuiceFS S3 Gateway supports creating user groups, similar to Linux user groups, and uses `mc admin group` for management. You can set one or more users to a group and grant permissions uniformly to the group. This usage is similar to user management.\n\n```Shell\n$ mc admin group -h\nNAME:\n mc admin group - manage groups\n\nUSAGE:\n mc admin group COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add users to a new or existing group\n remove remove group or members from a group\n info display group info\n list display list of groups\n enable enable a group\n disable disable a group\n```\n\n#### Anonymous access management\n\nIn addition to user-specific permissions, anonymous access management is also possible. This allows specific objects or buckets to be accessible to anyone. You can use the `mc policy` command to manage this functionality.\n\n```Shell\nName:\n mc policy - manage anonymous access to buckets and objects\n\nUSAGE:\n mc policy [FLAGS] set PERMISSION TARGET\n mc policy [FLAGS] set-json FILE TARGET\n mc policy [FLAGS] get TARGET\n mc policy [FLAGS] get-json TARGET\n mc policy [FLAGS] list TARGET\n\nPERMISSION:\n Allowed policies are: [none, download, upload, public].\n\nFILE:\n A valid S3 policy JSON filepath.\n\nEXAMPLES:\n 1. Set bucket to \"download\" on Amazon S3 cloud storage.\n $ mc policy set download s3/burningman2011\n\n 2. Set bucket to \"public\" on Amazon S3 cloud storage.\n $ mc policy set public s3/shared\n\n 3. Set bucket to \"upload\" on Amazon S3 cloud storage.\n $ mc policy set upload s3/incoming\n\n 4. Set policy to \"public\" for bucket with prefix on Amazon S3 cloud storage.\n $ mc policy set public s3/public-commons/images\n\n 5. Set a custom prefix based bucket policy on Amazon S3 cloud storage using a JSON file.\n $ mc policy set-json /path/to/policy.json s3/public-commons/images\n\n 6. Get bucket permissions.\n $ mc policy get s3/shared\n\n 7. Get bucket permissions in JSON format.\n $ mc policy get-json s3/shared\n\n 8. List policies set to a specified bucket.\n $ mc policy list s3/shared\n\n 9. List public object URLs recursively.\n $ mc policy --recursive links s3/shared/\n```\n\nThe gateway has built-in support for four types of anonymous permissions by default:\n\n- `none`: Disallows anonymous access (typically used to clear existing permissions).\n- `download`: Allows anyone to read.\n- `upload`: Allows anyone to write.\n- `public`: Allows anyone to read and write.\n\nThe following example shows how to set an object to allow anonymous downloads:\n\n```\n# Set testbucket1/afile for anonymous access.\nmc policy set download useradmin/testbucket1/afile\n\n# View specific permissions.\nmc policy get-json useradmin/testbucket1/afile\n\n$ mc policy --recursive links useradmin/testbucket1/\nhttp://127.0.0.1:9001/testbucket1/afile\n\n# Directly download the object.\nwget http://127.0.0.1:9001/testbucket1/afile\n\n# Clear download permission for afile.\nmc policy set none useradmin/testbucket1/afile\n```\n\n#### Configuration effective time\n\nAll management API updates for JuiceFS S3 Gateway take effect immediately and are persisted to the JuiceFS file system. Clients that accept these API requests also immediately reflect these changes.\n\nHowever, in a multi-server gateway setup, the situation is slightly different. This is because when the gateway handles request authentication, it uses in-memory cached information as the validation baseline. Otherwise, reading configuration file content for every request would pose unacceptable performance issues. However, caching also introduces potential inconsistencies between cached data and the configuration file.\n\nCurrently, JuiceFS S3 Gateway's cache refresh strategy involves forcibly updating the in-memory cache every 5 minutes (certain operations also trigger cache update operations). This ensures that configuration changes take effect within a maximum of 5 minutes in a multi-server setup. You can adjust this time by using the `--refresh-iam-interval` parameter. If immediate effect on a specific gateway is required, you can manually restart it.\n\n### Bucket event notifications\n\nYou can use bucket event notifications to monitor events happening on objects within a storage bucket and trigger certain actions in response.\n\nCurrently supported object event types include:\n\n- `s3:ObjectCreated:Put`\n- `s3:ObjectCreated:CompleteMultipartUpload`\n- `s3:ObjectAccessed:Head`\n- `s3:ObjectCreated:Post`\n- `s3:ObjectRemoved:Delete`\n- `s3:ObjectCreated:Copy`\n- `s3:ObjectAccessed:Get`\n\nSupported global events include:\n\n- `s3:BucketCreated`\n- `s3:BucketRemoved`\n\nYou can use the `mc` client tool with the event subcommand to set up and monitor event notifications. Notifications sent by MinIO for publishing events are in JSON format. See the [JSON structure](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html).\n\nTo reduce dependencies, JuiceFS S3 Gateway has cut support for certain event destination types. Currently, storage bucket events can be published to the following destinations:\n\n- Redis\n- MySQL\n- PostgreSQL\n- Webhooks\n\n```Shell\n$ mc admin config get myjfs | grep notify\nnotify_webhook publish bucket notifications to webhook endpoints\nnotify_mysql publish bucket notifications to MySQL databases\nnotify_postgres publish bucket notifications to Postgres databases\nnotify_redis publish bucket notifications to Redis datastores\n```\n\n:::note\nHere, assuming the JuiceFS file system name is 'images', enable the S3 Gateway service and define its alias as 'myjfs' in mc. For the S3 Gateway, the JuiceFS file system name 'images' serves as a bucket name.\n:::\n\n#### Use Redis to publish events\n\nRedis event destination supports two formats: `namespace` and `access`.\n\nIn the `namespace` format, the gateway synchronizes objects in the bucket to entries in a Redis hash. Each entry corresponds to an object in the storage bucket, with the key set to \"bucket name/object name\" and the value as JSON-formatted event data specific to that gateway object. Any updates or deletions of objects also update or delete corresponding entries in the hash.\n\nIn the `access` format, the gateway uses [RPUSH](https://redis.io/commands/rpush) to add events to a list. Each element in this list is a JSON-formatted list with two elements:\n\n- A timestamp string\n- A JSON object containing event data related to operations on the bucket\n\nIn this format, elements in the list are not updated or deleted.\n\nTo use notification destinations in `namespace` and `access` formats:\n\n1. Configure Redis with the gateway.\n\n Use the `mc admin config set` command to configure Redis as the event notification destination:\n\n ```Shell\n # Command-line parameters\n # mc admin config set myjfs notify_redis[:name] address=\"xxx\" format=\"namespace|access\" key=\"xxxx\" password=\"xxxx\" queue_dir=\"\" queue_limit=\"0\"\n # An example\n $ mc admin config set myjfs notify_redis:1 address=\"127.0.0.1:6379/1\" format=\"namespace\" key=\"bucketevents\" password=\"yoursecret\" queue_dir=\"\" queue_limit=\"0\"\n ```\n\n You can use `mc admin config get myjfs notify_redis` to view the configuration options. Different types of destinations have different configuration options. For Redis type, it has the following configuration options:\n\n ```Shell\n $ mc admin config get myjfs notify_redis\n notify_redis enable=off format=namespace address= key= password= queue_dir= queue_limit=0\n ```\n\n Here are the meanings of each configuration option:\n\n ```Shell\n notify_redis[:name] Supports setting multiple Redis instances with different names.\n address* (address) Address of the Redis server. For example: localhost:6379.\n key* (string) Redis key to store/update events. The key is created automatically.\n format* (namespace*|access) Determines the format type, either 'namespace' or 'access'; defaults to 'namespace'.\n password (string) Password for the Redis server.\n queue_dir (path) Directory to store unsent messages, for example, '/home/events'.\n queue_limit (number) Maximum limit of unsent messages. The default is '100000'.\n comment (sentence) Optional comment description.\n ```\n\n The gateway supports persistent event storage. Persistent storage backs up events when the Redis broker is offline and replays events when the broker comes back online. You can set the directory for event storage using the `queue_dir` field and the maximum limit for storage using `queue_limit`. For example, you can set `queue_dir` to `/home/events`, and you can set `queue_limit` to 1,000. By default, `queue_limit` is 100,000. Before updating the configuration, you can use the `mc admin config get` command to get the current configuration.\n\n ```Shell\n $ mc admin config get myjfs notify_redis\n notify_redis:1 address=\"127.0.0.1:6379/1\" format=\"namespace\" key=\"bucketevents\" password=\"yoursecret\" queue_dir=\"\" queue_limit=\"0\"\n\n # Effective after restart\n $ mc admin config set myjfs notify_redis:1 queue_limit=\"1000\"\n Successfully applied new settings.\n Please restart your server: 'mc admin service restart myjfs'.\n # Note that the `mc admin service restart myjfs` command cannot be used to restart. JuiceFS S3 Gateway does not currently support this functionality. When you see this prompt after configuring with `mc`, you need to manually restart JuiceFS S3 Gateway.\n ```\n\n After using the `mc admin config set` command to update the configuration, restart JuiceFS S3 Gateway to apply the changes. JuiceFS S3 Gateway will output a line similar to `SQS ARNs: arn:minio:sqs::1:redis`.\n\n Based on your needs, you can add multiple Redis destinations by providing the identifier for each Redis instance (like the \"1\" in the example \"notify_redis:1\") along with the configuration parameters for each instance.\n\n2. Enable bucket notifications.\n\n Now you can enable event notifications on a bucket named \"images.\" When a JPEG file is created or overwritten, a new key is created or an existing key is updated in the previously configured Redis hash. If an existing object is deleted, the corresponding key is also removed from the hash. Therefore, the rows in the Redis hash map to `.jpg` objects in the \"images\" bucket.\n\n To configure bucket notifications, you need to use the Amazon Resource Name (ARN) information outputted by the gateway in the previous steps. See more information about [ARNs](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html).\n\n You can use the `mc` tool to add these configuration details. Assuming the gateway service alias is myjfs, you can execute the following script:\n\n ```Shell\n mc event add myjfs/images arn:minio:sqs::1:redis --suffix .jpg\n mc event list myjfs/images\n arn:minio:sqs::1:redis s3:ObjectCreated:*,s3:ObjectRemoved:*,s3:ObjectAccessed:* Filter: suffix=\".jpg\"\n ```\n\n3. Verify Redis.\n\n Start the `redis-cli` Redis client program to check the content in Redis. Running the `monitor` Redis command will output every command executed on Redis.\n\n ```Shell\n redis-cli -a yoursecret\n 127.0.0.1:6379> monitor\n OK\n ```\n\n Upload a file named `myphoto.jpg` to the `images` bucket.\n\n ```Shell\n mc cp myphoto.jpg myjfs/images\n ```\n\n In the previous terminal, you can see the operations performed by the gateway on Redis:\n\n ```Shell\n 127.0.0.1:6379> monitor\n OK\n 1712562516.867831 [1 192.168.65.1:59280] \"hset\" \"bucketevents\" \"images/myphoto.jpg\" \"{\\\"Records\\\":[{\\\"eventVersion\\\":\\\"2.0\\\",\\\"eventSource\\\":\\\"minio:s3\\\",\\\"awsRegion\\\":\\\"\\\",\\\"eventTime\\\":\\\"2024-04-08T07:48:36.865Z\\\",\\\"eventName\\\":\\\"s3:ObjectCreated:Put\\\",\\\"userIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"requestParameters\\\":{\\\"principalId\\\":\\\"admin\\\",\\\"region\\\":\\\"\\\",\\\"sourceIPAddress\\\":\\\"127.0.0.1\\\"},\\\"responseElements\\\":{\\\"content-length\\\":\\\"0\\\",\\\"x-amz-request-id\\\":\\\"17C43E891887BA48\\\",\\\"x-minio-origin-endpoint\\\":\\\"http://127.0.0.1:9001\\\"},\\\"s3\\\":{\\\"s3SchemaVersion\\\":\\\"1.0\\\",\\\"configurationId\\\":\\\"Config\\\",\\\"bucket\\\":{\\\"name\\\":\\\"images\\\",\\\"ownerIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"arn\\\":\\\"arn:aws:s3:::images\\\"},\\\"object\\\":{\\\"key\\\":\\\"myphoto.jpg\\\",\\\"size\\\":4,\\\"eTag\\\":\\\"40b134ab8a3dee5dd9760a7805fd495c\\\",\\\"userMetadata\\\":{\\\"content-type\\\":\\\"image/jpeg\\\"},\\\"sequencer\\\":\\\"17C43E89196AE2A0\\\"}},\\\"source\\\":{\\\"host\\\":\\\"127.0.0.1\\\",\\\"port\\\":\\\"\\\",\\\"userAgent\\\":\\\"MinIO (darwin; arm64) minio-go/v7.0.11 mc/RELEASE.2021-04-22T17-40-00Z\\\"}}]}\"\n ```\n\n Here, you can see that the gateway executed the `HSET` command on the `minio_events` key.\n\n In the `access` format, `minio_events` is a list, and the gateway calls `RPUSH` to add it to the list. In the `monitor` command, you can see:\n\n ```Shell\n 127.0.0.1:6379> monitor\n OK\n 1712562751.922469 [1 192.168.65.1:61102] \"rpush\" \"aceesseventskey\" \"[{\\\"Event\\\":[{\\\"eventVersion\\\":\\\"2.0\\\",\\\"eventSource\\\":\\\"minio:s3\\\",\\\"awsRegion\\\":\\\"\\\",\\\"eventTime\\\":\\\"2024-04-08T07:52:31.921Z\\\",\\\"eventName\\\":\\\"s3:ObjectCreated:Put\\\",\\\"userIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"requestParameters\\\":{\\\"principalId\\\":\\\"admin\\\",\\\"region\\\":\\\"\\\",\\\"sourceIPAddress\\\":\\\"127.0.0.1\\\"},\\\"responseElements\\\":{\\\"content-length\\\":\\\"0\\\",\\\"x-amz-request-id\\\":\\\"17C43EBFD35A53B8\\\",\\\"x-minio-origin-endpoint\\\":\\\"http://127.0.0.1:9001\\\"},\\\"s3\\\":{\\\"s3SchemaVersion\\\":\\\"1.0\\\",\\\"configurationId\\\":\\\"Config\\\",\\\"bucket\\\":{\\\"name\\\":\\\"images\\\",\\\"ownerIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"arn\\\":\\\"arn:aws:s3:::images\\\"},\\\"object\\\":{\\\"key\\\":\\\"myphoto.jpg\\\",\\\"size\\\":4,\\\"eTag\\\":\\\"40b134ab8a3dee5dd9760a7805fd495c\\\",\\\"userMetadata\\\":{\\\"content-type\\\":\\\"image/jpeg\\\"},\\\"sequencer\\\":\\\"17C43EBFD3DACA70\\\"}},\\\"source\\\":{\\\"host\\\":\\\"127.0.0.1\\\",\\\"port\\\":\\\"\\\",\\\"userAgent\\\":\\\"MinIO (darwin; arm64) minio-go/v7.0.11 mc/RELEASE.2021-04-22T17-40-00Z\\\"}}],\\\"EventTime\\\":\\\"2024-04-08T07:52:31.921Z\\\"}]\"\n ```\n\n#### Use MySQL to publish events\n\nThe MySQL notification destination supports two formats: `namespace` and `access`.\n\nIf you use the `namespace` format, the gateway synchronizes objects in the bucket to rows in the database table. Each row has two columns:\n\n- `key_name`. It is the bucket name plus the object name.\n- `value`. It is the JSON-formatted event data about that gateway object.\n\nIf objects are updated or deleted, the corresponding rows in the table are also updated or deleted.\n\nIf you use the `access` format, the gateway adds events to the table. Rows have two columns:\n\n- `event_time`. It is the time the event occurred on the gateway server.\n- `event_data`. It is the JSON-formatted event data about that gateway object.\n\nIn this format, rows are not deleted or modified.\n\nThe following steps show how to use the notification destination in `namespace` format. The `access` format is similar and not further described here.\n\n1. Ensure the MySQL version meets the minimum requirements.\n\n JuiceFS S3 Gateway requires MySQL version 5.7.8 or above, because it uses the [JSON](https://dev.mysql.com/doc/refman/5.7/en/json.html) data type introduced in MySQL 5.7.8.\n\n2. Configure MySQL to the gateway.\n\n Use the `mc admin config set` command to configure MySQL as the event notification destination.\n\n ```Shell\n mc admin config set myjfs notify_mysql:myinstance table=\"minio_images\" dsn_string=\"root:123456@tcp(172.17.0.1:3306)/miniodb\"\n ```\n\n You can use `mc admin config get myjfs notify_mysql` to view the configuration options. Different destination types have different configuration options. For MySQL type, the following configuration options are available:\n\n ```shell\n $ mc admin config get myjfs notify_mysql\n format=namespace dsn_string= table= queue_dir= queue_limit=0 max_open_connections=2\n ```\n\n Here are the meanings of each configuration item:\n\n ```Shell\n KEY:\n notify_mysql[:name] Publish bucket notifications to the MySQL database. When multiple MySQL server endpoints are required, you can add a user-specified \"name\" to each configuration, for example, \"notify_mysql:myinstance.\"\n\n ARGS:\n dsn_string* (string) MySQL data source name connection string, for example, \":@tcp(:)/\".\n table* (string) Name of the database table to store/update events. The table is automatically created.\n format* (namespace*|access) 'namespace' or 'access.' The default is 'namespace.'\n queue_dir (path) The directory for storing unsent messages, for example, '/home/events'.\n queue_limit (number) The maximum limit of unsent messages. The default is '100000'.\n comment (sentence) Optional comment description.\n ```\n\n `dsn_string` is required and must be in the format `:@tcp(:)/`.\n\n MinIO supports persistent event storage. Persistent storage backs up events when the MySQL connection is offline and replays events when the broker comes back online. You can set the storage directory for events using the `queue_dir` field, and the maximum storage limit using `queue_limit`. For example, you can set `queue_dir` to `/home/events`, and `queue_limit` to 1,000. By default, `queue_limit` is set to 100,000.\n\n Before updating the configuration, you can use the `mc admin config get` command to get the current configuration.\n\n ```Shell\n $ mc admin config get myjfs/ notify_mysql\n notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0\n ```\n\n Update the MySQL notification configuration using the `mc admin config set` command with the `dsn_string` parameter:\n\n ```Shell\n mc admin config set myjfs notify_mysql:myinstance table=\"minio_images\" dsn_string=\"root:xxxx@tcp(127.0.0.1:3306)/miniodb\"\n ```\n\n You can add multiple MySQL server endpoints as needed, by providing the identifier of the MySQL instance (for example, \"myinstance\") and the configuration parameter information for each instance.\n\n After updating the configuration with the `mc admin config set` command, restart the gateway to apply the configuration changes. The gateway server will output a line during startup similar to `SQS ARNs: arn:minio:sqs::myinstance:mysql`.\n\n3. Enable bucket notifications.\n\n Now you can enable event notifications on a bucket named \"images.\" When a file is uploaded to the bucket, a new record is inserted into MySQL, or an existing record is updated. If an existing object is deleted, the corresponding record is also deleted from the MySQL table. Therefore, each row in the MySQL table corresponds to an object in the bucket.\n\n To configure bucket notifications, you need to use the ARN information outputted by MinIO in previous steps. See more information about [ARNs](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html).\n\n Assuming the gateway service alias is myjfs, you can execute the following script:\n\n ```Shell\n # Add notification configuration to the 'images' bucket using the MySQL ARN. The --suffix parameter is used to filter events.\n mc event add myjfs/images arn:minio:sqs::myinstance:mysql --suffix .jpg\n # Print the notification configuration on the 'images' bucket.\n mc event list myjfs/images\n arn:minio:sqs::myinstance:mysql s3:ObjectCreated:*,s3:ObjectRemoved:*,s3:ObjectAccessed:* Filter: suffix=”.jpg”\n ```\n\n4. Verify MySQL.\n\n Open a new terminal and upload a JPEG image to the `images` bucket:\n\n ```Shell\n mc cp myphoto.jpg myjfs/images\n ```\n\n Open a MySQL terminal and list all records in the `minio_images` table. You will find a newly inserted record.\n\n#### Use PostgreSQL to publish events\n\nThe method of publishing events using PostgreSQL is similar to publishing MinIO events using MySQL, with PostgreSQL version 9.5 or above required. The gateway uses PostgreSQL 9.5's [`INSERT ON CONFLICT`](https://www.postgresql.org/docs/9.5/static/sql-insert.html#SQL-ON-CONFLICT) (aka `UPSERT`) feature and 9.4's [`jsonb`](https://www.postgresql.org/docs/9.4/static/datatype-json.html) data type.\n\n#### Use a webhook to publish events\n\n[Webhooks](https://en.wikipedia.org/wiki/Webhook) use a push model to get data instead of continually pulling.\n\n1. Configure a webhook to the gateway.\n\n The gateway supports persistent event storage. Persistent storage backs up events when the webhook is offline and replays events when the broker comes back online. You can set the directory for event storage using the `queue_dir` field, and the maximum storage limit using `queue_limit`. For example, you can set `queue_dir` to `/home/events` and `queue_limit` to 1,000. By default, `queue_limit` is 100,000.\n\n ```Shell\n KEY:\n notify_webhook[:name] Publish bucket notifications to webhook endpoints.\n\n ARGS:\n endpoint* (url) Webhook server endpoint, for example, http://localhost:8080/minio/events.\n auth_token (string) Opaque token or JWT authorization token.\n queue_dir (path) The directory for storing unsent messages, for example, '/home/events'.\n queue_limit (number) The maximum limit of unsent messages. The default is '100000'.\n client_cert (string) The client certificate for mTLS authentication of the webhook.\n client_key (string) The client certificate key for mTLS authentication of the webhook.\n comment (sentence) Optional comment description.\n ```\n\n Use the `mc admin config set` command to update the configuration. The endpoint here is the service that listens for webhook notifications. Save the configuration file and restart the MinIO service to apply the changes. Note that when restarting MinIO, this endpoint must be up and accessible.\n\n ```Shell\n mc admin config set myjfs notify_webhook:1 queue_limit=\"0\" endpoint=\"http://localhost:3000\" queue_dir=\"\"\n ```\n\n2. Enable bucket notifications.\n\n Now you can enable event notifications. When a file is uploaded to the bucket, an event is triggered. Here, the ARN value is `arn:minio:sqs::1:webhook`. See more information about [ARNs](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html).\n\n ```Shell\n mc mb myjfs/images-thumbnail\n mc event add myjfs/images arn:minio:sqs::1:webhook --event put --suffix .jpg\n ```\n\n If the command report cannot create a bucket, please check if the S3 Gateway has enabled [multi-bucket support](#multi-bucket-support).\n\n3. Use Thumbnailer to verify.\n\n [Thumbnailer](https://github.com/minio/thumbnailer) is a project that generates thumbnails using MinIO's `listenBucketNotification` API. JuiceFS uses Thumbnailer to listen to gateway notifications. If a file is uploaded to the gateway service, Thumbnailer listens to that notification, generates a thumbnail, and uploads it to the gateway service.\n\n To install Thumbnailer:\n\n ```Shell\n git clone https://github.com/minio/thumbnailer/\n npm install\n ```\n\n Open the Thumbnailer's `config/webhook.json` configuration file, add the configuration for the MinIO server, and start Thumbnailer using:\n\n ```Shell\n NODE_ENV=webhook node thumbnail-webhook.js\n ```\n\n Thumbnailer runs on `http://localhost:3000/`.\n\n Next, configure the MinIO server to send messages to this URL (mentioned in step 1) and set up bucket notifications using `mc` (mentioned in step 2). Then upload an image to the gateway server:\n\n ```Shell\n mc cp ~/images.jpg myjfs/images\n .../images.jpg: 8.31 KB / 8.31 KB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 100.00% 59.42 KB/s 0s\n ```\n\n After a moment, use `mc ls` to check the content of the bucket. You will see a thumbnail.\n\n ```Shell\n mc ls myjfs/images-thumbnail\n [2017-02-08 11:39:40 IST] 992B images-thumbnail.jpg\n ```\n" }, { "path": "docs/en/guide/quota.md", "content": "---\ntitle: Storage Quota\nsidebar_position: 4\n---\n\nJuiceFS supports both total file system quota and subdirectory quota, both of which can be used to limit the available capacity and the number of available inodes. Both file system quota and directory quota are hard limits. When the total file system quota is exhausted, subsequent writes will return `ENOSPC` (No space left) error; and when the directory quota is exhausted, subsequent writes will return `EDQUOT` (Disk quota exceeded) error.\n\n:::tip\nThe storage quota settings are stored in the metadata engine for all mount points to read, and the client of each mount point will also cache its own used capacity and inodes and synchronize them with the metadata engine once per second. Meanwhile the client will read the latest usage value from the metadata engine every 10 seconds to synchronize the usage information among each mount point, but this information synchronization mechanism cannot guarantee that the usage data is counted accurately.\n:::\n\n## File system quota {#file-system-quota}\n\nFor Linux, the default capacity of a JuiceFS type file system is identified as `1.0P` by using the `df` command.\n\n```shell\n$ df -Th | grep juicefs\nJuiceFS:ujfs fuse.juicefs 1.0P 682M 1.0P 1% /mnt\n```\n\n:::note\nThe capacity of underlying object storage is usually unlimited, i.e., JuiceFS storage is unlimited. Therefore, the displayed capacity is just an estimate rather than the actual storage limit.\n:::\n\nThe `config` command that comes with the client allows you to view the details of a file system.\n\n```shell\n$ juicefs config $METAURL\n{\n \"Name\": \"ujfs\",\n \"UUID\": \"1aa6d290-279b-432f-b9b5-9d7fd597dec2\",\n \"Storage\": \"minio\",\n \"Bucket\": \"127.0.0.1:9000/jfs1\",\n \"AccessKey\": \"herald\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0,\n \"TrashDays\": 0\n}\n```\n\n### Limit total capacity {#limit-total-capacity}\n\nThe capacity limit (in GiB) can be set with `--capacity` when creating a file system, e.g. to create a file system with an available capacity of 100 GiB:\n\n```shell\njuicefs format --storage minio \\\n --bucket 127.0.0.1:9000/jfs1 \\\n ... \\\n --capacity 100 \\\n $METAURL myjfs\n```\n\nYou can also set a capacity limit for a created file system with the `config` command:\n\n```shell\n$ juicefs config $METAURL --capacity 100\n2022/01/27 12:31:39.506322 juicefs[16259] : Meta address: postgres://herald@127.0.0.1:5432/jfs1\n2022/01/27 12:31:39.521232 juicefs[16259] : The latency to database is too high: 14.771783ms\n capacity: 0 GiB -> 100 GiB\n```\n\nFor file systems that have been set with storage quota, the identification capacity becomes the quota capacity:\n\n```shell\n$ df -Th | grep juicefs\nJuiceFS:ujfs fuse.juicefs 100G 682M 100G 1% /mnt\n```\n\n### Limit the total number of inodes {#limit-total-number-of-inodes}\n\nOn Linux systems, each file (a folder is also a type of file) has an inode regardless of size, so limiting the number of inodes is equivalent to limiting the number of files.\n\nThe quota can be set with `--inodes` when creating the file system, e.g.\n\n```shell\njuicefs format --storage minio \\\n --bucket 127.0.0.1:9000/jfs1 \\\n ... \\\n --inodes 100 \\\n $METAURL myjfs\n```\n\nThe file system created by the above command allows only 100 files to be stored. However, there is no limit to the size of individual files. For example, it will still work if a single file is equivalent or even larger than 1 TB as long as the total number of files does not exceed 100.\n\nYou can also set a capacity quota for a created file system by using the `config` command:\n\n```shell\n$ juicefs config $METAURL --inodes 100\n2022/01/27 12:35:37.311465 juicefs[16407] : Meta address: postgres://herald@127.0.0.1:5432/jfs1\n2022/01/27 12:35:37.322991 juicefs[16407] : The latency to database is too high: 11.413961ms\n inodes: 0 -> 100\n```\n\n### Combine `--capacity` and `--inodes` {#limit-total-capacity-and-inodes}\n\nYou can combine `--capacity` and `--inodes` to set the capacity quota of a file system with more flexibility. For example, to create a file system that the total capacity limits to 100 TiB with only 100000 files to be stored:\n\n```shell\njuicefs format --storage minio \\\n --bucket 127.0.0.1:9000/jfs1 \\\n ... \\\n --capacity 102400 \\\n --inodes 100000 \\\n $METAURL myjfs\n```\n\nSimilarly, for the file systems that have been created, you can follow the settings below separately.\n\n```shell\njuicefs config $METAURL --capacity 102400\n```\n\n```shell\njuicefs config $METAURL --inodes 100000\n```\n\n:::tip\nThe client reads the latest storage quota settings from the metadata engine periodically to update the local settings. The refresh interval is controlled by the `--heartbeat` option (default: 12 seconds). Other mount points may take up to the heartbeat interval to update the quota setting.\n:::\n\n## Directory quota {#directory-quota}\n\nJuiceFS began to support directory-level storage quota since v1.1, and you can use the `juicefs quota` subcommand for directory quota management and query.\n\n:::tip\nThe usage statistic relies on the mount process, please do not use this feature until all writable mount processes are upgraded to v1.1.0.\n:::\n\n### Limit directory capacity {#limit-directory-capacity}\n\nYou can use `juicefs quota set $METAURL --path $DIR --capacity $N` to set the directory capacity limit in GiB. For example, to set a capacity quota of 1GiB for the directory `/test`:\n\n```shell\n$ juicefs quota set $METAURL --path /test --capacity 1\n+-------+---------+---------+------+-----------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+---------+---------+------+-----------+-------+-------+\n| /test | 1.0 GiB | 1.6 MiB | 0% | unlimited | 314 | |\n+-------+---------+---------+------+-----------+-------+-------+\n```\n\nAfter the setting is successful, you can see a table describing the current quota setting directory, quota size, current usage and other information.\n\n:::tip\nThe use of the `quota` subcommand does not require a local mount point, and it is expected that the input directory path is a path relative to the JuiceFS root directory rather than a local mount path. It may take a long time to set a quota for a large directory, because the current usage of the directory needs to be calculated.\n:::\n\nIf you need to query the quota and current usage of a certain directory, you can use the `juicefs quota get $METAURL --path $DIR` command:\n\n```shell\n$ juicefs quota get $METAURL --path /test\n+-------+---------+---------+------+-----------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+---------+---------+------+-----------+-------+-------+\n| /test | 1.0 GiB | 1.6 MiB | 0% | unlimited | 314 | |\n+-------+---------+---------+------+-----------+-------+-------+\n```\n\nYou can also use the `juicefs quota ls $METAURL` command to list all directory quotas.\n\n### Limit the total number of directory inodes\n\nYou can use `juicefs quota set $METAURL --path $DIR --inodes $N` to set the directory inode quota, the unit is one. For example, to set a quota of 400 inodes for the directory `/test`:\n\n```shell\n$ juicefs quota set $METAURL --path /test --inodes 400\n+-------+---------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+---------+---------+------+--------+-------+-------+\n| /test | 1.0 GiB | 1.6 MiB | 0% | 400 | 314 | 78% |\n+-------+---------+---------+------+--------+-------+-------+\n```\n\n### Limit capacity and inodes of directory {#limit-capacity-and-inodes-of-directory}\n\nYou can combine `--capacity` and `--inodes` to set the capacity limit of the directory more flexibly. For example, to set a quota of 10GiB and 1000 inodes for the `/test` directory:\n\n```shell\n$ juicefs quota set $METAURL --path /test --capacity 10 --inodes 1000\n+-------+--------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+--------+---------+------+--------+-------+-------+\n| /test | 10 GiB | 1.6 MiB | 0% | 1,000 | 314 | 31% |\n+-------+--------+---------+------+--------+-------+-------+\n```\n\nIn addition, you can also not limit the capacity of the directory and the number of inodes (set to `0` means unlimited), and only use the `quota` command to count the current usage of the directory:\n\n```shell\n$ juicefs quota set $METAURL --path /test --capacity 0 --inodes 0\n+-------+-----------+---------+------+-----------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+-----------+---------+------+-----------+-------+-------+\n| /test | unlimited | 1.6 MiB | | unlimited | 314 | |\n+-------+-----------+---------+------+-----------+-------+-------+\n```\n\n### Nested quota {#nested-quota}\n\nJuiceFS allows nested quota to be set on multiple levels of directories, client performs recursive lookup to ensure quota settings take effect on every level of directory. This means even if the parent directory is allocated a smaller quota, you can still set a larger quota on the child directory.\n\n### Subdirectory mount {#subdirectory-mount}\n\nJuiceFS supports mounting arbitrary subdirectories using [`--subdir`](../reference/command_reference.mdx#mount-metadata-options). If the directory quota is set for the mounted subdirectory, you can use the `df` command that comes with the system to view the directory quota and current usage. For example, the file system quota is 1PiB and 10M inodes, while the quota for the `/test` directory is 1GiB and 400 inodes. The output of the `df` command when mounted using the root directory is:\n\n```shell\n$ df -h\nFilesystem Size Used Avail Use% Mounted on\n...\nJuiceFS:myjfs 1.0P 1.6M 1.0P 1% /mnt/jfs\n\n$ df -i -h\nFilesystem Inodes IUsed IFree IUse% Mounted on\n...\nJuiceFS:myjfs 11M 315 10M 1% /mnt/jfs\n```\n\nWhen mounted using the `/test` subdirectory, the output of the `df` command is:\n\n```shell\n$ df -h\nFilesystem Size Used Avail Use% Mounted on\n...\nJuiceFS:myjfs 1.0G 1.6M 1023M 1% /mnt/jfs\n\n$ df -i -h\nFilesystem Inodes IUsed IFree IUse% Mounted on\n...\nJuiceFS:myjfs 400 314 86 79% /mnt/jfs\n```\n\n:::note\nWhen there is no quota set for the mounted subdirectory, JuiceFS will query up to find the nearest directory quota and return it to `df`. If directory quotas are set for multiple levels of parent directories, JuiceFS will return the minimum available capacity and number of inodes after calculation.\n:::\n\n### Usage check and fix {#usage-check-and-fix}\n\nSince directory usage updates are laggy and asynchronous, loss may occur under unusual circumstances (such as a client exiting unexpectedly). We can use the `juicefs quota check $METAURL --path $DIR` command to check or fix it:\n\n```shell\n$ juicefs quota check $METAURL --path /test\n2023/05/23 15:40:12.704576 juicefs[1638846] : quota of /test is consistent [base.go:839]\n+-------+--------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+--------+---------+------+--------+-------+-------+\n| /test | 10 GiB | 1.6 MiB | 0% | 1,000 | 314 | 31% |\n+-------+--------+---------+------+--------+-------+-------+\n```\n\nWhen the directory usage is correct, the current directory quota usage will be output; if it fails, the error log will be output:\n\n```shell\n$ juicefs quota check $METAURL --path /test\n2023/05/23 15:48:17.494604 juicefs[1639997] : /test: quota(314, 4.0 KiB) != summary(314, 1.6 MiB) [base.go:843]\n2023/05/23 15:48:17.494644 juicefs[1639997] : quota of /test is inconsistent, please repair it with --repair flag [main.go:31]\n```\n\nAt this point you can use the `--repair` option to repair directory usage:\n\n```shell\n$ juicefs quota check $METAURL --path /test --repair\n2023/05/23 15:50:08.737086 juicefs[1640281] : /test: quota(314, 4.0 KiB) != summary(314, 1.6 MiB) [base.go:843]\n2023/05/23 15:50:08.737123 juicefs[1640281] : repairing... [base.go:852]\n+-------+--------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+--------+---------+------+--------+-------+-------+\n| /test | 10 GiB | 1.6 MiB | 0% | 1,000 | 314 | 31% |\n+-------+--------+---------+------+--------+-------+-------+\n```\n" }, { "path": "docs/en/guide/sync.md", "content": "---\ntitle: Data Synchronization\nsidebar_position: 7\ndescription: Learn how to use juicefs sync for efficient data synchronization across supported storage systems, including object storage, JuiceFS, and local file systems.\n---\n\n[`juicefs sync`](../reference/command_reference.mdx#sync) is a powerful data synchronization tool that can copy data across all supported storage systems, including object storage, JuiceFS, and local file systems. You can freely copy data between any of these systems. It also supports syncing remote directories accessed via SSH, HDFS, and WebDAV. Advanced features include incremental synchronization, pattern matching (like rsync), and distributed syncing.\n\n:::tip Mixing Community and Enterprise Editions\n`juicefs sync` shares code between Community and Enterprise Editions. Therefore, even when you use different editions of the JuiceFS client, `sync` works normally. The only exception is when the [`jfs://`](#sync-without-mount-point) protocol header is involved. Due to the different metadata engine implementations in the Community and Enterprise Editions, clients from different editions cannot be mixed when using the `jfs://` protocol header.\n:::\n\n`juicefs sync` works like this:\n\n```shell\njuicefs sync [command options] SRC DST\n\n# Sync object from OSS to S3\njuicefs sync oss://mybucket.oss-cn-shanghai.aliyuncs.com s3://mybucket.s3.us-east-2.amazonaws.com\n\n# Sync objects from S3 to JuiceFS\njuicefs sync s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://VOL_NAME/\n\n# Copy all files ending with .gz\njuicefs sync --match-full-path --include='**.gz' --exclude='*' s3://xxx jfs://VOL_NAME/\n\n# Copy all files that do not end with .gz\njuicefs sync --match-full-path --exclude='**.gz' s3://xxx/ jfs://VOL_NAME/\n\n# Copy all files except the subdirectory named tempdir\njuicefs sync --match-full-path --include='*' s3://xxx/ jfs://VOL_NAME/\n```\n\n## Pattern matching {#pattern-matching}\n\nYou can use `--exclude` and `--include` for filtering. If no filtering rules are provided, all files are scanned and copied (`--include='*'` is the default). However, if you use the `--include` filter to match files with a specific pattern, you must also use `--exclude` to exclude other files. See the examples above for reference.\n\n:::tip\nWhen using multiple matching patterns, it may be difficult to determine whether a file will be synchronized due to the filtering logic. In such cases, it is recommended to add the `--dry --debug` option to preview the files selected for synchronization. If the results are not as expected, adjust the matching patterns accordingly.\n:::\n\n### Matching rules {#matching-rules}\n\nYou can use any word or substring for filtering, as well as these special patterns (similar to shell wildcards):\n\n+ A single `*` matches any character, but terminates at `/`.\n+ `**` matches any character, including `/`.\n+ `?` matches any single character except `/`.\n+ `[...]` matches a set of characters, such as `[a-z]` for any lowercase letter.\n+ `[^...]` excludes specified characters. For example, `[^abc]` matches any character except `a`, `b`, and `c`.\n\nIn addition:\n\n- If the matching pattern does not contain regex patterns, it tries to match the full file name. For example, `foo` matches `foo` and `xx/foo` but not `foo1`, `2foo`, or `foo/xx`, since none of them is a file named exactly `foo`.\n- If the matching pattern ends with `/`, it only matches directories, not files.\n- A pattern that starts with `/` stands for absolute path, so `/foo` matches the `foo` file at the root.\n\nHere are some examples of matching patterns:\n\n+ `--exclude='*.o'` excludes all files matching `*.o`.\n+ `--exclude='/foo/*/bar'` excludes `bar` files located two levels under `/foo`, such as `/foo/spam/bar`, but not `/foo/spam/eggs/bar`.\n+ `--exclude='/foo/**/bar'` excludes `bar` files at any level under `/foo`.\n\nThe `sync` command supports two filtering modes: *full path filtering* and *layer-by-layer filtering*. Both use `--include` and `--exclude` to filter files, but their behaviors are different. By default, `sync` employs the layer-by-layer filtering mode, which is more complicated but resembles rsync's usage. Therefore, it is only recommended for users familiar with rsync. For most people, `--match-full-path` is recommended because it is much easier to understand.\n\n### Full path filtering (recommended) 1.2.0 {#full-path-filtering-mode}\n\nSince v1.2.0, JuiceFS supports the `--match-full-path` option. This mode directly matches the full path of an object against all specified filters sequentially. Once a pattern matches, the result is returned (either \"include\" or \"exclude\"), and subsequent patterns are ignored.\n\nBelow is the workflow of full path filtering mode:\n\n![Full path filtering workflow](../images/sync-full-path-filtering-mode-flow-chart.svg)\n\nFor example, consider a file located at `a1/b1/c1.txt` and three matching patterns `--include 'a*.txt' --include 'c1.txt' --exclude 'c*.txt'`. In full path filtering mode:\nThe string `a1/b1/c1.txt` is first matched against `--include 'a*.txt'`. This fails because `*` does not match the `/` character (see [matching rules](#matching-rules)).\n`a1/b1/c1.txt` is then matched against `--include 'c1.txt'`, which succeeds. According to the mode's logic, subsequent patterns, such as `--exclude 'c*.txt'`, are ignored once a match is found. This file will be handled by the `sync` command.\n\nHere are some more examples:\n\n- `--exclude '/foo**'` excludes all files or directories whose root directory name starts with `foo`.\n- `--exclude '**foo/**'` excludes all directories ending with `foo`.\n- `--include '*/' --include '*.c' --exclude '*'` includes all directories and files with the `.c` extension while excluding everything else.\n- `--include 'foo/bar.c' --exclude '*'` includes only the `foo` directory and the `foo/bar.c` file.\n\n### Layer-by-layer filtering mode {#layer-by-layer-filtering-mode}\n\nIn layer-by-layer filtering mode, the full path is split into hierarchical levels, generating a sequence of strings. For example, a path like `a1/b1/c1.txt` is split into the sequence `a1`, `a1/b1`, and `a1/b1/c1.txt`. Each element in this sequence is processed as though in [\"full path filtering\"](#full-path-filtering-mode) mode.\n\nIf an element matches a certain pattern, two outcomes are possible:\n\n- If it is an exclude pattern, the *exclude* behavior is immediately returned as the final result.\n- If it is an include pattern, remaining patterns for that layer are skipped and the process moves on to the next layer.\n\nIf no patterns match at a particular layer, the process moves on to the next layer. **If \"exclude\" is not returned after all layers are processed, the scanned files are included (be \"handled\" by the `sync` command) by default.**\n\nBelow is the workflow for layer-by-layer filtering mode:\n\n![Layer-by-layer filtering workflow](../images/sync-layer-by-layer-filtering-mode-flow-chart.svg)\n\nFor example, given the file `a1/b1/c1.txt` and the patterns `--include 'a*.txt' --include 'c1.txt' --exclude 'c*.txt'`, in layer-by-layer filtering mode, the sequence is `a1`, `a1/b1`, and `a1/b1/c1.txt`. The specific matching steps are:\n\n1. At the first layer `a1`, no patterns match. Move on to the next layer.\n2. At the second layer `a1/b1`, no patterns match. Move to the next level.\n3. At the third layer `a1/b1/c1.txt`, the `--inlude 'c1.txt'` pattern matches. So as for the current state, this file will be handled, and the process will continue to the next layer.\n4. Since there is no next layer, `a1/b1/c1.txt` will be included and handled by this command.\n\nIn the example above, the matching is successful until the last layer. In addition, there may be two situations:\n\n- If the match is successful before the last layer, and the matching pattern is an exclude filter, the file is excluded as a final state, skipping all subsequent layers.\n- If all layers are processed but no matches occur, this file will be included.\n\nEssentially, this mode processes paths hierarchically, applying full path filtering at each layer. Each layer comes out with either a hit to exclude or a miss and continue to the next layer. The only way to get the file included is to process all layers of filtering.\n\nSome more examples:\n\n+ `--exclude /foo` excludes all files or directories named `foo` under the root directory.\n+ `--exclude foo/` excludes all directories named `foo`.\n+ For multi-level directories such as `dir_name/.../.../...`, all paths under `dir_name` will be processed according to the directory hierarchy. If the parent directory of a file is \"excluded,\" the file will not be handled, even if an include rule is subsequently specified for it. If you want this file to be included, you must guarantee that all its parent directories are not excluded. For example, `/some/path/this-file-will-not-be-synced` in the following example will not be included because its parent directory `some` has been excluded by the rule `--exclude '*'`:\n\n ```shell\n --include '/some/path/this-file-will-not-be-synced' \\\n --exclude '*'\n ```\n\nOne solution is to include all directories in the directory hierarchy by using the `--include '*/'` rule (which needs to be placed before the `--exclude '*'` rule). Alternatively, you can add include rules to each parent directory, for example:\n\n ```shell\n --include '/some/' \\\n --include '/some/path/' \\\n --include '/some/path/this-file-will-be-synced' \\\n --exclude '*'\n ```\n\n## Storage protocols {#storage-protocols}\n\nYou can sync data between any [supported storage system](../reference/how_to_set_up_object_storage.md), but note that if one of the endpoint is a JuiceFS volume, it it then recommended to [sync without mount point](#sync-without-mount-point) since it runs without FUSE overhead.\n\n### Sync without mount point 1.1 {#sync-without-mount-point}\n\nFor data migrations that involve JuiceFS, it's recommended use the `jfs://` protocol, rather than mount JuiceFS and access its local directory, which bypasses the FUSE mount point and access JuiceFS directly. Under large scale scenarios, bypassing FUSE can save precious resources and increase performance.\n\n```shell\nmyfs=redis://10.10.0.8:6379/1 juicefs sync s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/movies/ jfs://myfs/movies/\n```\n\n### Synchronize between object storage and JuiceFS {#synchronize-between-object-storage-and-juicefs}\n\nThe following command synchronizes `movies` directory from object storage to JuiceFS.\n\n```shell\n# mount JuiceFS\njuicefs mount -d redis://10.10.0.8:6379/1 /mnt/jfs\n# synchronize\njuicefs sync s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/movies/ /mnt/jfs/movies/\n```\n\nThe following command synchronizes `images` directory from JuiceFS to object storage.\n\n```shell\n# mount JuiceFS\njuicefs mount -d redis://10.10.0.8:6379/1 /mnt/jfs\n# synchronization\njuicefs sync /mnt/jfs/images/ s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/images/\n```\n\n### Synchronize between object storages {#synchronize-between-object-storages}\n\nThe following command synchronizes all of the data from object storage to another bucket.\n\n```shell\njuicefs sync s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com oss://ABCDEFG:HIJKLMN@bbb.oss-cn-hangzhou.aliyuncs.com\n```\n\n### Synchronize between local and remote servers {#synchronize-between-local-and-remote-servers}\n\nTo copy files between directories on a local computer, simply specify the source and destination paths. For example, to synchronize the `/media/` directory with the `/backup/` directory:\n\n```shell\njuicefs sync /media/ /backup/\n```\n\nIf you need to synchronize between servers, you can access the target server using the SFTP/SSH protocol. For example, to synchronize the local `/media/` directory with the `/backup/` directory on another server:\n\n```shell\njuicefs sync /media/ username@192.168.1.100:/backup/\n# Specify password (optional)\njuicefs sync /media/ \"username:password\"@192.168.1.100:/backup/\n```\n\nWhen using the SFTP/SSH protocol, if no password is specified, the sync task will prompt for the password. If you want to explicitly specify the username and password, you need to enclose them in double quotation marks, with a colon separating the username and password.\n\n## Sync behavior {#sync-behavior}\n\n### Incremental and full synchronization {#incremental-and-full-synchronization}\n\nBy default, `juicefs sync` performs incremental synchronization. It only overwrites files if their sizes are different. You can also use [`--update`](../reference/command_reference.mdx#sync) to overwrite files when the `mtime` of the source file has been updated. For scenarios with higher demand for data integrity, use [`--check-new`](../reference/command_reference.mdx#sync) or [`--check-all`](../reference/command_reference.mdx#sync) to perform byte-by-byte comparison between the source and the destination.\n\nFor full synchronization (where all files are synchronized regardless of their presence on the destination path), use [`--force-update`](../reference/command_reference.mdx#sync).\n\n### Directory structure and file permissions {#directory-structure-and-file-permissions}\n\nBy default, empty directories are not synchronized. To include them, use the `--dirs` option.\n\nIn addition, when migrating data between file systems such as local, SFTP, and HDFS, use the `--perms` option to synchronize file permissions.\n\n### Copy symbolic links {#copy-symbolic-links}\n\nFor synchronization between **local directories**, the `--links` option allows symbolic links to be copied as is, instead of resolving their targets. The synchronized symbolic link retains the original path stored in the source, regardless of whether the path is valid before or after the synchronization.\n\nNote:\n\n* The `mtime` of a symbolic link is not synchronized.\n* The `--check-new` and `--perms` options will be ignored when synchronizing symbolic links.\n\n### Data sync and compaction {#sync-and-compaction}\n\nFor sequential write scenarios, ensure each file write has at least a 4M (the default block size) buffer available. If the write concurrency is too high or the buffer size is too small, the client will not be able to maintain the desired \"writing by large chunks\" pattern. Instead, it could only write by small slices, which combined with compaction, could really deteriorate performance due to write amplification.\n\nCompaction can be monitored using `juicefs_compact_size_histogram_bytes`, If compaction traffic is substantial during a `sync` operation, consider the following optimizations:\n\n* If the object storage bandwidth is limited, avoid setting high concurrency (`--threads`). Instead, start with low concurrency and gradually increase it until you get the desired speed.\n\n* When the destination is a JuiceFS file system, use the `jfs://` protocol, because it bypasses the FUSE mount point (reducing overhead) and is already optimized for file fragmentation problems. See the next point for details.\n\n* When the destination is a JuiceFS file system, ensure the destination has sufficient available [buffer](https://github.com/juicedata/docs/pull/662/cache.md#buffer-size) capacity. Each write file handler must have at least 4MB of reserved memory. This means the `--buffer-size` should be at least 4 times the `--threads` value. If higher write concurrency is needed, consider setting it to 8 or 12 times the value. Depending on the destination file system's deployment model, you will use different methods to configure buffer size:\n\n * When the destination starts with the `jfs://` protocol, the JuiceFS client is part of the `juicefs sync` command itself. In this case, `--buffer-size` needs to be appended to the `juicefs sync` command.\n * When the destination is a FUSE mount point, the JuiceFS client runs as the `juicefs mount` process on the host machine. In this case, `--buffer-size` needs to be added directly to the mount command.\n\n* If you need to limit the bandwidth via `--bwlimit`, you must also lower the `--threads` value to avoid write fragmentation caused by concurrency congestion. Since storage systems come with different performance levels, exact calculations cannot be provided here. Therefore, it is recommended to start with low concurrency and adjust as needed.\n\n### Delete selected files\n\nUsing filters, you can even delete files by pattern via `juicefs sync`, the trick is to create an empty directory and use it as `SRC`.\n\nBelow are some examples which uses `--dry --debug` just to be cautious, they will not delete anything as long as `--dry` is specified, after the behavior is verified, remove the option to actually execute.\n\n```shell\nmkdir empty-dir\n# Delete all objects in mybucket except the .gz files\njuicefs sync ./empty-dir/ s3://mybucket.s3.us-east-2.amazonaws.com/ --match-full-path --delete-dst --exclude='**.gz' --include='*' --dry --debug\n# Delete all files ending with .gz in mybucket\njuicefs sync ./empty-dir/ s3://mybucket.s3.us-east-2.amazonaws.com/ --match-full-path --delete-dst --include='**.gz' --exclude='*' --dry --debug\n```\n\n## Accelerate synchronization {#accelerate-sync}\n\nBy default, `juicefs sync` starts 10 threads to run syncing jobs. You can set the `--threads` option to increase or decrease the number of threads as needed. However, adding threads beyond a system's resource limits may cause issues like out-of-memory errors. If performance is still insufficient, consider:\n\n* Check if `SRC` or `DST` storage systems have reached bandwidth limits. If either is constrained, increasing concurrency will not help.\n\n* Performing `juicefs sync` on a single host may be limited by host resources, such as CPU or network throttle. If this is the case, consider the following:\n\n * If a node with better hardware resources (such as CPU or network bandwidth) is available in your environment, consider using that node to run `juicefs sync` and access the source data via SSH. For example, `juicefs sync root@src:/data /jfs/data`.\n * Use [distributed synchronization](#distributed-sync) (introduced below).\n\n* If the synchronized data is mainly small files, and the `list` API of `SRC` storage system has excellent performance, the default single-threaded `list` of `juicefs sync` may become a bottleneck. You can enable [concurrent `list`](#concurrent-list) (introduced below).\n\n### Concurrent `list` {#concurrent-list}\n\nIf `Pending objects` in `juicefs sync` output remains 0, it means consumption is faster than production. You can increase `--list-threads` to enable concurrent `list` and then use `--list-depth` to control directory depth of `list`.\n\nFor example, if you are dealing with an object storage bucket used by JuiceFS, the directory structure is `//chunks/xxx/xxx/...`. In this case, setting `--list-depth=2` enables concurrent listing on `/chunks`.\n\n### Distributed synchronization {#distributed-sync}\n\nSynchronizing between two object storage services is essentially pulling data from one and pushing it to the other. The efficiency of the synchronization depends on the bandwidth between the client and the cloud.\n\n![JuiceFS-sync-single](../images/juicefs-sync-single.png)\n\nWhen copying large scale data, node bandwidth can easily bottleneck the synchronization process. For this scenario, `juicefs sync` provides a multi-machine concurrent solution, as shown in the figure below.\n\n![JuiceFS-sync-worker](../images/juicefs-sync-worker.png)\n\nThe manager node executes the `sync` command as the master and defines multiple worker nodes by setting the `--worker` option (the manager node also serves as a worker node). JuiceFS splits the workload and distributes it to workers for distributed synchronization. This increases the amount of data that can be processed per unit time, and the total bandwidth is also multiplied.\n\nWhen using distributed syncing, you should configure SSH logins so that the manager can access all worker nodes without a password. If the SSH port is not the default 22, you need to include that in the manager's `~/.ssh/config`. The manager will distribute the JuiceFS Client to all worker nodes, so they should all use the same architecture to avoid compatibility problems.\n\nFor example, to synchronize data between two object storage services:\n\n```shell\njuicefs sync --worker bob@192.168.1.20,tom@192.168.8.10 s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com oss://ABCDEFG:HIJKLMN@bbb.oss-cn-hangzhou.aliyuncs.com\n```\n\nThe synchronization workload between the two object storage services is shared by the manager machine and two workers, `bob@192.168.1.20` and `tom@192.168.8.10`.\n\nThe above command demonstrates object → object synchronization, if you need to sync via FUSE mount points, then you need to mount the file system in all worker nodes, and then run the following command to achieve distributed sync:\n\n```shell\n# Source file system needs better read performance, increase its buffer-size\nparallel-ssh -h hosts.txt -i juicefs mount -d redis://10.10.0.8:6379/1 /jfs-src --buffer-size=1024 --cache-size=0\n\n# Destination file system needs better write performance\nparallel-ssh -h hosts.txt -i juicefs mount -d redis://10.10.0.8:6379/1 /jfs-dst --buffer-size=1024 --cache-size=0 --max-uploads=50\n\n# Copy data\njuicefs sync --worker host1,host2 /jfs-src /jfs-dst\n```\n\n## Observation {#observation}\n\nWhen using `sync` to transfer large files, the progress bar might move slowly or get stuck. If this happens, you can observe the progress using other methods.\n\n`sync` is designed for scenarios involving a large number of files. Its progress bar only updates when a file has been transferred. In a large file scenario, each file is transferred slowly, so the progress bar updates infrequently or even appears stuck. This is worse for destinations without multipart upload support (such as `file`, `sftp`, and `jfs` schemes), where each file is transferred using a single thread.\n\nIf you notice the progress bar is not changing, use the methods below for monitoring and troubleshooting:\n\n* Add the [`--verbose` or `--debug`](../reference/command_reference.mdx#global-options) option to the `juicefs sync` command to print debug logs.\n\n* If either end is a JuiceFS mount point:\n\n * Use [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) to quickly check current I/O status.\n * Review the [client log](../administration/fault_diagnosis_and_analysis.md#client-log) (default path: `/var/log/juicefs.log`) for [slow requests or timeout errors](../administration/troubleshooting.md#io-error-object-storage).\n\n* If the destination is a local disk, check the directory for temporary files with `.jfs.xxx.tmp.xxx`. During the synchronization process, the transfer results are written to these temporary files. Once the transfer is complete, they are renamed to finalize the write. By monitoring the size changes of the temporary files, you can determine the current I/O status.\n\n* If both the source and destination are object storage systems, use tools like `nethogs` to check network I/O.\n\n* If none of the above methods provide useful debug information, please collect its goroutine and send it to Juicedata engineers:\n\n ```shell\n # Replace with the actual PID of the stuck sync process\n # This command will print its pprof listen port\n lsof -p | grep TCP | grep LISTEN\n # pprof port is typically 6061, but in the face of port conflict,\n # port number will be automatically increased\n curl -s localhost:6061/debug/pprof/goroutine?debug=1\n ```\n\n## Application scenarios {#application-scenarios}\n\n### Geo-disaster recovery backup {#geo-disaster-recovery-backup}\n\nGeo-disaster recovery backup backs up files, and thus the files stored in JuiceFS should be synchronized to other object storages. For example, synchronize files from JuiceFS to object storage:\n\n```shell\n# mount JuiceFS\njuicefs mount -d redis://10.10.0.8:6379/1 /mnt/jfs\n# synchronization\njuicefs sync /mnt/jfs/ s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/\n```\n\n### Build a JuiceFS data copy {#build-a-juicefs-data-copy}\n\nUnlike the file-oriented disaster recovery backup, the purpose of creating a copy of JuiceFS data is to establish a mirror with exactly the same content and structure as the JuiceFS data storage. When the object storage in use fails, you can switch to the data copy by modifying the configurations. Note that only the file data of the JuiceFS file system is replicated, and the metadata stored in the metadata engine still needs to be backed up.\n\nThis requires manipulating the underlying object storage directly to synchronize it with the target object storage. For example, to take the object storage as the data copy of a JuiceFS volume:\n\n```shell\njuicefs sync cos://ABCDEFG:HIJKLMN@ccc-125000.cos.ap-beijing.myqcloud.com oss://ABCDEFG:HIJKLMN@bbb.oss-cn-hangzhou.aliyuncs.com\n```\n\n### Sync across regions using S3 Gateway {#sync-across-region}\n\nWhen transferring a large number of small files across different regions via FUSE mount points, clients will inevitably talk to the metadata service in the opposite region via the public internet (or dedicated network connection with limited bandwidth). In such cases, metadata latency can become the bottleneck of the data transfer:\n\n![sync via public metadata service](../images/sync-public-metadata.svg)\n\nJuiceFS S3 Gateway is the solution in these scenarios: by deploying a gateway in the source region, metadata is accessed over a private network, minimizing metadata latency and delivering optimal performance for small-file-intensive workloads.\n\n![sync via gateway](../images/sync-via-gateway.svg)\n\nRead [S3 Gateway](../guide/gateway.md) to learn its deployment and use.\n" }, { "path": "docs/en/introduction/README.md", "content": "---\ntitle: Introduction to JuiceFS\nsidebar_position: 1\nslug: .\npagination_next: introduction/architecture\n---\n\n[**JuiceFS**](https://github.com/juicedata/juicefs) is an open-source, high-performance distributed file system designed for the cloud, released under the Apache License 2.0. By providing full [POSIX](https://en.wikipedia.org/wiki/POSIX) compatibility, it allows almost all kinds of object storage to be used as massive local disks and to be mounted and accessed on different hosts across platforms and regions.\n\nJuiceFS separates \"data\" and \"metadata\" storage. Files are split into chunks and stored in [object storage](../reference/how_to_set_up_object_storage.md#supported-object-storage) like Amazon S3. The corresponding metadata can be stored in various [databases](../reference/how_to_set_up_metadata_engine.md) such as Redis, MySQL, TiKV, and SQLite, based on the scenarios and requirements.\n\nJuiceFS provides rich APIs for various forms of data management, analysis, archiving, and backup. It seamlessly interfaces with big data, machine learning, artificial intelligence and other application platforms without modifying code, and delivers massive, elastic, and high-performance storage at low cost. With JuiceFS, you do not need to worry about availability, disaster recovery, monitoring, and scalability. This greatly reduces maintenance work and makes it an excellent choice for DevOps.\n\n## Features {#features}\n\n- **POSIX Compatible**: JuiceFS can be used like a local file system, making it easy to integrate with existing applications.\n- **HDFS Compatible**: JuiceFS is fully compatible with the [HDFS API](../deployment/hadoop_java_sdk.md), which can enhance metadata performance.\n- **S3 Compatible**: JuiceFS provides an [S3 gateway](../guide/gateway.md) to implement an S3-compatible access interface.\n- **Cloud-Native**: It is easy to use JuiceFS in Kubernetes via the [CSI Driver](../deployment/how_to_use_on_kubernetes.md).\n- **Distributed**: Each file system can be mounted on thousands of servers at the same time with high-performance concurrent reads and writes and shared data.\n- **Strong Consistency**: Any changes committed to files are immediately visible on all servers.\n- **Outstanding Performance**: JuiceFS achieves millisecond-level latency and nearly unlimited throughput depending on the object storage scale (see [performance test results](../benchmark/benchmark.md)).\n- **Data Security**: JuiceFS supports encryption in transit and encryption at rest (view [Details](../security/encryption.md)).\n- **File Lock**: JuiceFS supports BSD lock (flock) and POSIX lock (fcntl).\n- **Data Compression**: JuiceFS supports the [LZ4](https://lz4.github.io/lz4) and [Zstandard](https://facebook.github.io/zstd) compression algorithms to save storage space.\n\n## Scenarios {#scenarios}\n\nJuiceFS is designed for massive data storage and can be used as an alternative to many distributed file systems and network file systems, especially in the following scenarios:\n\n- **Big Data**: JuiceFS is compatible with HDFS and can be seamlessly integrated with mainstream computing engines such as Spark, Presto, and Hive, bringing much better performance than directly using object storage.\n- **Machine Learning**: JuiceFS is compatible with POSIX and supports all machine learning and deep learning frameworks. As a shareable file storage, JuiceFS can improve the efficiency of team management and data usage.\n- **Kubernetes**: JuiceFS supports Kubernetes CSI, providing decoupled persistent storage for pods so that your application can be stateless, also great for data sharing among containers.\n- **Shared Workspace**: JuiceFS file system can be mounted on any host, allowing concurrent read/write operations without limitations. Its POSIX compatibility ensures smooth data flow and supports scripting operations.\n- **Data Backup**: JuiceFS provides scalable storage space for backing up all kinds of data. With its shared mount feature, data from multiple hosts can be aggregated into one place and then backed up together.\n\n## Data privacy {#data-privacy}\n\nJuiceFS is an open-source software available on [GitHub](https://github.com/juicedata/juicefs). When using JuiceFS to store data, the data is split into chunks according to specific rules and stored in custom object storage or other storage media, and the corresponding metadata is stored in a custom database.\n\n## More info {#more-info}\n\n* **Use case**: For more use cases of similar scenarios, please visit [User Stories](https://juicefs.com/en/blog/user-stories).\n* **Join the community**: Welcome to join [Slack](https://go.juicefs.com/slack) to discuss with JuiceFS users.\n* **AI assistant**: If you encounter any problems, you are welcome to use the \"Ask AI\" feature (in the bottom right corner) to get assistance from the AI assistant. The knowledge base of the AI ​​assistant comes from documentation and related content on GitHub.\n" }, { "path": "docs/en/introduction/architecture.md", "content": "---\ntitle: Architecture\nsidebar_position: 2\nslug: /architecture\ndescription: This article introduces the technical architecture of JuiceFS and its technical advantages.\n---\n\nThe JuiceFS file system consists of three parts:\n\n![JuiceFS-arch](../images/juicefs-arch.svg)\n\n**JuiceFS Client**: The JuiceFS client handles all file I/O operations, including background tasks like data compaction and trash file expiration. It communicates with both the object storage and metadata engine. The client supports multiple access methods:\n\n- **FUSE**: JuiceFS file system can be mounted on a host in a POSIX-compatible manner, allowing the massive cloud storage to be used as local storage. For details, see [this document](https://juicefs.com/docs/community/getting-started/installation).\n- **Python SDK**: In scenarios where FUSE mounting is not feasible or where direct file system access from within a Python process is required, the Python SDK can read and write the file system directly. Furthermore, the Python SDK natively implements fsspec for easy integration with frameworks like Ray. For details, see [Python_SDK](https://juicefs.com/docs/community/deployment/python_sdk).\n- **Windows Client**: You can experience a file system performance close to that of a local one. For details, see [Use JuiceFS on Windows](https://juicefs.com/docs/community/tutorials/windows).\n- **Hadoop Java SDK**: JuiceFS can replace HDFS, providing Hadoop with cost-effective and abundant storage capacity.\nFor details, see [Use JuiceFS on Hadoop Ecosystem](https://juicefs.com/docs/community/hadoop_java_sdk).\n- **Kubernetes CSI Driver**: JuiceFS provides shared storage for containers in Kubernetes through its CSI Driver. For details, see [Introduction to JuiceFS CSI Driver](https://juicefs.com/docs/csi/introduction).\n- **S3 Gateway**: Applications using S3 as the storage layer can directly access the JuiceFS file system, and tools such as AWS CLI, s3cmd, and MinIO client can be used to access the JuiceFS file system at the same time. For details, see [JuiceFS S3 Gateway](https://juicefs.com/docs/community/guide/gateway).\n- **WebDAV Server**: Files in JuiceFS can be operated directly using the HTTP protocol.\n\n**Data Storage**: File data is split and stored in object storage. JuiceFS supports virtually all types of object storage, including typical self-hosted solutions like OpenStack Swift, Ceph, and MinIO.\n\n**Metadata Engine**: The Metadata Engine stores file metadata, which contains:\n\n- Common file system metadata: file name, size, permission information, creation and modification time, directory structure, file attribute, symbolic link, file lock.\n- JuiceFS-specific metadata: file data mapping, reference counting, client session, etc.\n\nJuiceFS supports a variety of common databases as the metadata engine, like Redis, TiKV, MySQL/MariaDB, PostgreSQL, and SQLite, and the list is still expanding. [Submit an issue](https://github.com/juicedata/juicefs/issues) if your favorite database is not supported.\n\n## How JuiceFS stores files {#how-juicefs-store-files}\n\nTraditional file systems use local disks to store both file data and metadata. However, JuiceFS formats data first and then stores it in the object storage, with the corresponding metadata being stored in the metadata engine.\n\nIn JuiceFS, each file is composed of one or more *chunks*. Each chunk has a maximum size of 64 MB. Regardless of the file's size, all reads and writes are located based on their offsets (the position in the file where the read or write operation occurs) to the corresponding chunk. This design enables JuiceFS to achieve excellent performance even with large files. As long as the total length of the file remains unchanged, the chunk division of the file remains fixed, regardless of how many modifications or writes the file undergoes.\n\n![File and chunks](../images/file-and-chunks.svg)\n\nChunks exist to optimize lookup and positioning, while the actual file writing is performed on *slices*. In JuiceFS, each slice represents a single continuous write, belongs to a specific chunk, and cannot overlap between adjacent chunks. This ensures that the slice length never exceeds 64 MB.\n\nFor example, if a file is generated through a continuous sequential write, each chunk contains only one slice. The figure above illustrates this scenario: a 160 MB file is sequentially written, resulting in three chunks, each containing only one slice.\n\nFile writing generates slices, and invoking `flush` persists these slices. `flush` can be explicitly called by the user, and even if not invoked, the JuiceFS client automatically performs `flush` at the appropriate time to prevent buffer overflow (refer to [buffer-size](../guide/cache.md#buffer-size)). When persisting to the object storage, slices are further split into individual *blocks* (default maximum size of 4 MB) to enable multi-threaded concurrent writes, thereby enhancing write performance. The previously mentioned chunks and slices are logical data structures, while blocks represent the final physical storage form and serve as the smallest storage unit for the object storage and disk cache.\n\n![Split slices to blocks](../images/slice-to-block.svg)\n\nAfter writing a file to JuiceFS, you cannot find the original file directly in the object storage. Instead, the storage bucket contains a `chunks` folder and a series of numbered directories and files. These numerically named object storage files are the blocks split and stored by JuiceFS. The mapping between these blocks, chunks, slices, and other metadata information (such as file names and sizes) is stored in the metadata engine. This decoupled design makes JuiceFS a high-performance file system.\n\n![How JuiceFS stores files](../images/how-juicefs-stores-files.svg)\n\nRegarding logical data structures, if a file is not generated through continuous sequential writes but through multiple append writes, each append write triggers a `flush` to initiate the upload, resulting in multiple slices. If the data size for each append write is less than 4 MB, the data blocks eventually stored in the object storage are smaller than 4 MB blocks.\n\n![Small append writes](../images/small-append.svg)\n\nDepending on the writing pattern, the arrangement of slices can be diverse:\n\n- If a file is repeatedly modified in the same part, it results in multiple overlapping slices.\n- If writes occur in non-overlapping parts, there will be gaps between slices.\n\nHowever complex the arrangement of slices may be, when reading a file, the most recent written slice is read for each file position. The figure below illustrates this concept: while slices may overlap, reading the file always occurs \"from top to bottom.\" This ensures that you see the latest state of the file.\n\n![Complicate pattern](../images/complicate-pattern.svg)\n\nDue to the potential overlapping of slices, JuiceFS [marks the valid data offset range for each slice](../development/internals.md#sliceref) in the reference relationship between chunks and slices. This approach informs the file system of the valid data in each slice.\n\nHowever, it is not difficult to imagine that looking up the \"most recently written slice within the current read range\" during file reading, especially with a large number of overlapping slices as shown in the figure, can significantly impact read performance. This leads to what we call \"file fragmentation.\" File fragmentation not only affects read performance but also increases space usage at various levels (object storage, metadata). Hence, whenever a write occurs, the client evaluates the file's fragmentation and runs the fragmentation compaction asynchronously, merging all slices within the same chunk into one.\n\n![File fragmentation compaction](../images/compaction.svg)\n\nAdditional technical aspects of JuiceFS storage design:\n\n* Irrespective of the file size, JuiceFS avoids storage merging to prevent read amplification and ensure optimal performance.\n* JuiceFS provides strong consistency guarantees while allowing tuning options with caching mechanisms tailored to specific use cases. For example, by configuring more aggressive metadata caching, a certain level of consistency can be traded for enhanced performance. For more details, see [Metadata cache](../guide/cache.md#metadata-cache).\n* JuiceFS supports the [\"Trash\"](../security/trash.md) functionality and enables it by default. After a file is deleted, it is retained for a certain period before being permanently cleared. This helps you avoid data loss caused by accidental deletion.\n" }, { "path": "docs/en/introduction/comparison/_category_.yml", "content": "position: 4\nlabel: \"Comparing with Others\"\n# collapsible: true \n# collapsed: true " }, { "path": "docs/en/introduction/comparison/juicefs_vs_3fs.md", "content": "---\nslug: /comparison/juicefs_vs_3fs\ndescription: This article compares the architectures, features, and innovations of DeepSeek 3FS and JuiceFS in AI storage scenarios.\n---\n\n# JuiceFS vs. 3FS\n\n3FS (Fire-Flyer File System) is a high-performance distributed file system designed for AI training and inference workloads, open-sourced by DeepSeek. It uses NVMe SSDs and RDMA networks to provide a shared storage layer, optimized for the demanding I/O requirements of large-scale AI applications.\n\nJuiceFS is a cloud-native distributed file system that stores data in object storage. The Community Edition, open-sourced on GitHub in 2021, integrates with multiple metadata services and supports diverse use cases. The Enterprise Edition, tailored for high-performance scenarios, is widely adopted in large-scale AI tasks, including generative AI, autonomous driving, quantitative finance, and biotechnology.\n\nThis document provides a comprehensive comparison between 3FS and JuiceFS in terms of architecture, file distribution, RPC framework, and features.\n\n## Architecture comparison\n\n### 3FS\n\n3FS employs an architecture designed for AI workloads with the following key components:\n\n- **Cluster manager**: Handles node membership changes and distributes cluster configurations to other components. Multiple cluster managers are deployed with one elected as primary for high availability.\n- **Metadata service**: Stateless services that handle file metadata operations, backed by FoundationDB—a transactional key-value database for storing metadata.\n- **Storage service**: Manages data storage using local NVMe SSDs with CRAQ (Chain Replication with Apportioned Queries) for data consistency.\n- **Clients**: Provides both FUSE client for POSIX compatibility and native client API for high-performance zero-copy operations.\n\nAll components communicate via RDMA for networking. Cluster configurations are stored in reliable distributed services like ZooKeeper or etcd.\n\n![3FS architecture](https://static1.juicefs.com/images/3FS_JiaGou.original.png)\n\n### JuiceFS\n\nJuiceFS uses a modular, cloud-native architecture that comprises three core components:\n\n- **Metadata engine**: Stores file metadata, including standard file system metadata and file data indexes. The Community Edition supports various databases including Redis, TiKV, MySQL, PostgreSQL, and FoundationDB. The Enterprise Edition uses a self-developed distributed metadata service.\n- **Data storage**: Generally an object storage service, which can be public cloud object storage or on-premises deployed object storage service. Supports integration with various storage backends.\n- **JuiceFS client**: Provides different access methods such as POSIX (FUSE), Hadoop SDK, CSI Driver, and S3 Gateway.\n\n![JuiceFS Community Edition architecture](../../images/juicefs-arch.svg)\n\n### Architectural differences\n\n#### Storage module\n\n3FS employs local NVMe SSDs for data storage and utilizes the CRAQ (Chain Replication with Apportioned Queries) algorithm to ensure data consistency. Replicas are organized into a chain where write requests start from the head and propagate sequentially to the tail. A write operation is confirmed only after reaching the tail. For read requests, any replica in the chain can be queried.\n\n![CRAQ consistency algorithm](https://static1.juicefs.com/images/CRAQ_YiZhiXingSuanFa.original.png)\n\nWhile this design introduces higher write latency due to sequential propagation, it prioritizes read performance, which is crucial for read-intensive AI workloads.\n\nIn contrast, JuiceFS uses object storage as its data storage solution, inheriting key advantages such as data reliability and consistency. The storage module provides standard object operation interfaces (GET/PUT/HEAD/LIST), enabling seamless integration with various storage backends. JuiceFS Community Edition provides local cache for AI scenario bandwidth requirements, while the Enterprise Edition uses distributed cache for larger aggregate read bandwidth needs.\n\n#### Metadata module\n\nIn 3FS, file attributes are stored as key-value pairs within a stateless, high-availability metadata service, backed by FoundationDB. FoundationDB ensures global ordering of keys and evenly distributes data across nodes via consistent hashing. To optimize directory listing efficiency, 3FS constructs dentry keys by combining a \"DENT\" prefix with the parent directory's inode number and file name.\n\nJuiceFS Community Edition provides a metadata module that offers a set of interfaces for metadata operations, supporting integration with various metadata services including key-value databases (Redis, TiKV), relational databases (MySQL, PostgreSQL), and FoundationDB. The Enterprise Edition employs a proprietary high-performance metadata service that dynamically balances data and hot operations based on workload patterns.\n\n#### Client\n\n3FS provides both a FUSE client and a native client API to bypass FUSE for direct data operations. The native client eliminates data copying introduced by the FUSE layer, reducing I/O latency and memory bandwidth overhead through zero-copy communication using shared memory and semaphores.\n\n![3FS native client API](https://static1.juicefs.com/images/3FS_NATIVE_Client_API.original.png)\n\n3FS uses `hf3fs_iov` to store shared memory attributes and `IoRing` for inter-process communication. The system creates virtual files and uses semaphores to facilitate communication between the user process and FUSE process.\n\nJuiceFS' FUSE client offers a more comprehensive implementation with features such as:\n\n- Immediate file length updates after successful object upload\n- BSD locks (flock) and POSIX locks (fcntl)\n- Advanced interfaces like `file_copy_range`, `readdirplus`, and `fallocate`\n\nBeyond the FUSE client, JuiceFS Community Edition also provides Java SDK, Python SDK, S3 Gateway, and CSI Driver for user-space execution, with the Enterprise Edition offering additional enterprise-grade features.\n\n## File distribution comparison\n\n### 3FS file distribution\n\n3FS uses fixed-size chunks, allowing clients to calculate which chunks an I/O request targets based on the file inode and request offset/length, avoiding database queries for each I/O operation. The chunk index is obtained through `offset/chunk_size`, and the chain index through `chunk_id%stripe`.\n\nTo address data imbalance, the first chain of each file is selected in a round-robin manner. When a file is created, chains are randomly sorted and stored in metadata.\n\n![3FS file distribution](https://static1.juicefs.com/images/3FS_WenJianFenBu.original.png)\n\n### JuiceFS file distribution\n\nJuiceFS manages data blocks according to chunk, slice, and block rules. Each chunk is fixed at 64MB for optimizing data search and positioning. Actual file write operations are performed on slices, which represent continuous write processes within chunks. Blocks (default 4MB) are the basic unit of physical storage in object storage and disk cache.\n\n![JuiceFS file distribution](../../images/file-and-chunks.svg)\n\nSlice is a unique structure in JuiceFS that records file write operations and persists them in object storage. Since object storage doesn't support in-place file modification, JuiceFS uses slices to update file content without rewriting entire files. All slices are written once, reducing reliance on underlying object storage consistency and simplifying cache system complexity.\n\n## 3FS RPC framework\n\n3FS implements a custom RPC framework using RDMA as the underlying network communication protocol, which JuiceFS currently doesn't support. The framework provides capabilities such as serialization and packet merging, using templates to implement reflection for data structure serialization.\n\n![3FS FUSE client RPC process](https://static1.juicefs.com/images/3FS_FUSE_Client_DiaoYong_MetadataFuWuDe_RPC_Guo.original.png)\n\nThe 3FS cache system consists of TLS (Thread-Local Storage) queues and global queues. Memory allocation from TLS queues requires no locks, while global queue access requires locking. Multiple RPC requests may be merged into one InfiniBand request for efficiency.\n\n## Feature comparison\n\n| Features | 3FS | JuiceFS Community | JuiceFS Enterprise |\n|----------|-----|-------------------|-------------------|\n| Metadata | Stateless metadata service + FoundationDB | External database | Self-developed high-performance distributed metadata engine (horizontally scalable) |\n| Data storage | Self-managed | Object storage | Object storage |\n| Redundancy | Multi-replica | Provided by object storage | Provided by object storage |\n| Data caching | None | Local cache | Self-developed high-performance multi-copy distributed cache |\n| Encryption | Not supported | Supported | Supported |\n| Compression | Not supported | Supported | Supported |\n| Quota management | Not supported | Supported | Supported |\n| Network protocol | RDMA | TCP | TCP |\n| Snapshots | Not supported | Supports cloning | Supports cloning |\n| POSIX ACL | Not supported | Supported | Supported |\n| POSIX compliance | Partial | Fully compatible | Fully compatible |\n| CSI Driver | No official support | Supported | Supported |\n| Clients | FUSE + native client | POSIX (FUSE), Java SDK, Python SDK, S3 Gateway | POSIX (FUSE), Java SDK, S3 Gateway, Python SDK |\n| Multi-cloud mirroring | Not supported | Not supported | Supported |\n| Cross-cloud/region replication | Not supported | Not supported | Supported |\n| Main maintainer | DeepSeek | Juicedata | Juicedata |\n| Development language | C++, Rust (local storage engine) | Go | Go |\n| License | MIT | Apache License 2.0 | Commercial |\n\n## Summary\n\nFor large-scale AI training, 3FS adopts a performance-first design approach:\n\n- **Local storage**: Uses local NVMe SSDs, requiring users to manage underlying storage infrastructure\n- **Zero-copy optimization**: Achieves zero-copy from client to NIC, reducing I/O latency and memory bandwidth usage via shared memory and semaphores\n- **RDMA networking**: Leverages RDMA for better networking performance\n- **Optimized I/O**: Enhances small I/O and metadata operations with TLS-backed I/O buffer pools and merged network requests\n\nWhile this approach can deliver performance improvements, it comes with higher costs and greater maintenance complexity.\n\nJuiceFS uses object storage as its backend, significantly reducing costs and simplifying maintenance. To meet AI workloads' performance demands:\n\n- **Enterprise Edition features**: Distributed caching, distributed metadata service, and Python SDK\n- **Upcoming optimizations**: v5.2 adds zero-copy over TCP for faster data transfers\n- **Cloud-native advantages**: Full POSIX compatibility, mature open-source ecosystem, and Kubernetes CSI support\n- **Enterprise capabilities**: Quotas, security management, and disaster recovery features\n" }, { "path": "docs/en/introduction/comparison/juicefs_vs_alluxio.md", "content": "---\nslug: /comparison/juicefs_vs_alluxio\ndescription: This article compares the main features of Alluxio and JuiceFS.\n---\n\n# JuiceFS vs. Alluxio\n\nAlluxio (/əˈlʌksio/) is a data access layer in the big data and machine learning ecosystem. Initially as the research project \"Tachyon,\" it was created at the University of California, Berkeley's [AMPLab](https://en.wikipedia.org/wiki/AMPLab) as creator's Ph.D. thesis in 2013. Alluxio was open sourced in 2014.\n\nThe following table compares the main features of Alluxio and JuiceFS.\n\n| Features | Alluxio | JuiceFS |\n| -------- | ------- | ------- |\n| Storage format | Object | Block |\n| Cache granularity | 64 MiB | 4 MiB |\n| Multi-tier cache | ✓ | ✓ |\n| Hadoop-compatible | ✓ | ✓ |\n| S3-compatible | ✓ | ✓ |\n| Kubernetes CSI Driver | ✓ | ✓ |\n| Hadoop data locality | ✓ | ✓ |\n| Fully POSIX-compatible | ✕ | ✓ |\n| Atomic metadata operation | ✕ | ✓ |\n| Consistency | ✕ | ✓ |\n| Data compression | ✕ | ✓ |\n| Data encryption | ✕ | ✓ |\n| Zero-effort operation | ✕ | ✓ |\n| Language | Java | Go |\n| Open source license | Apache License 2.0 | Apache License 2.0 |\n| Open source date | 2014 | 2021.1 |\n\n## Storage format\n\nJuiceFS has its own storage format, where files are divided into blocks, and they can be optionally encrypted and compressed before being uploaded to the object storage. For more details, see [How JuiceFS stores files](../architecture.md#how-juicefs-store-files).\n\nIn contrast, Alluxio stores files as _objects_ into UFS and does not split them into blocks like JuiceFS does.\n\n## Cache granularity\n\nJuiceFS has a smaller [default block size](../architecture.md#how-juicefs-store-files) of 4 MiB, which results in a finer granularity compared to Alluxio's 64 MiB. The smaller block size of JuiceFS is more beneficial for workloads involving random reads (e.g., Parquet and ORC), as it improves cache management efficiency.\n\n## Hadoop-compatible\n\nJuiceFS is [HDFS-compatible](../../deployment/hadoop_java_sdk.md), supporting not only Hadoop 2.x and Hadoop 3.x, but also various components in the Hadoop ecosystem.\n\n## Kubernetes CSI Driver\n\nJuiceFS provides [Kubernetes CSI Driver](https://github.com/juicedata/juicefs-csi-driver) for easy integration with Kubernetes environments. While Alluxio also offers [Kubernetes CSI Driver](https://github.com/Alluxio/alluxio-csi), it seems to have limited activity and lacks official support from Alluxio.\n\n## Fully POSIX-compatible\n\nJuiceFS is [fully POSIX-compatible](../../reference/posix_compatibility.md). A pjdfstest from [JD.com](https://www.slideshare.net/Alluxio/using-alluxio-posix-fuse-api-in-jdcom) shows that Alluxio did not pass the POSIX compatibility test. For example, Alluxio does not support symbolic links, truncate, fallocate, append, xattr, mkfifo, mknod and utimes. Besides the things covered by pjdfstest, JuiceFS also provides close-to-open consistency, atomic metadata operations, mmap, fallocate with punch hole, xattr, BSD locks (flock), and POSIX record locks (fcntl).\n\n## Atomic metadata operation\n\nIn Alluxio, a metadata operation involves two steps: modifying the state of the Alluxio master and sending a request to the UFS. This process is not atomic, and the state is unpredictable during execution or in case of failures. Additionally, Alluxio relies on UFS to implement metadata operations. For example, rename file operations will become copy and delete operations.\n\nThanks to [Redis transactions](https://redis.io/topics/transactions), **most metadata operations in JuiceFS are atomic**, for example, file renaming, file deletion, and directory renaming. You do not have to worry about the consistency and performance.\n\n## Consistency\n\nAlluxio loads metadata from the UFS as needed. It lacks information about UFS at startup. By default, Alluxio expects all modifications on UFS to be completed through Alluxio. If changes are made directly on UFS, you need to sync metadata between Alluxio and UFS either manually or periodically. As we have mentioned in [Atomic metadata operation](#atomic-metadata-operation) section, the two-step metadata operation may result in inconsistency.\n\nJuiceFS provides strong consistency for both metadata and data. **The metadata service of JuiceFS is the single source of truth, not a mirror of UFS.** The metadata service does not rely on object storage to obtain metadata, and object storage is just treated as unlimited block storage. This ensures there are no inconsistencies between JuiceFS and object storage.\n\n## Data compression\n\nJuiceFS supports data compression using [LZ4](https://lz4.github.io/lz4) or [Zstandard](https://facebook.github.io/zstd) for all your data, while Alluxio does not offer this feature.\n\n## Data encryption\n\nJuiceFS supports data encryption both in transit and at rest. Alluxio community edition lacks this feature, while it is available in the [enterprise edition](https://docs.alluxio.io/ee/user/stable/en/operation/Security.html#end-to-end-data-encryption).\n\n## Zero-effort operations\n\nAlluxio's architecture can be divided into three components: master, worker and client. A typical cluster consists of a single leading master, standby masters, a job master, standby job masters, workers, and job workers. You need to maintain all these masters and workers by yourself.\n\nJuiceFS uses Redis or [other databases](../../reference/how_to_set_up_metadata_engine.md) as the metadata engine. You could easily use the service managed by a public cloud provider as JuiceFS' metadata engine, without any operational overhead.\n" }, { "path": "docs/en/introduction/comparison/juicefs_vs_cephfs.md", "content": "---\nslug: /comparison/juicefs_vs_cephfs\ndescription: Ceph is a unified system that provides object storage, block storage and file storage. This article compares the similarities and differences between JuiceFS and Ceph.\n---\n\n# JuiceFS vs. CephFS\n\nThis document offers a comprehensive comparison between JuiceFS and CephFS. You will learn their similarities and differences in their system architectures and features.\n\n## Similarities\n\nBoth are highly reliable, high-performance, resilient distributed file systems with good POSIX compatibility, suitable for various scenarios.\n\n## Differences\n\n### System architecture\n\nBoth JuiceFS and CephFS employ an architecture that separates data and metadata, but they differ greatly in implementations.\n\n#### CephFS\n\nCephFS is a complete and independent system used mainly for private cloud deployments. Through CephFS, all file metadata and data are persistently stored in Ceph's distributed object store (RADOS).\n\n- Metadata\n - Metadata Server (MDS): Stateless and theoretically horizontally scalable. There are mature primary-secondary mechanisms, while concerns about performance and stability still exist in multi-primary deployments. Production environments typically adopt one-primary-multiple-secondary or multi-primary static isolation.\n - Persistent: Independent RADOS storage pools, usually used with SSDs or higher-performance hardware storage.\n- Data: Stored in one or more RADOS storage pools, supporting different configurations through _Layout_, such as chunk size (default 4 MiB) and redundancy (multi-copy, EC).\n- Client: Supports kernel client (`kcephfs`), user-state client (`ceph-fuse`) and libcephfs-based SDKs for C++, Python, etc.; recently the community has also provided a Windows client (`ceph-dokan`). VFS object for Samba and an FSAL module for NFS-Ganesha are also available in the ecosystem.\n\n#### JuiceFS\n\nJuiceFS provides a libjfs library, a FUSE client application, Java SDK, etc. It supports various metadata engines and object storages, and can be deployed in public, private, or hybrid cloud environments.\n\n- Metadata: Supports [various databases](../../reference/how_to_set_up_metadata_engine.md), including:\n - Redis and various variants of the Redis-compatible protocol (transaction supports are required)\n - SQL family: MySQL, PostgreSQL, SQLite, etc.\n - Distributed K/V storage: TiKV, FoundationDB, etcd\n - A self-developed engine: a JuiceFS fully managed service used on the public cloud.\n- Data: Supports over 30 types of [object storage](../../reference/how_to_set_up_object_storage.md) on the public cloud and can also be used with MinIO, Ceph RADOS, Ceph RGW, etc.\n- Clients: Supports Unix user-state mounting, Windows mounting, Java SDK with full HDFS semantic compatibility, [Python SDK](https://github.com/megvii-research/juicefs-python), and a built-in S3 gateway.\n\n### Features\n\n| Comparison basis | CephFS | JuiceFS |\n| ------------------------------- | --------------------- | --------------------- |\n| File chunking [1] | ✓ | ✓ |\n| Metadata transactions | ✓ | ✓ |\n| Strong consistency | ✓ | ✓ |\n| Kubernetes CSI Driver | ✓ | ✓ |\n| Hadoop-compatible | ✓ | ✓ |\n| Data compression [2] | ✓ | ✓ |\n| Data encryption [3] | ✓ | ✓ |\n| Snapshot | ✓ | ✕ |\n| Client data caching | ✕ | ✓ |\n| Hadoop data locality | ✕ | ✓ |\n| S3-compatible | ✕ | ✓ |\n| Quota | Directory level quota | Directory level quota |\n| Languages | C++ | Go |\n| License | LGPLv2.1 & LGPLv3 | Apache License 2.0 |\n\n#### [1] File chunking\n\nCephFS splits files by [`object_size`](https://docs.ceph.com/en/latest/cephfs/file-layouts/#reading-layouts-with-getfattr) (default 4MiB). Each chunk corresponds to a RADOS object. In contrast, JuiceFS splits files into 64MiB chunks and it further divides each chunk into logical slices during writing according to the actual situation. These slices are then split into logical blocks when writing to the object store, with each block corresponding to an object in the object storage. When handling overwrites, CephFS modifies corresponding objects directly, which is a complicated process. Especially, when the redundancy policy is EC or the data compression is enabled, part of the object content needs to be read first, modified in memory, and then written. This leads to great performance overhead. In comparison, JuiceFS handles overwrites by writing the updated data as new objects and modifying the metadata at the same time, which greatly improves the performance. Any redundant data generated during the process will go to garbage collection asynchronously.\n\n#### [2] Data compression\n\nStrictly speaking, CephFS itself does not provide data compression but relies on the BlueStore compression on the RADOS layer. JuiceFS, on the other hand, has already compressed data once before uploading a block to the object storage to reduce the capacity cost in the object storage. In other words, if you use JuiceFS to interact with RADOS, you compress a block both before and after it enters RADOS, twice in total. Also, as mentioned in **File chunking**, to guarantee overwrite performance, CephFS usually does not enable the BlueStore compression.\n\n#### [3] Data encryption\n\nOn network transport layer, Ceph encrypts data by using **Messenger v2**, while on data storage layer, the data encryption is done at OSD creation, which is similar to data compression.\n\nJuiceFS encrypts objects before uploading and decrypts them after downloading. This is completely transparent to the object storage.\n" }, { "path": "docs/en/introduction/comparison/juicefs_vs_glusterfs.md", "content": "---\ntitle: JuiceFS vs. GlusterFS\nslug: /comparison/juicefs_vs_glusterfs\ndescription: This document compares the design and features of GlusterFS and JuiceFS, helping you make an informed decision for selecting a storage solution.\n---\n\n[GlusterFS](https://github.com/gluster/glusterfs) is an open-source software-defined distributed storage solution. It can support data storage of PiB levels within a single cluster.\n\nJuiceFS is an open-source, high-performance distributed file system designed for the cloud. It delivers massive, elastic, and high-performance storage at low cost.\n\nThis document compares the key attributes of JuiceFS and GlusterFS in a table and then explores them in detail, offering insights to aid your team in the technology selection process. You can easily see their main differences in the table below and delve into specific topics you're interested in within this article.\n\n## A quick summary of GlusterFS vs. JuiceFS {#a-quick-summary-of-glusterfs-vs-juicefs}\n\nThe table below provides a quick overview of the differences between GlusterFS and JuiceFS:\n\n| **Comparison basis** | **GlusterFS** | **JuiceFS** |\n| :--- | :--- | :--- |\n| Metadata | Purely distributed | Independent database |\n| Data storage | Self-managed | Relies on object storage |\n| Large file handling | Doesn't split files | Splits large files |\n| Redundancy protection | Replication, erasure coding | Relies on object storage |\n| Data compression | Partial support | Supported |\n| Data encryption | Partial support | Supported |\n| POSIX compatibility | Full | Full |\n| NFS protocol | Not directly supported | Not directly supported |\n| CIFS protocol | Not directly supported | Not directly supported |\n| S3 protocol | Supported (but not updated) | Supported |\n| HDFS compatibility | Supported (but not updated) | Supported |\n| CSI Driver | Supported | Supported |\n| POSIX ACLs | Supported | Supported |\n| Cross-cluster replication | Supported | Relies on external service |\n| Directory quotas | Supported | Supported |\n| Snapshots | Supported | Not supported (but supports cloning) |\n| Trash | Supported | Supported |\n| Primary maintainer | Red Hat, Inc | Juicedata, Inc |\n| Development language | C | Go |\n| Open source license | GPLv2 and LGPLv3+ | Apache License 2.0 |\n\n## System architecture comparison {#system-architecture-comparison}\n\n### GlusterFS' architecture {#glusterfs-architectire}\n\nGlusterFS employs a fully distributed architecture without centralized nodes. A GlusterFS cluster consists of the server and the client. The server side manages and stores data, often referred to as the Trusted Storage Pool. This pool comprises a set of server nodes, each running two types of processes:\n\n* glusterd: One per node, which manages and distributes configuration.\n* glusterfsd: One per [brick](https://docs.gluster.org/en/latest/glossary/#Brick) (storage unit), which handles data requests and interfaces with the underlying file system.\n\nAll files on each brick can be considered a subset of GlusterFS. File content accessed directly through the brick or via GlusterFS clients is typically consistent. If GlusterFS experiences an exception, users can partially recover original data by integrating content from multiple bricks. Additionally, for fault tolerance during deployment, data is often redundantly protected. In GlusterFS, multiple bricks form a redundancy group, protecting data through replication or erasure coding. When a node experiences a failure, recovery can only be performed within the redundancy group, which may result in longer recovery times. When scaling a GlusterFS cluster, the scaling is typically performed on a redundancy group basis.\n\nThe client side, which mounts GlusterFS, presents a unified namespace to applications. The architecture diagram is as follows (source: [GlusterFS Architecture](https://docs.gluster.org/en/latest/Quick-Start-Guide/Architecture)):\n\n![GlusterFS architecture](../../images/glusterfs-architecture.jpg)\n\n### JuiceFS' architecture {#juicefs-architecture}\n\nJuiceFS adopts an architecture that separates its data and metadata storage. File data is split and stored in object storage systems like Amazon S3, while metadata is stored in a user-selected database like Redis or MySQL. By sharing the same database and object storage, JuiceFS achieves a strongly consistent distributed file system with features like full POSIX compatibility and high performance. For a more detailed introduction, see [the documentation](../architecture.md).\n\n![JuiceFS architecture](../../images/juicefs-arch-new.png)\n\n## Metadata management comparison {#metadata-management-comparison}\n\n### GlusterFS {#glusterfs}\n\nMetadata in GlusterFS is purely distributed, lacking a centralized metadata service. Clients use file name hashing to determine the associated brick. When requests require access across multiple bricks, for example, `mv` and `ls`, the client is responsible for coordination. While this design is simple, it can lead to performance bottlenecks as the system scales. For instance, listing a large directory might require accessing multiple bricks, and any latency in one brick can slow down the entire request. Additionally, ensuring metadata consistency when performing cross-brick modifications in the event of failures can be challenging, and severe failures may lead to split-brain scenarios, requiring [manual data recovery](https://docs.gluster.org/en/latest/Troubleshooting/resolving-splitbrain) to achieve a consistent version.\n\n### JuiceFS {#juicefs}\n\nJuiceFS metadata is stored in an independent database, which is called the metadata engine. Clients transform file metadata operations into transactions within this database, leveraging its transactional capabilities to ensure operation atomicity. This design simplifies JuiceFS implementation but places higher demands on the metadata engine. JuiceFS currently supports three categories of transactional databases. For details, see the [metadata engine document](../../reference/how_to_set_up_metadata_engine.md).\n\n## Data management comparison {#data-management-comparison}\n\nGlusterFS stores data by integrating multiple server nodes' bricks (typically built on local file systems like XFS). Therefore, it provides certain data management features, including distribution management, redundancy protection, fault switching, and silent error detection.\n\nJuiceFS, on the other hand, does not use physical disks directly but manages data through integration with various object storage systems. Most of its features rely on the capabilities of its object storage.\n\n### Large file splitting {#large-file-splitting}\n\nIn distributed systems, splitting large files into smaller chunks and storing them on different nodes is a common optimization technique. This often leads to higher concurrency and bandwidth when applications access such files.\n\n* GlusterFS does not split large files (although it used to support Striped Volumes for large files, this feature is no longer supported).\n* JuiceFS splits files into 64 MiB chunks by default, and each chunk is further divided into 4 MiB blocks based on the write pattern. For details, see [How JuiceFS stores files](../architecture.md#how-juicefs-store-files).\n\n### Redundancy protection {#redundancy-protection}\n\nGlusterFS supports both replication (Replicated Volume) and erasure coding (Dispersed Volume).\n\nJuiceFS relies on the redundancy capabilities of the underlying object storage it uses.\n\n### Data compression {#data-compression}\n\nGlusterFS:\n\n* Supports only transport-layer compression. Files are compressed by clients, transmitted to the server, and decompressed by the bricks.\n* Does not implement storage-layer compression but depends on the underlying file system used by the bricks, such as [ZFS](https://docs.gluster.org/en/latest/Administrator-Guide/Gluster-On-ZFS).\n\nJuiceFS supports both transport-layer and storage-layer compression. Data compression and decompression are performed on the client side.\n\n### Data encryption {#data-encryption}\n\nGlusterFS:\n\n* Supports only [transport-layer encryption](https://docs.gluster.org/en/latest/Administrator-Guide/SSL), relying on SSL/TLS.\n* Previously supported [storage-layer encryption](https://github.com/gluster/glusterfs-specs/blob/master/done/GlusterFS%203.5/Disk%20Encryption.md), but it is no longer supported.\n\nJuiceFS supports both [transport-layer and storage-layer encryption](../../security/encryption.md). Data encryption and decryption are performed on the client side.\n\n## Access protocols {#access-protocols}\n\n### POSIX compatibility {#posix-compatibility}\n\nBoth [GlusterFS](https://docs.gluster.org/en/latest/glossary) and [JuiceFS](../../reference/posix_compatibility.md) offer POSIX compatibility.\n\n### NFS protocol {#nfs-protocol}\n\nGlusterFS previously had embedded support for NFSv3 but now it is [no longer recommended](https://github.com/gluster/glusterfs-specs/blob/master/done/GlusterFS%203.8/gluster-nfs-off.md). Instead, it is suggested to export the mount point using an NFS server.\n\nJuiceFS does not provide direct support for NFS and requires mounting followed by [export via another NFS server](../../deployment/nfs.md).\n\n### CIFS protocol {#cifs-protocol}\n\nGlusterFS embeds support for Windows, Linux Samba clients, and macOS CLI access (excluding macOS Finder). However, it is recommended to [use Samba for exporting mount points](https://docs.gluster.org/en/latest/Administrator-Guide/Setting-Up-Clients/#testing-mounted-volumes).\n\nJuiceFS does not offer direct support for CIFS and requires mounting followed by [export via Samba](../../deployment/samba.md).\n\n### S3 protocol {#s3-protocol}\n\nGlusterFS supports S3 through the [`gluster-swift`](https://github.com/gluster/gluster-swift) project, but the project hasn't seen recent updates since November 2017.\n\nJuiceFS supports S3 through the [S3 gateway](../../guide/gateway.md).\n\n### HDFS compatibility {#hdfs-compatibility}\n\nGlusterFS offers HDFS compatibility through the [`glusterfs-hadoop`](https://github.com/gluster/glusterfs-hadoop) project, but the project hasn't seen recent updates since May 2015.\n\nJuiceFS provides [full compatibility with the HDFS API](../../deployment/hadoop_java_sdk.md).\n\n### CSI Driver {#csi-driver}\n\nGlusterFS [previously supported CSI Driver](https://github.com/gluster/gluster-csi-driver) but the latest version was released in November 2018, and the repository is marked as DEPRECATED.\n\nJuiceFS supports CSI Driver. For details, see the [document](https://juicefs.com/docs/csi/introduction).\n\n## Extended features {#extended-features}\n\n### POSIX ACLs {#posix-acls}\n\nIn Linux, file access permissions are typically controlled by three entities: the file owner, the group owner, and others. However, when more complex requirements arise, such as the need to assign specific permissions to a particular user within the others category, this standard mechanism does not work. POSIX Access Control Lists (ACLs) offer enhanced permission management capabilities, allowing you to assign permissions to any user or user group as needed.\n\nGlusterFS [supports ACLs](https://docs.gluster.org/en/main/Administrator-Guide/Access-Control-Lists), including access ACLs and default ACLs.\n\nJuiceFS supports the [POSIX ACLs](../../security/posix_acl.md) feature starting from v1.2.\n\n### Cross-cluster replication {#cross-cluster-replication}\n\nCross-cluster replication indicates replicating data between two independent clusters, often used for geographically distributed disaster recovery.\n\nGlusterFS [supports one-way asynchronous incremental replication](https://docs.gluster.org/en/main/Administrator-Guide/Geo-Replication) but requires both sides to use the same version of Gluster cluster.\n\nJuiceFS depends on the capabilities of the metadata engine and the object storage, allowing one-way replication.\n\n### Directory quotas {#directory-quotas}\n\nBoth [GlusterFS](https://docs.gluster.org/en/main/Administrator-Guide/Directory-Quota) and [JuiceFS](../../guide/quota.md#directory-quota) support directory quotas, including capacity and/or file count limits.\n\n### Snapshots {#snapshots}\n\nGlusterFS supports [volume-level snapshots](https://docs.gluster.org/en/main/Administrator-Guide/Managing-Snapshots) and requires all bricks to be deployed on LVM thinly provisioned volumes.\n\nJuiceFS does not support snapshots but offers [directory-level cloning](../../guide/clone.md).\n\n### Trash {#trash}\n\nGlusterFS [supports the trash functionality](https://docs.gluster.org/en/main/Administrator-Guide/Trash), which is disabled by default.\n\nJuiceFS [supports the trash functionality](../../security/trash.md), which is enabled by default.\n" }, { "path": "docs/en/introduction/comparison/juicefs_vs_lustre.md", "content": "---\nslug: /comparison/juicefs_vs_lustre\ndescription: This article compares the architecture, file distribution, and features of Lustre and JuiceFS.\n---\n\n# JuiceFS vs. Lustre\n\nLustre is a parallel distributed file system designed for HPC environments. Initially developed under U.S. government funding by national laboratories to support large-scale scientific and engineering computations, it's now maintained primarily by DataDirect Networks (DDN). Lustre is widely adopted in supercomputing centers, research institutions, and enterprise HPC clusters.\n\nJuiceFS is a cloud-native distributed file system that uses object storage to store data. The Community Edition supports integration with multiple metadata services and caters to diverse use cases. Its Enterprise Edition is specifically optimized for high-performance scenarios, with extensive applications in large-scale AI workloads including generative AI, autonomous driving, quantitative finance, and biotechnology.\n\nThis document provides a comprehensive comparison between Lustre and JuiceFS in terms of architecture, file distribution, and features.\n\n## Architecture comparison\n\n### Lustre\n\nLustre employs a traditional client-server architecture with the following core components:\n\n- **Metadata Servers (MDS)**: Handle namespace operations, such as file creation, deletion, and permission checks. Starting with version 2.4, Lustre introduced Distributed Namespace (DNE) to enable horizontal scaling by distributing different directories across multiple MDS within a single file system.\n- **Object Storage Servers (OSS)**: Manage actual data reads and writes, delivering high-performance large-scale read and write operations.\n- **Management Server (MGS)**: Acts as a global configuration registry, storing and distributing Lustre file system configuration information while remaining functionally independent of any specific Lustre instance.\n- **Clients**: Provides applications with access to the Lustre file system through a standard POSIX file operations interface.\n\nAll components are interconnected via LNet, Lustre's dedicated networking protocol, forming a unified and high-performance file system architecture.\n\n![Lustre architecture](https://static1.juicefs.com/images/Lustre_JiaGouTu_SWMlRaK.original.png)\n\n### JuiceFS\n\nJuiceFS uses a modular architecture that comprises three core components:\n\n- **Metadata engine**: Stores file system metadata, including standard file system metadata and file data indexes. The Community Edition supports various databases including Redis, TiKV, MySQL, PostgreSQL, and FoundationDB. The Enterprise Edition uses a self-developed high-performance metadata service.\n- **Data storage**: Primarily utilizes object storage services, which can be a public cloud object storage or an on-premises deployed object storage service. Supports over 30 types of object storage including AWS S3, Azure Blob, Google Cloud Storage, MinIO, and Ceph RADOS.\n- **Clients**: Provides multiple access protocols, such as POSIX (FUSE), Hadoop SDK, CSI Driver, S3 Gateway, and Python SDK.\n\n![JuiceFS Community Edition architecture](../../images/juicefs-arch.svg)\n\n### Architectural differences\n\n#### Client implementation\n\nLustre employs a C-language, kernel-space client architecture, while JuiceFS adopts a Go-based, user-space approach through file system in Userspace (FUSE). Because the Lustre client runs in kernel space, there is no need to perform context switching between user mode and kernel mode or additional memory copying when accessing the MDS or OSS. This significantly reduces the performance overhead caused by system calls and has certain advantages in throughput and latency.\n\nHowever, kernel-mode implementation also brings complexity to operation, maintenance, and debugging. Compared with user-mode development environments and debugging tools, kernel-mode tools have a higher threshold and are not easy for ordinary developers to master. JuiceFS's Go-based user-space implementation is easier to learn, maintain, and develop, with higher development efficiency and maintainability.\n\n#### Storage module\n\nLustre requires one or more shared disks to store file data. This design stems from the fact that its early versions did not support file level redundancy (FLR). To achieve high availability (HA), when a node goes offline, its file system must be mounted to a peer node, otherwise the data chunks on the node will be inaccessible. Therefore, the reliability of the data depends on the high availability mechanism of the shared storage itself or the software RAID implementation configured by the user.\n\nJuiceFS uses object storage as a data storage solution, thus enjoying several advantages brought by object storage, such as data reliability and consistency. Users can connect to specific storage systems according to their needs, including both object storage of mainstream cloud vendors and on-premises deployed object storage systems such as MinIO and Ceph RADOS. JuiceFS Community Edition provides local cache to cope with bandwidth requirements in AI scenarios, and the Enterprise Edition uses distributed cache to meet the needs of larger aggregate read bandwidth.\n\n#### Metadata module\n\nLustre's MDS high availability relies on the coordinated implementation of software and hardware:\n\n- **Hardware level**: The disks used by MDS need to be configured with RAID to avoid service unavailability due to single-point disk failure; the disks also need to have sharing capabilities so that when the primary node fails, the backup node can take over the disk resources.\n- **Software level**: Use Pacemaker and Corosync to build a high-availability cluster to ensure that only one MDS instance is active at any time.\n\nJuiceFS Community Edition provides a set of metadata operation interfaces that can access different metadata services, including databases like Redis, TiKV, MySQL, PostgreSQL, and FoundationDB. JuiceFS Enterprise Edition uses self-developed high-performance metadata services, which can balance data and hotspot operations according to load conditions to avoid the problem of metadata service hotspots being concentrated on certain nodes in large-scale training.\n\n## File distribution comparison\n\n### Lustre file distribution\n\n#### Normal file layout (NFL)\n\nLustre's initial file distribution mechanism segments files into multiple chunks distributed across object storage targets (OSTs) in a RAID 0-like striping pattern.\n\nKey distribution parameters:\n\n- **stripe count**: Determines the number of OSTs across which a file is striped. Higher values improve parallel access but increase scheduling and management overhead.\n- **stripe size**: Defines the chunk size written to each OST before switching to the next OST. This determines the granularity of each chunk.\n\n![Lustre NFL file distribution](https://static1.juicefs.com/images/Lustre_NFL_WenJianFenBuShiLi.original.png)\n\nThe figure above shows how a file with `stripe count = 3` and `stripe size = 1 MB` is distributed across multiple OSTs. Each data block (stripe) is allocated to different OSTs sequentially via round-robin scheduling.\n\nKey limitations include:\n\n- Configuration parameters are immutable after file creation\n- Can lead to ENOSPC (no space left) if any target OST runs out of space\n- May result in storage imbalance over time\n\n#### Progressive file layout (PFL)\n\nTo address the constraints of NFL, Lustre introduced progressive file layout (PFL), which allows defining different layout policies for different segments of the same file.\n\n![Lustre PFL file distribution](https://static1.juicefs.com/images/Lustre_PFL_WenJianFenBuShiLi.original.png)\n\nPFL provides advantages such as:\n\n- Dynamic adaptation to file growth\n- Mitigation of storage imbalance\n- Improved space efficiency and flexibility\n\nWhile PFL provides more adaptive layout strategies, Lustre integrates lazy initialization technology for more efficient dynamic resource scheduling to further address storage imbalance issues.\n\n#### File level redundancy (FLR)\n\nLustre introduced file level redundancy to simplify HA architecture and enhance fault tolerance. FLR allows configuring one or more replicas for each file to achieve file-level redundancy protection. During write operations, data is initially written to only one replica, while the others are marked as STALE. The system ensures data consistency through a synchronization process called Resync.\n\n### JuiceFS file distribution\n\nJuiceFS manages data blocks according to the rules of chunk, slice, and block. The size of each chunk is fixed at 64 MB, which optimizes data search and positioning. The actual file write operation is performed on slices. Each slice represents a continuous write process within a chunk, with length not exceeding 64 MB. A block (4 MB by default) is the basic unit of physical storage that implements the final storage of data in object storage and disk cache.\n\n![JuiceFS file distribution](../../images/file-and-chunks.svg)\n\nSlice in JuiceFS is a structure that is not common in other file systems. It records file write operations and persists them in object storage. Since object storage does not support in-place file modification, JuiceFS allows file content to be updated without rewriting the entire file by introducing the slice structure. When a file is modified, the system creates a new slice and updates the metadata after the slice is uploaded, pointing the file content to the new slice.\n\nAll slices of JuiceFS are written once, which reduces the reliance on the consistency of the underlying object storage and greatly simplifies the complexity of the cache system, making data consistency easier to ensure.\n\n## Feature comparison\n\n| Features | Lustre | JuiceFS Community | JuiceFS Enterprise |\n|----------|--------|-------------------|-------------------|\n| Metadata | Distributed metadata service | Independent database service | Proprietary high-performance distributed metadata engine (horizontally scalable) |\n| Metadata redundancy | Requires storage device support | Depends on the database used | Triple replication |\n| Data storage | Self-managed | Uses object storage | Uses object storage |\n| Data redundancy | Storage device or async replication | Provided by object storage | Provided by object storage |\n| Data caching | Client local cache | Client local cache | Proprietary high-performance multi-replica distributed cache |\n| Data encryption | Supported | Supported | Supported |\n| Data compression | Supported | Supported | Supported |\n| Quota management | Supported | Supported | Supported |\n| Network protocol | Multiple protocols supported | TCP | TCP |\n| Snapshots | File system-level snapshots | File-level snapshots | File-level snapshots |\n| POSIX ACL | Supported | Supported | Supported |\n| POSIX compliance | Compatible | Fully compatible | Fully compatible |\n| CSI Driver | Unofficially supported | Supported | Supported |\n| Client access | POSIX | POSIX (FUSE), Java SDK, S3 Gateway, Python SDK | POSIX (FUSE), Java SDK, S3 Gateway, Python SDK |\n| Multi-cloud mirroring | Not supported | Not supported | Supported |\n| Cross-cloud/region replication | Not supported | Not supported | Supported |\n| Primary maintainer | DDN | Juicedata | Juicedata |\n| Development language | C | Go | Go |\n| License | GPL 2.0 | Apache License 2.0 | Commercial software |\n\n## Summary\n\nLustre is a high-performance parallel distributed file system where clients run in kernel space, interacting directly with the MDS and OSS. This architecture eliminates context switching between user and kernel space, enabling exceptional performance in high-bandwidth I/O scenarios when combined with high-performance storage devices.\n\nHowever, running clients in kernel space increases operational complexity, requiring administrators to possess deep expertise in kernel debugging and underlying system troubleshooting. Additionally, Lustre's fixed-capacity storage approach and complex file distribution design demand meticulous planning and configuration for optimal resource utilization.\n\nJuiceFS is a cloud-native, user-space distributed file system that tightly integrates with object storage and natively supports Kubernetes CSI, simplifying deployment and management in cloud environments. Users can achieve elastic scaling and highly available data services in containerized environments without needing to manage underlying storage hardware or complex scheduling mechanisms. For performance optimization, JuiceFS Enterprise Edition employs distributed caching to significantly reduce object storage access latency and improve file operation responsiveness.\n\nFrom a cost perspective, Lustre requires high-performance dedicated storage hardware, resulting in substantial upfront investment and long-term maintenance expenses. In contrast, object storage offers greater cost efficiency, inherent scalability, and pay-as-you-go flexibility.\n\nBoth systems have their strengths: Lustre excels in traditional HPC environments requiring maximum performance, while JuiceFS provides better flexibility, easier management, and cost efficiency for cloud-native and AI workloads.\n" }, { "path": "docs/en/introduction/comparison/juicefs_vs_s3fs.md", "content": "---\nslug: /comparison/juicefs_vs_s3fs\ndescription: This document compares S3FS and JuiceFS, examining their product positioning, architecture, caching, and features.\n---\n\n# JuiceFS vs. S3FS\n\n[S3FS](https://github.com/s3fs-fuse/s3fs-fuse) is an open source tool developed in C++ that mounts S3 object storage locally via FUSE for read and write access as a local disk. In addition to Amazon S3, it supports all S3 API-compatible object stores.\n\nWhile both S3FS and JuiceFS share the basic functionality of mounting object storage buckets locally via FUSE and using them through POSIX interfaces, they differ significantly in functional details and technical implementation.\n\n## Product positioning\n\nS3FS is a utility that allows users to mount object storage buckets locally and read and write in a way that the users used to. It targets general use scenarios that are not sensitive to performance and network latency.\n\nJuiceFS is a distributed file system with a unique approach to data management and a series of technical optimizations for high performance, reliability, and security. It primarily addresses the storage needs of large volumes of data.\n\n## Architecture\n\nS3FS does not do special optimization for files. It acts as an access channel between local and object storage, allowing the same content to be seen on the local mount point and the object storage browser. This makes it easy to use cloud storage locally. On the other hand, with this simple architecture, retrieving, reading, and writing files with S3FS require direct interaction with the object store, and network latency can impact strongly on performance and user experience.\n\nJuiceFS uses a architecture that separates data and metadata. Files are split into data blocks according to specific rules before being uploaded to object storage, and the corresponding metadata is stored in a separate database. The advantage of this is that retrieval of files and modification of metadata such as file names can directly interact with the database with a faster response, bypassing the network latency impact of interacting with the object store.\n\nIn addition, when processing large files, although S3FS can solve the problem of transferring large files by uploading them in chunks, the nature of object storage dictates that appending files requires rewriting the entire object. For large files of tens or hundreds of gigabytes or even terabytes, repeated uploads waste a lot of time and bandwidth resources.\n\nJuiceFS avoids such problems by splitting individual files into chunks locally according to specific rules (default 4MiB) before uploading, regardless of their size. The rewriting and appending operations will eventually become new data blocks instead of modifying already generated data blocks. This greatly reduces the waste of time and bandwidth resources.\n\nFor a detailed description of the JuiceFS architecture, refer to the [documentation](../../introduction/architecture.md).\n\n## Caching\n\nS3FS supports disk caching, but it is disabled by default. Local caching can be enabled by specifying a cache path with `-o use_cache`. When caching is enabled, any file reads or writes will be written to the cache before the operation is actually performed. S3FS detects data changes via MD5 to ensure data correctness and reduce duplicate file downloads. Since all operations involved with S3FS require interactions with S3, whether the cache is enabled or not impacts significantly on its application experience.\n\nS3FS does not limit the cache capacity by default, which may cause the cache to fill up the disk when working with large buckets. You need to define the reserved disk space by `-o ensure_diskfree`. In addition, S3FS does not have a cache expiration and cleanup mechanism, so users need to manually clean up the cache periodically. Once the cache space is full, uncached file operations need to interact directly with the object storage, which will impact large file handling.\n\nJuiceFS uses a completely different caching approach than S3FS. First, JuiceFS guarantees data consistency. Secondly, JuiceFS defines a default disk cache usage limit of 100GiB, which can be freely adjusted by users as needed, and by default ensures that no more space is used when disk free space falls below 10%. When the cache usage limit reaches the upper limit, JuiceFS will automatically do cleanup using an LRU-like algorithm to ensure that cache is always available for subsequent read and write operations.\n\nFor more information on JuiceFS caching, see the [documentation](../../guide/cache.md).\n\n## Features\n\n| Comparison basis | S3FS | JuiceFS |\n|---------------------------|----------------------------------------------------------------|----------------------------------------------|\n| Data Storage | S3 | S3, other object storage, WebDAV, local disk |\n| Metadata Storage | No | Database |\n| Operating System | Linux, macOS | Linux, macOS, Windows |\n| Access Interface | POSIX | POSIX, HDFS API, S3 Gateway and CSI Driver |\n| POSIX Compatibility | Partially compatible | Fully compatible |\n| Shared Mounts | Supports but does not guarantee data integrity and consistency | Guarantee strong consistency |\n| Local Cache | ✓ | ✓ |\n| Symbol Links | ✓ | ✓ |\n| Standard Unix Permissions | ✓ | ✓ |\n| Strong Consistency | ✕ | ✓ |\n| Extended Attributes | ✕ | ✓ |\n| Hard Links | ✕ | ✓ |\n| File Chunking | ✕ | ✓ |\n| Atomic Operations | ✕ | ✓ |\n| Data Compression | ✕ | ✓ |\n| Client-side Encryption | ✕ | ✓ |\n| Development Language | C++ | Go |\n| Open Source License | GPL v2.0 | Apache License 2.0 |\n\n## Additional notes\n\n[OSSFS](https://github.com/aliyun/ossfs), [COSFS](https://github.com/tencentyun/cosfs), and [OBSFS](https://github.com/huaweicloud/huaweicloud-obs-obsfs) are all derivatives based on S3FS and have essentially the same functional features and usage as S3FS.\n" }, { "path": "docs/en/introduction/comparison/juicefs_vs_s3ql.md", "content": "---\nslug: /comparison/juicefs_vs_s3ql\n---\n\n# JuiceFS vs. S3QL\n\nSimilar to JuiceFS, S3QL is also an open source network file system driven by object storage and database. All data will be split into blocks and stored in object storage services such as Amazon S3, Backblaze B2, or OpenStack Swift, and the corresponding metadata will be stored in the database.\n\n## Common ground\n\n- Both support the standard POSIX file system interface through the FUSE module, so that massive cloud storage can be mounted locally and used like local storage.\n- Both provide standard file system features: hard links, symbolic links, extended attributes, file permissions.\n- Both support data compression and encryption, but the algorithms used are different.\n- Both support metadata backup, S3QL automatically backs up SQLite databases to object storage, and JuiceFS automatically exports metadata to JSON format files every hour and backs them up to object storage for easy recovery and migration between various metadata engines.\n\n## Differences\n\n- S3QL only supports SQLite. But JuiceFS supports more databases, such as Redis, TiKV, MySQL, PostgreSQL, and SQLite.\n- S3QL has no distributed capability and **does not** support multi-host shared mounting. JuiceFS is a typical distributed file system. When using a network-based database, it supports multi-host distributed mount read and write.\n- S3QL commits a data block to S3 when it has not been accessed for more than a few seconds. After a file closed or even fsynced, it is only guaranteed to stay in system memory, which may result in data loss if node fails. JuiceFS ensures high data durability, uploading all blocks synchronously when a file is closed.\n- S3QL provides data deduplication. Only one copy of the same data is stored, which can reduce the storage usage, but it will also increase the performance overhead of the system. JuiceFS pays more attention to performance, and it is too expensive to perform deduplication on large-scale data, so this function is temporarily not provided.\n\n| | **S3QL** | **JuiceFS** |\n| :------------------------ | :-------------------- | :---------------------------- |\n| Project status | Active development | Active development |\n| Metadata engine | SQLite | Redis, MySQL, SQLite, TiKV |\n| Storage engine | Object Storage, Local | Object Storage, WebDAV, Local |\n| Operating system | Unix-like | Linux, macOS, Windows |\n| Compression algorithm | LZMA, bzip2, gzip | LZ4, zstd |\n| Encryption algorithm | AES-256 | AES-GCM, RSA |\n| POSIX compatible | ✓ | ✓ |\n| Hard link | ✓ | ✓ |\n| Symbolic link | ✓ | ✓ |\n| Extended attributes | ✓ | ✓ |\n| Standard Unix permissions | ✓ | ✓ |\n| Data block | ✓ | ✓ |\n| Local cache | ✓ | ✓ |\n| Elastic storage | ✓ | ✓ |\n| Metadata backup | ✓ | ✓ |\n| Data deduplication | ✓ | ✕ |\n| Immutable trees | ✓ | ✕ |\n| Snapshots | ✓ | ✕ |\n| Share mount | ✕ | ✓ |\n| Hadoop SDK | ✕ | ✓ |\n| Kubernetes CSI Driver | ✕ | ✓ |\n| S3 gateway | ✕ | ✓ |\n| Language | Python | Go |\n| Open source license | GPLv3 | Apache License 2.0 |\n| Open source date | 2011 | 2021.1 |\n\n## Usability\n\nThis part mainly evaluates the ease of installing and using the two products.\n\n### Installation\n\nDuring the installation process, we use Rocky Linux 8.4 operating system (kernel version 4.18.0-305.12.1.el8_4.x86_64).\n\n#### S3QL\n\nS3QL is developed in Python and requires `python-devel` 3.7 or higher to be installed. In addition, at least the following dependencies must be satisfied: `fuse3-devel`, `gcc`, `pyfuse3`, `sqlite-devel`, `cryptography`, `defusedxml`, `apsw`, `dugong`. In addition, you need to pay special attention to Python's package dependencies and location issues.\n\nS3QL will install 12 binary programs in the system, and each program provides an independent function, as shown in the figure below.\n\n![S3QL-bin](../../images/s3ql-bin.jpg)\n\n#### JuiceFS\n\nJuiceFS is developed in Go and can be used directly by downloading the pre-compiled binary file. The JuiceFS client has only one binary program `juicefs`. You can just copy it to any executable path of the system, for example: `/usr/local/bin`.\n\n### Create and Mount a file system\n\nBoth S3QL and JuiceFS use database to store metadata. S3QL only supports SQLite databases, while JuiceFS supports databases such as Redis, TiKV, MySQL, MariaDB, PostgreSQL, and SQLite.\n\nHere we create a file system using S3QL and JuiceFS separately with locally created MinIO as object storage:\n\n#### S3QL\n\nS3QL uses `mkfs.s3ql` to create a file system:\n\n```shell\nmkfs.s3ql --plain --backend-options no-ssl -L s3ql s3c://127.0.0.1:9000/s3ql/\n```\n\nMount a file system using `mount.s3ql`:\n\n```shell\nmount.s3ql --compress none --backend-options no-ssl s3c://127.0.0.1:9000/s3ql/ mnt-s3ql\n```\n\nS3QL needs the access key of the object storage API to be interactively provided through the command line when creating and mounting a file system.\n\n#### JuiceFS\n\nJuiceFS uses the `format` subcommand to create a file system:\n\n```shell\njuicefs format --storage minio \\\n --bucket http://127.0.0.1:9000/myjfs \\\n --access-key minioadmin \\\n --secret-key minioadmin \\\n sqlite3://myjfs.db \\\n myjfs\n```\n\nMount a file system using `mount` subcommand:\n\n```shell\nsudo juicefs mount -d sqlite3://myjfs.db mnt-juicefs\n```\n\nJuiceFS only sets the object storage API access key when creating a file system, and the relevant information will be written into the metadata engine. After created, there is no need to repeatedly provide the object storage url, access key and other information.\n\n## Summary\n\n**S3QL** adopts the storage structure of object storage + SQLite. Storing data in blocks can not only improve the read and write efficiency of the file but also reduce the resource overhead when the file is modified. The advanced features such as snapshots, data deduplication, and data retention, as well as the default data compression and data encryption make S3QL very suitable for individuals to store files in cloud storage at a lower cost and with higher security.\n\n**JuiceFS** supports object storage, HDFS, WebDAV, and local disks as data storage engines, and supports popular databases such as Redis, TiKV, MySQL, MariaDB, PostgreSQL, and SQLite as metadata storage engines. It provides a standard POSIX file system interface through FUSE and a Java API, which can directly replace HDFS to provide storage for Hadoop. At the same time, it also provides [Kubernetes CSI Driver](https://github.com/juicedata/juicefs-csi-driver), which can be used as the storage layer of Kubernetes for data persistent storage. JuiceFS is a file system designed for enterprise-level distributed data storage scenarios. It is widely used in various scenarios such as big data analysis, machine learning, container shared storage, data sharing, and backup.\n" }, { "path": "docs/en/introduction/comparison/juicefs_vs_seaweedfs.md", "content": "---\ntitle: JuiceFS vs. SeaweedFS\nslug: /comparison/juicefs_vs_seaweedfs\ndescription: This document compares JuiceFS and SeaweedFS, covering their architecture, storage mechanisms, client protocols, and other advanced features.\n---\n\n[SeaweedFS](https://github.com/seaweedfs/seaweedfs) and [JuiceFS](https://github.com/juicedata/juicefs) are both open-source high-performance distributed file storage systems. They operate under the business-friendly Apache License 2.0. However, JuiceFS comes in two versions: a [Community Edition](https://juicefs.com/docs/community/introduction) and an [Enterprise Edition](https://juicefs.com/en/blog/solutions/juicefs-enterprise-edition-features-vs-community-edition), you can use JuiceFS Enterprise Edition as on-premises deployment, or [use Cloud Service](https://juicefs.com/docs/cloud) directly. The Enterprise Edition uses a proprietary metadata engine, while its client shares code extensively with the [Community Edition](https://github.com/juicedata/juicefs).\n\nThis document compares the key attributes of JuiceFS and SeaweedFS in a table and then explores them in detail. You can easily see their main differences in the table below and delve into specific topics you're interested in within this article. By highlighting their contrasts and evaluating their suitability for different use cases, this document aims to help you make informed decisions.\n\n## A quick summary of SeaweedFS vs. JuiceFS\n\n| Comparison basis | SeaweedFS | JuiceFS |\n| :--- | :--- | :--- |\n| Metadata engine | Supports multiple databases | The Community Edition supports various databases; the Enterprise Edition uses an in-house, high-performance metadata engine. |\n| Metadata operation atomicity | Not guaranteed | The Community Edition ensures atomicity through database transactions; the Enterprise Edition ensures atomicity within the metadata engine. |\n| Changelog | Supported | Exclusive to the Enterprise Edition |\n| Data storage | Self-contained | Relies on object storage |\n| Erasure coding | Supported | Relies on object storage |\n| Data consolidation | Supported | Relies on object storage |\n| File splitting | 8MB | 64MB logical blocks + 4MB physical storage blocks |\n| Tiered storage | Supported | Relies on object storage |\n| Data compression | Supported (based on file extensions) | Supported (configured globally) |\n| Storage encryption | Supported | Supported |\n| POSIX compatibility | Basic | Full |\n| S3 protocol | Basic | Basic |\n| WebDAV protocol | Supported | Supported |\n| HDFS compatibility | Basic | Full |\n| CSI Driver | Supported | Supported |\n| Client cache | Supported | Supported |\n| Cluster data replication | Unidirectional and bidirectional replication is supported | Exclusive to the Enterprise Edition, only unidirectional replication is supported |\n| Cloud data cache | Supported (manual synchronization) | Exclusive to the Enterprise Edition |\n| Trash | Unsupported | Supported |\n| Operations and monitoring | Supported | Supported |\n| Release date | April 2015 | January 2021 |\n| Primary maintainer | Individual (Chris Lu) | Company (Juicedata Inc.) |\n| Programming language | Go | Go |\n| Open source license | Apache License 2.0 | Apache License 2.0 |\n\n## The SeaweedFS architecture\n\nThe system consists of three components:\n\n- The volume servers, which store files in the underlying layer\n- The master servers, which manage the cluster\n- An optional component, filer, which provides additional features to the upper layer\n\n![SeaweedFS architecture](../../images/seaweedfs_arch_intro.png)\n\nIn the system operation, both the volume server and the master server are used for file storage:\n\n- The volume server focuses on data read and write operations.\n- The master server primarily functions as a management service for the cluster and volumes.\n\nIn terms of data access, SeaweedFS implements a similar approach to Haystack. A user-created volume in SeaweedFS corresponds to a large disk file (\"Superblock\" in the diagram below). Within this volume, all files written by the user (\"Needles\" in the diagram) are merged into the large disk file.\n\n![SeaweedFS Superblock](../../images/seaweedfs_superblock.png)\n\nData write and read process in SeaweedFS:\n\n1. Before a write operation, the client initiates a write request to the master server.\n2. SeaweedFS returns a File ID based on the current data volume. This ID is composed of three parts: \\. During the writing process, basic metadata information such as file length and chunk details is also written together with the data.\n3. After the write is completed, the caller needs to associate the file with the returned File ID and store this mapping in an external system such as MySQL.\n4. When reading data, since the volume index is already loaded in memory, the system can use the File ID to quickly retrieve all necessary information about the file's location (offset). This enables efficient file reading.\n\nOn top of the underlying storage services, SeaweedFS offers a component called filer, which interfaces with the volume server and the master server. It provides features like POSIX support, WebDAV, and the S3 API. Like JuiceFS, the filer needs to connect to an external database to store metadata information.\n\n## The JuiceFS architecture\n\nJuiceFS adopts an architecture that separates data and metadata storage:\n\n- File data is split and stored in object storage systems such as Amazon S3.\n- Metadata is stored in a user-selected database such as Redis or MySQL.\n\nThe client connects to the metadata engine for metadata services and writes actual data to object storage, achieving distributed file systems with strong consistency .\n\n![JuiceFS architecture](../../images/juicefs-arch-new.png)\n\nFor details about JuiceFS' architecture, see the [Architecture](../architecture.md) document.\n\n## Architecture comparison\n\n### Metadata\n\nBoth SeaweedFS and JuiceFS support storing file system metadata in external databases:\n\n- SeaweedFS supports up to [24 databases](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Stores).\n- JuiceFS has a high requirement for database transaction capabilities and currently supports [10 transactional databases across 3 categories](../../reference/how_to_set_up_metadata_engine.md).\n\n### Atomic operations\n\nJuiceFS ensures strict atomicity for every operation, which requires strong transaction capabilities from the metadata engine like Redis and MySQL. As a result, JuiceFS supports fewer databases.\n\nSeaweedFS provides weaker atomicity guarantees for operations. It only uses transactions of some databases (SQL, ArangoDB, and TiKV) during rename operations, with a lower requirement for database transaction capabilities. Additionally, during the rename operation, SeaweedFS does not lock the original directory or file during the metadata copying process. This may result in data loss under high loads.\n\n### Changelog and related features\n\nSeaweedFS generates changelog for all metadata operations. The changelog can be transmitted and replayed. This ensures data safety and enables features like file system data replication and operation auditing.\n\nSeaweedFS supports file system data replication between multiple clusters. It offers two asynchronous data replication modes:\n\n- Active-Active. In this mode, both clusters participate in read and write operations and they synchronize data bidirectionally. When there are more than two nodes in the cluster, certain operations such as renaming directories are subject to certain restrictions.\n- Active-Passive. In this mode, a primary-secondary relationship is established, and the passive side is read-only.\n\nBoth modes achieve consistency between different cluster data by transmitting and applying changelog. Each changelog has a signature to ensure that the same message is applied only once.\n\nThe JuiceFS Community Edition does not implement a changelog, but it can use its inherent data replication capabilities from the metadata engine and object storage to achieve file system mirroring. For example, both [MySQL](https://dev.mysql.com/doc/refman/8.0/en/replication.html) and [Redis](https://redis.io/docs/management/replication) only support data replication. When combined with [S3's object replication feature](https://docs.aws.amazon.com/AmazonS3/latest/userguide/replication.html), either of them can enable a setup similar to SeaweedFS' Active-Passive mode without relying on JuiceFS.\n\nIt's worth noting that the JuiceFS Enterprise Edition implements the metadata engine based on changelog. It supports [data replication](https://juicefs.com/docs/cloud/guide/replication) and [mirror file system](https://juicefs.com/docs/cloud/guide/mirror).\n\n## Storage comparison\n\nAs mentioned earlier, SeaweedFS' data storage is achieved through volume servers + master servers, supporting features like merging small data blocks and erasure coding.\n\nJuiceFS' data storage relies on object storage services, and relevant features are provided by the object storage.\n\n### File splitting\n\nBoth SeaweedFS and JuiceFS split files into smaller chunks before persisting them in the underlying data system:\n\n- SeaweedFS splits files into 8MB blocks. For extremely large files (over 8GB), it also stores the chunk index in the underlying data system.\n- JuiceFS uses 64MB logical data blocks (chunks), which are further divided into 4MB blocks to be uploaded to object storage. For details, see [How JuiceFS stores files](../architecture.md#how-juicefs-store-files).\n\n### Tiered storage\n\nFor newly created volumes, SeaweedFS stores data locally. For older volumes, SeaweedFS supports uploading them to the cloud to achieve [hot-cold data separation](https://github.com/seaweedfs/seaweedfs/wiki/Tiered-Storage).\n\nJuiceFS does not implement tiered storage but directly uses object storage's tiered management services, such as [Amazon S3 Glacier storage classes](https://aws.amazon.com/s3/storage-classes/glacier/?nc1=h_ls).\n\n### Data compression\n\nJuiceFS supports compressing all written data using LZ4 or Zstandard.\nSeaweedFS determines whether to compress data based on factors such as the file extension and file type.\n\n### Encryption\n\nBoth support encryption, including encryption during transmission and at rest:\n\n- SeaweedFS supports encryption both in transit and at rest. When data encryption is enabled, all data written to the volume server is encrypted using random keys. The corresponding key information is managed by the filer that maintains the file metadata. For details, see the [Wiki](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Data-Encryption).\n- For details about JuiceFS' encryption feature, see [Data Encryption](../../security/encryption.md).\n\n## Client protocol comparison\n\n### POSIX\n\nJuiceFS is [fully POSIX-compatible](../../reference/posix_compatibility.md), while SeaweedFS currently [partially implements POSIX compatibility](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount), with ongoing feature enhancements.\n\n### S3\n\nJuiceFS implements an [S3 gateway](https://juicefs.com/docs/community/s3_gateway), enabling direct access to the file system through the S3 API. It supports tools like s3cmd, AWS CLI, and MinIO Client (mc) for file system management.\n\nSeaweedFS currently [supports a subset of the S3 API](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API), covering common read, write, list, and delete requests, with some extensions for specific requests like reads.\n\n### WebDAV\n\nBoth support the WebDAV protocol. For details, see:\n\n- [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki/WebDAV)\n- [JuiceFS documentation](../../deployment/webdav.md)\n\n### HDFS\n\nJuiceFS is [fully compatible with the HDFS API](../../deployment/hadoop_java_sdk.md), including Hadoop 2.x, Hadoop 3.x, and various components within the Hadoop ecosystem.\n\nSeaweedFS offers [basic HDFS compatibility](https://github.com/seaweedfs/seaweedfs/wiki/Hadoop-Compatible-File-System). It lacks support for advanced operations like truncate, concat, checksum, and set attributes.\n\n### CSI Driver\n\nBoth support a CSI Driver. For details, see:\n\n- [SeaweedFS CSI Driver](https://github.com/seaweedfs/seaweedfs-csi-driver)\n- [JuiceFS CSI Driver](https://github.com/juicedata/juicefs-csi-driver)\n\n## Other advanced features\n\n### Client cache\n\nSeaweedFS client is equipped with [basic cache capabilities](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount), but its documentation weren't located at the time of writing, you can search for `cache` in the [source code](https://github.com/seaweedfs/seaweedfs/blob/master/weed/command/mount.go).\n\nJuiceFS' client supports [metadata and data caching](../../guide/cache.md), allowing users to optimize based on their application's needs.\n\n### Object storage gateway\n\nSeaweedFS can be used as an [object storage gateway](https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage), you can manually warm up specified data to local cache directory, while local modification is asynchronously uploaded to object storage.\n\nJuiceFS stores files in split form. Due to its architecture, it does not support serving as a cache for object storage or a cache layer. However, the JuiceFS Enterprise Edition has a standalone feature to provide caching services for existing data in object storage, which is similar to SeaweedFS' object storage gateway.\n\n### Trash\n\nBy default, JuiceFS enables the [Trash](../../security/trash.md) feature. To prevent accidental data loss and ensure data safety, deleted files are retained for a specified time.\nHowever, SeaweedFS does not support this feature.\n\n### Operations and maintenance\n\nBoth offer comprehensive maintenance and troubleshooting solutions:\n\n- JuiceFS provides [`juicefs stats`](../../administration/fault_diagnosis_and_analysis.md#stats) and [`juicefs profile`](../../administration/fault_diagnosis_and_analysis.md#profile) to let users view real-time performance metrics. It offers a [`metrics`](../../administration/monitoring.md#collect-metrics) API to integrate monitoring data into Prometheus for visualization and monitoring alerts in Grafana.\n- SeaweedFS uses [`weed shell`](https://github.com/seaweedfs/seaweedfs/wiki/weed-shell) to interactively execute maintenance tasks, such as checking the current cluster status and listing file directories. It also supports [push and pull](https://github.com/seaweedfs/seaweedfs/wiki/System-Metrics) approaches to integrate with Prometheus.\n" }, { "path": "docs/en/introduction/io_processing.md", "content": "---\ntitle: Data Processing Workflow\nsidebar_position: 3\nslug: /internals/io_processing\ndescription: This article introduces read and write implementation of JuiceFS, including how it splits files into chunks.\n---\n\n## Data writing process {#workflow-of-write}\n\nJuiceFS splits large files at multiple levels to improve I/O performance. See [how JuiceFS stores files](./architecture.md#how-juicefs-store-files). Files are initially divided into logical chunks (64 MiB each), which are isolated from each other and further broken down into slices. Slices are the data units for persistence. During a write request, data is stored in the client buffer as chunks/slices. A new slice is created if it does not overlap or adjoin any existing slices; otherwise, the affected existing slices are updated. On a flush operation, a slice is divided into blocks (4 MiB by default) and uploaded to the object storage. Metadata is updated upon successful upload.\n\nSequential writes are optimized, requiring only one continuously growing slice and one final flush. This maximizes object storage write performance. A simple [JuiceFS benchmark](../benchmark/performance_evaluation_guide.md) below shows sequentially writing a 1 GiB file with a 1 MiB I/O size at its first stage. The following figure shows the data flow in each component of the system.\n\n![internals-write](../images/internals-write.png)\n\nUse [`juicefs stats`](../reference/command_reference.mdx#stats) to obtain real-time performance monitoring metrics.\n\n![internals-stats](../images/internals-stats.png)\n\nThe first highlighted section in the above figure shows:\n\n- The average I/O size for writing to the object storage is `object.put / object.put_c = 4 MiB`. It is the same as the default block size.\n- The ratio of metadata transactions to object storage transactions is `meta.txn : object.put_c -= 1 : 16`. It means that a single slice flush requires 1 metadata update and 16 uploads to the object storage. Each flush operation transmits 64 MiB of data (4 MiB * 16), equivalent to the default chunk size.\n- The average request size in the FUSE layer approximately equals to `fuse.write / fuse.ops ~= 128 KiB`, matching the default request size limitation.\n\nGenerally, when JuiceFS writes a small file, the file is uploaded to the object storage upon file closure, and the I/O size is equal to the file size. In the third stage of the figure above, where 128 KiB small files are created, we can see that:\n\n- The size of data written to the object storage during PUT operations is 128 KiB, calculated by `object.put / object.put_c`.\n- The number of metadata transactions is approximately twice the number of PUT operations, since each file requires one create and one write.\n\nWhen JuiceFS uploads objects smaller than the block size, it simultaneously writes them into the [local cache](../guide/cache.md) to improve future performance. As shown in the third stage of the figure above, the write bandwidth of the `blockcache` is the same as that of the object storage. Since small files are cached, reading these files is extremely fast, as demonstrated in the fourth stage.\n\nWrite operations are immediately committed to the client buffer, resulting in very low write latency (typically just a few microseconds). The actual upload to the object storage is automatically triggered internally when certain conditions are met, such as when the size or number of slices exceeds their limit, or data stays in the buffer for too long. Explicit calls, such as closing a file or invoking `fsync`, can also trigger uploading.\n\nThe client buffer is only released after the data stored inside is uploaded. In scenarios with high write concurrency, if the buffer size (configured using [`--buffer-size`](../reference/command_reference.mdx#mount-data-cache-options)) is not big enough, or the object storage's performance insufficient, write blocking may occur, because the buffer cannot be released timely. The real-time buffer usage is shown in the `usage.buf` field in the metrics figure. To slow things down, The JuiceFS client introduces a 10 ms delay to every write when the buffer usage exceeds the threshold. If the buffer usage is over twice the threshold, new writes are completely suspended until the buffer is released. Therefore, if the write latency keeps increasing or the buffer usage has exceeded the threshold for a long while, you should increase `--buffer-size`. Also consider increasing the maximum number of upload concurrency ([`--max-uploads`](../reference/command_reference.mdx#mount-data-storage-options), defaults to 20), which improves the upload bandwidth, thus boosting buffer release.\n\n### Random writes {#random-write}\n\nJuiceFS supports random writes, including mmap-based random writes.\n\nNote that a block is an immutable object, because most object storage services don't support edit in blocks; they can only be re-uploaded and overwritten. Thus, when overwrites or random writes occur, JuiceFS avoids downloading the block for editing and re-uploading, which could cause serious I/O amplifications. Instead, writes are performed on new or existing slices. Relevant new blocks are uploaded to the object storage, and the new slice is appended to the slice list under the chunk. When a file is read, what the client sees is actually a consolidated view of all the slices.\n\nCompared to sequential writes, random writes in large files are more complicated. There could be a number of intermittent slices in a chunk, possibly all smaller than 4 MiB. Frequent random writes require frequent metadata updates, which in turn further impact performance. To improve read performance, JuiceFS schedules compaction tasks when the number of slices under a chunk exceeds the limit. You can also manually trigger compaction by running [`juicefs gc`](../administration/status_check_and_maintenance.md#gc).\n\n### Client write cache {#client-write-cache}\n\nClient write cache is also referred to as \"Writeback mode\" throughout the docs.\n\nFor scenarios that does not deem consistency and data security as top priorities, enabling client write cache is also an option to further improve performance. When client write cache is enabled, flush operations return immediately after writing data to the local cache directory. Then, local data is uploaded asynchronously to the object storage. In other words, the local cache directory is a cache layer for the object storage.\n\nLearn more in [Client Write Cache](../guide/cache.md#client-write-cache).\n\n## Data reading process {#workflow-of-read}\n\nJuiceFS supports sequential reads and random reads (including mmap-based random reads). During read requests, the object corresponding to the block is completely read through the `GetObject` API of the object storage, or only a certain range of data in the object may be read (e.g., the read range is limited by the `Range` parameter of [S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html)). Meanwhile, prefetching is performed (controlled by the [`--prefetch`](../reference/command_reference.mdx#mount) option) to download the complete data block into the local cache directory, as shown in the `blockcache` write speed in the second stage of the above metrics figure. This is very good for sequential reads as all cached data is utilized, maximizing the object storage access efficiency. The dataflow is illustrated in the figure below:\n\n![internals-read](../images/internals-read.png)\n\nAlthough prefetching works well for sequential reads, it might not be so effective for random reads on large files. It can cause read amplification and frequent cache eviction. Consider disabling prefetching using `--prefetch=0`. It is always hard to design cache strategy for random read scenarios. Two possible solutions are increasing the cache size to store all data locally or completely disabling the cache (`--cache-size=0`) and relying on a high-performance object storage service.\n\nReading small files (smaller than the block size) is much easier because the entire file can be read in a single request. Since small files are cached locally during the write process, future reads are fast.\n" }, { "path": "docs/en/reference/_common_options.mdx", "content": "#### Metadata related options {#mount-metadata-options}\n\n|Items|Description|\n|-|-|\n|`--subdir=value`|mount a sub-directory as root (default: \"\")|\n|`--backup-meta=3600`|interval (in seconds) to automatically backup metadata in the object storage (0 means disable backup) (default: \"3600\")|\n|`--backup-skip-trash` 1.2|skip files and directories in trash when backup metadata.|\n|`--heartbeat=12`|interval (in seconds) to send heartbeat; it's recommended that all clients use the same heartbeat value (default: \"12\")|\n|`--read-only`|Read-only mode, i.e. allow only lookup/read operations. Note that this option implies `--no-bgjob`, so read-only clients do not execute background jobs.|\n|`--no-bgjob`|Disable background jobs, default to false, which means clients by default carry out background jobs, including:
  • Clean up expired files in Trash (look for `cleanupDeletedFiles`, `cleanupTrash` in [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/base.go))
  • Delete slices that's not referenced (look for `cleanupSlices` in [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/base.go))
  • Clean up stale client sessions (look for `CleanStaleSessions` in [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/base.go))
Note that compaction isn't affected by this option, it happens automatically with file reads and writes, client will check if compaction is in need, and run in background (take Redis for example, look for `compactChunk` in [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/redis.go)).|\n|`--atime-mode=noatime` 1.1 |Control atime (last time the file was accessed) behavior, support the following modes:
  • `noatime` (default): set when the file is created or when `SetAttr` is explicitly called. Accessing and modifying the file will not affect atime, tracking atime comes at a performance cost, so this is the default behavior
  • `relatime`: update inode access times relative to mtime (last time when the file data was modified) or ctime (last time when file metadata was changed). Only update atime if atime was earlier than the current mtime or ctime, or the file's atime is more than 1 day old
  • `strictatime`: always update atime on access
|\n|`--skip-dir-nlink=20` 1.1 |number of retries after which the update of directory nlink will be skipped (used for tkv only, 0 means never) (default: 20)|\n|`--skip-dir-mtime=100ms` 1.2|skip updating attribute of a directory if the mtime difference is smaller than this value (default: 100ms)|\n|`--sort-dir` 1.3|sort entries within a directory by name|\n|`--fast-statfs` 1.3|performance of `statfs` is improved by using local caching to reduce metadata access, but accuracy may decrease (default: false)|\n\n#### Metadata cache related options {#mount-metadata-cache-options}\n\nFor metadata cache description and usage, refer to [Kernel metadata cache](../guide/cache.md#kernel-metadata-cache) and [Client memory metadata cache](../guide/cache.md#client-memory-metadata-cache).\n\n|Items|Description|\n|-|-|\n|`--attr-cache=1`|attributes cache timeout in seconds (default: 1), read [Kernel metadata cache](../guide/cache.md#kernel-metadata-cache)|\n|`--entry-cache=1`|file entry cache timeout in seconds (default: 1), read [Kernel metadata cache](../guide/cache.md#kernel-metadata-cache)|\n|`--dir-entry-cache=1`|dir entry cache timeout in seconds (default: 1), read [Kernel metadata cache](../guide/cache.md#kernel-metadata-cache)|\n|`--open-cache=0`|open file cache timeout in seconds (0 means disable this feature) (default: 0)|\n|`--open-cache-limit value` 1.1 |max number of open files to cache (soft limit, 0 means unlimited) (default: 10000)|\n|`--readdir-cache=false` 1.3, only for mount|enable directory entry cache (default: false, disable this feature)|\n|`--negative-entry-cache=0` 1.3, only for mount|negative lookup (return ENOENT) cache timeout in seconds (default: 0, means disable this feature)|\n\n#### Data storage related options {#mount-data-storage-options}\n\n|Items|Description|\n|-|-|\n|`--storage=file`|Object storage type (e.g. `s3`, `gs`, `oss`, `cos`) (default: `\"file\"`, refer to [documentation](../reference/how_to_set_up_object_storage.md#supported-object-storage) for all supported object storage types).|\n|`--bucket=value`|customized endpoint to access object storage|\n|`--storage-class value` 1.1 |the storage class for data written by current client|\n|`--get-timeout=60`|the max number of seconds to download an object (default: 60)|\n|`--put-timeout=60`|the max number of seconds to upload an object (default: 60)|\n|`--io-retries=10`|The number of retries when the network is abnormal and the number of retries for metadata requests are also controlled by this option. If the number of retries is exceeded, an `EIO Input/output error` error will be returned. (default: 10)|\n|`--max-uploads=20`|Upload concurrency, defaults to 20. This is already a reasonably high value for 4M writes, with such write pattern, increasing upload concurrency usually demands higher `--buffer-size`, learn more at [Read/Write Buffer](../guide/cache.md#buffer-size). But for random writes around 100K, 20 might not be enough and can cause congestion at high load, consider using a larger upload concurrency, or try to consolidate small writes in the application end. |\n|`--max-stage-write=0` 1.2|The maximum number of concurrent writes of data blocks to the cache disk asynchronously. If the maximum number of concurrent writes is reached, the object storage will be uploaded directly (this option is only valid when [\"Client write data cache\"](../guide/cache.md#client-write-cache) is enabled) (default value: 0, that is, no concurrency limit)|\n|`--max-deletes=10`|number of threads to delete objects (default: 10)|\n|`--upload-limit=0`|bandwidth limit for upload in Mbps (default: 0)|\n|`--download-limit=0`|bandwidth limit for download in Mbps (default: 0)|\n|`--check-storage`1.3|test storage before mounting to expose access issues early|\n\n#### Data cache related options {#mount-data-cache-options}\n\n|Items|Description|\n|-|-|\n|`--buffer-size=300`|total read/write buffering in MiB (default: 300), see [Read/Write buffer](../guide/cache.md#buffer-size)|\n|`--prefetch=1`|prefetch N blocks in parallel (default: 1), see [Client read data cache](../guide/cache.md#client-read-cache)|\n|`--writeback`|upload objects in background (default: false), see [Client write data cache](../guide/cache.md#client-write-cache)|\n|`--upload-delay=0`|When `--writeback` is enabled, you can use this option to add a delay to object storage upload, default to 0, meaning that upload will begin immediately after write. Different units are supported, including `s` (second), `m` (minute), `h` (hour). If files are deleted during this delay, upload will be skipped entirely, when using JuiceFS for temporary storage, use this option to reduce resource usage. Refer to [Client write data cache](../guide/cache.md#client-write-cache).|\n|`--upload-hours` 1.2|When `--writeback` is enabled, data blocks are only uploaded during the specified time of day. The format of the parameter is `,` (including \"start hour\", but not including \"end hour\", \"start hour\" must be less than or greater than \"end hour\"), where `` can range from 0 to 23. For example, `0,6` means that data blocks are only uploaded between 0:00 and 5:59 every day, and `23,3` means that data blocks are only uploaded between 23:00 every day and 2:59 the next day.|\n|`--cache-dir=value`|directory paths of local cache, use `:` (Linux, macOS) or `;` (Windows) to separate multiple paths (default: `$HOME/.juicefs/cache` or `/var/jfsCache`), see [Client read data cache](../guide/cache.md#client-read-cache)|\n|`--cache-mode value` 1.1 |file permissions for cached blocks (default: \"0600\")|\n|`--cache-size=102400`|size of cached object for read in MiB (default: 102400), see [Client read data cache](../guide/cache.md#client-read-cache)|\n|`--cache-items=0` 1.3 |max number of cached items (default is 0, which will be automatically calculated based on the `free‑space‑ratio`.)|\n|`--free-space-ratio=0.1`|min free space ratio (default: 0.1), if [Client write data cache](../guide/cache.md#client-write-cache) is enabled, this option also controls write cache size, see [Client read data cache](../guide/cache.md#client-read-cache)|\n|`--cache-partial-only`|cache random/small read only (default: false), see [Client read data cache](../guide/cache.md#client-read-cache)|\n|`--cache-large-write` 1.3|cache full blocks after uploading|\n|`--verify-cache-checksum=extend` 1.1 |Checksum level for cache data. After enabled, checksum will be calculated on divided parts of the cache blocks and stored on disks, which are used for verification during reads. The following strategies are supported:
  • `none`: Disable checksum verification, if local cache data is tampered, bad data will be read;
  • `full` (default before 1.3): Perform verification when reading the full block, use this for sequential read scenarios;
  • `shrink`: Perform verification on parts that's fully included within the read range, use this for random read scenarios;
  • `extend`: Perform verification on parts that fully include the read range, this causes read amplifications and is only used for random read scenarios demanding absolute data integrity. (default since 1.3)
|\n|`--cache-eviction=2-random` 1.1 |cache eviction policy (`none` or `2-random`) (default: \"2-random\")|\n|`--cache-scan-interval=1h` 1.1 |interval (in seconds) to scan cache-dir to rebuild in-memory index (default: \"1h\")|\n|`--cache-expire=0` 1.2|Cache blocks that have not been accessed for more than the set time, in seconds, will be automatically cleared (even if the value of `--cache-eviction` is `none`, these cache blocks will be deleted). A value of 0 means never expires (default: 0)|\n|`--max-readahead` 1.3|max buffering for read ahead in MiB|\n\n#### Metrics related options {#mount-metrics-options}\n\n||Items|Description|\n|-|-|\n|`--metrics=127.0.0.1:9567`|address to export metrics (default: `127.0.0.1:9567`)|\n|`--custom-labels`|custom labels for metrics, format: `key1:value1;key2:value2` (default: \"\")|\n|`--consul=127.0.0.1:8500`|Consul address to register (default: `127.0.0.1:8500`)|\n|`--no-usage-report`|do not send usage report (default: false)|\n\n#### Windows related options {#mount-windows-options}\n\n|Items|Description|\n|-|-|\n|`--o=`|Used to specify additional FUSE mount options. The actual supported options are determined by WinFsp.|\n|`--log=c:/juicefs.log` 1.3|Path to save JuiceFS logs (only effective when running in background mode).|\n|`-d` 1.3|Run in background mode. On Windows, enabling this will run JuiceFS as a system service. (Note: This requires administrator privileges and only one file system can be mounted at a time in this mode.)|\n|`--fuse-trace-log=c:/fuse.log` 1.3|Specifies the trace log path for WinFsp's FUSE layer callbacks. (Default: \"\")|\n|`--as-root`|A compatibility option that maps all file uid, gid, and write operations to the root user (uid=0).|\n|`--show-dot-files` 1.3|Show files that begin with a dot (.). By default, such files are treated as hidden.|\n|`--winfsp-threads=16` 1.3|Sets the number of threads WinFsp uses to handle kernel events. The default is min(CPU cores * 2, 16).|\n|`--report-case` 1.3|Configures whether JuiceFS should report the precise case of filenames when possible. For example, when opening aaa.txt that actually exists as AAA.txt, enabling this option allows JuiceFS to report the original case to the Windows kernel. (Note: Enabling this may affect performance.)|\n" }, { "path": "docs/en/reference/command_reference.mdx", "content": "---\ntitle: Command Reference\nsidebar_position: 1\nslug: /command_reference\ndescription: Descriptions, usage and examples of all commands and options included in JuiceFS Client.\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n\nimport CommonOptions from './_common_options.mdx';\n\nRunning `juicefs` by itself and it will print all available commands. In addition, you can add `-h/--help` flag after each command to get more information, e.g., `juicefs format -h`.\n\n```\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 1.2.0\n\nCOMMANDS:\n ADMIN:\n format Format a volume\n config Change configuration of a volume\n quota Manage directory quotas\n destroy Destroy an existing volume\n gc Garbage collector of objects in data storage\n fsck Check consistency of a volume\n restore restore files from trash\n dump Dump metadata into a JSON file\n load Load metadata from a previously dumped JSON file\n version Show version\n INSPECTOR:\n status Show status of a volume\n stats Show real time performance statistics of JuiceFS\n profile Show profiling of operations completed in JuiceFS\n info Show internal information of a path or inode\n debug Collect and display system static and runtime information\n summary Show tree summary of a directory\n SERVICE:\n mount Mount a volume\n umount Unmount a volume\n gateway Start an S3-compatible gateway\n webdav Start a WebDAV server\n TOOL:\n bench Run benchmarks on a path\n objbench Run benchmarks on an object storage\n warmup Build cache for target directories/files\n rmr Remove directories recursively\n sync Sync between two storages\n clone clone a file or directory without copying the underlying data\n compact Trigger compaction of chunks\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q show warning and errors only (default: false)\n --trace enable trace log (default: false)\n --log-id value append the given log id in log, use \"random\" to use random uuid\n --no-agent disable pprof (:6060) agent (default: false)\n --pyroscope value pyroscope address\n --no-color disable colors (default: false)\n --help, -h show help (default: false)\n --version, -V print version only (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\n## Auto completion {#auto-completion}\n\nTo enable commands completion, simply source the script provided within [`hack/autocomplete`](https://github.com/juicedata/juicefs/tree/main/hack/autocomplete) directory. For example:\n\n\n \n\n```shell\nsource hack/autocomplete/bash_autocomplete\n```\n\n \n \n\n```shell\nsource hack/autocomplete/zsh_autocomplete\n```\n\n \n\n\nPlease note the auto-completion is only enabled for the current session. If you want to apply it for all new sessions, add the `source` command to `.bashrc` or `.zshrc`:\n\n\n \n\n```shell\necho \"source path/to/bash_autocomplete\" >> ~/.bashrc\n```\n\n \n \n\n```shell\necho \"source path/to/zsh_autocomplete\" >> ~/.zshrc\n```\n\n \n\n\nAlternatively, if you are using bash on a Linux system, you may just copy the script to `/etc/bash_completion.d` and rename it to `juicefs`:\n\n```shell\ncp hack/autocomplete/bash_autocomplete /etc/bash_completion.d/juicefs\nsource /etc/bash_completion.d/juicefs\n```\n\n## Global options {#global-options}\n\n|Items|Description|\n|-|-|\n|`-v` `--verbose` `--debug`|Enable debug logs.|\n|`-q` `--quiet`|Show only warning and error logs.|\n|`--trace`|Enable more detailed debug logs than the `--debug` option.|\n|`--no-agent`|Disable pprof agent.|\n|`--pyroscope`|Config [Pyroscope](https://github.com/pyroscope-io/pyroscope) address, e.g. `http://localhost:4040`.|\n|`--no-color`|Disable log color.|\n\n\n## Admin {#admin}\n\n### `juicefs format` {#format}\n\nCreate and format a file system, if a volume already exists with the same `META-URL`, this command will skip the format step. To adjust configurations for existing volumes, use [`juicefs config`](#config).\n\n#### Synopsis\n\n```shell\njuicefs format [command options] META-URL NAME\n\n# Create a simple test volume (data will be stored in a local directory)\njuicefs format sqlite3://myjfs.db myjfs\n\n# Create a volume with Redis and S3\njuicefs format redis://localhost myjfs --storage=s3 --bucket=https://mybucket.s3.us-east-2.amazonaws.com\n\n# Create a volume with password protected MySQL\njuicefs format mysql://jfs:mypassword@(127.0.0.1:3306)/juicefs myjfs\n# A safer alternative\nMETA_PASSWORD=mypassword juicefs format mysql://jfs:@(127.0.0.1:3306)/juicefs myjfs\n# Provide password from file\nMETA_PASSWORD_FILE=/secret/mypassword.txt juicefs format mysql://jfs:@(127.0.0.1:3306)/juicefs myjfs\n\n# Create a volume with quota enabled\njuicefs format sqlite3://myjfs.db myjfs --inodes=1000000 --capacity=102400\n\n# Create a volume with trash disabled\njuicefs format sqlite3://myjfs.db myjfs --trash-days=0\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`META-URL`|Database URL for metadata storage, see [JuiceFS supported metadata engines](../reference/how_to_set_up_metadata_engine.md) for details.|\n|`NAME`|Name of the file system|\n|`--force`|overwrite existing format (default: false)|\n|`--no-update`|don't update existing volume (default: false)|\n\n#### Data storage options {#format-data-storage-options}\n\n|Items|Description|\n|-|-|\n|`--storage=file`|Object storage type (e.g. `s3`, `gs`, `oss`, `cos`) (default: `file`, refer to [documentation](../reference/how_to_set_up_object_storage.md#supported-object-storage) for all supported object storage types)|\n|`--bucket=/var/jfs`|A bucket URL to store data (default: `$HOME/.juicefs/local` or `/var/jfs`)|\n|`--access-key=value`|Access Key for object storage (can also be set via the environment variable `ACCESS_KEY`), see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#aksk) for more.|\n|`--secret-key value`|Secret Key for object storage (can also be set via the environment variable `SECRET_KEY`), see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#aksk) for more.|\n|`--session-token=value`|session token for object storage, see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#session-token) for more.|\n|`--storage-class value` 1.1 |the default storage class|\n\n#### Data format options {#format-data-format-options}\n\n|Items|Description|\n|-|-|\n|`--block-size=4M`|size of block in KiB (default: 4M). 4M is usually a better default value because many object storage services use 4M as their internal block size, thus using the same block size in JuiceFS usually yields better performance.|\n|`--compress=none`|compression algorithm, choose from `lz4`, `zstd`, `none` (default). Enabling compression will inevitably affect performance. Among the two supported algorithms, `lz4` offers a better performance, while `zstd` comes with a higher compression ratio, Google for their detailed comparison.|\n|`--encrypt-rsa-key=value`|A path to RSA private key (PEM)|\n|`--encrypt-algo=aes256gcm-rsa`|encrypt algorithm (aes256gcm-rsa, chacha20-rsa) (default: \"aes256gcm-rsa\")|\n|`--hash-prefix`|For most object storages, if object storage blocks are sequentially named, they will also be closely stored in the underlying physical regions. When loaded with intensive concurrent consecutive reads, this can cause hotspots and hinder object storage performance.

Enabling `--hash-prefix` will add a hash prefix to name of the blocks (slice ID mod 256, see [internal implementation](../development/internals.md#object-storage-naming-format)), this distributes data blocks evenly across actual object storage regions, offering more consistent performance. Obviously, this option dictates object naming pattern and **should be specified when a file system is created, and cannot be changed on-the-fly.**

Currently, [AWS S3](https://aws.amazon.com/about-aws/whats-new/2018/07/amazon-s3-announces-increased-request-rate-performance) had already made improvements and no longer require application side optimization, but for other types of object storages, this option still recommended for large scale scenarios.|\n|`--shards=0`|If your object storage limit speed in a bucket level (or you're using a self-hosted object storage with limited performance), you can store the blocks into N buckets by hash of key (default: 0), when N is greater than 0, `bucket` should to be in the form of `%d`, e.g. `--bucket \"juicefs-%d\"`. `--shards` cannot be changed afterwards and must be planned carefully ahead.|\n\n#### Management options {#format-management-options}\n\n|Items|Description|\n|-|-|\n|`--capacity=0`|storage space limit in GiB, default to 0 which means no limit. Capacity will include trash files, if [trash](../security/trash.md) is enabled.|\n|`--inodes=0`|Limit the number of inodes, default to 0 which means no limit.|\n|`--trash-days=1`|By default, delete files are put into [trash](../security/trash.md), this option controls the number of days before trash files are expired, default to 1, set to 0 to disable trash.|\n|`--enable-acl=true` 1.2|enable [POSIX ACL](../security/posix_acl.md)，it is irreversible. |\n\n### `juicefs config` {#config}\n\nChange config of a volume. Note that after updating some settings, the client may not take effect immediately, and it needs to wait for a certain period of time. The specific waiting time can be controlled by the [`--heartbeat`](#mount-metadata-options) option.\n\n#### Synopsis\n\n```shell\njuicefs config [command options] META-URL\n\n# Show the current configurations\njuicefs config redis://localhost\n\n# Change volume \"quota\"\njuicefs config redis://localhost --inodes 10000000 --capacity 1048576\n\n# Change maximum days before files in trash are deleted\njuicefs config redis://localhost --trash-days 7\n\n# Limit client version that is allowed to connect\njuicefs config redis://localhost --min-client-version 1.0.0 --max-client-version 1.1.0\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--yes, -y`|automatically answer 'yes' to all prompts and run non-interactively (default: false)|\n|`--force`|skip sanity check and force update the configurations (default: false)|\n\n#### Data storage options {#config-data-storage-options}\n\n|Items|Description|\n|-|-|\n|`--storage=file` 1.1 |Object storage type (e.g. `s3`, `gs`, `oss`, `cos`) (default: `\"file\"`, refer to [documentation](../reference/how_to_set_up_object_storage.md#supported-object-storage) for all supported object storage types).|\n|`--bucket=/var/jfs`|A bucket URL to store data (default: `$HOME/.juicefs/local` or `/var/jfs`)|\n|`--access-key=value`|Access Key for object storage (can also be set via the environment variable `ACCESS_KEY`), see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#aksk) for more.|\n|`--secret-key value`|Secret Key for object storage (can also be set via the environment variable `SECRET_KEY`), see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#aksk) for more.|\n|`--session-token=value`|session token for object storage, see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#session-token) for more.|\n|`--storage-class value` 1.1 |the default storage class|\n|`--upload-limit=0`|bandwidth limit for upload in Mbps (default: 0)|\n|`--download-limit=0`|bandwidth limit for download in Mbps (default: 0)|\n\n#### Management options {#config-management-options}\n\n|Items|Description|\n|-|-|\n|`--capacity value`|limit for space in GiB|\n|`--inodes value`|limit for number of inodes|\n|`--trash-days value`|number of days after which removed files will be permanently deleted|\n|`--enable-acl` 1.2|enable [POSIX ACL](../security/posix_acl.md) (irreversible), at the same time, the minimum client version allowed to connect will be upgraded to v1.2|\n|`--encrypt-secret`|encrypt the secret key if it was previously stored in plain format (default: false)|\n|`--min-client-version value` 1.1 |minimum client version allowed to connect|\n|`--max-client-version value` 1.1 |maximum client version allowed to connect|\n|`--dir-stats` 1.1 |enable dir stats, which is necessary for fast summary and dir quota (default: false)|\n\n### `juicefs quota` 1.1 {#quota}\n\nManage directory quotas\n\n#### Synopsis\n\n```shell\njuicefs quota command [command options] META-URL\n\n# Set quota to a directory\njuicefs quota set redis://localhost --path /dir1 --capacity 1 --inodes 100\n\n# Get quota of a directory\njuicefs quota get redis://localhost --path /dir1\n\n# List all directory quotas\njuicefs quota list redis://localhost\n\n# Delete quota of a directory\njuicefs quota delete redis://localhost --path /dir1\n\n# Check quota consistency of a directory\njuicefs quota check redis://localhost\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`META-URL`|Database URL for metadata storage, see \"[JuiceFS supported metadata engines](../reference/how_to_set_up_metadata_engine.md)\" for details.|\n|`--path value`|full path of the directory within the volume|\n|`--capacity value`|hard quota of the directory limiting its usage of space in GiB (default: 0)|\n|`--inodes value`|hard quota of the directory limiting its number of inodes (default: 0)|\n|`--repair`|repair inconsistent quota (default: false)|\n|`--strict`|calculate total usage of directory in strict mode (NOTE: may be slow for huge directory) (default: false)|\n\n### `juicefs destroy` {#destroy}\n\nDestroy an existing volume, will delete relevant data in metadata engine and object storage. See [How to destroy a file system](../administration/destroy.md).\n\n#### Synopsis\n\n```shell\njuicefs destroy [command options] META-URL UUID\n\njuicefs destroy redis://localhost e94d66a8-2339-4abd-b8d8-6812df737892\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--yes, -y` 1.1 |automatically answer 'yes' to all prompts and run non-interactively (default: false)|\n|`--force`|skip sanity check and force destroy the volume (default: false)|\n\n### `juicefs gc` {#gc}\n\nIf for some reason, a object storage block escape JuiceFS management completely, i.e. the metadata is gone, but the block still persists in the object storage, and cannot be released, this is called an \"object leak\". If this happens without any special file system manipulation, it could well indicate a bug within JuiceFS, file a [GitHub Issue](https://github.com/juicedata/juicefs/issues/new/choose) to let us know.\n\nMeanwhile, you can run this command to deal with leaked objects. It also deletes stale slices produced by file overwrites. See [Status Check & Maintenance](../administration/status_check_and_maintenance.md#gc).\n\n#### Synopsis\n\n```shell\njuicefs gc [command options] META-URL\n\n# Check only, no writable change\njuicefs gc redis://localhost\n\n# Trigger compaction of all slices\njuicefs gc redis://localhost --compact\n\n# Delete leaked objects\njuicefs gc redis://localhost --delete\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--compact`|compact all chunks with more than 1 slices (default: false).|\n|`--delete`|delete leaked objects (default: false)|\n|`--threads=10`|number of threads to delete leaked objects (default: 10)|\n\n### `juicefs fsck` {#fsck}\n\nCheck consistency of file system.\n\n#### Synopsis\n\n```shell\njuicefs fsck [command options] META-URL\n\njuicefs fsck redis://localhost\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--path value` 1.1 |absolute path within JuiceFS to check|\n|`--repair` 1.1 |repair specified path if it's broken (default: false)|\n|`--recursive, -r` 1.1 |recursively check or repair (default: false)|\n|`--sync-dir-stat` 1.1 |sync stat of all directories, even if they are existed and not broken (NOTE: it may take a long time for huge trees) (default: false)|\n\n### `juicefs restore` 1.1 {#restore}\n\nRebuild the tree structure for trash files, and put them back to original directories.\n\n#### Synopsis\n\n```shell\njuicefs restore [command options] META HOUR ...\n\njuicefs restore redis://localhost/1 2023-05-10-01\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--put-back value`|move the recovered files into original directory (default: false)|\n|`--threads value`|number of threads (default: 10)|\n\n### `juicefs dump` {#dump}\n\nDump metadata into a JSON file. Refer to [\"Metadata backup\"](../administration/metadata_dump_load.md#backup) for more information.\n\n#### Synopsis\n\n```shell\njuicefs dump [command options] META-URL [FILE]\n\n# Export metadata to meta-dump.json\njuicefs dump redis://localhost meta-dump.json\n\n# Export metadata for only one subdirectory of the file system\njuicefs dump redis://localhost sub-meta-dump.json --subdir /dir/in/jfs\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`META-URL`|Database URL for metadata storage, see [JuiceFS supported metadata engines](../reference/how_to_set_up_metadata_engine.md) for details.|\n|`FILE`|Export file path, if not specified, it will be exported to standard output. If the filename ends with `.gz`, it will be automatically compressed.|\n|`--subdir=path`|Only export metadata for the specified subdirectory.|\n|`--keep-secret-key` 1.1 |Export object storage authentication information, the default is `false`. Since it is exported in plain text, pay attention to data security when using it. If the export file does not contain object storage authentication information, you need to use [`juicefs config`](#config) to reconfigure object storage authentication information after the subsequent import is completed.|\n|`--threads=10` 1.2|number of threads to dump metadata. (default: 10)|\n|`--fast` 1.2|Use more memory to speedup dump.|\n|`--skip-trash` 1.2|Skip files and directories in trash.|\n\n### `juicefs load` {#load}\n\nLoad metadata from a previously dumped JSON file. Read [\"Metadata recovery and migration\"](../administration/metadata_dump_load.md#recovery-and-migration) to learn more.\n\n#### Synopsis\n\n```shell\njuicefs load [command options] META-URL [FILE]\n\n# Import the metadata backup file meta-dump.json to the database\njuicefs load redis://127.0.0.1:6379/1 meta-dump.json\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`META-URL`|Database URL for metadata storage, see [JuiceFS supported metadata engines](../reference/how_to_set_up_metadata_engine.md) for details.|\n|`FILE`|Import file path, if not specified, it will be imported from standard input. If the filename ends with `.gz`, it will be automatically decompressed.|\n|`--encrypt-rsa-key=path` 1.0.4 |The path to the RSA private key file used for encryption.|\n|`--encrypt-alg=aes256gcm-rsa` 1.0.4 |Encryption algorithm, the default is `aes256gcm-rsa`.|\n\n## Inspector {#inspector}\n\n### `juicefs status` {#status}\n\nShow status of JuiceFS.\n\n#### Synopsis\n\n```shell\njuicefs status [command options] META-URL\n\njuicefs status redis://localhost\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--session=0, -s 0`|show detailed information (sustained inodes, locks) of the specified session (SID) (default: 0)|\n|`--more, -m` 1.1 |show more statistic information, may take a long time (default: false)|\n\n### `juicefs stats` {#stats}\n\nShow runtime statistics, read [Real-time performance monitoring](../administration/fault_diagnosis_and_analysis.md#performance-monitor) for more.\n\n#### Synopsis\n\n```shell\njuicefs stats [command options] MOUNTPOINT\n\njuicefs stats /mnt/jfs\n\n# More metrics\njuicefs stats /mnt/jfs -l 1\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--schema=ufmco`|schema string that controls the output sections (`u`: usage, `f`: FUSE, `m`: metadata, `c`: block cache, `o`: object storage, `g`: Go) (default: `ufmco`)|\n|`--interval=1`|interval in seconds between each update (default: 1)|\n|`--verbosity=0`|verbosity level, 0 or 1 is enough for most cases (default: 0)|\n\n### `juicefs profile` {#profile}\n\nShow profiling of operations completed in JuiceFS, based on [access log](../administration/fault_diagnosis_and_analysis.md#access-log). read [Real-time performance monitoring](../administration/fault_diagnosis_and_analysis.md#performance-monitor) for more.\n\n#### Synopsis\n\n```shell\njuicefs profile [command options] MOUNTPOINT/LOGFILE\n\n# Monitor real time operations\njuicefs profile /mnt/jfs\n\n# Replay an access log\ncat /mnt/jfs/.accesslog > /tmp/jfs.alog\n# Press Ctrl-C to stop the \"cat\" command after some time\njuicefs profile /tmp/jfs.alog\n\n# Analyze an access log and print the total statistics immediately\njuicefs profile /tmp/jfs.alog --interval 0\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--uid=value, -u value`|only track specified UIDs (separated by comma)|\n|`--gid=value, -g value`|only track specified GIDs (separated by comma)|\n|`--pid=value, -p value`|only track specified PIDs (separated by comma)|\n|`--interval=2`|flush interval in seconds; set it to 0 when replaying a log file to get an immediate result (default: 2)|\n\n### `juicefs info` {#info}\n\nShow internal information for given paths or inodes.\n\n#### Synopsis\n\n```shell\njuicefs info [command options] PATH or INODE\n\n# Check a path\njuicefs info /mnt/jfs/foo\n\n# Check an inode\ncd /mnt/jfs\njuicefs info -i 100\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--inode, -i`|use inode instead of path (current dir should be inside JuiceFS) (default: false)|\n|`--recursive, -r`|get summary of directories recursively (NOTE: it may take a long time for huge trees) (default: false)|\n|`--strict` 1.1 |get accurate summary of directories (NOTE: it may take a long time for huge trees) (default: false)|\n|`--raw`|show internal raw information (default: false)|\n\n### `juicefs debug` 1.1 {#debug}\n\nIt collects and displays information from multiple dimensions such as the operating environment and system logs to help better locate errors\n\n#### Synopsis\n\n```shell\njuicefs debug [command options] MOUNTPOINT\n\n# Collect and display information about the mount point /mnt/jfs\njuicefs debug /mnt/jfs\n\n# Specify the output directory as /var/log\njuicefs debug --out-dir=/var/log /mnt/jfs\n\n# Get the last up to 1000 log entries\njuicefs debug --out-dir=/var/log --limit=1000 /mnt/jfs\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--out-dir=./debug/`|The output directory of the results, automatically created if the directory does not exist (default: `./debug/`)|\n|`--limit=value`|The number of log entries collected, from newest to oldest, if not specified, all entries will be collected|\n|`--stats-sec=5`|The number of seconds to sample .stats file (default: 5)|\n|`--trace-sec=5`|The number of seconds to sample trace metrics (default: 5)|\n|`--profile-sec=30`|The number of seconds to sample profile metrics (default: 30)|\n\n### `juicefs summary` 1.1 {#summary}\n\nIt is used to show tree summary of target directory.\n\n#### Synopsis\n\n```shell\njuicefs summary [command options] PATH\n\n# Show with path\njuicefs summary /mnt/jfs/foo\n\n# Show max depth of 5\njuicefs summary --depth 5 /mnt/jfs/foo\n\n# Show top 20 entries\njuicefs summary --entries 20 /mnt/jfs/foo\n\n# Show accurate result\njuicefs summary --strict /mnt/jfs/foo\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--depth value, -d value`|depth of tree to show (zero means only show root) (default: 2)|\n|`--entries value, -e value`|show top N entries (sort by size) (default: 10)|\n|`--strict`|show accurate summary, including directories and files (may be slow) (default: false)|\n|`--csv`|print summary in csv format (default: false)|\n\n## Service {#service}\n\n### `juicefs mount` {#mount}\n\nMount a volume. The volume must be formatted in advance.\n\nJuiceFS can be mounted by root or normal user, but due to their privilege differences, cache directory and log path will vary, read below descriptions for more.\n\n#### Synopsis\n\n```shell\njuicefs mount [command options] META-URL MOUNTPOINT\n\n# Mount in foreground\njuicefs mount redis://localhost /mnt/jfs\n\n# Mount in background with password protected Redis\njuicefs mount redis://:mypassword@localhost /mnt/jfs -d\n# A safer alternative\nMETA_PASSWORD=mypassword juicefs mount redis://localhost /mnt/jfs -d\n\n# Mount with a sub-directory as root\njuicefs mount redis://localhost /mnt/jfs --subdir /dir/in/jfs\n\n# Enable \"writeback\" mode, which improves performance at the risk of losing objects\njuicefs mount redis://localhost /mnt/jfs -d --writeback\n\n# Enable \"read-only\" mode\njuicefs mount redis://localhost /mnt/jfs -d --read-only\n\n# Disable metadata backup\njuicefs mount redis://localhost /mnt/jfs --backup-meta 0\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`META-URL`|Database URL for metadata storage, see [JuiceFS supported metadata engines](../reference/how_to_set_up_metadata_engine.md) for details.|\n|`MOUNTPOINT`|file system mount point, e.g. `/mnt/jfs`, `Z:`.|\n|`-d, --background`|run in background (default: false)|\n|`--no-syslog`|disable syslog (default: false)|\n|`--log=path`|path of log file when running in background (default: `$HOME/.juicefs/juicefs.log` or `/var/log/juicefs.log`)|\n|`--force`|force to mount even if the mount point is already mounted by the same filesystem.|\n|`--update-fstab` 1.1 |add / update entry in `/etc/fstab`, will create a symlink from `/sbin/mount.juicefs` to JuiceFS executable if not existing (default: false)|\n|`--disable-transparent-hugepage` 1.3 |Disable the kernel’s Transparent Huge Page (THP). In situations like memory pressure, keeping THP enabled may cause processes to hang. (default: false)|\n\n#### FUSE related options {#mount-fuse-options}\n\n|Items|Description|\n|-|-|\n|`--enable-xattr`|enable extended attributes (xattr) (default: false)|\n|`--enable-cap` 1.3|enable security.capability xattr (default: false)|\n|`--enable-selinux` 1.3|enable security.selinux xattr (default: false)|\n|`--enable-ioctl` 1.1 |enable ioctl (support GETFLAGS/SETFLAGS only) (default: false)|\n|`--root-squash value` 1.1 |mapping local root user (UID = 0) to another one specified as UID:GID|\n|`--all-squash value` 1.3 |mapping all users to another one specified as UID:GID|\n|`--umask value` 1.3 |umask for new file and directory in octal|\n|`--prefix-internal` 1.1 |add '.jfs' prefix to all internal files (default: false)|\n|`--max-fuse-io=128K` 1.3|maximum size for fuse request (default: 128K)|\n|`-o value`|other FUSE options, see [FUSE Mount Options](../reference/fuse_mount_options.md)|\n\n\n\n\n
\n\n#### {#mount-metadata-options}\n#### {#mount-metadata-cache-options}\n#### {#mount-data-storage-options}\n#### {#mount-data-cache-options}\n#### {#mount-metrics-options}\n\n
\n\n### `juicefs umount` {#umount}\n\nUnmount a volume.\n\n#### Synopsis\n\n```shell\njuicefs umount [command options] MOUNTPOINT\n\njuicefs umount /mnt/jfs\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`-f, --force`|force unmount a busy mount point (default: false)|\n|`--flush` 1.1 |wait for all staging chunks to be flushed (default: false)|\n\n### `juicefs gateway` {#gateway}\n\nStart an S3-compatible gateway, read [Deploy JuiceFS S3 Gateway](../guide/gateway.md) for more.\n\n#### Synopsis\n\n```shell\njuicefs gateway [command options] META-URL ADDRESS\n\nexport MINIO_ROOT_USER=admin\nexport MINIO_ROOT_PASSWORD=12345678\njuicefs gateway redis://localhost localhost:9000\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`META-URL`|Database URL for metadata storage, see [JuiceFS supported metadata engines](../reference/how_to_set_up_metadata_engine.md) for details.|\n|`ADDRESS`|S3 gateway address and listening port, for example: `localhost:9000`|\n|`--log value` 1.2|path for gateway log|\n|`--access-log=path`|path for JuiceFS access log.|\n|`--background, -d` 1.2|run in background (default: false)|\n|`--no-banner`|disable MinIO startup information (default: false)|\n|`--multi-buckets`|use top level of directories as buckets (default: false)|\n|`--keep-etag`|save the ETag for uploaded objects (default: false)|\n|`--umask=022`|umask for new file and directory in octal (default: 022)|\n|`--object-tag` 1.2|enable object tagging API|\n|`--domain value` 1.2|domain for virtual-host-style requests|\n|`--refresh-iam-interval=5m` 1.2|interval to reload gateway IAM from configuration (default: 5m)|\n\n\n\n### `juicefs webdav` {#webdav}\n\nStart a WebDAV server, refer to [Deploy WebDAV Server](../deployment/webdav.md) for more.\n\n#### Synopsis\n\n```shell\njuicefs webdav [command options] META-URL ADDRESS\n\njuicefs webdav redis://localhost localhost:9007\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`META-URL`|Database URL for metadata storage, see [JuiceFS supported metadata engines](../reference/how_to_set_up_metadata_engine.md) for details.|\n|`ADDRESS`|WebDAV address and listening port, for example: `localhost:9007`.|\n|`--cert-file` 1.1|certificate file for HTTPS|\n|`--key-file` 1.1|key file for HTTPS|\n|`--gzip`|compress served files via gzip (default: false)|\n|`--disallowList`|disallow list a directory (default: false)|\n|`--enable-proppatch` 1.3|enable proppatch method support|\n|`--log value` 1.2|path for WebDAV log|\n|`--access-log=path`|path for JuiceFS access log|\n|`--background, -d` 1.2|run in background (default: false)|\n|`--threads=50, -p 50` 1.3|number of threads for delete jobs (max 255)|\n\n\n\n## Tool {#tool}\n\n### `juicefs bench` {#bench}\n\nRun benchmark, including read/write/stat for big and small files.\nFor a detailed introduction to the `bench` subcommand, refer to the [documentation](../benchmark/performance_evaluation_guide.md#juicefs-bench).\n\n#### Synopsis\n\n```shell\njuicefs bench [command options] PATH\n\n# Run benchmarks with 4 threads\njuicefs bench /mnt/jfs -p 4\n\n# Run benchmarks of only small files\njuicefs bench /mnt/jfs --big-file-size 0\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--block-size=1`|block size in MiB (default: 1)|\n|`--big-file-size=1024`|size of big file in MiB (default: 1024)|\n|`--small-file-size=128`|size of small file in KiB (default: 128)|\n|`--small-file-count=100`|number of small files (default: 100)|\n|`--threads=1, -p 1`|number of concurrent threads (default: 1)|\n\n### `juicefs objbench` {#objbench}\n\nRun basic benchmarks on the target object storage to test if it works as expected. Read [documentation](../benchmark/performance_evaluation_guide.md#juicefs-objbench) for more.\n\n#### Synopsis\n\n```shell\njuicefs objbench [command options] BUCKET\n\n# Run benchmarks on S3\nACCESS_KEY=myAccessKey SECRET_KEY=mySecretKey juicefs objbench --storage=s3 https://mybucket.s3.us-east-2.amazonaws.com -p 6\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--storage=file`|Object storage type (e.g. `s3`, `gs`, `oss`, `cos`) (default: `file`, refer to [documentation](../reference/how_to_set_up_object_storage.md#supported-object-storage) for all supported object storage types)|\n|`--access-key=value`|Access Key for object storage (can also be set via the environment variable `ACCESS_KEY`), see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#aksk) for more.|\n|`--secret-key value`|Secret Key for object storage (can also be set via the environment variable `SECRET_KEY`), see [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md#aksk) for more.|\n|`--session-token value` 1.0|session token for object storage|\n|`--shards`1.3|If your object storage limit speed in a bucket level (or you're using a self-hosted object storage with limited performance), you can store the blocks into N buckets by hash of key (default: 0), when N is greater than 0, `bucket` should to be in the form of `%d`, e.g. `--bucket \"juicefs-%d\"`. `--shards` cannot be changed afterwards and must be planned carefully ahead.|\n|`--block-size=4096`|size of each IO block in KiB (default: 4096)|\n|`--big-object-size=1024`|size of each big object in MiB (default: 1024)|\n|`--small-object-size=128`|size of each small object in KiB (default: 128)|\n|`--small-objects=100`|number of small objects (default: 100)|\n|`--skip-functional-tests`|skip functional tests (default: false)|\n|`--threads=4, -p 4`|number of concurrent threads (default: 4)|\n\n### `juicefs warmup` {#warmup}\n\nDownload data to local cache in advance, to achieve better performance on application's first read. You can specify a mount point path to recursively warm-up all files under this path. You can also specify a file through the `--file` option to only warm-up the files contained in it.\n\nIf the files needing warming up resides in many different directories, you should specify their names in a text file, and pass to the `warmup` command using the `--file` option, allowing `juicefs warmup` to download concurrently, which is significantly faster than calling `juicefs warmup` multiple times, each with a single file.\n\n#### Synopsis\n\n```shell\njuicefs warmup [command options] [PATH ...]\n\n# Warm up all files in datadir\njuicefs warmup /mnt/jfs/datadir\n\n# Warm up selected files\necho '/jfs/f1\n/jfs/f2\n/jfs/f3' > /tmp/filelist.txt\njuicefs warmup -f /tmp/filelist.txt\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--file=path, -f path`|file containing a list of paths (each line is a file path)|\n|`--threads=50, -p 50`|number of concurrent workers, default to 50. Reduce this number in low bandwidth environment to avoid download timeouts|\n|`--background, -b`|run in background (default: false)|\n|`--evict` 1.2|evict cached blocks|\n|`--check` 1.2|check whether the data blocks are cached or not|\n\n### `juicefs rmr` {#rmr}\n\nRemove all the files and subdirectories, similar to `rm -rf`, except this command deals with metadata directly (bypassing kernel), thus is much faster.\n\nIf trash is enabled, deleted files are moved into trash. Read more at [Trash](../security/trash.md).\n\n#### Synopsis\n\n```shell\njuicefs rmr PATH ...\n\njuicefs rmr /mnt/jfs/foo\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--skip-trash`1.3|skip trash and delete files directly (requires root)|\n|`--threads=50, -p 50`1.3|number of threads for delete jobs (max 255)|\n\n### `juicefs sync` {#sync}\n\nSync between two storage, read [Data migration](../guide/sync.md) for more.\n\n#### Synopsis\n\n```shell\njuicefs sync [command options] SRC DST\n\n# Sync object from OSS to S3\njuicefs sync oss://mybucket.oss-cn-shanghai.aliyuncs.com s3://mybucket.s3.us-east-2.amazonaws.com\n\n# Sync objects from S3 to JuiceFS\njuicefs sync s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n\n# SRC: a1/b1,a2/b2,aaa/b1 DST: empty sync result: aaa/b1\njuicefs sync --exclude='a?/b*' s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n\n# SRC: a1/b1,a2/b2,aaa/b1 DST: empty sync result: a1/b1,aaa/b1\njuicefs sync --include='a1/b1' --exclude='a[1-9]/b*' s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n\n# SRC: a1/b1,a2/b2,aaa/b1,b1,b2 DST: empty sync result: b2\njuicefs sync --include='a1/b1' --exclude='a*' --include='b2' --exclude='b?' s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n```\n\nAs shown in the examples, the format of both source (`SRC`) and destination (`DST`) paths is:\n\n```\n[NAME://][ACCESS_KEY:SECRET_KEY[:TOKEN]@]BUCKET[.ENDPOINT][/PREFIX]\n```\n\nIn which:\n\n- `NAME`: JuiceFS supported data storage types like `s3`, `oss`, refer to [this document](../reference/how_to_set_up_object_storage.md#supported-object-storage) for a full list.\n- `ACCESS_KEY` and `SECRET_KEY`: The credential required to access the data storage, special characters need to be [URL encoded](https://www.w3schools.com/tags/ref_urlencode.ASP), e.g. `/` must be substituted with `%2F`. If you are not familiar with AKSK management, refer to [this document](../reference/how_to_set_up_object_storage.md#aksk).\n- `TOKEN` token used to access the object storage, as some object storage supports the use of temporary token to obtain permission for a limited time.\n- `BUCKET[.ENDPOINT]`: The access address of the data storage service. The format may be different for different storage types, and refer to [the document](../reference/how_to_set_up_object_storage.md#supported-object-storage).\n- `[/PREFIX]`: Optional, a prefix for the source and destination paths that can be used to limit synchronization of data only in certain paths.\n\n#### Selection related options {#sync-selection-related-options}\n\n|Items|Description|\n|-|-|\n|`--files-from` 1.3|Only synchronize the objects recorded in the given file, where each line is the relative path of the object to be synchronized. If the object is a directory, it is recommended to end with `/`.|\n|`--start=KEY, -s KEY, --end=KEY, -e KEY`|Provide object storage key range for syncing.|\n|`--end KEY, -e KEY`|the last `KEY` to sync|\n|`--exclude=PATTERN`|Exclude keys matching `PATTERN`. Refer to the [\"Filtering\"](../guide/sync.md#filtering) document to learn how to use it.|\n|`--include=PATTERN`|Include keys matching `PATTERN`, need to be used with `--exclude`. Refer to the [\"Filtering\"](../guide/sync.md#filtering) document to learn how to use it.|\n|`--match-full-path` 1.2|Use \"Full path filtering mode\", default is false. Refer to the [\"Filtering modes\"](../guide/sync.md#filtering-mode) document to learn how to use it.|\n|`--max-size-SIZE` 1.2|skip files larger than `SIZE`|\n|`--min-size-SIZE` 1.2|skip files smaller than `SIZE`|\n|`--max-age=DURATION` 1.2|Skip files whose last modification time exceeds `DURATION`, in seconds. For example, `--max-age=3600` means to synchronize only files that have been modified within 1 hour.|\n|`--min-age=DURATION` 1.2|Skip files whose last modification time is no more than `DURATION`, in seconds. For example, `--min-age=3600` means to synchronize only files whose last modification time is more than 1 hour from the current time.|\n|`--start-time` 1.3|skip files modified before start-time. example: 2006-01-02 15:04:05|\n|`--end-time` 1.3|skip files modified after end-time. example: 2006-01-02 15:04:05|\n|`--limit=-1`|Limit the number of objects that will be processed, default to -1 which means unlimited.|\n|`--update, -u`|Update existing files if the source files' `mtime` is newer, default to false.|\n|`--force-update, -f`|Always update existing file, default to false.|\n|`--existing, --ignore-non-existing` 1.1 |Skip creating new files on destination, default to false.|\n|`--ignore-existing` 1.1 |Skip updating files that already exist on destination, default to false.|\n\n#### Action related options {#sync-action-related-options}\n\n|Items|Description|\n|-|-|\n|`--dirs`|Sync empty directories as well.|\n|`--perms`|Preserve permissions, default to false.|\n|`--links, -l`|Copy symlinks as symlinks default to false.|\n|`--inplace` 1.2|When a file in the source path is modified, directly modify the file with the same name in the destination path instead of first writing a temporary file in the destination path and then atomically renaming the temporary file to the real file name. This option only makes sense when the `--update` option is enabled and the storage system of the destination path supports in-place modification of files (such as JuiceFS, HDFS, NFS). That is to say, if the storage system of the destination path is object storage, enable this option is invalid. (default: false)|\n|`--delete-src, --deleteSrc`|Delete objects that already exist in destination. Different from rsync, files won't be deleted at the first run, instead they will be deleted at the next run, after files are successfully copied to the destination.|\n|`--delete-dst, --deleteDst`|Delete extraneous objects from destination.|\n|`--check-all`|Verify the integrity of all files in source and destination, default to false. Comparison is done on byte streams, which comes at a performance cost.|\n|`--check-new`|Verify the integrity of newly copied files, default to false. Comparison is done on byte streams, which comes at a performance cost.|\n|`--check-change` 1.3|Verify whether the data has changed before and after synchronization, default is false. Based on file size and mtime, which is lightweight.|\n|`--max-failure`1.3 |max number of allowed failed files (-1 for unlimited)|\n|`--dry`|Don't actually copy any file.|\n\n#### Storage related options {#sync-storage-related-options}\n\n|Items|Description|\n|-|-|\n|`--threads=10, -p 10`|Number of concurrent threads, default to 10.|\n|`--list-threads=1` 1.1 |Number of `list` threads, default to 1. Read [concurrent `list`](../guide/sync.md#concurrent-list) to learn its usage.|\n|`--list-depth=1` 1.1 |Depth of concurrent `list` operation, default to 1. Read [concurrent `list`](../guide/sync.md#concurrent-list) to learn its usage.|\n|`--no-https`|Do not use HTTPS, default to false.|\n|`--storage-class value` 1.1 |the storage class for destination|\n|`--bwlimit=0`|Limit bandwidth in Mbps default to 0 which means unlimited.|\n\n#### Cluster related options {#sync-cluster-related-options}\n\n|Items| Description|\n|-|-|\n|`--manager-addr=ADDR`| The listening address of the Manager node in distributed synchronization mode in the format: `:[port]`. If not specified, it listens on a random port. If this option is omitted, it listens on a random local IPv4 address and a random port. |\n|`--worker=ADDR,ADDR`| Worker node addresses used in distributed syncing, comma separated. |\n\n#### Metrics related options {#sync-metircs-related-options}\n\n|Items|Description|\n|-|-|\n|`--metrics value` 1.2|address to export metrics (default: \"127.0.0.1:9567\")|\n|`--consul value` 1.2|Consul address to register (default: \"127.0.0.1:8500\")|\n\n### `juicefs clone` 1.1 {#clone}\n\nQuickly clone directories or files within a single JuiceFS mount point. The cloning process involves copying only the metadata without copying the data blocks, making it extremely fast. Read [Clone Files or Directories](../guide/clone.md) for more.\n\n#### Synopsis\n\n```shell\njuicefs clone [command options] SRC DST\n\n# Clone a file\njuicefs clone /mnt/jfs/file1 /mnt/jfs/file2\n\n# Clone a directory\njuicefs clone /mnt/jfs/dir1 /mnt/jfs/dir2\n\n# Preserve the UID, GID, and mode of the file\njuicefs clone -p /mnt/jfs/file1 /mnt/jfs/file2\n```\n\n#### Options\n\n|Items|Description|\n|-|-|\n|`--preserve, -p`|By default, the executor's UID and GID are used for the clone result, and the mode is recalculated based on the user's umask. Use this option to preserve the UID, GID, and mode of the file.|\n\n### `juicefs compact` 1.2 {#compact}\n\nPerforms fragmentation optimization, merging, or cleaning of non-contiguous slices in the given directory to improve read performance. For detailed information, refer to [「Status Check and Maintenance」](../administration/status_check_and_maintenance.md).\n\n#### Overview\n\n```shell\njuicefs compact [command options] PATH\n\n# Perform fragmentation optimization on the specified directory\njuicefs compact /mnt/jfs\n```\n\n#### Parameters\n\n| Item | Description |\n|-|-|\n| `--threads, -p` | Number of threads to concurrently execute tasks (default: 10) |\n" }, { "path": "docs/en/reference/fuse_mount_options.md", "content": "---\ntitle: FUSE Mount Options\nsidebar_position: 5\nslug: /fuse_mount_options\n---\n\nJuiceFS provides several access methods, FUSE is the common one, which is the way to mount the file system locally using the `juicefs mount` command. Users can add FUSE mount options for more granular control.\n\nThis guide describes the common FUSE mount options for JuiceFS, with two ways to add mount options:\n\n1. Run [`juicefs mount`](../reference/command_reference.mdx#mount), and use `-o` to specify multiple options separated by commas.\n\n ```bash\n juicefs mount -d -o allow_other,writeback_cache sqlite3://myjfs.db ~/jfs\n ```\n\n2. When writing `/etc/fstab` items, add FUSE options directly to the `options` field, with multiple options separated by commas.\n\n ```\n # \n redis://localhost:6379/1 /jfs juicefs _netdev,writeback_cache 0 0\n ```\n\n## default_permissions\n\nThis option is automatically enabled when JuiceFS is mounted and does not need to be explicitly specified. It will enable the kernel's file access checks, which are performed outside the filesystem. When enabled, both the kernel checks and the file system checks must succeed before further operations.\n\n:::tip\nThe kernel performs standard Unix permission checks based on mode bits, UID/GID, and directory entry ownership.\n:::\n\n## allow_other\n\nBy default FUSE only allows access to the user mounting the file system. `allow_other` option overrides this behavior to allow access for other users. When mounting JuiceFS using root, `allow_other` is automatically assumed (search for `AllowOther` in [`fuse.go`](https://github.com/juicedata/juicefs/blob/main/pkg/fuse/fuse.go)). When mounting by non-root users, you'll need to first modify `/etc/fuse.conf` and enable `user_allow_other`, and then add `allow_other` to the mount command.\n\n## writeback_cache\n\n:::note\nThis mount option requires at least version 3.15 Linux kernel\n:::\n\nFUSE supports [\"writeback-cache mode\"](https://www.kernel.org/doc/Documentation/filesystems/fuse-io.txt), which means the `write()` syscall can often complete rapidly. It's recommended to enable this mount option when write small data (e.g. 100 bytes) frequently.\n\n## user_id and group_id\n\nThese options are used to specify the owner ID and owner group ID of the file system (as distinct from the UID and GID of a file or directory) for higher-level permission validation. If the allow_other option is specified, this option will not work. e.g. `sudo juicefs mount -o user_id=100,group_id=100`.\n\n## debug\n\nThis option will output Debug information from the low-level library (`go-fuse`) to `juicefs.log`.\n\n:::note\nThis option will output debug information for the low-level library (`go-fuse`) to `juicefs.log`. Note that this option is different from the global `-debug` option for the JuiceFS client, where the former outputs debug information for the `go-fuse` library and the latter outputs debug information for the JuiceFS client. see the documentation [Fault Diagnosis and Analysis](../administration/fault_diagnosis_and_analysis.md).\n:::\n" }, { "path": "docs/en/reference/how_to_set_up_metadata_engine.md", "content": "---\ntitle: How to Set Up Metadata Engine\nsidebar_position: 2\nslug: /databases_for_metadata\ndescription: JuiceFS supports Redis, TiKV, PostgreSQL, MySQL and other databases as metadata engines, and this article describes how to set up and use them.\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n:::tip\n`META_PASSWORD` is supported from JuiceFS v1.0. You should [upgrade](../administration/upgrade.md) if you're still using older versions.\n:::\n\nJuiceFS is a decoupled structure that separates data and metadata. Metadata can be stored in any supported database (called Metadata Engine). Many databases are supported and they all comes with different performance and intended scenarios, refer to [our docs](../benchmark/metadata_engines_benchmark.md) for comparison.\n\n## The storage usage of metadata {#storage-usage}\n\nThe storage space required for metadata is related to the length of the file name, the type and length of the file, and extended attributes. It is difficult to accurately estimate the metadata storage space requirements of a file system. For simplicity, we can approximate based on the storage space required for a single small file without extended attributes.\n\n- **Key-Value Database** (e.g. Redis, TiKV): 300 bytes/file\n- **Relational Database** (e.g. SQLite, MySQL, PostgreSQL): 600 bytes/file\n\nWhen the average file is larger (over 64MB), or the file is frequently modified and has a lot of fragments, or there are many extended attributes, or the average file name is long (over 50 bytes), more storage space is needed.\n\nWhen you need to migrate between two types of metadata engines, you can use this method to estimate the required storage space. For example, if you want to migrate the metadata engine from a relational database (MySQL) to a key-value database (Redis), and the current usage of MySQL is 30GB, then the target Redis needs to prepare at least 15GB or more of memory. The reverse is also true.\n\n## Redis Compatible Database\n\n### Redis\n\nJuiceFS requires Redis version 4.0 and above. Redis Cluster is also supported, but in order to avoid transactions across different Redis instances, JuiceFS puts all metadata for one file system on a single Redis instance.\n\n:::tip Redis Cluster Key Prefix\nWhen using Redis Cluster, the database number in the URL is used as a **key prefix** rather than for actual database selection (since Redis Cluster only supports database 0). The prefix format is `{N}` (e.g., `{1}`, `{2}`), which uses Redis hash tags to ensure all keys for one volume are routed to the same slot. This allows multiple JuiceFS file systems to share a single Redis Cluster:\n\n```shell\n# Different volumes use different DB numbers as key prefixes\njuicefs format redis://cluster:6379/1 volume1 # keys prefixed with {1}\njuicefs format redis://cluster:6379/2 volume2 # keys prefixed with {2}\n```\n\nYou can verify the keys in Redis Cluster using:\n\n```shell\nredis-cli -c -h -p 6379 keys '{1}*' # list all keys for volume with prefix {1}\n```\n\n:::\n\nTo ensure metadata security, JuiceFS requires [`maxmemory-policy noeviction`](https://redis.io/docs/reference/eviction/), otherwise it will try to set it to `noeviction` when starting JuiceFS, and will print a warning log if it fails. Refer to [Redis Best practices](../administration/metadata/redis_best_practices.md) for more.\n\n#### Create a file system\n\nWhen using Redis as the metadata storage engine, the following format is usually used to access the database:\n\n\n \n\n```\nredis[s]://[:@][:]/\n```\n\n \n \n\n```\nunix://[:@]?db=\n```\n\n \n\n\nWhere `[]` enclosed are optional and the rest are mandatory.\n\n- If the [TLS](https://redis.io/docs/manual/security/encryption) feature of Redis is enabled, the protocol header needs to use `rediss://`, otherwise use `redis://`.\n- `` is introduced after Redis 6.0 and can be ignored if there is no username, but the `:` colon in front of the password needs to be kept, e.g. `redis://:@:6379/1`.\n- The default port number on which Redis listens is `6379`, which can be left blank if the default port number is not changed, e.g. `redis://:@/1`.\n- Redis supports multiple [logical databases](https://redis.io/commands/select), please replace `` with the actual database number used.\n- If you need to connect to Redis Sentinel, the format will be slightly different, refer to [Redis Best Practices](../administration/metadata/redis_best_practices.md#high-availability) for details.\n- If username / password contains special characters, use single quote to avoid unexpected shell interpretations, or use the `REDIS_PASSWORD` environment.\n\n:::tip\nA Redis instance can, by default, create a total of 16 logical databases, with each of these databases eligible for the creation of a singular JuiceFS file system. Thus, under ordinary circumstances, a single Redis instance may be utilized to form up to 16 JuiceFS file systems. However, it is crucial to note that the logical databases intended for use with JuiceFS must not be shared with other applications, as doing so could lead to data inconsistencies.\n:::\n\nFor example, the following command will create a JuiceFS file system named `pics`, using the database No. `1` in Redis to store metadata:\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"redis://:mypassword@192.168.1.6:6379/1\" \\\n pics\n```\n\nFor security purposes, it is recommended to pass the password using the environment variable `META_PASSWORD` or `REDIS_PASSWORD`, e.g.\n\n```shell\nexport META_PASSWORD=mypassword\n```\n\nSimilarly, the password can be provided from a file using:\n\n```shell\nexport META_PASSWORD_FILE=/secret/mypassword.txt\n```\n\nThen there is no need to set a password in the metadata URL.\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"redis://192.168.1.6:6379/1\" \\\n pics\n```\n\n#### Mount a file system\n\nIf you need to share the same file system across multiple nodes, ensure that all nodes has access to the Metadata Engine.\n\n```shell\njuicefs mount -d \"redis://:mypassword@192.168.1.6:6379/1\" /mnt/jfs\n```\n\nPassing passwords with the `META_PASSWORD` or `REDIS_PASSWORD` environment variables is also supported.\n\n```shell\nexport META_PASSWORD=mypassword\njuicefs mount -d \"redis://192.168.1.6:6379/1\" /mnt/jfs\n```\n\nSimilarly, the password can be provided from a file using as follows:\n\n```shell\nexport META_PASSWORD_FILE=/secret/mypassword.txt\njuicefs mount -d \"redis://192.168.1.6:6379/1\" /mnt/jfs\n```\n\n#### Set up TLS\n\nJuiceFS supports both TLS server-side encryption authentication and mTLS mutual encryption authentication connections to Redis. When connecting to Redis via TLS or mTLS, use the `rediss://` protocol header. However, when using TLS server-side encryption authentication, it is not necessary to specify the client certificate and private key.\n\n:::note\nUsing Redis mTLS requires JuiceFS version 1.1.0 and above\n:::\n\nIf Redis server has enabled mTLS feature, it is necessary to provide client certificate, private key, and CA certificate that issued the client certificate to connect. In JuiceFS, mTLS can be used in the following way:\n\n```shell\njuicefs format --storage s3 \\\n ... \\\n \"rediss://192.168.1.6:6379/1?tls-cert-file=/etc/certs/client.crt&tls-key-file=/etc/certs/client.key&tls-ca-cert-file=/etc/certs/ca.crt\"\n pics\n```\n\nIn the code mentioned above, we use the `rediss://` protocol header to enable mTLS functionality, and then use the following options to specify the path of the client certificate:\n\n- `tls-cert-file=`: The path of the client certificate.\n- `tls-key-file=`: The path of the private key.\n- `tls-ca-cert-file=`: The path of the CA certificate. It is optional. If it is not specified, the system CA certificate will be used.\n- `insecure-skip-verify=true` It can skip verifying the server certificate.\n\nWhen specifying options in a URL, start with the `?` symbol and use the `&` symbol to separate multiple options, for example: `?tls-cert-file=client.crt&tls-key-file=client.key`.\n\nIn the above example, `/etc/certs` is just a directory name. Replace it with your actual certificate directory when using it, which can be a relative or absolute path.\n\n### KeyDB\n\n[KeyDB](https://keydb.dev) is an open source fork of Redis, developed to stay aligned with the Redis community. KeyDB implements multi-threading support, better memory utilization, and greater throughput on top of Redis, and also supports [Active Replication](https://github.com/JohnSully/KeyDB/wiki/Active-Replication), i.e., the Active Active feature.\n\n:::note\nSame as Redis, the Active Replication is asynchronous, which may cause consistency issues. So use with caution!\n:::\n\nWhen being used as metadata storage engine for Juice, KeyDB is used exactly in the same way as Redis. So please refer to the [Redis](#redis) section for usage.\n\n## Key-Value Database\n\n### BadgerDB\n\n[BadgerDB](https://github.com/dgraph-io/badger) is an embedded, persistent, and standalone Key-Value database developed in pure Go. The database files are stored locally in the specified directory.\n\nWhen using BadgerDB as the JuiceFS metadata storage engine, use `badger://` to specify the database path.\n\n#### Create a file system\n\nYou only need to create a file system for use, and there is no need to create a BadgerDB database in advance.\n\n```shell\njuicefs format badger://$HOME/badger-data myjfs\n```\n\nThis command creates `badger-data` as a database directory in the `home` directory of the current user, which is used as metadata storage for JuiceFS.\n\n#### Mount a file system\n\nThe database path needs to be specified when mounting the file system.\n\n```shell\njuicefs mount -d badger://$HOME/badger-data /mnt/jfs\n```\n\n:::tip\nBadgerDB only allows single-process access. If you need to perform operations like `gc`, `fsck`, `dump`, and `load`, you need to unmount the file system first.\n:::\n\n### TiKV\n\n[TiKV](https://tikv.org) is a distributed transactional Key-Value database. It is originally developed by PingCAP as the storage layer for their flagship product TiDB. Now TiKV is an independent open source project, and is also a granduated project of CNCF.\n\nBy using the official tool TiUP, you can easily build a local playground for testing (refer [here](https://tikv.org/docs/latest/concepts/tikv-in-5-minutes) for details). Production environment generally requires at least three hosts to store three data replicas (refer to the [official document](https://tikv.org/docs/latest/deploy/install/install) for all deployment steps).\n\n:::note\nIt's recommended to use dedicated TiKV 5.0+ cluster as the metadata engine for JuiceFS.\n:::\n\n#### Create a file system\n\nWhen using TiKV as the metadata storage engine, parameters needs to be specified as the following format:\n\n```shell\ntikv://[,...]/\n```\n\nThe `prefix` is a user-defined string, which can be used to distinguish multiple file systems or applications when they share the same TiKV cluster. For example:\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"tikv://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs\" \\\n pics\n```\n\n#### Set up TLS\n\nIf you need to enable TLS, you can set the TLS configuration item by adding the query parameter after the metadata URL. Currently supported configuration items:\n\n| Name | Value |\n|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `ca` | CA root certificate, used to connect TiKV/PD with TLS |\n| `cert` | certificate file path, used to connect TiKV/PD with TLS |\n| `key` | private key file path, used to connect TiKV/PD with TLS |\n| `verify-cn` | verify component caller's identity, [reference link](https://docs.pingcap.com/tidb/stable/enable-tls-between-components#verify-component-callers-identity) |\n\nFor example:\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"tikv://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs?ca=/path/to/ca.pem&cert=/path/to/tikv-server.pem&key=/path/to/tikv-server-key.pem&verify-cn=CN1,CN2\" \\\n pics\n```\n\n#### Mount a file system\n\n```shell\njuicefs mount -d \"tikv://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs\" /mnt/jfs\n```\n\n### etcd\n\n[etcd](https://etcd.io) is a small-scale key-value database with high availability and reliability, which can be used as metadata storage for JuiceFS.\n\n#### Create a file system\n\nWhen using etcd as the metadata engine, the `Meta-URL` parameter needs to be specified in the following format:\n\n```\netcd://[user:password@][,...]/\n```\n\nWhere `user` and `password` are required when etcd enables user authentication. The `prefix` is a user-defined string. When multiple file systems or applications share an etcd cluster, setting the prefix can avoid confusion and conflict. An example is as follows:\n\n```shell\njuicefs format etcd://user:password@192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs pics\n```\n\n#### Set up TLS\n\nIf you need to enable TLS, set the TLS configuration item by adding the query parameter after the metadata URL, use absolute path for certificate files to avoid file not found error.\n\n| Name | Value |\n|------------------------|-----------------------|\n| `cacert` | CA root certificate |\n| `cert` | certificate file path |\n| `key` | private key file path |\n| `server-name` | name of server |\n| `insecure-skip-verify` | 1 |\n\nFor example:\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"etcd://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs?cert=/path/to/ca.pem&cacert=/path/to/etcd-server.pem&key=/path/to/etcd-key.pem&server-name=etcd\" \\\n pics\n```\n\n#### Mount a file system\n\n```shell\njuicefs mount -d \"etcd://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs\" /mnt/jfs\n```\n\n:::note\nWhen mounting to the background, the path to the certificate needs to use an absolute path.\n:::\n\n### FoundationDB 1.1\n\n[FoundationDB](https://www.foundationdb.org) is a distributed database that can hold large-scale structured data on multiple clustered servers. The database system focuses on high performance, high scalability, and good fault tolerance. Using FoundationDB as the metadata engine requires its client library, so by default it is not enabled in the JuiceFS released binaries. If you need to use it, please compile it yourself.\n\n#### Compile JuiceFS\n\nFirst, you need to install the FoundationDB client library (refer to the [official documentation](https://apple.github.io/foundationdb/api-general.html#installing-client-binaries) for more details):\n\n\n \n\n```shell\ncurl -O https://github.com/apple/foundationdb/releases/download/6.3.25/foundationdb-clients_6.3.25-1_amd64.deb\nsudo dpkg -i foundationdb-clients_6.3.25-1_amd64.deb\n```\n\n \n \n\n```shell\ncurl -O https://github.com/apple/foundationdb/releases/download/6.3.25/foundationdb-clients-6.3.25-1.el7.x86_64.rpm\nsudo rpm -Uvh foundationdb-clients-6.3.25-1.el7.x86_64.rpm\n```\n\n \n\n\nThen, compile JuiceFS supporting FoundationDB:\n\n```shell\nmake juicefs.fdb\n```\n\n#### Create a file system\n\nWhen using FoundationDB as the metadata engine, the `Meta-URL` parameter needs to be specified in the following format:\n\n```uri\nfdb://[config file address]?prefix=\n```\n\nThe `` is the FoundationDB configuration file path, which is used to connect to the FoundationDB server. The `` is a user-defined string, which can be used to distinguish multiple file systems or applications when they share the same FoundationDB cluster. For example:\n\n```shell\njuicefs.fdb format \\\n --storage s3 \\\n ... \\\n \"fdb:///etc/foundationdb/fdb.cluster?prefix=jfs\" \\\n pics\n```\n\n#### Set up TLS\n\nIf you need to enable TLS, the general steps are as follows. For details, please refer to [official documentation](https://apple.github.io/foundationdb/tls.html).\n\n##### Use OpenSSL to generate a CA certificate\n\n```shell\nopenssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout private.key -out cert.crt\ncat cert.crt private.key > fdb.pem\n```\n\n##### Configure TLS\n\n| Command-line Option | Client Option | Environment Variable | Purpose |\n|------------------------|--------------------|----------------------------|----------------------------------------------------------------------------|\n| `tls_certificate_file` | `TLS_cert_path` | `FDB_TLS_CERTIFICATE_FILE` | Path to the file from which the local certificates can be loaded |\n| `tls_key_file` | `TLS_key_path` | `FDB_TLS_KEY_FILE` | Path to the file from which to load the private key |\n| `tls_verify_peers` | `tls_verify_peers` | `FDB_TLS_VERIFY_PEERS` | The byte-string for the verification of peer certificates and sessions |\n| `tls_password` | `tls_password` | `FDB_TLS_PASSWORD` | The byte-string representing the passcode for unencrypting the private key |\n| `tls_ca_file` | `TLS_ca_path` | `FDB_TLS_CA_FILE` | Path to the file containing the CA certificates to trust |\n\n##### Configure the server\n\nThe TLS parameters can be configured in `foundationdb.conf` or environment variables, as shown in the following configuration files (emphasis on the `[foundationdb.4500]` configuration).\n\n```ini title=\"foundationdb.conf\"\n[fdbmonitor]\nuser = foundationdb\ngroup = foundationdb\n\n[general]\nrestart-delay = 60\n## by default, restart-backoff = restart-delay-reset-interval = restart-delay\n# initial-restart-delay = 0\n# restart-backoff = 60\n# restart-delay-reset-interval = 60\ncluster-file = /etc/foundationdb/fdb.cluster\n# delete-envvars =\n# kill-on-configuration-change = true\n## Default parameters for individual fdbserver processes\n\n[fdbserver]\ncommand = /usr/sbin/fdbserver\n#public-address = auto:$ID\n#listen-address = public\ndatadir = /var/lib/foundationdb/data/$ID\nlogdir = /var/log/foundationdb\n# logsize = 10MiB\n# maxlogssize = 100MiB\n# machine-id =\n# datacenter-id =\n# class =\n# memory = 8GiB\n# storage-memory = 1GiB\n# cache-memory = 2GiB\n# metrics-cluster =\n# metrics-prefix =\n\n[fdbserver.4500]\nPublic - address = 127.0.0.1:4500: TLS\nlisten-address = public\ntls_certificate_file = /etc/foundationdb/fdb.pem\ntls_ca_file = /etc/foundationdb/cert.crt\ntls_key_file = /etc/foundationdb/private.key\ntls_verify_peers= Check.Valid=0\n\n[backup_agent]\ncommand = /usr/lib/foundationdb/backup_agent/backup_agent\nlogdir = /var/log/foundationdb\n\n[backup_agent.1]\n```\n\nIn addition, you need to add the suffix `:tls` after the address in `fdb.cluster`, `fdb.cluster` is as follows:\n\n```uri title=\"fdb.cluster\"\nU6pT9Jhl:ClZfjAWM@127.0.0.1:4500:tls\n```\n\n##### Configure the client\n\nYou need to configure TLS parameters and `fdb.cluster` on the client machine, `fdbcli` is the same.\n\nConnected by `fdbcli`:\n\n```shell\nfdbcli --tls_certificate_file=/etc/foundationdb/fdb.pem \\\n --tls_ca_file=/etc/foundationdb/cert.crt \\\n --tls_key_file=/etc/foundationdb/private.key \\\n --tls_verify_peers=Check.Valid=0\n```\n\nConnected by API (`fdbcli` also applies):\n\n```shell\nexport FDB_TLS_CERTIFICATE_FILE=/etc/foundationdb/fdb.pem \\\nexport FDB_TLS_CA_FILE=/etc/foundationdb/cert.crt \\\nexport FDB_TLS_KEY_FILE=/etc/foundationdb/private.key \\\nexport FDB_TLS_VERIFY_PEERS=Check.Valid=0\n```\n\n#### Mount a file system\n\n```shell\njuicefs.fdb mount -d \\\n \"fdb:///etc/foundationdb/fdb.cluster?prefix=jfs\" \\\n /mnt/jfs\n```\n\n## SQL Database\n\nEach database can only be used by one JuiceFS file system by default. If you want multiple file systems to share a database, you can achieve this by adding a `table_prefix` 1.3 query parameter in the META-URL to add different table prefixes for different file systems.\nFor example: `mysql://user:mypassword@(192.168.1.6:3306)/juicefs?table_prefix=volume1`\n\n### MySQL\n\n[MySQL](https://www.mysql.com) is one of the most popular open source relational databases, and is often preferred for web applications.\n\n>[MariaDB](https://mariadb.org) is an open source branch of MySQL, maintained by the original developers of MySQL. With its high compatibility with MySQL, setting up the Meta engine in MariaDB uses the same parameters and configurations as MySQL.\n>\n>[OceanBase](https://en.oceanbase.com) is a self-developed distributed relational database designed for processing massive data and high-concurrency transactions. It features high performance, strong consistency, and high availability. OceanBase is also highly compatible with MySQL, allowing the metadata engine to be configured in the same way.\n\n#### Create a file system\n\nWhen using MySQL as the metadata storage engine, you need to create a database manually before create the file system. The command with the following format is usually used to access the database:\n\n\n \n\n```\nmysql://[:]@(:3306)/\n```\n\n \n \n\n```\nmysql://[:]@unix()/\n```\n\n \n\n\n:::note\n\n1. Don't leave out the `()` brackets on either side of the URL.\n2. Special characters in passwords do not require url encoding\n\n:::\n\nFor example:\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs\" \\\n pics\n```\n\nA more secure approach would be to pass the database password through the environment variable `META_PASSWORD`:\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user@(192.168.1.6:3306)/juicefs\" \\\n pics\n```\n\nOr equivalently:\n\n```shell\nexport META_PASSWORD_FILE=\"/secret/mypassword.txt\"\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user@(192.168.1.6:3306)/juicefs\" \\\n pics\n```\n\nTo connect to a TLS enabled MySQL server, pass the `tls=true` parameter (or `tls=skip-verify` if using a self-signed certificate):\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs?tls=true\" \\\n pics\n```\n\n#### Mount a file system\n\n```shell\njuicefs mount -d \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\nPassing password with the `META_PASSWORD` environment variable is also supported when mounting a file system.\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs mount -d \"mysql://user@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\nPassing the password using a file is also supported as follows:\n\n```shell\nexport META_PASSWORD_FILE=\"/secret/mypassword.txt\"\njuicefs mount -d \"mysql://user@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\nTo connect to a TLS enabled MySQL server, pass the `tls=true` parameter (or `tls=skip-verify` if using a self-signed certificate):\n\n```shell\njuicefs mount -d \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs?tls=true\" /mnt/jfs\n```\n\nFor more examples of MySQL database address format, please refer to [Go-MySQL-Driver](https://github.com/Go-SQL-Driver/MySQL/#examples).\n\n### PostgreSQL\n\n[PostgreSQL](https://www.postgresql.org) is a powerful open source relational database with a perfect ecosystem and rich application scenarios, and it also works as the metadata engine of JuiceFS.\n\nMany cloud computing platforms offer hosted PostgreSQL database services, or you can deploy one yourself by following the [Usage Wizard](https://www.postgresqltutorial.com/postgresql-getting-started).\n\nOther PostgreSQL-compatible databases (such as CockroachDB) can also be used as metadata engine.\n\n#### Create a file system\n\nWhen using PostgreSQL as the metadata storage engine, you need to create a database manually before creating the file system by following the format below:\n\n\n \n\n```\npostgres://[username][:]@[:5432]/[?parameters]\n```\n\n \n \n\n```\npostgres://[username][:]@/?host=[¶meters]\n```\n\n \n\n\nWhere `[]` enclosed are optional and the rest are mandatory.\n\nFor example:\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"postgres://user:mypassword@192.168.1.6:5432/juicefs\" \\\n pics\n```\n\nA more secure approach would be to pass the database password through the environment variable `META_PASSWORD`:\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs format \\\n --storage s3 \\\n ... \\\n \"postgres://user@192.168.1.6:5432/juicefs\" \\\n pics\n```\n\nThe password can also be passed using a file as follows:\n\n```shell\nexport META_PASSWORD_FILE=\"/secret/mypassword.txt\"\njuicefs format \\\n --storage s3 \\\n ... \\\n \"postgres://user@192.168.1.6:5432/juicefs\" \\\n pics\n```\n\n:::note\n\n1. JuiceFS uses public [schema](https://www.postgresql.org/docs/current/ddl-schemas.html) by default, if you want to use a `non-public schema`, you need to specify `search_path` in the connection string parameter. e.g `postgres://user:mypassword@192.168.1.6:5432/juicefs?search_path=pguser1`\n2. If the `public schema` is not the first hit in the `search_path` configured on the PostgreSQL server, the `search_path` parameter must be explicitly set in the connection string.\n3. The `search_path` connection parameter can be set to multiple schemas natively, but currently JuiceFS only supports setting one. `postgres://user:mypassword@192.168.1.6:5432/juicefs?search_path=pguser1,public` will be considered illegal.\n4. Special characters in the password need to be replaced by url encoding. For example, `|` needs to be replaced with `%7C`.\n\n:::\n\n#### Mount a file system\n\n```shell\njuicefs mount -d \"postgres://user:mypassword@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\nPassing password with the `META_PASSWORD` environment variable is also supported when mounting a file system.\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs mount -d \"postgres://user@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\nPassing a password using a file is also supported as follows:\n\n```shell\nexport META_PASSWORD_FILE=\"/secret/mypassword.txt\"\njuicefs mount -d \"postgres://user@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\n#### Troubleshooting\n\nThe JuiceFS client connects to PostgreSQL via SSL encryption by default. If you encountered an error saying `pq: SSL is not enabled on the server`, you need to enable SSL encryption for PostgreSQL according to your own business scenario, or you can disable it by adding a parameter to the metadata URL Validation.\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"postgres://user@192.168.1.6:5432/juicefs?sslmode=disable\" \\\n pics\n```\n\nAdditional parameters can be appended to the metadata URL. More details can be seen [here](https://pkg.go.dev/github.com/lib/pq#hdr-Connection_String_Parameters).\n\n### SQLite\n\n[SQLite](https://sqlite.org) is a widely used small, fast, single-file, reliable and full-featured SQL database engine.\n\nThe SQLite database has only one file, which is very flexible to create and use. When using SQLite as the JuiceFS metadata storage engine, there is no need to create a database file in advance, and you can directly create a file system:\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"sqlite3://my-jfs.db\" \\\n pics\n```\n\nExecuting the above command will automatically create a database file named `my-jfs.db` in the current directory. **Please keep this file properly**!\n\nMount the file system:\n\n```shell\njuicefs mount -d \"sqlite3://my-jfs.db\" /mnt/jfs/\n```\n\nPlease note the location of the database file, if it is not in the current directory, you need to specify the absolute path to the database file, e.g.\n\n```shell\njuicefs mount -d \"sqlite3:///home/herald/my-jfs.db\" /mnt/jfs/\n```\n\nOne can also add driver supported [PRAGMA Statements](https://www.sqlite.org/pragma.html) to the connection string like:\n\n```shell\n\"sqlite3://my-jfs.db?cache=shared&_busy_timeout=5000\"\n```\n\nFor more examples of SQLite database address format, please refer to [Go-SQLite3 Driver](https://github.com/mattn/go-sqlite3#connection-string).\n\n:::note\nSince SQLite is a single-file database, usually only the host where the database is located can access it. Therefore, SQLite database is more suitable for standalone use. For multiple servers sharing the same file system, it is recommended to use databases such as Redis or MySQL.\n:::\n" }, { "path": "docs/en/reference/how_to_set_up_object_storage.md", "content": "---\ntitle: How to Set Up Object Storage\nsidebar_position: 3\ndescription: This article introduces the object storages supported by JuiceFS and how to configure and use it.\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\nAs you can learn from [JuiceFS Technical Architecture](../introduction/architecture.md), JuiceFS is a distributed file system with data and metadata stored separately. JuiceFS uses object storage as the main data storage and uses databases such as Redis, PostgreSQL and MySQL as metadata storage.\n\n## Storage options {#storage-options}\n\nWhen creating a JuiceFS file system, there are following options to set up the storage:\n\n- `--storage`: Specify the type of storage to be used by the file system, e.g. `--storage s3`\n- `--bucket`: Specify the storage access address, e.g. `--bucket https://myjuicefs.s3.us-east-2.amazonaws.com`\n- `--access-key` and `--secret-key`: Specify the authentication information when accessing the storage\n\nFor example, the following command uses Amazon S3 object storage to create a file system:\n\n```shell\njuicefs format --storage s3 \\\n --bucket https://myjuicefs.s3.us-east-2.amazonaws.com \\\n --access-key abcdefghijklmn \\\n --secret-key nmlkjihgfedAcBdEfg \\\n redis://192.168.1.6/1 \\\n myjfs\n```\n\n## Other options {#other-options}\n\nWhen executing the `juicefs format` or `juicefs mount` command, you can set some special options in the form of URL parameters in the `--bucket` option, such as `tls-insecure-skip-verify=true` in `https://myjuicefs.s3.us-east-2.amazonaws.com?tls-insecure-skip-verify=true` is to skip the certificate verification of HTTPS requests.\n\nClient certificates are also supported as they are commonly used for mTLS connections, for example:\n`https://myjuicefs.s3.us-east-2.amazonaws.com?ca-certs=./path/to/ca&ssl-cert=./path/to/cert&ssl-key=./path/to/privatekey`\n\n## Enable data sharding {#enable-data-sharding}\n\nWhen creating a file system, multiple buckets can be defined as the underlying storage of the file system through the [`--shards`](../reference/command_reference.mdx#format-data-format-options) option. In this way, the system will distribute the files to multiple buckets based on the hashed value of the file name. Data sharding technology can distribute the load of concurrent writing of large-scale data to multiple buckets, thereby improving the writing performance.\n\nThe following are points to note when using the data sharding function:\n\n- The `--shards` option accepts an integer between 0 and 256, indicating how many Buckets the files will be scattered into. The default value is 0, indicating that the data sharding function is not enabled.\n- Only multiple buckets under the same object storage can be used.\n- The integer wildcard `%d` needs to be used to specify the buckets, for example, `\"http://192.168.1.18:9000/myjfs-%d\"`. Buckets can be created in advance in this format, or automatically created by the JuiceFS client when creating a file system.\n- The data sharding is set at the time of creation and cannot be modified after creation. You cannot increase or decrease the number of buckets, nor cancel the shards function.\n\nFor example, the following command creates a file system with 4 shards.\n\n```shell\njuicefs format --storage s3 \\\n --shards 4 \\\n --bucket \"https://myjfs-%d.s3.us-east-2.amazonaws.com\" \\\n ...\n```\n\nAfter executing the above command, the JuiceFS client will create 4 buckets named `myjfs-0`, `myjfs-1`, `myjfs-2`, and `myjfs-3`.\n\n## Access Key and Secret Key {#aksk}\n\nIn general, object storages are authenticated with Access Key ID and Access Key Secret. For JuiceFS file system, they are provided by options `--access-key` and `--secret-key` (or AK, SK for short).\n\nIt is more secure to pass credentials via environment variables `ACCESS_KEY` and `SECRET_KEY` instead of explicitly specifying the options `--access-key` and `--secret-key` in the command line when creating a filesystem, e.g.,\n\n```shell\nexport ACCESS_KEY=abcdefghijklmn\nexport SECRET_KEY=nmlkjihgfedAcBdEfg\njuicefs format --storage s3 \\\n --bucket https://myjuicefs.s3.us-east-2.amazonaws.com \\\n redis://192.168.1.6/1 \\\n myjfs\n```\n\nPublic clouds typically allow users to create IAM (Identity and Access Management) roles, such as [AWS IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) or [Alibaba Cloud RAM role](https://www.alibabacloud.com/help/doc-detail/110376.htm), which can be assigned to VM instances. If the cloud server instance already has read and write access to the object storage, there is no need to specify `--access-key` and `--secret-key`.\n\n## Use temporary access credentials {#session-token}\n\nPermanent access credentials generally have two parts, Access Key, Secret Key, while temporary access credentials generally include three parts, Access Key, Secret Key and token, and temporary access credentials have an expiration time, usually between a few minutes and a few hours.\n\n### How to get temporary credentials {#how-to-get-temporary-credentials}\n\nDifferent cloud vendors have different acquisition methods. Generally, the Access Key, Secret Key and ARN representing the permission boundary of the temporary access credential are required as parameters to request access to the STS server of the cloud service vendor to obtain the temporary access credential. This process can generally be simplified by the SDK provided by the cloud vendor. For example, Amazon S3 can refer to this [link](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) to obtain temporary credentials, and Alibaba Cloud OSS can refer to this [link](https://www.alibabacloud.com/help/en/object-storage-service/latest/use-a-temporary-credential-provided-by-sts-to-access-oss).\n\n### How to set up object storage with temporary access credentials {#how-to-set-up-object-storage-with-temporary-access-credentials}\n\nThe way of using temporary credentials is not much different from using permanent credentials. When formatting the file system, pass the Access Key, Secret Key, and token of the temporary credentials through `--access-key`, `--secret-key`, `--session-token` can set the value. E.g:\n\n```bash\njuicefs format \\\n --storage oss \\\n --access-key xxxx \\\n --secret-key xxxx \\\n --session-token xxxx \\\n --bucket https://bucketName.oss-cn-hangzhou.aliyuncs.com \\\n redis://localhost:6379/1 \\\n test1\n```\n\nSince temporary credentials expire quickly, the key is how to update the temporary credentials that JuiceFS uses after `format` the file system before the temporary credentials expire. The credential update process is divided into two steps:\n\n1. Before the temporary certificate expires, apply for a new temporary certificate;\n2. Without stopping the running JuiceFS, use the `juicefs config Meta-URL --access-key xxxx --secret-key xxxx --session-token xxxx` command to hot update the access credentials.\n\nNewly mounted clients will use the new credentials directly, and all clients already running will also update their credentials within a minute. The entire update process will not affect the running business. Due to the short expiration time of the temporary credentials, the above steps need to **be executed in a long-term loop** to ensure that the JuiceFS service can access the object storage normally.\n\n## Internal and public endpoint {#internal-and-public-endpoint}\n\nTypically, object storage services provide a unified URL for access, but the cloud platform usually provides both internal and external endpoints. For example, the platform cloud services that meet the criteria will automatically resolve requests to the internal endpoint of the object storage. This offers you a lower latency, and internal network traffic is free.\n\nSome cloud computing platforms also distinguish between internal and public networks, but instead of providing a unified access URL, they provide separate internal Endpoint and public Endpoint addresses.\n\nJuiceFS also provides flexible support for this object storage service that distinguishes between internal and public addresses. For scenarios where the same file system is shared, the object storage is accessed through internal Endpoint on the servers that meet the criteria, and other computers are accessed through public Endpoint, which can be used as follows:\n\n- **When creating a file system**: It is recommended to use internal Endpoint address for `--bucket`\n- **When mounting a file system**: For clients that do not satisfy the internal line, you can specify a public Endpoint address to `--bucket`.\n\nCreating a file system using an internal Endpoint ensures better performance and lower latency, and for clients that cannot be accessed through an internal address, you can specify a public Endpoint to mount with the option `--bucket`.\n\n## Storage class 1.1 {#storage-class}\n\nObject storage usually supports multiple storage classes, such as standard storage, infrequent access storage, and archive storage. Different storage classes will have different prices and availability, you can set the default storage class with the [`--storage-class`](../reference/command_reference.mdx#format-data-storage-options) option when creating the JuiceFS file system, or set a new storage class with the [`--storage-class`](../reference/command_reference.mdx#mount-data-storage-options) option when mounting the JuiceFS file system. Please refer to the user manual of the object storage you are using to see how to set the value of the `--storage-class` option (such as [Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-StorageClass)).\n\n:::note\nWhen using certain storage classes (such as archive and deep archive), the data cannot be accessed immediately, and the data needs to be restored in advance and accessed after a period of time.\n:::\n\n:::note\nWhen using certain storage classes (such as infrequent access), there are minimum bill units, and additional charges may be incurred for reading data. Please refer to the user manual of the object storage you are using for details.\n:::\n\n## Using proxy {#using-proxy}\n\nIf the network environment where the client is located is affected by firewall policies or other factors that require access to external object storage services through a proxy, the corresponding proxy settings are different for different operating systems. Please refer to the corresponding user manual for settings.\n\nOn Linux, for example, the proxy can be set by creating `http_proxy` and `https_proxy` environment variables.\n\n```shell\nexport http_proxy=http://localhost:8035/\nexport https_proxy=http://localhost:8035/\njuicefs format \\\n --storage s3 \\\n ... \\\n myjfs\n```\n\n## Supported object storage {#supported-object-storage}\n\nIf you wish to use a storage system that is not listed, feel free to submit a requirement [issue](https://github.com/juicedata/juicefs/issues).\n\n| Name | Value |\n|:-----------------------------------------------------------:|:----------:|\n| [Amazon S3](#amazon-s3) | `s3` |\n| [Google Cloud Storage](#google-cloud) | `gs` |\n| [Azure Blob Storage](#azure-blob-storage) | `wasb` |\n| [Backblaze B2](#backblaze-b2) | `b2` |\n| [IBM Cloud Object Storage](#ibm-cloud-object-storage) | `ibmcos` |\n| [Oracle Cloud Object Storage](#oracle-cloud-object-storage) | `s3` |\n| [Scaleway Object Storage](#scaleway-object-storage) | `scw` |\n| [DigitalOcean Spaces](#digitalocean-spaces) | `space` |\n| [Wasabi](#wasabi) | `wasabi` |\n| [Telnyx Cloud Storage](#telnyx) | `s3` |\n| [Storj DCS](#storj-dcs) | `s3` |\n| [Vultr Object Storage](#vultr-object-storage) | `s3` |\n| [Cloudflare R2](#r2) | `s3` |\n| [Bunny Storage](#bunny) | `bunny` |\n| [Alibaba Cloud OSS](#alibaba-cloud-oss) | `oss` |\n| [Tencent Cloud COS](#tencent-cloud-cos) | `cos` |\n| [Huawei Cloud OBS](#huawei-cloud-obs) | `obs` |\n| [Baidu Object Storage](#baidu-object-storage) | `bos` |\n| [Volcano Engine TOS](#volcano-engine-tos) | `tos` |\n| [Kingsoft Cloud KS3](#kingsoft-cloud-ks3) | `ks3` |\n| [QingStor](#qingstor) | `qingstor` |\n| [Qiniu](#qiniu) | `qiniu` |\n| [CTYun OOS](#ctyun-oos) | `oos` |\n| [ECloud Object Storage](#ecloud-object-storage) | `eos` |\n| [JD Cloud OSS](#jd-cloud-oss) | `s3` |\n| [UCloud US3](#ucloud-us3) | `ufile` |\n| [Ceph RADOS](#ceph-rados) | `ceph` |\n| [Ceph RGW](#ceph-rgw) | `s3` |\n| [Gluster](#gluster) | `gluster` |\n| [Swift](#swift) | `swift` |\n| [MinIO](#minio) | `minio` |\n| [WebDAV](#webdav) | `webdav` |\n| [HDFS](#hdfs) | `hdfs` |\n| [Apache Ozone](#apache-ozone) | `s3` |\n| [Redis](#redis) | `redis` |\n| [TiKV](#tikv) | `tikv` |\n| [etcd](#etcd) | `etcd` |\n| [SQLite](#sqlite) | `sqlite3` |\n| [MySQL](#mysql) | `mysql` |\n| [PostgreSQL](#postgresql) | `postgres` |\n| [Local disk](#local-disk) | `file` |\n| [SFTP/SSH](#sftp) | `sftp` |\n\n### Amazon S3\n\nS3 supports [two styles of endpoint URI](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html): virtual hosted-style and path-style. The difference is:\n\n- Virtual-hosted-style: `https://.s3..amazonaws.com`\n- Path-style: `https://s3..amazonaws.com/`\n\nThe `` should be replaced with specific region code, e.g. the region code of US East (N. Virginia) is `us-east-1`. All the available region codes can be found [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions).\n\n:::note\nFor AWS users in China, you need add `.cn` to the host, i.e. `amazonaws.com.cn`, and check [this document](https://docs.amazonaws.cn/en_us/aws/latest/userguide/endpoints-arns.html) for region code.\n:::\n\n:::note\nIf the S3 bucket has public access (anonymous access is supported), please set `--access-key` to `anonymous`.\n:::\n\nIn JuiceFS both the two styles are supported to specify the bucket address, for example:\n\n\n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://.s3..amazonaws.com \\\n ... \\\n myjfs\n```\n\n \n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://s3..amazonaws.com/ \\\n ... \\\n myjfs\n```\n\n \n\n\nYou can also set `--storage` to `s3` to connect to S3-compatible object storage, e.g.:\n\n\n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n \n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https:/// \\\n ... \\\n myjfs\n```\n\n \n\n\n:::tip\nThe format of the option `--bucket` for all S3 compatible object storage services is `https://.` or `https:///`. The default `region` is `us-east-1`. When a different `region` is required, it can be set manually via the environment variable `AWS_REGION` or `AWS_DEFAULT_REGION`.\n:::\n\n### Google Cloud Storage {#google-cloud}\n\nGoogle Cloud uses [IAM](https://cloud.google.com/iam/docs/overview) to manage permissions for accessing resources. Through authorizing [service accounts](https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-gcloud), you can have a fine-grained control of the access rights of cloud servers and object storage.\n\nFor cloud servers and object storage that belong to the same service account, as long as the account grants access to the relevant resources, there is no need to provide authentication information when creating a JuiceFS file system, and the cloud platform will automatically complete authentication.\n\nFor cases where you want to access the object storage from outside the Google Cloud Platform, for example, to create a JuiceFS file system on your local computer using Google Cloud Storage, you need to configure authentication information. Since Google Cloud Storage does not use Access Key ID and Access Key Secret, but rather the JSON key file of the service account to authenticate the identity.\n\nPlease refer to [\"Authentication as a service account\"](https://cloud.google.com/docs/authentication/production) to create JSON key file for the service account and download it to the local computer, and define the path to the key file via the environment variable `GOOGLE_APPLICATION_ CREDENTIALS`, e.g.:\n\n```shell\nexport GOOGLE_APPLICATION_CREDENTIALS=\"$HOME/service-account-file.json\"\n```\n\nYou can write the command to create environment variables to `~/.bashrc` or `~/.profile` and have the shell set it automatically every time you start.\n\nOnce you have configured the environment variables for passing key information, the commands to create a file system locally and on Google Cloud Server are identical. For example,\n\n```bash\njuicefs format \\\n --storage gs \\\n --bucket [.region] \\\n ... \\\n myjfs\n```\n\nAs you can see, there is no need to include authentication information in the command, and the client will authenticate the access to the object storage through the JSON key file set in the previous environment variable. Also, since the bucket name is [globally unique](https://cloud.google.com/storage/docs/naming-buckets#considerations), when creating a file system, you only need to specify the bucket name in the option `--bucket`.\n\n### Azure Blob Storage\n\nTo use Azure Blob Storage as data storage of JuiceFS, please [check the documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage) to learn how to view the storage account name and access key, which correspond to the values ​​of the `--access-key` and `--secret-key` options, respectively.\n\nThe `--bucket` option is set in the format `https://.`, please replace `` with the name of the actual blob container and `` with `core.windows.net` (Azure Global) or `core.chinacloudapi.cn` (Azure China). For example:\n\n```bash\njuicefs format \\\n --storage wasb \\\n --bucket https://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\nIn addition to providing authorization information through the options `--access-key` and `--secret-key`, you could also create a [connection string](https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string) and set the environment variable `AZURE_STORAGE_CONNECTION_STRING`. For example:\n\n```bash\n# Use connection string\nexport AZURE_STORAGE_CONNECTION_STRING=\"DefaultEndpointsProtocol=https;AccountName=XXX;AccountKey=XXX;EndpointSuffix=core.windows.net\"\njuicefs format \\\n --storage wasb \\\n --bucket https:// \\\n ... \\\n myjfs\n```\n\n:::note\nFor Azure users in China, the value of `EndpointSuffix` is `core.chinacloudapi.cn`.\n:::\n\n### Backblaze B2\n\nTo use Backblaze B2 as a data storage for JuiceFS, you need to create [application key](https://www.backblaze.com/b2/docs/application_keys.html) first. **Application Key ID** and **Application Key** corresponds to Access Key and Secret Key, respectively.\n\nBackblaze B2 supports two access interfaces: the B2 native API and the S3-compatible API.\n\n#### B2 native API\n\nThe storage type should be set to `b2`, and only the bucket name needs to be set in the option `--bucket`. For example:\n\n```bash\njuicefs format \\\n --storage b2 \\\n --bucket \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n#### S3-compatible API\n\nThe storage type should be set to `s3`, and the full bucket address in the option `bucket` needs to be specified. For example:\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://s3.eu-central-003.backblazeb2.com/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### IBM Cloud Object Storage\n\nWhen creating JuiceFS file system using IBM Cloud Object Storage, you first need to create an [API key](https://cloud.ibm.com/docs/account?topic=account-manapikey) and an [instance ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID). The \"API key\" and \"instance ID\" are the equivalent of access key and secret key, respectively.\n\nIBM Cloud Object Storage provides [multiple endpoints](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-endpoints) for each region, depending on your network (e.g. public or private). Thus, please choose an appropriate endpoint. For example:\n\n```bash\njuicefs format \\\n --storage ibmcos \\\n --bucket https://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Oracle Cloud Object Storage\n\nOracle Cloud Object Storage supports S3 compatible access. Please refer to [official documentation](https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) for more information.\n\nThe `endpoint` format for this object storage is: `${namespace}.compat.objectstorage.${region}.oraclecloud.com`, for example:\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Scaleway Object Storage\n\nPlease follow [this document](https://www.scaleway.com/en/docs/generate-api-keys) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://.s3..scw.cloud`. Remember to replace `` with specific region code, e.g. the region code of \"Amsterdam, The Netherlands\" is `nl-ams`. All available region codes can be found [here](https://www.scaleway.com/en/docs/object-storage-feature/#-Core-Concepts). For example:\n\n```bash\njuicefs format \\\n --storage scw \\\n --bucket https://.s3..scw.cloud \\\n ... \\\n myjfs\n```\n\n### DigitalOcean Spaces\n\nPlease follow [this document](https://www.digitalocean.com/community/tutorials/how-to-create-a-digitalocean-space-and-api-key) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://..digitaloceanspaces.com`. Please replace `` with specific region code, e.g. `nyc3`. All available region codes can be found [here](https://www.digitalocean.com/docs/spaces/#regional-availability). For example:\n\n```bash\njuicefs format \\\n --storage space \\\n --bucket https://..digitaloceanspaces.com \\\n ... \\\n myjfs\n```\n\n### Wasabi\n\nPlease follow [this document](https://wasabi-support.zendesk.com/hc/en-us/articles/360019677192-Creating-a-Root-Access-Key-and-Secret-Key) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://.s3..wasabisys.com`, replace `` with specific region code, e.g. the region code of US East 1 (N. Virginia) is `us-east-1`. All available region codes can be found [here](https://wasabi-support.zendesk.com/hc/en-us/articles/360.15.26031-What-are-the-service-URLs-for-Wasabi-s-different-regions-). For example:\n\n```bash\njuicefs format \\\n --storage wasabi \\\n --bucket https://.s3..wasabisys.com \\\n ... \\\n myjfs\n```\n\n:::note\nFor users in Tokyo (ap-northeast-1) region, please refer to [this document](https://wasabi-support.zendesk.com/hc/en-us/articles/360039372392-How-do-I-access-the-Wasabi-Tokyo-ap-northeast-1-storage-region-) to learn how to get appropriate endpoint URI.***\n:::\n\n### Telnyx\n\nPrerequisites\n\n- A [Telnyx account](https://telnyx.com/sign-up)\n- [API key](https://portal.telnyx.com/#/app/api-keys) – this will be used as both `access-key` and `secret-key`\n\nSet up JuiceFS:\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://.telnyxstorage.com/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\nAvailable regional endpoints are [here](https://developers.telnyx.com/docs/cloud-storage/api-endpoints).\n\n### Storj DCS\n\nPlease refer to [this document](https://docs.storj.io/api-reference/s3-compatible-gateway) to learn how to create access key and secret key.\n\nStorj DCS is an S3-compatible storage, using `s3` for option `--storage`. The setting format of the option `--bucket` is `https://gateway..storjshare.io/`, and please replace `` with the corresponding region code you need. There are currently three available regions: `us1`, `ap1` and `eu1`. For example:\n\n```shell\njuicefs format \\\n --storage s3 \\\n --bucket https://gateway..storjshare.io/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n:::caution\nStorj DCS [ListObjects](https://github.com/storj/gateway-st/blob/main/docs/s3-compatibility.md#listobjects) API is not fully S3 compatible (result list is not sorted), so some features of JuiceFS do not work. For example, `juicefs gc`, `juicefs fsck`, `juicefs sync`, `juicefs destroy`. And when using `juicefs mount`, you need to disable [automatic-backup](../administration/metadata_dump_load.md#backup-automatically) function by adding `--backup-meta 0`.\n:::\n\n### Vultr Object Storage\n\nVultr Object Storage is an S3-compatible storage, using `s3` for `--storage` option. The format of the option `--bucket` is `https://..vultrobjects.com/`. For example:\n\n```shell\njuicefs format \\\n --storage s3 \\\n --bucket https://.ewr1.vultrobjects.com/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\nPlease find the access and secret keys for object storage [in the customer portal](https://my.vultr.com/objectstorage).\n\n### Cloudflare R2 {#r2}\n\nR2 is Cloudflare's object storage service and provides an S3-compatible API, so usage is the same as Amazon S3. Please refer to [Documentation](https://developers.cloudflare.com/r2/data-access/s3-api/tokens) to learn how to create Access Key and Secret Key.\n\n```shell\njuicefs format \\\n --storage s3 \\\n --bucket https://.r2.cloudflarestorage.com/myjfs \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\nFor production, it is recommended to pass key information via the `ACCESS_KEY` and `SECRET_KEY` environment variables, e.g.\n\n```shell\nexport ACCESS_KEY=\nexport SECRET_KEY=\njuicefs format \\\n --storage s3 \\\n --bucket https://.r2.cloudflarestorage.com/myjfs \\\n ... \\\n myjfs\n```\n\n:::caution\nCloudflare R2 `ListObjects` API is not fully S3 compatible (result list is not sorted), so some features of JuiceFS do not work. For example, `juicefs gc`, `juicefs fsck`, `juicefs sync`, `juicefs destroy`. And when using `juicefs mount`, you need to disable [automatic-backup](../administration/metadata_dump_load.md#backup-automatically) function by adding `--backup-meta 0`.\n:::\n\n### Bunny Storage {#bunny}\n\nBunny Storage offers a non-S3 compatible object storage with multiple performance tiers and many storage regions. It uses [it uses a custom API](https://docs.bunny.net/reference/storage-api).\n\nThis is not included by default, please build it with tag `bunny`\n\n#### Usage\n\nCreate a Storage Zone and use the Zone Name with the Hostname of the Location separated by a dot as Bucket name and the `Write Password` as Secret Key.\n\n```shell\njuicefs format \\\n --storage bunny \\\n --secret-key \"write-password\" \\\n --bucket \"https://uk.storage.bunnycdn.com/myzone\" \\ # https:///\n myjfs\n```\n\n### Alibaba Cloud OSS\n\nPlease follow [this document](https://www.alibabacloud.com/help/doc-detail/125558.htm) to learn how to get access key and secret key. If you have already created [RAM role](https://www.alibabacloud.com/help/doc-detail/110376.htm) and assigned it to a VM instance, you could omit the options `--access-key` and `--secret-key`.\n\nAlibaba Cloud also supports using [Security Token Service (STS)](https://www.alibabacloud.com/help/doc-detail/100624.htm) to authorize temporary access to OSS. If you wanna use STS, you should omit the options `--access-key` and `--secret-key` and set environment variables `ALICLOUD_ACCESS_KEY_ID`, `ALICLOUD_ACCESS_KEY_SECRET` and `SECURITY_TOKEN`instead, for example:\n\n```bash\n# Use Security Token Service (STS)\nexport ALICLOUD_ACCESS_KEY_ID=XXX\nexport ALICLOUD_ACCESS_KEY_SECRET=XXX\nexport SECURITY_TOKEN=XXX\njuicefs format \\\n --storage oss \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\nOSS provides [multiple endpoints](https://www.alibabacloud.com/help/doc-detail/31834.htm) for each region, depending on your network (e.g. public or internal network). Please choose an appropriate endpoint.\n\nIf you are creating a file system on AliCloud's server, you can specify the bucket name directly in the option `--bucket`. For example.\n\n```bash\n# Running within Alibaba Cloud\njuicefs format \\\n --storage oss \\\n --bucket \\\n ... \\\n myjfs\n```\n\n### Tencent Cloud COS\n\nThe naming rule of bucket in Tencent Cloud is `-`, so you must append `APPID` to the bucket name. Please follow [this document](https://intl.cloud.tencent.com/document/product/436/13312) to learn how to get `APPID`.\n\nThe full format of `--bucket` option is `https://-.cos..myqcloud.com`, and please replace `` with specific region code. E.g. the region code of Shanghai is `ap-shanghai`. You could find all available region codes [here](https://intl.cloud.tencent.com/document/product/436/6224). For example:\n\n```bash\njuicefs format \\\n --storage cos \\\n --bucket https://-.cos..myqcloud.com \\\n ... \\\n myjfs\n```\n\nIf you are creating a file system on Tencent Cloud's server, you can specify the bucket name directly in the option `--bucket`. For example.\n\n```bash\n# Running within Tencent Cloud\njuicefs format \\\n --storage cos \\\n --bucket - \\\n ... \\\n myjfs\n```\n\n### Huawei Cloud OBS\n\nPlease follow [this document](https://support.huaweicloud.com/usermanual-ca/zh-cn_topic_0046606340.html) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://.obs..myhuaweicloud.com`, and please replace `` with specific region code. E.g. the region code of Beijing 1 is `cn-north-1`. You could find all available region codes [here](https://developer.huaweicloud.com/endpoint?OBS). For example:\n\n```bash\njuicefs format \\\n --storage obs \\\n --bucket https://.obs..myhuaweicloud.com \\\n ... \\\n myjfs\n```\n\nIf you are creating a file system on Huawei Cloud's server, you can specify the bucket name directly in the option `--bucket`. For example,\n\n```bash\n# Running within Huawei Cloud\njuicefs format \\\n --storage obs \\\n --bucket \\\n ... \\\n myjfs\n```\n\n### Baidu Object Storage\n\nPlease follow [this document](https://cloud.baidu.com/doc/Reference/s/9jwvz2egb) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://..bcebos.com`, and please replace `` with specific region code. E.g. the region code of Beijing is `bj`. You could find all available region codes [here](https://cloud.baidu.com/doc/BOS/s/Ck1rk80hn#%E8%AE%BF%E9%97%AE%E5%9F%9F%E5%90%8D%EF%BC%88endpoint%EF%BC%89). For example:\n\n```bash\njuicefs format \\\n --storage bos \\\n --bucket https://..bcebos.com \\\n ... \\\n myjfs\n```\n\nIf you are creating a file system on Baidu Cloud's server, you can specify the bucket name directly in the option `--bucket`. For example,\n\n```bash\n# Running within Baidu Cloud\njuicefs format \\\n --storage bos \\\n --bucket \\\n ... \\\n myjfs\n```\n\n### Volcano Engine TOS 1.0.3 {#volcano-engine-tos}\n\nPlease follow [this document](https://www.volcengine.com/docs/6291/65568) to learn how to get access key and secret key.\n\nThe TOS provides [multiple endpoints](https://www.volcengine.com/docs/6349/107356) for each region, depending on your network (e.g. public or internal). Please choose an appropriate endpoint. For example:\n\n```bash\njuicefs format \\\n --storage tos \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### Kingsoft Cloud KS3\n\nPlease follow [this document](https://docs.ksyun.com/documents/1386) to learn how to get access key and secret key.\n\nKS3 provides [multiple endpoints](https://docs.ksyun.com/documents/6761) for each region, depending on your network (e.g. public or internal). Please choose an appropriate endpoint. For example:\n\n```bash\njuicefs format \\\n --storage ks3 \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### QingStor\n\nPlease follow [this document](https://docsv3.qingcloud.com/storage/object-storage/api/practices/signature/#%E8%8E%B7%E5%8F%96-access-key) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://..qingstor.com`, replace `` with specific region code. E.g. the region code of Beijing 3-A is `pek3a`. You could find all available region codes [here](https://docs.qingcloud.com/qingstor/#%E5%8C%BA%E5%9F%9F%E5%8F%8A%E8%AE%BF%E9%97%AE%E5%9F%9F%E5%90%8D). For example:\n\n```bash\njuicefs format \\\n --storage qingstor \\\n --bucket https://..qingstor.com \\\n ... \\\n myjfs\n```\n\n:::note\nThe format of `--bucket` option for all QingStor compatible object storage services is `http://.`.\n:::\n\n### Qiniu\n\nPlease follow [this document](https://developer.qiniu.com/af/kb/1479/how-to-access-or-locate-the-access-key-and-secret-key) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://.s3-.qiniucs.com`, replace `` with specific region code. E.g. the region code of China East is `cn-east-1`. You could find all available region codes [here](https://developer.qiniu.com/kodo/4088/s3-access-domainname). For example:\n\n```bash\njuicefs format \\\n --storage qiniu \\\n --bucket https://.s3-.qiniucs.com \\\n ... \\\n myjfs\n```\n\n### CTYun OOS\n\nPlease follow [this document](https://www.ctyun.cn/help2/10000101/10473683) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://.`, For example:\n\n```bash\njuicefs format \\\n --storage oos \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### ECloud Object Storage\n\nPlease follow [this document](https://ecloud.10086.cn/op-help-center/doc/article/24501) to learn how to get access key and secret key.\n\nECloud Object Storage provides [multiple endpoints](https://ecloud.10086.cn/op-help-center/doc/article/40956) for each region, depending on your network (e.g. public or internal). Please choose an appropriate endpoint. For example:\n\n```bash\njuicefs format \\\n --storage eos \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### JD Cloud OSS\n\nPlease follow [this document](https://docs.jdcloud.com/cn/account-management/accesskey-management) to learn how to get access key and secret key.\n\nThe `--bucket` option format is `https://..jdcloud-oss.com`，and please replace `` with specific region code. You could find all available region codes [here](https://docs.jdcloud.com/cn/object-storage-service/oss-endpont-list). For example:\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://..jdcloud-oss.com \\\n ... \\\n myjfs\n```\n\n### UCloud US3\n\nPlease follow [this document](https://docs.ucloud.cn/uai-censor/access/key) to learn how to get access key and secret key.\n\nUS3 (formerly UFile) provides [multiple endpoints](https://docs.ucloud.cn/ufile/introduction/region) for each region, depending on your network (e.g. public or internal). Please choose an appropriate endpoint. For example:\n\n```bash\njuicefs format \\\n --storage ufile \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### Ceph RADOS\n\n:::note\nJuiceFS v1.0 uses `go-ceph` v0.4.0, which supports Ceph Luminous (v12.2.x) and above.\nJuiceFS v1.1 uses `go-ceph` v0.18.0, which supports Ceph Octopus (v15.2.x) and above.\nMake sure that JuiceFS matches your Ceph and `librados` version, see [`go-ceph`](https://github.com/ceph/go-ceph#supported-ceph-versions).\n:::\n\nThe [Ceph Storage Cluster](https://docs.ceph.com/en/latest/rados) has a messaging layer protocol that enables clients to interact with a Ceph Monitor and a Ceph OSD Daemon. The [`librados`](https://docs.ceph.com/en/latest/rados/api/librados-intro) API enables you to interact with the two types of daemons:\n\n- The [Ceph Monitor](https://docs.ceph.com/en/latest/rados/configuration/common/#monitors), which maintains a master copy of the cluster map.\n- The [Ceph OSD Daemon (OSD)](https://docs.ceph.com/en/latest/rados/configuration/common/#osds), which stores data as objects on a storage node.\n\nJuiceFS supports the use of native Ceph APIs based on `librados`. You need to install `librados` library and build `juicefs` binary separately.\n\nFirst, install a `librados` that matches the version of your Ceph installation, For example, if Ceph version is Octopus (v15.2.x), then it is recommended to use `librados` v15.2.x.\n\n\n \n\n```bash\nsudo apt-get install librados-dev\n```\n\n \n \n\n```bash\nsudo yum install librados2-devel\n```\n\n \n\n\nThen compile JuiceFS for Ceph (make sure you have Go 1.20+ and GCC 5.4+ installed):\n\n```bash\nmake juicefs.ceph\n```\n\nWhen using with Ceph, the JuiceFS Client object storage related options are interpreted differently:\n\n* `--bucket` stands for the Ceph storage pool, the format is `ceph://`. A [pool](https://docs.ceph.com/en/latest/rados/operations/pools) is a logical partition for storing objects. Create a pool before use.\n* `--access-key` stands for the Ceph cluster name, the default value is `ceph`.\n* `--secret-key` option is [Ceph client user name](https://docs.ceph.com/en/latest/rados/operations/user-management), the default user name is `client.admin`.\n\nIn order to reach Ceph Monitor, `librados` reads Ceph configuration file by searching default locations and the first found will be used. The locations are:\n\n- `CEPH_CONF` environment variable\n- `/etc/ceph/ceph.conf`\n- `~/.ceph/config`\n- `ceph.conf` in the current working directory\n\nSince these additional Ceph configuration files are needed during the mount, CSI Driver users need to [upload them to Kubernetes, and map to the mount pod](https://juicefs.com/docs/csi/guide/pv/#mount-pod-extra-files).\n\nTo format a volume, run:\n\n```bash\njuicefs.ceph format \\\n --storage ceph \\\n --bucket ceph:// \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Ceph RGW\n\n[Ceph Object Gateway](https://ceph.io/ceph-storage/object-storage) is an object storage interface built on top of `librados` to provide applications with a RESTful gateway to Ceph Storage Clusters. Ceph Object Gateway supports S3-compatible interface, so we could set `--storage` to `s3` directly.\n\nThe `--bucket` option format is `http://.` (virtual hosted-style). For example:\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket http://. \\\n ... \\\n myjfs\n```\n\n### Gluster\n\n[Gluster](https://github.com/gluster/glusterfs) is a software defined distributed storage that can scale to several petabytes. JuiceFS communicates with Gluster via the `libgfapi` library, so it needs to be built separately before used.\n\nFirst, install `libgfapi` (version 6.0 - 10.1, [10.4+ is not supported yet](https://github.com/juicedata/juicefs/issues/4043))\n\n\n \n\n```bash\nsudo apt-get install uuid-dev libglusterfs-dev glusterfs-common\n```\n\n \n \n\n```bash\nsudo yum install glusterfs glusterfs-api-devel glusterfs-libs\n```\n\n \n\n\nThen compile JuiceFS supporting Gluster:\n\n```bash\nmake juicefs.gluster\n```\n\nNow we can create a JuiceFS volume on Gluster:\n\n```bash\njuicefs format \\\n --storage gluster \\\n --bucket host1,host2,host3/gv0 \\\n ... \\\n myjfs\n```\n\nThe format of `--bucket` option is `/`. Please note the `volume_name` here is the name of Gluster volume, and has nothing to do with the name of JuiceFS volume.\n\n### Swift\n\n[OpenStack Swift](https://github.com/openstack/swift) is a distributed object storage system designed to scale from a single machine to thousands of servers. Swift is optimized for multi-tenancy and high concurrency. Swift is ideal for backups, web and mobile content, and any other unstructured data that can grow without bound.\n\nThe `--bucket` option format is `http://.`. A container defines a namespace for objects.\n\n**Currently, JuiceFS only supports [Swift V1 authentication](https://www.swiftstack.com/docs/cookbooks/swift_usage/auth.html).**\n\nThe value of `--access-key` option is username. The value of `--secret-key` option is password. For example:\n\n```bash\njuicefs format \\\n --storage swift \\\n --bucket http://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### MinIO\n\n[MinIO](https://min.io) is an open source lightweight object storage, compatible with Amazon S3 API.\n\nIt is easy to run a MinIO instance locally using Docker. For example, the following command sets and maps port `9900` for the console with `--console-address \":9900\"` and also maps the data path for the MinIO to the `minio-data` folder in the current directory, which can be modified if needed.\n\n```shell\nsudo docker run -d --name minio \\\n -p 9000:9000 \\\n -p 9900:9900 \\\n -e \"MINIO_ROOT_USER=minioadmin\" \\\n -e \"MINIO_ROOT_PASSWORD=minioadmin\" \\\n -v $PWD/minio-data:/data \\\n --restart unless-stopped \\\n minio/minio server /data --console-address \":9900\"\n```\n\nAfter container is up and running, you can access:\n\n- **MinIO API**: [http://127.0.0.1:9000](http://127.0.0.1:9000), this is the object storage service address used by JuiceFS\n- **MinIO UI**: [http://127.0.0.1:9900](http://127.0.0.1:9900), this is used to manage the object storage itself, not related to JuiceFS\n\nThe initial Access Key and Secret Key of the object storage are both `minioadmin`.\n\nWhen using MinIO as data storage for JuiceFS, set the option `--storage` to `minio`.\n\n```bash\njuicefs format \\\n --storage minio \\\n --bucket http://127.0.0.1:9000/ \\\n --access-key minioadmin \\\n --secret-key minioadmin \\\n ... \\\n myjfs\n```\n\n:::note\n\n1. Currently, JuiceFS only supports path-style MinIO URI addresses, e.g., `http://127.0.0.1:9000/myjfs`.\n1. The `MINIO_REGION` environment variable can be used to set the region of MinIO, if not set, the default is `us-east-1`.\n1. When using Multi-Node MinIO deployment, consider setting using a DNS address in the service endpoint, resolving to all MinIO Node IPs, as a simple load-balancer, e.g. `http://minio.example.com:9000/myjfs`\n\n:::\n\n### WebDAV\n\n[WebDAV](https://en.wikipedia.org/wiki/WebDAV) is an extension of the Hypertext Transfer Protocol (HTTP)\nthat facilitates collaborative editing and management of documents stored on the WWW server among users.\nFrom JuiceFS v0.15+, JuiceFS can use a storage that speaks WebDAV as a data storage.\n\nYou need to set `--storage` to `webdav`, and `--bucket` to the endpoint of WebDAV. If basic authorization is enabled, username and password should be provided as `--access-key` and `--secret-key`, for example:\n\n```bash\njuicefs format \\\n --storage webdav \\\n --bucket http:/// \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### HDFS\n\n[HDFS](https://hadoop.apache.org) is the file system for Hadoop, which can be used as the object storage for JuiceFS.\n\nWhen HDFS is used, `--access-key` can be used to specify the `username`, and `hdfs` is usually the default superuser. For example:\n\n```bash\njuicefs format \\\n --storage hdfs \\\n --bucket namenode1:8020 \\\n --access-key hdfs \\\n ... \\\n myjfs\n```\n\nWhen `--access-key` is not specified on formatting, JuiceFS will use the current user of `juicefs mount` or Hadoop SDK to access HDFS. It will hang and fail with IO error eventually, if the current user don't have enough permission to read/write the blocks in HDFS.\n\nJuiceFS will try to load configurations for HDFS client based on `$HADOOP_CONF_DIR` or `$HADOOP_HOME`. If an empty value is provided to `--bucket`, the default HDFS found in Hadoop configurations will be used.\n\nbucket format:\n\n- `[hdfs://]namenode:port[/path]`\n\nfor HA cluster:\n\n- `[hdfs://]namenode1:port,namenode2:port[/path]`\n- `[hdfs://]nameservice[/path]`\n\nFor HDFS which enable Kerberos, `KRB5KEYTAB` and `KRB5PRINCIPAL` environment var can be used to set keytab and principal.\n\n### Apache Ozone\n\nApache Ozone is a scalable, redundant, and distributed object storage for Hadoop. It supports S3-compatible interface, so we could set `--storage` to `s3` directly.\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket http:///\\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Redis\n\n[Redis](https://redis.io) can be used as both metadata storage for JuiceFS and as data storage, but when using Redis as a data storage, it is recommended not to store large-scale data.\n\n#### Standalone\n\nThe `--bucket` option format is `redis://:/`. The value of `--access-key` option is username. The value of `--secret-key` option is password. For example:\n\n```bash\njuicefs format \\\n --storage redis \\\n --bucket redis://:/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n#### Redis Sentinel\n\nIn Redis Sentinel mode, the format of the `--bucket` option is `redis[s]://MASTER_NAME,SENTINEL_ADDR[,SENTINEL_ADDR]:SENTINEL_PORT[/DB]`. Sentinel's password needs to be declared through the `SENTINEL_PASSWORD_FOR_OBJ` environment variable. For example:\n\n```bash\nexport SENTINEL_PASSWORD_FOR_OBJ=sentinel_password\njuicefs format \\\n --storage redis \\\n --bucket redis://masterName,1.2.3.4,1.2.5.6:26379/2 \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n#### Redis Cluster\n\nIn Redis Cluster mode, the format of `--bucket` option is `redis[s]://ADDR:PORT,[ADDR:PORT],[ADDR:PORT]`. For example:\n\n```bash\njuicefs format \\\n --storage redis \\\n --bucket redis://127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002 \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### TiKV\n\n[TiKV](https://tikv.org) is a highly scalable, low latency, and easy to use key-value database. It provides both raw and ACID-compliant transactional key-value API.\n\nTiKV can be used as both metadata storage and data storage for JuiceFS.\n\n:::note\nIt's recommended to use dedicated TiKV 5.0+ cluster as the data storage for JuiceFS.\n:::\n\nThe `--bucket` option format is `:,:,:`, and `` is the address of Placement Driver (PD). The options `--access-key` and `--secret-key` have no effect and can be omitted. For example:\n\n```bash\njuicefs format \\\n --storage tikv \\\n --bucket \":,:,:\" \\\n ... \\\n myjfs\n```\n\n:::note\nDon't use the same TiKV cluster for both metadata and data, because JuiceFS uses non-transactional protocol (RawKV) for objects and transactional protocol (TnxKV) for metadata. The TxnKV protocol has special encoding for keys, so they may overlap with keys even they has different prefixes. BTW, it's recommended to enable [Titan](https://tikv.org/docs/latest/deploy/configure/titan) in TiKV for data cluster.\n:::\n\n#### Set up TLS\n\nIf you need to enable TLS, you can set the TLS configuration item by adding the query parameter after the bucket URL. Currently supported configuration items:\n\n| Name | Value |\n|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `ca` | CA root certificate, used to connect TiKV/PD with TLS |\n| `cert` | certificate file path, used to connect TiKV/PD with TLS |\n| `key` | private key file path, used to connect TiKV/PD with TLS |\n| `verify-cn` | verify component caller's identity, [reference link](https://docs.pingcap.com/tidb/dev/enable-tls-between-components#verify-component-callers-identity) |\n\nFor example:\n\n```bash\njuicefs format \\\n --storage tikv \\\n --bucket \":,:,:?ca=/path/to/ca.pem&cert=/path/to/tikv-server.pem&key=/path/to/tikv-server-key.pem&verify-cn=CN1,CN2\" \\\n ... \\\n myjfs\n```\n\n### etcd\n\n[etcd](https://etcd.io) is a small-scale key-value database with high availability and reliability, which can be used as both the metadata storage of JuiceFS and the data storage of JuiceFS.\n\netcd will [limit](https://etcd.io/docs/latest/dev-guide/limit) a single request to no more than 1.5MB by default, you need to change the block size (`--block-size` option) of JuiceFS to 1MB or even lower.\n\nThe `--bucket` option needs to fill in the etcd address, the format is similar to `:,:,:`. The `--access-key` and `--secret-key` options are filled with username and password, which can be omitted when etcd does not enable user authentication. E.g:\n\n```bash\njuicefs format \\\n --storage etcd \\\n --block-size 1024 \\ # This option is very important\n --bucket \":,:,:/prefix\" \\\n --access-key myname \\\n --secret-key mypass \\\n ... \\\n myjfs\n```\n\n#### Set up TLS\n\nIf you need to enable TLS, you can set the TLS configuration item by adding the query parameter after the bucket URL. Currently supported configuration items:\n\n| Name | Value |\n|------------------------|-----------------------|\n| `cacert` | CA root certificate |\n| `cert` | certificate file path |\n| `key` | private key file path |\n| `server-name` | name of server |\n| `insecure-skip-verify` | 1 |\n\nFor example:\n\n```bash\njuicefs format \\\n --storage etcd \\\n --bucket \":,:,:?cacert=/path/to/ca.pem&cert=/path/to/server.pem&key=/path/to/key.pem&server-name=etcd\" \\\n ... \\\n myjfs\n```\n\n:::note\nThe path to the certificate needs to be an absolute path, and make sure that all machines that need to mount can use this path to access them.\n:::\n\n### SQLite\n\n[SQLite](https://sqlite.org) is a small, fast, single-file, reliable, full-featured single-file SQL database engine widely used around the world.\n\nWhen using SQLite as a data store, you only need to specify its absolute path.\n\n```shell\njuicefs format \\\n --storage sqlite3 \\\n --bucket /path/to/sqlite3.db \\\n ... \\\n myjfs\n```\n\n:::note\nSince SQLite is an embedded database, only the host where the database is located can access it, and cannot be used in multi-machine sharing scenarios. If a relative path is used when formatting, it will cause problems when mounting, please use an absolute path.\n:::\n\n### MySQL\n\n[MySQL](https://www.mysql.com) is one of the popular open source relational databases, often used as the database of choice for web applications, both as a metadata engine for JuiceFS and for storing files data. MySQL-compatible [MariaDB](https://mariadb.org), [TiDB](https://github.com/pingcap/tidb), etc. can be used as data storage.\n\nWhen using MySQL as a data storage, you need to create a database in advance and add the desired permissions, specify the access address through the `--bucket` option, specify the user name through the `--access-key` option, and specify the password through the `--secret-key` option. An example is as follows:\n\n```shell\njuicefs format \\\n --storage mysql \\\n --bucket (:3306)/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\nAfter the file system is created, JuiceFS creates a table named `jfs_blob` in the database to store the data.\n\n:::note\nDon't miss the parentheses `()` in the `--bucket` parameter.\n:::\n\n### PostgreSQL\n\n[PostgreSQL](https://www.postgresql.org) is a powerful open source relational database with a complete ecology and rich application scenarios. It can be used as both the metadata engine of JuiceFS and the data storage. Other databases compatible with the PostgreSQL protocol (such as [CockroachDB](https://github.com/cockroachdb/cockroach), etc.) can also be used as data storage.\n\nWhen creating a file system, you need to create a database and add the corresponding read and write permissions. Use the `--bucket` option to specify the address of the data, use the `--access-key` option to specify the username, and use the `--secret-key` option to specify the password. An example is as follows:\n\n```shell\njuicefs format \\\n --storage postgres \\\n --bucket :/[?parameters] \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\nAfter the file system is created, JuiceFS creates a table named `jfs_blob` in the database to store the data.\n\n#### Troubleshooting\n\nThe JuiceFS client uses SSL encryption to connect to PostgreSQL by default. If the connection error `pq: SSL is not enabled on the server` indicates that the database does not have SSL enabled. You can enable SSL encryption for PostgreSQL according to your business scenario, or you can add the parameter `sslmode=disable` to the bucket URL to disable encryption verification.\n\n### Local disk\n\nWhen creating JuiceFS storage, if no storage type is specified, the local disk will be used to store data by default. The default storage path for root user is `/var/jfs`, and `~/.juicefs/local` is for ordinary users.\n\nFor example, using the local Redis database and local disk to create a JuiceFS storage named `test`:\n\n```shell\njuicefs format redis://localhost:6379/1 test\n```\n\nLocal storage is usually only used to help users understand how JuiceFS works and to give users an experience on the basic features of JuiceFS. The created JuiceFS storage cannot be mounted by other clients within the network and can only be used on a single machine.\n\n### SFTP/SSH {#sftp}\n\nSFTP - Secure File Transfer Protocol, It is not a type of storage. To be precise, JuiceFS reads and writes to disks on remote hosts via SFTP/SSH, thus allowing any SSH-enabled operating system to be used as a data storage for JuiceFS.\n\nFor example, the following command uses the SFTP protocol to connect to the remote server `192.168.1.11` and creates the `myjfs/` folder in the `$HOME` directory of user `tom` as the data storage of JuiceFS.\n\n```shell\njuicefs format \\\n --storage sftp \\\n --bucket 192.168.1.11:myjfs/ \\\n --access-key tom \\\n --secret-key 123456 \\\n ...\n redis://localhost:6379/1 myjfs\n```\n\n#### Notes\n\n- `--bucket` is used to set the server address and storage path in the format `[sftp://]:[port]:`. Note that the directory name should end with `/`, and the port number is optionally defaulted to `22`, e.g. `192.168.1.11:22:myjfs/`.\n- `--access-key` set the username of the remote server\n- `--secret-key` set the password of the remote server\n\n### NFS {#nfs}\n\nNFS - Network File System, is a commonly used file-sharing service in Unix-like operating systems. It allows computers within a network to access remote files as if they were local files.\n\nJuiceFS supports using NFS as the underlying storage to build a file system, offering two usage methods: local mount and direct mode.\n\n#### Local Mount\n\nJuiceFS v1.1 and earlier versions only support using NFS as underlying storage via local mount. This method requires mounting the directory on the NFS server locally first, and then using it as a local disk to create the JuiceFS file system.\n\nFor example, first mount the `/srv/data` directory from the remote NFS server `192.168.1.11` to the local `/mnt/data` directory, and then access it in `file` mode.\n\n```shell\n$ sudo mount -t nfs 192.168.1.11:/srv/data /mnt/data\n$ sudo juicefs format \\\n --storage file \\\n --bucket /mnt/data \\\n ... \\\n redis://localhost:6379/1 myjfs\n```\n\nFrom JuiceFS's perspective, the locally mounted NFS is still a local disk, so the `--storage` option is set to `file`.\n\nSimilarly, because the underlying storage can only be accessed on the mounted device, to share access across multiple devices, you need to mount the NFS share on each device separately, or provide external access through network-based methods such as WebDAV or S3 Gateway.\n\n#### Direct Mode\n\nJuiceFS v1.2 and later versions support using NFS as the underlying storage in direct mode. This method does not require pre-mounting the NFS directory locally but accesses the shared directory directly through the built-in NFS protocol in the JuiceFS client.\n\nFor example, the remote server's `/etc/exports` configuration file exports the following NFS share:\n\n```\n/srv/data 192.168.1.0/24(rw,sync,no_subtree_check)\n```\n\nYou can directly use the JuiceFS client to connect to the `/srv/data` directory on the NFS server to create the file system:\n\n```shell\n$ sudo juicefs format \\\n --storage nfs \\\n --bucket 192.168.1.11:/srv/data \\\n ... \\\n redis://localhost:6379/1 myjfs\n```\n\nIn direct mode, the `--storage` option is set to `nfs`, and the `--bucket` option is set to the NFS server address and shared directory. The JuiceFS client will directly connect to the directory on the NFS server to read and write data.\n\n**A few considerations:**\n\n1. JuiceFS direct mode currently only supports the NFSv3 protocol.\n2. The JuiceFS client needs permission to access the NFS shared directory.\n3. NFS by default enables the `root_squash` feature, which maps root access to the NFS share to the `nobody` user by default. To avoid permission issues with NFS shares, you can set the owner of the shared directory to `nobody:nogroup` or configure the NFS share with the `no_root_squash` option to disable permission squashing.\n" }, { "path": "docs/en/reference/p8s_metrics.md", "content": "---\ntitle: JuiceFS Metrics\nsidebar_position: 4\n---\n\nIf you haven't yet set up monitoring for JuiceFS, read [monitoring and data visualization\"](../administration/monitoring.md) to learn how.\n\n## Global labels {#global-labels}\n\n| Name | Description |\n| ---- | ----------- |\n| `vol_name` | Volume name |\n| `instance` | Client host name in format `:`. Refer to [official document](https://prometheus.io/docs/concepts/jobs_instances) for more information |\n| `mp` | Mount point path, if metrics are reported through [Prometheus Pushgateway](https://github.com/prometheus/pushgateway), for example, [JuiceFS Hadoop Java SDK](../administration/monitoring.md#hadoop), `mp` will be `sdk-` |\n\n## File system {#file-system}\n\n### Metrics\n\n| Name | Description | Unit |\n|-------------------------------|----------------------------------------|------|\n| `juicefs_used_space` | Total used space | byte |\n| `juicefs_used_inodes` | Total number of inodes | |\n\n## Operating system {#operating-system}\n\n### Metrics\n\n| Name | Description | Unit |\n| ---- | ----------- | ---- |\n| `juicefs_uptime` | Total running time | second |\n| `juicefs_cpu_usage` | Accumulated CPU usage | second |\n| `juicefs_memory` | Used memory | byte |\n\n## Metadata engine {#metadata-engine}\n\n### Metrics\n\n| Name | Description | Unit |\n| ---- | ----------- | ---- |\n| `juicefs_transaction_durations_histogram_seconds` | Transactions latency distributions | second |\n| `juicefs_transaction_restart` | Number of times a transaction restarted | |\n\n## FUSE {#fuse}\n\n### Metrics\n\n| Name | Description | Unit |\n| ---- | ----------- | ---- |\n| `juicefs_fuse_read_size_bytes` | Size distributions of read request | byte |\n| `juicefs_fuse_written_size_bytes` | Size distributions of write request | byte |\n| `juicefs_fuse_ops_durations_histogram_seconds` | Operations latency distributions | second |\n| `juicefs_fuse_open_handlers` | Number of open files and directories | |\n\n## SDK {#sdk}\n\n### Metrics\n\n| Name | Description | Unit |\n| ---- | ----------- | ---- |\n| `juicefs_sdk_read_size_bytes` | Size distributions of read request | byte |\n| `juicefs_sdk_written_size_bytes` | Size distributions of write request | byte |\n| `juicefs_sdk_ops_durations_histogram_seconds` | Operations latency distributions | second |\n\n## Cache {#cache}\n\n### Metrics\n\n| Name | Description | Unit |\n|:----------------------------------------|---------------------------------------------|--------|\n| `juicefs_blockcache_blocks` | Number of cached blocks | |\n| `juicefs_blockcache_bytes` | Size of cached blocks | byte |\n| `juicefs_blockcache_hits` | Count of cached block hits | |\n| `juicefs_blockcache_miss` | Count of cached block miss | |\n| `juicefs_blockcache_writes` | Count of cached block writes | |\n| `juicefs_blockcache_drops` | Count of cached block drops | |\n| `juicefs_blockcache_evicts` | Count of cached block evicts | |\n| `juicefs_blockcache_hit_bytes` | Size of cached block hits | byte |\n| `juicefs_blockcache_miss_bytes` | Size of cached block miss | byte |\n| `juicefs_blockcache_write_bytes` | Size of cached block writes | byte |\n| `juicefs_blockcache_read_hist_seconds` | Latency distributions of read cached block | second |\n| `juicefs_blockcache_write_hist_seconds` | Latency distributions of write cached block | second |\n| `juicefs_staging_blocks` | Number of blocks in the staging path | |\n| `juicefs_staging_block_bytes` | Total bytes of blocks in the staging path | byte |\n| `juicefs_staging_block_delay_seconds` | Total seconds of delay for staging blocks | second |\n\n## Object storage {#object-storage}\n\n### Labels\n\n| Name | Description |\n| ---- | ----------- |\n| `method` | Method to request object storage (e.g. GET, PUT, HEAD, DELETE) |\n\n### Metrics\n\n| Name | Description | Unit |\n| ---- | ----------- | ---- |\n| `juicefs_object_request_durations_histogram_seconds` | Object storage request latency distributions | second |\n| `juicefs_object_request_errors` | Count of failed requests to object storage | |\n| `juicefs_object_request_data_bytes` | Size of requests to object storage | byte |\n\n## Internal {#internal}\n\n### Metrics\n\n| Name | Description | Unit |\n|----------------------------------------| ----------- | ---- |\n| `juicefs_compact_size_histogram_bytes` | Size distributions of compacted data | byte |\n| `juicefs_used_read_buffer_size_bytes` | size of currently used buffer for read | |\n\n## Data synchronization {#sync}\n\n### Metrics\n\n| Name | Description | Unit |\n|-|-|-|\n| `juicefs_sync_scanned` | Number of all objects scanned from the source | |\n| `juicefs_sync_handled` | Number of objects from the source that have been processed | |\n| `juicefs_sync_pending` | Number of objects waiting to be synchronized | |\n| `juicefs_sync_copied` | Number of objects that have been synchronized | |\n| `juicefs_sync_copied_bytes` | Total size of data that has been synchronized | byte |\n| `juicefs_sync_skipped` | Number of objects that skipped during synchronization | |\n| `juicefs_sync_failed` | Number of objects that failed during synchronization | |\n| `juicefs_sync_deleted` | Number of objects that deleted during synchronization | |\n| `juicefs_sync_checked` | Number of objects that have been verified by checksum during synchronization | |\n| `juicefs_sync_checked_bytes` | Total size of data that has been verified by checksum during synchronization | byte |\n" }, { "path": "docs/en/reference/posix_compatibility.md", "content": "---\ntitle: POSIX Compatibility\nsidebar_position: 6\nslug: /posix_compatibility\ndescription: Learn how JuiceFS ensures POSIX compatibility through testing with pjdfstest and LTP.\n---\n\nJuiceFS ensures POSIX compatibility by using [pjdfstest](https://github.com/pjd/pjdfstest) and [Linux Test Project (LTP)](https://github.com/linux-test-project/ltp) for testing.\n\n## Pjdfstest\n\nPjdfstest is a test suite that helps to test POSIX system calls. JuiceFS passed all of its latest 8,813 tests:\n\n```\nAll tests successful.\n\nTest Summary Report\n-------------------\n/root/soft/pjdfstest/tests/chown/00.t (Wstat: 0 Tests: 1323 Failed: 0)\n TODO passed: 693, 697, 708-709, 714-715, 729, 733\nFiles=235, Tests=8813, 233 wallclock secs ( 2.77 usr 0.38 sys + 2.57 cusr 3.93 csys = 9.65 CPU)\nResult: PASS\n```\n\n:::note\nWhen running pjdfstest, you must disable the JuiceFS trash, because the test deletes files directly rather than moving them to the trash. The JuiceFS trash is enabled by default. To disable it, run `juicefs config --trash-days 0`.\n:::\n\nBesides the features covered by pjdfstest, JuiceFS provides:\n\n- Close-to-open consistency. It ensures that once a file is written and closed, the written data is accessible in the following open and read operations. Within the same mount point, all written data can be read immediately.\n- Rename and all other metadata operations are atomic, guaranteed by the transactional nature of metadata engines.\n- Open files remain accessible after being unlinked from the same mount point.\n- Mmap (tested with FSx).\n- Fallocate with punch hole support.\n- Extended attributes (xattr).\n- BSD locks (flock).\n- POSIX traditional record locks (fcntl).\n\n:::note\nPOSIX record locks are classified as **traditional locks** (\"process-associated\") and **OFD locks** (open file description locks). Their locking operation commands are `F_SETLK` and `F_OFD_SETLK` respectively. Due to the implementation of the FUSE kernel module, JuiceFS currently only supports traditional record locks. More details can be found at: [https://man7.org/linux/man-pages/man2/fcntl.2.html](https://man7.org/linux/man-pages/man2/fcntl.2.html).\n:::\n\n## LTP\n\nLTP is a joint project developed and maintained by IBM, Cisco, Fujitsu, and others.\n\n> The project goal is to deliver tests to the open source community that validates the reliability, robustness, and stability of Linux.\n>\n> The LTP testsuite contains a collection of tools for testing the Linux kernel and related features. Our goal is to improve the Linux kernel and system libraries by bringing test automation to the testing effort.\n\nJuiceFS passed most of the file system related tests.\n\n### Test environment\n\n- Host: Amazon EC2: c5d.xlarge (4C 8G)\n- OS: Ubuntu 20.04.1 LTS (Kernel `5.4.0-1029-aws`)\n- Object storage: Amazon S3\n- JuiceFS version: 0.17-dev (2021-09-16 292f2b65)\n\n### Test steps\n\n1. Download the LTP [release](https://github.com/linux-test-project/ltp/releases/download/20210524/ltp-full-20210524.tar.bz2) from GitHub.\n2. Unarchive, compile, and install LTP:\n\n ```bash\n tar -jvxf ltp-full-20210524.tar.bz2\n cd ltp-full-20210524\n ./configure\n make all\n make install\n ```\n\n3. Change the directory to `/opt/ltp` where the test tools are installed:\n\n ```bash\n cd /opt/ltp\n ```\n\n The test definition files are located under `runtest`. To speed up testing, we delete some pressure cases and unrelated cases in `fs` and `syscalls` (refer to [Appendix](#appendix), modified files are saved as `fs-jfs` and `syscalls-jfs`), then execute:\n\n ```bash\n ./runltp -d /mnt/jfs -f fs_bind,fs_perms_simple,fsx,io,smoketest,fs-jfs,syscalls-jfs\n ```\n\n### Test result\n\n```bash\nTestcase Result Exit Value\n-------- ------ ----------\nfcntl17 FAIL 7\nfcntl17_64 FAIL 7\ngetxattr05 CONF 32\nioctl_loop05 FAIL 4\nioctl_ns07 FAIL 1\nlseek11 CONF 32\nopen14 CONF 32\nopenat03 CONF 32\nsetxattr03 FAIL 6\n\n-----------------------------------------------\nTotal Tests: 1270\nTotal Skipped Tests: 4\nTotal Failures: 5\nKernel Version: 5.4.0-1029-aws\nMachine Architecture: x86_64\n```\n\nHere are causes of the skipped and failed tests:\n\n- fcntl17, fcntl17_64: These tests require the file system to automatically detect deadlocks when trying to add POSIX locks. JuiceFS does not support it yet.\n- getxattr05: This test requires extended ACLs, which are not yet supported by JuiceFS.\n- ioctl_loop05, ioctl_ns07, setxattr03: These tests require `ioctl`, which is not yet supported by JuiceFS.\n- lseek11: This test requires `lseek` to handle `SEEK_DATA` and `SEEK_HOLE` flags. JuiceFS uses a kernel general function, which does not support these two flags.\n- open14, openat03: These tests require `open` to handle the `O_TMPFILE` flag. It is not supported by FUSE and thus not by JuiceFS.\n\n### Appendix\n\nHere are deleted cases in `fs` and `syscalls`:\n\n```bash\n# fs --> fs-jfs\ngf01 growfiles -W gf01 -b -e 1 -u -i 0 -L 20 -w -C 1 -l -I r -T 10 -f glseek20 -S 2 -d $TMPDIR\ngf02 growfiles -W gf02 -b -e 1 -L 10 -i 100 -I p -S 2 -u -f gf03_ -d $TMPDIR\ngf03 growfiles -W gf03 -b -e 1 -g 1 -i 1 -S 150 -u -f gf05_ -d $TMPDIR\ngf04 growfiles -W gf04 -b -e 1 -g 4090 -i 500 -t 39000 -u -f gf06_ -d $TMPDIR\ngf05 growfiles -W gf05 -b -e 1 -g 5000 -i 500 -t 49900 -T10 -c9 -I p -u -f gf07_ -d $TMPDIR\ngf06 growfiles -W gf06 -b -e 1 -u -r 1-5000 -R 0--1 -i 0 -L 30 -C 1 -f g_rand10 -S 2 -d $TMPDIR\ngf07 growfiles -W gf07 -b -e 1 -u -r 1-5000 -R 0--2 -i 0 -L 30 -C 1 -I p -f g_rand13 -S 2 -d $TMPDIR\ngf08 growfiles -W gf08 -b -e 1 -u -r 1-5000 -R 0--2 -i 0 -L 30 -C 1 -f g_rand11 -S 2 -d $TMPDIR\ngf09 growfiles -W gf09 -b -e 1 -u -r 1-5000 -R 0--1 -i 0 -L 30 -C 1 -I p -f g_rand12 -S 2 -d $TMPDIR\ngf10 growfiles -W gf10 -b -e 1 -u -r 1-5000 -i 0 -L 30 -C 1 -I l -f g_lio14 -S 2 -d $TMPDIR\ngf11 growfiles -W gf11 -b -e 1 -u -r 1-5000 -i 0 -L 30 -C 1 -I L -f g_lio15 -S 2 -d $TMPDIR\ngf12 mkfifo $TMPDIR/gffifo17; growfiles -b -W gf12 -e 1 -u -i 0 -L 30 $TMPDIR/gffifo17\ngf13 mkfifo $TMPDIR/gffifo18; growfiles -b -W gf13 -e 1 -u -i 0 -L 30 -I r -r 1-4096 $TMPDIR/gffifo18\ngf14 growfiles -W gf14 -b -e 1 -u -i 0 -L 20 -w -l -C 1 -T 10 -f glseek19 -S 2 -d $TMPDIR\ngf15 growfiles -W gf15 -b -e 1 -u -r 1-49600 -I r -u -i 0 -L 120 -f Lgfile1 -d $TMPDIR\ngf16 growfiles -W gf16 -b -e 1 -i 0 -L 120 -u -g 4090 -T 101 -t 408990 -l -C 10 -c 1000 -S 10 -f Lgf02_ -d $TMPDIR\ngf17 growfiles -W gf17 -b -e 1 -i 0 -L 120 -u -g 5000 -T 101 -t 499990 -l -C 10 -c 1000 -S 10 -f Lgf03_ -d $TMPDIR\ngf18 growfiles -W gf18 -b -e 1 -i 0 -L 120 -w -u -r 10-5000 -I r -l -S 2 -f Lgf04_ -d $TMPDIR\ngf19 growfiles -W gf19 -b -e 1 -g 5000 -i 500 -t 49900 -T10 -c9 -I p -o O_RDWR,O_CREAT,O_TRUNC -u -f gf08i_ -d $TMPDIR\ngf20 growfiles -W gf20 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 1-256000:512 -R 512-256000 -T 4 -f gfbigio-$$ -d $TMPDIR\ngf21 growfiles -W gf21 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -T 10 -t 20480 -f gf-bld-$$ -d $TMPDIR\ngf22 growfiles -W gf22 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -T 10 -t 20480 -f gf-bldf-$$ -d $TMPDIR\ngf23 growfiles -W gf23 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 512-64000:1024 -R 1-384000 -T 4 -f gf-inf-$$ -d $TMPDIR\ngf24 growfiles -W gf24 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -f gf-jbld-$$ -d $TMPDIR\ngf25 growfiles -W gf25 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 1024000-2048000:2048 -R 4095-2048000 -T 1 -f gf-large-gs-$$ -d $TMPDIR\ngf26 growfiles -W gf26 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 128-32768:128 -R 512-64000 -T 4 -f gfsmallio-$$ -d $TMPDIR\ngf27 growfiles -W gf27 -b -D 0 -w -g 8b -C 1 -b -i 1000 -u -f gfsparse-1-$$ -d $TMPDIR\ngf28 growfiles -W gf28 -b -D 0 -w -g 16b -C 1 -b -i 1000 -u -f gfsparse-2-$$ -d $TMPDIR\ngf29 growfiles -W gf29 -b -D 0 -r 1-4096 -R 0-33554432 -i 0 -L 60 -C 1 -u -f gfsparse-3-$$ -d $TMPDIR\ngf30 growfiles -W gf30 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -o O_RDWR,O_CREAT,O_SYNC -g 20480 -T 10 -t 20480 -f gf-sync-$$ -d $TMPDIR\nrwtest01 export LTPROOT; rwtest -N rwtest01 -c -q -i 60s -f sync 10%25000:$TMPDIR/rw-sync-$$\nrwtest02 export LTPROOT; rwtest -N rwtest02 -c -q -i 60s -f buffered 10%25000:$TMPDIR/rw-buffered-$$\nrwtest03 export LTPROOT; rwtest -N rwtest03 -c -q -i 60s -n 2 -f buffered -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-buff-$$\nrwtest04 export LTPROOT; rwtest -N rwtest04 -c -q -i 60s -n 2 -f sync -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-sync-$$\nrwtest05 export LTPROOT; rwtest -N rwtest05 -c -q -i 50 -T 64b 500b:$TMPDIR/rwtest01%f\niogen01 export LTPROOT; rwtest -N iogen01 -i 120s -s read,write -Da -Dv -n 2 500b:$TMPDIR/doio.f1.$$ 1000b:$TMPDIR/doio.f2.$$\nquota_remount_test01 quota_remount_test01.sh\nisofs isofs.sh\n\n# syscalls --> syscalls-jfs\nbpf_prog05 bpf_prog05\ncacheflush01 cacheflush01\nchown01_16 chown01_16\nchown02_16 chown02_16\nchown03_16 chown03_16\nchown04_16 chown04_16\nchown05_16 chown05_16\nclock_nanosleep03 clock_nanosleep03\nclock_gettime03 clock_gettime03\nleapsec01 leapsec01\nclose_range01 close_range01\nclose_range02 close_range02\nfallocate06 fallocate06\nfchown01_16 fchown01_16\nfchown02_16 fchown02_16\nfchown03_16 fchown03_16\nfchown04_16 fchown04_16\nfchown05_16 fchown05_16\nfcntl06 fcntl06\nfcntl06_64 fcntl06_64\ngetegid01_16 getegid01_16\ngetegid02_16 getegid02_16\ngeteuid01_16 geteuid01_16\ngeteuid02_16 geteuid02_16\ngetgid01_16 getgid01_16\ngetgid03_16 getgid03_16\ngetgroups01_16 getgroups01_16\ngetgroups03_16 getgroups03_16\ngetresgid01_16 getresgid01_16\ngetresgid02_16 getresgid02_16\ngetresgid03_16 getresgid03_16\ngetresuid01_16 getresuid01_16\ngetresuid02_16 getresuid02_16\ngetresuid03_16 getresuid03_16\ngetrusage04 getrusage04\ngetuid01_16 getuid01_16\ngetuid03_16 getuid03_16\nioctl_sg01 ioctl_sg01\nfanotify16 fanotify16\nfanotify18 fanotify18\nfanotify19 fanotify19\nlchown01_16 lchown01_16\nlchown02_16 lchown02_16\nlchown03_16 lchown03_16\nmbind02 mbind02\nmbind03 mbind03\nmbind04 mbind04\nmigrate_pages02 migrate_pages02\nmigrate_pages03 migrate_pages03\nmodify_ldt01 modify_ldt01\nmodify_ldt02 modify_ldt02\nmodify_ldt03 modify_ldt03\nmove_pages01 move_pages01\nmove_pages02 move_pages02\nmove_pages03 move_pages03\nmove_pages04 move_pages04\nmove_pages05 move_pages05\nmove_pages06 move_pages06\nmove_pages07 move_pages07\nmove_pages09 move_pages09\nmove_pages10 move_pages10\nmove_pages11 move_pages11\nmove_pages12 move_pages12\nmsgctl05 msgctl05\nmsgstress04 msgstress04\nopenat201 openat201\nopenat202 openat202\nopenat203 openat203\nmadvise06 madvise06\nmadvise09 madvise09\nptrace04 ptrace04\nquotactl01 quotactl01\nquotactl04 quotactl04\nquotactl06 quotactl06\nreaddir21 readdir21\nrecvmsg03 recvmsg03\nsbrk03 sbrk03\nsemctl08 semctl08\nsemctl09 semctl09\nset_mempolicy01 set_mempolicy01\nset_mempolicy02 set_mempolicy02\nset_mempolicy03 set_mempolicy03\nset_mempolicy04 set_mempolicy04\nset_thread_area01 set_thread_area01\nsetfsgid01_16 setfsgid01_16\nsetfsgid02_16 setfsgid02_16\nsetfsgid03_16 setfsgid03_16\nsetfsuid01_16 setfsuid01_16\nsetfsuid02_16 setfsuid02_16\nsetfsuid03_16 setfsuid03_16\nsetfsuid04_16 setfsuid04_16\nsetgid01_16 setgid01_16\nsetgid02_16 setgid02_16\nsetgid03_16 setgid03_16\nsgetmask01 sgetmask01\nsetgroups01_16 setgroups01_16\nsetgroups02_16 setgroups02_16\nsetgroups03_16 setgroups03_16\nsetgroups04_16 setgroups04_16\nsetregid01_16 setregid01_16\nsetregid02_16 setregid02_16\nsetregid03_16 setregid03_16\nsetregid04_16 setregid04_16\nsetresgid01_16 setresgid01_16\nsetresgid02_16 setresgid02_16\nsetresgid03_16 setresgid03_16\nsetresgid04_16 setresgid04_16\nsetresuid01_16 setresuid01_16\nsetresuid02_16 setresuid02_16\nsetresuid03_16 setresuid03_16\nsetresuid04_16 setresuid04_16\nsetresuid05_16 setresuid05_16\nsetreuid01_16 setreuid01_16\nsetreuid02_16 setreuid02_16\nsetreuid03_16 setreuid03_16\nsetreuid04_16 setreuid04_16\nsetreuid05_16 setreuid05_16\nsetreuid06_16 setreuid06_16\nsetreuid07_16 setreuid07_16\nsetuid01_16 setuid01_16\nsetuid03_16 setuid03_16\nsetuid04_16 setuid04_16\nshmctl06 shmctl06\nsocketcall01 socketcall01\nsocketcall02 socketcall02\nsocketcall03 socketcall03\nssetmask01 ssetmask01\nswapoff01 swapoff01\nswapoff02 swapoff02\nswapon01 swapon01\nswapon02 swapon02\nswapon03 swapon03\nswitch01 endian_switch01\nsysinfo03 sysinfo03\ntimerfd04 timerfd04\nperf_event_open02 perf_event_open02\nstatx07 statx07\nio_uring02 io_uring02\n```\n" }, { "path": "docs/en/reference/redis-csc.md", "content": "# Redis Client-Side Caching Support in JuiceFS\r\n\r\nStarting with version 6.0, Redis provides [Client-Side Caching](https://redis.io/docs/latest/develop/reference/client-side-caching) which allows clients to maintain local caches of data in a faster and more efficient way. JuiceFS includes full support for this feature, offering significant performance improvements for metadata operations.\r\n\r\n## How it works\r\n\r\nRedis Client-Side Caching (CSC) works by:\r\n\r\n1. The client enables tracking mode with `CLIENT TRACKING ON BCAST`\r\n2. The client caches data locally after reading it from Redis\r\n3. Redis notifies the client when cached keys are modified by any client\r\n4. The client invalidates those keys in its local cache\r\n\r\nThis results in reduced network traffic, lower latency, and higher throughput.\r\n\r\n## Configuration\r\n\r\nJuiceFS supports Redis CSC through the following options in the metadata URL:\r\n\r\n```shell\r\n--meta-url=\"redis://localhost/1?client-cache=true\" # Enable client-side caching (always BCAST mode) \r\n--meta-url=\"redis://localhost/1?client-cache=true&client-cache-size=500\" # Set cache size (default 12800) \r\n--meta-url=\"redis://localhost/1?client-cache=true&client-cache-expire=60s\" # Set cache expiration (default: 60s)\r\n```\r\n\r\n### Options\r\n\r\n- `client-cache`: Enables client-side caching in BCAST mode (set to any value except \"false\")\r\n- `client-cache-size`: Maximum cache size (default: 12800)\r\n- `client-cache-expire`: Cache expiration time (default: 60s)\r\n- `client-cache-preload`: Number of file objects under the root directory preloaded after mounting. (default: 0)\r\n\r\nWhen client-side caching is enabled, JuiceFS caches:\r\n\r\n1. **Inode attributes**: File/directory metadata like permissions, size, timestamps\r\n2. **Directory entries**: Name to inode mappings for faster lookups\r\n\r\n> **Note:** Redis Client Side Cache requires Redis server version 6.0 or higher. Using this feature with older Redis versions will result in errors.\r\n\r\n### Preloading Cache\r\n\r\nWhen client-side caching is enabled and `client-cache-preload` is set, JuiceFS will preload the file-object attributes and entries under the root directory after mounting. This lazy preloading happens in the background and helps to:\r\n\r\n1. Warm up the cache for common operations\r\n2. Reduce latency for initial file system operations\r\n3. Provide better performance from the moment the file system is mounted\r\n\r\nThe preloading process intelligently prioritizes the most important inodes by:\r\n\r\n1. Starting with the root directory\r\n2. Loading the most frequently accessed top-level directories and files\r\n3. Recursively exploring important subdirectories\r\n\r\nThe preloading process runs in a background goroutine with fail-safe mechanisms and won't block or affect normal file system operations.\r\n\r\n## Modes\r\n\r\nJuiceFS uses BCAST mode for simplicity and reliability:\r\n\r\n- **BCAST mode**: All keys accessed by the client are tracked and notifications are sent for any changes.\r\n\r\nBCAST mode provides the simplest implementation while ensuring cache coherence across all clients.\r\n\r\n## Requirements\r\n\r\n- Redis server version 6.0 or higher\r\n- JuiceFS with CSC support enabled\r\n\r\n## Performance Considerations\r\n\r\n1. The default 12800 cache size should be sufficient for most workloads\r\n2. For very large filesystems with millions of files, you may benefit from increasing the cache size\r\n3. The cache is most effective for metadata-heavy workloads with many repeated operations\r\n4. For very write-heavy workloads, consider disabling CSC as invalidation traffic may offset benefits\r\n\r\n## Troubleshooting\r\n\r\nIf you experience crashes or instability with CSC enabled:\r\n\r\n1. Update to the latest JuiceFS version which contains important fixes for CSC\r\n2. Try reducing the cache size with `client-cache-size`\r\n3. Check Redis server logs for any memory or client tracking issues\r\n4. Make sure your Redis server version is 6.0 or higher\r\n5. If problems persist, disable CSC by removing the `client-cache` parameter\r\n\r\nJuiceFS includes robust error handling for various Redis CSC-specific responses to ensure stable operation even when Redis sends unexpected response formats due to client tracking.\r\n\r\n## References\r\n\r\n- [Redis Client-Side Caching Documentation](https://redis.io/docs/latest/develop/reference/client-side-caching)\r\n" }, { "path": "docs/en/reference/spec-limits.md", "content": "---\nsidebar_position: 7\n---\n\n# Specification Limits\n\n## File System Limits\n\nBelow are theoretical limits for JuiceFS, in real use, performance and file system size will be limited by the metadata engine and object storage of your choice.\n\n* Directory tree depth: unlimited\n* File name length: 255 Bytes\n* Symbolic link length: 4096 Bytes\n* Number of hard links: 2^31\n* Number of files in single directory: 2^31\n* Number of files in a single volume: unlimited\n* Single file size: 2^(26+31)\n* Total file size: 4EiB\n" }, { "path": "docs/en/release_notes.md", "content": "# Release Notes\n\n:::tip\nFor all versions, please see [GitHub Releases](https://github.com/juicedata/juicefs/releases).\n:::\n\n## Version number {#version-number}\n\nJuiceFS Community Edition uses [semantic versioning](https://semver.org) to label its releases. Each version number consists of three numbers in the format `x.y.z`, representing the major version number (x), the minor version number (y), and the patch number (z).\n\n1. **Major version number (x)**: When the major version number is greater than or equal to `1`, it indicates that the version is suitable for production environments. When the major version number changes, it indicates that this version may have added major features, architectural changes, or data format changes that are not backward compatible. For example, `v0.8.3` → `v1.0.0` means production-ready, `v1.0.0` → `v2.0.0` represents an architectural or functional change.\n2. **Minor version number (y)**: The minor version number indicates that the version adds some new features, performance optimizations, bug fixes, etc. that can be backward compatible. For example, `v1.0.0` → `v1.1.0`.\n3. **Patch version number (z)**: The patch version number indicates a minor update or bug fix for the software, which is only some minor changes or fixes to existing features and will not affect the compatibility of the softwares. For example, `v1.0.3` → `v1.0.4`.\n\n## Upgrade {#changes}\n\nJuiceFS client has only one binary file, so usually you only need to replace the old binary with the new one when upgrading JuiceFS.\n\n### JuiceFS v1.1\n\n:::tip\nIf you are using JuiceFS version prior to v1.0, please [upgrade to v1.0](#juicefs-v10) first.\n:::\n\nIn v1.1 (specifically, v1.1.0-beta2) JuiceFS added [**Directory Statistics**](https://juicefs.com/docs/community/guide/dir-stats) and [**Directory Quota**](https://juicefs.com/docs/community/guide/quota#directory-quota). These two features were not available in older versions of the client, and writing with the old client when they were turned on would result in large deviations in the statistics. When upgrading to v1.1, if you do not intend to enable these two new features, you can simply replace the client without additional action. If you do, it is recommended that you read the following content before upgrading.\n\n#### Default configuration\n\nThe default configurations for these two features are:\n\n- For newly created filesystems they are automatically enabled.\n\n- For existing filesystems, they are disabled.\n - Directory statistics can be enabled independently by `juicefs config` command.\n - When setting directory quotas the directory statistics will be enabled automatically.\n\n#### Recommended Upgrade Steps\n\n1. Upgrade all client binaries to v1.1 version.\n2. Deny re-connections from versions prior to v1.1: `juicefs config META-URL --min-client-version 1.1.0-A`.\n3. Restart the service at a proper time (remount, restart gateway, etc.)\n4. Make sure that all online clients are version v1.1 or higher: `juicefs status META-URL | grep -w Version`\n5. Enable the new features, see [Directory Statistics](https://juicefs.com/docs/community/guide/dir-stats) and [Directory Quota](https://juicefs.com/docs/community/guide/quota#directory-quota).\n\n### JuiceFS v1.0\n\nJuiceFS has two compatibility changes in version v1.0 (specifically, v1.0.0-beta3). If you are using an older version of the client, it is recommended that you read the following content before upgrading.\n\n#### SQL: Update table schema to support encoding other than UTF-8\n\nJuiceFS v1.0 has changed the table schema to support encoding other than UTF-8. For existing file systems, you need to upgrade the table schema manually to support that. It's recommended to upgrade all clients first and then the table schema.\n\n:::note\nTable schema upgrades are optional, and they are required only if you need to use non-UTF-8 characters. In addition, database performance may degrade when upgrading SQL table schemas, affecting running services.\n:::\n\n##### MySQL/MariaDB\n\n```sql\nalter table jfs_edge\n modify name varbinary(255) not null;\nalter table jfs_symlink\n modify target varbinary(4096) not null;\n```\n\n##### PostgreSQL\n\n```sql\nalter table jfs_edge\n alter column name type bytea using name::bytea;\nalter table jfs_symlink\n alter column target type bytea using target::bytea;\n```\n\n##### SQLite\n\nSQLite does not support modifying columns, but you can migrate columns by `dump` and `load` commands, refer to [JuiceFS Metadata Backup and Recovery](administration/metadata_dump_load.md) for details.\n\n#### New session management format\n\nJuiceFS v1.0 uses a new session management format. The previous versions of clients cannot see the sessions generated by v1.0 clients via `juicefs status` or `juicefs destroy`, whereas the new versions are able to see all the sessions.\n" }, { "path": "docs/en/security/encryption.md", "content": "---\nsidebar_position: 1\n---\n# Data Encryption\n\nJuiceFS provides data encryption from two aspects:\n\n1. Data Encryption In Transit\n2. Data Encryption At Rest\n\n## Data Encryption In Transit {#in-transit}\n\nRunning JuiceFS generally involves the network connection between database and object storage, which is determined by the architecture of JuiceFS. As long as the servers support encryption connections, JuiceFS can be accessed through the encrypted channel.\n\n### Connect to object storage via HTTPS\n\nPublic cloud object storage generally supports both HTTP and HTTPS. If no scheme is specified, JuiceFS uses HTTPS by default. For example, the client will identify the bucket in following command as `https://myjfs.s3.ap-southeast-1.amazonaws.com`.\n\n```shell {2}\njuicefs format --storage s3 \\\n --bucket myjfs.s3.ap-southeast-1.amazonaws.com \\\n ...\n```\n\nWith the above command, the client will recognize the bucket as `https://myjfs.s3.ap-southeast-1.amazonaws.com`.\n\nIn the case where server and object storage run on the same VPC network, explicitly set the URL scheme to `http` if you don't need an encrypted connection, e.g., `--bucket http://myjfs.s3.ap-southeast-1.amazonaws.com`.\n\n### Connect to database via TLS/SSL\n\nFor [all the supported metadata engines](../reference/how_to_set_up_metadata_engine.md), as long as the database supports encryption and has been configured with encryption such as TLS/SSL, JuiceFS can connect to the database through its encrypted channel. For instance, a Redis database configured with TLS can use `rediss://` for connecting.\n\n```shell {3}\njuicefs format --storage s3 \\\n --bucket myjfs.s3.ap-southeast-1.amazonaws.com \\\n \"rediss://myredis.ap-southeast-1.amazonaws.com:6379/1\" myjfs\n```\n\n## Data Encryption At Rest {#at-rest}\n\nJuiceFS provides Data Encryption At Rest support, which encrypts first, then uploads. All files stored in JuiceFS will be encrypted locally and then uploaded to object storage, effectively preventing data leakage when the object storage itself is compromised.\n\nJuiceFS Data Encryption At Rest adopts a hybrid encryption architecture: symmetric encryption handles data encryption, while asymmetric encryption handles key protection. You only need to provide an private key when creating the file system to enable data encryption functionality, and provide the private key password through the `JFS_RSA_PASSPHRASE` environment variable. In usage, the mount point is completely transparent to applications, meaning the encryption and decryption processes will not affect file system access.\n\n:::caution\nThe cached data on the client-side is **NOT** encrypted. Only the root user or owner can access this data. To encrypt the cached data, you can put the cached directory in an encrypted file system or block storage.\n:::\n\n### Encryption Principles\n\n#### Encryption Architecture Design\n\nJuiceFS adopts a **hybrid encryption architecture** with two encryption layers:\n\n1. **Data Encryption Layer** (Symmetric Encryption - AES-256-GCM or ChaCha20-Poly1305 or SM4-GCM)\n - **Purpose**: Actually encrypts user data content\n - **Mechanism**: Each block generates a unique symmetric key `S` + random seed `N` (both use 256-bit keys)\n - **Advantage**: Both AES-256-GCM and ChaCha20-Poly1305 provide high-speed encryption and integrity verification (AEAD)\n - **Standard**: 256-bit key strength complies with NIST security standards, ChaCha20-Poly1305 is an RFC 8439 standard algorithm\n\n2. **Key Protection Layer** (Asymmetric Encryption)\n - **Purpose**: Protects the secure distribution and storage of symmetric keys\n - **Mechanism**: Uses private key `M` to encrypt each data block's symmetric key `S`\n - **Advantage**: Solves key distribution challenges and avoids key reuse risks\n - **Scheme**: Supports private keys in PKCS#1 or PKCS#8 formats.\n\nUsers need to create a global private key `M` for the file system in advance. Each object stored in the object storage will have its own random symmetric key `S`.\n\nSymbol explanation:\n\n- `M` represents private key created by user\n- `S` represents 256-bit symmetric key generated by the JuiceFS for each file object\n- `N` represents random seed generated by the JuiceFS for each file object\n- `K` represents the cipher text of `S` encrypted with private key `M`\n\n![Encryption At-rest](../images/encryption.png)\n\n#### Data Encryption Process\n\n- Before writing to object storage, data blocks are compressed using LZ4 or Zstandard.\n- A random 256-bit symmetric key `S` and a random seed `N` are generated for each data block.\n- Each data block is encrypted into `encrypted_data` using AES-256-GCM or ChaCha20-Poly1305 or SM4-GCM algorithm with key `S` and seed `N`.\n- To avoid the symmetric key `S` from being transmitted in clear text over the network, the symmetric key `S` is encrypted into the cipher text `K` with the RSA private key `M`.\n- The encrypted data `encrypted_data`, the ciphertext `K`, and the random seed `N` are combined into an object and then written to the object storage.\n\n#### Data Decryption Process\n\n- Read the entire encrypted object (it may be a bit larger than 4MB).\n- Parse the object data to get the ciphertext `K`, the random seed `N`, and the encrypted data `encrypted_data`.\n- Decrypt `K` with private key to get symmetric key `S`.\n- Decrypt the data `encrypted_data` based on AES-256-GCM or ChaCha20-Poly1305 or SM4-GCM using `S` and `N` to get the data block plaintext.\n- Decompress the data block.\n\n### Enable Data Encryption At Rest\n\n:::note\nData Encryption At Rest must be enabled when creating file system. The file system that was created without Data Encryption At Rest enabled cannot enable it later.\n:::\n\nThe steps to enable Data Encryption At Rest are:\n\n1. Create a private key\n2. Create an encrypted file system using the private key\n3. Mount the file system\n\n#### Step 1: Create a private key\n\nThe private key is crucial for Data Encryption At Rest and is generally manually generated using OpenSSL. The following command will generate a 2048-bit RSA private key named `my-priv-key.pem` in the current directory using the aes256 algorithm:\n\n```shell\nopenssl genrsa -out my-priv-key.pem -aes256 2048\n```\n\nSince the `aes256` encryption algorithm is used, the command line will require you to provide a `Passphrase` of at least 4 characters for this private key. You can simply think of it as a password used to encrypt the RSA private key file itself, which is also the last security safeguard for the RSA private key file.\n\n:::caution Special Attention\nThe security of the private key is extremely important, and special attention needs to be paid to the following points:\n\n- **Passphrase Leakage Risk**: If the private key's passphrase is leaked, attackers may decrypt the private key stored in the metadata engine, thereby jeopardizing the security of all encrypted data\n- **Private Key File Leakage**: If the encrypted private key file itself is leaked along with the passphrase, it will lead to serious security risks\n- **Data Irrecoverability**: If the correct passphrase cannot be provided to access the private key stored in the metadata engine, **all encrypted data will be permanently lost and unrecoverable**\n\nIt is recommended to focus on protecting the security of the passphrase and pass it through environment variables to avoid leakage in command line history.\n:::\n\n#### Step 2: Create an encrypted file system\n\nCreating an encrypted file system requires using the `--encrypt-rsa-key` option to specify the private key. The provided private key content will be written to the metadata engine. You need to use the environment variable `JFS_RSA_PASSPHRASE` to specify the private key's passphrase.\n\nJuiceFS supports two encryption algorithm combinations, which can be specified via the `--encrypt-algo` option:\n\n- `aes256gcm-rsa` (default): Uses AES-256-GCM + RSA (or other private key)\n- `chacha20-rsa`: Uses ChaCha20-Poly1305 + RSA (or other private key)\n- `sm4gcm`: Uses SM4-GCM + SM2 (or other private key)\n\n1. Set passphrase using environment variable\n\n ```shell\n export JFS_RSA_PASSPHRASE=the-passwd-for-rsa\n ```\n\n2. Create file system (using default AES-256-GCM encryption)\n\n ```shell {2}\n juicefs format --storage s3 \\\n --encrypt-rsa-key my-priv-key.pem \\\n ...\n ```\n\n Or explicitly specify ChaCha20-Poly1305 encryption:\n\n ```shell {2,3}\n juicefs format --storage s3 \\\n --encrypt-rsa-key my-priv-key.pem \\\n --encrypt-algo chacha20-rsa \\\n ...\n ```\n\n3. (Optional) Delete local private key file\n\n JuiceFS securely stores the private key content in the metadata engine during file system formatting. Therefore, after completing file system creation (unless there are specific compliance requirements), we recommend deleting your local private key file:\n\n ```shell\n rm my-priv-key.pem\n ```\n\n This way, you only need to ensure the security of the `JFS_RSA_PASSPHRASE` environment variable, and subsequent file system mounting and access only require providing the correct passphrase.\n\n If you need to retain the private key file due to compliance requirements or other reasons, please ensure the private key file is stored in a secure location with strict access permissions, and keep the private key file and passphrase separately.\n\n#### Step 3: Mount file system\n\nThere is no need to specify extra options while mounting an encrypted file system. However, the passphrase of the private key needs to be set before mounting using environment variable.\n\n1. Set passphrase using environment variable\n\n ```shell\n export JFS_RSA_PASSPHRASE=the-passwd-for-rsa\n ```\n\n2. Mount file system\n\n ```shell\n juicefs mount redis://127.0.0.1:6379/1 /mnt/myjfs\n ```\n\n### Performance Considerations\n\nEnabling encryption does introduce some performance overhead, but modern hardware technologies have made this impact quite manageable. The specific performance impact depends on workload type, hardware configuration (particularly CPU encryption instruction set support), and data access patterns.\n\nModern CPUs have specialized hardware optimizations for TLS, HTTPS, and AES-256 encryption technologies. In particular, modern Intel and AMD processors include AES-NI instruction sets that can perform AES encryption operations at near-native speeds, significantly reducing the performance impact of data encryption.\n\n#### Encryption Algorithm Selection Recommendations\n\n**AES-256-GCM** (default choice):\n\n- Excellent performance on modern CPUs with AES-NI instruction set support\n- Widely supported and validated industry standard\n- Suitable for most production environments\n\n**ChaCha20-Poly1305**:\n\n- May provide better performance on CPUs without AES-NI support\n- Suitable for ARM architectures or older x86 processors\n- Better resistance against timing attacks\n- Preferred algorithm by companies like Google for mobile devices and certain server environments\n\nWhen selecting encryption keys, we recommend using RSA-2048 keys, which provide a good balance between security strength and performance. RSA-4096 provides higher security, but its decryption operations are slower and may impact performance in high-concurrency read scenarios.\n\nIt's worth mentioning that encrypted data will be slightly larger than the original data, primarily because both AES-256-GCM and ChaCha20-Poly1305 encryption algorithms require adding authentication tags (16 bytes) and other encryption metadata.\n\n### Security Best Practices\n\nThe security of an encryption scheme depends not only on the algorithms themselves but also on how encryption keys are properly managed and used. Here are some important security practice recommendations:\n\n**Key management is at the core of security**. The passphrase you set for your private key should be strong enough—we recommend using at least 16 characters with a combination of uppercase and lowercase letters, numbers, and special symbols. We recommend passing the passphrase through environment variables to avoid leakage in command line history.\n\nWhile regularly rotating keys is a good practice, it's important to note that changing private keys requires reformatting the entire file system. Therefore, when planning key rotation strategies, you need to balance security requirements with business continuity.\n\n**Access control is equally important**. Ensure your metadata engine (whether Redis, MySQL, or another database) is configured with appropriate authentication and authorization mechanisms. Object storage access permissions should also follow the principle of least privilege, granting only necessary operational permissions.\n\nAt the network level, try to use VPC or private networks to isolate communication traffic between the metadata engine and object storage, reducing the risk of man-in-the-middle attacks.\n\n**Monitoring and auditing** can help you detect abnormal situations promptly. We recommend logging all encryption-related operations, regularly checking key usage patterns, and establishing abnormal access detection mechanisms. This way, even if a security incident occurs, you can respond quickly and take appropriate measures.\n\n### Important Considerations\n\nWhen using JuiceFS encryption features, there are several important technical limitations to be aware of:\n\nFirst, client-side local cached data is **NOT encrypted**. Although only root users or file owners can access this cached data, if your use case requires end-to-end full encryption, you'll need to consider additional protection measures, such as placing the cache directory on an encrypted file system or block storage.\n\nSecondly, encryption functionality has some inherent limitations. File metadata (such as filenames, sizes, permissions, etc.) is not encrypted, and decrypted data exists in plaintext in memory. Most importantly, once encryption is enabled for a file system, it cannot be turned off—encryption is an irreversible operation.\n\nIn deployment planning, please consider that encryption brings additional CPU and memory overhead. To ensure optimal compatibility and stability, we recommend that all clients accessing encrypted file systems use the same or compatible versions of JuiceFS.\n\n### Usage Scenario Analysis\n\nJuiceFS encryption features are particularly suitable for these scenarios: protecting sensitive data in cloud object storage, meeting compliance requirements such as GDPR and HIPAA, long-term secure storage of important business data, and achieving data isolation in multi-tenant environments.\n\nHowever, if you need client-side local cache encryption, or want to add encryption functionality to existing file systems later, this solution may not be suitable. Similarly, for applications with extremely demanding performance requirements, or scenarios that require frequent key rotation but cannot accept reformatting, careful consideration is needed.\n" }, { "path": "docs/en/security/posix_acl.md", "content": "---\ntitle: POSIX ACLs\ndescription: Learn about POSIX ACL support in JuiceFS and how to enable and use ACL permissions.\nsidebar_position: 1\n---\n\nPOSIX ACLs (Portable Operating System Interface for Unix - Access Control Lists) are an access control mechanism in Unix-like operating systems that allows for finer-grained control over file and directory access permissions.\n\nThis document introduces how to enable and use POSIX ACL permissions in JuiceFS.\n\n## Versions and compatibility requirements\n\n* Since version 1.2, JuiceFS has supported POSIX ACLs.\n* All client versions can mount volumes without ACLs enabled, regardless of their creation by new or old client versions.\n* Once ACLs are enabled, they cannot be disabled. Therefore, the `--enable-acl` option is tied to the volume.\n\n:::caution\nIf you plan to use ACL functionality, it is recommended to upgrade all clients to the latest version to avoid potential issues with older versions affecting the accuracy of ACLs.\n:::\n\n## Enable ACLs\n\nAs mentioned earlier, you can enable ACLs when creating a new volume or on an existing volume using a new version of the client.\n\n### Create a new volume and enable ACLs\n\nExecute the following command to create a new volume and enable ACLs:\n\n```shell\njuicefs format --enable-acl sqlite3://myjfs.db myjfs\n```\n\n### Enable ACLs on an existing volume\n\nUse the `config` command to enable ACL functionality on an existing volume:\n\n```\njuicefs config --enable-acl sqlite3://myjfs.db\n```\n\n## Usage\n\nTo set ACL permissions for a file or directory, you can use the `setfacl` command, for example:\n\n```\nsetfacl -m u:alice:rw- /mnt/jfs/file\n```\n\nFor detailed rules, guidelines, and implementation of POSIX ACLs, see:\n\n* [POSIX Access Control Lists on Linux](https://www.usenix.org/legacy/publications/library/proceedings/usenix03/tech/freenix03/full_papers/gruenbacher/gruenbacher_html/main.html)\n* [setfacl](https://linux.die.net/man/1/setfacl)\n* [How We Optimized ACL Implementation for Minimal Performance Impact](https://juicefs.com/en/blog/engineering/access-control-list)\n\n## Notes\n\n* ACL permission checks require [Linux kernel 4.9](https://lkml.iu.edu/hypermail/linux/kernel/1610.0/01531.html) or later.\n* Enabling ACLs may impact performance. However, due to memory cache optimization, most usage scenarios experience minimal performance degradation.\n" }, { "path": "docs/en/security/trash.md", "content": "---\nsidebar_position: 2\n---\n# Trash\n\n:::note\nThis feature requires at least JuiceFS v1.0.0, for previous versions, you need to upgrade all JuiceFS clients, and then enable trash using the `config` subcommand, introduced in below sections.\n:::\n\nJuiceFS enables the trash feature by default, files deleted will be moved in a hidden directory named `.trash` under the file system root, and kept for specified period of time before expiration. Until actual expiration, file system usage (check using `df -h`) will not change, this is also true with the corresponding object storage data.\n\nWhen using `juicefs format` command to initialize JuiceFS volume, users are allowed to specify `--trash-days ` to set the number of days which files are kept in the `.trash` directory. Within this period, user-removed files are not actually purged, so the file system usage shown in the output of `df` command will not decrease, and the blocks in the object storage will still exist.\n\nTo control the expiration settings, use the [`--trash-days`](../reference/command_reference.mdx#format) option which is available for both `juicefs format` and `juicefs config`:\n\n```shell\n# Creating a new file system\njuicefs format META-URL myjfs --trash-days=7\n\n# Modify an existing file system\njuicefs config META-URL --trash-days=7\n\n# Set to 0 to disable Trash\njuicefs config META-URL --trash-days=0\n```\n\nIn addition, the automatic cleaning of the trash relies on the background job of the JuiceFS client. To ensure that the background job can be executed properly, at least one online mount point is required, and the [`--no-bgjob`](../reference/command_reference.mdx#mount-metadata-options) parameter should not be used when mounting the file system.\n\n## Recover files {#recover}\n\nWhen files are deleted, they will be moved to a directory that takes up the format of `.trash/YYYY-MM-DD-HH/[parent inode]-[file inode]-[file name]`, where `YYYY-MM-DD-HH` is the UTC time of the deletion. You can locate the deleted files and recover them if you remember when they are deleted.\n\nIf you have found the desired files in Trash, you can recover them using `mv`:\n\n```shells\nmv .trash/2022-11-30-10/[parent inode]-[file inode]-[file name] .\n```\n\nFiles within the Trash directory lost all their directory structure information, and are stored in a \"flatten\" style, however the parent directory inode is preserved in the file name, if you have forgotten the file name, look for parent directory inode using [`juicefs info`](../reference/command_reference.mdx#info), and then track down the desired files.\n\nAssuming the mount point being `/jfs`, and you've accidentally deleted `/jfs/data/config.json`, but you cannot directly recover this `config.json` because you've forgotten its name, use the following procedure to locate the parent directory inode, and then locate the corresponding trash files.\n\n```shell\n# Use the info subcommand to locate the parent directory inode\njuicefs info /jfs/data\n\n# Note the \"inode\" field in above output, assuming the inode of /jfs/data is 3\n# Find all its files within the Trash directory using the find command\nfind /jfs/.trash -name '3-*'\n\n# Recover all files under that directory\nmv /jfs/.trash/2022-11-30-10/3-* /jfs/data\n```\n\nKeep in mind that only the root user have write access to the Trash directory, so the method introduced above is only available to the root user. If a normal user happens to have read permission to these deleted files, they can also recover them via a read-only method like `cp`, although this obviously wastes storage capacity.\n\nIf you accidentally delete a complicated structured directory, using solely `mv` to recover can be a disaster, for example:\n\n```shell\n$ tree data\ndata\n├── app1\n│   └── config\n│   └── config.json\n└── app2\n └── config\n └── config.json\n\n# Delete the above complicated data directory\n$ juicefs rmr data\n\n# Files will be flattened inside the Trash directory\n$ tree .trash/2023-08-14-05\n.trash/2023-08-14-05\n├── 1-12-data\n├── 12-13-app1\n├── 12-15-app2\n├── 13-14-config\n├── 14-17-config.json\n├── 15-16-config\n└── 16-18-config.json\n```\n\nTo resolve such inconvenience, JuiceFS v1.1 provides the [`restore`](../reference/command_reference.mdx#restore) subcommand to quickly restore deleted files, while preserving its original directory structure. Run this procedure as follows:\n\n```shell\n# Run the restore command to reconstruct directory structure within the Trash\n$ juicefs restore $META_URL 2023-08-14-05\n\n# Preview the rebuilt directory structure, and determine the recovery scope\n# You can either recover the entire directory using the below --put-back command, or just a subdir using mv\n$ tree .trash/2023-08-14-05\n.trash/2023-08-14-05\n└── 1-12-data\n ├── app1\n │   └── config\n │   └── config.json\n └── app2\n └── config\n └── config.json\n\n# Add --put-back to recover deleted files\njuicefs restore $META_URL 2023-08-14-05 --put-back\n```\n\n## Permanently delete files {#purge}\n\nWhen files in the trash directory reach their expiration time, they will be automatically cleaned up. It is important to note that the file cleaning is performed by the background job of the JuiceFS client, which is scheduled to run every hour by default. Therefore, when there are a large number of expired files, the cleaning speed of the object storage may not be as fast as expected, and it may take some time to see the change in storage capacity.\n\nIf you want to permanently delete files before their expiration time, you need to have `root` privileges and use [`juicefs rmr`](../reference/command_reference.mdx#rmr) or the system's built-in `rm` command to delete the files in the `.trash` directory, so that storage space can be immediately released.\n\nFor example, to permanently delete a directory in the trash:\n\n```shell\njuicefs rmr .trash/2022-11-30-10/\n```\n\nIf you want to delete expired files more quickly, you can mount multiple mount points to exceed the deletion speed limit of a single client.\n\n## Selectively skipping trash {#skip}\n\nIt is possible to skip the trash and permanently delete files directly. The 's' flag using `chattr` can be set on files or directories to enable this feature. When a file or directory has the 's' flag set, the file or directory will be permanently deleted when removed, bypassing the trash. New files or directories created under a directory with the 's' flag will also inherit this behavior. Existing JuiceFS files or directories moved into a directory with the 's' flag set will not inherit the flag.\n\nYou will need to enable the mount option `--enable-ioctl` to allow adjusting file attributes using `chattr`.\n\n## Trash and slices {#gc}\n\nApart from user deleted files, there's another type of data which also resides in Trash, which isn't directly visible from the `.trash` directory, they are stale slices created by file edits and overwrites. Read more in [How JuiceFS stores files](../introduction/architecture.md#how-juicefs-store-files). To sum up, if applications constantly delete or overwrite files, object storage usage will exceed file system usage.\n\nAlthough stale slices cannot be browsed or manipulated, you can use [`juicefs status`](../reference/command_reference.mdx#status) to observe its scale:\n\n```shell\n# The Trash Slices field displayed below is the number of stale slices\n$ juicefs status META-URL --more\n...\n Trash Files: 0 0.0/s\n Trash Files: 0.0 b (0 Bytes) 0.0 b/s\n Pending Deleted Files: 0 0.0/s\n Pending Deleted Files: 0.0 b (0 Bytes) 0.0 b/s\n Trash Slices: 27 26322.2/s\n Trash Slices: 783.0 b (783 Bytes) 753.1 KiB/s\nPending Deleted Slices: 0 0.0/s\nPending Deleted Slices: 0.0 b (0 Bytes) 0.0 b/s\n...\n```\n\nStale slices are also kept according to the expiration settings, this adds another layer of data security: if files are erroneously edited or overwritten, original state can be recovered through metadata backups (provided that you have already set up metadata backup). If you do need to rollback this type of accident overwrites, you need to obtain a copy of the metadata backup, and then mount using this copy, so that you can visit the file system in its older state, and recover any files before they are tampered. See [Metadata Backup & Recovery](../administration/metadata_dump_load.md) for more.\n\nDue to its invisibility, stale slices can grow to a very large size, if you do need to delete them, follow below procedure:\n\n```shell\n# Temporarily disable Trash\njuicefs config META-URL --trash-days 0\n\n# Optionally run compaction\njuicefs gc --compact\n\n# Purge leaked objects\njuicefs gc --delete\n\n# Do not forget to re-enable Trash upon completion\n```\n\n## Access privileges {#permission}\n\nAll users are allowed to browse the trash directory and see the full list of removed files. However, only root has write privilege to the `.trash` directory. Since JuiceFS keeps the original permission modes even for the trashed files, normal users can read files that they have permission to.\n\nSeveral caveats on Trash privileges:\n\n* When JuiceFS Client is started by a non-root user, add the `-o allow_root` option or trash cannot be emptied normally.\n* The `.trash` directory can only be accessed from the file system root, thus not available for sub-directory mount points.\n* User cannot create new files inside the trash directory, and only root are allowed to move or delete files in trash.\n" }, { "path": "docs/en/tutorials/aliyun.md", "content": "---\ntitle: Use JuiceFS on Alibaba Cloud\nsidebar_position: 7\nslug: /clouds/aliyun\ndescription: Learn how to use JuiceFS on Alibaba Cloud.\n---\n\nAs shown in the figure below, JuiceFS is driven by both the database and the object storage. The files stored in JuiceFS are split into fixed-size data blocks and stored in the object store according to certain rules, while the metadata corresponding to the data is stored in the database.\n\nThe metadata is stored completely independently. Retrieval and processing of files do not directly manipulate the data in the object storage. Instead, operations are performed first on the metadata in the database. Interaction with the object storage only occurs when data changes.\n\nThis design can effectively reduce the cost of the object storage in terms of the number of requests. It also allows users to significantly experience the performance improvement brought by JuiceFS.\n\n![JuiceFS-arch-new](../images/juicefs-aliyun.png)\n\nThis document introduces how to use JuiceFS on Alibaba Cloud.\n\n## Preparation\n\nFrom the previous architecture description, you can know that JuiceFS needs to be used together with database and object storage. Here we directly use the Alibaba Cloud ECS cloud server, combined with cloud database and OSS object storage.\n\nWhen you create cloud computing resources, try to choose in the same region, so that resources can access each other through intranet and avoid using public network to incur additional traffic costs.\n\n### ECS\n\nJuiceFS has no special requirements for server hardware. Generally speaking, entry-level cloud servers can also use JuiceFS stably. Typically, you just need to choose the one that can meet your own application requirements.\n\nIn particular, you do not need to buy a new server or reinstall the system to use JuiceFS. JuiceFS is not application invasive and does not cause any interference with your existing systems and programs. You can install and use JuiceFS on your running server.\n\nBy default, JuiceFS takes up 1 GB of hard disk space for caching, and you can adjust the size of the cache space as needed. This cache is a data buffer layer between the client and the object storage. You can get better performance by choosing a cloud drive with better performance.\n\nIn terms of operating system, JuiceFS can be installed on all operating systems provided by Alibaba Cloud ECS.\n\n**The ECS specification used in this document are as follows:**\n\n| **Instance specification** | ecs.t5-lc1m1.small |\n| -------------------------- | -------------------------- |\n| **CPU** | 1 core |\n| **MEMORY** | 1 GB |\n| **Storage** | 40 GB |\n| **OS** | Ubuntu Server 20.04 64-bit |\n| **Location** | Shanghai |\n\n### Cloud database\n\nJuiceFS stores all the metadata corresponding to the data in a separate database, which currently supports Redis, MySQL, PostgreSQL, SQLite, and OceanBase.\n\nDepending on the database type, the performance and reliability of metadata are different. For example, Redis runs entirely in memory. While it provides the ultimate performance, it is difficult to operate and maintain and has low reliability. SQLite is a single-file relational database with low performance and is not suitable for large-scale data storage. However, it is configuration-free and suitable for a small amount of data storage on a single machine. In contrast, OceanBase is a distributed relational database that delivers high performance while ensuring data consistency and high reliability (RTO < 8 seconds). It is particularly well-suited for scenarios in industries such as finance, retail, and telecommunications, where transactional consistency and distributed capabilities are critical. By integrating with JuiceFS, OceanBase enhances the efficiency, reduces the latency, and improves the stability of handling massive metadata, meeting the demanding requirements of modern distributed storage systems for underlying databases.\n\nIf you just want to evaluate the functionality of JuiceFS, you can build the database manually on ECS. If you want to use JuiceFS in a production environment, and you don't have a professional database operation and maintenance team, the cloud database service is usually a better choice.\n\nYou can also use cloud database services provided on other platforms if you wish. But in this case, you have to expose the database port to the public network, which may have some security risks.\n\nIf you must access the database through the public network, you can enhance the security of your data by strictly limiting the IP addresses that are allowed to access the database through the whitelist feature provided by the cloud database console.\n\nOn the other hand, if you cannot successfully connect to the cloud database through the public network, you can check the whitelist of the database.\n\n| Database | Redis | MySQL/PostgreSQL | SQLite | OceanBase |\n| :-------------: | :-----------------------------------------------------: | :----------------------------------------------------------: | :----------------------------------------------------------: | :----------------------------------------------------------: |\n| **Performance** | High | Medium | Low | High |\n| **Management** | High | Medium | Low | Medium |\n| **Reliability** | Low | Medium | Low | High |\n| **Scenario** | Massive data, distributed high-frequency reads and writes | Massive data, distributed low- and medium-frequency reads and writes | Low-frequency reads and writes in single machine for small amounts of data | Distributed scenarios, strong transaction consistency, and high reliability requirements |\n\n**This document uses [ApsaraDB for Redis](https://www.alibabacloud.com/product/apsaradb-for-redis), and the following pseudo address is compiled for demonstration purposes only:**\n\n| Redis version | 5.0 Community Edition |\n|----------------------------|----------------------------------------|\n| **Instance specification** | 256M Standard master-replica instances |\n| **Connection address** | `herald-sh-abc.redis.rds.aliyuncs.com` |\n| **Available zone** | Shanghai |\n\n### Object Storage OSS\n\nJuiceFS stores all data in object storage, which supports almost all object storage services. However, to get the best performance, when using Alibaba Cloud ECS, OSS object storage is usually the optimal choice. However, you must choose ECS and OSS buckets in the same region so that they can be accessed through intranet. This has low latency and does not require additional traffic costs.\n\nYou can also use object storage services provided by other cloud platforms if you wish, but this is not recommended. This is because accessing object storage from other cloud platforms through ECS needs the public network, and object storage will incur traffic costs. In addition, the access latency will be higher compared to this, which may affect the performance of JuiceFS.\n\nAlibaba Cloud OSS has different storage levels. Since JuiceFS needs to interact with object storage frequently, it is recommended to use standard tier. You can use it with OSS resource pack to reduce the cost of using object storage.\n\n### API access secret key\n\nAlibaba Cloud OSS needs to be accessed through an API. You need to prepare an access key pair, including an AccessKey ID and an AccessKey secret. [Click here](https://www.alibabacloud.com/help/doc-detail/125558.htm) to see how to obtain the access key pair.\n\n> **Security advisory**: Explicit use of the API access secret key may lead to key compromise. It is recommended to assign a [RAM role](https://www.alibabacloud.com/help/doc-detail/110376.htm) to the cloud server. Once an ECS is granted access to the OSS, the API access key is no longer required to access the OSS.\n\n## Installation\n\nWe are currently using Ubuntu Server 20.04 64-bit, so you can download the latest version of the client by running the following command:\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\nAlternatively, you can choose another version by visiting the [JuiceFS GitHub Releases](https://github.com/juicedata/juicefs/releases) page.\n\nExecute the command, and you will see the help message returned by JuiceFS. This means that the client installation was successful.\n\n```shell\n$ juicefs\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 0.15.2 (2021-07-07T05:51:36Z 4c16847)\n\nCOMMANDS:\n format format a volume\n mount mount a volume\n umount unmount a volume\n gateway S3-compatible gateway\n sync sync between two storage\n rmr remove directories recursively\n info show internal information for paths or inodes\n bench run benchmark to read/write/stat big/small files\n gc collect any leaked objects\n fsck Check consistency of file system\n profile analyze access log\n status show status of JuiceFS\n warmup build cache for target directories/files\n dump dump metadata into a JSON file\n load load metadata from a previously dumped JSON file\n help, h Shows a list of commands or help for one command\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q only warning and errors (default: false)\n --trace enable trace log (default: false)\n --no-agent disable pprof (:6060) agent (default: false)\n --help, -h show help (default: false)\n --version, -V print only the version (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\nJuiceFS has good cross-platform compatibility and supports Linux, Windows, and macOS. This document focuses on installing and using JuiceFS on Linux. For installation instructions on other systems, [check this document](../getting-started/installation.md).\n\n## Create JuiceFS storage\n\nOnce the JuiceFS client is installed, you can create the JuiceFS storage using the Redis database and OSS object storage that you prepared earlier.\n\nTechnically speaking, this step should be called \"Format a volume.\" However, given that many users may not understand or care about the standard file system terminology, we will refer to the process simply as \"Create JuiceFS storage.\"\n\nThe following command creates a storage named `mystor`, which is a file system, using the `format` subcommand provided by the JuiceFS client:\n\n```shell\n$ juicefs format \\\n --storage oss \\\n --bucket https:// \\\n --access-key \\\n --secret-key \\\n redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n mystor\n```\n\n**Option description:**\n\n- `--storage`: Specifies the type of object storage. [Click here](../reference/how_to_set_up_object_storage.md) to view the object storage services supported by JuiceFS.\n- `--bucket`: Bucket domain name of the object storage. When using OSS, just fill in the bucket name. There is no need to fill in the full domain name. JuiceFS will automatically identify and fill in the complete address.\n- `--access-key` and `--secret-key`: The secret key pair to access the object storage API. [Click here](https://www.alibabacloud.com/help/doc-detail/125558.htm) for instructions on obtaining these keys.\n\n> Redis 6.0 authentication requires username and password parameters in the format of `redis://username:password@redis-server-url:6379/1`. Currently, Alibaba Cloud Redis only provides Reids 4.0 and 5.0 versions, which require only a password for authentication. When setting the Redis server address, leave the username empty, like this: `redis://:password@redis-server-url:6379/1`.\n\nWhen you are using the RAM role to bind to the ECS, you can create JuiceFS storage by specifying `--storage` and `--bucket` without providing the API access key. The command can be rewritten as follows:\n\n```shell\n$ juicefs format \\\n --storage oss \\\n --bucket https://mytest.oss-cn-shanghai.aliyuncs.com \\\n redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n mystor\n```\n\nA successful creation of the file system will yield output similar to the following:\n\n```shell\n2021/07/13 16:37:14.264445 juicefs[22290] : Meta address: redis://@herald-sh-abc.redis.rds.aliyuncs.com:6379/1\n2021/07/13 16:37:14.277632 juicefs[22290] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/07/13 16:37:14.281432 juicefs[22290] : Ping redis: 3.609453ms\n2021/07/13 16:37:14.527879 juicefs[22290] : Data uses oss://mytest/mystor/\n2021/07/13 16:37:14.593450 juicefs[22290] : Volume is formatted as {Name:mystor UUID:4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b Storage:oss Bucket:https://mytest340 AccessKey:LTAI4G4v6ioGzQXy56m3XDkG SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n## Mount JuiceFS\n\nWhen the file system is created, the information related to the object storage is stored in the database. Therefore, you do not need to enter information such as the bucket domain and secret key when mounting.\n\nUse the `mount` subcommand to mount the file system to the `/mnt/jfs` directory.\n\n```shell\nsudo juicefs mount -d redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1 /mnt/jfs\n```\n\n> **Note**: When mounting the file system, only the Redis database address is required; the file system name is not necessary. The default cache path is `/var/jfsCache`. Make sure the current user has sufficient read/write permissions.\n\nOutput similar to the following means that the file system was mounted successfully:\n\n```shell\n2021/07/13 16:40:37.088847 juicefs[22307] : Meta address: redis://@herald-sh-abc.redis.rds.aliyuncs.com/1\n2021/07/13 16:40:37.101279 juicefs[22307] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/07/13 16:40:37.104870 juicefs[22307] : Ping redis: 3.408807ms\n2021/07/13 16:40:37.384977 juicefs[22307] : Data use oss://mytest/mystor/\n2021/07/13 16:40:37.387412 juicefs[22307] : Disk cache (/var/jfsCache/4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b/): capacity (1024 MB), free ratio (10%), max pending pages (15)\n.2021/07/13 16:40:38.410742 juicefs[22307] : OK, mystor is ready at /mnt/jfs\n```\n\nYou can use the `df` command to see how the file system is mounted:\n\n```shell\n$ df -Th\nFile system type capacity used usable used% mount point\nJuiceFS:mystor fuse.juicefs 1.0P 64K 1.0P 1% /mnt/jfs\n```\n\nAfter the file system is successfully mounted, you can store data in the `/mnt/jfs` directory as if you were using a local hard drive.\n\n> **Multi-host sharing**: JuiceFS storage supports being mounted by multiple cloud servers at the same time. You can install the JuiceFS client on other could servers and then use the `redis://:@herald-sh-abc.redis.rds.aliyuncs. com:6379/1` database address to mount the file system on each host.\n\n## File system status\n\nUse the `status` subcommand of the JuiceFS client to view basic information and connection status of a file system.\n\n```shell\n$ juicefs status redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1\n\n2021/07/13 16:56:17.143503 juicefs[22415] : Meta address: redis://@herald-sh-abc.redis.rds.aliyuncs.com:6379/1\n2021/07/13 16:56:17.157972 juicefs[22415] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/07/13 16:56:17.161533 juicefs[22415] : Ping redis: 3.392906ms\n{\n \"Setting\": {\n \"Name\": \"mystor\",\n \"UUID\": \"4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b\",\n \"Storage\": \"oss\",\n \"Bucket\": \"https://mytest\",\n \"AccessKey\": \"\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0\n },\n \"Sessions\": [\n {\n \"Sid\": 3,\n \"Heartbeat\": \"2021-07-13T16:55:38+08:00\",\n \"Version\": \"0.15.2 (2021-07-07T05:51:36Z 4c16847)\",\n \"Hostname\": \"demo-test-sh\",\n \"MountPoint\": \"/mnt/jfs\",\n \"ProcessID\": 22330\n }\n ]\n}\n```\n\n## Unmount JuiceFS\n\nYou can unmount the file system using the `umount` command provided by the JuiceFS client, for example:\n\n```shell\nsudo juicefs umount /mnt/jfs\n```\n\n> **Note**: Forcelly unmounting a file system in use may result in data corruption or loss. Therefore, proceed with caution.\n\n## Auto-mount on boot\n\nFor details on auto-mounting JuiceFS at boot time, see [Mount JuiceFS at Boot Time](../administration/mount_at_boot.md).\n" }, { "path": "docs/en/tutorials/aws.md", "content": "---\ntitle: Use JuiceFS on AWS\nsidebar_position: 4\nslug: /clouds/aws\n---\n\nAmazon Web Services (AWS) is a leading global cloud computing platform that offers a wide range of cloud computing services. With its extensive product line, AWS provides flexible options for creating and utilizing JuiceFS file systems.\n\n## Where can JuiceFS be used? {#where-can-juicefs-be-used}\n\nJuiceFS has a rich set of API interfaces. For AWS, JuiceFS can typically be used in the following products:\n\n- **Amazon EC2**: Use by mounting the JuiceFS file system\n- **Amazon Elastic Kubernetes Service (EKS)**: Utilizing the JuiceFS CSI Driver\n- **Amazon EMR**: Using the JuiceFS Hadoop Java SDK\n\n## Preparation {#preparation}\n\nA JuiceFS file system consists of two parts:\n\n1. **Object Storage**: Used for data storage.\n2. **Metadata Engine**: A database used for storing metadata.\n\nDepending on specific requirements, you can choose to use fully managed databases and S3 object storage on AWS, or deploy them on EC2 and EKS by yourself.\n\n:::tip\nThis article focuses on the method of creating a JuiceFS file system using AWS fully managed services. For self-hosted scenarios, please refer to the [\"JuiceFS Supported Metadata Engines\"](../reference/how_to_set_up_metadata_engine.md) and [\"JuiceFS Supported Object Storage\"](../reference/how_to_set_up_object_storage.md) guides, as well as the corresponding program documentation.\n:::\n\n### Object storage {#object-storage}\n\nS3 is the object storage service provided by AWS. You can create a bucket in the corresponding region as needed, or authorize the JuiceFS client to automatically create a bucket through [IAM roles](../reference/how_to_set_up_object_storage.md#aksk).\n\nAmazon S3 provides various [storage classes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html), for example:\n\n- **S3 Standard**: Standard storage, suitable for general-purpose storage with frequent data access, offering real-time access with no retrieval costs.\n- **S3 Standard-IA**: Infrequent Access (IA) storage, suitable for data that is accessed less frequently but needs to be stored for the long term, offering real-time access with retrieval costs.\n- **S3 Glacier**: Archive storage, suitable for data that is rarely accessed and requires retrieval (thawing) before access.\n\nYou can set the storage class when creating or mounting the JuiceFS file system, please refer to [documentation](../reference/how_to_set_up_object_storage.md#storage-class) for details. It is recommended to choose the standard storage class first. Although other storage classes may have lower unit storage prices, they often come with minimum storage duration requirements and retrieval costs.\n\nFurthermore, accessing object storage services requires authentication using Access Key (a.k.a. access key ID) and Secret Key (a.k.a. secret access key). You can refer to the document [\"Managing access keys for IAM users\"](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) for creating the necessary policies. When accessing S3 from an EC2 cloud server, you can also assign an [IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) to the EC2 instance to enable the S3 API to be called without using access keys.\n\n### Database {#database}\n\nAWS offers various network-based fully managed databases that can be used to build the JuiceFS metadata engine, mainly including:\n\n- **Amazon MemoryDB for Redis** (hereinafter referred to as MemoryDB): A durable Redis in-memory database service that provides extremely fast performance.\n- **Amazon RDS**: Fully managed databases such as MariaDB, MySQL, PostgreSQL, and more.\n\n:::note\nAlthough Amazon ElastiCache for Redis (hereinafter referred to as ElastiCache) also provides services compatible with the Redis protocol, compared with MemoryDB, ElastiCache cannot provide \"strong consistency guarantee\", so MemoryDB is recommended.\n:::\n\n## Using JuiceFS on EC2 {#using-juicefs-on-ec2}\n\n### Installing the JuiceFS client {#installing-the-juicefs-client}\n\nPlease refer to the [Installation](../getting-started/installation.md) documentation to install the latest JuiceFS Community Edition client based on the operating system used by your EC2 instance.\n\nFor example, if you are using a Linux system, you can use the one-liner installation script to automatically install the client:\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n### Creating a File System {#creating-a-file-system}\n\n#### Preparing object storage {#preparing-object-storage}\n\nYou can assign an IAM role with [AmazonS3FullAccess](https://docs.aws.amazon.com/AmazonS3/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-amazons3fullaccess) permission to your EC2 instance, allowing it to create and use S3 Buckets directly without using Access Key and Secret Key.\n\n#### Preparing the database {#preparing-the-database}\n\nHere we take MemoryDB as an example, please refer to [\"Redis Best Practices\"](../administration/metadata/redis_best_practices.md) and AWS documentation to create a database.\n\nIn order to allow EC2 instances to access the Redis cluster, you need to create them in the same VPC or add rules to the security group of the Redis cluster to allow access from the EC2 instance.\n\n:::note\nIf you are creating a Redis 7.0 version cluster, you will need to install JuiceFS version 1.1 or above on the client side.\n:::\n\n#### Formatting file system {#formatting-file-system}\n\n```shell\njuicefs format --storage s3 \\\n --bucket https://s3.ap-east-1.amazonaws.com/myjfs \\\n rediss://clustercfg.myredis.hc79sw.memorydb.ap-east-1.amazonaws.com:6379/1 \\\n myjfs\n```\n\n### Mounting file system {#mounting-file-system}\n\n```shell\nsudo juicefs mount -d \\\n rediss://clustercfg.myredis.hc79sw.memorydb.ap-east-1.amazonaws.com:6379/1 \\\n /mnt/myjfs\n```\n\nTo mount and use the file system created by authorizing S3 access through an IAM role from outside of AWS, you will need to use `juicefs config` to add the Access Key and Secret Key for the file system.\n\n```shell\njuicefs config \\\n --access-key= \\\n --secret-key= \\\n rediss://clustercfg.myredis.hc79sw.memorydb.ap-east-1.amazonaws.com:6379/1\n```\n\n### Mounting at boot {#mounting-at-boot}\n\nPlease refer to the document [Mount JuiceFS at Boot](../administration/mount_at_boot.md) for details on how to automatically mount JuiceFS at boot.\n\n## Using JuiceFS on Amazon EKS {#using-juicefs-on-amazon-eks}\n\nAmazon EKS supports [three types of node](https://docs.aws.amazon.com/eks/latest/userguide/eks-compute.html):\n\n- **EKS managed node groups**: Use Amazon EC2 as compute nodes\n- **Self-managed nodes**: Use Amazon EC2 as compute nodes\n- **Fargate**: A serverless compute engine\n\nJuiceFS CSI Driver is not currently supported on Fargate. Please create a cluster using \"EKS managed node groups\" or \"self-managed nodes\" to use JuiceFS CSI Driver.\n\nAmazon EKS is a standard Kubernetes cluster and can be managed using tools such as `eksctl`, `kubectl`, and `helm`. For installation and usage instructions, please refer to the [JuiceFS CSI Driver documentation](/docs/csi/introduction).\n\n## Using JuiceFS on Amazon EMR {#using-juicefs-on-amazon-emr}\n\nPlease refer to the document [\"Using JuiceFS in Hadoop Ecosystem\"](../deployment/hadoop_java_sdk.md) for instructions.\n" }, { "path": "docs/en/tutorials/digitalocean.md", "content": "---\ntitle: Use JuiceFS on DigitalOcean\nsidebar_position: 6\nslug: /clouds/digitalocean\n---\n\nJuiceFS is designed for the cloud, using the cloud platform out-of-the-box storage and database services, and can be configured and put into use in as little as a few minutes. This article uses the DigitalOcean as an example to introduce how to quickly and easily install and use JuiceFS on the cloud computing platform.\n\n## Preparation\n\nJuiceFS is powered by a combination of storage and database, so the things you need to prepare should include.\n\n### 1. Cloud Server\n\nThe cloud server on DigitalOcean is called Droplet. If you already have a Droplet, you do not need to purchase a new one separately in order to use JuiceFS. Whichever cloud server needs to use JuiceFS storage on it, install the JuiceFS client for it.\n\n#### Hardware Specifications\n\nJuiceFS has no special hardware requirements, and any size Droplet can be used stably. However, it is recommended to choose a better performing SSD and reserve at least 1GB for JuiceFS to use as local cache.\n\n#### Operating System\n\nJuiceFS supports Linux, BSD, macOS and Windows. In this article, we will take Ubuntu Server 20.04 as an example.\n\n### 2. Object Storage\n\nJuiceFS uses object storage to store all your data, and using Spaces on DigitalOcean is the easiest solution. Spaces is an S3-compatible object storage service that works right out of the box. It is recommended to choose the same region as Droplet to get the best access speed and also to avoid additional traffic costs.\n\nOf course, you can also use an object storage service from another platform or build it manually using Ceph or MinIO. In short, you are free to choose the object storage you want to use, just make sure that the JuiceFS client can access the object storage.\n\nHere, we created a Spaces storage bucket named `juicefs` with the region `sgp1` in Singapore, and it is accessible at:\n\n- `https://juicefs.sgp1.digitaloceanspaces.com`\n\nIn addition, you also need to create `Spaces access keys` in the API menu, which JuiceFS needs to access the Spaces API.\n\n### 3. Database\n\nUnlike normal file systems, JuiceFS stores all metadata corresponding to the data in a separate database, and the larger the size of the stored data, the better the performance. Currently, JuiceFS supports common databases such as Redis, TiKV, MySQL/MariaDB, PostgreSQL, SQLite, etc., while support for other databases is under continuous development. If the database you need is not supported at the moment, please submit [issue](https://github.com/juicedata/juicefs/issues) feedback.\n\nEach database has its own advantages and disadvantages in terms of performance, size and reliability, and you should choose according to the actual needs of the scenario.\n\nDon't worry about the choice of database, the JuiceFS client provides a metadata migration feature that allows you to easily export and migrate metadata from one database to another.\n\nFor this article, we use DigitalOcean's Redis 6 database hosting service, choose `Singapore`, and select the same VPC private network as the existing Droplet. It takes about 5 minutes to create the Redis, and we follow the setup wizard to initialize the database.\n\n![DigitalOcean-Redis-guide](../images/digitalocean-redis-guide.png)\n\nBy default, the Redis allows all inbound connections. For security reasons, you should select the Droplet that have access to the Redis in the security setting section of the setup wizard in the `Add trusted sources`, that is, only allow the selected host to access the Redis.\n\nIn the setting of the eviction policy, it is recommended to select `noeviction`, that is, when the memory is exhausted, only errors are reported and no data is evictioned.\n\n> **Note**: In order to ensure the safety and integrity of metadata, please do not select `allkeys-lru` and `allkey-random` for the eviction policy.\n\nThe access address of the Redis can be found in the `Connection Details` of the console. If all computing resources are in DigitalOcean, it is recommended to use the VPC private network for connection first, which can maximize security.\n\n![DigitalOcean-Redis-url](../images/digitalocean-redis-url.png)\n\n## Installation and Use\n\n### 1. Install JuiceFS client\n\nWe currently using Ubuntu Server 20.04, execute the following command to install the latest version of the client.\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\nExecute the command and see the command help information returned to `juicefs`, which means that the client is installed successfully.\n\n```shell\n$ juicefs\n\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 0.16.2 (2021-08-25T04:01:15Z 29d6fee)\n\nCOMMANDS:\n format format a volume\n mount mount a volume\n umount unmount a volume\n gateway S3-compatible gateway\n sync sync between two storage\n rmr remove directories recursively\n info show internal information for paths or inodes\n bench run benchmark to read/write/stat big/small files\n gc collect any leaked objects\n fsck Check consistency of file system\n profile analyze access log\n stats show runtime stats\n status show status of JuiceFS\n warmup build cache for target directories/files\n dump dump metadata into a JSON file\n load load metadata from a previously dumped JSON file\n help, h Shows a list of commands or help for one command\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q only warning and errors (default: false)\n --trace enable trace log (default: false)\n --no-agent disable pprof (:6060) agent (default: false)\n --help, -h show help (default: false)\n --version, -V print only the version (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\nIn addition, you can also visit the [JuiceFS GitHub Releases](https://github.com/juicedata/juicefs/releases) page to select other versions for manual installation.\n\n### 2. Create a file system\n\nTo create a file system, use the `format` subcommand, the format is:\n\n```shell\njuicefs format [command options] META-URL NAME\n```\n\nThe following command creates a file system named `mystor`:\n\n```shell\n$ juicefs format \\\n --storage space \\\n --bucket https://juicefs.sgp1.digitaloceanspaces.com \\\n --access-key \\\n --secret-key \\\n rediss://default:your-password@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1 \\\n mystor\n```\n\n**Parameter Description:**\n\n- `--storage`: Specify the data storage engine, here is `space`, click here to view all [supported storage](../reference/how_to_set_up_object_storage.md).\n- `--bucket`: Specify the bucket access address.\n- `--access-key` and `--secret-key`: Specify the secret key for accessing the object storage API.\n- The Redis managed by DigitalOcean needs to be accessed with TLS/SSL encryption, so it needs to use the `rediss://` protocol header. The `/1` added at the end of the link represents the use of Redis's No. 1 database.\n\nIf you see output similar to the following, it means that the file system is created successfully.\n\n```shell\n2021/08/23 16:36:28.450686 juicefs[2869028] : Meta address: rediss://default@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:36:28.481251 juicefs[2869028] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/08/23 16:36:28.481763 juicefs[2869028] : Ping redis: 331.706µs\n2021/08/23 16:36:28.482266 juicefs[2869028] : Data uses space://juicefs/mystor/\n2021/08/23 16:36:28.534677 juicefs[2869028] : Volume is formatted as {Name:mystor UUID:6b0452fc-0502-404c-b163-c9ab577ec766 Storage:space Bucket:https://juicefs.sgp1.digitaloceanspaces.com AccessKey:7G7WQBY2QUCBQC5H2DGK SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n### 3. Mount a file system\n\nTo mount a file system, use the `mount` subcommand, and use the `-d` parameter to mount it as a daemon. The following command mounts the newly created file system to the `mnt` directory under the current directory:\n\n```shell\nsudo juicefs mount -d \\\n rediss://default:your-password@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1 mnt\n```\n\nThe purpose of using `sudo` to perform the mount operation is to allow JuiceFS to have the authority to create a cache directory under `/var/`. Please note that when mounting the file system, you only need to specify the `database address` and the `mount point`, not the name of the file system.\n\nIf you see output similar to the following, it means that the file system is mounted successfully.\n\n```shell\n2021/08/23 16:39:14.202151 juicefs[2869081] : Meta address: rediss://default@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:39:14.234925 juicefs[2869081] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/08/23 16:39:14.235536 juicefs[2869081] : Ping redis: 446.247µs\n2021/08/23 16:39:14.236231 juicefs[2869081] : Data use space://juicefs/mystor/\n2021/08/23 16:39:14.236540 juicefs[2869081] : Disk cache (/var/jfsCache/6b0452fc-0502-404c-b163-c9ab577ec766/): capacity (1024 MB), free ratio (10%), max pending pages (15)\n2021/08/23 16:39:14.738416 juicefs[2869081] : OK, mystor is ready at mnt\n```\n\nUse the `df` command to see the mounting status of the file system:\n\n```shell\n$ df -Th\nFile system type capacity used usable used% mount point\nJuiceFS:mystor fuse.juicefs 1.0P 64K 1.0P 1% /home/herald/mnt\n```\n\nAs you can see from the output information of the mount command, JuiceFS defaults to sets 1024 MB as the local cache. Setting a larger cache can make JuiceFS have better performance. You can set the cache (in MiB) through the `--cache-size` option when mounting a file system. For example, set a 20GB local cache:\n\n```shell\nsudo juicefs mount -d --cache-size 20000 \\\n rediss://default:your-password@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1 mnt\n```\n\nAfter the file system is mounted, you can store data in the `~/mnt` directory just like using a local hard disk.\n\n### 4. File system status\n\nUse the `status` subcommand to view the basic information and connection status of a file system. You only need to specify the database URL.\n\n```shell\n$ juicefs status rediss://default:bn8l7ui2cun4iaji@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:48:48.567046 juicefs[2869156] : Meta address: rediss://default@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:48:48.597513 juicefs[2869156] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/08/23 16:48:48.598193 juicefs[2869156] : Ping redis: 491.003µs\n{\n \"Setting\": {\n \"Name\": \"mystor\",\n \"UUID\": \"6b0452fc-0502-404c-b163-c9ab577ec766\",\n \"Storage\": \"space\",\n \"Bucket\": \"https://juicefs.sgp1.digitaloceanspaces.com\",\n \"AccessKey\": \"7G7WQBY2QUCBQC5H2DGK\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0\n },\n \"Sessions\": [\n {\n \"Sid\": 1,\n \"Heartbeat\": \"2021-08-23T16:46:14+08:00\",\n \"Version\": \"0.16.2 (2021-08-25T04:01:15Z 29d6fee)\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869091\n },\n {\n \"Sid\": 2,\n \"Heartbeat\": \"2021-08-23T16:47:59+08:00\",\n \"Version\": \"0.16.2 (2021-08-25T04:01:15Z 29d6fee)\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869146\n }\n ]\n}\n```\n\n### 5. Unmount a file system\n\nUse the `umount` subcommand to unmount a file system, for example:\n\n```shell\nsudo juicefs umount ~/mnt\n```\n\n> **Note**: Force unmount the file system in use may cause data damage or loss, please be careful to operate.\n\n### 6. Auto-mount on boot\n\nPlease refer to [\"Mount JuiceFS at Boot Time\"](../administration/mount_at_boot.md) for more details.\n\n### 7. Multi-host shared\n\nThe JuiceFS file system supports being mounted by multiple cloud servers at the same time, and there is no requirement for the geographic location of the cloud server. It can easily realize the real-time data sharing of servers between the same platform, between cross-cloud platforms, and between public and private clouds.\n\nNot only that, the shared mount of JuiceFS can also provide strong data consistency guarantee. When multiple servers mount the same file system, the writes confirmed on the file system will be visible in real time on all hosts.\n\nTo use the shared mount, it is important to ensure that the database and object storage service that make up the file system can be accessed by each host to mount it. In the demonstration environment of this article, the Spaces object storage is open to the entire Internet, and it can be read and written through the API as long as the correct access key is used. But for the Redis database managed by DigitalOcean, you need to configure the access strategy reasonably to ensure that the hosts outside the platform have access permissions.\n\nWhen you mount the same file system on multiple hosts, first create a file system on any host, then install the JuiceFS client on every hosts, and use the same database address to mount it with the `mount` command. Pay special attention to the fact that the file system only needs to be created once, and there should be no need to repeat file system creation operations on other hosts.\n" }, { "path": "docs/en/tutorials/juicefs_on_colab.md", "content": "---\ntitle: Use JuiceFS on Colab with Google Cloud SQL and GCS\nsidebar_position: 5\nslug: /juicefs_on_colab\ndescription: Learn how to use JuiceFS on Google Colab with Google Cloud SQL and GCS, facilitating convenient file storage and sharing in a distributed manner.\n---\n\n[Colaboratory](https://colab.research.google.com), or \"Colab\" for short, is a product by Google Research. Colab enables users to write and execute arbitrary Python code through the browser. It is particularly well suited for machine learning, data analysis, and educational purposes.\n\nColab supports Google Drive for uploading files to or downloading files from Colab instances. However, in some cases, Google Drive might not be that convenient to use with Colab. This is where JuiceFS can a valuable tool, enabling easy file synchronization between Colab instances or between a Colab instance and a local or on-premises machine.\n\nA demo Colab notebook using JuiceFS is available [here](https://colab.research.google.com/drive/1wA8vRwqiihXkI6ViDU8Ud868UeYtmCo5).\n\nThis document outlines the necessary steps for using JuiceFS in the Colab environment. We use Google Cloud SQL as the JuiceFS metadata engine and Google Cloud Storage (GCS) as the JuiceFS object storage.\n\nFor other types of metadata engines or object storages, see [How to Set Up a Metadata Engine](../reference/how_to_set_up_metadata_engine.md)\nand [How to Set Up Object Storage](../reference/how_to_set_up_object_storage.md).\n\nMany of the steps mentioned here will be quite similar to\nthe [Getting Started document](../getting-started/for_distributed.md), which you can also use for reference.\n\n## Summary of steps\n\n1. Format a `juicefs` file system from any machine or instance with access to Google Cloud resources.\n2. Mount the `juicefs` file system in a Colab Notebook\n3. Store sharing files across machines and platforms.\n\n## Prerequisites\n\nThis demo uses Google Cloud Platform's Cloud SQL and Google Cloud Storage (GCS) to create a high-performance file storage system of JuiceFS. You need a Google Cloud Platform account to follow this demo document.\n\nIf you have another cloud vendor's resources (such as AWS RDBS and S3), you can still use this guide as a reference and with other reference documents provided by JuiceFS to achieve a similar solution.\n\nTo make JuiceFS reach the best performance, you might also want the Colab instance is in the same zone or close to the region where Cloud SQL and GCS are deployed. The tutorial works for a randomly hosted Colab instance, but you might notice slower performance due to the latency between the Colab instance and the Cloud SQL/GCS regions. To start Colab instances in a specific region, see [instructions for starting a GCE VM on Colab via GCP Marketplace](https://research.google.com/colaboratory/marketplace.html).\n\nBefore diving into the detailed steps, ensure you have the following resources ready:\n\n* A Google Cloud Platform account ready and a *project* created. This demo uses a GCP project\nnamed `juicefs-learning`.\n* A Cloud SQL (Postgres) ready for use. This demo uses the `juicefs-learning:europe-west1:juicefs-sql-example-1` instance as the metadata service.\n* A GCS bucket created as the object storage service. This demo uses `gs://juicefs-bucket-example-1` as the bucket to store file chunks.\n* An IAM service account or an authorized user account that has write access to the Postgres server and GCS buckets.\n\n## Detailed steps\n\n### Step 1: Format and mount a JuiceFS file system folder\n\nThis step needs to be done only once, and you can choose to execute it on any machine or instance where you have good connectivity and access to your Google Cloud resources.\n\n1. Use `gcloud auth application-default login` to prepare a local credential, or use `GOOGLE_APPLICATION_CREDENTIALS` to set up a JSON key file.\n\n2. Use [`cloud_sql_proxy`](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy) to open a port (in\nthis case, 5432) locally to expose your cloud Postgres service to your local machine:\n\n ```shell\n gcloud auth application-default login\n\n # Or set up the json key file via GOOGLE_APPLICATION_CREDENTIALS=/path/to/key\n\n cloud_sql_proxy -instances=juicefs-learning:europe-west1:juicefs-sql-example-1=tcp:0.0.0.0:5432\n ```\n\n3. Use the following command to create a new file system named `myvolume` using the `juicefs format` command. Later, you can mount this file system on any other machines or instances where you have access to your cloud resources.\n\n You can download `juicefs` [here](https://github.com/juicedata/juicefs/releases).\n\n ```shell\n juicefs format \\\n --storage gs \\\n --bucket gs://juicefs-bucket-example-1 \\\n \"postgres://postgres:mushroom1@localhost:5432/juicefs?sslmode=disable\" \\\n myvolume\n ```\n\nNote that this step is only required once on any machine you prefer to work on.\n\n### Step 2: Mount the JuiceFS file system on Colab\n\nOnce you have completed Step 1, it means you already have a JuiceFS file system (named `myvolume` in this case) defined and ready to be used.\n\nNow, let's open a Colab page and execute the following commands to mount our file system into a folder named `mnt`.\n\nFirstly, download the `juicefs` binary and do the same as Step 1 to get GCP credentials and open the Cloud SQL proxy.\n\nNote that the following commands are run in the Colab environment, so there is a `!` mark at the beginning for running shell commands.\n\n1. Download `juicefs` to the Colab runtime instance:\n\n ```shell\n ! curl -sSL https://d.juicefs.com/install | sh -\n ```\n\n2. Set up Google Cloud credentials:\n\n ```shell\n ! gcloud auth application-default login\n ```\n\n3. Open `cloud_sql_proxy`:\n\n ```shell\n ! wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O cloud_sql_proxy\n ! chmod +x cloud_sql_proxy\n ! GOOGLE_APPLICATION_CREDENTIALS=/content/.config/application_default_credentials.json nohup ./cloud_sql_proxy -instances=juicefs-learning:europe-west1:juicefs-sql-example-1=tcp:0.0.0.0:5432 >> cloud_sql_proxy.log &\n ```\n\n4. Mount the `myvolumn` JuiceFS file system onto the `mnt` folder:\n\n ```shell\n ! GOOGLE_APPLICATION_CREDENTIALS=/content/.config/application_default_credentials.json nohup juicefs mount \"postgres://postgres:mushroom1@localhost:5432/juicefs?sslmode=disable\" mnt > juicefs.log &\n ```\n\nNow you should be able to use the `mnt` folder as if it were a local file system folder to write and read folders and files in it.\n\n### Step 3: Load data at another time or on another instance\n\nWith data stored in the JuiceFS file system in Step 2, you can repeat all the operations mentioned in Step 2 at any time on any other machines to access the previously stored data or to store more data into it.\n\nCongratulations! Now you have learned how to use JuiceFS, specifically with Google Colab to\nconveniently share and store data files in a distributed fashion.\n\nFeel free to explore a demo Colab notebook using JuiceFS [here](https://colab.research.google.com/drive/1wA8vRwqiihXkI6ViDU8Ud868UeYtmCo5).\n\nHappy coding :)\n" }, { "path": "docs/en/tutorials/juicefs_on_k3s.md", "content": "---\ntitle: Use JuiceFS on K3s\nsidebar_position: 2\nslug: /juicefs_on_k3s\n---\n\n[K3s](https://k3s.io) is a functionally optimized lightweight Kubernetes distribution that is fully compatible with Kubernetes.In other words, almost all operations performed on Kubernetes can also be executed on K3s. K3s packages the entire container orchestration system into a binary program with a size of less than 100MB, significantly reducing the environment dependencies and installation steps required to deploy Kubernetes production clusters. Compared to Kubernetes, K3s has lower performance requirements for the operating system.\n\nIn this article, we will build a K3s cluster with two nodes, install and configure [JuiceFS CSI Driver](https://github.com/juicedata/juicefs-csi-driver) for the cluster, and lastly create an NGINX Pod for verification.\n\n## Deploy a K3s cluster\n\nK3s has very low **minimum requirements** for hardware:\n\n- **Memory**: 512MB+ (recommend 1GB+)\n- **CPU**: 1 core\n\nWhen deploying a production cluster, it is recommended to start with a minimum hardware configuration of 4 cores and 8GB of memory per node. For more detailed information, please refer to the [Hardware Requirements](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#hardware) documentation.\n\n### K3s server node\n\nThe IP address of the server node is: `192.168.1.35`\n\nYou can use the official script provided by K3s to deploy the server node on a regular Linux distribution.\n\n```shell\ncurl -sfL https://get.k3s.io | sh -\n```\n\nAfter the deployment is successful, the K3s service will automatically start, and kubectl and other tools will also be installed at the same time.\n\nYou can execute the following command to view the status of the node:\n\n```shell\n$ sudo kubectl get nodes\nNAME STATUS ROLES AGE VERSION\nk3s-s1 Ready control-plane,master 28h v1.21.4+k3s1\n```\n\nGet the `node-token`:\n\n```shell\nsudo -u root cat /var/lib/rancher/k3s/server/node-token\n```\n\n### K3s worker node\n\nThe IP address of the worker node is: `192.168.1.36`\n\nExecute the following command and change the value of `K3S_URL` to the IP or domain name of the server node (the default port is `6443`). Replace the value of `K3S_TOKEN` with the `node-token` obtained from the server node.\n\n```shell\ncurl -sfL https://get.k3s.io | K3S_URL=http://192.168.1.35:6443 K3S_TOKEN=K1041f7c4fabcdefghijklmnopqrste2ec338b7300674f::server:3d0ab12800000000000000006328bbd80 sh -\n```\n\nAfter the deployment is successful, go back to the server node to check the node status:\n\n```shell\n$ sudo kubectl get nodes\nNAME STATUS ROLES AGE VERSION\nk3s-s1 Ready control-plane,master 28h v1.21.4+k3s1\nk3s-n1 Ready 28h v1.21.4+k3s1\n```\n\n## Install CSI Driver\n\nIt is consistent with the method of [Use JuiceFS on Kubernetes](../deployment/how_to_use_on_kubernetes.md). Therefore, you can install CSI Driver through Helm or kubectl.\n\nHere we use kubectl as an example. Execute the following command to install the CSI Driver:\n\n```shell\nkubectl apply -f https://raw.githubusercontent.com/juicedata/juicefs-csi-driver/master/deploy/k8s.yaml\n```\n\n### Create Storage Class\n\nCopy and modify the following code to create a configuration file, for example: `juicefs-sc.yaml`\n\n```yaml\napiVersion: v1\nkind: Secret\nmetadata:\n name: juicefs-sc-secret\n namespace: kube-system\ntype: Opaque\nstringData:\n name: \"test\"\n metaurl: \"redis://juicefs.afyq4z.0001.use1.cache.amazonaws.com/3\"\n storage: \"s3\"\n bucket: \"https://juicefs-test.s3.us-east-1.amazonaws.com\"\n access-key: \"\"\n secret-key: \"\"\n---\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: juicefs-sc\nprovisioner: csi.juicefs.com\nreclaimPolicy: Retain\nvolumeBindingMode: Immediate\nparameters:\n csi.storage.k8s.io/node-publish-secret-name: juicefs-sc-secret\n csi.storage.k8s.io/node-publish-secret-namespace: kube-system\n csi.storage.k8s.io/provisioner-secret-name: juicefs-sc-secret\n csi.storage.k8s.io/provisioner-secret-namespace: kube-system\n```\n\nThe `stringData` part of the configuration file is used to set the information related to the JuiceFS file system. It will create the file system based on the information you specify. When you need to use the pre-created file system in the storage class, you only need to fill in the `name` and `metaurl`, and the other items can be deleted or the value can be left blank.\n\nExecute the command to deploy the storage class:\n\n```shell\nkubectl apply -f juicefs-sc.yaml\n```\n\nView storage class status:\n\n```shell\n$ sudo kubectl get sc\nNAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE\nlocal-path (default) rancher.io/local-path Delete WaitForFirstConsumer false 28h\njuicefs-sc csi.juicefs.com Retain Immediate false 28h\n```\n\n> **Note**: A storage class is associated with a JuiceFS file system. You can create as many storage classes as you need, but be aware of the storage class name in the configuration file as the same name can cause conflicts.\n\n## Use JuiceFS to persist NGINX data\n\nNext, deploy an NGINX Pod using a persistent storage declared by the JuiceFS storage class.\n\n### Deployment\n\nCreate a configuration file, for example: `deployment.yaml`\n\n```yaml\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: web-pvc\nspec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n storage: 10Pi\n storageClassName: juicefs-sc\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-run\n labels:\n app: nginx\nspec:\n replicas: 2\n selector:\n matchLabels:\n app: nginx\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: linuxserver/nginx\n ports:\n - containerPort: 80\n volumeMounts:\n - mountPath: /config\n name: web-data\n volumes:\n - name: web-data\n persistentVolumeClaim:\n claimName: web-pvc\n```\n\nDeploy it:\n\n```\nsudo kubectl apply -f deployment.yaml\n```\n\n### Service\n\nCreate a configuration file, for example: `service.yaml`\n\n```yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: nginx-run-service\nspec:\n selector:\n app: nginx\n ports:\n - name: http\n port: 80\n```\n\nDeploy it:\n\n```shell\nsudo kubectl apply -f service.yaml\n```\n\n### Ingress\n\nK3s is pre-installed with traefik-ingress by default. Create an ingress for NGINX through the following configuration. For example: `ingress.yaml`\n\n```yaml\napiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: nginx-run-ingress\n annotations:\n traefik.ingress.kubernetes.io/router.entrypoints: web\nspec:\n rules:\n - http:\n paths:\n - pathType: Prefix\n path: \"/web\"\n backend:\n service:\n name: nginx-run-service\n port:\n number: 80\n```\n\nDeploy it:\n\n```shell\nsudo kubectl apply -f ingress.yaml\n```\n\n### Visit\n\nAfter the deployment is completed, use the host on the same LAN to access any cluster node, and then you will see the NGINX welcome page.\n\n![K3s-NGINX-welcome](../images/k3s-nginx-welcome.png)\n\nNext, check whether the container has successfully mounted JuiceFS, and execute the following command to check the Pod status:\n\n```shell\n$ sudo kubectl get pods\nNAME READY STATUS RESTARTS AGE\nnginx-run-7d6fb7d6df-qhr2m 1/1 Running 0 28h\nnginx-run-7d6fb7d6df-5hpv7 1/1 Running 0 24h\n```\n\nExecuting the following command will show the file system mount status of any Pod:\n\n```shell\n$ sudo kubectl exec nginx-run-7d6fb7d6df-qhr2m -- df -Th\nFilesystem Type Size Used Avail Use% Mounted on\noverlay overlay 20G 3.2G 17G 17% /\ntmpfs tmpfs 64M 0 64M 0% /dev\ntmpfs tmpfs 2.0G 0 2.0G 0% /sys/fs/cgroup\nJuiceFS:jfs fuse.juicefs 1.0P 174M 1.0P 1% /config\n/dev/sda1 ext4 20G 3.2G 17G 17% /etc/hosts\nshm tmpfs 64M 0 64M 0% /dev/shm\ntmpfs tmpfs 2.0G 12K 2.0G 1% /run/secrets/kubernetes.io/serviceaccount\ntmpfs tmpfs 2.0G 0 2.0G 0% /proc/acpi\ntmpfs tmpfs 2.0G 0 2.0G 0% /proc/scsi\ntmpfs tmpfs 2.0G 0 2.0G 0% /sys/firmware\n```\n\nAs you can see, the file system named `jfs` has been mounted to the `/config` directory of the container, and the used space is 174M.\n\nThis indicates that the Pods in the cluster have been successfully configured and used JuiceFS to persist data.\n" }, { "path": "docs/en/tutorials/juicefs_on_kubesphere.md", "content": "---\ntitle: Use JuiceFS on KubeSphere\nsidebar_position: 3\nslug: /juicefs_on_kubesphere\n---\n\n[KubeSphere](https://kubesphere.com.cn) is an application-centric multi-tenant container platform built on Kubernetes. It provides full-stack IT automated operation and maintenance capabilities and simplifies the DevOps workflow of the enterprise.\n\nKubeSphere provides a friendly wizard-style operation interface for operation and maintenance, even users who are not experienced in Kubernetes can start management and use relatively easily. It provides a Helm-based application market that can easily install various Kubernetes applications under a graphical interface.\n\nThis article will introduce how to deploy JuiceFS CSI Driver in KubeSphere with one click to provide data persistence for various applications on the cluster.\n\n## Prerequisites\n\n1. **Install KubeSphere**\n\n There are two ways to install KubeSphere. One is installing in Linux, you can refer to the document: [All-in-One Installation of Kubernetes and KubeSphere on Linux](https://kubesphere.com.cn/en/docs/quick-start/all-in-one-on-linux) , One is installing in Kubernetes, you can refer to the document: [Minimal KubeSphere on Kubernetes](https://kubesphere.com.cn/en/docs/quick-start/minimal-kubesphere-on-k8s)\n\n2. **Enable app store in KubeSphere**\n\n You can refer to the documentation for enabling the app store in KubeSphere: [KubeSphere App Store](https://kubesphere.com.cn/en/docs/pluggable-components/app-store)\n\n## Install JuiceFS CSI Driver\n\nIf the version of KubeSphere is v3.2.0 and above, you can install CSI Driver directly in the app store, skip the \"Configure Application Template/Application Repository\" step, and go directly to the \"Install\" step; if the KubeSphere version is lower than v3.2.0, follow the steps below to configure application templates/application repository.\n\n### Configure of Application Template/Application Repository\n\nTo install JuiceFS CSI Driver, you first need to create an application template. There are two methods.\n\n#### Method one: Application Repository\n\nClick in the workspace to enter the application management, select \"App Repositories\", click the create button to add JuiceFS CSI Repository, fill in:\n\n- Repository name: `juicefs-csi-driver`\n- Index URL: `https://juicedata.github.io/charts/`\n\n![kubesphere_app_shop_en](../images/kubesphere_app_shop_en.png)\n\n#### Method two: Application Template\n\nDownload the chart compressed package from the JuiceFS CSI Driver warehouse: [https://github.com/juicedata/juicefs-csi-driver/releases](https://github.com/juicedata/juicefs-csi-driver/releases).\n\nIn the \"Workspace\", click to enter the \"App Management\", select \"App Templates\", click \"create\", upload the chart compression package:\n\n![kubesphere_app_template_en](../images/kubesphere_app_template_en.png)\n\n### Install\n\nSelect \"Project\" where you want to deploy in the \"Workspace\" (the project in KubeSphere is the namespace in K8s), select \"Apps\", click the \"create\" button, select \"From App Store\", and then Select `juicefs`:\n\n![kubesphere_shop_juicefs_en](../images/kubesphere_shop_juicefs_en.png)\n\nIf KubeSphere version is lower than v3.2.0, select button \"From App Template\" according to the application template configured in the previous step:\n\n![kubesphere_install_csi_en](../images/kubesphere_install_csi_en.png)\n\nIt's the same after entering the configuration modification page, modify the following two places:\n\n- namespace: Change to the corresponding project name\n- storageClass.backend:\n The `backend` part is used to define the backend database and object storage of the file system. Refer to [\"Create a File System\"](../getting-started/standalone.md#juicefs-format) for related content.\n\nYou can also quickly create databases (such as Redis) and object storage (such as MinIO) by KubeSphere's app store. For example, build on the KubeSphere platform Redis: Select \"Apps\" in the current project, click the \"create\" button, select \"From App Store\", select \"Redis\", and then quickly deploy. The access URL of Redis can be the service name of the deployed application, as follows:\n\n![kubesphere_redis_en](../images/kubesphere_redis_en.png)\n\nDeploying MinIO on the KubeSphere platform is a similar process, but you can modify the accessKey and secretKey of MinIO before deploying MinIO, and you need to remember the configured values. As shown below:\n\n![kubesphere_create_minio_en](../images/kubesphere_create_minio_en.png)\n\n> Attention: If there are permissions error when deploying MinIO, you can set the `securityContext.enables` in the configuration to false.\n\nMinIO's access URL can be the service name of the deployed application, as follows:\n\n![kubesphere_minio_en](../images/kubesphere_minio_en.png)\n\nAfter both Redis and MinIO are set up, you can fill in the `backend` value of JuiceFS CSI Driver.\n\n1. `metaurl` is the database address of Redis just created, the access address of Redis can be the service name corresponding to the Redis application, such as `redis://redis-rzxoz6:6379/1`\n2. `storage` is type of storage for the object, such as `minio`\n3. `bucket` is the available bucket of MinIO just created (JuiceFS will automatically create it, no need to create it manually), the access address of MinIO can be the service name corresponding to the MinIO application, such as `http://minio-qkp9my:9000/minio/test`\n4. `accessKey` and `secretKey` are the accessKey and secretKey of MinIO just created\n\n![kubesphere_update_csi_en](../images/kubesphere_update_csi_en.png)\n\nAfter the configuration is modified, click \"Install\".\n\n## Usage\n\n### Deploy application\n\nThe JuiceFS CSI Driver installed above has created a `StorageClass`, for example, the `StorageClass` created above is `juicefs-sc` , Can be used directly.\n\nThen you need to create a PVC. In \"Project\", select \"Storage Management\", then select \"Storage Volume\", click the \" Create\" button to create a PVC, and select `juicefs-sc` for the \"StorageClass\", as follows:\n\n![kubesphere_pvc_en](../images/kubesphere_pvc_en.png)\n\nAfter the PVC is created, in the \"Apps\" of \"Project\", select \"Workloads\", click \"Create\" button to deploy the workload, and fill in your favorite name on the \"Basic Information\" page; the \"Container Image\" page can fill in the mirror image `centos`; Start command `sh,-c,while true; do echo $(date -u) >> /data/out.txt; sleep 5; done`; \"Mount Volume\" select \"Existing Volume\", and then select PVC created in one step, fill in the path in the container with `/data` as follows:\n\n![kubesphere_deployment_en](../images/kubesphere_deployment_en.png)\n\n![kubesphere_workload_en](../images/kubesphere_workload_en.png)\n\nAfter the deployment completed, you can see the running pod:\n\n![kubesphere_pod_en](../images/kubesphere_pod_en.png)\n\n### Create StorageClass\n\nIf you did not create a `StorageClass` when installing JuiceFS CSI Driver, or you need to create a new one, you can follow the steps below:\n\nAfter preparing the metadata service and object storage service, create a new `Secret`. On the \"Platform Management\" page, select \"Configuration\", select \"Secret\", and click the \"Create\" button to create a new one:\n\n![kubesphere_secret_en](../images/kubesphere_secret_en.png)\n\nFill in the metadata service and object storage information in \"Data Settings\", as follows:\n\n![kubesphere_update_secret_en](../images/kubesphere_update_secret_en.png)\n\nAfter creating `Secret`, create `StorageClass`, select \"Storage\" on the \"Platform Management\" page, select \"Storage Classes\", click the \"Create\" button to create a new one, and select \"Custom\" for \"Storage Class\":\n\n![kubesphere_sc_create_en](../images/kubesphere_sc_create_en.png)\n\nThe setting page information is as follows, where \"Storage System\" fills in `csi.juicefs.com`, and 4 more parameters are set:\n\n- `csi.storage.k8s.io/provisioner-secret-name`: secret name\n- `csi.storage.k8s.io/provisioner-secret-namespace`: project of secret\n- `csi.storage.k8s.io/node-publish-secret-name`: secret name\n- `csi.storage.k8s.io/node-publish-secret-namespace`: project of secret\n\n![kubesphere_sc_update_en](../images/kubesphere_sc_update_en.png)\n\nAfter clicking the \"Create\" button, the `StorageClass` is created.\n" }, { "path": "docs/en/tutorials/juicefs_on_rancher.md", "content": "---\ntitle: Use JuiceFS on Rancher\nsidebar_position: 2\nslug: /juicefs_on_rancher\n---\n\n[Rancher](https://rancher.com) is an enterprise-level Kubernetes cluster management system, which can be used to quickly complete the deployment of Kubernetes clusters on various cloud computing platforms.\n\nRancher provides a browser-based management interface, even users who are not experienced in Kubernetes can start to manage and use easily. It is preset with Helm-based application market by default, and various Kubernetes applications can be installed very easy under the graphical interface.\n\nThis article will introduce how to deploy Rancher on a Linux system and create a Kubernetes cluster with it, and then deploy JuiceFS CSI Driver with one click through the application market, thereby providing data persistence for various applications on the cluster.\n\n## Install Rancher\n\nRancher can be installed on almost all modern Linux distributions. It can be installed directly on the operating system, or on Docker, Kubernetes, K3s or RKE. The installation is \"Product-Ready\" no matter which environment it is installed in.\n\nHere we choose to install Rancher on Docker, with the following requirements:\n\n- **Operating System**: Linux system with x86-64 architecture\n- **Memory**: 4GB or more\n- **Docker**: 19.03+\n\nRun the following command to install Rancher:\n\n```shell\nsudo docker run --privileged -d --restart=unless-stopped -p 80:80 -p 443:443 rancher/rancher\n```\n\nAfter the container is created, Rancher's management interface can be opened by accessing the IP address of the host.\n\n![Rancher-welcome](../images/rancher-welcome.jpeg)\n\n## Create a Kubernetes cluster\n\nAfter Rancher is installed, you can see that it has deployed a K3s cluster in the current container, and Rancher related resources are running in this internal K3s cluster, but we don't need to pay attention to this cluster now.\n\nNext, start to create a Kubernetes cluster. In the Cluster section of the welcome page, click `Create` to create a cluster. Rancher supports the creation of Kubernetes clusters on major cloud computing platforms. Here we need to create a cluster directly on Rancher's host, so choose `Custom`. Then fill in the cluster name according to the wizard and select the Kubernetes version.\n\n![Rancher-cluster-create](../images/rancher-cluster-create.jpg)\n\nIn the `Cluster Options` page, select the node role to be created, then copy the generated command and execute it on the target host.\n\n![Rancher-cluster-options](../images/rancher-cluster-options.jpg)\n\nAfter the cluster is created, it will be displayed in Rancher's cluster list.\n\n![Rancher-clusters](../images/rancher-clusters.jpg)\n\n## One-click installation of JuiceFS CSI Driver\n\nIn the cluster list, click to enter the Kubernetes cluster, click on the left navigation menu to expand `Apps & Marketplace` -> `Chart Repositories`, click the `Create` button to add JuiceFS CSI repository, fill in:\n\n- **Name**: `juicefs`\n- **Index URL**: `https://juicedata.github.io/charts/`\n\n![Rancher-new-repo](../images/rancher-new-repo.jpg)\n\nAnd then, you can see the new repository in the list.\n\n![Rancher-repos](../images/rancher-repos.jpg)\n\nThen click to open the `Apps & Marketplace` → `Charts` from the left menu, type `juicefs` in the search bar, and then click to open `juicefs-csi-driver`.\n\n![Rancher-chart-search](../images/rancher-chart-search.jpg)\n\nClick the \"Install\" button on the application details page, the latest version will be installed by default, or you can click to switch to the historical version to install.\n\n![Rancher-chart-info](../images/rancher-chart-info.jpg)\n\nThe installation wizard has two steps:\n\n### Step 1: Set up the `Namespace`\n\nThe JuiceFS CSI Driver defaults to `kube-system`, and there is no need to set this step.\n\n### Step 2: Adjust configuration parameters\n\nThis page provides a YAML editor, you can adjust JuiceFS-related information according to your needs. Usually you only need to modify the `storageClasses` part, where the `backend` part is used to define the backend database and object storage of the file system. If you are using an existing file system, you only need to fill in the two items `metaurl` and `name`, for example:\n\n```yaml\n...\nstorageClasses:\n - backend:\n accessKey: ''\n bucket: ''\n metaurl: 'redis://:mypasswd@efgh123.redis.rds.aliyuncs.com/1'\n name: myjfs\n secretKey: ''\n storage: ''\n enabled: true\n name: juicefs-sc\n reclaimPolicy: Retain\n...\n```\n\n> **Tip**: If you have multiple JuiceFS file systems that need to be associated with different storageClasses in the Kubernetes cluster, you can add storageClass configuration items after the `storageClasses` array. Pay attention to modify the name of the storage class to avoid conflicts.\n\nClick \"Install\" and wait for the application installation to complete.\n\n![Rancher-chart-installed](../images/rancher-chart-installed.jpg)\n\n## Use JuiceFS to persist data\n\nWhen deploying an application, specify `juicefs-sc` in the storage configuration.\n\n![Rancher-PVC](../images/rancher-pvc.jpg)\n" }, { "path": "docs/en/tutorials/juicefs_on_wsl.md", "content": "---\ntitle: Use JuiceFS on WSL\nsidebar_position: 9\n---\n\nWSL is called Windows Subsystem for Linux, which means Windows subsystem for Linux. It allows you to run most GNU/Linux native commands, tools, and programs in a Windows environment without the additional hardware overhead of using a virtual machine or dual system.\n\n## Installing WSL\n\nUsing WSL requires Windows 10 2004 or higher or Windows 11.\n\nTo check the current system version, you can call up the Run program by pressing Win + R. Type and run `winver`.\n\n![WSL/winver-en](../images/wsl/winver-en.png)\n\nAfter confirming the Windows version, open PowerShell or Windows Command Prompt as an administrator and run the installation command.\n\n```powershell\nwsl --install\n```\n\nThis command will download the latest Linux kernel, install and set WSL 2 as the default version, and install the Linux distribution (Ubuntu by default).\n\nYou can also specify the distribution to be installed directly at:\n\n```powershell\nwsl --install -d ubuntu\n```\n\n:::tip\n`wsl --list --online` to view all available distributions.\n:::\n\n## Setting up Linux users and passwords\n\nOnce the WSL installation is complete, you can find the newly installed Linux distribution in the Start menu.\n\n![WSL/startmenu-en](../images/wsl/startmenu-en.png)\n\nBy clicking on the Ubuntu subsystem shortcut, WSL will open the terminal of the Linux subsystem. The first time you run it, you will be asked to set the user and password for managing the Linux subsystem, just follow the prompts.\n\n![WSL/init](../images/wsl/init.png)\n\nThere are several points to note about the username and password set here:\n\n- This user is dedicated to the administration of this Linux subsystem and is not related to the users on the Windows system.\n- This user will be the default user of the Linux subsystem and will be automatically logged in at boot time.\n- this user will be considered as the administrator of the Linux subsystem and will be allowed to execute `sudo` commands.\n- Multiple Linux subsystems are allowed to run at the same time in WSL, and each subsystem needs to have an administrative user.\n\n## Using JuiceFS in WSL\n\nUsing JuiceFS in WSL means using JuiceFS on a Linux system, and here is an example of the Community Edition.\n\n### Install the client\n\nInstall the JuiceFS client on the Linux subsystem by executing the following command.\n\n ```shell\n curl -sSL https://d.juicefs.com/install | sh -\n ```\n\n### Creating a file system\n\nJuiceFS is a distributed file system with data and metadata separated, usually using object storage as data storage and Redis, PostgreSQL or MySQL as metadata storage. It is assumed here that the following materials have been prepared.\n\n#### Object Storage\n\nView \"[JuiceFS Supported Object Storage](../reference/how_to_set_up_object_storage.md)\"\n\n- **Bucket Endpoint**: `https://myjfs.oss-cn-shanghai.aliyuncs.com`\n- **Access Key ID**: `ABCDEFGHIJKLMNopqXYZ`\n- **Access Key Secret**: `ZYXwvutsrqpoNMLkJiHgfeDCBA`\n\n#### Database\n\nView \"[JuiceFS Supported Metadata Engines](../reference/how_to_set_up_metadata_engine.md)\"\n\n- **Database URL**: `myjfs-sh-abc.redis.rds.aliyuncs.com:6379`\n- **Database Password**: `mypassword`\n\nWrite private information to environment variables:\n\n```shell\nexport ACCESS_KEY=ABCDEFGHIJKLMNopqXYZ\nexport SECRET_KEY=ZYXwvutsrqpoNMLkJiHgfeDCBA\nexport REDIS_PASSWORD=mypassword\n```\n\nCreate a file system named `myjfs`:\n\n```shell\njuicefs format \\\n --storage oss \\\n --bucket https://myjfs.oss-cn-shanghai.aliyuncs.com \\\n redis://myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n myjfs\n```\n\n### Mount and use\n\nWrite the database password to the environment variable:\n\n```shell\nexport REDIS_PASSWORD=mypassword\n```\n\n:::note\nOnce the file system is created successfully, the corresponding key information will be written to the database and the JuiceFS client will automatically read it from the database when the file system is mounted, so there is no need to set it again.\n:::\n\nMount the file system to `mnt` in the user's home directory:\n\n```shell\nsudo juicefs mount -d redis://myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1 $HOME/mnt\n```\n\nIf you need to access the JuiceFS file system mounted on a Linux subsystem from a Windows system, find the Linux subsystem in the list on the left side of Explorer, then find and open the mount point path.\n\n![WSL/access-jfs-from-win-en](../images/wsl/access-jfs-from-win-en.png)\n\nFor more information on the use of JuiceFS, please refer to the official documentation.\n\n## WSL Storage Performance\n\nWSL bridges the Windows and Linux subsystems, allowing them to access each other's files stored on each other's systems.\n\n![WSL/Windows-to-Linux-en](../images/wsl/windows-to-linux-en.png)\n\nNote, however, that accessing the Linux subsystem from Windows or accessing Windows from the Linux subsystem is bound to incur some performance overhead due to switching between systems. Therefore, the recommended practice is to decide where to store the files depending on the system where the program is located, and for programs in the Linux subsystem, the files it will be processing should also be stored in the Linux subsystem for better performance.\n\nIn the Linux subsystem, WSL mounts each Windows drive to `/mnt`, for example, the mount point for the C: drive in the Linux subsystem is `/mnt/c`.\n\n![WSL/mount-point](../images/wsl/mount-point.png)\n\nTo ensure optimal performance, when using JuiceFS in WSL, both the storage and cache paths should be set in the Linux subsystem. In other words, you should avoid setting the storage or cache on a Windows partition mount point like `/mnt/c`.\n\nUsing the `bench` benchmarking tool that comes with JuiceFS, the results show that mounting a file system to Windows (e.g. `/mnt/c`) has about 30% lower performance than mounting it inside a Linux subsystem (e.g. `$HOME/mnt`).\n\n## Known Issues\n\nWhen copying files to a Linux subsystem via Windows Explorer, WSL automatically appends a file of the same name with the `Zone.Identifier` identifier to each file. This is an NTFS file system security mechanism intended to track the origin of external files, but it is a bug for WSL and has been reported to the Microsoft development team on GitHub [#7456](https://github.com/microsoft/WSL/issues/7456).\n\nThis issue also affects the same problem when saving files to a mounted JuiceFS file system in the Linux subsystem via Windows Explorer. However, reading and writing JuiceFS file systems inside the Linux subsystem is not affected by this bug.\n\n![WSL/zone-identifier-en](../images/wsl/zone-identifier-en.png)\n" }, { "path": "docs/en/tutorials/qcloud.md", "content": "---\ntitle: Use JuiceFS on Tencent Cloud\nsidebar_position: 8\nslug: /clouds/qcloud\n---\n\nJuiceFS needs to be used with database and object storage together. Here we directly use Tencent Cloud's CVM cloud server, combined with cloud database and COS object storage.\n\n## Preparation\n\nWhen creating cloud computing resources, try to choose the same region, so that resources can access each other through intranet and avoid extra traffic costs by using public network.\n\n### 1. CVM\n\nJuiceFS has no special requirements for server hardware, and the minimum specification of CVM can use JuiceFS stably, usually you just need to choose the configuration that can meet your business.\n\nIn particular, you do not need to buy a new server or reinstall the system to use JuiceFS, JuiceFS is not business invasive and will not cause any interference with your existing systems and programs, you can install and use JuiceFS on your running server.\n\nBy default, JuiceFS takes up 1GB of hard disk space for caching, and you can adjust the size of the cache space as needed. This cache is a data buffer layer between the client and the object storage, and you can get better performance by choosing a cloud drive with better performance.\n\nJuiceFS can be installed on all operating systems provided by Tencent Cloud CVM.\n\n**The specifications of CVM used in this article are as follows:**\n\n| Server Specifications | |\n| --------------------- | ------------------------ |\n| **CPU** | 1 Core |\n| **RAM** | 2 GB |\n| **Storage** | 50 GB |\n| **OS** | Ubuntu Server 20.04 64-bit |\n| **Location** | Shanghai 5 |\n\n### 2. Database\n\nJuiceFS will store all the metadata corresponding to the data in a separate database, and the supported databases are Redis, MySQL, PostgreSQL, TiKV and SQLite.\n\nDepending on the database type, the performance and reliability of metadata varies. For example, Redis runs entirely on memory, which provides the ultimate performance, but is difficult to operate and maintain, and has relatively low reliability. SQLite is a single-file relational database with low performance and is not suitable for large-scale data storage, but it is configuration-free and suitable for scenarios with small amounts of data storage.\n\nIf you are just evaluating the capabilities of JuiceFS, you can manually build the database for use in the CVM. When you want to use JuiceFS in a production environment, the cloud database service of Tencent Cloud is usually a better choice if you don't have a professional database operation and maintenance team.\n\nOf course, you can also use cloud database services provided on other cloud platforms if you wish.However, in this case, you can only access the cloud database through the public network, which means that you must expose the database port to the public network, which has some security risks and requires special attention.\n\nIf you must access the database through the public network, you can enhance the security of your data by strictly limiting the IP addresses that are allowed to access the database through the whitelist feature provided by the cloud database console. On the other hand, if you cannot connect to the cloud database through the public network, then you can check the whitelist of the database.\n\n| Database | Redis | MySQL/PostgreSQL | SQLite |\n| :-------------: | :-----------------------------------------------------: | :----------------------------------------------------------: | :----------------------------------------------------------: |\n| **Performance** | High | Medium | Low |\n| **Management** | High | Medium | Low |\n| **Reliability** | Low | Medium | Low |\n| **Scenario** | Massive data, distributed high-frequency read and write | Massive data, distributed low and medium frequency read and write | Low frequency read and write in single machine for small amount of data |\n\n**This article uses the TencentDB for Redis, which is accessed through a VPC private network interacting with the CVM:**\n\n| Redis version | 5.0 community edition |\n| --------------------------- | ------------------------------------------ |\n| **Instance Specification** | 1GB Memory Edition (standard architecture) |\n| **Connection Address** | 192.168.5.5:6379 |\n| **Available Zone** | Shanghai 5 |\n\nNote that the database connection address depends on the VPC network settings you create, and that creating a Redis instance automatically gets the address in the network segment you define.\n\n### 3. Object Storage COS\n\nJuiceFS stores all data in object storage, and it supports almost all object storage services. However, for the best performance, when using Tencent Cloud CVM, pairing it with Tencent Cloud COS Object Storage is usually the optimal choice. However, please note that selecting CVM and COS Bucket in the same region so that they can be accessed through Tencent Cloud's intranet not only has low latency, but also does not require additional traffic costs.\n\n> **Hint**: The unique access address provided by Tencent Cloud COS supports both intranet and extranet access. When accessing through the intranet, COS will automatically resolve to the intranet IP, and the traffic generated at this time is all intranet traffic, which will not incur traffic costs.\n\nOf course, if you want, you can also use object storage services provided by other cloud platforms, but it is not recommended to do so. First of all, if you access the object storage of other cloud platforms through Tencent Cloud CVM, you have to take the public network, and the object storage will incur traffic costs, and the access latency will be higher compared to this, which may affect the performance of JuiceFS.\n\nTencent Cloud COS has different storage levels, and since JuiceFS needs to interact with object storage frequently, it is recommended to use standard storage. You can use it with COS resource package to reduce the cost.\n\n### API Access Secret Key\n\nTencent Cloud COS needs to be accessed through API, you need to prepare the access secret key, including `Access Key ID` and `Access Key Secret`, [click here to view](https://intl.cloud.tencent.com/document/product/598/32675) to get the way.\n\n> **Security Advisory**: Explicit use of the API access secret key may lead to key compromise and it is recommended to assign [CAM Service Role](https://intl.cloud.tencent.com/document/product/598/19420) to the cloud server. Once a CVM has been granted COS operation privileges, it can access the COS without using the API access key.\n\n## Installation\n\nHere we are using Ubuntu Server 20.04 64-bit system, and the latest version of the client can be installed by running the following command.\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\nYou can also choose another version by visiting the [JuiceFS GitHub Releases](https://github.com/juicedata/juicefs/releases) page.\n\nExecute the command and see the help message `juicefs` returned, which means the client installation is successful.\n\n```shell\n$ juicefs\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 0.15.2 (2021-07-07T05:51:36Z 4c16847)\n\nCOMMANDS:\n format format a volume\n mount mount a volume\n umount unmount a volume\n gateway S3-compatible gateway\n sync sync between two storage\n rmr remove directories recursively\n info show internal information for paths or inodes\n bench run benchmark to read/write/stat big/small files\n gc collect any leaked objects\n fsck Check consistency of file system\n profile analyze access log\n status show status of JuiceFS\n warmup build cache for target directories/files\n dump dump metadata into a JSON file\n load load metadata from a previously dumped JSON file\n help, h Shows a list of commands or help for one command\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q only warning and errors (default: false)\n --trace enable trace log (default: false)\n --no-agent disable pprof (:6060) agent (default: false)\n --help, -h show help (default: false)\n --version, -V print only the version (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\nJuiceFS has good cross-platform compatibility and is supported on Linux, Windows and macOS. This article focuses on the installation and use of JuiceFS on Linux, if you need to know how to install it on other systems, please [check the documentation](../getting-started/installation.md).\n\n## Creating JuiceFS\n\nOnce the JuiceFS client is installed, you can now create the JuiceFS storage using the Redis database and COS you prepared earlier.\n\nTechnically speaking, this step should be called \"Format a volume\". However, since many users may not understand or care about the standard file system terminology, we will simply call the process \"Create JuiceFS Storage\".\n\nThe following command creates a storage called `mystor`, i.e., a file system, using the `format` subcommand provided by the JuiceFS client.\n\n```shell\n$ juicefs format \\\n --storage cos \\\n --bucket https:// \\\n --access-key \\\n --secret-key \\\n redis://:@192.168.5.5:6379/1 \\\n mystor\n```\n\n**Option description:**\n\n- `--storage`: Specify the type of object storage.\n- `---bucket`: Bucket access domain of the object store, which can be found in the COS management console.\n- `--access-key` and `--secret-key`: the secret key pair for accessing the Object Storage API, [click here to view](https://intl.cloud.tencent.com/document/product/598/32675) to get it.\n\n> Redis 6.0 authentication requires two parameters, username and password, and the address format is `redis://username:password@redis-server-url:6379/1`. Currently, the Redis version of Tencent Cloud Database only provides Reids 4.0 and 5.0, which only requires a password for authentication. When setting the Redis server address, you only need to leave the username empty, for example: `redis://:password@redis-server-url:6379/1`\n\nOutput like the following means the file system was created successfully.\n\n```shell\n2021/07/30 11:44:31.904157 juicefs[44060] : Meta address: redis://@192.168.5.5:6379/1\n2021/07/30 11:44:31.907083 juicefs[44060] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/07/30 11:44:31.907634 juicefs[44060] : Ping redis: 474.98µs\n2021/07/30 11:44:31.907850 juicefs[44060] : Data uses cos://juice-0000000000/mystor/\n2021/07/30 11:44:32.149692 juicefs[44060] : Volume is formatted as {Name:mystor UUID:dbf05314-57af-4a2c-8ac1-19329d73170c Storage:cos Bucket:https://juice-0000000000.cos.ap-shanghai.myqcloud.com AccessKey:AKIDGLxxxxxxxxxxxxxxxxxxZ8QRBdpkOkp SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n## Mount JuiceFS\n\nWhen the file system is created, the information related to the object storage is stored in the database, so there is no need to enter information such as the bucket domain and secret key when mounting.\n\nUse the `mount` subcommand to mount the file system to the `/mnt/jfs` directory.\n\n```shell\nsudo juicefs mount -d redis://:@192.168.5.5:6379/1 /mnt/jfs\n```\n\n> **Note**: When mounting the file system, only the Redis database address is required, not the file system name. The default cache path is `/var/jfsCache`, please make sure the current user has enough read/write permissions.\n\nOutput similar to the following means that the file system was mounted successfully.\n\n```shell\n2021/07/30 11:49:56.842211 juicefs[44175] : Meta address: redis://@192.168.5.5:6379/1\n2021/07/30 11:49:56.845100 juicefs[44175] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/07/30 11:49:56.845562 juicefs[44175] : Ping redis: 383.157µs\n2021/07/30 11:49:56.846164 juicefs[44175] : Data use cos://juice-0000000000/mystor/\n2021/07/30 11:49:56.846731 juicefs[44175] : Disk cache (/var/jfsCache/dbf05314-57af-4a2c-8ac1-19329d73170c/): capacity (1024 MB), free ratio (10%), max pending pages (15)\n2021/07/30 11:49:57.354763 juicefs[44175] : OK, mystor is ready at /mnt/jfs\n```\n\nUsing the `df` command, you can see how the file system is mounted.\n\n```shell\n$ df -Th\nFile system type capacity used usable used% mount point\nJuiceFS:mystor fuse.juicefs 1.0P 64K 1.0P 1% /mnt/jfs\n```\n\nAfter the file system is successfully mounted, you can now store data in the `/mnt/jfs` directory as if you were using a local hard drive.\n\n> **Multi-Host Sharing**: JuiceFS storage supports being mounted by multiple cloud servers at the same time. You can install the JuiceFS client on other could server and then use `redis://:@herald-sh-abc.redis.rds.aliyuncs. com:6379/1` database address to mount the file system on each host.\n\n## File System Status\n\nUse the `status` subcommand of the JuiceFS client to view basic information and connection status of a file system.\n\n```shell\n$ juicefs status redis://:@192.168.5.5:6379/1\n\n2021/07/30 11:51:17.864767 juicefs[44196] : Meta address: redis://@192.168.5.5:6379/1\n2021/07/30 11:51:17.866619 juicefs[44196] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/07/30 11:51:17.867092 juicefs[44196] : Ping redis: 379.391µs\n{\n \"Setting\": {\n \"Name\": \"mystor\",\n \"UUID\": \"dbf05314-57af-4a2c-8ac1-19329d73170c\",\n \"Storage\": \"cos\",\n \"Bucket\": \"https://juice-0000000000.cos.ap-shanghai.myqcloud.com\",\n \"AccessKey\": \"AKIDGLxxxxxxxxxxxxxxxxx8QRBdpkOkp\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0\n },\n \"Sessions\": [\n {\n \"Sid\": 1,\n \"Heartbeat\": \"2021-07-30T11:49:56+08:00\",\n \"Version\": \"0.15.2 (2021-07-07T05:51:36Z 4c16847)\",\n \"Hostname\": \"VM-5-6-ubuntu\",\n \"MountPoint\": \"/mnt/jfs\",\n \"ProcessID\": 44175\n },\n {\n \"Sid\": 3,\n \"Heartbeat\": \"2021-07-30T11:50:56+08:00\",\n \"Version\": \"0.15.2 (2021-07-07T05:51:36Z 4c16847)\",\n \"Hostname\": \"VM-5-6-ubuntu\",\n \"MountPoint\": \"/mnt/jfs\",\n \"ProcessID\": 44185\n }\n ]\n}\n```\n\n## Unmount JuiceFS\n\nThe file system can be unmounted using the `umount` command provided by the JuiceFS client, e.g.\n\n```shell\nsudo juicefs umount /mnt/jfs\n```\n\n> **Note**: Forced unmount of the file system in use may result in data corruption or loss, so please be sure to proceed with caution.\n\n## Auto-mount on boot\n\nPlease refer to [\"Mount JuiceFS at Boot Time\"](../administration/mount_at_boot.md) for more details.\n" }, { "path": "docs/en/tutorials/windows.md", "content": "---\ntitle: Using JuiceFS on Windows\nsidebar_position: 1\n---\n\n## Quick Start Video\n\n
\n \n
\n\n## Install JuiceFS Client\n\n:::tip Environment Dependency\nOn Windows, JuiceFS relies on WinFsp to mount the file system. You can download the latest version from the [WinFsp Repository](https://github.com/winfsp/winfsp). After installation, it is recommended to restart your computer to ensure all components are loaded properly.\n:::\n\nThe [installation guide](../getting-started/installation.md#windows) introduces various ways to install JuiceFS on Windows. Here, we detail the manual installation process.\n\n### Step 1: Download JuiceFS Client\n\nGo to the project's [Release page](https://github.com/juicedata/juicefs/releases) and download the latest JuiceFS client, for example, `juicefs-1.3.0-windows-amd64.tar.gz`.\n\n### Step 2: Create Program Directory\n\nFor better management, it is recommended to create a dedicated directory for the JuiceFS client. For example, create a folder named `juicefs` under `C:\\`, and place the extracted `juicefs.exe` inside.\n\n### Step 3: Configure Environment Variables\n\nTo conveniently use the `juicefs` command in the command line, add the JuiceFS client directory to your system's environment variables:\n\n1. Right-click \"This PC\" or \"Computer\" and select \"Properties\";\n2. Click \"Advanced system settings\";\n3. In the \"System Properties\" window, click the \"Environment Variables\" button;\n4. In the \"System variables\" section, find the variable named `Path`, select it and click \"Edit\";\n5. In the edit window, click \"New\" and enter the JuiceFS client directory path, e.g., `C:\\juicefs`;\n6. Click \"OK\" to save changes.\n\n![Windows Environment Variable Settings](https://static1.juicefs.com/docs/windows-path-en.png)\n\n### Step 4: Verify Installation\n\nAfter installation, verify the JuiceFS client via the command line. Open Command Prompt (CMD) or PowerShell and enter:\n\n```bash\njuicefs version\n```\n\nIf installed successfully, you should see output similar to:\n\n```\njuicefs version 1.3.0+2025-07-03.30190ca1094d2\n```\n\n## Create and Mount File System\n\nThe steps to create and mount a JuiceFS file system are similar to other operating systems, but pay attention to Windows command line syntax and path formats.\n\n### Create File System\n\n```shell\njuicefs format --storage oss `\n --bucket https://your-bucket.oss-cn-region.aliyuncs.com `\n --access-key your-access-key `\n --secret-key your-secret-key `\n redis://your-redis-host:6379/0 `\n mywinfs\n```\n\n> Unlike Linux, Windows command lines use backticks (`) for line continuation.\n\n### Mount File System\n\nOn Windows, the mount point must be an unused drive letter (such as X, Y, Z, etc.). This differs from Linux and macOS, which mount file systems to directories.\n\n```shell\njuicefs mount -d redis://your-redis-host:6379/0 X:\n```\n\n## Environment Variable Configuration\n\nFor security, to avoid entering passwords in plain text, you can store sensitive information in environment variables. When mounting the file system or enabling S3 Gateway, the client will automatically read from these variables.\n\nCommon environment variables for JuiceFS on Windows:\n\n| Variable Name | Description |\n|-------------------------|----------------------------|\n| `META_PASSWORD` | Metadata engine password |\n| `MINIO_ROOT_USER` | S3 Gateway Access Key |\n| `MINIO_ROOT_PASSWORD` | S3 Gateway Secret Key |\n\nSet these variables directly in the command line:\n\n```cmd\nset META_PASSWORD=your_password\nset MINIO_ROOT_USER=your_access_key\nset MINIO_ROOT_PASSWORD=your_secret_key\n```\n\nNote: This method only works for the current session. Once the window is closed, the variables are lost and need to be reset.\n\n### Persist Environment Variables\n\nTo automatically load these variables every time Windows starts, set them as system environment variables:\n\n1. **Open System Environment Variable Settings**\n - Press `Win + S`, search for and open \"Edit the system environment variables\".\n - Click the \"Environment Variables\" button.\n\n ![System Environment Variable Settings](https://static1.juicefs.com/docs/win_env_01.png)\n\n2. **Create System-Level Environment Variable**\n - In the \"System variables\" area, click \"New\".\n - **Variable name**: e.g., `META_PASSWORD`\n - **Variable value**: Enter the password or key\n - Click \"OK\" to save.\n\n ![Add Environment Variable](https://static1.juicefs.com/docs/win_env_02.png)\n\n ![Add Environment Variable](https://static1.juicefs.com/docs/win_env_03.png)\n\n3. **Verify Environment Variable**\n\n Reopen the terminal and try mounting the file system without specifying the password. If successful, the environment variable is effective.\n\n## Auto-Mount on Startup\n\nThere are several ways to enable auto-mount on startup in Windows. This section introduces the method using \"Task Scheduler\".\n\n1. Open \"Task Scheduler\" and click \"Create Task\".\n\n ![Task Scheduler](https://static1.juicefs.com/docs/task_00.png)\n\n2. In the \"General\" tab, set the task name (e.g., `JuiceFS_AutoMount`) and check \"Run with highest privileges\".\n\n ![General Settings](https://static1.juicefs.com/docs/task_01.png)\n\n3. Switch to the \"Triggers\" tab, click \"New\", and select \"At system startup\" as the trigger.\n\n ![Trigger Settings](https://static1.juicefs.com/docs/task_02.png)\n\n4. Switch to the \"Actions\" tab, click \"New\", and fill in:\n\n - **Program/script**: Browse to select the JuiceFS client path (e.g., `C:\\juicefs\\juicefs.exe`).\n - **Arguments**: Enter the mount command parameters. It is recommended to use system environment variables for the metadata engine password to avoid plain text input here.\n\n ![Action Settings](https://static1.juicefs.com/docs/task_03.png)\n\n5. In the \"Conditions\" tab, check \"Start only if the network connection is available\" to ensure the mount operation runs when the network is ready.\n\n ![Condition Settings](https://static1.juicefs.com/docs/task_04.png)\n\n6. Click \"OK\" to save the task.\n\n**Notes:**\n\n- Ensure the mount command parameters are correct; do not include the password in the command (it is stored in environment variables).\n- To unmount the file system: right-click the mounted drive letter and select \"Disconnect\".\n" }, { "path": "docs/zh_cn/administration/destroy.md", "content": "---\ntitle: 销毁文件系统\nsidebar_position: 8\n---\n\nJuiceFS 客户端提供了 `destroy` 命令用以彻底销毁一个文件系统，销毁操作将会产生以下结果：\n\n- 清空此文件系统的全部元数据记录；\n- 清空此文件系统的全部数据块\n\n销毁文件系统的命令格式如下：\n\n```shell\njuicefs destroy \n```\n\n- ``：元数据引擎的 URL 地址；\n- ``：文件系统的 UUID。\n\n## 查找文件系统的 UUID\n\nJuiceFS 客户端的 `status` 命令可以查看一个文件系统的详细信息，只需指定文件系统的元数据引擎 URL 即可，例如：\n\n```shell {8}\n$ juicefs status redis://127.0.0.1:6379\n\n2022/01/26 21:41:37.577645 juicefs[31181] : Meta address: redis://127.0.0.1:6379\n2022/01/26 21:41:37.578238 juicefs[31181] : Ping redis: 55.041µs\n{\n \"Setting\": {\n \"Name\": \"macjfs\",\n \"UUID\": \"eabb96d5-7228-461e-9240-fddbf2b576d8\",\n \"Storage\": \"file\",\n \"Bucket\": \"jfs/\",\n \"AccessKey\": \"\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0,\n \"TrashDays\": 1\n },\n ...\n}\n```\n\n## 销毁文件系统\n\n:::danger 危险操作\n销毁操作将导致文件系统关联的数据库记录和对象存储中的数据全部被清空，请务必先备份重要数据后再操作！\n:::\n\n```shell {1}\n$ juicefs destroy redis://127.0.0.1:6379 eabb96d5-7228-461e-9240-fddbf2b576d8\n\n2022/01/26 21:52:17.488987 juicefs[31518] : Meta address: redis://127.0.0.1:6379\n2022/01/26 21:52:17.489668 juicefs[31518] : Ping redis: 55.542µs\n volume name: macjfs\n volume UUID: eabb96d5-7228-461e-9240-fddbf2b576d8\ndata storage: file://jfs/\n used bytes: 18620416\n used inodes: 23\nWARNING: The target volume will be destroyed permanently, including:\nWARNING: 1. objects in the data storage\nWARNING: 2. entries in the metadata engine\nProceed anyway? [y/N]: y\ndeleting objects: 68\nThe volume has been destroyed! You may need to delete cache directory manually.\n```\n\n在销毁文件系统时，客户端会发出确认提示，请务必仔细核对文件系统信息，确认无误后输入 `y` 确认。\n\n## 常见错误\n\n```shell\n2022/01/26 21:47:30.949149 juicefs[31483] : 1 sessions are active, please disconnect them first\n```\n\n如果收到类似上面的错误提示，说明文件系统没有被妥善卸载，请检查并确认卸载了所有挂载点后再行操作。\n" }, { "path": "docs/zh_cn/administration/fault_diagnosis_and_analysis.md", "content": "---\ntitle: 问题排查方法\nsidebar_position: 5\nslug: /fault_diagnosis_and_analysis\ndescription: 本文介绍 JuiceFS 挂载点、CSI 驱动、Hadoop Java SDK、S3 网关等客户端的问题排查方法。\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n## 客户端日志 {#client-log}\n\nJuiceFS 客户端在运行过程中会输出日志用于故障诊断，日志等级从低到高分别是：DEBUG、INFO、WARNING、ERROR、FATAL，默认只输出 INFO 级别以上的日志。如果需要输出 DEBUG 级别的日志，需要在运行 JuiceFS 客户端时显式开启，如加上 `--debug` 选项。\n\n不同 JuiceFS 客户端获取日志的方式不同，以下分别介绍。\n\n### 挂载点\n\n当挂载 JuiceFS 文件系统时加上了 [`-d` 选项](../reference/command_reference.mdx#mount)（表示后台运行），日志会同时输出到系统日志和本地日志文件，取决于挂载文件系统时的运行用户，本地日志文件的路径稍有区别。root 用户对应的日志文件路径是 `/var/log/juicefs.log`，非 root 用户的日志文件路径是 `$HOME/.juicefs/juicefs.log`，具体请参见 [`--log` 选项](../reference/command_reference.mdx#mount)。\n\n取决于你使用的操作系统，你可以通过不同的命令获取系统日志或直接读取本地日志文件：\n\n\n \n\n```bash\ntail -n 100 /var/log/juicefs.log\n```\n\n \n \n\n```bash\nsyslog | grep 'juicefs'\n```\n\n \n \n\n```bash\ncat /var/log/syslog | grep 'juicefs'\n```\n\n \n \n\n```bash\ncat /var/log/messages | grep 'juicefs'\n```\n\n \n\n\n你可以使用 `grep` 命令过滤显示不同等级的日志信息，从而进行性能统计和故障追踪，例如：\n\n```shell\ncat /var/log/syslog | grep 'juicefs' | grep ''\n```\n\n### Kubernetes CSI 驱动\n\n根据你使用的 JuiceFS CSI 驱动版本会有不同的获取日志的方式，具体请参考 [CSI 驱动文档](https://juicefs.com/docs/zh/csi/troubleshooting)。\n\n### S3 网关\n\nS3 网关仅支持在前台运行，因此客户端日志会直接输出到终端。如果你是在 Kubernetes 中部署 S3 网关，需要查看对应 pod 的日志。\n\n### Hadoop Java SDK\n\n使用 JuiceFS Hadoop Java SDK 的应用进程（如 Spark executor）的日志中会包含 JuiceFS 客户端日志，因为和应用自身产生的日志混杂在一起，需要通过特定关键词来过滤筛选（如 `juicefs`，注意这里忽略了大小写）。\n\n## 文件系统访问日志 {#access-log}\n\n每个 JuiceFS 客户端都有一个访问日志，其中详细记录了文件系统上的所有操作，如操作类型、用户 ID、用户组 ID、文件 inode 及其花费的时间。访问日志可以有多种用途，如性能分析、审计、故障诊断。\n\n### 日志格式\n\n访问日志的示例格式如下：\n\n```\n2021.01.15 08:26:11.003330 [uid:0,gid:0,pid:4403] write (17669,8666,4993160): OK <0.000010>\n```\n\n其中每一列的含义为：\n\n- `2021.01.15 08:26:11.003330`：当前操作的时间\n- `[uid:0,gid:0,pid:4403]`：当前操作的用户 ID、用户组 ID、进程 ID\n- `write`：操作类型\n- `(17669,8666,4993160)`：当前操作类型的输入参数，如示例中的 `write` 操作的输入参数分别为写入文件的 inode、写入数据的大小、写入文件的偏移。不同操作类型的参数不同，具体请参考 [`vfs.go`](https://github.com/juicedata/juicefs/blob/main/pkg/vfs/vfs.go) 文件。\n- `OK`：当前操作是否成功，如果不成功会输出具体的失败信息。\n- `<0.000010>`：当前操作花费的时间（以秒为单位）\n\n访问日志量很大，直接阅读难以把握系统性能情况，推荐使用 [`juicefs profile`](#profile) 直接基于日志进行性能可视化分析。\n\n不同 JuiceFS 客户端获取访问日志的方式不同，以下分别介绍。\n\n### 挂载点\n\n在 JuiceFS 文件系统挂载点的根目录中有一个名为 `.accesslog` 的虚拟文件，通过 `cat` 命令可以查看其中的内容（命令不会退出），例如（假设挂载点根目录为 `/jfs`）：\n\n```bash\ncat /jfs/.accesslog\n```\n\n```output\n2021.01.15 08:26:11.003330 [uid:0,gid:0,pid:4403] write (17669,8666,4993160): OK <0.000010>\n2021.01.15 08:26:11.003473 [uid:0,gid:0,pid:4403] write (17675,198,997439): OK <0.000014>\n2021.01.15 08:26:11.003616 [uid:0,gid:0,pid:4403] write (17666,390,951582): OK <0.000006>\n```\n\n### Kubernetes CSI 驱动\n\n请参考 [CSI 驱动文档](https://juicefs.com/docs/zh/csi/troubleshooting)及根据你使用的 JuiceFS CSI 驱动版本来找到 mount pod 或者 CSI 驱动 pod，在 pod 内的 JuiceFS 文件系统挂载点根目录查看 `.accesslog` 文件即可。Pod 内的挂载点路径为 `/jfs/`，假设 mount pod 的名称叫 `juicefs-1.2.3.4-pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373`，`` 为 `pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373`，可以使用如下命令查看：\n\n```bash\nkubectl -n kube-system exec juicefs-1.2.3.4-pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373 -- cat /jfs/pvc-d4b8fb4f-2c0b-48e8-a2dc-530799435373/.accesslog\n```\n\n### S3 网关\n\n需要在启动 S3 网关时新增 [`--access-log` 选项](../reference/command_reference.mdx#gateway)，指定访问日志输出的路径，默认 S3 网关不输出访问日志。\n\n### Hadoop Java SDK\n\n需要在 JuiceFS Hadoop Java SDK 的[客户端配置](../deployment/hadoop_java_sdk.md#其它配置)中新增 `juicefs.access-log` 配置项，指定访问日志输出的路径，默认不输出访问日志。\n\n## 使用 debug 子命令收集各类信息 {#debug}\n\n`juicefs debug` 子命令可以自动搜集指定挂载点的各类信息，方便进行故障诊断。\n\n```shell\njuicefs debug \n```\n\n该命令会收集以下信息：\n\n1. JuiceFS version\n2. 操作系统版本与内核版本\n3. JuiceFS .config 内部文件内容\n4. JuiceFS .stat 内部文件的内容并且在 5s 后再记录一次\n5. mount 命令行参数\n6. Go pprof\n7. JuiceFS 日志（默认最后 5000 行）\n\n默认会在当前目录下创建 debug 目录，并将收集到的信息保存在该目录下。下面是一个示例：\n\n```shell\n$ juicefs debug /tmp/mountpoint\n\n$ tree ./debug\n./debug\n├── tmp-test1-20230609104324\n│ ├── config.txt\n│ ├── juicefs.log\n│ ├── pprof\n│ │ ├── juicefs.allocs.pb.gz\n│ │ ├── juicefs.block.pb.gz\n│ │ ├── juicefs.cmdline.txt\n│ │ ├── juicefs.goroutine.pb.gz\n│ │ ├── juicefs.goroutine.stack.txt\n│ │ ├── juicefs.heap.pb.gz\n│ │ ├── juicefs.mutex.pb.gz\n│ │ ├── juicefs.profile.30s.pb.gz\n│ │ ├── juicefs.threadcreate.pb.gz\n│ │ └── juicefs.trace.5s.pb.gz\n│ ├── stats.5s.txt\n│ ├── stats.txt\n│ └── system-info.log\n└── tmp-test1-20230609104324.zip\n```\n\n## 实时性能监控 {#performance-monitor}\n\nJuiceFS 客户端提供 `profile` 和 `stats` 两个子命令来对性能数据进行可视化呈现。其中，`profile` 命令通过读取[「文件系统请求日志」](#access-log)进行汇总输出，而 `stats` 则依赖[客户端监控数据](../administration/monitoring.md)。\n\n### `juicefs profile` {#profile}\n\n[`juicefs profile`](../reference/command_reference.mdx#profile) 会对[「文件系统访问日志」](#access-log)进行汇总，运行 `juicefs profile MOUNTPOINT` 命令，便能看到根据最新访问日志获取的各个文件系统操作的实时统计信息：\n\n![JuiceFS-profiling](../images/juicefs-profiling.gif)\n\n除了对挂载点进行实时分析，该命令还提供回放模式，可以对预先收集的日志进行回放分析：\n\n```shell\n# 预先收集日志\ncat /jfs/.accesslog > /tmp/juicefs.accesslog\n\n# 性能问题复现后，重放日志，分析各调用耗时，找出性能瓶颈\njuicefs profile /tmp/juicefs.accesslog\n```\n\n如果认为回放日志的速度太快，可以用 Enter/Return 暂停／继续回放。如果太慢，则设置 `--interval 0` 来立即回放整个日志文件并直接显示统计结果。\n\n如果只对某个用户或进程感兴趣，可以通过指定其 ID 来过滤掉其他用户或进程。例如：\n\n```bash\njuicefs profile /tmp/juicefs.accesslog --uid 12345\n```\n\n### `juicefs stats` {#stats}\n\n[`juicefs stats`](../reference/command_reference.mdx#stats) 命令通过读取 JuiceFS 客户端的监控数据，以类似 Linux `dstat` 工具的形式实时打印各个指标的每秒变化情况：\n\n![juicefs_stats_watcher](../images/juicefs_stats_watcher.png)\n\n各个板块指标介绍：\n\n#### `usage`\n\n- `cpu`：进程的 CPU 使用率。\n- `mem`：进程的物理内存使用量。\n- `buf`：进程已使用的[读写缓冲区](../guide/cache.md#buffer-size)大小，如果该数值逼近甚至超过客户端所设置的 [`--buffer-size`](../reference/command_reference.mdx#mount-data-cache-options)，说明读写缓冲区空间不足，需要视情况扩大，或者降低应用读写负载。\n- `cache`：内部指标，无需关注。\n\n#### `fuse`\n\n- `ops`/`lat`：通过 FUSE 接口处理的每秒请求数及其平均时延，单位为毫秒。\n- `read`/`write`：通过 FUSE 接口处理的读写带宽。\n\n#### `meta`\n\n- `ops`/`lat`：每秒处理的元数据请求数和平均时延，单位为毫秒。注意部分能在缓存中直接处理的元数据请求未列入统计，以更好地体现客户端与元数据引擎交互的耗时。\n- `txn`/`lat`：元数据引擎每秒处理的写事务个数及其平均时延，单位为毫秒。只读请求如 `getattr` 只会计入 `ops` 而不会计入 `txn`。\n- `retry`：元数据引擎每秒重试写事务的次数。\n\n#### `blockcache`\n\n`blockcache` 代表本地数据缓存，如果读请求已经被内核缓存，那么流量将不会体现在 `blockcache` 相关指标下。因此如果反复读取相同文件，却发现持续产生 `blockcache` 流量，说明文件始终未能被内核页缓存收录，考虑往该方向排查（比如内存吃紧，不足以缓存更多文件）。\n\n- `read`/`write`：客户端本地数据缓存的每秒读写流量。\n\n#### `object`\n\n`object` 代表与对象存储相关指标，在缓存场景下，读请求穿透到对象存储，将会明显降低读性能，可以用该指标来断定数据是否完整缓存。另一方面，通过对比 GET 请求流量和 FUSE 读流量的关系，也能初步判断[读放大](./troubleshooting.md#read-amplification)的情况。\n\n- `get`/`get_c`/`lat`：对象存储每秒处理读请求的带宽值，请求个数及其平均时延（单位为毫秒）。\n- `put`/`put_c`/`lat`：对象存储每秒处理写请求的带宽值，请求个数及其平均时延（单位为毫秒）。\n- `del_c`/`lat`：对象存储每秒处理删除请求的个数和平均时延（单位为毫秒）。\n\n## 用 pprof 获取运行时信息 {#runtime-information}\n\nJuiceFS 客户端默认会通过 [pprof](https://pkg.go.dev/net/http/pprof) 在本地监听一个 TCP 端口用以获取运行时信息，如 Goroutine 堆栈信息、CPU 性能统计、内存分配统计。你可以通过挂载点下的 `.config` 文件查看当前 JuiceFS 客户端监听的具体端口号：\n\n```shell\n# 假设挂载点是 /jfs\n$ cat /jfs/.config | grep 'DebugAgent'\n \"DebugAgent\": \"127.0.0.1:6064\",\n```\n\n默认 pprof 监听的端口号范围是从 6060 开始至 6099 结束，从上面的示例中可以看到实际的端口号是 6064。在获取到监听端口号以后就可以通过 `http://localhost:/debug/pprof` 地址查看所有可供查询的运行时信息，一些重要的运行时信息如下：\n\n- Goroutine 堆栈信息：`http://localhost:/debug/pprof/goroutine?debug=1`\n- CPU 性能统计：`http://localhost:/debug/pprof/profile?seconds=30`\n- 内存分配统计：`http://localhost:/debug/pprof/heap`\n\n为了便于分析这些运行时信息，可以将它们保存到本地，例如：\n\n```bash\ncurl 'http://localhost:/debug/pprof/goroutine?debug=1' > juicefs.goroutine.txt\n```\n\n```bash\ncurl 'http://localhost:/debug/pprof/profile?seconds=30' > juicefs.cpu.pb.gz\n```\n\n```bash\ncurl 'http://localhost:/debug/pprof/heap' > juicefs.heap.pb.gz\n```\n\n:::tip 建议\n你也可以使用 `juicefs debug` 命令自动收集这些运行时信息并保存到本地，默认保存到当前目录下的 `debug` 目录中，例如：\n\n```bash\njuicefs debug /mnt/jfs\n```\n\n关于 `juicefs debug` 命令的更多信息，请查看[命令参考](../reference/command_reference.mdx#debug)。\n:::\n\n如果你安装了 `go` 命令，那么可以通过 `go tool pprof` 命令直接分析，例如分析 CPU 性能统计：\n\n```bash\n$ go tool pprof 'http://localhost:/debug/pprof/profile?seconds=30'\nFetching profile over HTTP from http://localhost:/debug/pprof/profile?seconds=30\nSaved profile in /Users/xxx/pprof/pprof.samples.cpu.001.pb.gz\nType: cpu\nTime: Dec 17, 2021 at 1:41pm (CST)\nDuration: 30.12s, Total samples = 32.06s (106.42%)\nEntering interactive mode (type \"help\" for commands, \"o\" for options)\n(pprof) top\nShowing nodes accounting for 30.57s, 95.35% of 32.06s total\nDropped 285 nodes (cum <= 0.16s)\nShowing top 10 nodes out of 192\n flat flat% sum% cum cum%\n 14.73s 45.95% 45.95% 14.74s 45.98% runtime.cgocall\n 7.39s 23.05% 69.00% 7.41s 23.11% syscall.syscall\n 2.92s 9.11% 78.10% 2.92s 9.11% runtime.pthread_cond_wait\n 2.35s 7.33% 85.43% 2.35s 7.33% runtime.pthread_cond_signal\n 1.13s 3.52% 88.96% 1.14s 3.56% runtime.nanotime1\n 0.77s 2.40% 91.36% 0.77s 2.40% syscall.Syscall\n 0.49s 1.53% 92.89% 0.49s 1.53% runtime.memmove\n 0.31s 0.97% 93.86% 0.31s 0.97% runtime.kevent\n 0.27s 0.84% 94.70% 0.27s 0.84% runtime.usleep\n 0.21s 0.66% 95.35% 0.21s 0.66% runtime.madvise\n```\n\n也可以将运行时信息导出为可视化图表，以更加直观的方式进行分析。可视化图表支持导出为多种格式，如 HTML、PDF、SVG、PNG 等。例如导出内存分配统计信息为 PDF 文件的命令如下：\n\n:::note 注意\n导出为可视化图表功能依赖 [Graphviz](https://graphviz.org)，请先将它安装好。\n:::\n\n```bash\ngo tool pprof -pdf 'http://localhost:/debug/pprof/heap' > juicefs.heap.pdf\n```\n\n关于 pprof 的更多信息，请查看[官方文档](https://github.com/google/pprof/blob/main/doc/README.md)。\n\n### 使用 Pyroscope 进行性能剖析 {#use-pyroscope}\n\n![Pyroscope](../images/pyroscope.png)\n\n[Pyroscope](https://github.com/pyroscope-io/pyroscope) 是一个开源的持续性能剖析平台。它能够帮你：\n\n+ 找出源代码中的性能问题和瓶颈\n+ 解决 CPU 利用率高的问题\n+ 理解应用程序的调用树（call tree）\n+ 追踪随一段时间内变化的情况\n\nJuiceFS 支持使用 `--pyroscope` 选项传入 Pyroscope 服务端地址，指标以每隔 10 秒的频率推送到服务端。如果服务端开启了权限校验，校验信息 API Key 可以通过环境变量 `PYROSCOPE_AUTH_TOKEN` 传入：\n\n```bash\nexport PYROSCOPE_AUTH_TOKEN=xxxxxxxxxxxxxxxx\njuicefs mount --pyroscope http://localhost:4040 redis://localhost /mnt/jfs\njuicefs dump --pyroscope http://localhost:4040 redis://localhost dump.json\n```\n" }, { "path": "docs/zh_cn/administration/metadata/_category_.yml", "content": "label: \"Metadata Engine Best Practices\"\nposition: 1" }, { "path": "docs/zh_cn/administration/metadata/etcd_best_practices.md", "content": "---\nsidebar_label: etcd\nsidebar_position: 4\nslug: /etcd_best_practices\n---\n\n# etcd 最佳实践\n\n## 数据规模\n\netcd 默认设置了 2GB 的[存储配额](https://etcd.io/docs/latest/op-guide/maintenance/#space-quota)，大概能够支撑存储两百万文件的元数据，可以通过 `--quota-backend-bytes` 选项进行调整，[官方建议](https://etcd.io/docs/latest/dev-guide/limit)不要超过 8GB。\n\n默认情况下，etcd 会保留所有数据的修改历史，直到数据量超过存储配额导致无法提供服务，建议加上如下选项启用[自动数据合并](https://etcd.io/docs/latest/op-guide/maintenance/#auto-compaction)：\n\n```\n--auto-compaction-mode revision --auto-compaction-retention 1000000\n```\n\n当数据量达到配额导致无法写入时，可以通过手动压缩（`etcdctl compact`）和整理碎片（`etcdctl defrag`）的方式来减少容量。**强烈建议对 etcd 集群的节点逐个进行这些操作，否则可能会导致整个 etcd 集群不可用。**\n\n## 性能\n\netcd 提供强一致的读写访问，并且所有操作都会涉及到多机事务以及磁盘的数据持久化。**建议使用高性能的 SSD 来部署**，否则会影响到文件系统的性能。更多硬件配置建议请参考[官方文档](https://etcd.io/docs/latest/op-guide/hardware)。\n\n如果 etcd 集群都有掉电保护，或者其它能够保证不会导致所有节点同时宕机的措施，也可以通过 `--unsafe-no-fsync` 选项关闭数据同步落盘，以降低访问时延提高文件系统的性能。**此时如果有两个节点同时宕机，会有数据丢失风险。**\n\n## Kubernetes\n\n建议在 Kubernetes 环境中搭建独立的 etcd 服务供 JuiceFS 使用，而不是使用集群中默认的 etcd 服务，避免当文件系统访问压力高时影响 Kubernetes 集群的稳定性。\n" }, { "path": "docs/zh_cn/administration/metadata/fdb_best_practices.md", "content": "---\nsidebar_label: FoundationDB\nsidebar_position: 6\nslug: /fdb_best_practices\n---\n\n# FoundationDB 最佳实践\n\nfdb 支持横向扩容，一旦数据存储达到集群的最高负载，只需要在集群中添加新的机器即可。配置集群的详细教程可见官网 [https://apple.github.io/foundationdb/configuration.html](https://apple.github.io/foundationdb/configuration.html) ，对于不同场景不同机器数量的性能测试可见 [https://apple.github.io/foundationdb/benchmarking.html](https://apple.github.io/foundationdb/benchmarking.html)。\n\n## 系统要求\n\n- 以下 64 位操作系统之一\n - 受支持的 Linux 发行版\n - RHEL/CentOS 6.x and 7.x\n - Ubuntu 12.04 或更高版本\n - 未受支持的 Linux 发行版\n - 内核版本介于 2.6.33 和 3.0.x（含）或 3.7 或更高版本之间\n - 最好是.deb 或者.rpm\n - macOS 10.7 或更高版本\n- 每个 fdbserver 需要至少 4GB 内存\n- 存储\n - 存储数据小于内存时使用内存存储引擎\n - 存储数据大于内存时使用 SSD 存储引擎\n\n## 如何配置 FoundationDB\n\n### 在单机上配置 FoundationDB\n\n**[Ubuntu](https://apple.github.io/foundationdb/getting-started-linux.html)**\n\n```\n//下载server和client deb包\nwget https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb\nwget https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-server_6.3.23-1_amd64.deb\n//安装\nsudo dpkg -i foundationdb-clients_6.3.23-1_amd64.deb \\\nfoundationdb-server_6.3.23-1_amd64.deb\n```\n\n**[RHEL/CentOS6/CentOS7](https://apple.github.io/foundationdb/getting-started-linux.html)**\n\n```\n//下载server和client rpm包\nwget https://github.com/apple/foundationdb/releases/download/6.3.12/foundationdb-clients-6.3.23-1.el7.x86_64.rpm\nwget https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-server-6.3.23-1.el7.x86_64.rpm\n//安装\nsudo rpm -Uvh foundationdb-clients-6.3.23-1.el7.x86_64.rpm \\\nfoundationdb-server-6.3.23-1.el7.x86_64.rpm\n```\n\n**[macOS](https://apple.github.io/foundationdb/getting-started-linux.html)**\n\n详情请移步 FoundationDB 官网\n\n### [在多台机器上配置 FoundationDB 集群](https://apple.github.io/foundationdb/administration.html#adding-machines-to-a-cluster)\n\n> 部署单台机器的步骤与上述一致。\n\n- 首先在每台机器上部署好单个 FoundationDB\n- 选择一个节点将其 fdb.cluster 文件修改（路径默认`/etc/foundationdb/fdb.cluster`），此文件由一行字符串组成，格式为 description:ID@IP:PORT,IP:PORT,...，仅添加其他机器的 IP:PORT 即可。\n- 将此修改完的 fdb.cluster 拷贝到其他节点\n- 将机器重启（`sudo service foundationdb restart`）\n\n## 冗余模式\n\nFoundationDB 支持多种冗余模式。这些模式定义了存储要求、所需的集群大小和故障恢复能力，用户可根据不同的机器配置选择相对应的冗余模式。要更改冗余模式，请使用 的 configure 命令 fdbcli。示例如下：\n\n```\nuser@host$ fdbcli\nUsing cluster file `/etc/foundationdb/fdb.cluster'.\n\nThe database is available.\n\nWelcome to the fdbcli. For help, type `help'.\nfdb> configure double\nConfiguration changed.\n```\n\n### `single` mode（1-2 台机器）\n\nFoundationDB 不复制数据，只需要一台物理机器就可以进行处理。由于数据没有被复制，数据库没有容错能力。\n\n建议在单个开发机器上进行测试时使用此模式。(单模式将用于由两台或两台以上计算机组成的集群，并将数据进行分区以提高性能，但集群不会容忍任何机器的丢失)\n\n### `double` mode（3-4 台机器）\n\nFoundationDB 将数据复制到两台机器上，因此需要两台或两台以上的机器进行处理。一台机器的丢失可以在不丢失数据的情况下存活，但如果最初只有两台机器，则数据库将不可用，直到恢复第二台机器、添加另一台机器或更改复制模式。\n\n### `triple` mode（5+ 台机器）\n\nFoundationDB 将数据复制到三台机器上，并且至少需要三台可用的机器才能进行处理。对于一个数据中心中有五台或更多机器的集群，推荐使用这种模式。\n\n## 存储引擎\n\nfdb 提供`ssd`和`memory`两种存储引擎，根据数据量大小来选择不同的存储引擎。我们在实际测试中发现两种存储引擎的性能相差不大，而`ssd`存储引擎支持较大的数据量，故推荐使用`ssd`存储引擎。\n\n```\nuser@host$ fdbcli\nUsing cluster file `/etc/foundationdb/fdb.cluster'.\n\nThe database is available.\n\nWelcome to the fdbcli. For help, type `help'.\nfdb> configure ssd\nConfiguration changed.\n```\n\n### `ssd` 存储引擎（推荐）\n\n数据以 B 树的格式存储在磁盘中，一般使用固态硬盘而非机械硬盘。当有合适的磁盘硬件时，这个引擎更加健壮，因为它可以存储大量数据。\n\n关于性能，固态硬盘提供了很不错的随机读写性能，再加上热点数据的缓存，基本上于`memory`存储引擎相差无几，对于`JUICEFS`的元数据存储也是极力推荐使用`ssd`存储引擎。\n\n需要注意的是，固态硬盘在损坏之后数据有可能不可恢复，所以需要注意硬盘的磨损程度以更换新的硬盘。\n\n由于该存储引擎是针对于 SSD（固态硬盘），因此如果使用的机械硬盘，性能会受到很大影响。\n\n### `memory` 存储引擎\n\n数据存储在内存中，其通过顺序写日志的方式对数据进行持久化，数据库重启时通过回放日志的方式来进行数据恢复，此过程一般需要一些时间（几秒钟到几分钟）。\n\n默认情况下，每个使用内存存储引擎的进程只能存储 1GB 的数据 (包括开销)。这个限制可以通过在`foundationdb.conf`中记录的`storage_memory`参数来更改。\n" }, { "path": "docs/zh_cn/administration/metadata/mysql_best_practices.md", "content": "---\nsidebar_label: MySQL\nsidebar_position: 2\nslug: /mysql_best_practices\n---\n# MySQL 最佳实践\n\n对于数据与元数据分离存储的分布式文件系统，元数据的读写性能直接影响整个系统的工作效率，元数据的安全也直接关系着整个系统的数据安全。\n\n在生产环境中，建议您优先选择云计算平台提供的托管型云数据库，并搭配恰当的高可用性架构。\n\n不论自行搭建，还是采用云数据库，使用 JuiceFS 应该始终关注元数据的完整和安全。\n\n## 通过环境变量传递数据库信息\n\n虽然直接在元数据 URL 中设置数据库密码简单方便，但日志或程序输出中可能会泄漏密码，为了保证数据安全，应该始终通过环境变量传递数据库密码。\n\n环境变量名称可以自由定义，例如：\n\n```shell\nexport $MYSQL_PASSWORD=mypassword\n```\n\n在元数据 URL 中通过环境变量传递数据库密码：\n\n```shell\njuicefs mount -d \"mysql://user:$MYSQL_PASSWORD@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\n或者使用指定的环境变量 (META_PASSWORD) ，例如：\n\n```shell\nexport $META_PASSWORD=mypassword\n```\n\n在元数据 URL 中通过直接省略密码：\n\n```shell\njuicefs mount -d \"mysql://user:@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\n## 连接数控制\n\nMySQL 后端采用多线程模式，每一个连接对应后端一个线程，控制数据库的连接总数和减少数据库连接的动态创建都是非常必要的。JuiceFS 提供 4 个数据库连接相关的控制选项：\n\n- max_open_conns：控制当前挂载点到数据库的最大连接数，默认值为 0，表示没有限制。如果设置了一个固定值，并且所有连接都被使用了，新的请求就需要等待其他请求释放数据库连接，过小的值可能会影响性能，请根据实际业务压力情况动态调整。\n- max_idle_conns：控制当前挂载点到数据库的最大空闲连接数，默认值为 CPU 的逻辑核心数的两倍。如果设置的值过大，这些连接一直空闲着，可能会消耗或浪费后端的资源，引起后端连接数过高，导致其他挂载点需要新建连接时无法连接成功。\n- max_idle_time：一个连接的最长空闲时间，默认值为 300 秒。如果一个连接一直未被使用，和后端数据库无任何交互，超过指定时间后，会自动断开连接，以节约后端资源。设置过小的值可能会引起频繁地创建数据据连接，影响性能。\n- max_life_time：一个连接的最大生命周期，默认为 0，表示无限制。一个数据库连接会被各种请求循环复用，在服务请求的过程中会申请一些临时资源，比如内存等，可能存在清理不干净或资源碎片的情况，可以考虑设置一个合理的生命周期，达到周期并且服务完当前请求后会自动断开来优化资源使用。\n\n可在元数据 URL 中直接传递上述控制选项：\n\n```shell\njuicefs mount -d \"mysql://user:@(192.168.1.6:3306)/juicefs?max_open_conns=30&max_life_time=3600\" /mnt/jfs\n```\n\n请参考 Go 模块文档 [Database/SQL](https://pkg.go.dev/database/sql#SetConnMaxIdleTime) 了解更多信息。\n\n## 定期备份\n\n请参考官方手册 [Chapter 9. Backup and Recovery](https://dev.mysql.com/doc/refman/8.0/en/backup-and-recovery.html) 了解如何备份和恢复数据库。\n\n建议制定数据库备份计划，并遵照计划定期备份 MySQL 数据库，与此同时，还应该在实验环境中尝试恢复数据，确认备份是有效的。\n\n## 高可用\n\nMySQL 官方文档 [Chapter 19. Replication](https://dev.mysql.com/doc/refman/8.0/en/replication.html) 和 [Chapter 20. Group Replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication.html) 是常用的数据库高可用方案，请根据实际业务需要选择恰当的高可用方案。\n\n:::note 注意\nJuiceFS 需要使用[事务功能]来保证元数据操作的原子性，因此需要使用支持事务的存储引擎，例如 [InnoDB](https://dev.mysql.com/doc/refman/8.0/en/innodb-storage-engine.html) 。一些基于 MySQL 的 Shared Nothing 分布式架构可能会存在事务的兼容性问题，目前未对分布式架构做 JuiceFS 元数据做兼容性研发和测试。\n:::\n" }, { "path": "docs/zh_cn/administration/metadata/postgresql_best_practices.md", "content": "---\nsidebar_label: PostgreSQL\nsidebar_position: 3\nslug: /postgresql_best_practices\n---\n# PostgreSQL 最佳实践\n\n对于数据与元数据分离存储的分布式文件系统，元数据的读写性能直接影响整个系统的工作效率，元数据的安全也直接关系着整个系统的数据安全。\n\n在生产环境中，建议您优先选择云计算平台提供的托管型云数据库，并搭配恰当的高可用性架构。\n\n不论自行搭建，还是采用云数据库，使用 JuiceFS 应该始终关注元数据的完整和安全。\n\n## 通信安全\n\n默认情况下，JuiceFS 客户端会采用 SSL 加密协议连接 PostgreSQL，如果数据库未启用 SSL 加密，则需要在元数据 URL 中需要附加 `sslmode=disable` 参数。\n\n建议配置并始终开启数据库服务端 SSL 加密。\n\n## 通过环境变量传递数据库信息\n\n虽然直接在元数据 URL 中设置数据库密码简单方便，但日志或程序输出中可能会泄漏密码，为了保证数据安全，应该始终通过环境变量传递数据库密码。\n\n环境变量名称可以自由定义，例如：\n\n```shell\nexport $PG_PASSWD=mypassword\n```\n\n在元数据 URL 中通过环境变量传递数据库密码：\n\n```shell\njuicefs mount -d \"postgres://user:$PG_PASSWD@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\n## 连接数控制\n\nPostgreSQL 后端采用多进程模式，每一个连接对应后端一个进程，控制数据库的连接总数和减少数据库连接的动态创建都是非常必要的。JuiceFS 提供 4 个数据库连接相关的控制选项：\n\n- max_open_conns：控制当前挂载点到数据库的最大连接数，默认值为 0，表示没有限制。如果设置了一个固定值，并且所有连接都被使用了，新的请求就需要等待其他请求释放数据库连接，过小的值可能会影响性能，请根据实际业务压力情况动态调整。\n- max_idle_conns：控制当前挂载点到数据库的最大空闲连接数，默认值为 CPU 的逻辑核心数的两倍。如果设置的值过大，这些连接一直空闲着，可能会消耗或浪费后端的资源，引起后端连接数过高，导致其他挂载点需要新建连接时无法连接成功。\n- max_idle_time：一个连接的最长空闲时间，默认值为 300 秒。如果一个连接一直未被使用，和后端数据库无任何交互，超过指定时间后，会自动断开连接，以节约后端资源。设置过小的值可能会引起频繁地创建数据据连接，影响性能。\n- max_life_time：一个连接的最大生命周期，默认为 0，表示无限制。一个数据库连接会被各种请求循环复用，在服务请求的过程中会申请一些临时资源，比如内存等，可能存在清理不干净或资源碎片的情况，可以考虑设置一个合理的生命周期，达到周期并且服务完当前请求后会自动断开来优化资源使用。\n\n可在元数据 URL 中直接传递上述控制选项：\n\n```shell\njuicefs mount -d \"postgres://user:$PG_PASSWD@192.168.1.6:5432/juicefs?max_open_conns=30&max_life_time=3600\" /mnt/jfs\n```\n\n请参考 Go 模块文档 [Database/SQL](https://pkg.go.dev/database/sql#SetConnMaxIdleTime) 了解更多信息。\n\n## 定期备份\n\n请参考官方手册 [Chapter 26. Backup and Restore](https://www.postgresql.org/docs/current/backup.html) 了解如何备份和恢复数据库。\n\n建议制定数据库备份计划，并遵照计划定期备份 PostgreSQL 数据库，与此同时，还应该在实验环境中尝试恢复数据，确认备份是有效的。\n\n## 使用连接池\n\n连接池是客户端与数据库之间的中间层，由它作为中介提升连接效率，降低短连接的损耗。常用的连接池有 [PgBouncer](https://www.pgbouncer.org) 和 [Pgpool-II](https://www.pgpool.net) 。\n\n## 高可用\n\nPostgreSQL 官方文档 [High Availability, Load Balancing, and Replication](https://www.postgresql.org/docs/current/different-replication-solutions.html) 对比了几种常用的数据库高可用方案，请根据实际业务需要选择恰当的高可用方案。\n\n:::note 注意\nJuiceFS 使用[事务](https://www.postgresql.org/docs/current/tutorial-transactions.html)保证元数据操作的原子性。由于 PostgreSQL 尚不支持 Multi-Shard (Distributed) 分布式事务，因此请勿将多服务器分布式架构用于 JuiceFS 元数据存储。\n:::\n" }, { "path": "docs/zh_cn/administration/metadata/redis_best_practices.md", "content": "---\nsidebar_label: Redis\nsidebar_position: 1\nslug: /redis_best_practices\n---\n\n# Redis 最佳实践\n\n为保证元数据服务稳定，我们建议使用云平台提供的 Redis 托管服务，详情查看[「推荐的 Redis 托管服务」](#推荐的-redis-托管服务)。\n\n## 内存使用量\n\nJuiceFS 元数据引擎的使用空间主要与文件系统中的文件数量有关，根据我们的经验，每一个文件的元数据会大约占用 300 字节内存。因此，如果要存储 1 亿个文件，大约需要 30GiB 内存。\n\n你可以通过 Redis 的 [`INFO memory`](https://redis.io/commands/info) 命令查看具体的内存使用量，例如：\n\n```\n> INFO memory\nused_memory: 19167628056\nused_memory_human: 17.85G\nused_memory_rss: 20684886016\nused_memory_rss_human: 19.26G\n...\nused_memory_overhead: 5727954464\n...\nused_memory_dataset: 13439673592\nused_memory_dataset_perc: 70.12%\n```\n\n其中 `used_memory_rss` 是 Redis 实际使用的总内存大小，这里既包含了存储在 Redis 中的数据大小（也就是上面的 `used_memory_dataset`），也包含了一些 Redis 的[系统开销](https://redis.io/commands/memory-stats)（也就是上面的 `used_memory_overhead`）。前面提到每个文件的元数据大约占用 300 字节是通过 `used_memory_dataset` 来计算的，如果你发现你的 JuiceFS 文件系统中单个文件元数据占用空间远大于 300 字节，可以尝试运行 [`juicefs gc`](../../reference/command_reference.mdx#gc) 命令来清理可能存在的冗余数据。\n\n## 数据可用性\n\n### 哨兵模式 {#sentinel-mode}\n\n[Redis 哨兵](https://redis.io/docs/manual/sentinel) 是 Redis 官方的高可用解决方案，它提供以下功能：\n\n- **监控**，哨兵会不断检查您的 master 实例和 replica 实例是否按预期工作。\n- **通知**，当受监控的 Redis 实例出现问题时，哨兵可以通过 API 通知系统管理员或其他计算机程序。\n- **自动故障转移**，如果 master 没有按预期工作，哨兵可以启动一个故障转移过程，其中一个 replica 被提升为 master，其他的副本被重新配置为使用新的 master，应用程序在连接 Redis 服务器时会被告知新的地址。\n- **配置提供程序**，哨兵会充当客户端服务发现的权威来源：客户端连接到哨兵以获取当前 Redis 主节点的地址。如果发生故障转移，哨兵会报告新地址。\n\n**Redis 2.8 开始提供稳定版本的 Redis 哨兵**。Redis 2.6 提供的第一版 Redis 哨兵已被弃用，不建议使用。\n\n在使用 Redis 哨兵之前，先了解一些[基础知识](https://redis.io/docs/manual/sentinel#fundamental-things-to-know-about-sentinel-before-deploying)：\n\n1. 您至少需要三个哨兵实例才能进行稳健的部署。\n2. 这三个哨兵实例应放置在彼此独立的计算机或虚拟机中。例如，分别位于不同的可用区域上的不同物理服务器或虚拟机上。\n3. **由于 Redis 使用异步复制，无法保证在发生故障时能够保留已确认的写入。** 然而，有一些部署 哨兵的方法，可以使丢失写入的窗口限于某些时刻，当然还有其他不太安全的部署方法。\n4. 如果您不在开发环境中经常进行测试，就无法确保 HA 的设置是安全的。在条件允许的情况，如果能够在生产环境中进行验证则更好。错误的配置往往都是在你难以预期和响应的时间出现（比如，凌晨 3 点你的 master 节点悄然罢工）。\n5. **哨兵、Docker 或其他形式的网络地址转换或端口映射应谨慎混用**：Docker 执行端口重映射，会破坏其他哨兵进程的哨兵自动发现以及 master 的 replicas 列表。\n\n更多信息请阅读[官方文档](https://redis.io/docs/manual/sentinel)。\n\n部署了 Redis 服务器和哨兵以后，`META-URL` 可以指定为 `redis[s]://[[USER]:PASSWORD@]MASTER_NAME,SENTINEL_ADDR[,SENTINEL_ADDR]:SENTINEL_PORT[/DB]`，例如：\n\n```shell\njuicefs mount redis://:password@masterName,1.2.3.4,1.2.5.6:26379/2 ~/jfs\n```\n\n:::tip 提示\n对于 JuiceFS v0.16 及以上版本，URL 中提供的密码会用于连接 Redis 服务器，哨兵的密码需要用环境变量 `SENTINEL_PASSWORD` 指定。对于更早的版本，URL 中的密码会同时用于连接 Redis 服务器和哨兵，也可以通过环境变量 `SENTINEL_PASSWORD` 和 `REDIS_PASSWORD` 来覆盖。\n:::\n\n自 JuiceFS v1.0.0 版本开始，支持在挂载文件系统时仅连接 Redis 的副本节点，以降低 Redis 主节点的负载。为了开启这个特性，必须以只读模式挂载 JuiceFS 文件系统（即设置 `--read-only` 挂载选项），并通过 Redis 哨兵连接元数据引擎，最后需要在元数据 URL 末尾加上 `?route-read=replica`，例如：`redis://:password@masterName,1.2.3.4,1.2.5.6:26379/2?route-read=replica`。\n\n需要注意由于 Redis 主节点的数据是异步复制到副本节点，因此有可能读到的元数据不是最新的。\n\n### 集群模式 {#cluster-mode}\n\n:::note 注意\n此特性需要使用 1.0.0 及以上版本的 JuiceFS\n:::\n\nJuiceFS 同样支持集群模式的 Redis 作为元数据引擎，Redis 集群模式的 `META-URL` 为 `redis[s]://[[USER]:PASSWORD@]ADDR:PORT,[ADDR:PORT],[ADDR:PORT][/DB]`，例如：\n\n```shell\njuicefs format redis://127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002/1 myjfs\n```\n\n:::tip 提示\nRedis 集群不再支持多数据库，而是将所有 keys 分散到 16384 个 hash slots 中，再将这些 hash slots 打散到多个 Redis master 节点来存储。JuiceFS 利用了 Redis 集群的 [Hash Tag](https://redis.io/docs/reference/cluster-spec/#hash-tags) 特性，通过将 `{DB}` 作为 key 的前缀来将一个文件系统中的所有 keys 都存放在同一个 hash slot，以保证集群模式下操作的事务性。另外，通过设置不同的 `DB` 可以让一个 Redis 集群同时作为多个 JuiceFS 的元数据库。\n:::\n\n## 数据持久性\n\nRedis 提供了不同范围的[持久性](https://redis.io/docs/manual/persistence)选项：\n\n- **RDB**：以指定的时间间隔生成当前数据集的快照。\n- **AOF**：记录服务器收到的每一个写操作，在服务器启动时重建原始数据集。命令使用与 Redis 协议本身相同的格式以追加写（append-only）的方式记录。当日志变得太大时，Redis 能够在后台重写日志。\n- **RDB+AOF** 建议：组合使用 AOF 和 RDB。在这种情况下，当 Redis 重新启动时，AOF 文件将用于重建原始数据集，因为它保证是最完整的。\n\n当使用 AOF 时，您可以有不同的 fsync 策略：\n\n1. 没有 fsync；\n2. 每秒 fsync 默认；\n3. 每次查询 fsync。\n\n默认策略「每秒 fsync」是不错的选择（fsync 是使用后台线程执行的，当没有 fsync 正在进行时，主线程会努力执行写入），**但你可能丢失最近一秒钟的写入**。\n\nRedis 对数据备份非常友好，因为您可以在数据库运行时复制 RDB 文件，RDB 一旦生成就永远不会被修改，当它被生成时，它使用一个临时名称，并且只有在新快照完成时才使用 `rename` 原子地重命名到其最终目的地。您还可以复制 AOF 文件以创建备份。\n\n更多信息请阅读[官方文档](https://redis.io/docs/manual/persistence)。\n\n## 备份 Redis 数据\n\n磁盘可能会损坏，虚拟机可能出意外，即使采用 RBD+AOF 模式，**依然需要定期备份 Redis 数据**。\n\n默认情况下，Redis 将数据集的快照保存在磁盘上，名为 `dump.rdb` 的二进制文件中。你可以根据需要，将 Redis 配置为当数据集至少发生 M 次变化时，每 N 秒保存一次，也可以手动调用 [`SAVE`](https://redis.io/commands/save) 或 [`BGSAVE`](https://redis.io/commands/bgsave) 命令。\n\nRedis 对数据备份非常友好，因为您可以在数据库运行时复制 RDB 文件：RDB 一旦生成就永远不会被修改，当它被生成时，它使用一个临时名称，并且只有在新快照完成时才使用 `rename(2)` 原子地重命名到其最终目的地。\n\n这意味着在服务器运行时复制 RDB 文件是完全安全的。以下是我们的建议：\n\n- 在您的服务器中创建一个 cron 任务，在一个目录中创建 RDB 文件的每小时快照，并在另一个目录中创建每日快照。\n- 每次 cron 脚本运行时，请务必调用 `find` 命令以确保删除太旧的快照：例如，您可以保留最近 48 小时的每小时快照，以及一至两个月的每日快照。要确保使用数据和时间信息来命名快照。\n- 确保每天至少一次将 RDB 快照从运行 Redis 的实例传输至 _数据中心以外_ 或至少传输至 _物理机以外_。\n\n更多信息请阅读[官方文档](https://redis.io/docs/manual/persistence)。\n\n## 恢复 Redis 数据\n\n当生成 AOF 或者 RDB 备份文件以后，可以将备份文件拷贝到新 Redis 实例的 `dir` 配置对应的路径中来恢复数据，你可以通过 [`CONFIG GET dir`](https://redis.io/commands/config-get) 命令获取当前 Redis 实例的配置信息。\n\n如果 AOF 和 RDB 同时开启，Redis 启动时会优先使用 AOF 文件来恢复数据，因为 AOF 保证是最完整的数据。\n\n在恢复完 Redis 数据以后，可以继续通过新的 Redis 地址使用 JuiceFS 文件系统。建议运行 [`juicefs fsck`](../../reference/command_reference.mdx#fsck) 命令检查文件系统数据的完整性。\n\n## 推荐的 Redis 托管服务\n\n### Amazon MemoryDB for Redis\n\n[Amazon MemoryDB for Redis](https://aws.amazon.com/memorydb) 是一种持久的内存数据库服务，可提供超快的性能。MemoryDB 与 Redis 兼容，使用 MemoryDB，你的所有数据都存储在内存中，这使你能够实现微秒级读取和数毫秒的写入延迟和高吞吐。MemoryDB 还使用多可用区事务日志跨多个可用区持久存储数据，以实现快速故障切换、数据库恢复和节点重启。\n\n### Google Cloud Memorystore for Redis\n\n[Google Cloud Memorystore for Redis](https://cloud.google.com/memorystore/docs/redis) 是针对 Google Cloud 的完全托管的 Redis 服务。通过利用高度可扩展、可用且安全的 Redis 服务，在 Google Cloud 上运行的应用程序可以实现卓越的性能，而无需管理复杂的 Redis 部署。\n\n### Azure Cache for Redis\n\n[Azure Cache for Redis](https://azure.microsoft.com/en-us/services/cache) 是一个完全托管的内存缓存，支持高性能和可扩展的架构。使用它来创建云或混合部署，以亚毫秒级延迟处理每秒数百万个请求——所有这些都具有托管服务的配置、安全性和可用性优势。\n\n### 阿里云云数据库 Redis 版\n\n[阿里云云数据库 Redis 版](https://www.aliyun.com/product/kvstore)是一种兼容原生 Redis 协议的数据库服务。它支持混合内存和硬盘以实现数据持久性。云数据库 Redis 版提供高可用的热备架构，可扩展以满足高性能、低延迟的读写操作需求。\n\n### 腾讯云云数据库 Redis\n\n[腾讯云云数据库 Redis](https://cloud.tencent.com/product/crs) 是一种兼容 Redis 协议的缓存和存储服务。丰富多样的数据结构选项，帮助您开发不同类型的业务场景，提供主从热备份、容灾自动切换、数据备份、故障转移、实例监控、在线等一整套数据库服务缩放和数据回滚。\n\n## 使用 Redis 兼容的产品\n\n如果想要使用 Redis 兼容产品作为元数据引擎，需要确认是否完整支持 JuiceFS 所需的以下 Redis 数据类型和命令。\n\n### JuiceFS 使用到的 Redis 数据类型\n\n+ [String](https://redis.io/docs/data-types/strings)\n+ [Set](https://redis.io/docs/data-types/sets)\n+ [Sorted Set](https://redis.io/docs/data-types/sorted-sets)\n+ [Hash](https://redis.io/docs/data-types/hashes)\n+ [List](https://redis.io/docs/data-types/lists)\n\n### JuiceFS 使用到的 Redis 特性\n\n+ [管道](https://redis.io/docs/manual/pipelining)\n\n### JuiceFS 使用到的 Redis 命令\n\n#### String\n\n+ [DECRBY](https://redis.io/commands/decrby)\n+ [DEL](https://redis.io/commands/del)\n+ [GET](https://redis.io/commands/get)\n+ [INCR](https://redis.io/commands/incr)\n+ [INCRBY](https://redis.io/commands/incrby)\n+ [DECR](https://redis.io/commands/decr)\n+ [MGET](https://redis.io/commands/mget)\n+ [MSET](https://redis.io/commands/mset)\n+ [SETNX](https://redis.io/commands/setnx)\n+ [SET](https://redis.io/commands/set)\n\n#### Set\n\n+ [SADD](https://redis.io/commands/sadd)\n+ [SMEMBERS](https://redis.io/commands/smembers)\n+ [SREM](https://redis.io/commands/srem)\n\n#### Sorted Set\n\n+ [ZADD](https://redis.io/commands/zadd)\n+ [ZRANGEBYSCORE](https://redis.io/commands/zrangebyscore)\n+ [ZRANGE](https://redis.io/commands/zrange)\n+ [ZREM](https://redis.io/commands/zrem)\n+ [ZSCORE](https://redis.io/commands/zscore)\n\n#### Hash\n\n+ [HDEL](https://redis.io/commands/hdel)\n+ [HEXISTS](https://redis.io/commands/hexists)\n+ [HGETALL](https://redis.io/commands/hgetall)\n+ [HGET](https://redis.io/commands/hget)\n+ [HINCRBY](https://redis.io/commands/hincrby)\n+ [HKEYS](https://redis.io/commands/hkeys)\n+ [HSCAN](https://redis.io/commands/hscan)\n+ [HSETNX](https://redis.io/commands/hsetnx)\n+ [HSET](https://redis.io/commands/hset)（需要支持设置多个 field 和 value）\n\n#### List\n\n+ [LLEN](https://redis.io/commands/llen)\n+ [LPUSH](https://redis.io/commands/lpush)\n+ [LRANGE](https://redis.io/commands/lrange)\n+ [LTRIM](https://redis.io/commands/ltrim)\n+ [RPUSHX](https://redis.io/commands/rpushx)\n+ [RPUSH](https://redis.io/commands/rpush)\n+ [SCAN](https://redis.io/commands/scan)\n\n#### 事务\n\n+ [EXEC](https://redis.io/commands/exec)\n+ [MULTI](https://redis.io/commands/multi)\n+ [WATCH](https://redis.io/commands/watch)\n+ [UNWATCH](https://redis.io/commands/unwatch)\n\n#### 连接管理\n\n+ [PING](https://redis.io/commands/ping)\n\n#### 服务管理\n\n+ [CONFIG GET](https://redis.io/commands/config-get)\n+ [CONFIG SET](https://redis.io/commands/config-set)\n+ [DBSIZE](https://redis.io/commands/dbsize)\n+ [FLUSHDB](https://redis.io/commands/flushdb)（可选）\n+ [INFO](https://redis.io/commands/info)\n\n#### 集群管理\n\n+ [CLUSTER INFO](https://redis.io/commands/cluster-info)\n\n#### 脚本（可选）\n\n+ [EVALSHA](https://redis.io/commands/evalsha)\n+ [SCRIPT LOAD](https://redis.io/commands/script-load)\n" }, { "path": "docs/zh_cn/administration/metadata/tikv_best_practices.md", "content": "---\nsidebar_label: TiKV\nsidebar_position: 5\nslug: /tikv_best_practices\n---\n# TiKV 最佳实践\n\nTiKV 通过 Raft 协议保证多副本数据一致性以及高可用，所以建议生产环境中至少部署三个以上副本以保证数据安全和服务稳定。\nTiKV 有很好的横向扩容能力，适用于大规模且对性能有一定要求的文件系统场景。\n\n## 垃圾回收\n\nTiKV 原生支持了 MVCC（多版本并发控制）机制，当新写入的数据覆盖旧的数据时，旧的数据不会被替换掉，而是与新写入的数据同时保留，并以时间戳来区分版本。垃圾回收 (GC) 的任务便是清理不再需要的旧数据。\n\n### JuiceFS 的配置\n\nTiKV 根据一个集群变量 `safe-point`（时间戳）来决定是否要清理某个时间之前的旧版本数据。JuiceFS 在 v1.0.4 之前不会设置`safe-point`，TiKV 元数据引擎需要依赖 TiDB 才能正常进行垃圾回收。而在 v1.0.4 之后，JuiceFS 客户端会周期性地设置 `safe-point`，默认会清除三小时之前的旧版本数据，这个时间可在挂载时通过 meta url 的 `gc-interval` 设置。\n\n- 默认 `gc-interval` 的挂载 log\n\n```bash\n> sudo ./juicefs mount tikv://localhost:2379 ~/mnt/jfs\n2023/04/06 20:23:34.741432 juicefs[17286] : Meta address: tikv://localhost:2379 [interface.go:491]\n2023/04/06 20:23:34.741561 juicefs[17286] : TiKV gc interval is set to 3h0m0s [tkv_tikv.go:84]\n...\n```\n\n- 设置 `gc-interval` 后的挂载 log\n\n```bash\n> sudo ./juicefs mount tikv://localhost:2379\\?gc-interval=1h ~/mnt/jfs\n2023/04/06 20:25:58.134999 juicefs[17395] : Meta address: tikv://localhost:2379?gc-interval=1h [interface.go:491]\n2023/04/06 20:25:58.135113 juicefs[17395] : TiKV gc interval is set to 1h0m0s [tkv_tikv.go:84]\n...\n```\n\n#### 主动设置 `safe-point`\n\nJuiceFS 客户端会周期性设置 `safe-point`，除此之外我们也可以通过 gc 子命令来主动设置。\n\n```bash\n> ./juicefs gc -v tikv://localhost:2379\\?gc-interval=1h --delete\n...\n2023/04/06 20:41:57.145692 juicefs[18531] : TiKV GC returns new safe point: 440606737600086016 (2023-04-06 19:41:57.139 +0800 CST) [tkv_tikv.go:248]\n...\n```\n\n:::tip 提示\n此命令同时会清理 JuiceFS 产生的「泄漏对象」和「待清理对象」，请参考[状态检查 & 维护](../status_check_and_maintenance.md#gc)以确认您是否应该使用。\n:::\n\n### TiKV 的垃圾回收模式\n\n- gc-worker\n\n可以在通过 TiKV 配置来启用 gc-worker。gc-worker 模式下垃圾会被及时回收，但大量额外的磁盘读写可能会影响元数据引擎性能。\n\n```toml\n[gc]\nenable-compaction-filter = false\n```\n\n- compaction-filter\n\nTiKV 默认通过 [compaction-filter](https://docs.pingcap.com/zh/tidb/dev/garbage-collection-configuration#gc-in-compaction-filter-%E6%9C%BA%E5%88%B6) 进行垃圾回收，由 RocksDB 的 Compaction 过程来进行 GC，而不再使用一个单独的 GC worker 线程。这样做的好处是避免了 GC 引起的额外磁盘读取，以及避免清理掉的旧版本残留大量删除标记影响顺序扫描性能。\n\n由于此回收模式依赖 RocksDB compaction，所以设置`safe-point`之后垃圾并不会被及时回收，需要后续持续写入触发 compaction 才能进行 GC。如果您需要主动触发 GC，可以通过 [`tikv-ctl`](https://docs.pingcap.com/zh/tidb/dev/tikv-control) 工具主动进行集群 compaction，从而触发全局 GC。\n\n```bash\n> tikv-ctl --pd 127.0.0.1:2379 compact-cluster -b -c default,lock,write\n```\n\n## 元数据备份\n\n对于大规模文件系统，需要调高 [tikv_gc_life_time](https://docs.pingcap.com/zh/tidb/stable/dev-guide-timeouts-in-tidb#gc-%E8%B6%85%E6%97%B6) 参数，否则可能会因为 `GC life time is shorter than transaction duration` 导致备份失败。\n\n## 运行环境与调优\n\n### 硬件选型\n\n根据[TiDB 软件和硬件环境建议配置](https://docs.pingcap.com/zh/tidb/stable/hardware-and-software-requirements)，TiKV 支持部署和运行在 Intel x86-64 架构的 64 位通用硬件服务器平台或者 ARM 架构的硬件服务器平台。对于开发、测试及生产环境的服务器硬件配置（不包含操作系统 OS 本身的占用）有以下要求和建议：\n\n+ **开发与测试环境**\n\n| 组件 |CPU| 内存 | 本地存储 | 网络 | 实例数量 (最低要求)|\n|-|-|-|-|-|-|\n|PD|4 核 +|8 GB+|SAS, 200 GB+| 千兆网卡 |1|\n|TiKV|8 核 +|32 GB+|SSD, 200 GB+| 千兆网卡 |3|\n\n:::note 说明\n\n+ 如进行性能相关的测试，避免采用低性能存储和网络硬件配置，防止对测试结果的正确性产生干扰。\n+ TiKV 的 SSD 盘推荐使用 NVME 接口以保证读写更快。\n\n:::\n\n+ **生产环境**\n\n| 组件 |CPU| 内存 | 本地存储 | 网络 | 实例数量 (最低要求)|\n|-|-|-|-|-|-|\n|PD|8 核 +|16 GB+|SSD| 万兆网卡（2 块最佳）|3|\n|TiKV|16 核 +|64 GB+|SSD| 万兆网卡（2 块最佳）|3|\n\n:::note 说明\nTiKV 硬盘大小配置建议 PCI-E SSD 不超过 2 TB，普通 SSD 不超过 1.5 TB。\n:::\n\n### 网络要求\n\nTiKV 正常运行需要网络环境提供如下的网络端口配置要求，管理员可根据实际环境中组件部署的方案，在网络侧和主机侧开放相关端口：\n\n| 组件 | 默认端口 | 说明 |\n|-|-|-|\n|TiKV|20160|TiKV 通信端口 |\n|TiKV|20180|TiKV 状态信息上报通信端口 |\n|PD|2379| 提供 TiDB 和 PD 通信端口 |\n|PD|2380|PD 集群节点间通信端口 |\n\n### 磁盘空间要求\n\n| 组件 | 磁盘空间要求 | 健康水位使用率 |\n|-|-|-|\n|PD| 数据盘和日志盘建议最少各预留 20 GB| 低于 90%|\n|TiKV| 数据盘和日志盘建议最少各预留 100 GB| 低于 80%|\n\n## 硬件调优\n\n各种数据库官方都有硬件有一定要求，TiKV 等组件都有最低的 CPU、内存、硬盘、网卡要求。本章节在满足这些需求的基础上，探讨下硬件参数优化，主要参考[数据库硬件调优](https://tidb.net/book/tidb-monthly/2022/2022-03/usercase/tuning-hardware)。\n\n### CPU\n\n+ **CPU 选型**\n\n可以分为计算型和存储型。计算型往往需要更多的 CPU 核心和更高的主频。存储型的 CPU 可能就配置稍微低些。对于计算型和存储型 CPU 选择，拿 JuiceFS 的使用场景来说，PD 和 TiKV 以存储型为主，没有太高的计算负载，可以提前规划使得硬件采购更加合理，节省成本。\n\n+ **CPU 架构：X86/ARM**\n\nX86 架构出现在 intel/AMD 的 CPU 架构中，采用复杂指令集，也是目前最主流服务器的 CPU 架构。ARM 架构 CPU 在手机，mac 笔记本，以及华为等国产服务器厂商中出现。目前各大公司主要采购的是 X86-64 架构的 CPU，也对 ARM 服务器进行了 web 和数据库应用的验证。TiKV 对两种架构均有支持，可根据实际部署情况进行选择。\n\n+ **Numa 绑核**\n\n多核心 CPU 的各核心会被分配到不同的 NUMA node，每个 NUMA node 都有自己专属/本地的主存，访问本地的主存比其跨 NUMA node 访问内存更快，开启 NUMA 会优先就近使用内存。在单机多节点部署时推荐此配置。\n\n+ **CPU-动态节能技术**\n\ncpufreq 是一个动态调整 CPU 频率的模块，可支持五种模式。为保证服务性能应选用 performance 模式，将 CPU 频率固定工作在其支持的最高运行频率上，从而获取最佳的性能，一般都是默认 powersave，可以通过 cpupower frequency-set 修改。\n\n### Memory\n\n+ **关闭 Swap**\n\nswap 用硬盘来承接到达一定阀值的内存访问，由 `vm.swappiness` 参数控制，默认 60，也就是系统内存使用到 40% 时开始使用，TiKV 运行需要有足够的内存。如果内存不足，不建议使用 swap 作为内存不足的缓冲，因为这会降低性能。建议关闭系统 swap。\n\n+ **设置`min_free_kbytes`**\n\n`min_free_kbytes` 内核参数控制了多少内存应该保持空闲而不被文件系统缓存占用。通常情况下，内核会用文件系统缓存占据几乎所有的空闲内存，并根据需要释放内存供进程分配。由于数据库会共享内存中执行大量的分配，默认的内核值可能会导致意外的 OOM（Out-of-Memory kill），在总内存大于 40G 的情况下，建议将该参数配置为至少 1GB，但是不建议超过总内存的 5%，这可以确保 Linux 始终保持足够的内存可用。\n\n+ **关闭透明大页（Transparent Huge Pages，THP）**\n\n数据库的内存访问模式往往是稀疏的而非连续的。当高阶内存碎片化比较严重时，分配 THP 页面会出现较高的延迟，若开启针对 THP 的直接内存规整功能，也会出现系统 CPU 使用率激增的现象，因此建议关闭 THP。\n\n+ **调整虚拟内存 `dirty_ratio`/`dirty_background_ratio` 参数**\n\n`dirty_ratio` 是绝对的脏页百分比值限限制。当脏的 page cache 总量达到系统内存总量的这一百分比后，系统将开始使用 pdflush 操作将脏的 page cache 写入磁盘。默认值为 20％，也就是说如果到达该值时可能会导致应用进程的 IO 等待，通常不需调整。\n\n`dirty_background_ratio` 百分比值。当脏的 page cache 总量达到系统内存总量的这一百分比后，系统开始在后台将脏的 page cache 写入磁盘。默认值为 10％，如果后台刷脏页的慢，而数据写的快就容易触发 dirty_ratio 的限制。通常不需调整。对于高性能 SSD，比如 NVMe 设备来说，设置较低的值有利于提高内存回收时的效率。\n\n### 数据存储\n\n#### 硬盘选型\n\n1. SAS 一般跟 RAID 卡搭配，实现 raid 0/1/10/5 等阵列扩展。\n2. SATA 支持热插拔，接口最高 6G/s。\n3. PCIE 传输速率更高 8G/s，但是支持多通道，可以线性扩展速率。之前网卡/显卡都在用。上面 3 个接口协议不同，AHCI 转为 SAS 和 SATA 设计，NVMe 协议为 PCIE SSD 设计性能更优。一般核心的 + 高 I/O 的数据库都采用该类型 SSD。\n4. 持久内存：傲腾，它提供丰富的底层接口，成本很高，对于需要极致写入性能的，可以考虑。\n\n#### I/O 调度算法\n\n##### noop(no operation)\n\nnoop 调度算法是内核中最简单的 IO 调度算法。noop 调度算法将 IO 请求放入到一个 FIFO 队列中，然后逐个执行这些 IO 请求，当然对于一些在磁盘上连续的 IO 请求，noop 调度会适当做一些合并。这个调度算法特别适合那些不希望调度器重新组织 IO 请求顺序的应用，因为内核的 I/O 调度操作会导致性能损失。NVMe SSD 这种高速 I/O 设备可以直接将请求下发给硬件，从而获取更好的性能。\n\n##### CFQ(Completely Fair Queuing)\n\nCFQ 尝试提供由发起 I/O 进程决定的公平的 I/O 调度，该算法为每一个进程分配一个时间窗口，在该时间窗口内，允许进程发出 IO 请求。通过时间窗口在不同进程间的移动，保证了对于所有进程而言都有公平的发出 IO 请求的机会，假如少数进程存在大量密集的 I/O 请求的情况，会出现明显的 I/O 性能下降。\n\n##### deadline\n\ndeadline 调度算法主要针对 I/O 请求的延时，每个 I/O 请求都被附加一个最后执行期限。读请求和写请求被分成了两个队列，默认优先处理读 IO，除非写快到 deadline 时才调度。当系统中存在的 I/O 请求进程数量比较少时，与 CFQ 算法相比，deadline 算法可以提供较高的 I/O 吞吐率。\n\n## 常见问题\n\n### 多机并发读写同一个目录，如何避免持续的事务重启现象？\n\n当多客户端在同一个目录下频繁创建/删除子目录时，可能会出现持续的事务重启现象。JuiceFS v1.1 版本开始提供 `--skip-dir-nlink value` 挂载选项，用以指定跳过目录的 nlink 检查之前的重试次数，默认为 20 次。可以适当调小该值，或者设置为 0 禁止重试，从而避免持续的事务重启现象，详情参考[元数据相关的挂载选项](https://juicefs.com/docs/zh/community/command_reference#mount-metadata-options)。\n" }, { "path": "docs/zh_cn/administration/metadata_dump_load.md", "content": "---\ntitle: 元数据备份和恢复\nsidebar_position: 2\nslug: /metadata_dump_load\n---\n\n:::tip 提示\n\n- JuiceFS v1.0.0 开始支持元数据自动备份\n- JuiceFS v1.0.4 开始支持通过 `load` 命令恢复加密的元数据备份\n- JuiceFS v1.3.0 开始支持二进制格式的元数据备份和恢复\n\n:::\n\nJuiceFS 支持[多种元数据引擎](../reference/how_to_set_up_metadata_engine.md)，且各引擎内部的数据管理格式各有不同。为了便于管理，JuiceFS 提供了 [`dump`](../reference/command_reference.mdx#dump) 命令允许将所有元数据以统一格式写入到 JSON 或二进制文件进行备份。同时，JuiceFS 也提供了 [`load`](../reference/command_reference.mdx#load) 命令，允许将备份恢复或迁移到任意元数据存储引擎。这个导出导入流程也可以用来将 JuiceFS 社区版文件系统迁移到企业版（参考[企业版文档](https://juicefs.com/docs/zh/cloud/administration/metadata_dump_load)），反之亦然。\n\n## 快速上手视频\n\n
\n \n
\n\n## 元数据备份 {#backup}\n\n:::note 注意\n\n* `juicefs dump` 不提供全局时间点快照的功能，如果在导出过程中业务仍在写入，最终结果会包含不同时间点的信息，对于特定应用（比如数据库），这可能意味着导出文件不可用。如果对一致性有更高要求，可能需要在导出前确保应用停写。\n* 对大规模文件系统，如果直接在线上环境进行导出，可能影响业务稳定性。\n\n:::\n\n## 文件格式\n\nJuiceFS 支持两种格式的元数据备份：JSON 格式和二进制格式。二进制格式在 v1.3.0 版本中引入，主要用于大规模文件系统的导入导出和迁移。二进制格式的备份体积更小，内存占用更低，并且支持并发导入导出。\n\n| 格式类型 | 结构特点 | 适用场景 | 体积大小 | 内存占用 | 版本要求 |\n|------------|------------------|--------------------|------------------|------------------|------------|\n| **JSON 格式** | 完整目录树结构，易读 | 中小规模文件系统；问题定位 | 较大 | 较高 | 所有版本 |\n| **二进制格式** | 扁平化结构，高效紧凑 | 大规模导入导出和迁移 | 约为 JSON 的 1/3 | < 1GiB（1 亿文件） | v1.3.0+ |\n\n### 手动备份 {#backup-manually}\n\n使用 JuiceFS 客户端提供的 `dump` 命令可以将元数据导出到文件，例如：\n\n```shell\n# 导出为 JSON 格式\njuicefs dump redis://192.168.1.6:6379/1 meta-dump\n\n# 导出为二进制格式\njuicefs dump redis://192.168.1.6:6379/1 meta-dump --binary\n```\n\n上例中 `meta-dump` 是导出的备份文件，你可以随意调整它的文件名和扩展名。特别地，如果文件的扩展名为 `.gz`（如 `meta-dump.gz`），将会使用 Gzip 算法对导出的数据进行压缩。将会使用 Gzip 算法对导出的数据进行压缩。v1.3 版本之后也支持 Zstandard 压缩算法，使用 `.zstd` 作为扩展名。\n\n`dump` 命令默认从根目录 `/` 开始，深度遍历目录树下所有文件，将每个文件的元数据信息按 JSON 格式进行输出。出于数据安全的考虑，对象存储的认证信息不会被导出，但可以通过 `--keep-secret-key` 选项保留。\n\n`juicefs dump` 的价值在于它能将完整的元数据信息以统一的 JSON 格式导出，便于管理和保存，而且不同的元数据存储引擎都可以识别并导入。\n\n在实际应用中，`dump` 命令与数据库自带的备份工具应该共同使用，相辅相成。比如，Redis 有 [Redis RDB](https://redis.io/topics/persistence#backing-up-redis-data)，MySQL 有 [`mysqldump`](https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/mysqldump-sql-format.html) 等。\n\n### 自动备份 {#backup-automatically}\n\n从 JuiceFS v1.0.0 开始，不论文件系统通过 `mount` 命令挂载，还是通过 JuiceFS S3 网关及 Hadoop Java SDK 访问，客户端每小时都会自动备份元数据并拷贝到对象存储。\n\n备份的文件存储在对象存储的 `meta` 目录中，它是一个独立于数据存储的目录，在挂载点中不可见，也不会与数据存储之间产生影响，用对象存储的文件浏览器即可查看和管理。\n\n![meta-auto-backup-list](../images/meta-auto-backup-list.png)\n\n默认情况下，JuiceFS 客户端每小时备份一次元数据，自动备份的频率可以在挂载文件系统时通过 `--backup-meta` 选项进行调整，例如，要设置为每 8 个小时执行一次自动备份：\n\n```shell\njuicefs mount -d --backup-meta 8h redis://127.0.0.1:6379/1 /mnt\n```\n\n备份频率可以精确到秒，支持的单位如下：\n\n- `h`：精确到小时，如 `1h`；\n- `m`：精确到分钟，如 `30m`、`1h30m`；\n- `s`：精确到秒，如 `50s`、`30m50s`、`1h30m50s`;\n\n值得一提的是，备份操作耗时会随着文件系统内文件数的增多而增加，因此当文件数较多（默认为达到一百万）且自动备份频率为默认值 1 小时的情况下 JuiceFS 会自动跳过元数据备份，并打印相应的告警日志。此时可以选择挂载一个新客户端并设置较大的 `--backup-meta` 参数来重新启用自动备份。\n\n作为参考，当使用 Redis 作为元数据引擎时，备份一百万文件的元数据大约需要 1 分钟，消耗约 1GB 内存。\n\n:::caution 注意\n使用 `--read-only` 只读挂载时，元数据不会自动备份。\n:::\n\n#### 自动备份策略\n\n虽然自动备份元数据成为了客户端的默认动作，但在多主机共享挂载同一个文件系统时并不会发生备份冲突。\n\nJuiceFS 维护了一个全局的时间戳，确保同一时刻只有一个客户端执行备份操作。当客户端之间设置了不同的备份周期，那么就会以周期最短的设置为准进行备份。\n\n#### 备份清理策略\n\nJuiceFS 会按照以下规则定期清理备份：\n\n- 保留 2 天以内全部的备份；\n- 超过 2 天不足 2 周的，保留每天中的 1 个备份；\n- 超过 2 周不足 2 月的，保留每周中的 1 个备份；\n- 超过 2 个月的，保留每个月中的 1 个备份。\n\n## 元数据恢复与迁移 {#recovery-and-migration}\n\n使用 [`load`](../reference/command_reference.mdx#load) 命令可以将 `dump` 命令导出的元数据恢复到一个空数据库中，比如：\n\n```shell\n# 从 JSON 文件导入\njuicefs load redis://192.168.1.6:6379/1 meta-dump\n\n# 从二进制备份导入\njuicefs load redis://192.168.1.6:6379/1 meta-dump --binary\n```\n\n导入元数据时，JuiceFS 会重新计算文件系统的统计信息，包括空间使用量、inode 计数器等，最后在数据库中生成一份全局一致的元数据。如果你对 JuiceFS 的元数据设计有深入理解，还可以在恢复前对元数据备份文件进行修改，以此来进行调试。\n\n`dump` 命令导出的 JSON 格式数据是统一且通用的，所有元数据引擎都能识别和导入。因此，你不但可以把备份恢复到原有类型的数据库中，还可以恢复到其它数据库，从而实现元数据引擎的迁移。\n\n例如将元数据从 Redis 迁移到 MySQL：\n\n1. 从 Redis 导出元数据备份：\n\n ```shell\n juicefs dump redis://192.168.1.6:6379/1 meta-dump.json\n ```\n\n1. 将元数据恢复到一个全新的 MySQL 数据库：\n\n ```shell\n juicefs load mysql://user:password@(192.168.1.6:3306)/juicefs meta-dump.json\n ```\n\n另外，也可以通过系统的管道直接迁移：\n\n```shell\njuicefs dump redis://192.168.1.6:6379/1 | juicefs load mysql://user:password@(192.168.1.6:3306)/juicefs\n```\n\n需要注意的是，由于 `dump` 导出的备份中默认排除了对象存储的 API 访问密钥，不论恢复还是迁移元数据，完成操作后都需要使用 [`juicefs config`](../reference/command_reference.mdx#config) 命令把文件系统关联的对象存储的认证信息再添加回去，例如：\n\n```shell\njuicefs config --secret-key xxxxx mysql://user:password@(192.168.1.6:3306)/juicefs\n```\n\n### 加密文件系统 {#encrypted-file-system}\n\n对于[加密的文件系统](../security/encryption.md)，所有文件都会在本地加密后才上传到后端对象存储，包括元数据自动备份文件，也会加密后才上传至对象存储。这与 `dump` 命令不同，`dump` 导出的元数据永远是明文的。\n\n对于加密文件系统，在恢复自动备份的元数据时需要额外设置 `JFS_RSA_PASSPHRASE` 环境变量，以及指定 RSA 私钥和加密算法：\n\n```shell\nexport JFS_RSA_PASSPHRASE=xxxxxx\njuicefs load \\\n --encrypt-rsa-key my-private.pem \\\n --encrypt-algo aes256gcm-rsa \\\n redis://192.168.1.6:6379/1 \\\n dump-2023-03-16-090750.json.gz\n```\n\n## 元数据检视 {#inspection}\n\n除了可以导出完整的元数据信息，`dump` 命令还支持导出特定子目录中的元数据。可以直观地查看到指定目录树下所有文件的内部信息，因此常被用来辅助排查问题。\n\n```shell\njuicefs dump redis://192.168.1.6:6379/1 meta-dump.json --subdir /path/in/juicefs\n```\n\n另外，也可以使用 `jq` 等工具对导出文件进行分析。\n\n### 二进制备份内容分析与排查\n\n二进制备份还支持直接查看类型统计、分段（Segment）信息等：\n\n```shell\n# 查看备份元数据类型统计信息\njuicefs load meta-dump --binary --stat\n\n# 查看备份元数据 Segments 信息（获取 offset）\njuicefs load meta-dump --binary --stat --offset=-1\n\n# 查看备份元数据指定 Segment（指定 offset）信息\njuicefs load meta-dump --binary --stat --offset=123416309\n```\n\n示例输出：\n\n```\nBackup Version: 1\n-----------------------\nName | Num\n-----------------------\nacl | 0\nchunk | 1111179\ncounter | 6\ndelFile | 0\nedge | 1112124\nformat | 1\n…\nSegment: format\nValue: {\n\"Name\": \"test2\",\n\"UUID\": \"15b92123-1395-40e4-a5aa-edb38918985a\",\n\"Storage\": \"file\",\n\"Bucket\": \"/home/hjf/.juicefs/local/\",\n\"BlockSize\": 4096,\n\"Compression\": \"none\",\n\"EncryptAlgo\": \"aes256gcm-rsa\",\n\"TrashDays\": 1,\n\"MetaVersion\": 1,\n\"MinClientVersion\": \"1.1.0-A\",\n\"DirStats\": true,\n\"EnableACL\": false\n}\n```\n\n> 二进制备份为 PB 格式，也可自定义工具对备份进行校验和查看。\n" }, { "path": "docs/zh_cn/administration/monitoring.md", "content": "---\ntitle: 监控与数据可视化\nsidebar_position: 3\ndescription: 了解 JuiceFS 的监控指标，以及如何通过 Prometheus 和 Grafana 实现数据可视化。\n---\n\nJuiceFS 提供了丰富的监控指标，本文介绍如何收集这些指标，并通过 Prometheus 和 Grafana 实现类似下图的可视化监控系统。\n\n![grafana_dashboard](../images/grafana_dashboard.png)\n\n搭建流程大致如下：\n\n1. 配置 Prometheus 抓取 JuiceFS 监控指标\n2. 让 Grafana 读取 Prometheus 中的监控数据\n3. 用 JuiceFS 官方的 Grafana 仪表盘模板展现监控指标\n\n:::tip 提示\n本文使用开源版的 Grafana 和 Prometheus 作为例子，如果你想使用 Grafana Cloud 来构建可视化监控系统，可以参考这篇文章 [「如何使用 Grafana 监控文件系统状态」](https://juicefs.com/zh-cn/blog/usage-tips/use-grafana-monitor-file-system-status)。\n:::\n\n## 快速上手视频\n\n
\n \n
\n\n## 1. 配置 Prometheus 抓取 JuiceFS 监控指标 {#add-scrape-config}\n\nJuiceFS 挂载后，默认会通过 `http://localhost:9567/metrics` 地址实时输出 Prometheus 格式的指标数据。为了查看各项指标在一个时间范围内的状态变化，需要搭建 Prometheus 并配置定时抓取和保存这些指标数据。\n\n![Prometheus-client-data](../images/prometheus-client-data.jpg)\n\n不同挂载或访问方式（如 FUSE 挂载、CSI 驱动、S3 网关、Hadoop SDK 等）收集指标数据的方式略有区别，详见[「收集监控指标」](#collect-metrics)。\n\n这里以最常见的 FUSE 挂载方式为例介绍，如果还没安装 Prometheus，可以参考[官方文档](https://prometheus.io/docs/prometheus/latest/installation)。\n\n编辑 [`prometheus.yml`](https://prometheus.io/docs/prometheus/latest/configuration/configuration) 配置文件，在抓取配置部分（`scrape_configs`）添加新的任务，定义 JuiceFS 客户端输出监控指标的地址：\n\n```yaml {20-22}\nglobal:\n scrape_interval: 15s\n evaluation_interval: 15s\n\nalerting:\n alertmanagers:\n - static_configs:\n - targets:\n # - alertmanager:9093\n\nrule_files:\n # - \"first_rules.yml\"\n # - \"second_rules.yml\"\n\nscrape_configs:\n - job_name: \"prometheus\"\n static_configs:\n - targets: [\"localhost:9090\"]\n\n - job_name: \"juicefs\"\n static_configs:\n - targets: [\"localhost:9567\"]\n```\n\n启动 Prometheus 服务：\n\n```shell\n./prometheus --config.file=prometheus.yml\n```\n\n访问 `http://localhost:9090` 即可看到 Prometheus 的界面。\n\n## 2. 让 Grafana 读取 Prometheus 中的监控数据 {#grafana}\n\nPrometheus 开始抓取 JuiceFS 的监控指标后，接下来要配置 Grafana 读取 Prometheus 中的数据。\n\n如果还没安装 Grafana，可以参考[官方文档](https://grafana.com/docs/grafana/latest/installation)。\n\n在 Grafana 中新建 Prometheus 类型的数据源：\n\n- **Name**：为了便于识别，可以填写文件系统的名称。\n- **URL**：Prometheus 的数据接口，默认为 `http://localhost:9090`。\n\n![Grafana-data-source](../images/grafana-data-source.jpg)\n\n## 3. 用 JuiceFS 官方的 Grafana 仪表盘模板展现监控指标 {#grafana-dashboard}\n\n在 Grafana Dashboard 仓库中可以找到 JuiceFS 官方维护的仪表盘模板，可以直接在 Grafana 中通过 `https://grafana.com/grafana/dashboards/20794/` 链接导入，也可以通过 ID `20794` 导入。\n\nGrafana 仪表盘如下图：\n\n![grafana_dashboard](../images/grafana_dashboard.png)\n\n## 收集监控指标 {#collect-metrics}\n\n根据部署 JuiceFS 方式的不同可以有不同的收集监控指标的方法，下面分别介绍。\n\n### FUSE 挂载 {#mount-point}\n\n当通过 [`juicefs mount`](../reference/command_reference.mdx#mount) 命令挂载 JuiceFS 文件系统后，可以通过 `http://localhost:9567/metrics` 这个地址收集监控指标，你也可以通过 `--metrics` 选项自定义。如：\n\n```shell\njuicefs mount --metrics localhost:9567 ...\n```\n\n你可以使用命令行工具查看这些监控指标：\n\n```shell\ncurl http://localhost:9567/metrics\n```\n\n除此之外，每个 JuiceFS 文件系统的根目录还有一个叫做 `.stats` 的隐藏文件，通过这个文件也可以查看监控指标。例如（这里假设挂载点的路径是 `/jfs`）：\n\n```shell\ncat /jfs/.stats\n```\n\n:::tip 提示\n如果想要实时查看监控指标，可以使用 [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) 命令。\n:::\n\n### Kubernetes {#kubernetes}\n\n参考 [CSI 驱动文档](https://juicefs.com/docs/zh/csi/administration/going-production#monitoring)。\n\n### S3 网关 {#s3-gateway}\n\n:::note 注意\n该特性需要运行 0.17.1 及以上版本 JuiceFS 客户端\n:::\n\n[JuiceFS S3 网关](../guide/gateway.md)默认会在 `http://localhost:9567/metrics` 这个地址提供监控指标，你也可以通过 `--metrics` 选项自定义。如：\n\n```shell\njuicefs gateway --metrics localhost:9567 ...\n```\n\n如果你是[在 Kubernetes 中部署](../guide/gateway.md#deploy-in-kubernetes) JuiceFS S3 网关，可以参考 [Kubernetes](#kubernetes) 小节的 Prometheus 配置来收集监控指标（区别主要在于 `__meta_kubernetes_pod_label_app_kubernetes_io_name` 这个标签的正则表达式），例如：\n\n```yaml {6-8}\nscrape_configs:\n - job_name: 'juicefs-s3-gateway'\n kubernetes_sd_configs:\n - role: pod\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]\n action: keep\n regex: juicefs-s3-gateway\n - source_labels: [__address__]\n action: replace\n regex: ([^:]+)(:\\d+)?\n replacement: $1:9567\n target_label: __address__\n - source_labels: [__meta_kubernetes_pod_node_name]\n target_label: node\n action: replace\n```\n\n#### 通过 Prometheus Operator 收集 {#prometheus-operator}\n\n[Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) 让用户在 Kubernetes 环境中能够快速部署和管理 Prometheus，借助 Prometheus Operator 提供的 `ServiceMonitor` CRD 可以自动生成抓取配置。例如（假设 JuiceFS S3 网关的 `Service` 部署在 `kube-system` 名字空间）：\n\n```yaml\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n name: juicefs-s3-gateway\nspec:\n namespaceSelector:\n matchNames:\n - kube-system\n selector:\n matchLabels:\n app.kubernetes.io/name: juicefs-s3-gateway\n endpoints:\n - port: metrics\n```\n\n有关 Prometheus Operator 的更多信息，请查看[官方文档](https://prometheus-operator.dev/docs/user-guides/getting-started)。\n\n### Hadoop Java SDK {#hadoop}\n\n[JuiceFS Hadoop Java SDK](../deployment/hadoop_java_sdk.md) 支持把监控指标上报到 [Pushgateway](https://github.com/prometheus/pushgateway) 或者 [Graphite](https://graphiteapp.org)。\n\n#### Pushgateway\n\n启用指标上报到 Pushgateway：\n\n```xml\n\n juicefs.push-gateway\n host:port\n\n```\n\n同时可以通过 `juicefs.push-interval` 配置修改上报指标的频率，默认为 10 秒上报一次。\n\n:::info 说明\n根据 [Pushgateway 官方文档](https://github.com/prometheus/pushgateway/blob/master/README.md#configure-the-pushgateway-as-a-target-to-scrape)的建议，Prometheus 的[抓取配置](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)中需要设置 `honor_labels: true`。\n\n需要特别注意，Prometheus 从 Pushgateway 抓取的指标的时间戳不是 JuiceFS Hadoop Java SDK 上报时的时间，而是抓取时的时间，具体请参考 [Pushgateway 官方文档](https://github.com/prometheus/pushgateway/blob/master/README.md#about-timestamps)。\n\n默认情况下 Pushgateway 只会在内存中保存指标，如果需要持久化到磁盘上，可以通过 `--persistence.file` 选项指定保存的文件路径以及 `--persistence.interval` 选项指定保存到文件的频率（默认 5 分钟保存一次）。\n:::\n\n:::note 注意\n每一个使用 JuiceFS Hadoop Java SDK 的进程会有唯一的指标，而 Pushgateway 会一直记住所有收集到的指标，导致指标数持续积累占用过多内存，也会使得 Prometheus 抓取指标时变慢，建议定期清理 Pushgateway 上的指标。\n\n定期使用下面的命令清理 Pushgateway 的指标数据，清空指标不影响运行中的 JuiceFS Hadoop Java SDK 持续上报数据。**注意 Pushgateway 启动时必须指定 `--web.enable-admin-api` 选项，同时以下命令会清空 Pushgateway 中的所有监控指标。**\n\n```bash\ncurl -X PUT http://host:9091/api/v1/admin/wipe\n```\n\n:::\n\n有关 Pushgateway 的更多信息，请查看[官方文档](https://github.com/prometheus/pushgateway/blob/master/README.md)。\n\n#### Graphite\n\n启用指标上报到 Graphite：\n\n```xml\n\n juicefs.push-graphite\n host:port\n\n```\n\n同时可以通过 `juicefs.push-interval` 配置修改上报指标的频率，默认为 10 秒上报一次。\n\nJuiceFS Hadoop Java SDK 支持的所有配置参数请参考[文档](../deployment/hadoop_java_sdk.md#客户端配置参数)。\n\n### 使用 Consul 作为注册中心 {#use-consul}\n\n:::note 注意\n该特性需要运行 1.0.0 及以上版本 JuiceFS 客户端\n:::\n\nJuiceFS 支持使用 Consul 作为监控指标 API 的注册中心，默认的 Consul 地址是 `127.0.0.1:8500`，你也可以通过 `--consul` 选项自定义。如：\n\n```shell\njuicefs mount --consul 1.2.3.4:8500 ...\n```\n\n当配置了 Consul 地址以后，`--metrics` 选项不再需要配置，JuiceFS 将会根据自身网络与端口情况自动配置监控指标 URL。如果同时设置了 `--metrics`，则会优先尝试监听配置的 URL。\n\n注册到 Consul 上的每个服务，其[服务名](https://developer.hashicorp.com/consul/docs/services/configuration/services-configuration-reference#name)都为 `juicefs`，[服务 ID](https://developer.hashicorp.com/consul/docs/services/configuration/services-configuration-reference#id) 的格式为 `:`，例如：`127.0.0.1:/tmp/jfs`。\n\n每个服务的 [`meta`](https://developer.hashicorp.com/consul/docs/services/configuration/services-configuration-reference#meta) 都包含 `hostname` 与 `mountpoint` 两个 key，对应的值分别表示挂载点所在的主机名和挂载点路径。特别地，S3 网关的 `mountpoint` 值总是为 `s3gateway`。\n\n成功注册到 Consul 上以后，需要在 `prometheus.yml` 中新增 [`consul_sd_config`](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) 配置，在 `services` 中填写 `juicefs`。\n\n## 监控指标索引 {#metrics-reference}\n\n参考[「JuiceFS 监控指标」](../reference/p8s_metrics.md)。\n" }, { "path": "docs/zh_cn/administration/mount_at_boot.md", "content": "---\ntitle: 启动时自动挂载 JuiceFS\nsidebar_position: 3\nslug: /mount_juicefs_at_boot_time\n---\n\n在确认挂载成功，可以正常使用以后，可以参考本节内容设置开机自动挂载。\n\n## Linux\n\n从 JuiceFS v1.1.0 开始，挂载命令的 `--update-fstab` 选项能自动帮你设置好开机自动挂载：\n\n```bash\n$ sudo juicefs mount --update-fstab --max-uploads=50 --writeback --cache-size 204800 \n$ grep /etc/fstab\n juicefs _netdev,max-uploads=50,writeback,cache-size=204800 0 0\n$ ls -l /sbin/mount.juicefs\nlrwxrwxrwx 1 root root 29 Aug 11 16:43 /sbin/mount.juicefs -> /usr/local/bin/juicefs\n```\n\n如果你有意自行控制，请注意：\n\n* 需要创建一个从 `/sbin/mount.juicefs` 到 JuiceFS 可执行文件的软链接，比如 `ln -s /usr/local/bin/juicefs /sbin/mount.juicefs`。\n* 挂载命令所包含的各种选项，也需要在 fstab options 列加以声明，注意去掉 `-` 前缀，并将选项取值以 `=` 连接，举例说明：\n\n ```bash\n $ sudo juicefs mount --update-fstab --max-uploads=50 --writeback --cache-size 204800 -o max_read=99 /jfs\n # -o 是 FUSE options，在 fstab 中需特殊对待\n $ grep jfs /etc/fstab\n redis://localhost:6379/1 /jfs juicefs _netdev,max-uploads=50,max_read=99,writeback,cache-size=204800 0 0\n ```\n\n:::tip 提示\n默认情况下，CentOS 6 在启动后不会自动挂载网络文件系统，你可以使用下面的命令开启它：\n\n```bash\nsudo chkconfig --add netfs\n```\n\n:::\n\n### 使用 systemd.mount 实现自动挂载\n\n基于安全考虑，JuiceFS 将命令行中的一些选项隐藏在环境变量中，所以像数据库访问密码、S3 访问密钥和密钥等设置不能直接应用于 `/etc/fstab` 文件。在这种情况下，你可以使用 systemd 来挂载 JuiceFS 实例。\n\n以下是如何设置 systemd 配置文件的步骤：\n\n1. 创建文件 `/etc/systemd/system/juicefs.mount`，并添加以下内容：\n\n ```conf\n [Unit]\n Description=Juicefs\n Before=docker.service\n\n [Mount]\n Environment=\"ALICLOUD_ACCESS_KEY_ID=mykey\" \"ALICLOUD_ACCESS_KEY_SECRET=mysecret\" \"META_PASSWORD=mypassword\"\n What=mysql://juicefs@(mysql.host:3306)/juicefs\n Where=/juicefs\n Type=juicefs\n Options=_netdev,allow_other,writeback_cache\n\n [Install]\n WantedBy=remote-fs.target\n WantedBy=multi-user.target\n ```\n\n 你可以根据需要更改环境变量、挂载选项等。\n\n2. 使用以下命令启用和启动 JuiceFS 挂载：\n\n ```sh\n ln -s /usr/local/bin/juicefs /sbin/mount.juicefs\n systemctl enable juicefs.mount\n systemctl start juicefs.mount\n ```\n\n完成这些步骤后，就可以访问 `/juicefs` 目录来存取文件了。\n\n## macOS\n\n在 `~/Library/LaunchAgents` 下创建名为 `io.juicefs..plist` 的文件。替换 `` 为 JuiceFS 文件系统的名字。添加如下内容到文件中（再次替换 `NAME`、`PATH-TO-JUICEFS`、`META-URL`、`MOUNTPOINT` 和 `MOUNT-OPTIONS` 为适当的值）：\n\n```xml\n\n\n\n\n Label\n io.juicefs.NAME\n ProgramArguments\n \n PATH-TO-JUICEFS\n mount\n META-URL\n MOUNTPOINT\n MOUNT-OPTIONS\n \n RunAtLoad\n \n\n\n```\n\n:::tip 提示\n如果有多个挂载选项可以分为多行依次设置，例如：\n\n```xml\n --max-uploads\n 50\n --cache-size\n 204800\n```\n\n:::\n\n使用以下命令加载上一步创建的文件，并测试加载是否成功。**请确保元数据引擎已正常运行。**\n\n```bash\nlaunchctl load ~/Library/LaunchAgents/io.juicefs..plist\nlaunchctl start ~/Library/LaunchAgents/io.juicefs.\nls \n```\n\n如果挂载失败，可以将以下配置添加到 `io.juicefs..plist` 文件来调试：\n\n```xml\n StandardOutPath\n /tmp/juicefs.out\n StandardErrorPath\n /tmp/juicefs.err\n```\n\n使用以下命令重新加载最新的配置并检查输出：\n\n```bash\nlaunchctl unload ~/Library/LaunchAgents/io.juicefs..plist\nlaunchctl load ~/Library/LaunchAgents/io.juicefs..plist\ncat /tmp/juicefs.out\ncat /tmp/juicefs.err\n```\n\n如果你是使用 Homebrew 安装的 Redis 服务，你可以使用以下命令让其在机器启动时启动它：\n\n```bash\nbrew services start redis\n```\n\n然后添加以下配置到 `io.juicefs..plist` 文件确保 Redis 服务已经启动：\n\n```xml\n KeepAlive\n \n OtherJobEnabled\n homebrew.mxcl.redis\n \n```\n" }, { "path": "docs/zh_cn/administration/status_check_and_maintenance.md", "content": "---\ntitle: 状态检查 & 维护\nsidebar_position: 4\n---\n\n任何一种存储系统在投入使用之后都需要定期进行检查和维护，尽早发现并修复潜在的问题，从而保证文件系统可靠运行、存储的数据完整一致。\n\nJuiceFS 提供了一系列检查和维护文件系统的工具，不但可以帮助我们了解文件系统的基本信息、运行状态，还能够帮助我们更容易地发现和修复潜在的问题。\n\n## status\n\n`juicefs status` 命令用来查看一个 JuiceFS 文件系统的基本信息，所有活跃的会话状态（包括挂载、SDK 访问、S3 网关、WebDAV 连接）以及统计信息。\n\n文件系统的基本信息中包括名称、UUID、存储类型、对象存储 Bucket、回收站状态等；统计信息默认有文件系统的配额与用量。\n\n```shell\njuicefs status redis://xxx.cache.amazonaws.com:6379/1\n```\n\n```json\n{\n \"Setting\": {\n \"Name\": \"myjfs\",\n \"UUID\": \"6b0452fc-0502-404c-b163-c9ab577ec766\",\n \"Storage\": \"s3\",\n \"Bucket\": \"https://xxx.s3.amazonaws.com\",\n \"AccessKey\": \"xxx\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"TrashDays\": 1,\n \"MetaVersion\": 1\n },\n \"Sessions\": [\n {\n \"Sid\": 2,\n \"Heartbeat\": \"2021-08-23T16:47:59+08:00\",\n \"Version\": \"1.0.0+2022-08-08.cf0c269\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869146\n }\n ],\n \"Statistic\": {\n \"UsedSpace\": 4886528,\n \"AvailableSpace\": 1125899901956096,\n \"UsedInodes\": 643,\n \"AvailableInodes\": 10485760,\n }\n}\n```\n\n通过 `--session, -s` 选项指定会话的 `Sid` 可以进一步显示该会话的更多信息：\n\n```shell\njuicefs status --session 2 redis://xxx.cache.amazonaws.com:6379/1\n```\n\n```json\n{\n \"Sid\": 2,\n \"Heartbeat\": \"2021-08-23T16:47:59+08:00\",\n \"Version\": \"1.0.0+2022-08-08.cf0c269\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869146\n}\n```\n\n根据会话的状态，信息中还可能包括：\n\n- Sustained inodes：这些是已经被删掉的文件，但是因为在这个会话中已经被打开，因此会被暂时保留直至文件关闭。\n- Flocks：被这个会话加锁的文件的 BSD 锁信息\n- Plocks：被这个会话加锁的文件的 POSIX 锁信息\n\n通过 `--more, -m` 选项扫描 trash 中的文件和 slice，以及延迟删除的文件和 slice：\n\n```shell\njuicefs status -m redis://xxx.cache.amazonaws.com:6379/1\n```\n\n```json\n{\n \"Setting\": {\n \"Name\": \"myjfs\",\n \"UUID\": \"6b0452fc-0502-404c-b163-c9ab577ec766\",\n \"Storage\": \"s3\",\n \"Bucket\": \"https://xxx.s3.amazonaws.com\",\n \"AccessKey\": \"xxx\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"TrashDays\": 1,\n \"MetaVersion\": 1\n },\n \"Sessions\": [\n {\n \"Sid\": 2,\n \"Heartbeat\": \"2021-08-23T16:47:59+08:00\",\n \"Version\": \"1.0.0+2022-08-08.cf0c269\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869146\n }\n ],\n \"Statistic\": {\n \"UsedSpace\": 4886528,\n \"AvailableSpace\": 1125899901956096,\n \"UsedInodes\": 643,\n \"AvailableInodes\": 10485760,\n \"TrashFileCount\": 277,\n \"TrashFileSize\": 1152597,\n \"PendingDeletedFileCount\": 156,\n \"PendingDeletedFileSize\": 1313577,\n \"TrashSliceCount\": 581,\n \"TrashSliceSize\": 1845292,\n \"PendingDeletedSliceCount\": 1378,\n \"PendingDeletedSliceSize\": 26245344,\n }\n```\n\n## info\n\n`juicefs info` 用于检查指定文件或目录的元数据信息，其中包括该文件对应的每个 block 在对象存储上的对象路径以及作用于该文件的 flock 与 plock。\n\n### 检查一个文件的元数据\n\n```shell\n$ juicefs info mnt/luggage-6255515.jpg\n\nmnt/luggage-6255515.jpg :\n inode: 36\n files: 1\n dirs: 0\n length: 789.02 KiB (807955 Bytes)\n size: 792.00 KiB (811008 Bytes)\n path: /luggage-6255515.jpg\nobjects:\n+------------+------------------------------+--------+--------+--------+\n| chunkIndex | objectName | size | offset | length |\n+------------+------------------------------+--------+--------+--------+\n| 0 | myjfs/chunks/0/0/80_0_807955 | 807955 | 0 | 807955 |\n+------------+------------------------------+--------+--------+--------+\nflocks:\n+-----+----------------------+------+\n| Sid | Owner | Type |\n+-----+----------------------+------+\n| 4 | 14034871352581537016 | W |\n+-----+----------------------+------+\n```\n\n### 检查一个目录的元数据\n\n该命令默认只检查一层目录：\n\n```shell\n$ juicefs info ./mnt\n\nmnt :\n inode: 1\n files: 9\n dirs: 4\n length: 2.41 MiB (2532102 Bytes)\n size: 2.44 MiB (2555904 Bytes)\n path: /\n```\n\n如果希望递归检查所有子目录，需要指定 `--recursive, -r` 选项：\n\n```shell\n$ juicefs info -r ./mnt\n\n./mnt :\n inode: 1\n files: 33\n dirs: 4\n length: 80.29 MiB (84191037 Bytes)\n size: 80.34 MiB (84242432 Bytes)\n path: /\n```\n\n默认情况下 `juicefs info -r` 在 `fast` 模式下运行，它结果中的目录用量不一定精准。如果你怀疑其准确性，可以使用 `--strict` 选项查看精准用量：\n\n```shell\n$ juicefs info -r ./mnt --strict\n\n./mnt :\n inode: 1\n files: 33\n dirs: 4\n length: 80.29 MiB (84191037 Bytes)\n size: 80.34 MiB (84242432 Bytes)\n path: /\n```\n\n### 使用 inode 检查元数据\n\n还可以通过 inode 来反向查找文件路径及数据块的信息，但需要先进入挂载点：\n\n```shell\n~ $ cd mnt\n~/mnt $ juicefs info -i 36\n\n36 :\n inode: 36\n files: 1\n dirs: 0\n length: 789.02 KiB (807955 Bytes)\n size: 792.00 KiB (811008 Bytes)\n path: /luggage-6255515.jpg\nobjects:\n+------------+------------------------------+--------+--------+--------+\n| chunkIndex | objectName | size | offset | length |\n+------------+------------------------------+--------+--------+--------+\n| 0 | myjfs/chunks/0/0/80_0_807955 | 807955 | 0 | 807955 |\n+------------+------------------------------+--------+--------+--------+\n```\n\n## summary\n\nJuiceFS 1.1.0 之后支持 `summary` 子命令，可以递归列出目录树和各层的使用量：\n\n```bash\n$ juicefs summary /mnt/jfs/\n+---------------------------+---------+------+-------+\n| PATH | SIZE | DIRS | FILES |\n+---------------------------+---------+------+-------+\n| / | 1.0 GiB | 100 | 445 |\n| d/ | 1.0 GiB | 1 | 1 |\n| d/test1 | 1.0 GiB | 0 | 1 |\n| pjdfstest/ | 2.8 MiB | 39 | 304 |\n| pjdfstest/tests/ | 1.1 MiB | 18 | 240 |\n| pjdfstest/autom4te.cache/ | 692 KiB | 1 | 7 |\n| pjdfstest/.git/ | 432 KiB | 17 | 26 |\n| pjdfstest/configure | 176 KiB | 0 | 1 |\n| pjdfstest/config.log | 84 KiB | 0 | 1 |\n| pjdfstest/pjdfstest.o | 80 KiB | 0 | 1 |\n| pjdfstest/pjdfstest | 68 KiB | 0 | 1 |\n| pjdfstest/aclocal.m4 | 44 KiB | 0 | 1 |\n| pjdfstest/pjdfstest.c | 40 KiB | 0 | 1 |\n| pjdfstest/config.status | 36 KiB | 0 | 1 |\n| pjdfstest/... | 164 KiB | 2 | 24 |\n| roa/ | 2.3 MiB | 59 | 140 |\n| roa/.git/ | 1.4 MiB | 17 | 26 |\n| roa/roa/ | 252 KiB | 9 | 30 |\n| roa/integration/ | 148 KiB | 13 | 22 |\n| roa/roa-core/ | 124 KiB | 4 | 17 |\n| roa/Cargo.lock | 84 KiB | 0 | 1 |\n| roa/roa-async-std/ | 36 KiB | 2 | 6 |\n| roa/.github/ | 32 KiB | 2 | 6 |\n| roa/examples/ | 32 KiB | 1 | 7 |\n| roa/roa-diesel/ | 32 KiB | 2 | 5 |\n| roa/assets/ | 28 KiB | 2 | 5 |\n| roa/... | 108 KiB | 6 | 15 |\n+---------------------------+---------+------+-------+\n```\n\n可以使用 `--depth value, -d value` 和 `--entries value, -e value` 选项控制目录层级和每层显示的最大数量：\n\n```bash\n$ juicefs summary /mnt/jfs/ -d 3 -e 3\n+------------------------------------+---------+------+-------+\n| PATH | SIZE | DIRS | FILES |\n+------------------------------------+---------+------+-------+\n| / | 1.0 GiB | 100 | 445 |\n| d/ | 1.0 GiB | 1 | 1 |\n| d/test1 | 1.0 GiB | 0 | 1 |\n| pjdfstest/ | 2.8 MiB | 39 | 304 |\n| pjdfstest/tests/ | 1.1 MiB | 18 | 240 |\n| pjdfstest/tests/open/ | 112 KiB | 1 | 26 |\n| pjdfstest/tests/rename/ | 112 KiB | 1 | 25 |\n| pjdfstest/tests/link/ | 76 KiB | 1 | 18 |\n| pjdfstest/tests/... | 776 KiB | 14 | 171 |\n| pjdfstest/autom4te.cache/ | 692 KiB | 1 | 7 |\n| pjdfstest/autom4te.cache/output.0 | 180 KiB | 0 | 1 |\n| pjdfstest/autom4te.cache/output.1 | 180 KiB | 0 | 1 |\n| pjdfstest/autom4te.cache/output.2 | 180 KiB | 0 | 1 |\n| pjdfstest/autom4te.cache/... | 148 KiB | 0 | 4 |\n| pjdfstest/.git/ | 432 KiB | 17 | 26 |\n| pjdfstest/.git/objects/ | 252 KiB | 3 | 2 |\n| pjdfstest/.git/hooks/ | 64 KiB | 1 | 13 |\n| pjdfstest/.git/logs/ | 32 KiB | 5 | 3 |\n| pjdfstest/.git/... | 80 KiB | 7 | 8 |\n| pjdfstest/... | 692 KiB | 2 | 31 |\n| roa/ | 2.3 MiB | 59 | 140 |\n| roa/.git/ | 1.4 MiB | 17 | 26 |\n| roa/.git/objects/ | 1.3 MiB | 3 | 2 |\n| roa/.git/hooks/ | 64 KiB | 1 | 13 |\n| roa/.git/logs/ | 32 KiB | 5 | 3 |\n| roa/.git/... | 72 KiB | 7 | 8 |\n| roa/roa/ | 252 KiB | 9 | 30 |\n| roa/roa/src/ | 228 KiB | 7 | 27 |\n| roa/roa/README.md | 8.0 KiB | 0 | 1 |\n| roa/roa/templates/ | 8.0 KiB | 1 | 1 |\n| roa/roa/... | 4.0 KiB | 0 | 1 |\n| roa/integration/ | 148 KiB | 13 | 22 |\n| roa/integration/diesel-example/ | 52 KiB | 4 | 9 |\n| roa/integration/multipart-example/ | 36 KiB | 4 | 5 |\n| roa/integration/juniper-example/ | 32 KiB | 2 | 5 |\n| roa/integration/... | 24 KiB | 2 | 3 |\n| roa/... | 476 KiB | 19 | 62 |\n+------------------------------------+---------+------+-------+\n```\n\n此命令也支持标准 csv 输出，用于其它软件解析：\n\n```bash\n$ juicefs summary /mnt/jfs/ --csv\nPATH,SIZE,DIRS,FILES\n/,1079132160,100,445\nd/,1073745920,1,1\nd/test1,1073741824,0,1\npjdfstest/,2969600,39,304\npjdfstest/tests/,1105920,18,240\npjdfstest/autom4te.cache/,708608,1,7\npjdfstest/.git/,442368,17,26\npjdfstest/configure,180224,0,1\npjdfstest/config.log,86016,0,1\npjdfstest/pjdfstest.o,81920,0,1\npjdfstest/pjdfstest,69632,0,1\npjdfstest/aclocal.m4,45056,0,1\npjdfstest/pjdfstest.c,40960,0,1\npjdfstest/config.status,36864,0,1\npjdfstest/...,167936,2,24\nroa/,2412544,59,140\nroa/.git/,1511424,17,26\nroa/roa/,258048,9,30\nroa/integration/,151552,13,22\nroa/roa-core/,126976,4,17\nroa/Cargo.lock,86016,0,1\nroa/roa-async-std/,36864,2,6\nroa/.github/,32768,2,6\nroa/examples/,32768,1,7\nroa/roa-diesel/,32768,2,5\nroa/assets/,28672,2,5\nroa/...,110592,6,15\n```\n\n默认情况下 `juicefs summary` 在 `fast` 模式下运行，它结果中的目录用量不一定精准。如果你怀疑其准确性，可以使用 `--strict` 选项查看精准用量。\n\n## gc {#gc}\n\n`juicefs gc` 是一个用来处理「对象泄漏」与「待清理对象」，以及因为覆盖写而产生的碎片数据的工具。它以元数据信息为基准与对象存储中的数据进行逐一扫描比对，从而找出或清理对象存储上需要处理的数据块。\n\n:::info 说明\n**对象泄漏**是指数据块在对象存储，但元数据引擎中没有对应的记录的情况。对象泄漏极少出现，成因可能是程序 bug、元数据引擎或对象存储的未预期问题、断电、断网等等。\n**待清理对象**是指被原数据引擎标记为删除但还未清理的对象。待删除对象很常见，比如到期的 trash 文件与 slice 和延迟删除的文件与 slice。\n:::\n\n:::tip 提示\n虽然几乎不会出现对象泄漏的情况，但你仍然可以根据需要进行相应例行检查。文件在上传到对象存储时可能产生临时的中间文件，它们会在写入完成后被清理。为了避免中间文件被误判为泄漏的对象，`juicefs gc` 默认会跳过最近 1 个小时上传的文件。可以通过 `JFS_GC_SKIPPEDTIME` 环境变量调整跳过的时间范围（单位为秒）。例如设置跳过最近 2 个小时的文件：`export JFS_GC_SKIPPEDTIME=7200`。\n:::\n\n:::tip 提示\n因为 `juicefs gc` 命令会扫描对象存储中的所有对象，所以对于数据量较大的文件系统执行这个命令会有一定开销。另外使用此命令之前请确保您不需要回滚到旧版本元数据，并且建议您备份对象存储数据。\n:::\n\n### 扫描\n\n默认情况下 `juicefs gc` 仅执行扫描：\n\n```shell\n$ juicefs gc sqlite3://myjfs.db\nPending deleted files: 0 0.0/s \n Pending deleted data: 0.0 b (0 Bytes) 0.0 b/s \nCleaned pending files: 0 0.0/s \n Cleaned pending data: 0.0 b (0 Bytes) 0.0 b/s \n Listed slices: 4437 82800.0/s \n Trash slices: 0 0.0/s \n Trash data: 0.0 b (0 Bytes) 0.0 b/s \n Cleaned trash slices: 0 0.0/s \n Cleaned trash data: 0.0 b (0 Bytes) 0.0 b/s \n Scanned objects: 4741/4741 [==============================================================] 387369.2/s used: 12.247821ms\n Valid objects: 4741 395521.0/s \n Valid data: 1.7 GiB (1846388716 Bytes) 143.6 GiB/s \n Compacted objects: 0 0.0/s \n Compacted data: 0.0 b (0 Bytes) 0.0 b/s \n Leaked objects: 0 0.0/s \n Leaked data: 0.0 b (0 Bytes) 0.0 b/s \n Skipped objects: 0 0.0/s \n Skipped data: 0.0 b (0 Bytes) 0.0 b/s \n2023/06/09 10:14:33.683384 juicefs[280403] : scanned 4741 objects, 4741 valid, 0 compacted (0 bytes), 0 leaked (0 bytes), 0 delslices (0 bytes), 0 delfiles (0 bytes), 0 skipped (0 bytes) [gc.go:379]\n```\n\n### 清理\n\n当 `juicefs gc` 命令扫描到了「泄漏的对象」或「待清理对象」，可以通过 `--delete` 选项对它们进行清理。客户端默认启动 10 个线程执行清理操作，可以使用 `--threads, -p` 选项来调整线程数量。\n\n```shell\n$ juicefs gc sqlite3://myjfs.db --delete\nCleaned pending slices: 0 0.0/s \n Pending deleted files: 0 0.0/s \n Pending deleted data: 0.0 b (0 Bytes) 0.0 b/s \n Cleaned pending files: 0 0.0/s \n Cleaned pending data: 0.0 b (0 Bytes) 0.0 b/s \n Cleaned trash: 0 0.0/s \nCleaned detached nodes: 0 0.0/s \n Listed slices: 4437 75803.6/s \n Trash slices: 0 0.0/s \n Trash data: 0.0 b (0 Bytes) 0.0 b/s \n Cleaned trash slices: 0 0.0/s \n Cleaned trash data: 0.0 b (0 Bytes) 0.0 b/s \n Scanned objects: 4741/4741 [==============================================================] 337630.2/s used: 14.056704ms\n Valid objects: 4741 345974.4/s \n Valid data: 1.7 GiB (1846388716 Bytes) 125.6 GiB/s \n Compacted objects: 0 0.0/s \n Compacted data: 0.0 b (0 Bytes) 0.0 b/s \n Leaked objects: 0 0.0/s \n Leaked data: 0.0 b (0 Bytes) 0.0 b/s \n Skipped objects: 0 0.0/s \n Skipped data: 0.0 b (0 Bytes) 0.0 b/s \n2023/06/09 10:15:49.819995 juicefs[280474] : scanned 4741 objects, 4741 valid, 0 compacted (0 bytes), 0 leaked (0 bytes), 0 delslices (0 bytes), 0 delfiles (0 bytes), 0 skipped (0 bytes) [gc.go:379]\n```\n\n随后可以再执行一次 `juicefs gc` 检查是否清理成功。\n\n## fsck\n\n`juicefs fsck` 是一个以数据块为基准与元数据进行逐一扫描比对的工具，主要用来修复文件系统内可能发生而且可以修复的各种问题。它可以帮你找到元数据引擎中存在记录，但对象存储中没有对应数据块的情况，还可以检查文件的属性信息是否存在。\n\n```shell {5}\n$ juicefs fsck sqlite3://myjfs2.db\n\n2022/11/10 17:31:19.062348 juicefs[26158] : Meta address: sqlite3://myjfs2.db [interface.go:402]\n2022/11/10 17:31:19.063132 juicefs[26158] : Data use file:///Users/herald/.juicefs/local/myjfs/ [fsck.go:73]\n2022/11/10 17:31:19.065857 juicefs[26158] : can't find block 0/1/1063_0_2693747 for file /david-bruno-silva-Z19vToWBDIc-unsplash.jpg: stat /Users/herald/.juicefs/local/myjfs/chunks/0/1/1063_0_2693747: no such file or directory [fsck.go:146]\n Found blocks count: 68\n Found blocks bytes: 34.24 MiB (35904042 Bytes)\n Listed slices count: 65\nScanned slices count: 65 / 65 [=======================================] done\nScanned slices bytes: 36.81 MiB (38597789 Bytes)\n Lost blocks count: 1\n Lost blocks bytes: 2.57 MiB (2693747 Bytes)\n2022/11/10 17:31:19.066243 juicefs[26158] : 1 objects are lost (2693747 bytes), 1 broken files:\n INODE: PATH\n 57: /david-bruno-silva-Z19vToWBDIc-unsplash.jpg [fsck.go:168]\n```\n\n从结果可以看到，`juicefs fsck` 扫描发现文件系统中因为丢失了数据块致使一个文件损坏。\n\n虽然结果表明后端存储中的文件已经损坏，但还是有必要去挂载点查验一下文件是否可以访问，因为 JuiceFS 会在本地缓存最近访问过的文件数据，文件损坏之前的版本如果已经缓存在本地，则可以将缓存的文件数据块重新上传以避免丢失数据。你可以在缓存目录（即 `--cache-dir` 选项对应的路径）中根据 `juicefs fsck` 命令输出的数据块路径查找是否存在缓存数据，例如上面例子中丢失的数据块路径为 `0/1/1063_0_2693747`。\n\n### 强制同步目录用量\n\n在[目录用量统计](../guide/dir-stats.md)中我们介绍了这个新功能。虽然 fsck 默认会发现以及修复明显损坏的目录用量，但目录用量仍有可能不精准。我们可以使用 `--sync-dir-stat` 选项来强制检查或修复目录用量：\n\n```bash\n$ juicefs fsck redis://localhost --path /d --sync-dir-stat\n2023/06/07 15:59:14.080820 juicefs[228395] : Meta address: redis://localhost [interface.go:494]\n2023/06/07 15:59:14.082555 juicefs[228395] : Ping redis latency: 49.904µs [redis.go:3569]\n2023/06/07 15:59:14.083412 juicefs[228395] : usage stat of /d should be &{1073741824 1073741824 1}, but got &{0 0 0} [base.go:2026]\n2023/06/07 15:59:14.083443 juicefs[228395] : Stat of path /d (inode 10701) should be synced, please re-run with '--path /d --repair --sync-dir-stat' to fix it [base.go:2041]\n2023/06/07 15:59:14.083473 juicefs[228395] : some errors occurred, please check the log of fsck [main.go:31]\n\n$ juicefs fsck redis://localhost --path /d --repair --sync-dir-stat\n2023/06/07 16:00:43.043851 juicefs[228487] : Meta address: redis://localhost [interface.go:494]\n2023/06/07 16:00:43.051556 juicefs[228487] : Ping redis latency: 577.29µs [redis.go:3569]\n\n# 成功修复\n$ juicefs fsck redis://localhost --path /d --sync-dir-stat\n2023/06/07 16:01:08.401972 juicefs[228547] : Meta address: redis://localhost [interface.go:494]\n2023/06/07 16:01:08.404041 juicefs[228547] : Ping redis latency: 85.566µs [redis.go:3569]\n```\n\n## compact {#compact}\n\n`juicefs compact` 是 v1.2 版本中新增的功能，它是一个用来处理因为覆盖写而产生的碎片数据的工具。它将随机写产生的大量不连续的 slice 进行合并或清理，从而提升文件系统的读性能。\n\n相比于 `juicefs gc` 对整个文件系统进行垃圾回收和碎片整理，`juicefs compact` 可指定目录处理因为覆盖写而产生的碎片数据。\n\n```shell\njuicefs compact /mnt/jfs/foo\n```\n\n另外，可以使用 `-p, --threads` 选项指定并发线程数，以加快处理速度。默认值为 10，可以根据实际情况调整。\n\n```shell\njuicefs compact /mnt/jfs/foo -p 20\n```\n" }, { "path": "docs/zh_cn/administration/sync_accounts_between_multiple_hosts.md", "content": "---\ntitle: 多主机间同步账户\nsidebar_position: 7\nslug: /sync_accounts_between_multiple_hosts\n---\n\nJuiceFS 支持 Unix 文件权限，以目录或文件的粒度管理权限。该行为与本地文件系统相同。\n\n为了让用户获得直观一致的权限管理体验（例如，用户 A 在主机 X 中访问的文件，在主机 Y 中也应该可以用相同的用户身份访问），想要访问 JuiceFS 存储的同一个用户，应该在所有主机上具有相同的 UID 和 GID。\n\n在这里，我们提供了一个简单的 [Ansible](https://www.ansible.com/community) playbook 来演示如何确保一个帐户在多个主机上具有相同的 UID 和 GID。\n\n:::note 注意\n如果你是在 Hadoop 环境使用 JuiceFS，除了在多主机间同步账户以外，也可以指定一个全局的用户列表和所属用户组文件，具体请参见[这里](../deployment/hadoop_java_sdk.md#其它配置)。\n:::\n\n## 安装 Ansible\n\n选择一个主机作为 [控制节点](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements)，它可以使用 `ssh` 以 `root` 或其他在 sudo 用户组的身份，访问所有。在此主机上安装 Ansible。阅读 [安装 Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#installing-ansible) 了解更多安装细节。\n\n## 确保所有主机上的帐户相同\n\n创建一个空目录 `account-sync` ，将下面的内容保存在该目录下的 `play.yaml` 中。\n\n```yaml\n---\n- hosts: all\n tasks:\n - name: \"Ensure group {{ group }} with gid {{ gid }} exists\"\n group:\n name: \"{{ group }}\"\n gid: \"{{ gid }}\"\n state: present\n\n - name: \"Ensure user {{ user }} with uid {{ uid }} exists\"\n user:\n name: \"{{ user }}\"\n uid: \"{{ uid }}\"\n group: \"{{ gid }}\"\n state: present\n```\n\n在该目录下创建一个名为 `hosts` 的文件，将所有需要创建账号的主机的 IP 地址放置在该文件中，每行一个 IP。\n\n在这里，我们确保在 2 台主机上使用 UID 1200 的帐户 `alice` 和 GID 500 的 `staff` 组：\n\n```shell\n~/account-sync$ cat hosts\n172.16.255.163\n172.16.255.180\n~/account-sync$ ansible-playbook -i hosts -u root --ssh-extra-args \"-o StrictHostKeyChecking=no\" \\\n--extra-vars \"group=staff gid=500 user=alice uid=1200\" play.yaml\n\nPLAY [all] ************************************************************************************************\n\nTASK [Gathering Facts] ************************************************************************************\nok: [172.16.255.180]\nok: [172.16.255.163]\n\nTASK [Ensure group staff with gid 500 exists] *************************************************************\nok: [172.16.255.163]\nok: [172.16.255.180]\n\nTASK [Ensure user alice with uid 1200 exists] *************************************************************\nchanged: [172.16.255.180]\nchanged: [172.16.255.163]\n\nPLAY RECAP ************************************************************************************************\n172.16.255.163 : ok=3 changed=1 unreachable=0 failed=0\n172.16.255.180 : ok=3 changed=1 unreachable=0 failed=0\n```\n\n现在已经在这 2 台主机上创建了新帐户 `alice:staff`。\n\n如果指定的 UID 或 GID 已分配给某些主机上的另一个用户或组，则创建将失败。\n\n```shell\n~/account-sync$ ansible-playbook -i hosts -u root --ssh-extra-args \"-o StrictHostKeyChecking=no\" \\\n--extra-vars \"group=ubuntu gid=1000 user=ubuntu uid=1000\" play.yaml\n\nPLAY [all] ************************************************************************************************\n\nTASK [Gathering Facts] ************************************************************************************\nok: [172.16.255.180]\nok: [172.16.255.163]\n\nTASK [Ensure group ubuntu with gid 1000 exists] ***********************************************************\nok: [172.16.255.163]\nfatal: [172.16.255.180]: FAILED! => {\"changed\": false, \"msg\": \"groupmod: GID '1000' already exists\\n\", \"name\": \"ubuntu\"}\n\nTASK [Ensure user ubuntu with uid 1000 exists] ************************************************************\nok: [172.16.255.163]\n to retry, use: --limit @/home/ubuntu/account-sync/play.retry\n\nPLAY RECAP ************************************************************************************************\n172.16.255.163 : ok=3 changed=0 unreachable=0 failed=0\n172.16.255.180 : ok=1 changed=0 unreachable=0 failed=1\n```\n\n在上面的示例中，组 ID 1000 已分配给主机 `172.16.255.180` 上的另一个组，我们应该 **更改 GID** 或 **删除主机 `172.16.255.180` 上 GID 为 1000** 的组，然后再次运行 playbook。\n\n:::caution 注意\n如果用户帐户已经存在于主机上，并且我们将其更改为另一个 UID 或 GID 值，则用户可能会失去对他们以前拥有的文件和目录的权限。例如：\n\n```shell\n$ ls -l /tmp/hello.txt\n-rw-r--r-- 1 alice staff 6 Apr 26 21:43 /tmp/hello.txt\n$ id alice\nuid=1200(alice) gid=500(staff) groups=500(staff)\n```\n\n我们将 alice 的 UID 从 1200 改为 1201\n\n```shell\n~/account-sync$ ansible-playbook -i hosts -u root --ssh-extra-args \"-o StrictHostKeyChecking=no\" \\\n--extra-vars \"group=staff gid=500 user=alice uid=1201\" play.yaml\n```\n\n现在我们没有权限删除这个文件，因为它的所有者不是 alice：\n\n```shell\n$ ls -l /tmp/hello.txt\n-rw-r--r-- 1 1200 staff 6 Apr 26 21:43 /tmp/hello.txt\n$ rm /tmp/hello.txt\nrm: remove write-protected regular file '/tmp/hello.txt'? y\nrm: cannot remove '/tmp/hello.txt': Operation not permitted\n```\n\n:::\n" }, { "path": "docs/zh_cn/administration/troubleshooting.md", "content": "---\ntitle: 问题排查案例\nsidebar_position: 6\n---\n\n这里收录常见问题的具体排查步骤。\n\n## 创建文件系统（format）错误 {#format-error}\n\n### 无法重复创建文件系统 {#create-file-system-repeatedly}\n\n元数据引擎已经执行了 `juicefs format`，再次执行时可能无法更新之前的某些配置，将会报错：\n\n```\ncannot update volume XXX from XXX to XXX\n```\n\n这种情况需要清理元数据引擎中对应的数据，再重试。\n\n### Redis URL 格式错误 {#invalid-redis-url}\n\n使用的 Redis 版本小于 6.0.0 时，执行 `juicefs format` 命令如果指定了 `username` 参数，将会报错：\n\n```\nformat: ERR wrong number of arguments for 'auth' command\n```\n\n只有 Redis 6.0.0 版本以后才支持指定 `username`，因此你需要省略 URL 中的 `username` 参数，例如 `redis://:password@host:6379/1`。\n\n### Redis 哨兵（Sentinel）模式 NOAUTH 错误 {#redis-sentinel-noauth-error}\n\n如果使用 [Redis 哨兵模式](../administration/metadata/redis_best_practices.md#sentinel-mode)时遇到以下错误：\n\n```\nsentinel: GetMasterAddrByName master=\"xxx\" failed: NOAUTH Authentication required.\n```\n\n请确认是否为 Redis 哨兵实例[设置了密码](https://redis.io/docs/management/sentinel/#configuring-sentinel-instances-with-authentication)，如果设置了，那么需要通过 `SENTINEL_PASSWORD` 环境变量单独配置连接哨兵实例的密码，元数据引擎 URL 里的密码只会用于连接 Redis 服务器。\n\n## 权限问题导致挂载错误 {#mount-permission-error}\n\n使用 [Docker bind mounts](https://docs.docker.com/storage/bind-mounts) 把宿主机上的一个目录挂载到容器中时，可能遇到下方错误：\n\n```\ndocker: Error response from daemon: error while creating mount source path 'XXX': mkdir XXX: file exists.\n```\n\n这往往是因为使用了非 root 用户执行 `juicefs mount` 命令，进而导致 Docker 没有权限访问这个目录。这个问题有两种解决方法：\n\n* 用 root 用户执行 `juicefs mount` 命令\n* 在 FUSE 的配置文件，以及挂载命令中增加 [`allow_other`](../reference/fuse_mount_options.md#allow_other) 挂载选项。\n\n使用普通用户执行 `juicefs mount` 命令时，可能遇到下方错误：\n\n```\nfuse: fuse: exec: \"/bin/fusermount\": stat /bin/fusermount: no such file or directory\n```\n\n这个错误仅在普通用户执行挂载时出现，意味着找不到 `fusermount` 这个命令。此问题有两种解决方法：\n\n* 用 root 用户执行 `juicefs mount` 命令\n* 安装 `fuse` 包（例如 `apt-get install fuse`、`yum install fuse`）\n\n而如果当前用户不具备 `fusermount` 命令的执行权限，则还会遇到以下错误：\n\n```\nfuse: fuse: fork/exec /usr/bin/fusermount: permission denied\n```\n\n此时可以通过下面的命令检查 `fusermount` 命令的权限：\n\n```shell\n# 只有 root 用户和 fuse 用户组的用户有权限执行\n$ ls -l /usr/bin/fusermount\n-rwsr-x---. 1 root fuse 27968 Dec 7 2011 /usr/bin/fusermount\n\n# 所有用户都有权限执行\n$ ls -l /usr/bin/fusermount\n-rwsr-xr-x 1 root root 32096 Oct 30 2018 /usr/bin/fusermount\n```\n\n## 读写慢与读写失败 {#read-write-error}\n\n### 与对象存储通信不畅（网速慢） {#io-error-object-storage}\n\n如果无法访问对象存储，或者仅仅是网速太慢，JuiceFS 客户端也会发生读写错误。你也可以在日志中找到相应的报错。\n\n```text\n# 上传块的速度不符合预期\n: slow request: PUT chunks/0/0/1_0_4194304 (%!s(), 20.512s)\n\n# flush 超时通常意味着对象存储上传失败\n: flush 9902558 timeout after waited 8m0s\n: pending slice 9902558-80: ...\n```\n\n如果是网络异常导致无法访问，或者对象存储本身出现服务异常，问题排查相对简单。但在如果是在低带宽场景下希望优化 JuiceFS 的使用体验，需要留意的事情就稍微多一些。\n\n首先，在网速慢的时候，JuiceFS 客户端上传／下载文件容易超时（类似上方的错误日志），这种情况下可以考虑：\n\n* 降低上传并发度，比如 [`--max-uploads=1`](../reference/command_reference.mdx#mount-data-storage-options)，避免上传超时。\n* 降低读写缓冲区大小，比如 [`--buffer-size=64`](../reference/command_reference.mdx#mount-data-cache-options) 或者更小。当带宽充裕时，增大读写缓冲区能提升并发性能。但在低带宽场景下使用过大的读写缓冲区，`flush` 的上传时间会很长，因此容易超时。\n* 默认 GET／PUT 请求超时时间为 60 秒，因此增大 `--get-timeout` 以及 `--put-timeout`，可以改善读写超时的情况。\n\n此外，低带宽环境下需要慎用[「客户端写缓存」](../guide/cache.md#client-write-cache)特性。先简单介绍一下 JuiceFS 的后台任务设计：每个 JuiceFS 客户端默认都启用后台任务，后台任务中会执行碎片合并（compaction）、异步删除等工作，而如果节点网络状况太差，则会降低系统整体性能。更糟的是如果该节点还启用了客户端写缓存，则容易出现碎片合并后上传缓慢，导致其他节点无法读取该文件的危险情况：\n\n```text\n# 由于 writeback，碎片合并后的结果迟迟上传不成功，导致其他节点读取文件报错\n: read file 14029704: input/output error\n: slow operation: read (14029704,131072,0): input/output error (0) <74.147891>\n: fail to read sliceId 1771585458 (off:4194304, size:4194304, clen: 37746372): get chunks/0/0/1_0_4194304: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=\"The specified key does not exist.\", RequestId=62E8FB058C0B5C3134CB80B6\n```\n\n为了避免此类问题，我们推荐在低带宽节点上禁用后台任务，也就是为挂载命令添加 [`--no-bgjob`](../reference/command_reference.mdx#mount-metadata-options) 参数。\n\n### 警告日志：找不到对象存储块 {#warning-log-block-not-found-in-object-storage}\n\n规模化使用 JuiceFS 时，往往会在客户端日志中看到类似以下警告：\n\n```\n: fail to read sliceId 1771585458 (off:4194304, size:4194304, clen: 37746372): get chunks/0/0/1_0_4194304: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=\"The specified key does not exist.\", RequestId=62E8FB058C0B5C3134CB80B6\n```\n\n出现这一类警告时，如果并未伴随着访问异常（比如日志中出现 `input/output error`），其实不必特意关注，客户端会自行重试，往往不影响文件访问。\n\n这行警告日志的含义是：访问 slice 出错了，因为对应的某个对象存储块不存在，对象存储返回了 `NoSuchKey` 错误。出现此类异常的可能原因有下：\n\n* JuiceFS 客户端会异步运行碎片合并（Compaction），碎片合并完成后，文件与对象存储数据块（Block）的关系随之改变，但此时可能其他客户端正在读取该文件，因此随即报错。\n* 某些客户端开启了[「写缓存」](../guide/cache.md#client-write-cache)，文件已经写入，提交到了元数据服务，但对应的对象存储 Block 却并未上传完成（比如[网速慢](#io-error-object-storage)），导致其他客户端在读取该文件时，对象存储返回数据不存在。\n\n再次强调，如果并未出现应用端访问异常，则可安全忽略此类警告。\n\n## 读放大 {#read-amplification}\n\n在 JuiceFS 中，一个典型的读放大现象是：对象存储的下行流量，远大于实际读文件的速度。比方说 JuiceFS 客户端的读吞吐为 200MiB/s，但是在 S3 观察到了 2GiB/s 的下行流量。\n\nJuiceFS 中内置了[预读](../guide/cache.md#client-read-cache)（prefetch）机制：随机读 block 的某一段，会触发整个 block 下载，这个默认开启的读优化策略，在某些场景下会带来读放大。了解这个设计以后，我们就可以开始排查了。\n\n结合先前问题排查方法一章中介绍的[访问日志](./fault_diagnosis_and_analysis.md#access-log)知识，我们可以采集一些访问日志来分析程序的读模式，然后针对性地调整配置。下面是一个实际生产环境案例的排查过程：\n\n```shell\n# 收集一段时间的访问日志，比如 30 秒：\ncat /jfs/.accesslog | grep -v \"^#$\" >> access.log\n\n# 用 wc、grep 等工具简单统计发现，访问日志中大多都是 read 请求：\nwc -l access.log\ngrep \"read (\" access.log | wc -l\n\n# 选取一个文件，通过 inode 追踪其访问模式，read 的输入参数里，第一个就是 inode：\ngrep \"read (148153116,\" access.log\n```\n\n采集到该文件的访问日志如下：\n\n```\n2022.09.22 08:55:21.013121 [uid:0,gid:0,pid:0] read (148153116,131072,28668010496): OK (131072) <1.309992>\n2022.09.22 08:55:21.577944 [uid:0,gid:0,pid:0] read (148153116,131072,14342746112): OK (131072) <1.385073>\n2022.09.22 08:55:22.098133 [uid:0,gid:0,pid:0] read (148153116,131072,35781816320): OK (131072) <1.301371>\n2022.09.22 08:55:22.883285 [uid:0,gid:0,pid:0] read (148153116,131072,3570397184): OK (131072) <1.305064>\n2022.09.22 08:55:23.362654 [uid:0,gid:0,pid:0] read (148153116,131072,100420673536): OK (131072) <1.264290>\n2022.09.22 08:55:24.068733 [uid:0,gid:0,pid:0] read (148153116,131072,48602152960): OK (131072) <1.185206>\n2022.09.22 08:55:25.351035 [uid:0,gid:0,pid:0] read (148153116,131072,60529270784): OK (131072) <1.282066>\n2022.09.22 08:55:26.631518 [uid:0,gid:0,pid:0] read (148153116,131072,4255297536): OK (131072) <1.280236>\n2022.09.22 08:55:27.724882 [uid:0,gid:0,pid:0] read (148153116,131072,715698176): OK (131072) <1.093108>\n2022.09.22 08:55:31.049944 [uid:0,gid:0,pid:0] read (148153116,131072,8233349120): OK (131072) <1.020763>\n2022.09.22 08:55:32.055613 [uid:0,gid:0,pid:0] read (148153116,131072,119523176448): OK (131072) <1.005430>\n2022.09.22 08:55:32.056935 [uid:0,gid:0,pid:0] read (148153116,131072,44287774720): OK (131072) <0.001099>\n2022.09.22 08:55:33.045164 [uid:0,gid:0,pid:0] read (148153116,131072,1323794432): OK (131072) <0.988074>\n2022.09.22 08:55:36.502687 [uid:0,gid:0,pid:0] read (148153116,131072,47760637952): OK (131072) <1.184290>\n2022.09.22 08:55:38.525879 [uid:0,gid:0,pid:0] read (148153116,131072,53434183680): OK (131072) <0.096732>\n```\n\n对着日志观察下来，发现读文件的行为大体上是「频繁随机小读」。我们尤其注意到 offset（也就是 `read` 的第三个参数）跳跃巨大，说明相邻的读操作之间跨度很大，难以利用到预读提前下载下来的数据（默认的块大小是 4MiB，换算为 4194304 字节的 offset）。也正因此，我们建议将 `--prefetch` 调整为 0（让预读并发度为 0，也就是禁用该行为），并重新挂载。这样一来，在该场景下的读放大问题得到很好的改善。\n\n## 内存占用过高 {#memory-optimization}\n\n如果 JuiceFS 客户端内存占用过高，考虑按照以下方向进行排查调优，但也请注意，内存优化势必不是免费的，每一项设置调整都将带来相应的开销，请在调整前做好充分的测试与验证。\n\n* 读写缓冲区（也就是 `--buffer-size`）的大小，直接与 JuiceFS 客户端内存占用相关，因此可以通过降低读写缓冲区大小来减少内存占用，但请注意降低以后可能同时也会对读写性能造成影响。更多详见[「读写缓冲区」](../guide/cache.md#buffer-size)。\n* JuiceFS 挂载客户端是一个 Go 程序，因此也可以通过降低 `GOGC`（默认 100）来令 Go 在运行时执行更为激进的垃圾回收（将带来更多 CPU 消耗，甚至直接影响性能）。详见[「Go Runtime」](https://pkg.go.dev/runtime#hdr-Environment_Variables)。\n* 如果你使用自建的 Ceph RADOS 作为 JuiceFS 的数据存储，可以考虑将 glibc 替换为 [TCMalloc](https://google.github.io/tcmalloc)，后者有着更高效的内存管理实现，能在该场景下有效降低堆外内存占用。\n\n## 卸载错误 {#unmount-error}\n\n卸载 JuiceFS 文件系统时，如果某个文件或者目录正在被使用，那么卸载将会报错（下方假设挂载点为 `/jfs`）：\n\n```shell\n# Linux\numount: /jfs: target is busy.\n (In some cases useful info about processes that use\n the device is found by lsof(8) or fuser(1))\n\n# macOS\nResource busy -- try 'diskutil unmount'\n```\n\n这种情况下可以：\n\n* 用类似 `lsof /jfs` 的命令，找出该文件系统下正在使用的文件，然后按需处置对应的进程（比如强制退出），然后再次尝试卸载。\n* 用 `echo 1 > /sys/fs/fuse/connections/[device-number]/abort` 强制关闭 FUSE 连接，然后再次尝试卸载。其中 `[device-number]` 也许需要你用 `lsof /jfs` 手动确认，不过本机只有一个 FUSE 挂载点的话，那么 `/sys/fs/fuse/connections` 下也只会包含一个目录，不必特意确认。\n* 如果并不关心已经打开的文件，只想要尽快卸载，也可以运行 `juicefs umount --force` 来强制卸载，不过注意，强制卸载在 Linux、macOS 上的行为并不一致：\n * 对 Linux 而言，`juicefs umount --force` 意味着 `umount --lazy`，文件系统会被卸载，但已打开的文件不会关闭，而是等进程退出后再退出 FUSE 客户端。\n * 对 macOS 而言，`juicefs umount --force` 意味着 `umount -f`，文件系统会被强制卸载，已打开的文件会强制关闭。\n\n## 系统自动 mount 不生效 {#netmount}\n\n管理员一般通过 `--update-fstab` 更新 `/etc/fstab` 以确保系统在重启后自动 mount JuiceFS 文件系统，但某些最小化的 Linux 发行版本如 Alpine, 可能在其基础镜像中缺少 netmount 或类似功能的包。这个包对于网络文件系统是必要的。如果缺少 netmount 包，系统在重启后无法在 `/etc/fstab` 中自动挂载 JuiceFS 文件系统。为了解决这个问题，需安装 netmount 包并启动相关服务。\n以 Alpine 为例：\n\n```bash\n# use --update-fstab to add juicefs mount to /etc/fstab\n\n# install and enable netmount service\napk add openrc\n\nrc-update add netmount boot\n# * service netmount added to runlevel boot\n\n rc-service netmount start\n# / # rc-service netmount start\n# * Mounting network filesystems ...\n\n```\n\n## 开发相关问题 {#development-related-issues}\n\n编译 JuiceFS 需要 GCC 5.4 及以上版本，版本过低可能导致类似下方报错：\n\n```\n/go/pkg/tool/linux_amd64/link: running gcc failed: exit status 1\n/go/pkg/tool/linux_amd64/compile: signal: killed\n```\n\n如果编译环境与运行环境的 glibc 版本不同，会发生如下报错：\n\n```\n$ juicefs\njuicefs: /lib/aarch64-linux-gnu/libc.so.6: version 'GLIBC_2.28' not found (required by juicefs)\n```\n\n这需要你在运行环境重新编译 JuiceFS 客户端，大部分 Linux 发行版都预置了 glibc，你可以用 `ldd --version` 确认其版本。\n" }, { "path": "docs/zh_cn/administration/upgrade.md", "content": "---\nsidebar_position: 9\n---\n\n# 客户端升级\n\n不同 JuiceFS 客户端的升级方式不同，以下分别介绍。\n\n## 挂载点\n\n### 普通升级\n\nJuiceFS 客户端只有一个二进制程序，升级新版只需用新版程序替换旧版程序即可。\n\n- **使用预编译客户端**：可以参照[「安装」](../getting-started/installation.md#install-the-pre-compiled-client)文档中相应系统的安装方法，下载最新的客户端，覆盖旧版客户端即可。\n- **手动编译客户端**：可以拉取最新的源代码重新编译，覆盖旧版客户端即可，具体请参考[「安装」](../getting-started/installation.md#manually-compiling)文档。\n\n:::caution 注意\n对于已经使用旧版 JuiceFS 客户端挂载好的文件系统，需要先[卸载文件系统](../getting-started/for_distributed.md#7-卸载文件系统)，然后用新版 JuiceFS 客户端重新挂载。\n\n卸载文件系统时需确保没有任何应用正在访问，否则将会卸载失败。不可强行卸载文件系统，有可能造成应用无法继续正常访问。\n:::\n\n### 平滑升级\n\nJuiceFS 在 v1.2 版本中开始支持平滑升级功能，即在相同的挂载点再次挂载 JuiceFS 即可实现业务无感的客户端平滑升级。另外该功能还可以用来动态的调整挂载参数。\n\n下面举例说明两个常用的场景\n\n1. 客户端升级\n 比如当前存在 `juicefs mount` 进程 `juicefs mount redis://127.0.0.1:6379/0 /mnt/jfs -d`，现希望在不卸载挂载点的情况下部署新的 JuiceFS 客户端，可以执行以下步骤：\n\n ```shell\n # 1. 备份当前二进制\n cp juicefs juicefs.bak\n \n # 2. 下载新的二进制覆盖当前 juicefs 二进制\n \n # 3. 再次执行 juicefs mount 命令完成平滑升级\n juicefs mount redis://127.0.0.1:6379/0 /mnt/jfs -d\n ```\n\n2. 动态调整挂载参数\n\n 比如当前存在 `juicefs mount` 进程 `juicefs mount redis://127.0.0.1:6379/0 /mnt/jfs -d`，现希望在不卸载挂载点的情况下将日志级别调整为 debug，可以执行以下命令：\n\n```shell\n# 调整日志级别\njuicefs mount redis://127.0.0.1:6379/0 /mnt/jfs --debug -d\n```\n\n一些注意事项：\n\n1. 平滑升级要求新旧进程的 JuiceFS 客户端版本都至少为 v1.2 版本。\n\n2. 新的挂载参数中的 FUSE 参数应该与旧的挂载参数保持一致，否则平滑升级会在当前挂载点上继续覆盖挂载。\n\n3. `enable-xattr` 开启时，平滑升级会在当前挂载点上继续覆盖挂载。\n\n## Kubernetes CSI 驱动\n\n请参考[官方文档](https://juicefs.com/docs/zh/csi/upgrade-csi-driver)了解如何升级 JuiceFS CSI 驱动。\n\n## S3 网关\n\n与[挂载点](#挂载点)一样，升级 S3 网关也是使用新版程序替换旧版程序即可。\n\n如果是[通过 Kubernetes 部署](../guide/gateway.md#deploy-in-kubernetes)，则需要根据具体部署的方式来升级，以下详细介绍。\n\n### 通过 kubectl 升级\n\n下载并修改 S3 网关[部署 YAML](https://github.com/juicedata/juicefs/blob/main/deploy/juicefs-s3-gateway.yaml) 中的 `juicedata/juicefs-csi-driver` 镜像标签为想要升级的版本（关于所有版本的详细说明请参考[这里](https://github.com/juicedata/juicefs-csi-driver/releases)），然后运行以下命令：\n\n```shell\nkubectl apply -f ./juicefs-s3-gateway.yaml\n```\n\n### 通过 Helm 升级\n\n请依次运行以下命令以升级 S3 网关：\n\n```shell\nhelm repo update\nhelm upgrade juicefs-s3-gateway juicefs-s3-gateway/juicefs-s3-gateway -n kube-system -f ./values.yaml\n```\n\n## Hadoop Java SDK\n\n请参考[「安装与编译客户端」](../deployment/hadoop_java_sdk.md#安装与编译客户端)文档了解如何安装新版本的 Hadoop Java SDK，然后根据[「部署客户端」](../deployment/hadoop_java_sdk.md#部署客户端)的步骤重新部署新版本客户端即可完成升级。\n\n:::note 注意\n某些组件必须重启以后才能使用新版本的 Hadoop Java SDK，具体请参考[「重启服务」](../deployment/hadoop_java_sdk.md#重启服务)文档。\n:::\n" }, { "path": "docs/zh_cn/benchmark/benchmark.md", "content": "---\ntitle: 常规测试\nsidebar_position: 1\nslug: .\ndescription: 本文介绍使用 FIO、mdtest 以及 JuiceFS 自带的 bench 命令对文件系统进行性能测试。\n---\n\n本章介绍的测试中使用 [Redis](https://redis.io) 作为元数据存储引擎。在该测试条件下，JuiceFS 拥有十倍于 Amazon [EFS](https://aws.amazon.com/efs) 和 [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 的性能表现。\n\n### 基础测试\n\nJuiceFS 提供了 `bench` 子命令来运行一些基本的基准测试，用以评估 JuiceFS 在当前环境的运行情况：\n\n![JuiceFS Bench](../images/juicefs-bench.png)\n\n### 吞吐量\n\n使用 [fio](https://github.com/axboe/fio) 在 JuiceFS、[EFS](https://aws.amazon.com/efs) 和 [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 上执行连续读写测试，结果如下：\n\n[![Sequential Read Write Benchmark](../images/sequential-read-write-benchmark.svg)](../images/sequential-read-write-benchmark.svg)\n\n结果表明，JuiceFS 可以提供比另外两个工具大 10 倍的吞吐量，[了解更多](fio.md)。\n\n### 元数据 IOPS\n\n使用 [mdtest](https://github.com/hpc/ior) 在 JuiceFS、[EFS](https://aws.amazon.com/efs) 和 [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 上执行简易的 mdtest 基准测试，结果如下：\n\n[![Metadata Benchmark](../images/metadata-benchmark.svg)](../images/metadata-benchmark.svg)\n\n结果表明，JuiceFS 可以提供比另外两个工具更高的元数据 IOPS，[了解更多](mdtest.md)。\n\n### 分析测试结果\n\n如遇性能问题，阅读[「实时性能监控」](../administration/fault_diagnosis_and_analysis.md#performance-monitor)了解如何排查。\n" }, { "path": "docs/zh_cn/benchmark/fio.md", "content": "---\ntitle: fio 基准测试\nsidebar_position: 7\nslug: /fio\n---\n\n:::tip 提示\nJuiceFS v1.0+ 默认启用了回收站，基准测试会在文件系统中创建和删除临时文件，这些文件最终会被转存到回收站 `.trash` 占用存储空间，为了避免这种情况，可以在基准测试之前关闭回收站 `juicefs config META-URL --trash-days 0`，详情参考[回收站](../security/trash.md)。\n:::\n\n## 测试方法\n\n使用 [fio](https://github.com/axboe/fio) 在 JuiceFS、[EFS](https://aws.amazon.com/efs) 和 [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 上执行顺序读、顺序写基准测试。\n\n## 测试工具\n\n以下测试使用的工具为 fio 3.1。\n\n顺序读测试 (任务数：1):\n\n```\nfio --name=sequential-read --directory=/s3fs --rw=read --refill_buffers --bs=4M --size=4G\nfio --name=sequential-read --directory=/efs --rw=read --refill_buffers --bs=4M --size=4G\nfio --name=sequential-read --directory=/jfs --rw=read --refill_buffers --bs=4M --size=4G\n```\n\n顺序写测试 (任务数：1):\n\n```\nfio --name=sequential-write --directory=/s3fs --rw=write --refill_buffers --bs=4M --size=4G --end_fsync=1\nfio --name=sequential-write --directory=/efs --rw=write --refill_buffers --bs=4M --size=4G --end_fsync=1\nfio --name=sequential-write --directory=/jfs --rw=write --refill_buffers --bs=4M --size=4G --end_fsync=1\n```\n\n顺序读测试 (任务数：16):\n\n```\nfio --name=big-file-multi-read --directory=/s3fs --rw=read --refill_buffers --bs=4M --size=4G --numjobs=16\nfio --name=big-file-multi-read --directory=/efs --rw=read --refill_buffers --bs=4M --size=4G --numjobs=16\nfio --name=big-file-multi-read --directory=/jfs --rw=read --refill_buffers --bs=4M --size=4G --numjobs=16\n```\n\n顺序写测试 (任务数：16):\n\n```\nfio --name=big-file-multi-write --directory=/s3fs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=16 --end_fsync=1\nfio --name=big-file-multi-write --directory=/efs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=16 --end_fsync=1\nfio --name=big-file-multi-write --directory=/jfs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=16 --end_fsync=1\n```\n\n## 测试环境\n\n以下测试结果均使用 fio 在亚马逊云 c5d.18xlarge EC2 (72 CPU, 144G RAM) 实例得出，操作系统采用 Ubuntu 18.04 LTS (Kernel 5.4.0) ，JuiceFS 使用同主机的本地 Redis (version 4.0.9) 实例存储元数据。\n\nJuiceFS 挂载命令：\n\n```\n./juicefs format --storage=s3 --bucket=https://.s3..amazonaws.com localhost benchmark\n./juicefs mount --max-uploads=150 --io-retries=20 localhost /jfs\n```\n\nEFS 挂载命令 (与配置说明中一致):\n\n```\nmount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport, .efs..amazonaws.com:/ /efs\n```\n\nS3FS (version 1.82) 挂载命令：\n\n```\ns3fs :/s3fs /s3fs -o host=https://s3..amazonaws.com,endpoint=,passwd_file=${HOME}/.passwd-s3fs\n```\n\n## 测试结果\n\n![Sequential Read Write Benchmark](../images/sequential-read-write-benchmark.svg)\n" }, { "path": "docs/zh_cn/benchmark/mdtest.md", "content": "---\ntitle: mdtest 基准测试\nsidebar_position: 8\nslug: /mdtest\n---\n\n:::tip 提示\nJuiceFS v1.0+ 默认启用了回收站，基准测试会在文件系统中创建和删除临时文件，这些文件最终会被转存到回收站 `.trash` 占用存储空间，为了避免这种情况，可以在基准测试之前关闭回收站 `juicefs config META-URL --trash-days 0`，详情参考[回收站](../security/trash.md)。\n:::\n\n## 测试方法\n\n使用 [mdtest](https://github.com/hpc/ior)，分别在 JuiceFS、[EFS](https://aws.amazon.com/efs) 和 [S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 上执行元数据性能测试。\n\n## 测试工具\n\n以下测试使用 mdtest 3.4。\n调整 mdtest 的参数以确保命令可以在 5 分钟内完成。\n\n```\n./mdtest -d /s3fs/mdtest -b 6 -I 8 -z 2\n./mdtest -d /efs/mdtest -b 6 -I 8 -z 4\n./mdtest -d /jfs/mdtest -b 6 -I 8 -z 4\n```\n\n## 测试环境\n\n在下面的测试结果中，所有 mdtest 均在亚马逊云 c5.large EC2 实例（2 CPU，4G RAM），Ubuntu 18.04 LTS（Kernel 5.4.0）系统上进行，JuiceFS 使用的 Redis（4.0.9 版本）实例运行在相同区域的 c5.large EC2 实例上。\n\nJuiceFS 挂载命令：\n\n```\n./juicefs format --storage=s3 --bucket=https://.s3..amazonaws.com localhost benchmark\nnohup ./juicefs mount localhost /jfs &\n```\n\nEFS 挂载命令 (与配置说明保持一致)：\n\n```\nmount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport, .efs..amazonaws.com:/ /efs\n```\n\nS3FS (version 1.82) 挂载命令：\n\n```\ns3fs :/s3fs /s3fs -o host=https://s3..amazonaws.com,endpoint=,passwd_file=${HOME}/.passwd-s3fs\n```\n\n## 测试结果\n\n![Metadata Benchmark](../images/metadata-benchmark.svg)\n\n### S3FS\n\n```\nmdtest-3.4.0+dev was launched with 1 total task(s) on 1 node(s)\nCommand line used: ./mdtest '-d' '/s3fs/mdtest' '-b' '6' '-I' '8' '-z' '2'\nWARNING: Read bytes is 0, thus, a read test will actually just open/close.\nPath : /s3fs/mdtest\nFS : 256.0 TiB Used FS: 0.0% Inodes: 0.0 Mi Used Inodes: -nan%\nNodemap: 1\n1 tasks, 344 files/directories\n\nSUMMARY rate: (of 1 iterations)\n Operation Max Min Mean Std Dev\n --------- --- --- ---- -------\n Directory creation : 5.977 5.977 5.977 0.000\n Directory stat : 435.898 435.898 435.898 0.000\n Directory removal : 8.969 8.969 8.969 0.000\n File creation : 5.696 5.696 5.696 0.000\n File stat : 68.692 68.692 68.692 0.000\n File read : 33.931 33.931 33.931 0.000\n File removal : 23.658 23.658 23.658 0.000\n Tree creation : 5.951 5.951 5.951 0.000\n Tree removal : 9.889 9.889 9.889 0.000\n```\n\n### EFS\n\n```\nmdtest-3.4.0+dev was launched with 1 total task(s) on 1 node(s)\nCommand line used: ./mdtest '-d' '/efs/mdtest' '-b' '6' '-I' '8' '-z' '4'\nWARNING: Read bytes is 0, thus, a read test will actually just open/close.\nPath : /efs/mdtest\nFS : 8388608.0 TiB Used FS: 0.0% Inodes: 0.0 Mi Used Inodes: -nan%\nNodemap: 1\n1 tasks, 12440 files/directories\n\nSUMMARY rate: (of 1 iterations)\n Operation Max Min Mean Std Dev\n --------- --- --- ---- -------\n Directory creation : 192.301 192.301 192.301 0.000\n Directory stat : 1311.166 1311.166 1311.166 0.000\n Directory removal : 213.132 213.132 213.132 0.000\n File creation : 179.293 179.293 179.293 0.000\n File stat : 915.230 915.230 915.230 0.000\n File read : 371.012 371.012 371.012 0.000\n File removal : 217.498 217.498 217.498 0.000\n Tree creation : 187.906 187.906 187.906 0.000\n Tree removal : 218.357 218.357 218.357 0.000\n```\n\n### JuiceFS\n\n```\nmdtest-3.4.0+dev was launched with 1 total task(s) on 1 node(s)\nCommand line used: ./mdtest '-d' '/jfs/mdtest' '-b' '6' '-I' '8' '-z' '4'\nWARNING: Read bytes is 0, thus, a read test will actually just open/close.\nPath : /jfs/mdtest\nFS : 1024.0 TiB Used FS: 0.0% Inodes: 10.0 Mi Used Inodes: 0.0%\nNodemap: 1\n1 tasks, 12440 files/directories\n\nSUMMARY rate: (of 1 iterations)\n Operation Max Min Mean Std Dev\n --------- --- --- ---- -------\n Directory creation : 1416.582 1416.582 1416.582 0.000\n Directory stat : 3810.083 3810.083 3810.083 0.000\n Directory removal : 1115.108 1115.108 1115.108 0.000\n File creation : 1410.288 1410.288 1410.288 0.000\n File stat : 5023.227 5023.227 5023.227 0.000\n File read : 3487.947 3487.947 3487.947 0.000\n File removal : 1163.371 1163.371 1163.371 0.000\n Tree creation : 1503.004 1503.004 1503.004 0.000\n Tree removal : 1119.806 1119.806 1119.806 0.000\n```\n" }, { "path": "docs/zh_cn/benchmark/metadata_engines_benchmark.md", "content": "---\ntitle: 元数据引擎性能测试\nsidebar_position: 6\nslug: /metadata_engines_benchmark\ndescription: 本文采用亚马逊云的真实环境，介绍如何对 JuiceFS 的各种元数据引擎性能进行测试和评估。\n---\n\n首先展示结论：\n\n- 对于纯元数据操作，MySQL 耗时约为 Redis 的 2～4 倍；TiKV 性能与 MySQL 接近，大部分场景下略优于 MySQL；etcd 的耗时约为 TiKV 的 1.5 倍\n- 对于小 IO（～100 KiB）压力，使用 MySQL 引擎的操作总耗时大约是使用 Redis 引擎总耗时的 1～3 倍；TiKV 和 etcd 的耗时与 MySQL 接近\n- 对于大 IO（～4 MiB）压力，使用不同元数据引擎的总耗时未见明显差异（此时对象存储成为瓶颈）\n\n:::note 注意\n\n1. Redis 可以通过将 `appendfsync` 配置项由 `always` 改为 `everysec`，牺牲少量可靠性来换取一定的性能提升。更多信息可参见[这里](https://redis.io/docs/manual/persistence)。\n2. 测试中 Redis 和 MySQL 数据均仅在本地存储单副本，TiKV 和 etcd 数据会在三个节点间通过 Raft 协议存储三副本。\n\n:::\n\n以下提供了测试的具体细节。这些测试都运行在相同的对象存储（用来存放数据）、客户端和元数据节点上，只有元数据引擎不同。\n\n## 测试环境\n\n### JuiceFS 版本\n\n1.1.0-beta1+2023-06-08.5ef17ba0\n\n### 对象存储\n\nAmazon S3\n\n### 客户端节点\n\n- Amazon c5.xlarge：4 vCPUs，8 GiB 内存，最高 10 Gigabit 网络\n- Ubuntu 20.04.1 LTS\n\n### 元数据节点\n\n- Amazon c5d.xlarge：4 vCPUs，8 GiB 内存，最高 10 Gigabit 网络，100 GB SSD（为元数据引擎提供本地存储）\n- Ubuntu 20.04.1 LTS\n- SSD 数据盘被格式化为 ext4 文件系统并挂载到 `/data` 目录\n\n### 元数据引擎\n\n#### Redis\n\n- 版本：[7.0.9](https://download.redis.io/releases/redis-7.0.9.tar.gz)\n- 配置：\n - `appendonly`：`yes`\n - `appendfsync`：分别测试了 `always` 和 `everysec`\n - `dir`：`/data/redis`\n\n#### MySQL\n\n- 版本：8.0.25\n- `/var/lib/mysql` 目录被绑定挂载到 `/data/mysql`\n\n#### PostgreSQL\n\n- 版本：15.3\n- 数据目录被更改到 `/data/pgdata`\n\n#### TiKV\n\n- 版本：6.5.3\n- 配置：\n - `deploy_dir`：`/data/tikv-deploy`\n - `data_dir`：`/data/tikv-data`\n\n#### etcd\n\n- 版本：3.3.25\n- 配置：\n - `data-dir`：`/data/etcd`\n\n#### FoundationDB\n\n- 版本：6.3.23\n- 配置：\n - `data-dir`：`/data/fdb`\n\n## 测试工具\n\n每种元数据引擎都会运行以下所有测试。\n\n### Golang Benchmark\n\n在源码中提供了简单的元数据基准测试：[`pkg/meta/benchmarks_test.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/benchmarks_test.go)\n\n### JuiceFS Bench\n\nJuiceFS 提供了一个基础的性能测试命令：\n\n```bash\njuicefs bench /mnt/jfs -p 4\n```\n\n### mdtest\n\n- 版本：mdtest-3.3.0\n\n在 3 个客户端节点上并发执行测试：\n\n```bash\n$ cat myhost\nclient1 slots=4\nclient2 slots=4\nclient3 slots=4\n```\n\n测试命令：\n\nmeta only\n\n```shell\nmpirun --use-hwthread-cpus --allow-run-as-root -np 12 --hostfile myhost --map-by slot /root/mdtest -b 3 -z 1 -I 100 -u -d /mnt/jfs\n```\n\n12000 * 100KiB files\n\n```shell\nmpirun --use-hwthread-cpus --allow-run-as-root -np 12 --hostfile myhost --map-by slot /root/mdtest -F -w 102400 -I 1000 -z 0 -u -d /mnt/jfs\n```\n\n### fio\n\n- 版本：fio-3.28\n\n```bash\nfio --name=big-write --directory=/mnt/jfs --rw=write --refill_buffers --bs=4M --size=4G --numjobs=4 --end_fsync=1 --group_reporting\n```\n\n## 测试结果\n\n### Golang Benchmark\n\n- 展示了操作耗时（单位为 微秒/op），数值越小越好\n- 括号内数字是该指标对比 Redis-Always 的倍数（`always` 和 `everysec` 均是 Redis 配置项 `appendfsync` 的可选值）\n- 由于元数据缓存缘故，目前 `Read` 接口测试数据均小于 1 微秒，暂无对比意义\n\n | | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n |--------------|--------------|----------------|--------------|--------------|------------|--------------|--------------|\n | mkdir | 558 | 468 (0.8) | 2042 (3.7) | 1076 (1.9) | 1237 (2.2) | 1916 (3.4) | 1842 (3.3) |\n | mvdir | 693 | 621 (0.9) | 2693 (3.9) | 1459 (2.1) | 1414 (2.0) | 2486 (3.6) | 1895 (2.7) |\n | rmdir | 717 | 648 (0.9) | 3050 (4.3) | 1697 (2.4) | 1641 (2.3) | 2980 (4.2) | 2088 (2.9) |\n | readdir_10 | 280 | 288 (1.0) | 1350 (4.8) | 1098 (3.9) | 995 (3.6) | 1757 (6.3) | 1744 (6.2) |\n | readdir_1k | 1490 | 1547 (1.0) | 18779 (12.6) | 18414 (12.4) | 5834 (3.9) | 15809 (10.6) | 15276 (10.3) |\n | mknod | 562 | 464 (0.8) | 1547 (2.8) | 849 (1.5) | 1211 (2.2) | 1838 (3.3) | 1763 (3.1) |\n | create | 570 | 455 (0.8) | 1570 (2.8) | 844 (1.5) | 1209 (2.1) | 1849 (3.2) | 1761 (3.1) |\n | rename | 728 | 627 (0.9) | 2735 (3.8) | 1478 (2.0) | 1419 (1.9) | 2445 (3.4) | 1911 (2.6) |\n | unlink | 658 | 567 (0.9) | 2365 (3.6) | 1280 (1.9) | 1443 (2.2) | 2461 (3.7) | 1940 (2.9) |\n | lookup | 173 | 178 (1.0) | 557 (3.2) | 375 (2.2) | 608 (3.5) | 1054 (6.1) | 1029 (5.9) |\n | getattr | 87 | 86 (1.0) | 530 (6.1) | 350 (4.0) | 306 (3.5) | 536 (6.2) | 504 (5.8) |\n | setattr | 471 | 345 (0.7) | 1029 (2.2) | 571 (1.2) | 1001 (2.1) | 1279 (2.7) | 1596 (3.4) |\n | access | 87 | 89 (1.0) | 518 (6.0) | 356 (4.1) | 307 (3.5) | 534 (6.1) | 526 (6.0) |\n | setxattr | 393 | 262 (0.7) | 992 (2.5) | 534 (1.4) | 800 (2.0) | 717 (1.8) | 1300 (3.3) |\n | getxattr | 84 | 87 (1.0) | 494 (5.9) | 333 (4.0) | 303 (3.6) | 529 (6.3) | 511 (6.1) |\n | removexattr | 215 | 96 (0.4) | 697 (3.2) | 385 (1.8) | 1007 (4.7) | 1336 (6.2) | 1597 (7.4) |\n | listxattr_1 | 85 | 87 (1.0) | 516 (6.1) | 342 (4.0) | 303 (3.6) | 531 (6.2) | 515 (6.1) |\n | listxattr_10 | 87 | 91 (1.0) | 561 (6.4) | 383 (4.4) | 322 (3.7) | 565 (6.5) | 529 (6.1) |\n | link | 680 | 545 (0.8) | 2435 (3.6) | 1375 (2.0) | 1732 (2.5) | 3058 (4.5) | 2402 (3.5) |\n | symlink | 580 | 448 (0.8) | 1785 (3.1) | 954 (1.6) | 1224 (2.1) | 1897 (3.3) | 1764 (3.0) |\n | newchunk | 0 | 0 (0.0) | 1 (0.0) | 1 (0.0) | 1 (0.0) | 1 (0.0) | 2 (0.0) |\n | write | 553 | 369 (0.7) | 2352 (4.3) | 1183 (2.1) | 1573 (2.8) | 1788 (3.2) | 1747 (3.2) |\n | read_1 | 0 | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) |\n | read_10 | 0 | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) | 0 (0.0) |\n\n### JuiceFS Bench\n\n| | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n|------------------|------------------|------------------|-----------------|-----------------|-----------------|-----------------|-----------------|\n| Write big file | 730.84 MiB/s | 731.93 MiB/s | 729.00 MiB/s | 744.47 MiB/s | 730.01 MiB/s | 746.07 MiB/s | 744.70 MiB/s |\n| Read big file | 923.98 MiB/s | 892.99 MiB/s | 905.93 MiB/s | 895.88 MiB/s | 918.19 MiB/s | 939.63 MiB/s | 948.81 MiB/s |\n| Write small file | 95.20 files/s | 109.10 files/s | 82.30 files/s | 86.40 files/s | 101.20 files/s | 95.80 files/s | 94.60 files/s |\n| Read small file | 1242.80 files/s | 937.30 files/s | 752.40 files/s | 1857.90 files/s | 681.50 files/s | 1229.10 files/s | 1301.40 files/s |\n| Stat file | 12313.80 files/s | 11989.50 files/s | 3583.10 files/s | 7845.80 files/s | 4211.20 files/s | 2836.60 files/s | 3400.00 files/s |\n| FUSE operation | 0.41 ms/op | 0.40 ms/op | 0.46 ms/op | 0.44 ms/op | 0.41 ms/op | 0.41 ms/op | 0.44 ms/op |\n| Update meta | 2.45 ms/op | 1.76 ms/op | 2.46 ms/op | 1.78 ms/op | 3.76 ms/op | 3.40 ms/op | 2.87 ms/op |\n\n### mdtest\n\n 展示了操作速率（每秒 OPS 数），数值越大越好\n\n| | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n|--------------------|--------------|----------------|----------|------------|-----------|----------|--------------|\n| **EMPTY FILES** | | | | | | | |\n| Directory creation | 4901.342 | 9990.029 | 1252.421 | 4091.934 | 4041.304 | 1910.768 | 3065.578 |\n| Directory stat | 289992.466 | 379692.576 | 9359.278 | 69384.097 | 49465.223 | 6500.178 | 17746.670 |\n| Directory removal | 5131.614 | 10356.293 | 902.077 | 1254.890 | 3210.518 | 1450.842 | 2460.604 |\n| File creation | 5472.628 | 9984.824 | 1326.613 | 4726.582 | 4053.610 | 1801.956 | 2908.526 |\n| File stat | 288951.216 | 253218.558 | 9135.571 | 233148.252 | 50432.658 | 6276.787 | 14939.411 |\n| File read | 64560.148 | 60861.397 | 8445.953 | 20013.027 | 18411.280 | 9094.627 | 11087.931 |\n| File removal | 6084.791 | 12221.083 | 1073.063 | 3961.855 | 3742.269 | 1648.734 | 2214.311 |\n| Tree creation | 80.121 | 83.546 | 34.420 | 61.937 | 77.875 | 56.299 | 74.982 |\n| Tree removal | 218.535 | 95.599 | 42.330 | 44.696 | 114.414 | 76.002 | 64.036 |\n| **SMALL FILES** | | | | | | | |\n| File creation | 295.067 | 312.182 | 275.588 | 289.627 | 307.121 | 275.578 | 263.487 |\n| File stat | 54069.827 | 52800.108 | 8760.709 | 19841.728 | 14076.214 | 8214.318 | 10009.670 |\n| File read | 62341.568 | 57998.398 | 4639.571 | 19244.678 | 23376.733 | 5477.754 | 6533.787 |\n| File removal | 5615.018 | 11573.415 | 1061.600 | 3907.740 | 3411.663 | 1024.421 | 1750.613 |\n| Tree creation | 57.860 | 57.080 | 23.723 | 52.621 | 44.590 | 19.998 | 11.243 |\n| Tree removal | 96.756 | 65.279 | 23.227 | 19.511 | 27.616 | 17.868 | 10.571 |\n\n### fio\n\n| | Redis-Always | Redis-Everysec | MySQL | PostgreSQL | TiKV | etcd | FoundationDB |\n|-----------------|--------------|----------------|-----------|------------|-----------|-----------|--------------|\n| Write bandwidth | 729 MiB/s | 737 MiB/s | 736 MiB/s | 768 MiB/s | 731 MiB/s | 738 MiB/s | 745 MiB/s |\n" }, { "path": "docs/zh_cn/benchmark/performance_evaluation_guide.md", "content": "---\ntitle: 性能评估指南\nsidebar_position: 2\nslug: /performance_evaluation_guide\n---\n\n在进行性能测试之前，最好写下该使用场景的大致描述，包括：\n\n1. 对接的应用是什么？比如 Apache Spark、PyTorch 或者是自己写的程序等\n2. 应用运行的资源配置，包括 CPU、内存、网络，以及节点规模\n3. 预计的数据规模，包括文件数量和容量\n4. 文件的大小和访问模式（大文件或者小文件，顺序读写或者随机读写）\n5. 对性能的要求，比如每秒要写入或者读取的数据量、访问的 QPS 或者操作的延迟等\n\n以上这些内容越清晰、越详细，就越容易制定合适的测试计划，以及需要关注的性能指标，来判断应用对存储系统各方面的需求，包括 JuiceFS 元数据配置、网络带宽要求、配置参数等。当然，在一开始就清晰地写出上面所有的内容并不容易，有些内容可以在测试过程中逐渐明确，**但是在一次完整的测试结束时，以上使用场景描述以及相对应的测试方法、测试数据、测试结果都应该是完整的**。\n\n如果上面的内容还不明确，不要紧，JuiceFS 内置的测试工具可以一行命令得到单机基准性能的核心指标。同时本文还会介绍两个 JuiceFS 内置的性能分析工具，在做更复杂的测试时，这两个工具能帮你简单清晰的分析出 JuiceFS 性能表现背后的原因。\n\n## 性能测试快速上手\n\n以下示例介绍 JuiceFS 内置的 bench 工具的基本用法。\n\n### 环境配置\n\n- 测试主机：Amazon EC2 c5.xlarge 一台\n- 操作系统：Ubuntu 20.04.1 LTS (Kernel `5.4.0-1029-aws`)\n- 元数据引擎：Redis 6.2.3, 存储（dir）配置在系统盘\n- 对象存储：Amazon S3\n- JuiceFS version：0.17-dev (2021-09-23 2ec2badf)\n\n### 注意事项\n\nJuiceFS v1.0+ 默认启用了回收站，基准测试会在文件系统中创建和删除临时文件，这些文件最终会被转存到回收站 `.trash` 占用存储空间，为了避免这种情况，可以在基准测试之前关闭回收站 `juicefs config META-URL --trash-days 0`，详情参考[回收站](../security/trash.md)。\n\n### `juicefs bench`\n\n[`juicefs bench`](../reference/command_reference.mdx#bench) 命令可以帮助你快速完成单机性能测试，通过测试结果判断环境配置和性能表现是否正常。假设你已经把 JuiceFS 挂载到了测试机器的 `/mnt/jfs` 位置（如果在 JuiceFS 初始化、挂载方面需要帮助，请参考[创建文件系统](../getting-started/standalone.md#juicefs-format)），执行以下命令即可（推荐 `-p` 参数设置为测试机器的 CPU 核数）：\n\n```bash\njuicefs bench /mnt/jfs -p 4\n```\n\n测试结果以表格形式呈现，其中 `ITEM` 代表测试的项目，`VALUE` 代表每秒的处理能力（吞吐量、文件数、操作数等），`COST` 代表每个文件或操作所需的时间。\n\n各项性能指标会显示为绿色、黄色或红色区分性能表现。若您的结果中有红色指标，请先检查相关配置，需要帮助可以在 [GitHub Discussions](https://github.com/juicedata/juicefs/discussions) 详细描述你的问题。\n\n![bench](../images/bench-guide-bench.png)\n\n`juicefs bench` 基准性能测试的具体流程如下（它的实现逻辑非常简单，有兴趣了解细节的可以直接看[源码](https://github.com/juicedata/juicefs/blob/main/cmd/bench.go))：\n\n1. N 并发各写 1 个 1 GiB 的大文件，IO 大小为 1 MiB\n2. N 并发各读 1 个之前写的 1 GiB 的大文件，IO 大小为 1 MiB\n3. N 并发各写 100 个 128 KiB 的小文件，IO 大小为 128 KiB\n4. N 并发各读 100 个之前写的 128 KiB 的小文件，IO 大小为 128 KiB\n5. N 并发各 stat 100 个之前写的 128 KiB 的小文件\n6. 清理测试用的临时目录\n\n并发数 N 的值即由 `bench` 命令中的 `-p` 参数指定。\n\n在这用 AWS 提供的几种常用存储类型做个性能比较：\n\n- EFS 1TiB 容量时，读 150MiB/s，写 50MiB/s，价格是 $0.08/GB-month\n- EBS st1 是吞吐优化型 HDD，最大吞吐 500MiB/s，最大 IOPS（1MiB I/O）500，最大容量 16TiB，价格是 $0.045/GB-month\n- EBS gp2 是通用型 SSD，最大吞吐 250MiB/s，最大 IOPS（16KiB I/O）16000，最大容量 16TiB，价格是 $0.10/GB-month\n\n不难看出，在上面的测试中，JuiceFS 的顺序读写能力明显优于 AWS EFS，吞吐能力也超过了常用的 EBS。但是写小文件的速度不算快，因为每写一个文件都需要将数据持久化到 S3 中，调用对象存储 API 通常有 10~30ms 的固定开销。\n\n:::note 注\nAmazon EFS 的性能与容量线性相关（[参考官方文档](https://docs.aws.amazon.com/efs/latest/ug/performance.html#performancemodes)），这样就不适合用在小数据量高吞吐的场景中。\n:::\n\n:::note 注\n价格参考 [AWS 美东区（US East, Ohio Region）](https://aws.amazon.com/ebs/pricing/?nc1=h_ls)，不同 Region 的价格有细微差异。\n:::\n\n:::note 注\n以上数据来自 [AWS 官方文档](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html)，性能指标为最大值，EBS 的实际性能与卷容量和挂载 EC2 实例类型相关，总的来说是越大容量，搭配约高配置的 EC2，得到的 EBS 性能越好，但不超过上面提到的最大值。\n:::\n\n### `juicefs objbench`\n\n[`juicefs objbench`](../reference/command_reference.mdx#objbench) 命令可以运行一些关于对象存储的测试，用以评估其作为 JuiceFS 的后端存储时的运行情况。以测试 Amazon S3 为例：\n\n```bash\njuicefs objbench \\\n --storage s3 \\\n --access-key myAccessKey \\\n --secret-key mySecretKey \\\n https://mybucket.s3.us-east-2.amazonaws.com\n```\n\n测试结果如下图所示：\n\n![JuiceFS Bench](../images/objbench.png)\n\n其中，结果显示为 `not support` 代表所测试的对象存储不支持该项功能。\n\n#### 测试流程\n\n首先会对对象存储的接口进行功能测试，以下为测试用例：\n\n1. 创建 bucket\n2. 上传对象\n3. 下载对象\n4. 下载不存在的对象\n5. 获取对象部分内容\n6. 获取对象元信息\n7. 删除对象\n8. 删除不存在对象\n9. 列举对象\n10. 上传大对象\n11. 上传空对象\n12. 分块上传\n13. 更改文件拥有者与所属组（需要 `root` 权限运行）\n14. 更改文件权限\n15. 更改文件的 mtime（最后修改时间）\n\n然后进行性能测试：\n\n1. 将 `--small-objects` 个 `--small-object-size` 大小的对象，以 `--threads` 个并发上传\n2. 下载步骤 1 中上传的对象并检查内容\n3. 将 `--big-object-size` 大小的对象按照 `--block-size` 的大小拆分后以 `--threads` 并发度上传\n4. 下载步骤 3 中上传的对象并检查内容，然后清理步骤 3 上传到对象存储的所有对象\n5. 以 `--threads` 并发度列举对象存储中所有的对象 100 次\n6. 以 `--threads` 并发度获取步骤 1 中上传的所有对象的元信息\n7. 以 `--threads` 并发度更改步骤 1 中上传的所有对象的 mtime（最后修改时间）\n8. 以 `--threads` 并发度更改步骤 1 中上传的所有对象的权限\n9. 以 `--threads` 并发度更改步骤 1 中上传的所有对象的拥有者与所属组（需要 `root` 权限运行）\n10. 以 `--threads` 并发度删除步骤 1 中上传的所有对象\n\n最后清理测试的文件。\n\n## 性能观测和分析工具\n\n接下来介绍两个性能观测和分析工具，是 JuiceFS 测试、使用、调优过程中必备的利器。\n\n### `juicefs stats`\n\n[`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) 命令是一个实时统计 JuiceFS 性能指标的工具，类似 Linux 系统的 `dstat` 命令，可以实时显示 JuiceFS 客户端的指标变化。执行 `juicefs bench` 时，在另一个会话中执行以下命令：\n\n```bash\njuicefs stats /mnt/jfs --verbosity 1\n```\n\n结果如下，可以将其与上述基准测试流程对照来看，更易理解：\n\n![bench-guide-stats](../images/bench-guide-stats.png)\n\n其中各项指标具体含义参考 [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats)。\n\n### `juicefs profile`\n\n[`juicefs profile`](../administration/fault_diagnosis_and_analysis.md#profile) 命令可以基于[访问日志](../administration/fault_diagnosis_and_analysis.md#access-log)进行性能数据统计，来直观了解 JuiceFS 的运行情况。执行 `juicefs bench` 时，在另一个会话中执行以下命令：\n\n```bash\ncat /mnt/jfs/.accesslog > juicefs.accesslog\n```\n\n其中 `.accesslog` 是一个虚拟文件，它平时不会产生任何数据，只有在读取（如执行 `cat`）时才会有 JuiceFS 的访问日志输出。结束后使用 Ctrl + C 结束 `cat` 命令，并运行：\n\n```bash\njuicefs profile juicefs.accesslog --interval 0\n```\n\n其中 `--interval` 参数设置访问日志的采样间隔，设为 0 时用于快速重放一个指定的日志文件，生成统计信息，如下图所示：\n\n![bench-guide-profile](../images/bench-guide-profile.png)\n\n从之前基准测试流程描述可知，本次测试过程一共创建了 `(1 + 100) * 4 = 404` 个文件，每个文件都经历了「创建 → 写入 → 关闭 → 打开 → 读取 → 关闭 → 删除」的过程，因此一共有：\n\n- 404 次 `create`，`open` 和 `unlink` 请求\n- 808 次 `flush` 请求：每当文件关闭时会自动调用一次 `flush`\n- 33168 次 `write`/`read` 请求：每个大文件写入了 1024 个 1 MiB IO，而在 FUSE 层请求的默认最大值为 128 KiB，也就是说每个应用 IO 会被拆分成 8 个 FUSE 请求，因此一共有 `(1024 * 8 + 100) * 4 = 33168` 个请求。读 IO 与之类似，计数也相同。\n\n以上这些值均能与 `profile` 的结果完全对应上。另外，结果中还显示 `write` 的平均时延非常小（45 微秒），而主要耗时点在 `flush`。这是因为 JuiceFS 的 `write` 默认先写入内存缓冲区，在文件关闭时再调用 `flush` 上传数据到对象存储，与预期吻合。\n\n## 其他测试工具配置示例\n\n:::tip 提示\nJuiceFS v1.0+ 默认启用了回收站，基准测试会在文件系统中创建和删除临时文件，这些文件最终会被转存到回收站 `.trash` 占用存储空间，为了避免这种情况，可以在基准测试之前关闭回收站 `juicefs config META-URL --trash-days 0`，详情参考[回收站](../security/trash.md)。\n:::\n\n### Fio 单机性能测试\n\nFio 是业界常用的一个性能测试工具，完成 JuiceFS bench 后可以用它来做更复杂的性能测试。\n\n#### 环境配置\n\n与 [JuiceFS Bench](#环境配置) 测试环境一致。\n\n#### 测试任务\n\n执行下面四个 Fio 任务，分别进行顺序写、顺序读、随机写、随机读测试。\n\n顺序写\n\n```shell\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=write --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\n顺序读\n\n```bash\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=read --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\n随机写\n\n```shell\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=randwrite --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\n随机读\n\n```shell\nfio --name=jfs-test --directory=/mnt/jfs --ioengine=libaio --rw=randread --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting\n```\n\n参数说明：\n\n- `--name`：用户指定的测试名称，会影响测试文件名\n- `--directory`：测试目录\n- `--ioengine`：测试时下发 IO 的方式；通常用 libaio 即可\n- `--rw`：常用的有 read，write，randread，randwrite，分别代表顺序读写和随机读写\n- `--bs`：每次 IO 的大小\n- `--size`：每个线程的 IO 总大小；通常就等于测试文件的大小\n- `--numjobs`：测试并发线程数；默认每个线程单独跑一个测试文件\n- `--direct`：在打开文件时添加 `O_DIRECT` 标记位，不使用系统缓冲，可以使测试结果更稳定准确\n\n结果如下：\n\n```bash\n# Sequential\nWRITE: bw=703MiB/s (737MB/s), 703MiB/s-703MiB/s (737MB/s-737MB/s), io=4096MiB (4295MB), run=5825-5825msec\nREAD: bw=817MiB/s (856MB/s), 817MiB/s-817MiB/s (856MB/s-856MB/s), io=4096MiB (4295MB), run=5015-5015msec\n\n# Random\nWRITE: bw=285MiB/s (298MB/s), 285MiB/s-285MiB/s (298MB/s-298MB/s), io=4096MiB (4295MB), run=14395-14395msec\nREAD: bw=93.6MiB/s (98.1MB/s), 93.6MiB/s-93.6MiB/s (98.1MB/s-98.1MB/s), io=4096MiB (4295MB), run=43773-43773msec\n```\n\n### Vdbench 多机性能测试\n\nVdbench 也是业界常见的文件系统评测工具，且很好地支持了多机并发测试。\n\n#### 测试环境\n\n与 [JuiceFS Bench](#环境配置) 测试环境类似，只是多开了两台同配置主机，一共三台。\n\n#### 准备工作\n\n需要在每个节点相同路径下安装 vdbench：\n\n1. [官网](https://www.oracle.com/downloads/server-storage/vdbench-downloads.html)下载 50406 版本\n2. 安装 Java：`apt-get install openjdk-8-jre`\n3. 测试 vdbench 安装成功：`./vdbench -t`\n\n然后，假设三个节点名称分别为 node0，node1 和 node2，则需在 node0 上创建配置文件，如下（测试大量小文件读写）：\n\n```bash\n$ cat jfs-test\nhd=default,vdbench=/root/vdbench50406,user=root\nhd=h0,system=node0\nhd=h1,system=node1\nhd=h2,system=node2\n\nfsd=fsd1,anchor=/mnt/jfs/vdbench,depth=1,width=100,files=3000,size=128k,shared=yes\n\nfwd=default,fsd=fsd1,operation=read,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd1,host=h0\nfwd=fwd2,host=h1\nfwd=fwd3,host=h2\n\nrd=rd1,fwd=fwd*,fwdrate=max,format=yes,elapsed=300,interval=1\n```\n\n参数说明：\n\n- `vdbench=/root/vdbench50406`：指定了 vdbench 工具的安装路径\n- `anchor=/mnt/jfs/vdbench`：指定了每个节点上运行测试任务的路径\n- `depth=1,width=100,files=3000,size=128k`：定义了测试任务文件树结构，即测试目录下再创建 100 个目录，每个目录内包含 3000 个 128 KiB 大小的文件，一共 30 万个文件\n- `operation=read,xfersize=128k,fileio=random,fileselect=random`：定义了实际的测试任务，即随机选择文件下发 128 KiB 大小的读请求\n\n结果如下：\n\n```\nFILE_CREATES Files created: 300,000 498/sec\nREAD_OPENS Files opened for read activity: 188,317 627/sec\n```\n\n系统整体创建 128 KiB 文件速度为每秒 498 个，读取文件速度为每秒 627 个。\n\n#### 其他参考示例\n\n以下是一些本地简单评估文件系统性能时可用的配置文件，以供参考；具体测试集规模和并发数可根据实际情况调整。\n\n##### 顺序读写大文件\n\n文件大小均为 1GiB，其中 `fwd1` 是顺序写大文件，`fwd2` 是顺序读大文件。\n\n```bash\n$ cat local-big\nfsd=fsd1,anchor=/mnt/jfs/local-big,depth=1,width=1,files=4,size=1g,openflags=o_direct\n\nfwd=fwd1,fsd=fsd1,operation=write,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4\nfwd=fwd2,fsd=fsd1,operation=read,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4\n\nrd=rd1,fwd=fwd1,fwdrate=max,format=restart,elapsed=120,interval=1\nrd=rd2,fwd=fwd2,fwdrate=max,format=restart,elapsed=120,interval=1\n```\n\n##### 随机读写小文件\n\n文件大小均为 128KiB，其中 `fwd1` 是随机写小文件，`fwd2` 是随机读小文件，`fwd3` 是混合读写小文件（读写比 = 7:3）。\n\n```bash\n$ cat local-small\nfsd=fsd1,anchor=/mnt/jfs/local-small,depth=1,width=20,files=2000,size=128k,openflags=o_direct\n\nfwd=fwd1,fsd=fsd1,operation=write,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd2,fsd=fsd1,operation=read,xfersize=128k,fileio=random,fileselect=random,threads=4\nfwd=fwd3,fsd=fsd1,rdpct=70,xfersize=128k,fileio=random,fileselect=random,threads=4\n\nrd=rd1,fwd=fwd1,fwdrate=max,format=restart,elapsed=120,interval=1\nrd=rd2,fwd=fwd2,fwdrate=max,format=restart,elapsed=120,interval=1\nrd=rd3,fwd=fwd3,fwdrate=max,format=restart,elapsed=120,interval=1\n```\n" }, { "path": "docs/zh_cn/community/_roadmap.md", "content": "---\ntitle: 路线图\nsidebar_position: 3\n---\n" }, { "path": "docs/zh_cn/community/adopters.md", "content": "---\ntitle: 使用者\nsidebar_position: 1\nslug: /adopters\n---\n\n| 公司／团队 | 行业 & 使用场景 | 案例 |\n|-----------|------------------|-----------------|\n| [之江实验室](https://www.zhejianglab.com) | 研究机构，AI | [之江实验室：如何基于 JuiceFS 为超异构算力集群构建存储层](https://juicefs.com/zh-cn/blog/user-stories/high-performance-scale-out-heterogeneous-computing-power-cluster-storage) |\n| [人民大学](http://www.ruc.edu.cn) | 大学，AI | [从 HPC 到 AI：探索文件系统的发展及性能评估](https://juicefs.com/zh-cn/blog/user-stories/hpc-ai-file-systems-performance-development) |\n| [中山大学](https://www.sysu.edu.cn) | 大学，AI | [基于 JuiceFS 构建高校 AI 存储方案：高并发、系统稳定、运维简单](https://juicefs.com/zh-cn/blog/user-stories/juicefs-vs-nfs-ai-storage) |\n| [电子科技大学](https://www.uestc.edu.cn) | 大学，AI | |\n| [Character.AI](https://character.ai) | GenerativeAI | |\n| [BentoML](https://bentoml.com) | GenerativeAI | [BentoML：如何使用 JuiceFS 加速大模型加载](https://juicefs.com/zh-cn/blog/user-stories/bentoml-use-juicefs-accelerate-large-model-loading) |\n| [NAVER](https://www.naver.com) | 互联网服务，AI | [韩国国民搜索 NAVER：为 AI 平台引入存储方案 JuiceFS](https://juicefs.com/zh-cn/blog/user-stories/naver-storage-solution-juicefs-ai-platforms) |\n| [云知声](https://www.unisound.com) | AI | [AI 场景存储优化：云知声超算平台基于 JuiceFS 的存储实践](https://juicefs.com/zh-cn/blog/juicefs-support-ai-storage-at-unisound) |\n| [地平线](https://horizon.ai) | 汽车，AI | |\n| [卓驭科技](https://www.zyt.com/zh) | 汽车，AI | |\n| [理想汽车](https://www.lixiang.com) | 汽车，大数据，AI | [JuiceFS 在理想汽车的使用和展望](https://juicefs.com/zh-cn/blog/li-auto-with-juicefs)，
[理想汽车：从 Hadoop 到云原生的演进与思考](https://juicefs.com/zh-cn/blog/liauto-case-hadoop-cloudnatrive) |\n| [蔚来汽车](https://www.nio.cn) | 汽车，AI | |\n| [上汽集团](https://www.saicmotor.com/chinese) | 汽车，AI | [上汽云 x JuiceFS：iGear 用了这个小魔法，模型训练速度提升 300%](https://juicefs.com/zh-cn/blog/performance-boost-3x-on-igear-platform) |\n| [Plus.AI](https://plus.ai) | 汽车，AI | |\n| [五菱汽车](https://wuling.com) | 汽车，大数据 | |\n| [驭势科技](https://www.uisee.com) | 汽车，AI | |\n| [长安汽车梧桐车联](https://www.auto-pai.com) | AI，大数据 | |\n| [小米](https://www.mi.com) | 消费电子，AI | [小米云原生文件存储平台化实践：支撑 AI 训练、大模型、容器平台多项业务](https://juicefs.com/zh-cn/blog/user-stories/cloud-native-file-storage-platform-as-ai-training-large-models-container-platforms) |\n| [vivo](https://www.vivo.com) | AI | [vivo AI 计算平台的轩辕文件存储实践](https://www.infoq.cn/article/3oFSOWfYGsX5h7xzsIe6) |\n| [DJI 大疆创新](https://www.dji.com/cn) | 消费电子，AI | |\n| [安克创新](https://cn.anker-in.com) | 消费电子，AI | |\n| [OPPO](https://www.oppo.com) | 消费电子，共享文件存储 | |\n| [顺丰速运](https://www.sf-express.com) | 物流，AI，共享文件存储 | |\n| [思谋 SmartMore](https://cn.smartmore.com) | AI | [思谋科技：构建易于运维的 AI 训练平台](https://juicefs.com/zh-cn/blog/user-stories/easy-operate-ai-training-platform-storage-selection) |\n| [旷视](https://megvii.com) | AI | |\n| [商汤科技](https://www.sensetime.com/cn) | AI | |\n| [云从科技](https://www.cloudwalk.com) | AI | |\n| [思必驰](https://www.aispeech.com) | AI | |\n| [Clobotics](https://clobotics.com) | 机器人，AI | [Clobotics 计算机视觉场景存储实践：多云架构、POSIX 全兼容、低运维的统一存储](https://juicefs.com/zh-cn/blog/user-stories/clobotics-posix-multi-cloud-storage) |\n| [刻行 coSence](https://www.coscene.io) | 机器人，AI | [机器人行业数据闭环实践：从对象存储到 JuiceFS](https://juicefs.com/zh-cn/blog/user-stories/data-object-storag--to-juicefs) |\n| [海柔创新](https://www.hairobotics.cn) | 机器人，AI | [海柔仿真系统存储实践：混合云架构下实现高可用与极简运维](https://juicefs.com/zh-cn/blog/user-stories/multi-cloud-storage-high-availability) |\n| [蝉妈妈数据](https://www.chanmama.com) | AI | |\n| [酷家乐](https://www.kujiale.com) | AI | |\n| [TP-LINK](https://www.tp-link.com) | AI | |\n| [Fal](https://fal.ai) | GenerativeAI | |\n| [Lepton AI](https://www.lepton.ai) | GenerativeAI | [加速 AI 训推：Lepton AI 如何构建多租户、低延迟云存储平台](https://juicefs.com/zh-cn/blog/user-stories/lepton-ai-build-multi-tenant-low-latency-cloud-storage-platform) |\n| [Graviti Diffus](https://www.diffus.graviti.com) | GenerativeAI | |\n| [建信金融科技](https://www.ccbft.com) | 金融科技，AI | |\n| [平安银行](https://pingan.com) | 金融科技，大数据 | |\n| [同盾](https://tongdun.cn) | 金融科技，大数据 | |\n| [尧信](https://www.yaoxinhd.com) | 金融科技，大数据，共享文件存储 | |\n| [米筐](https://www.ricequant.com) | 金融科技，AI | |\n| [移动云](https://ecloud.he.chinamobile.com) | AI，大数据 | [移动云使用 JuiceFS 支持 Apache HBase 增效降本的探索](https://juicefs.com/zh-cn/blog/juicefs-support-hbase-at-chinamobile-cloud) |\n| [中国电信](http://www.chinatelecom.com.cn) | 大数据 | [存算分离实践：JuiceFS 在中国电信日均 PB 级数据场景的应用](https://juicefs.com/zh-cn/blog/user-stories/applicatio-of-juicefs-in-china-telecoms-daily-average-pb-data-scenario) |\n| [火山引擎](https://www.volcengine.com) | 共享文件存储，特效渲染 | [JuiceFS 在火山引擎边缘计算的应用实践](https://juicefs.com/zh-cn/blog/user-stories/how-juicefs-accelerates-edge-rendering-performance-in-volcengine) |\n| [金山云](https://www.ksyun.com) | AI，大数据 | [金山云：基于 JuiceFS 的 Elasticsearch 温冷热数据管理实践](https://juicefs.com/zh-cn/blog/user-stories/juicefs-elasticsearch-cold-heat-data-management) |\n| [腾讯](https://www.tencent.com) | 互联网服务，AI | |\n| [百度](https://home.baidu.com/home/index) | 互联网服务，大数据 | |\n| [知乎](https://www.zhihu.com) | 互联网服务，大数据 | [知乎 x JuiceFS：利用 JuiceFS 给 Flink 容器启动加速](https://juicefs.com/zh-cn/blog/zhihu-flink-with-juicefs)，
[利用 JuiceFS 动态注入 Protobuf JAR 包](https://zhuanlan.zhihu.com/p/586120009)，
[知乎：多云架构下大模型训练，如何保障存储稳定性](https://juicefs.com/zh-cn/blog/user-stories/data-storage-multi-cloud-zhihu-model-training-juicefs) |\n| [好未来](https://www.100tal.com) | 互联网服务，AI | [好未来：多云环境下基于 JuiceFS 建设低运维模型仓库](https://juicefs.com/zh-cn/blog/user-stories/multi-cloud-storage-juicefs-model-stroage) |\n| [Shopee](https://shopee.com) | 电商，大数据 | [Shopee x JuiceFS：ClickHouse 冷热数据分离存储架构与实践](https://juicefs.com/zh-cn/blog/shopee-clickhouse-with-juicefs) |\n| [京东](https://jd.com) | 电商，大数据 | |\n| [Grab](https://grab.com/sg) | 出行服务，大数据 | |\n| [深势科技](https://www.dp.tech) | 生物科技，AI | [深势科技分享 AI 企业多云存储架构实践](https://juicefs.com/zh-cn/blog/dptech-ai-storage-in-multi-cloud-practice) |\n| [MemVerge](https://memverge.com) | 生物科技，共享文件存储 | [MemVerge：小文件写入性能 5 倍于 S3FS，JuiceFS 加速生信研究](https://juicefs.com/zh-cn/blog/user-stories/memverge-s3fs-juicefs) |\n| [百图生科](https://www.biotu.com) | 生物科技，共享文件存储 | |\n| [MDI 生物实验室](https://mdibl.org) | 生物科技，高性能文件存储 | |\n| [劳伦斯伯克利实验室](https://www.lbl.gov) | 生物科技，高性能文件存储 | |\n| [美国自然历史博物馆](https://www.amnh.org) | 非盈利组织，高性能文件存储 | |\n| [阿拉贡国家实验室](https://www.anl.gov) | 非盈利组织，高性能文件存储 | |\n| [溯源精微](https://www.geneway.cn) | 生物科技，共享文件存储 | |\n| [国家超级计算济南中心](https://www.nsccjn.cn) | 超算，DevOps | |\n| [网易游戏](https://game.163.com) | 游戏，大数据，AI | [网易互娱出海之旅：大数据平台上云架构设计与实践](https://juicefs.com/zh-cn/blog/user-stories/hadoop-compatible-storage-big-data-cloud-platform-s3) |\n| [米哈游](https://www.mihoyo.com) | 游戏，共享文件存储 | |\n| [嘉谊互娱](http://www.joyient.com) | 游戏，共享文件存储，特效渲染 | |\n| [一面数据](https://www.yimian.com.cn) | 大数据 | [一面数据：Hadoop 迁移云上架构设计与实践](https://juicefs.com/zh-cn/blog/yimiancase) |\n| [携程旅行](https://www.ctrip.com) | 互联网服务，大数据，AI | [突破存储数据量限制，JuiceFS 在携程海量冷数据场景下的实践](https://juicefs.com/zh-cn/blog/xiecheng-case)，[稳定且高性价比的大模型存储：携程 10PB 级 JuiceFS 工程实践](https://juicefs.com/zh-cn/blog/user-stories/trip-10pb-level-llm-stroage-juicefs-practice) |\n| [同程旅行](https://ly.com) | 互联网服务，共享文件存储 | [从 CephFS 到 JuiceFS：同程旅行亿级文件存储平台构建之路](https://juicefs.com/zh-cn/blog/user-stories/cephfs-vs-juicefs-draco-travel-file-storage) |\n| [贝壳](https://ke.com) | 互联网服务，AI | [贝壳找房：为 AI 平台打造混合多云的存储加速底座](https://juicefs.com/zh-cn/blog/user-stories/beike-ai-platform-multi-cloud-storage) |\n| [Jerry](https://getjerry.com) | 互联网服务，大数据 | [北美科技企业 Jerry：基于 JuiceFS 构建 ClickHouse 主从架构](https://juicefs.com/zh-cn/blog/user-stories/jerry-clickhouse-read-write-separation-juicefs-primary-replica-architecture) |\n| [航天宏图](https://www.piesat.cn) | 遥感，共享文件存储 | |\n| [天桐互动](https://www.kuaidianyuedu.com) | AI，共享文件存储 | |\n| [视源股份](http://www.cvte.com) | 共享文件存储 | |\n| [多点 DMALL](https://www.dmall.com) | 零售，大数据 | [多点 DMALL：大数据存算分离下的存储架构探索与实践](https://juicefs.com/zh-cn/blog/user-stories/separation-of-storage--computing-building-cloud-native-big-data-platform) |\n| [百胜中国](https://www.yumchina.com) | 零售，AI | |\n| [酷数智能](http://www.kurudata.com) | 共享文件存储 | |\n| [朗新集团](https://www.longshine.com) | 大数据，共享文件存储 | |\n| [网易邮箱](https://mail.163.com) | 互联网服务，大数据 | |\n| [南昌维网科技](https://www.vwell.cn) | 共享数据存储 | |\n| [声网](https://www.agora.io/cn) | 大数据 | |\n| [南京鹏云网络](https://www.pengyunnetwork.cn) | 共享文件存储 | |\n| [聚云位智 LinkoopDB](http://www.datapps.cn) | 大数据 | |\n| [国家天文科学数据中心](https://nadc.china-vo.org) | 共享文件存储 | |\n| [艾莎医学](https://www.ashermed.com) | 共享文件存储 | |\n| [NodeReal](https://nodereal.io) | 共享文件存储 | |\n| [不鸣科技](https://www.boomingtech.com) | 共享文件存储 | |\n| [博依特科技](https://www.poi-t.com) | 大数据 | |\n| [九曳供应链](https://www.jiuyescm.com) | 大数据 | |\n| [摩登天空](https://www.modernsky.com/home) | 共享文件存储 | |\n| [酷狗音乐](https://www.kugou.com) | 共享文件存储 | |\n| [东方财富](https://www.eastmoney.com) | 共享文件存储 | |\n| [Texas A&M 大学](https://www.tamu.edu) | 大学 | |\n| [Simon Fraser 大学](https://www.sfu.ca) | 大学 | |\n| [堪培拉大学](https://www.canberra.edu.au) | 大学 | |\n\n欢迎你在使用 JuiceFS 后，向大家分享你的使用经验，可以直接向这个列表提交 Pull Request，或者联系我们 [`hello@juicedata.io`](mailto:hello@juicedata.io)。\n" }, { "path": "docs/zh_cn/community/articles.md", "content": "---\ntitle: JuiceFS 文章合集\nsidebar_position: 2\nslug: /articles\n---\n\nJuiceFS 广泛适用于各种数据存储和共享场景，本页汇总来自世界各地用户使用 JuiceFS 的实践和相关技术文章，欢迎大家共同维护这个列表。\n\n## AI\n\n- [韩国国民搜索 NAVER：使用 JuiceFS 打通 Hadoop 与 Kubernetes 存储实践](https://juicefs.com/zh-cn/blog/user-stories/naver-juicefs-hadoop-kubernetes-storage)，2026-02-12，Nam Kyung-wan@NAVER\n- [海量小文件 + 多云协同：地瓜机器人 JuiceFS 存储优化之路](https://juicefs.com/zh-cn/blog/user-stories/horizon-robotics-juicefs-small-file-multi-cloud-optimization)，2026-02-06，赵晗@地瓜机器人\n- [3D-AIGC 存储架构演进：从 NFS、GlusterFS 到 JuiceFS](https://juicefs.com/zh-cn/blog/user-stories/3d-aigc-storage-evolution-juicefs)，2026-01-05，李威宇@光影焕像\n- [JuiceFS + MinIO：Ariste AI 量化投资高性能存储实践](https://juicefs.com/zh-cn/blog/user-stories/juicefs-minio-ariste-ai-quant-storage)，2025-12-08，高玉堂@Ariste AI\n- [NAS、对象存储与 JuiceFS：百亿量化基金的存储选型实践](https://juicefs.com/zh-cn/blog/solutions/quant-fund-storage-selection-nas-object-juicefs)，2025-11-20，蔡敏\n- [基于 JuiceFS 构建 AI 推理：多模态复杂 I/O、跨云与多租户支持](https://juicefs.com/zh-cn/blog/solutions/juicefs-ai-inference-multi-modal-cross-cloud-multi-tenant)，2025-10-17，李少杰\n- [九识智能：基于 JuiceFS 的自动驾驶多云亿级文件存储](https://juicefs.com/zh-cn/blog/user-stories/intsig-juicefs-autonomous-driving-multi-cloud-storage)，2025-09-24，邓君宇@九识智能\n- [稿定科技：多云架构下的 AI 存储挑战与 JuiceFS 实践](https://juicefs.com/zh-cn/blog/user-stories/gaoding-ai-storage-challenges-multi-cloud-juicefs)，2025-08-08，可加@稿定科技\n- [从资源闲置到弹性高吞吐，JuiceFS 如何构建 70GB/s 吞吐的缓存池？](https://juicefs.com/zh-cn/blog/solutions/building-high-throughput-cache-pool-resilience-with-juicefs)，2025-07-25，蔡敏\n- [多模态“卷王”阶跃星辰：如何利用 JuiceFS 打造高效经济的大模型存储平台](https://juicefs.com/zh-cn/blog/user-stories/stepfun-ai-use-juicefs-create-multimodal-learning-storage-platform)，2025-07-23，缪昌新@阶跃星辰\n- [合合信息：基于 JuiceFS 构建统一存储，支撑 PB 级 AI 训练](https://juicefs.com/zh-cn/blog/user-stories/intsig-use-juicefs-build-unified-storage-support-pb-ai-training)，2025-07-17，唐义凡@合合信息\n- [中国科学院计算所：从 NFS 到 JuiceFS，大模型训推平台存储演进之路](https://juicefs.com/zh-cn/blog/user-stories/nfs-vs-juicefs-llm-storage)，2025-05-14，孙玮\n- [百图生科：基于 JuiceFS 构建生命科学大模型存储平台，成本降 90%](https://juicefs.com/zh-cn/blog/user-stories/biomap-juicefs-building-llm-storage)，2025-05-07，郑泽东@百图生科\n- [稳定且高性价比的大模型存储：携程 10PB 级 JuiceFS 工程实践](https://juicefs.com/zh-cn/blog/user-stories/trip-10pb-level-llm-stroage-juicefs-practice)，2025-03-10，吴松林@携程\n- [加速 AI 训推：Lepton AI 如何构建多租户、低延迟云存储平台](https://juicefs.com/zh-cn/blog/user-stories/lepton-ai-build-multi-tenant-low-latency-cloud-storage-platform)，2025-01-17，丁聪@Lepton AI\n- [多云架构，JuiceFS 如何实现一致性与低延迟的数据分发？](https://juicefs.com/zh-cn/blog/solutions/juicefs-multi-cloud-consistency-low-latency)，2025-01-10，蔡敏\n- [从 CephFS 到 JuiceFS：同程旅行亿级文件存储平台构建之路](https://juicefs.com/zh-cn/blog/user-stories/cephfs-vs-juicefs-draco-travel-file-storage)，2024-12-13，位传海@同程旅行\n- [vivo 轩辕文件系统：AI 计算平台存储性能优化实践](https://juicefs.com/zh-cn/blog/user-stories/vivo-ai)，2024-10-25，于相洋@vivo\n- [大模型存储选型 & JuiceFS 在关键环节性能详解](https://juicefs.com/zh-cn/blog/solutions/large-model-storage-performance-juicefs)，2024-10-09，李少杰\n- [MiniMax：如何基于 JuiceFS 构建高性能、低成本的大模型 AI 平台？](https://juicefs.com/zh-cn/blog/user-stories/minimax-juicefs-ai)，2024-08-30\n- [JuiceFS 在多云架构中加速大模型推理](https://juicefs.com/zh-cn/blog/solutions/data-storage-multi-cloud-model-training-juicefs)，2024-08-23，高昌健\n- [基于 JuiceFS 构建高校 AI 存储方案：高并发、系统稳定、运维简单](https://juicefs.com/zh-cn/blog/user-stories/juicefs-vs-nfs-ai-storage)，2024-06-26，徐国昊@中山大学\n- [贝壳找房：为 AI 平台打造混合多云的存储加速底座](https://juicefs.com/zh-cn/blog/user-stories/beike-ai-platform-multi-cloud-storage)，2024-06-12，王天庆@贝壳找房\n- [北美科技企业 Jerry：基于 JuiceFS 构建 ClickHouse 主从架构](https://juicefs.com/zh-cn/blog/user-stories/jerry-clickhouse-read-write-separation-juicefs-primary-replica-architecture)，2024-05-17，马涛@Jerry\n- [大模型存储实践：性能、成本与多云](https://juicefs.com/zh-cn/blog/solutions/large-model-storage-performance-multi-cloud)，2024-04-07，苏锐\n- [知乎：多云架构下大模型训练，如何保障存储稳定性](https://juicefs.com/zh-cn/blog/user-stories/data-storage-multi-cloud-zhihu-model-training-juicefs)，2024-03-28，王新\n- [BentoML：如何使用 JuiceFS 加速大模型加载](https://juicefs.com/zh-cn/blog/user-stories/bentoml-use-juicefs-accelerate-large-model-loading)，2024-02-21，管锡鹏\n- [韩国国民搜索 NAVER：为 AI 平台引入存储方案 JuiceFS](https://juicefs.com/zh-cn/blog/user-stories/naver-storage-solution-juicefs-ai-platforms)，2023-12-28，Nam Kyung-wan@NAVER\n- [机器人行业数据闭环实践：从对象存储到 JuiceFS](https://juicefs.com/zh-cn/blog/user-stories/data-object-storag--to-juicefs)，2023-12-13，宋巨超@刻行\n- [JuiceFS 在自动驾驶行业多云架构中的实践](https://juicefs.com/zh-cn/blog/user-stories/data-storage-multi-cloud-autonomous-driving-juicefs)，2023-10-27\n- [构建易于运维的 AI 训练平台：存储选型与最佳实践](https://juicefs.com/zh-cn/blog/user-stories/easy-operate-ai-training-platform-storage-selection)，2023-08-04，孙冀川@思谋科技\n- [之江实验室：如何基于 JuiceFS 为超异构算力集群构建存储层](https://juicefs.com/zh-cn/blog/user-stories/high-performance-scale-out-heterogeneous-computing-power-cluster-storage)，2023-06-09，洪晨@之江实验室\n- [加速 AI 训练，如何在云上实现灵活的弹性吞吐](https://juicefs.com/zh-cn/blog/solutions/accelerate-ai-training-flexible-elastic-throughput-cloud)，2023-05-06，苏锐\n- [如何借助分布式存储 JuiceFS 加速 AI 模型训练](https://juicefs.com/zh-cn/blog/usage-tips/how-to-use-juicefs-to-speed-up-ai-model-training)，2023-04-25，高昌健\n- [云原生数据交付平台 Kuda 在 AI 场景下的模型分发实践](https://xie.infoq.cn/article/7b41c7ab9e8bdf51e9910b8a9)，2023-01-30，Geek_c4ea78\n- [vivo AI 计算平台的轩辕文件存储实践](https://www.infoq.cn/article/3oFSOWfYGsX5h7xzsIe6)，2022-10-18，彭毅格@vivo AI 计算平台团队\n- [深势科技分享 AI 企业多云存储架构实践](https://juicefs.com/zh-cn/blog/user-stories/dptech-ai-storage-in-multi-cloud-practice)，2022-07-06，李样兵@深势科技\n- [AI 场景存储优化：云知声超算平台基于 JuiceFS 的存储实践](https://juicefs.com/zh-cn/blog/user-stories/juicefs-support-ai-storage-at-unisound)，2022-06-28，吕冬冬@云知声\n- [上汽云 x JuiceFS：iGear 用了这个小魔法，模型训练速度提升 300%](https://juicefs.com/zh-cn/blog/user-stories/performance-boost-3x-on-igear-platform)，2022-01-27，上汽云 iGear\n- [PaddlePaddle x JuiceFS : 全新缓存组件，大幅加速云上飞桨分布式训练作业](https://juicefs.com/zh-cn/blog/solutions/juicefs-helps-paddlepaddle-boosting-performance)，2022-01-06，百度 PaddlePaddle 团队\n- [如何在 Kubernetes 集群中玩转 Fluid + JuiceFS](https://juicefs.com/zh-cn/blog/solutions/fluid-with-juicefs)，2021-12-01，吕冬冬@云知声 & 朱唯唯@Juicedata\n- [百亿级小文件存储，JuiceFS 在自动驾驶行业的最佳实践](https://juicefs.com/zh-cn/blog/user-stories/ten-billion-level-small-files-storage-juicefs-best-practice-in-the-autonomous-driving-industry)，2021-10-28，高昌健\n- [初探云原生下的 AI 分布式文件系统-JuiceFS](https://mp.weixin.qq.com/s/AiI0lVgFwycmK9Rl-6KW4w)，2021-08-18，屈骏@梯度科技\n- [如何借助 JuiceFS 为 AI 模型训练提速 7 倍](https://juicefs.com/blog/cn/posts/how-to-use-juicefs-to-speed-up-ai-model-training-by-7-times)\n\n## 大数据\n\n- [基于 JuiceFS 的大数据平台上云：存储成本省 85%，性能媲美 HDFS](https://juicefs.com/zh-cn/blog/user-stories/hdfs-to-object-storage-juicefs)，2024-01-10，JuiceFS 资深用户\n- [多点 DMALL：大数据存算分离下的存储架构探索与实践](https://juicefs.com/zh-cn/blog/user-stories/separation-of-storage--computing-building-cloud-native-big-data-platform), 2023-08-16，李铭@多点\n- [网易互娱出海之旅：大数据平台上云架构设计与实践](https://juicefs.com/zh-cn/blog/user-stories/hadoop-compatible-storage-big-data-cloud-platform-s3)，2023-08-09，柯维鸿@网易互娱\n- [云上大数据存储：探究 JuiceFS 与 HDFS 的异同](https://juicefs.com/zh-cn/blog/engineering/similarities-and-differences-between-hdfs-and-juicefs-structures)，2023-04-04，汤友棚\n- [Protobuf 在知乎大数据场景的应用，利用 JuiceFS 动态注入 JAR 包](https://zhuanlan.zhihu.com/p/586120009)，2022-11-23，胡梦宇@知乎\n- [金山云：基于 JuiceFS 的 Elasticsearch 温冷热数据管理实践](https://juicefs.com/zh-cn/blog/user-stories/juicefs-elasticsearch-cold-heat-data-management)，2022-11-17，侯学峰@金山云\n- [JuiceFS 替代 HDFS，苦 HDFS 小文件久矣](https://zhuanlan.zhihu.com/p/569586606)，2022-10-08，久耶供应链 大数据总监\n- [JuiceFS 在 Elasticsearch/ClickHouse 温冷数据存储中的实践](https://juicefs.com/zh-cn/blog/solutions/juicefs-elasticsearch-clickhouse-hot-cold-data-storage)，2022-09-30，高昌健\n- [从 Hadoop 到云原生，大数据平台如何做存算分离](https://juicefs.com/zh-cn/blog/solutions/hadoop-to-cloud-native-separation-of-compute-and-storage-for-big-data-platform)，2022-09-14，苏锐\n- [理想汽车：从 Hadoop 到云原生的演进与思考](https://juicefs.com/zh-cn/blog/user-stories/li-auto-case-hadoop-cloud-native)，2022-08-30，聂磊@理想汽车\n- [一面数据：Hadoop 迁移云上架构设计与实践](https://juicefs.com/zh-cn/blog/user-stories/yimiancase)，2022-07-28，刘畅&李阳良@一面数据\n- [移动云使用 JuiceFS 支持 Apache HBase 增效降本的探索](https://juicefs.com/zh-cn/blog/user-stories/juicefs-support-hbase-at-chinamobile-cloud)，2022-05-31，陈海峰@移动云\n- [JuiceFS 在数据湖存储架构上的探索](https://juicefs.com/zh-cn/blog/solutions/juicefs-exploration-on-data-lake-storage-architecture)，2022-04-28，高昌健\n- [JuiceFS 在理想汽车的使用和展望](https://juicefs.com/zh-cn/blog/user-stories/li-auto-with-juicefs)，2022-01-21，聂磊@理想汽车\n- [JuiceFS + HDFS 权限问题定位](https://mp.weixin.qq.com/s/9mIMPuljL-UxP9t7-3dKxw)，2021-12-31，李阳良@一面数据\n- [知乎 x JuiceFS：利用 JuiceFS 给 Flink 容器启动加速](https://juicefs.com/zh-cn/blog/user-stories/zhihu-flink-with-juicefs)，2021-11-22，胡梦宇@知乎\n- [Elasticsearch 存储成本省 60%，稿定科技干货分享](https://juicefs.com/zh-cn/blog/user-stories/gaoding-with-juicefs)，2021-10-09，稿定 SRE 团队\n- [Shopee x JuiceFS：ClickHouse 冷热数据分离存储架构与实践](https://juicefs.com/zh-cn/blog/user-stories/shopee-clickhouse-with-juicefs)，2021-10-09，Teng@Shopee\n- [JuiceFS on AWS EMR](https://www.youtube.com/watch?v=PFNOcqiW4-M&t=3s), Youtube video, Pahud Dev\n- [JuiceFS 加速 Spark Shuffle](https://mp.weixin.qq.com/s/JGa2eYqM8db_OMU7SzZw8A)，2021-03-09，RespectM\n- [JuiceFS 如何帮助趣头条超大规模 HDFS 降负载](https://juicefs.com/blog/cn/posts/qutoutiao-big-data-platform-user-case)\n- [环球易购数据平台如何做到既提速又省钱？](https://juicefs.com/blog/cn/posts/globalegrow-big-data-platform-user-case)\n- [JuiceFS 在大搜车数据平台的实践](https://juicefs.com/blog/cn/posts/juicefs-practice-in-souche)\n- [使用 AWS Cloudformation 在 Amazon EMR 中一分钟配置 JuiceFS](https://aws.amazon.com/cn/blogs/china/use-aws-cloudformation-to-configure-juicefs-in-amazon-emr-in-one-minute)\n- [使用 JuiceFS 在云上优化 Kylin 4.0 的存储性能](https://juicefs.com/blog/cn/posts/optimize-kylin-on-juicefs)\n- [ClickHouse 存算分离架构探索](https://juicefs.com/blog/cn/posts/clickhouse-disaggregated-storage-and-compute-practice)\n- [存算分离实践：JuiceFS 在中国电信日均 PB 级数据场景的应用](https://juicefs.com/zh-cn/blog/user-stories/applicatio-of-juicefs-in-china-telecoms-daily-average-pb-data-scenario)\n\n## 云原生 & Kubernetes\n\n- [加速 AI 训推：Lepton AI 如何构建多租户、低延迟云存储平台](https://juicefs.com/zh-cn/blog/user-stories/lepton-ai-build-multi-tenant-low-latency-cloud-storage-platform)，2025-01-17，丁聪@Lepton AI\n- [海柔仿真系统存储实践：混合云架构下实现高可用与极简运维](https://juicefs.com/zh-cn/blog/user-stories/multi-cloud-storage-high-availability)，2024-11-08，吴森栋@海柔创新\n- [好未来：多云环境下基于 JuiceFS 建设低运维模型仓库](https://juicefs.com/zh-cn/blog/user-stories/multi-cloud-storage-juicefs-model-stroage)，2024-11-06，贺龙华@好未来\n- [小米云原生文件存储平台化实践：支撑 AI 训练、大模型、容器平台多项业务](https://juicefs.com/zh-cn/blog/user-stories/cloud-native-file-storage-platform-as-ai-training-large-models-container-platforms)，2023-09-22，孙佳朋@小米\n- [大模型训练：K8s 环境中数千节点存储最佳实践](https://juicefs.com/zh-cn/blog/usage-tips/large-model-storage-kubernetes)，2024-09-25，朱唯唯\n- [Clobotics 计算机视觉场景存储实践：多云架构、POSIX 全兼容、低运维的统一存储](https://juicefs.com/zh-cn/blog/user-stories/clobotics-posix-multi-cloud-storage)，2024-08-30，Jonnas@Clobotics\n- [如何在 Kubernetes 中使用 ClickHouse 和 JuiceFS](https://juicefs.com/zh-cn/blog/usage-tips/kubernetes-clickhouse-juicefs)，2024-08-02，Vitaliy Zakaznikov\n- [Kubernetes 数据持久化：从零开始使用 JuiceFS CSI Driver](https://juicefs.com/zh-cn/blog/usage-tips/kubernetes-juicefs-csi-driver)，2023-12-11，于鸿儒\n- [从本地到云端：豆瓣如何使用 JuiceFS 实现统一的数据存储](https://juicefs.com/zh-cn/blog/user-stories/scalable-computing-unified-data-storage-ops-cloud-spark-k8s-juicefs)，2023-05-10，曹丰宇@豆瓣\n- [Sidecar-详解 JuiceFS CSI Driver 新模式](https://juicefs.com/zh-cn/blog/usage-tips/explain-in-detail-juicefs-csi-driver-sidecar)，2023-02-22，朱唯唯\n- [存储更弹性，详解 Fluid“ECI 环境数据访问”新功能](https://juicefs.com/zh-cn/blog/solutions/fluid-eci-juicefs)，2022-09-05，朱唯唯\n- [基于 JuiceFS 的 KubeSphere DevOps 项目数据迁移方案](https://mp.weixin.qq.com/s/RgUHRUrL0u-J9nVqwOfS8Q)，2022-08-04，尹珉@数跑科技\n- [JuiceFS 在火山引擎边缘计算的应用实践](https://juicefs.com/zh-cn/blog/user-stories/how-juicefs-accelerates-edge-rendering-performance-in-volcengine)，2023-02-17\n，何兰州\n- [使用 KubeSphere 应用商店 5 分钟内快速部署 JuiceFS](https://juicefs.com/zh-cn/blog/solutions/kubesphere-with-juicefs)，2021-11-19，尹珉@杭州数跑科技 & 朱唯唯@Juicedata\n- [JuiceFS CSI Driver 的最佳实践](https://juicefs.com/zh-cn/blog/engineering/csi-driver-best-practices)，2021-11-08，朱唯唯\n- [JuiceFS CSI Driver v0.10 全新架构解读](https://juicefs.com/zh-cn/blog/engineering/juicefs-csi-driver-v010)，2021-07-28，朱唯唯\n\n## 数据共享\n\n- [Ollama + JuiceFS：一次拉取，到处运行](https://juicefs.com/zh-cn/blog/usage-tips/ollama-juicefs)，2024-09-09，朱唯唯\n- [Conda + JuiceFS：增强 AI 开发环境共享能力](https://juicefs.com/zh-cn/blog/usage-tips/conda-juicefs-enhance-ai)，2024-12-04，于鸿儒\n- [云上使用 Stable Diffusion，模型数据如何共享和存储？](https://juicefs.com/zh-cn/blog/usage-tips/share-store-model-data-stable-diffusion-cloud)，2023-06-16，于鸿儒\n- [基于 JuiceFS 搭建 Milvus 分布式集群](https://juicefs.com/blog/cn/posts/build-milvus-distributed-cluster-based-on-juicefs)\n- [如何解决 NAS 单点故障还顺便省了 90% 的成本？](https://juicefs.com/blog/cn/posts/modao-replace-nas-with-juicefs)\n\n## 数据备份、迁移与恢复\n\n- [JuiceFS v1.3-Beta1：一亿文件备份分钟级完成，性能优化全解析](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v13-beta1-backup)，2025-05-21，黄杰烽\n- [基于 JuiceFS 的低成本 Elasticsearch 云上备份存储](https://juicefs.com/zh-cn/blog/user-stories/low-cost-elasticsearch-cloud-backup-storage-juicefs)，2023-11-15，uuwang（正能量云）@杭州火石创造\n- [突破存储数据量限制，JuiceFS 在携程海量冷数据场景下的实践](https://juicefs.com/zh-cn/blog/user-stories/xiecheng-case)，2022-08-29，妙成 & 小峰\n- [40+ 倍提升，详解 JuiceFS 元数据备份恢复性能优化方法](https://juicefs.com/zh-cn/blog/engineering/juicefs-load-and-dump-optimization)，2022-07-13，执剑\n- [利用 JuiceFS 把 MySQL 备份验证性能提升 10 倍](https://juicefs.com/blog/cn/posts/optimize-xtrabackup-prepare-by-oplog)\n- [跨云数据搬迁利器：Juicesync](https://juicefs.com/blog/cn/posts/juicesync)\n- [下厨房基于 JuiceFS 的 MySQL 备份实践](https://juicefs.com/blog/cn/posts/xiachufang-mysql-backup-practice-on-juicefs)\n- [如何用 JuiceFS 归档备份 NGINX 日志](https://juicefs.com/blog/cn/posts/backup-nginx-logs-on-juicefs)\n\n## 原理解析\n\n- [JuiceFS sync 原理解析与性能优化，企业级数据同步利器](https://juicefs.com/zh-cn/blog/engineering/juicefs-sync-principle-performance-optimization)，2025-11-26，执剑\n- [深入解析 JuiceFS 垃圾回收机制](https://juicefs.com/zh-cn/blog/engineering/juicefs-gc-mechanism)，2025-10-30，许誉超\n- [JuiceFS writeback：写加速机制与适用场景解析](https://juicefs.com/zh-cn/blog/solutions/juicesfs-writeback-analysis)，2025-08-25，蔡敏\n- [JuiceFS on Windows: 首个 Beta 版的探索与优化原理](https://juicefs.com/zh-cn/blog/engineering/juicefs-on-windows-beta)，2025-08-04，陈杰\n- [JuiceFS v1.3-Beta2：Apache Ranger 集成与权限控制原理](https://juicefs.com/zh-cn/blog/release-notes/juicefs-1-3-ranger)，2025-06-06，汤友棚\n- [深度解析 JuiceFS 权限管理：Linux 多种安全机制全兼容](https://juicefs.com/zh-cn/blog/engineering/linux-file-system-juicefs-access-management)，2025-06-12，黄杰烽\n- [JuiceFS v1.3-Beta1：一亿文件备份分钟级完成，性能优化全解析](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v13-beta1-backup)，2025-05-21，黄杰烽\n- [代码级解析：JuiceFS 元数据、数据存储设计原理](https://juicefs.com/zh-cn/blog/engineering/juicefs-metadata-data-stroage-designed)，2024-11-25，Arthur\n- [JuiceFS CSI：Mount Pod 的平滑升级及其实现原理](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-csi-mount-pod-smooth-upgrade)，2024-10-30，朱唯唯\n- [一文详解 JuiceFS 读性能：预读、预取、缓存、FUSE 和对象存储](https://juicefs.com/zh-cn/blog/engineering/juicefs-read-performance)，2024-07-26，莫飞虎\n- [平滑升级功能详解，不停服即可更新](https://juicefs.com/zh-cn/blog/engineering/smooth-upgrade)，2024-05-07，执剑\n- [JuiceFS 目录配额功能设计详解](https://juicefs.com/zh-cn/blog/engineering/design-juicefs-directory-quotas)，2023-10-09，Sandy\n- [JuiceFS CSI Driver 架构设计详解](https://juicefs.com/zh-cn/blog/engineering/juicefs-csi-driver-arch-design)，2022-03-23，朱唯唯\n- [JuiceFS 数据加密原理](https://juicefs.com/zh-cn/blog/engineering/juicefs-encryption)，2021-12-23，Sandy\n\n## 教程、使用指南、评测及其他\n\n- [JuiceFS 企业版 5.3 特性详解：单文件系统支持超 5,000 亿文件，首次引入 RDMA](https://juicefs.com/zh-cn/blog/release-notes/juicefs-enterprise-5-3-500b-files-rdma-support)，2026-01-29，Sandy\n- [仅两台缓存节点，如何支撑 1.45TB/s 大吞吐业务 #JuiceFS 优化实践](https://juicefs.com/zh-cn/blog/solutions/how-2-cache-nodes-support-tbs-throughput)，2026-01-16，蔡敏\n- [从 MLPerf Storage v2.0 看 AI 训练中的存储性能与扩展能力](https://juicefs.com/zh-cn/blog/engineering/juicefs-mlperf-storage-v2-ai-training-storage-performance)，2025-09-17，莫飞虎\n- [实现 TB 级聚合带宽，JuiceFS 分布式缓存网络优化实践](https://juicefs.com/zh-cn/blog/engineering/tb-bandwidth-juicefs-distributed-cache-optimization)，2025-09-03，莫飞虎\n- [3000 台 JuiceFS Windows 客户端性能评估](https://juicefs.com/zh-cn/blog/solutions/juicefs-windows--performance-test)，2025-08-06，蔡敏\n- [探索 LanceDB 在多种存储方案下的查询效率](https://juicefs.com/zh-cn/blog/solutions/lancedb-query-performance-across-storage-solutions)，2025-07-30，白伯纯\n- [JuiceFS 社区版 V1.3 正式发布：支持 Python SDK、亿级备份加速、SQL 和 Windows 全面优化](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v13-ga)，2025-07-08\n- [Lustre 与 JuiceFS：架构设计、文件分布与特性比较](https://juicefs.com/zh-cn/blog/engineering/lustre-vs-juicefs)，2025-06-18，刘庆\n- [JuiceFS 企业版 5.2：迈入千亿文件时代，稳定性与性能再升级，首次支持 Windows 客户端](https://juicefs.com/zh-cn/blog/release-notes/juicefs-enterprise-edition-v52)，2025-05-28\n- [JuiceFS v1.3-beta1：新增 Python SDK，特定场景性能 3 倍于 FUSE](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v13-beta1-python-sdk)，2025-05-09，莫飞虎\n- [JuiceFS v1.3-beta1：全面优化 SQL 数据库支持，十亿级元数据管理新选项](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v13-beta1-sql)，2025-04-23，楼方鑫\n- [DeepSeek 3FS 与 JuiceFS：架构与特性比较](https://juicefs.com/zh-cn/blog/engineering/deepseek-3fs-vs-juicefs)，2025-03-18，刘庆\n- [FUSE，从内核到用户态文件系统的设计之路](https://juicefs.com/zh-cn/blog/engineering/fuse-file-system-design)，2025-02-27，许誉超\n- [告别“服务器繁忙”：JuiceFS 助力打造专属 DeepSeek 牧场](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-deepseek--storage)，2025-02-08，朱唯唯\n- [缓存管理自动化：JuiceFS 企业版 Cache Group Operator 新特性发布](https://juicefs.com/zh-cn/blog/release-notes/juicefs-cache-group-operator)，2024-12-26，张旭辉\n- [代码级解析：JuiceFS 元数据、数据存储设计原理](https://juicefs.com/zh-cn/blog/engineering/juicefs-metadata-data-stroage-designed)，2024-11-25，Arthur\n- [使用 JuiceFS 快照功能实现数据库发布与端到端测试](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-snapshot-database-test)，2024-11-15，马涛@Jerry\n- [详解 JuiceFS 在多云架构下的数据同步与一致性](https://juicefs.com/zh-cn/blog/solutions/juicefs-mirror)，2024-10-18\n- [全新 JuiceFS Python SDK 快速上手](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-python-sdk)，2024-10-14，于鸿儒\n- [Hugging Face + JuiceFS：多用户多节点环境下提升模型加载效率](https://juicefs.com/zh-cn/blog/usage-tips/huggingface-juicefs)，2024-09-29，于鸿儒\n- [JuiceFS 企业版 5.1：新增可写镜像、Python SDK 多项特性，强化 AI 场景支持](https://juicefs.com/zh-cn/blog/release-notes/juicefs-enterprise-edition-v51)，2024-09-14\n- [性能、成本与 POSIX 兼容性比较：JuiceFS vs EFS vs FSx for Lustre](https://juicefs.com/zh-cn/blog/engineering/juicefs-vs-efs-fsx-for-lustre)，2024-09-04，白伯纯\n- [如何判断数据库和对象存储是否被 JuiceFS 使用？](https://juicefs.com/zh-cn/blog/usage-tips/database-object-storage-used-by-juicefs)，2024-08-16，于鸿儒\n- [MemVerge：小文件写入性能 5 倍于 S3FS，JuiceFS 加速生信研究](https://juicefs.com/zh-cn/blog/user-stories/memverge-s3fs-juicefs)，2024-07-24，Jon Jiang@MemVerge\n- [JuiceFS 直连 NFS 新功能介绍，赋能 NAS 进行 AI 训练](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-nfs-nas-ai)，2024-07-19，于鸿儒\n- [SeaweedFS + TiKV 部署保姆级教程](https://juicefs.com/zh-cn/blog/usage-tips/seaweedfs-tikv)，2024-07-12，杨进豪@思谋科技\n- [JuiceFS 社区版 v1.2 发布，新增企业级权限管理、平滑升级功能](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v12)，2024-06-21\n- [JuiceFS S3 Gateway 新功能上手指南](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-s3-gateway)，2024-06-05，于鸿儒\n- [JuiceFS POSIX ACL 权限管理上手指南](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-posix-acl-permission-management-guide)，2024-05-23，于鸿儒\n- [详解 JuiceFS sync 新功能，选择性同步增强与多场景性能优化](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-sync)，2024-05-15，执剑\n- [JuiceFS v1.2-beta 1: ACL 功能全解析，更精细的权限控制](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v12-beta-1-acl)，2024-04-26，黄杰烽\n- [JuiceFS v1.2-beta1，Gateway 升级，多用户场景权限管理更灵活](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v12-beta1-gateway)，2024-04-22，执剑\n- [如何使用 Grafana 监控文件系统状态](https://juicefs.com/zh-cn/blog/usage-tips/use-grafana-monitor-file-system-status)，2024-04-12，于鸿儒\n- [在 Google Colab 中使用 JuiceFS](https://juicefs.com/zh-cn/blog/community/google-colab-juicefs)，2024-03-22，Jet\n- [从 HPC 到 AI：探索文件系统的发展及性能评估](https://juicefs.com/zh-cn/blog/user-stories/hpc-ai-file-systems-performance-development)，2024-03-06，鲁蔚征\n- [千卡利用率超 98%，详解 JuiceFS 在权威 AI 测试中的实现策略](https://juicefs.com/zh-cn/blog/engineering/juicefs-mlperf-test)，2024-02-28，莫飞虎\n- [极限挑战：使用 Go 打造百亿级文件系统的实践之旅](https://juicefs.com/zh-cn/blog/engineering/go-build-billion-file-system)，2024-02-02，Sandy\n- [详解新功能 JuiceFS CSI Dashboard：简化云上环境的问题排查流程](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-csi-dashboard)，2023-12-29，李晨曦\n- [JuiceFS 用户必备的 6 个技巧](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-user-skills)，2023-11-22，于鸿儒\n- [手把手教你搭建 Ceph 集群、对接 JuiceFS 文件系统](https://juicefs.com/zh-cn/blog/usage-tips/ceph-juicefs)，2023-11-20\n- [JuiceFS 企业版 5.0 新特性速览](https://juicefs.com/zh-cn/blog/release-notes/juicefs-enterprise-edition-v5)，2023-11-17\n- [POSIX 真的不适合对象存储吗](https://juicefs.com/zh-cn/blog/engineering/posix-object-store-suitable-file-system)，2023-10-24，于鸿儒\n- [浅析 GlusterFS 与 JuiceFS 的架构异同](https://juicefs.com/zh-cn/blog/engineering/similarities-and-differences-between-glusterfs-and-juicefs-structures)，2023-08-23，Sandy\n- [如何基于 JuiceFS 配置 Samba 和 NFS 共享？](https://juicefs.com/zh-cn/blog/usage-tips/configure-samba-and-nfs-shares-based-juicefs)，2023-08-04，于鸿儒\n- [从架构到特性：JuiceFS 企业版首次全面解析](https://juicefs.com/zh-cn/blog/solutions/juicefs-enterprise-edition-features-vs-community-edition)，2023-06-06，高昌健\n- [浅析三款大规模分布式文件系统架构设计](https://juicefs.com/zh-cn/blog/engineering/large-scale-distributed-filesystem-comparison)，2023-03-08，高昌健\n- [浅析 SeaweedFS 与 JuiceFS 架构异同](https://juicefs.com/zh-cn/blog/engineering/similarities-and-differences-between-seaweedfs-and-juicefs-structures)，2023-02-10，陈杰\n- [分布式文件系统 JuiceFS 测试总结](https://mp.weixin.qq.com/s/XFWQASQFt5FISip-mrYG4Q)，2022-09-13，邹秋波\n- [JuiceFS 元数据引擎选型指南](https://juicefs.com/zh-cn/blog/usage-tips/juicefs-metadata-engine-selection-guide)，2022-10-12，Sandy\n- [GitHub Codespaces 上分离计算和存储？ #JuiceFS 花式玩法#](https://mp.weixin.qq.com/s/geoYkruj6lkXOns7bib-qA)，2022-08-19，张俊帆\n- [浅析 Redis 作为 JuiceFS 元数据引擎的优劣势](https://juicefs.com/zh-cn/blog/usage-tips/introduce-redis-as-juicefs-metadata-engine)，2022-07-22，高昌健\n- [如何使用 etcd 实现分布式 /etc 目录](https://juicefs.com/zh-cn/blog/usage-tips/make-distributed-etc-directory-with-etcd-and-juicefs)，2022-06-23，朱唯唯\n- [社区投稿｜小团队如何妙用 JuiceFS](https://mp.weixin.qq.com/s/AAw1I6f36h1pZjLELtQCow)，2022-04-01，timfeirg\n- [在 Windows 上如何后台运行 JuiceFS](https://mp.weixin.qq.com/s/nMqCuit4zRoNCK4m-b0hxA)，2022-03-10，秦牧羊\n- [JuiceFS 导出/导入元数据的优化之路](https://www.youtube.com/watch?v=MDMitDtLly4), Youtube Video\n- [初探 JuiceFS](https://mp.weixin.qq.com/s/jTBAcmUiBMBvTutdOUHpcA)，2021-11-28，ahnselina\n- [JuiceFS 源码阅读 - 上](https://mp.weixin.qq.com/s/mdqFJLpaJ249rUUEnRiP3Q)，2021-06-24，秦牧羊\n- [JuiceFS 你应该知道的一些事](https://mp.weixin.qq.com/s/6ylBmUXy_3aQggznl65nHg)，2021-01-15，祝威廉@Kyligence\n\n## 内容收录\n\n如果你也想把自己的 JuiceFS 应用方案添加到这份案例列表中，可以采用以下几种投稿方式：\n\n### GitHub 投稿\n\n你可以通过 GitHub 创建本仓库的分支，将你的案例网页链接添加到相应的分类中，提交 Pull Request 申请，等待审核和分支合并。\n\n### 社交媒体投稿\n\n你可以加入 JuiceFS 官方的 [Slack 频道](https://go.juicefs.com/slack)，任何一位工作人员都可以接洽案例投稿事宜。\n" }, { "path": "docs/zh_cn/community/integrations.md", "content": "---\ntitle: 社区集成\nsidebar_position: 2\nslug: /integrations\n---\n\n## SDK\n\n- [旷视科技](https://megvii.com) 团队贡献了 [Python SDK](https://github.com/megvii-research/juicefs-python)。\n\n## AI\n\n- [云知声](https://www.unisound.com) 团队参与开发 [Fluid](https://github.com/fluid-cloudnative/fluid) JuiceFSRuntime 缓存引擎，具体请参考[文档](https://github.com/fluid-cloudnative/fluid/blob/master/docs/zh/samples/juicefs_runtime.md) 。\n- [PaddlePaddle](https://github.com/paddlepaddle/paddle) 团队已将 JuiceFS 缓存加速特性集成到 [Paddle Operator](https://github.com/PaddleFlow/paddle-operator) 中，具体请参考[文档](https://github.com/PaddleFlow/paddle-operator/blob/sampleset/docs/zh_CN/ext-overview.md)。\n- 通过 JuiceFS 可以轻松搭建一个 [Milvus](https://milvus.io) 向量搜索引擎，Milvus 团队已经撰写了官方 [案例](https://zilliz.com/blog/building-a-milvus-cluster-based-on-juicefs) 与 [教程](https://tutorials.milvus.io/en-juicefs/index.html?index=..%2F..index#0)。\n\n## 大数据\n\n- 大数据 OLAP 分析引擎 [Apache Kylin 4.0](http://kylin.apache.org) 可以使用 JuiceFS 在所有公有云上轻松部署存储计算分离架构的集群，请看 [视频分享](https://www.bilibili.com/video/BV1c54y1W72S) 和 [案例文章](https://juicefs.com/zh-cn/blog/optimize-kylin-on-juicefs)。\n- [Apache Hudi](https://hudi.apache.org) 自 v0.10.0 版本开始支持 JuiceFS，你可以参考[官方文档](https://hudi.apache.org/docs/jfs_hoodie)了解如何配置 JuiceFS。\n\n## DevOps\n\n- [Terraform Provider for JuiceFS](https://github.com/toowoxx/terraform-provider-juicefs) 由 Toowoxx IT GmbH 贡献，他们是一家来自德国的 IT 服务公司。\n\n## Alfred\n\nJuiceFS 文档站集成了 Alfred，可以快速搜索 JuiceFS 文档。\n\n![JuiceFS Alfred Workflow](../images/workflow-root.png)\n\n只需在 Alfred 中输入关键字（默认：jfs）并提供查询即可查看 JuiceFS 文档的即时搜索结果。\n\n### 安装\n\n安装 Alfred 5 的 JuiceFS workflow： [下载最新版本](https://github.com/zwwhdls/juicefs-alfred-workflow/releases/download/v0.2.0/JuiceFS.Search.alfredworkflow)\n\n### 使用\n\n可以搜索 JuiceFS 的所有文档，包括社区、企业和 CSI：\n\n```\n# 查询 JuiceFS 社区版文档\njfs ce \n# 查询 JuiceFS 企业版文档\njfs ee \n# 查询 JuiceFS CSI 文档\njfs csi \n```\n\n![JuiceFS Alfred Workflow demo](../images/workflow-demo.gif)\n\n### Workflow 可配置的变量\n\n- `API_KEY`：JuiceFS 文档使用的 algolia 的 API 密钥，使用默认值就可以。\n- `LANGUAGE`：要搜索的 JuiceFS 文档的语言（en/zh），默认为 en。\n- `HITS_PER_PAGE`：每次搜索的点击量，默认值为 10。\n\n![JuiceFS Alfred Workflow configuration](../images/configuration.png)\n" }, { "path": "docs/zh_cn/community/usage_tracking.md", "content": "---\ntitle: 用量上报\nsidebar_position: 4\n---\n\nJuiceFS 默认会收集并上报 **「匿名」** 的使用数据。这些数据仅仅包含核心指标（如版本号、文件系统大小），不会包含任何用户信息或者敏感数据。你可以查看[这里](https://github.com/juicedata/juicefs/blob/main/pkg/usage/usage.go)检查相关代码。\n\n这些数据帮助我们理解社区如何使用这个项目。你可以简单地通过 `--no-usage-report` 选项关闭用量上报：\n\n```\njuicefs mount --no-usage-report\n```\n" }, { "path": "docs/zh_cn/deployment/_share_via_nfs.md", "content": "---\nsidebar_label: 配置 NFS 共享\nsidebar_position: 5\n---\n# 通过 NFS 共享 JuiceFS 存储\n" }, { "path": "docs/zh_cn/deployment/_share_via_smb.md", "content": "---\nsidebar_label: 配置 SMB 共享\nsidebar_position: 6\n---\n# 通过 SMB 共享 JuiceFS 存储\n" }, { "path": "docs/zh_cn/deployment/automation.md", "content": "---\ntitle: 自动化部署\nsidebar_position: 7\n---\n\n面对大量节点需要安装并挂载 JuiceFS 时，可以用本章介绍的方法进行自动化部署。\n\n下方示范仅用于挂载，因此你需要提前[创建好 JuiceFS 文件系统](../getting-started/standalone.md#juicefs-format)。\n\n## Ansible\n\n使用 [Ansible](https://ansible.com) 在本机挂载 JuiceFS 文件系统的 playbook 样例如下：\n\n```yaml\n- hosts: localhost\n tasks:\n - set_fact:\n # 根据实际情况修改\n meta_url: sqlite3:///tmp/myjfs.db\n jfs_path: /jfs\n jfs_pkg: /tmp/juicefs-ce.tar.gz\n jfs_bin_dir: /usr/local/bin\n\n - get_url:\n # 根据实际情况替换成需要的下载链接\n url: https://d.juicefs.com/juicefs/releases/download/v1.0.2/juicefs-1.0.2-linux-amd64.tar.gz\n dest: \"{{jfs_pkg}}\"\n\n - ansible.builtin.unarchive:\n src: \"{{jfs_pkg}}\"\n dest: \"{{jfs_bin_dir}}\"\n include:\n - juicefs\n\n - name: Create symbolic for fstab\n ansible.builtin.file:\n src: \"{{jfs_bin_dir}}/juicefs\"\n dest: \"/sbin/mount.juicefs\"\n state: link\n\n - name: Mount JuiceFS and create fstab entry\n mount:\n path: \"{{jfs_path}}\"\n src: \"{{meta_url}}\"\n fstype: juicefs\n opts: _netdev\n state: mounted\n```\n" }, { "path": "docs/zh_cn/deployment/hadoop_java_sdk.md", "content": "---\ntitle: 在 Hadoop 生态使用 JuiceFS\nsidebar_position: 3\nslug: /hadoop_java_sdk\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\nJuiceFS 提供与 HDFS 接口[高度兼容](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/filesystem/introduction.html)的 Java 客户端，Hadoop 生态中的各种应用都可以在不改变代码的情况下，平滑地使用 JuiceFS 存储数据。\n\n## 环境要求\n\n### 1. Hadoop 及相关组件\n\nJuiceFS Hadoop Java SDK 同时兼容 Hadoop 2.x、Hadoop 3.x，以及 Hadoop 生态中的各种主流组件。\n\n### 2. 用户权限\n\nJuiceFS 默认使用本地的「用户／UID」及「用户组／GID」映射，在分布式环境下使用时，为了避免权限问题，请参考[文档](../administration/sync_accounts_between_multiple_hosts.md)将需要使用的「用户／UID」及「用户组／GID」同步到所有 Hadoop 节点。也可以通过定义一个全局的用户和用户组文件使得集群中的所有节点共享权限配置，相关配置请查看[这里](#其它配置)。\n\n### 3. 文件系统\n\n通过 JuiceFS Java 客户端为 Hadoop 生态提供存储，需要提前创建 JuiceFS 文件系统。部署 Java 客户端时，在配置文件中指定已创建文件系统的元数据引擎地址。\n\n创建文件系统可以参考 [JuiceFS 快速上手指南](../getting-started/installation.md)。\n\n:::note 注意\n如果要在分布式环境中使用 JuiceFS，创建文件系统时，请合理规划要使用的对象存储和数据库，确保它们可以被每个集群节点正常访问。\n:::\n\n### 4. 内存资源\n\n根据计算任务（如 Spark executor）的读写负载，JuiceFS Hadoop Java SDK 可能需要额外使用 4 * [`juicefs.memory-size`](#io-配置) 的堆外内存用来加速读写性能。默认情况下，建议为计算任务至少配置 1.2GB 的堆外内存。\n\n### 5. Java 运行时版本\n\nJuiceFS Hadoop Java SDK 默认使用 JDK 8 编译，如果需要在高版本的 Java 运行时中使用（如 Java 17），需在 JVM 参数中增加以下选项以允许使用反射 API：\n\n```shell\n--add-exports=java.base/sun.nio.ch=ALL-UNNAMED\n```\n\n更多关于以上选项的说明请参考[官方文档](https://docs.oracle.com/en/java/javase/17/migrate/migrating-jdk-8-later-jdk-releases.html#GUID-7BB28E4D-99B3-4078-BDC4-FC24180CE82B)。\n\n## 安装与编译客户端\n\n### 安装预编译客户端\n\n请参考[「安装」](../getting-started/installation.md#install-the-pre-compiled-client)文档了解如何下载预编译的 JuiceFS Hadoop Java SDK。\n\n### 手动编译客户端\n\n:::note 注意\n不论为哪个系统环境编译客户端，编译后的 JAR 文件都为相同的名称，且只能部署在匹配的系统环境中，例如在 Linux 中编译则只能用于 Linux 环境。另外，由于编译的包依赖 glibc，建议尽量使用低版本的系统进行编译，这样可以获得更好的兼容性。\n:::\n\n编译依赖以下工具：\n\n- [Go](https://golang.org) 1.15+（中国用户建议使用 [Goproxy China 镜像加速](https://github.com/goproxy/goproxy.cn)）\n- JDK 8+\n- [Maven](https://maven.apache.org) 3.3+（中国用户建议使用[阿里云镜像加速](https://maven.aliyun.com)）\n- Git\n- make\n- GCC 5.4+\n\n#### Linux 和 macOS\n\n克隆仓库：\n\n```shell\ngit clone https://github.com/juicedata/juicefs.git\n```\n\n进入目录，执行编译：\n\n```shell\ncd juicefs/sdk/java\nmake\n```\n\n:::note 注意\n如果使用 Ceph 的 RADOS 作为 JuiceFS 的存储引擎，需要先安装 `librados-dev` 包。\n:::\n\n```shell\ncd juicefs/sdk/java\nmake ceph\n```\n\n编译完成后，可以在 `sdk/java/target` 目录中找到编译好的 `JAR` 文件，包括两个版本：\n\n- 包含第三方依赖的包：`juicefs-hadoop-X.Y.Z.jar`\n- 不包含第三方依赖的包：`original-juicefs-hadoop-X.Y.Z.jar`\n\n建议使用包含第三方依赖的版本。\n\n#### Windows\n\n用于 Windows 环境的客户端需要在 Linux 或 macOS 系统上通过交叉编译的方式获得，编译依赖 [mingw-w64](https://www.mingw-w64.org)，需要提前安装。\n\n与编译面向 Linux 和 macOS 客户端的步骤相同，比如在 Ubuntu 系统上，先安装 `mingw-w64` 包，解决依赖问题：\n\n```shell\nsudo apt install mingw-w64\n```\n\n克隆并进入 JuiceFS 源代码目录，执行以下代码进行编译：\n\n```shell\ncd juicefs/sdk/java\n```\n\n```shell\nmake win\n```\n\n## 部署客户端\n\n让 Hadoop 生态各组件能够正确识别 JuiceFS，需要进行以下配置：\n\n1. 将编译好的 JAR 文件和 `$JAVA_HOME/lib/tools.jar` 放置到组件的 `classpath` 内，常见大数据平台和组件的安装路径见下表。\n2. 将 JuiceFS 相关配置写入配置文件（通常是 `core-site.xml`），详见[客户端配置参数](#客户端配置参数)。\n\n建议将 JAR 文件放置在一个统一的位置，其他位置通过符号链接进行调用。\n\n### 大数据平台\n\n| 名称 | 安装路径 |\n| ---- | ---- |\n| CDH | `/opt/cloudera/parcels/CDH/lib/hadoop/lib`

`/opt/cloudera/parcels/CDH/spark/jars`

`/var/lib/impala` |\n| HDP | `/usr/hdp/current/hadoop-client/lib`

`/usr/hdp/current/hive-client/auxlib`

`/usr/hdp/current/spark2-client/jars` |\n| Amazon EMR | `/usr/lib/hadoop/lib`

`/usr/lib/spark/jars`

`/usr/lib/hive/auxlib` |\n| 阿里云 EMR | `/opt/apps/ecm/service/hadoop/*/package/hadoop*/share/hadoop/common/lib`

`/opt/apps/ecm/service/spark/*/package/spark*/jars`

`/opt/apps/ecm/service/presto/*/package/presto*/plugin/hive-hadoop2`

`/opt/apps/ecm/service/hive/*/package/apache-hive*/lib`

`/opt/apps/ecm/service/impala/*/package/impala*/lib` |\n| 腾讯云 EMR | `/usr/local/service/hadoop/share/hadoop/common/lib`

`/usr/local/service/presto/plugin/hive-hadoop2`

`/usr/local/service/spark/jars`

`/usr/local/service/hive/auxlib` |\n| UCloud UHadoop | `/home/hadoop/share/hadoop/common/lib`

`/home/hadoop/hive/auxlib`

`/home/hadoop/spark/jars`

`/home/hadoop/presto/plugin/hive-hadoop2` |\n| 百度云 EMR | `/opt/bmr/hadoop/share/hadoop/common/lib`

`/opt/bmr/hive/auxlib`

`/opt/bmr/spark2/jars` |\n\n### 社区开源组件\n\n| 名称 | 安装路径 |\n|-----------|-----------------------------------------------------------------------------------------|\n| Hadoop | `${HADOOP_HOME}/share/hadoop/common/lib/`, `${HADOOP_HOME}/share/hadoop/mapreduce/lib/` |\n| Spark | `${SPARK_HOME}/jars` |\n| Presto | `${PRESTO_HOME}/plugin/hive-hadoop2` |\n| Trino | `${TRINO_HOME}/plugin/hive` |\n| Flink | `${FLINK_HOME}/lib` |\n| StarRocks | `${StarRocks_HOME}/fe/lib/`, `${StarRocks_HOME}/be/lib/hadoop/common/lib` |\n\n### 客户端配置参数\n\n请参考以下表格设置 JuiceFS 文件系统相关参数，并写入配置文件，一般是 `core-site.xml`。\n\n#### 核心配置\n\n| 配置项 | 默认值 | 描述 |\n| -------------------------------- | ---------------------------- | ------------------------------------------------------------ |\n| `fs.jfs.impl` | `io.juicefs.JuiceFileSystem` | 指定要使用的存储实现，默认使用 `jfs://` 作为 scheme。如想要使用其它 scheme（例如 `cfs://`），则修改为 `fs.cfs.impl` 即可。无论使用的 scheme 是什么，访问的都是 JuiceFS 中的数据。 |\n| `fs.AbstractFileSystem.jfs.impl` | `io.juicefs.JuiceFS` | 指定要使用的存储实现，默认使用 `jfs://` 作为 scheme。如想要使用其它 scheme（例如 `cfs://`），则修改为 `fs.AbstractFileSystem.cfs.impl` 即可。无论使用的 scheme 是什么，访问的都是 JuiceFS 中的数据。 |\n| `juicefs.meta` | | 指定预先创建好的 JuiceFS 文件系统的元数据引擎地址。可以通过 `juicefs.{vol_name}.meta` 格式为客户端同时配置多个文件系统。具体请参考[「多文件系统配置」](#多文件系统配置)。 |\n\n#### 缓存配置\n\n| 配置项 | 默认值 | 描述 |\n|------------------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `juicefs.cache-dir` | | 设置本地缓存目录，可以指定多个文件夹，用冒号 `:` 分隔，也可以使用通配符（比如 `*` ）。**请预先创建好这些目录，并给予 `0777` 权限，便于多个应用共享缓存数据。** |\n| `juicefs.cache-size` | 0 | 设置本地缓存目录的容量，单位 MiB，默认为 0，即不开启缓存。如果配置了多个缓存目录，该值代表所有缓存目录容量的总和。 |\n| `juicefs.cache-full-block` | `true` | 是否缓存所有读取的数据块，`false` 表示只缓存随机读的数据块。 |\n| `juicefs.free-space` | 0.1 | 本地缓存目录的最小可用空间比例，默认保留 10% 剩余空间。 |\n| `juicefs.open-cache` | 0 | 缓存打开的文件元数据（单位：秒），0 表示关闭 |\n| `juicefs.attr-cache` | 0 | 目录和文件属性缓存的过期时间（单位：秒） |\n| `juicefs.entry-cache` | 0 | 文件项缓存的过期时间（单位：秒） |\n| `juicefs.dir-entry-cache` | 0 | 目录项缓存的过期时间（单位：秒） |\n| `juicefs.discover-nodes-url` | | 指定发现集群缓存节点列表的方式，每 10 分钟刷新一次。

  • YARN：`yarn`
  • Spark Standalone：`http://spark-master:web-ui-port/json/`
  • Spark ThriftServer：`http://thrift-server:4040/api/v1/applications/`
  • Presto：`http://coordinator:discovery-uri-port/v1/service/presto/`
  • 文件系统：`jfs://{VOLUME}/etc/nodes`，需手动建立此文件，并将节点的主机名一条一行写入此文件
|\n\n#### I/O 配置\n\n| 配置项 | 默认值 | 描述 |\n|--------------------------|---------|------------------------|\n| `juicefs.max-uploads` | 20 | 上传数据的最大连接数 |\n| `juicefs.max-downloads` | 200 | 下载连接的最大数量 |\n| `juicefs.max-deletes` | 10 | 删除数据的最大连接数 |\n| `juicefs.get-timeout` | 5 | 下载一个对象的超时时间，单位为秒。 |\n| `juicefs.put-timeout` | 60 | 上传一个对象的超时时间，单位为秒。 |\n| `juicefs.memory-size` | 300 | 读写数据的缓冲区最大空间，单位为 MiB。 |\n| `juicefs.prefetch` | 1 | 预读数据块的线程数 |\n| `juicefs.upload-limit` | 0 | 上传带宽限制，单位为 Mbps，默认不限制。 |\n| `juicefs.download-limit` | 0 | 下载带宽限制，单位为 Mbps，默认不限制。 |\n| `juicefs.io-retries` | 10 | IO 失败重试次数 |\n| `juicefs.writeback` | `false` | 是否后台异步上传数据 |\n\n#### 其它配置\n\n| 配置项 | 默认值 | 描述 |\n|---------------------------|--------------|-------------------------------------------------------------------------------------------------------------|\n| `juicefs.bucket` | | 为对象存储指定跟格式化时不同的访问地址 |\n| `juicefs.debug` | `false` | 是否开启 debug 日志 |\n| `juicefs.access-log` | | 访问日志的路径。需要所有应用都有写权限，可以配置为 `/tmp/juicefs.access.log`。该文件会自动轮转，保留最近 7 个文件。 |\n| `juicefs.superuser` | `hdfs` | 超级用户 |\n| `juicefs.supergroup` | `supergroup` | 超级用户组 |\n| `juicefs.users` | `null` | 用户名以及 UID 列表文件的地址，比如 `jfs://name/etc/users`。文件格式为 `:`，一行一个用户。 |\n| `juicefs.groups` | `null` | 用户组、GID 以及组成员列表文件的地址，比如 `jfs://name/etc/groups`。文件格式为 `::,`，一行一个用户组。 |\n| `juicefs.umask` | `null` | 创建文件和目录的 umask 值（如 `0022`），如果没有此配置，默认值是 `fs.permissions.umask-mode`。 |\n| `juicefs.push-gateway` | | [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) 地址，格式为 `:`。 |\n| `juicefs.push-auth` | | [Prometheus 基本认证](https://prometheus.io/docs/guides/basic-auth)信息，格式为 `:`。 |\n| `juicefs.push-graphite` | | [Graphite](https://graphiteapp.org) 地址，格式为 `:`。 |\n| `juicefs.push-interval` | 10 | 指标推送的时间间隔，单位为秒。 |\n| `juicefs.push-labels` | | 指标额外标签，格式为 `key1:value1;key2:value2`。 |\n| `juicefs.fast-resolve` | `true` | 是否开启快速元数据查找（通过 Redis Lua 脚本实现） |\n| `juicefs.no-usage-report` | `false` | 是否上报数据。仅上版本号等使用量数据，不包含任何用户信息。 |\n| `juicefs.block.size` | `134217728` | 单位为字节，同 HDFS 的 `dfs.blocksize`，默认 128 MB |\n| `juicefs.file.checksum` | `false` | DistCp 使用 `-update` 参数时，是否计算文件 Checksum |\n| `juicefs.no-bgjob` | `false` | 是否关闭后台任务（清理、备份等） |\n| `juicefs.backup-meta` | 3600 | 自动将 JuiceFS 元数据备份到对象存储间隔（单位：秒），设置为 0 关闭自动备份 |\n|`juicefs.backup-skip-trash`| `false` | 备份元数据时忽略回收站中的文件和目录。 |\n| `juicefs.heartbeat` | 12 | 客户端和元数据引擎之间的心跳间隔（单位：秒），建议所有客户端都设置一样 |\n| `juicefs.skip-dir-mtime` | 100ms | 修改父目录 mtime 间隔。 |\n| `juicefs.subdir` | | 仅允许访问此目录的子路径。可以指定多个路径，使用逗号分隔。所有其他路径，包括根目录或同级目录，都将被拒绝访问。 |\n\n#### 多文件系统配置\n\n当需要同时使用多个 JuiceFS 文件系统时，上述所有配置项均可对特定文件系统进行指定，只需要将文件系统名字放在配置项的中间，比如下面示例中的 `jfs1` 和 `jfs2`：\n\n```xml\n\n juicefs.jfs1.meta\n redis://jfs1.host:port/1\n\n\n juicefs.jfs2.meta\n redis://jfs2.host:port/1\n\n```\n\n#### 配置示例\n\n以下是一个常用的配置示例，请替换 `juicefs.meta` 配置中的 `{HOST}`、`{PORT}` 和 `{DB}` 变量为实际的值。\n\n```xml\n\n fs.jfs.impl\n io.juicefs.JuiceFileSystem\n\n\n fs.AbstractFileSystem.jfs.impl\n io.juicefs.JuiceFS\n\n\n juicefs.meta\n redis://{HOST}:{PORT}/{DB}\n\n\n juicefs.cache-dir\n /data*/jfs\n\n\n juicefs.cache-size\n 1024\n\n\n juicefs.access-log\n /tmp/juicefs.access.log\n\n```\n\n## Hadoop 环境配置\n\n请参照前述各项配置表，将配置参数加入到 Hadoop 配置文件 `core-site.xml` 中。\n\n### CDH6\n\n如果使用的是 CDH 6 版本，除了修改 `core-site` 外，还需要通过 YARN 服务界面修改 `mapreduce.application.classpath`，增加：\n\n```shell\n$HADOOP_COMMON_HOME/lib/juicefs-hadoop.jar\n```\n\n### HDP\n\n除了修改 `core-site` 外，还需要通过 MapReduce2 服务界面修改配置 `mapreduce.application.classpath`，在末尾增加（变量无需替换）：\n\n```shell\n/usr/hdp/${hdp.version}/hadoop/lib/juicefs-hadoop.jar\n```\n\n### Flink\n\n将配置参数加入 `conf/flink-conf.yaml`。如果只是在 Flink 中使用 JuiceFS, 可以不在 Hadoop 环境配置 JuiceFS，只需要配置 Flink 客户端即可。\n\n#### 在阿里云实时平台 Flink SQL 使用 JuiceFS\n\n1. 创建 Maven 项目，根据 Flink 不同版本引入如下依赖\n\n ```xml\n \n \n io.juicefs\n juicefs-hadoop\n {JUICEFS_HADOOP_VERSION}\n \n\n \n \n org.apache.flink\n flink-table-runtime-blink_2.12\n 1.13.5\n provided\n \n\n \n \n org.apache.flink\n flink-table-common\n 1.15.2\n provided\n \n \n org.apache.flink\n flink-connector-files\n 1.15.2\n provided\n \n \n ```\n\n2. 创建一个 Java class\n\n ```java\n public class JuiceFileSystemTableFactory extends FileSystemTableFactory {\n @Override\n public String factoryIdentifier() {\n return \"juicefs\";\n }\n }\n ```\n\n3. Flink table connector 是使用 Java’s Service Provider Interfaces (SPI) 加载自定义实现。\n在 resources 按照如下结构创建文件\n\n ```\n ## for flink-1.13\n src/main/resources\n ├── META-INF\n │ └── services\n │ └── org.apache.flink.table.factories.Factory\n ```\n\n `org.apache.flink.table.factories.Factory` 文件内容：\n\n ```\n {YOUR_PACKAGE}.JuiceFileSystemTableFactory\n ```\n\n4. 将填写有 JuiceFS 配置的 core-site.xml 放到 src/main/resources 内：\n\n ```xml\n \n \n fs.juicefs.impl\n io.juicefs.JuiceFileSystem\n \n \n juicefs.meta\n redis://xxx.redis.rds.aliyuncs.com:6379/0\n \n ...\n \n ```\n\n :::note 注意\n 由于 `jfs://` scheme 被阿里其他文件系统占用，所以需要配置 `fs.juicefs.impl` 类为 JuiceFS 的实现类，并在后续路径使用 `juicefs://` 协议。\n :::\n\n5. 打包，确保 JAR 内包含 resources 目录下内容\n6. 通过阿里云实时计算平台控制台->应用->作业开发->connectors 界面上传 JAR 文件\n7. 测试，将如下 SQL 上线运行，可以在 JuiceFS 的 `tmp/tbl` 目录下发现写入内容\n\n ```sql\n CREATE TEMPORARY TABLE datagen_source(\n name VARCHAR\n ) WITH (\n 'connector' = 'datagen',\n 'number-of-rows' = '100'\n );\n\n CREATE TEMPORARY TABLE jfs_sink (name string)\n with (\n 'connector' = 'juicefs', 'path' = 'juicefs://{VOL_NAME}/tmp/tbl', 'format' = 'csv'\n );\n\n INSERT INTO jfs_sink\n SELECT\n name\n from datagen_source;\n ```\n\n### Hudi\n\n:::note 注意\nHudi 自 v0.10.0 版本开始支持 JuiceFS，请确保使用正确的版本。\n:::\n\n请参考[「Hudi 官方文档」](https://hudi.apache.org/docs/jfs_hoodie)了解如何配置 JuiceFS。\n\n### Kafka Connect\n\n可以使用 Kafka Connect 和 HDFS Sink Connector（[HDFS 2](https://docs.confluent.io/kafka-connect-hdfs/current/overview.html)、[HDFS 3](https://docs.confluent.io/kafka-connect-hdfs3-sink/current/overview.html)）将数据落盘存储到 JuiceFS。\n\n首先需要将 JuiceFS 的 SDK 添加到 Kafka Connect 的 `classpath` 内，如 `/usr/share/java/confluentinc-kafka-connect-hdfs/lib`。\n\n在新建 Connect Sink 任务时，做如下配置：\n\n- 指定 `hadoop.conf.dir` 为包含 `core-site.xml` 配置文件的目录，若没有运行在 Hadoop 环境，可创建一个单独目录，如 `/usr/local/juicefs/hadoop`，然后将与 JuiceFS 相关的配置添加到 `core-site.xml`。\n- 指定 `store.url` 为以 `jfs://` 开头的路径\n\n举例：\n\n```ini\n# 省略其他配置项...\nhadoop.conf.dir=/path/to/hadoop-conf\nstore.url=jfs://path/to/store\n```\n\n### HBase\n\nJuiceFS 适合存储 HBase 的 HFile，但不适合用来保存它的事务日志（WAL），因为将日志持久化到对象存储的时间会远高于持久化到 HDFS 的 DataNode 的内存中。\n\n建议部署一个小的 HDFS 集群来存放 WAL，HFile 文件则存储在 JuiceFS 上。\n\n#### 新建 HBase 集群\n\n修改 `hbase-site.xml` 配置：\n\n```xml title=\"hbase-site.xml\"\n\n hbase.rootdir\n jfs://{vol_name}/hbase\n\n\n hbase.wal.dir\n hdfs://{ns}/hbase-wal\n\n```\n\n#### 修改原有 HBase 集群\n\n除了修改上述配置项外，由于 HBase 集群已经在 ZooKeeper 里存储了部分数据，为了避免冲突，有以下两种方式解决：\n\n1. 删除原集群\n\n 通过 ZooKeeper 客户端删除 `zookeeper.znode.parent` 配置的 znode（默认 `/hbase`）。\n\n :::note 注意\n 此操作将会删除原有 HBase 上面的所有数据\n :::\n\n2. 使用新的 znode\n\n 保留原 HBase 集群的 znode，以便后续可以恢复。然后为 `zookeeper.znode.parent` 配置一个新的值：\n\n ```xml title=\"hbase-site.xml\"\n \n zookeeper.znode.parent\n /hbase-jfs\n \n ```\n\n### 重启服务\n\n当需要使用以下组件访问 JuiceFS 数据时，需要重启相关服务。\n\n:::note 注意\n在重启之前需要保证 JuiceFS 配置已经写入配置文件，通常可以查看机器上各组件配置的 `core-site.xml` 里面是否有 JuiceFS 相关配置。\n:::\n\n| 组件名 | 服务名 |\n| ------ | -------------------------- |\n| Hive | HiveServer
Metastore |\n| Spark | ThriftServer |\n| Presto | Coordinator
Worker |\n| Impala | Catalog Server
Daemon |\n| HBase | Master
RegionServer |\n\nHDFS、Hue、ZooKeeper 等服务无需重启。\n\n若访问 JuiceFS 出现 `Class io.juicefs.JuiceFileSystem not found` 或 `No FilesSystem for scheme: jfs` 错误，请参考 [FAQ](#faq)。\n\n### 回收站\n\nJuiceFS Hadoop Java SDK 同样也有和 HDFS 一样的回收站功能，需要通过设置 `fs.trash.interval` 和 `fs.trash.checkpoint.interval` 开启，请参考 [HDFS 文档](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html#File_Deletes_and_Undeletes)了解更多信息。\n\n## 环境验证\n\nJuiceFS Java 客户端部署完成以后，可以采用以下方式验证部署是否成功。\n\n### Hadoop CLI\n\n```bash\nhadoop fs -ls jfs://{JFS_NAME}/\n```\n\n:::info 说明\n这里的 `JFS_NAME` 是创建 JuiceFS 文件系统时指定的名称。\n:::\n\n### Hive\n\n```sql\nCREATE TABLE IF NOT EXISTS person\n(\n name STRING,\n age INT\n) LOCATION 'jfs://{JFS_NAME}/tmp/person';\n```\n\n### Java/Scala 项目\n\n1. 新增 Maven 或 Gradle 依赖：\n\n \n \n\n ```xml\n \n org.apache.hadoop\n hadoop-common\n {HADOOP_VERSION}\n provided\n \n \n io.juicefs\n juicefs-hadoop\n {JUICEFS_HADOOP_VERSION}\n provided\n \n ```\n\n \n \n\n ```groovy\n dependencies {\n implementation 'org.apache.hadoop:hadoop-common:${hadoopVersion}'\n implementation 'io.juicefs:juicefs-hadoop:${juicefsHadoopVersion}'\n }\n ```\n\n \n \n\n2. 使用以下示例代码验证：\n\n\n ```java\n package demo;\n\n import org.apache.hadoop.conf.Configuration;\n import org.apache.hadoop.fs.FileStatus;\n import org.apache.hadoop.fs.FileSystem;\n import org.apache.hadoop.fs.Path;\n\n public class JuiceFSDemo {\n public static void main(String[] args) throws Exception {\n Configuration conf = new Configuration();\n conf.set(\"fs.jfs.impl\", \"io.juicefs.JuiceFileSystem\");\n conf.set(\"juicefs.meta\", \"redis://127.0.0.1:6379/0\"); // JuiceFS 元数据引擎地址\n Path p = new Path(\"jfs://{JFS_NAME}/\"); // 请替换 {JFS_NAME} 为正确的值\n FileSystem jfs = p.getFileSystem(conf);\n FileStatus[] fileStatuses = jfs.listStatus(p);\n // 遍历 JuiceFS 文件系统并打印文件路径\n for (FileStatus status : fileStatuses) {\n System.out.println(status.getPath());\n }\n }\n }\n ```\n\n\n## 监控指标收集\n\n请查看[「监控」](../administration/monitoring.md)文档了解如何收集及展示 JuiceFS 监控指标\n\n## 从 HDFS 迁移数据到 JuiceFS\n\n从 HDFS 迁移数据到 JuiceFS，一般是使用 DistCp 来拷贝数据，它支持数据校验 (Checksum) 来保证数据的正确性。\n\nDistCp 是使用 HDFS 的 `getFileChecksum()` 接口来获得文件的校验码，然后对比拷贝后的文件的校验码来确保数据是一样的。\n\nHadoop 默认使用的 Checksum 算法是 MD5-MD5-CRC32, 严重依赖 HDFS 的实现细节。它是根据文件目前的分块形式，使用 MD5-CRC32 算法汇总每一个数据块的 Checksum（把每一个 64K 的 block 的 CRC32 校验码汇总，再算一个 MD5），然后再用 MD5 计算校验码。如果 HDFS 集群的分块大小不同，就没法用这个算法进行比较。\n\n为了兼容 HDFS，JuiceFS 也实现了该 MD5-MD5-CRC32 算法，它会将文件的数据读一遍，用同样的算法计算得到一个 checksum，用于比较。\n\n因为 JuiceFS 是基于对象存储实现的，后者已经通过多种 Checksum 机制保证了数据完整性，JuiceFS 默认没有启用上面的 Checksum 算法，需要通过 `juicefs.file.checksum` 配置来启用。\n\n因为该算法依赖于相同的分块大小，需要通过 `juicefs.block.size` 配置将分块大小设置为跟 HDFS 一样（默认值是 `dfs.blocksize`，它的默认值是 128MB）。\n\n另外，HDFS 里支持给每一个文件设置不同的分块大小，而 JuiceFS 不支持，如果启用 Checksum 校验的话会导致拷贝部分文件失败（因为分块大小不同），JuiceFS Hadoop Java SDK 对 DistCp 打了一个热补丁（需要 `tools.jar`）来跳过这些分块不同的文件（不做比较，而不是抛异常）。\n\n## 基准测试\n\n以下提供了一系列方法，使用 JuiceFS 客户端内置的压测工具，对已经成功部署了客户端环境进行性能测试。\n\n### 1. 本地测试\n\n#### 元数据性能\n\n- **create**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench create -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n 此命令会 create 10000 个空文件\n\n- **open**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench open -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n 此命令会 open 10000 个文件，并不读取数据\n\n- **rename**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench rename -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n- **delete**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench delete -files 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench -local\n ```\n\n- **参考值**\n\n | 操作 | TPS | 时延（ms） |\n | ------ | ---- | ---- |\n | create | 644 | 1.55 |\n | open | 3467 | 0.29 |\n | rename | 483 | 2.07 |\n | delete | 506 | 1.97 |\n\n#### I/O 性能\n\n- **顺序写**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -write -size 20000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO -local\n ```\n\n- **顺序读**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -read -size 20000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO -local\n ```\n\n 如果多次运行此命令，可能会出现数据被缓存到了系统缓存而导致读取速度非常快，只需清除 JuiceFS 的本地磁盘缓存即可\n\n- **参考值**\n\n | 操作 | 吞吐（MB/s） |\n | ------ | ---- |\n | write | 647 |\n | read | 111 |\n\n如果机器的网络带宽比较低，则一般能达到网络带宽瓶颈\n\n### 2. 分布式测试\n\n以下命令会启动 MapReduce 分布式任务程序对元数据和 IO 性能进行测试，测试时需要保证集群有足够的资源能够同时启动所需的 map 任务。\n\n本项测试使用的计算资源：\n\n- **服务器**：3 台 4 核 32 GB 内存的云服务器，突发带宽 5Gbit/s。\n- **数据库**：阿里云 Redis 5.0 社区 4G 主从版\n\n#### 元数据性能\n\n- **create**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench create -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 此命令会启动 10 个 map task，每个 task 有 10 个线程，每个线程会创建 1000 个空文件，总共 100000 个空文件\n\n- **open**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench open -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 此命令会启动 10 个 map task，每个 task 有 10 个线程，每个线程会 open 1000 个文件，总共 open 100000 个文件\n\n- **rename**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench rename -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 此命令会启动 10 个 map task，每个 task 有 10 个线程，每个线程会 rename 1000 个文件，总共 rename 100000 个文件\n\n- **delete**\n\n ```shell\n hadoop jar juicefs-hadoop.jar nnbench delete -maps 10 -threads 10 -files 1000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/NNBench\n ```\n\n 此命令会启动 10 个 map task，每个 task 有 10 个线程，每个线程会 delete 1000 个文件，总共 delete 100000 个文件\n\n- **参考值**\n\n - 10 并发\n\n | 操作 | IOPS | 时延（ms） |\n | ------ | ---- | ---- |\n | create | 4178 | 2.2 |\n | open | 9407 | 0.8 |\n | rename | 3197 | 2.9 |\n | delete | 3060 | 3.0 |\n\n - 100 并发\n\n | 操作 | IOPS | 时延（ms） |\n | ------ | ---- | ---- |\n | create | 11773 | 7.9 |\n | open | 34083 | 2.4 |\n | rename | 8995 | 10.8 |\n | delete | 7191 | 13.6 |\n\n#### I/O 性能\n\n- **连续写**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -write -maps 10 -size 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO\n ```\n\n 此命令会启动 10 个 map task，每个 task 写入 10000MB 的数据\n\n- **连续读**\n\n ```shell\n hadoop jar juicefs-hadoop.jar dfsio -read -maps 10 -size 10000 -baseDir jfs://{JFS_NAME}/tmp/benchmarks/DFSIO\n ```\n\n 此命令会启动 10 个 map task，每个 task 读取 10000MB 的数据\n\n- **参考值**\n\n | 操作 | 平均吞吐（MB/s） | 总吞吐（MB/s） |\n | ------ | ---- | ---- |\n | write | 198 | 1835 |\n | read | 124 | 1234 |\n\n### 3. TPC-DS\n\n测试数据集 100GB 规模，测试 Parquet 和 ORC 两种文件格式。\n\n本次测试仅测试前 10 个查询。\n\n使用 Spark Thrift JDBC/ODBC Server 开启 Spark 常驻进程，然后通过 Beeline 连接提交任务。\n\n#### 测试硬件\n\n| 节点类型 | 机器型号 | CPU | 内存 | 磁盘 | 数量 |\n| ------ | ------------------- | ---- | ------ | ---------------------------------- | ---- |\n| Master | 阿里云 ecs.r6.xlarge | 4 | 32GiB | 系统盘：100GiB | 1 |\n| Core | 阿里云 ecs.r6.xlarge | 4 | 32GiB | 系统盘：100GiB
数据盘：500GiB 高效云盘 x 2 | 3 |\n\n#### 软件配置\n\n##### Spark Thrift JDBC/ODBC Server\n\n```shell\n${SPARK_HOME}/sbin/start-thriftserver.sh \\\n --master yarn \\\n --driver-memory 8g \\\n --executor-memory 10g \\\n --executor-cores 3 \\\n --num-executors 3 \\\n --conf spark.locality.wait=100 \\\n --conf spark.sql.crossJoin.enabled=true \\\n --hiveconf hive.server2.thrift.port=10001\n```\n\n##### JuiceFS 缓存配置\n\nCore 节点的 2 块数据盘挂载在 `/data01` 和 `/data02` 目录下，`core-site.xml` 配置如下：\n\n```xml\n\n juicefs.cache-size\n 200000\n\n\n juicefs.cache-dir\n /data*/jfscache\n\n\n juicefs.cache-full-block\n false\n\n\n juicefs.discover-nodes-url\n yarn\n\n\n juicefs.attr-cache\n 3\n\n\n juicefs.entry-cache\n 3\n\n\n juicefs.dir-entry-cache\n 3\n\n```\n\n#### 测试\n\n任务提交的命令如下：\n\n```shell\n${SPARK_HOME}/bin/beeline -u jdbc:hive2://localhost:10001/${DATABASE} \\\n -n hadoop \\\n -f query{i}.sql\n```\n\n#### 结果\n\nJuiceFS 可以使用本地磁盘作为缓存加速数据访问，以下数据是分别使用 Redis 和 TiKV 作为 JuiceFS 的元数据引擎跑 4 次后的结果（单位秒）。\n\n##### ORC\n\n| Queries | JuiceFS (Redis) | JuiceFS (TiKV) | HDFS |\n| ------- | --------------- | -------------- | ---- |\n| q1 | 20 | 20 | 20 |\n| q2 | 28 | 33 | 26 |\n| q3 | 24 | 27 | 28 |\n| q4 | 300 | 309 | 290 |\n| q5 | 116 | 117 | 91 |\n| q6 | 37 | 42 | 41 |\n| q7 | 24 | 28 | 23 |\n| q8 | 13 | 15 | 16 |\n| q9 | 87 | 112 | 89 |\n| q10 | 23 | 24 | 22 |\n\n![orc](../images/spark_ql_orc.png)\n\n##### Parquet\n\n| Queries | JuiceFS (Redis) | JuiceFS (TiKV) | HDFS |\n| ------- | --------------- | -------------- | ---- |\n| q1 | 33 | 35 | 39 |\n| q2 | 28 | 32 | 31 |\n| q3 | 23 | 25 | 24 |\n| q4 | 273 | 284 | 266 |\n| q5 | 96 | 107 | 94 |\n| q6 | 36 | 35 | 42 |\n| q7 | 28 | 30 | 24 |\n| q8 | 11 | 12 | 14 |\n| q9 | 85 | 97 | 77 |\n| q10 | 24 | 28 | 38 |\n\n![parquet](../images/spark_sql_parquet.png)\n\n## 使用 Apache Ranger 进行权限管控（ v1.3支持 ）\n\nJuiceFS 当前支持对接 Apache Ranger 的 `HDFS` 模块进行路径的权限管控。仅 Hadoop Java SDK 支持该功能。\n\n### 1. 相关配置\n\nApache Ranger 的配置统一放在 META 数据库内。可以通过以下方法开启 Ranger 的权限管控：\n\n```shell\n# format 的时候指定 ranger 配置\njuicefs format META-URL NAME --ranger-rest-url http://localhost:6080 --ranger-service jfs\n\n# 已有的文件系统增加 ranger 配置\njuicefs config META-URL --ranger-rest-url http://localhost:6080 --ranger-service jfs\n\n# 关闭 ranger\njuicefs config META-URL --ranger-rest-url \"\" --ranger-service jfs \"\"\n```\n\n### 2. 环境及依赖\n\n考虑到使用的方便性，JuiceFS 将 Ranger 所有依赖的包均打包到 JuiceFS 的 SDK 中。如果遇到 Apache Ranger 的版本冲突问题，可能需要修改版本重新编译。\n\n### 3. 使用提示\n\n#### 3.1 Ranger版本\n\n当前代码测试基于`Ranger2.3`和`Ranger2.4`版本，因除`HDFS`模块鉴权外并未使用其他特性，理论上其他版本均适用。\n\n#### 3.2 Ranger Audit\n\n当前仅支持鉴权功能，`Ranger Audit`功能已关闭。\n\n#### 3.3 Ranger其他参数\n\n为提升使用效率，当前仅开放连接 Ranger 最核心的参数。\n\n#### 3.4 安全性问题\n\n因项目代码完全开源，无法避免用户通过替换`ranger-rest-url`等参数的方式扰乱安全管控。如需更严格的管控，建议自主编译代码，通过将相关安全参数进行加密处理等方式解决。\n\n## FAQ\n\n### 1. 出现 `Class io.juicefs.JuiceFileSystem not found` 异常\n\n出现这个异常的原因是 `juicefs-hadoop.jar` 没有被加载，可以用 `lsof -p {pid} | grep juicefs` 查看 JAR 文件是否被加载。需要检查 JAR 文件是否被正确地放置在各个组件的 classpath 里面，并且保证 JAR 文件有可读权限。\n\n另外，在某些发行版 Hadoop 环境中，需要修改 `mapred-site.xml` 中的 `mapreduce.application.classpath` 参数，添加 `juicefs-hadoop.jar` 的路径。\n\n### 2. 出现 `No FilesSystem for scheme: jfs` 异常\n\n出现这个异常的原因是 `core-site.xml` 配置文件中的 JuiceFS 配置没有被读取到，需要检查组件配置的 `core-site.xml` 中是否有 JuiceFS 相关配置。\n\n### 3. JuiceFS 与 HDFS 的用户权限管理有何相同和不同之处？\n\nJuiceFS 也是使用「用户／用户组」的方式管理文件权限，默认使用的是本地的用户和用户组。为了保证分布式计算时不同节点的权限统一，可以通过 `juicefs.users` 和 `juicefs.groups` 配置全局的「用户／UID」和「用户组／GID」映射。\n\n### 4. 数据删除后都是直接存储在 JuiceFS 的 `.trash` 目录，虽然文件都在但是很难像 HDFS 那样简单通过 `mv` 命令就能恢复数据，是否有某种办法可以达到类似 HDFS 回收站的效果？\n\n在 Hadoop 应用场景下，仍然保留了类似于 HDFS 回收站的功能。需要通过 `fs.trash.interval` 以及 `fs.trash.checkpoint.interval` 配置来显式开启，请参考[文档](#回收站)了解更多信息。\n\n### 5. 设置 `juicefs.discover-nodes-url` 这个参数有什么好处？\n\n在 HDFS 里面，每个数据块会有 [`BlockLocation`](https://hadoop.apache.org/docs/current/api/org/apache/hadoop/fs/BlockLocation.html) 信息，计算引擎会利用此信息尽量将计算任务调度到数据所存储的节点。JuiceFS 会通过一致性哈希算法为每个数据块计算出对应的 `BlockLocation`，这样第二次读取相同的数据时，计算引擎有可能将计算任务调度到相同的机器上，就可以利用第一次计算时缓存在本地磁盘的数据来加速数据访问。\n\n此算法需要事先知道所有的计算节点信息，`juicefs.discover-nodes-url` 参数就是用来获得这些计算节点信息的。\n\n### 6. 对于采用 Kerberos 认证的 CDH 集群，社区版 JuiceFS 目前能否支持呢？\n\n不支持。JuiceFS 不会校验 Kerberos 用户的合法性，但是可以使用通过 Kerberos 认证的用户名。\n" }, { "path": "docs/zh_cn/deployment/how_to_use_on_kubernetes.md", "content": "---\ntitle: Kubernetes 使用 JuiceFS\nsidebar_position: 2\nslug: /how_to_use_on_kubernetes\n---\n\nJuiceFS 非常适合用作 Kubernetes 集群的存储层，阅读本文以了解如何使用。\n\n## 以 `hostPath` 方式挂载 JuiceFS\n\n如果你仅仅需要在 Kubernetes 容器中简单使用 JuiceFS，没有其他任何复杂要求（比如隔离性、权限控制），那么完全可以以 [`hostPath` 卷](https://kubernetes.io/zh-cn/docs/concepts/storage/volumes/#hostpath) 的方式使用 JuiceFS，搭建起来也十分简单：\n\n1. 在 Kubernetes 节点上统一安装、挂载 JuiceFS，如果节点众多，考虑[自动化部署](./automation.md)。\n1. 在 pod 定义中使用 `hostPath` 卷，直接将宿主机上的 JuiceFS 子目录挂载到容器中：\n\n ```yaml {8-16}\n apiVersion: v1\n kind: Pod\n metadata:\n name: juicefs-app\n spec:\n containers:\n - ...\n volumeMounts:\n - name: jfs-data\n mountPath: /opt/app-data\n volumes:\n - name: jfs-data\n hostPath:\n # 假设挂载点为 /jfs\n path: \"/jfs/myapp/\"\n type: Directory\n ```\n\n相比以 CSI 驱动的方式来使用 JuiceFS，`hostPath` 更为简单直接，出问题也更易排查，但也要注意：\n\n* 为求管理方便，一般所有容器都在使用同一个宿主机挂载点，缺乏隔离可能导致数据安全问题，未来也无法在不同应用中单独调整 JuiceFS 挂载参数。请谨慎评估。\n* 所有节点都需要提前挂载 JuiceFS，因此集群加入新节点，需要在初始化流程里进行安装和挂载，否则新节点没有 JuiceFS 挂载点，容器将无法创建。\n* 宿主机上的 JuiceFS 挂载进程所占用的系统资源（如 CPU、内存等）不受 Kubernetes 控制，有可能占用较多宿主机资源。可以考虑用 [`system-reserved`](https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/reserve-compute-resources/#system-reserved) 来适当调整 Kubernetes 的系统资源预留值，为 JuiceFS 挂载进程预留更多资源。\n* 如果宿主机上的 JuiceFS 挂载进程意外退出，将会导致应用 pod 无法正常访问挂载点，此时需要重新挂载 JuiceFS 文件系统并重建应用 pod。作为对比，JuiceFS CSI 驱动提供[「挂载点自动恢复」](https://juicefs.com/docs/zh/csi/recover-failed-mountpoint)功能来解决这个问题。\n* 如果你使用 Docker 作为 Kubernetes 容器运行环境，最好令 JuiceFS 先于 Docker 启动，否则在节点重启的时候，偶尔可能出现容器启动时，JuiceFS 尚未挂载好的情况，此时便会因该依赖问题启动失败。以 systemd 为例，可以用下方 unit file 来配置启动顺序：\n\n ```systemd title=\"/etc/systemd/system/docker.service.d/override.conf\"\n [Unit]\n # 请使用下方命令确定 JuiceFS 挂载服务的名称（例如 jfs.mount）：\n # systemctl list-units | grep \"\\.mount\"\n After=network-online.target firewalld.service containerd.service jfs.mount\n ```\n\n## JuiceFS CSI 驱动\n\n在 Kubernetes 中使用 JuiceFS，请阅读[「JuiceFS CSI 驱动文档」](https://juicefs.com/docs/zh/csi/introduction)。\n\n## 在容器中挂载 JuiceFS\n\n某些情况下，你可能需要在容器中直接挂载 JuiceFS 存储，这需要在容器中使用 JuiceFS 客户端，你可以参考以下 `Dockerfile` 样本将 JuiceFS 客户端集成到应用镜像：\n\n```dockerfile title=\"Dockerfile\"\nFROM alpine:latest\nLABEL maintainer=\"Juicedata \"\n\n# Install JuiceFS client\nRUN apk add --no-cache curl && \\\n JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '\"' -f 4 | tr -d 'v') && \\\n wget \"https://github.com/juicedata/juicefs/releases/download/v${JFS_LATEST_TAG}/juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" && \\\n tar -zxf \"juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" && \\\n install juicefs /usr/bin && \\\n rm juicefs \"juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" && \\\n rm -rf /var/cache/apk/* && \\\n apk del curl\n\nENTRYPOINT [\"/usr/bin/juicefs\", \"mount\"]\n```\n\n由于 JuiceFS 需要使用 FUSE 设备挂载文件系统，因此在创建 Pod 时需要允许容器在特权模式下运行：\n\n```yaml {19-20}\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-run\nspec:\n selector:\n matchLabels:\n app: nginx\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: linuxserver/nginx\n ports:\n - containerPort: 80\n securityContext:\n privileged: true\n```\n\n:::caution 注意\n容器启用 `privileged: true` 特权模式以后，就具备了访问宿主机所有设备的权限，即拥有了对宿主机内核的完全控制权限。使用不当会带来严重的安全隐患，请您在使用此方式之前进行充分的安全评估。\n:::\n" }, { "path": "docs/zh_cn/deployment/juicefs_on_docker.md", "content": "---\ntitle: 在 Docker 中使用 JuiceFS\nsidebar_position: 6\nslug: /juicefs_on_docker\ndescription: 在 Docker 中以不同方式使用 JuiceFS，包括卷映射、卷插件，以及容器中挂载。\n---\n\n在 Docker 中使用 JuiceFS 文件系统，可以通过卷插件或直接在容器中运行客户端。\n\n## 使用卷插件 {#volume-plugin}\n\n如果你对挂载管理有一定要求，比如希望通过 Docker 来管理挂载点，方便不同的应用容器使用不同的 JuiceFS 文件系统，则可以使用[卷插件](https://github.com/juicedata/docker-volume-juicefs)（Docker volume plugin）。\n\nDocker 插件通常是以镜像形式提供的，[JuiceFS 卷插件镜像](https://hub.docker.com/r/juicedata/juicefs)中内置了 [JuiceFS 社区版](../introduction/README.md)和 [JuiceFS 云服务](https://juicefs.com/docs/zh/cloud)客户端，安装以后，便能够运行卷插件，在 Docker 中创建 JuiceFS Volume。\n\n通过下面的命令安装插件，按照提示为 FUSE 提供必要的权限：\n\n```shell\ndocker plugin install juicedata/juicefs\n```\n\n你可以使用以下命令管理卷插件：\n\n```shell\n# 停用插件\ndocker plugin disable juicedata/juicefs\n\n# 升级插件（需先停用）\ndocker plugin upgrade juicedata/juicefs\ndocker plugin enable juicedata/juicefs\n\n# 卸载插件\ndocker plugin rm juicedata/juicefs\n```\n\n### 创建存储卷 {#create-volume}\n\n请将以下命令中的 ``、``、``、``、``、`` 替换成你自己的文件系统配置。\n\n```shell\ndocker volume create -d juicedata/juicefs \\\n -o name= \\\n -o metaurl= \\\n -o storage= \\\n -o bucket= \\\n -o access-key= \\\n -o secret-key= \\\n jfsvolume\n```\n\n对于已经预先创建好的文件系统，在用其创建卷插件时，只需指定文件系统名称和数据库地址，例如：\n\n```shell\ndocker volume create -d juicedata/juicefs \\\n -o name= \\\n -o metaurl= \\\n jfsvolume\n```\n\n如果需要在挂载文件系统时传入额外的环境变量（比如 [Google 云](../reference/how_to_set_up_object_storage.md#google-cloud)），可以对上方命令追加类似 `-o env=FOO=bar,SPAM=egg` 的参数。\n\n### 使用和管理 {#usage-and-management}\n\n```shell\n# 创建容器时挂载卷\ndocker run -it -v jfsvolume:/opt busybox ls /opt\n\n# 卸载后，可以操作删除存储卷，注意这仅仅是删除 Docker 中的对应资源，并不影响 JuiceFS 中存储的数据\ndocker volume rm jfsvolume\n```\n\n### 在 Docker Compose 中使用卷插件 {#using-plugin-in-docker-compose}\n\n下面是在 `docker compose` 中使用 JuiceFS 卷插件的示例：\n\n```yaml\nversion: '3'\nservices:\nbusybox:\n image: busybox\n command: \"ls /jfs\"\n volumes:\n - jfsvolume:/jfs\nvolumes:\n jfsvolume:\n driver: juicedata/juicefs\n driver_opts:\n name: ${VOL_NAME}\n # 因为 SQLite 在插件容器本地路径创建数据库文件，\n # sqlite:// 将在服务重启时失败。\n # （详见 https://github.com/juicedata/docker-volume-juicefs/issues/37）\n metaurl: ${META_URL}\n storage: ${STORAGE_TYPE}\n bucket: ${BUCKET}\n access-key: ${ACCESS_KEY}\n secret-key: ${SECRET_KEY}\n # 如有需要，可以用 env 传入额外环境变量\n # env: FOO=bar,SPAM=egg\n```\n\n使用和管理：\n\n```shell\n# 启动服务\ndocker-compose up\n\n# 关闭服务并从 Docker 中卸载 JuiceFS 文件系统\ndocker-compose down --volumes\n```\n\n### 卷插件问题排查 {#troubleshooting}\n\n无法正常工作时，推荐先[升级卷插件](#volume-plugin)，然后根据问题情况查看日志。\n\n* 收集 JuiceFS 客户端日志，日志位于 Docker volume plugin 容器内，需要进入容器采集：\n\n ```shell\n # 确认 docker plugins runtime 目录，根据实际情况可能与下方示范不同\n # ls 打印出来的目录就是容器目录，名称为容器 ID\n ls /run/docker/plugins/runtime-root/plugins.moby\n\n # 打印 plugin 容器信息\n # 如果打印出的容器列表为空，说明 plugin 容器创建失败\n # 阅读下方查看 plugin 启动日志继续排查\n runc --root /run/docker/plugins/runtime-root/plugins.moby list\n\n # 进入容器，打印日志\n runc --root /run/docker/plugins/runtime-root/plugins.moby exec 452d2c0cf3fd45e73a93a2f2b00d03ed28dd2bc0c58669cca9d4039e8866f99f cat /var/log/juicefs.log\n ```\n\n 如果发现容器不存在（`ls` 发现目录为空），或者在最后打印日志的阶段发现 `juicefs.log` 不存在，那么多半是挂载本身就失败了，继续查看 plugin 自身的日志寻找原因。\n\n* 收集 plugin 日志，以 systemd 为例：\n\n ```shell\n journalctl -f -u docker | grep \"plugin=\"\n ```\n\n 如果 plugin 调用 `juicefs` 发生错误，或者 plugin 自身报错，均会在日志里有所体现。\n\n## 在容器中使用 JuiceFS 客户端 {#mount-juicefs-in-docker}\n\n相比卷插件，直接在容器中使用 JuiceFS 客户端更加灵活，可以在容器中直接挂载 JuiceFS 文件系统，也可以通过 S3 Gateway、WebDAV 开放文件系统访问。\n\n### 方式一：自行构建镜像\n\nJuiceFS 客户端是一个独立的二进制程序，同时提供 AMD64 和 ARM64 架构的版本，可以在 Dockerfile 中定义下载安装 JuiceFS 客户端的命令，例如：\n\n```Dockerfile\nFROM ubuntu:22.04\n...\n# 使用官方一键安装脚本\nRUN curl -sSL https://d.juicefs.com/install | sh - \n```\n\n更多内容详见[「定制容器镜像」](https://juicefs.com/docs/zh/csi/guide/custom-image)。\n\n### 方式二：使用官方维护的镜像\n\nJuiceFS 官方维护的镜像 [`juicedata/mount`](https://hub.docker.com/r/juicedata/mount) ，可以通过 tag 指定所需要的版本。**社区版 tag 为 ce**，例如：latest、ce-v1.1.2、ce-nightly。`latest` 标签仅包含最新的社区版，`nightly` 标签指向最新的开发版本，详情查看 [Docker hub 的 tags 页面](https://hub.docker.com/r/juicedata/mount/tags)。\n\n开始之前，你需要先准备好[对象存储](../reference/how_to_set_up_object_storage.md)和[元数据引擎](../reference/how_to_set_up_metadata_engine.md)。\n\n#### 创建文件系统\n\n通过一个临时容器创建文件系统，例如：\n\n```sh\ndocker run --rm \\\n juicedata/mount:ce-v1.1.2 juicefs format \\\n --storage s3 \\\n --bucket https://xxx.your-s3-endpoint.com \\\n --access-key=ACCESSKEY \\\n --secret-key=SECRETKEY \\\n rediss://user:password@xxx.your-redis-server.com:6379/1 myjfs\n```\n\n请将 `--storage`、`--bucket`、`--access-key`、`--secret-key` 以及元数据引擎的 URL 替换成你自己的配置。\n\n#### 直接在容器中挂载文件系统\n\n创建一个容器并将 JuiceFS 文件系统到挂载到容器中，例如：\n\n```sh\ndocker run --privileged --name myjfs \\\n juicedata/mount:ce-v1.1.2 juicefs mount \\\n rediss://user:password@xxx.your-redis-server.com:6379/1 /mnt\n```\n\n请将元数据引擎的 URL 替换成你自己的配置，`/mnt` 是挂载点，可以根据需要修改。由于需要使用 FUSE，所以还需要 `--privileged` 权限。\n\n#### 通过 Docker Compose 挂载文件系统\n\n下面是一个使用 Docker Compose 的示例，请将元数据引擎的 URL 和挂载点替换成你自己的配置。\n\n```yaml\nversion: \"3\"\nservices:\n busybox:\n image: busybox\n command: \"ls /jfs\"\n volumes:\n - ./mnt:/jfs\n depends_on:\n juicefs:\n condition: service_healthy\n\n juicefs:\n image: juicedata/mount:ce-v1.1.2\n container_name: myjfs\n volumes:\n - ./mnt:/mnt:rw,rshared\n cap_add:\n - SYS_ADMIN\n devices:\n - /dev/fuse\n security_opt: \n - apparmor:unconfined\n command: [\"juicefs\", \"mount\", \"rediss://user:password@xxx.your-redis-server.com:6379/1\", \"/mnt\"]\n restart: unless-stopped\n healthcheck:\n test: [\"CMD-SHELL\", \"cat /mnt/.control\"]\n interval: 60s\n retries: 5\n start_period: 30s\n timeout: 10s\n```\n\n在容器中，JuiceFS 文件系统挂载到了 `/mnt` 目录，又通过配置文件中的 volumes 部分将容器中的 `/mnt` 映射到宿主机的 `./mnt` 目录，这样就可以实现在宿主机直接访问容器中挂载的 JuiceFS 文件系统。同时通过 depends_on 和 volumes 的结合可以将目录再次挂载进其余容器中使用\n\n#### 通过 S3 Gateway 开放文件系统访问\n\n下面是一个将 JuiceFS 以 S3 Gateway 方式开放访问的示例，请将 `MINIO_ROOT_USER`、`MINIO_ROOT_PASSWORD`、元数据引擎的 URL、监听的地址和端口号替换成你自己的配置。\n\n```yaml\nversion: \"3\"\nservices:\n s3-gateway:\n image: juicedata/mount:ce-v1.1.2\n container_name: juicefs-s3-gateway\n environment:\n - MINIO_ROOT_USER=your-username\n - MINIO_ROOT_PASSWORD=your-password\n ports:\n - \"9090:9090\"\n command: [\"juicefs\", \"gateway\", \"rediss://user:password@xxx.your-redis-server.com:6379/1\", \"0.0.0.0:9090\"]\n restart: unless-stopped\n```\n\n使用宿主机的 `9090` 端口即可打开 S3 Gateway 的控制台，用相同的地址通过 S3 客户端或者 SDK 读写 JuiceFS 文件系统。\n" }, { "path": "docs/zh_cn/deployment/nfs.md", "content": "---\ntitle: 创建 NFS 共享\nsidebar_position: 9\ndescription: 本文介绍如何通过 NFS 共享 JuiceFS 文件系统中的目录。\n---\n\nNFS（Network File System）是一种网络文件共享协议，允许不同计算机之间通过网络共享文件和目录。它最初由 Sun Microsystems 开发，是一种在 Unix 和类 Unix 系统之间进行文件共享的标准方式。NFS 协议允许客户端像访问本地文件系统一样访问远程文件系统，从而实现透明的远程文件访问。\n\n当需要将 JuiceFS 文件系统中的目录通过 NFS 共享时，只需使用 `juicefs mount` 命令挂载，然后使用 JuiceFS 挂载点或子目录创建 NFS 共享即可。\n\n:::note\n`juicefs mount` 以 FUSE 接口的形式挂载为本地的用户态文件系统，与本地文件系统在形态和用法上无异，因此可以直接被用于创建 NFS 共享。\n:::\n\n## 第 1 步：安装 NFS\n\n配置 NFS 共享需要分别在服务端和客户端安装相应的软件包，以 Ubuntu/Debian 系统为例：\n\n### 1. 服务端安装\n\n创建 NFS 共享的主机（JuiceFS 文件系统也挂载在该服务器上）。\n\n```shell\nsudo apt install nfs-kernel-server\n```\n\n### 2. 客户端安装\n\n所有需要访问 NFS 的 Linux 主机都需要安装客户端。\n\n```shell\nsudo apt install nfs-common\n```\n\n## 第 2 步：创建共享\n\n这里假设 JuiceFS 在服务端系统的挂载点是 `/mnt/myjfs`，比如要将其中的 `media` 子目录设置为 NFS 共享，可以在服务端系统的 `/etc/exports` 文件中添加如下配置：\n\n```\n\"/mnt/myjfs/media\" *(rw,sync,no_subtree_check,fsid=1)\n```\n\nNFS 共享配置的语法为：\n\n```\n (options)\n```\n\n比如要将这个共享设置为仅允许 `192.168.1.0/24` 这个 IP 段的主机挂载且避免挤压 root 权限，则可以修改为：\n\n```\n\"/mnt/myjfs/media\" 192.168.1.0/24(rw,async,no_subtree_check,no_root_squash,fsid=1)\n```\n\n### 共享选项说明\n\n**其中涉及的共享选项：**\n\n- `rw`：代表允许读和写，如果只允许读则使用 `ro`。\n- `sync` 与 `async`：`sync` 为同步写入，当向 NFS 共享写入文件时，客户端会等待服务端确认数据写入成功后再进行后续操作。`async` 为异步写入，写入操作是异步的，在写数据到 NFS 共享时，客户端不会等待服务器确认是否成功写入，而是立即执行后续操作。\n- `no_subtree_check`：禁用子目录检查，这将允许客户端挂载共享目录的父目录和子目录，会降低一些安全性但能提高 NFS 的兼容性。也可以设置为 `subtree_check` 来启用子目录检查，这样仅允许客户端挂载共享目录和它的子目录。\n- `no_root_squash`：用于控制客户端 root 用户访问 NFS 共享时的身份映射行为。默认情况下，客户端以 root 身份挂载 NFS 共享时，服务端会将其映射为非特权用户（通常是 nobody 或 nfsnobody），这被称为 root 挤压。设置该选项后，则取消这种权限挤压，从而让客户端拥有服务端相同的 root 用户权限。该选项有一定安全风险，建议谨慎使用。\n- `fsid`：文件系统标识符，用于在 NFS 上标识不同的文件系统。在 NFSv4 中，NFS 的根目录所在的文件系统被定义为 fsid=0，其他文件系统需要在它之下且编号唯一。在这里，JuiceFS 就是一个外挂的 FUSE 文件系统，因此需要给它设置一个唯一的标识。\n\n### async 与 sync 模式的选择\n\n对于 NFS 共享而言，sync（同步写入）模式可以提高数据的可靠性，但总是需要等待服务器确认成功写入才会执行下一个操作，这势必会导致写入速度降低。对于 JuiceFS 这种基于云上对象存储的文件系统，还需要进一步考虑网络延时的影响，使用 sync 模式往往会导致较低的写入性能。\n\n通常情况下，在使用 JuiceFS 创建 NFS 共享时，建议将写入模式设置为 async（异步写入），从而避免损失写入性能。如果为了保证数据可靠性而必须使用 sync 模式时，建议为 JuiceFS 设置容量充足的高性能 SSD 磁盘作为本地缓存，并开启 writeback 写缓存模式。\n" }, { "path": "docs/zh_cn/deployment/production_deployment_recommendations.md", "content": "---\nsidebar_position: 1\nslug: /production_deployment_recommendations\ndescription: 本文面向即将把 JuiceFS 部署到生产环境的用户参考，提供一系列环境配置建议。\n---\n\n# 生产环境部署建议\n\n本文档提供在生产环境中部署 JuiceFS 社区版的建议，主要涉及监控指标收集、元数据自动备份、回收站配置、客户端后台任务、客户端日志滚动和命令行自动补全等方面，以确保文件系统的稳定性和可靠性。\n\n## 监控指标收集与可视化\n\n务必收集 JuiceFS 客户端的监控指标，并通过 Grafana 可视化，以便实时监控文件系统的性能和健康状态。具体请参考[文档](../administration/monitoring.md)。\n\n## 元数据自动备份\n\n:::tip 提示\n元数据自动备份是自 JuiceFS v1.0.0 版本开始加入的特性\n:::\n\n元数据对 JuiceFS 文件系统非常关键，一旦丢失或损坏将可能影响大批文件甚至整个文件系统。因此必须对元数据进行定期备份。\n\n元数据自动备份特性默认开启，备份间隔为 1 小时，备份的元数据会经过压缩后存储至对应的对象存储中（与文件系统的数据隔离）。备份由 JuiceFS 客户端执行，备份期间会导致其 CPU 和内存使用量上升，默认情况下可认为会在所有客户端中随机选择一个执行备份操作。\n\n特别注意默认情况下当文件系统的**文件数达到一百万**时，元数据自动备份功能将会关闭，需要配置一个更大的备份间隔（`--backup-meta` 选项）才会再次开启。备份间隔每个客户端独立配置，设置 `--backup-meta 0` 则表示关闭元数据自动备份特性。\n\n:::note 注意\n备份元数据所需的时间取决于具体的元数据引擎，不同元数据引擎会有不同的性能表现。\n:::\n\n有关元数据自动备份的详细介绍请参考[文档](../administration/metadata_dump_load.md#backup-automatically)，你也可以手动备份元数据。除此之外，也请遵照你所使用的元数据引擎的运维建议对数据进行定期备份。\n\n## 回收站\n\n:::tip 提示\n回收站是自 JuiceFS v1.0.0 版本开始加入的特性\n:::\n\n回收站默认开启，文件被删除后的保留时间默认配置为 1 天，可以有效防止数据被误删除时造成的数据丢失风险。\n\n不过回收站开启以后也可能带来一些副作用，如果应用需要经常删除文件或者频繁覆盖写文件，会导致对象存储使用量远大于文件系统用量。这本质上是因为 JuiceFS 客户端会将对象存储上被删除的文件或者覆盖写时产生的需要垃圾回收的数据块持续保留一段时间。因此，在部署 JuiceFS 至生产环境时就应该考虑好合适的回收站配置，回收站保留时间可以通过以下方式配置（如果将 `--trash-days` 设置为 `0` 则表示关闭回收站特性）：\n\n- 新建文件系统：通过 `juicefs format` 的 `--trash-days ` 选项设置\n- 已有文件系统：通过 `juicefs config` 的 `--trash-days ` 选项修改\n\n有关回收站的详细介绍请参考[文档](../security/trash.md)。\n\n## 客户端后台任务\n\nJuiceFS 文件系统通过客户端维护后台任务，可以自动执行清理待删除文件和对象、清理回收站中的过期文件和碎片、清理长时间未响应的客户端会话等任务等。\n\n同一个 JuiceFS 文件系统的所有客户端在运行过程中共享一个后台任务集，每个任务定时执行，且具体执行的客户端随机选择。具体的后台任务包括：\n\n1. 清理待删除的文件和对象\n2. 清理回收站中的过期文件和碎片\n3. 清理长时间未响应的客户端会话\n4. 自动备份元数据\n\n由于这些任务执行时会占用一定资源，因此可以为业务较繁重的客户端配置 `--no-bgjob` 选项来禁止其参与后台任务。\n\n:::note 注意\n请保证至少有一个 JuiceFS 客户端可以执行后台任务\n:::\n\n## 客户端日志滚动\n\n当后台运行 JuiceFS 挂载点时，客户端默认会将日志输出到本地文件中。取决于挂载文件系统时的运行用户，本地日志文件的路径稍有区别。root 用户对应的日志文件路径是 `/var/log/juicefs.log`，非 root 用户的日志文件路径是 `$HOME/.juicefs/juicefs.log`。\n\n本地日志文件默认不会滚动，生产环境中为了确保日志文件不占用过多磁盘空间需要手动配置。以下是一个日志滚动的示例配置：\n\n```text title=\"/etc/logrotate.d/juicefs\"\n/var/log/juicefs.log {\n daily\n rotate 7\n compress\n delaycompress\n missingok\n notifempty\n copytruncate\n}\n```\n\n通过 `logrotate -d` 命令可以验证配置文件的正确性：\n\n```shell\nlogrotate -d /etc/logrotate.d/juicefs\n```\n\n有关日志滚动配置的详细介绍请参考[文档](https://linux.die.net/man/8/logrotate)。\n\n## 命令行自动补全\n\nJuiceFS 为 Bash 和 Zsh 提供了命令行自动补全脚本，方便在命令行中使用 `juicefs` 命令，具体请参考[文档](../reference/command_reference.mdx#auto-completion)。\n" }, { "path": "docs/zh_cn/deployment/python_sdk.md", "content": "---\ntitle: Python SDK\nsidebar_position: 6\n---\n\nJuiceFS 社区版从 v1.3.0 引入 Python SDK，适合无法使用 FUSE 挂载的容器化或虚拟化环境使用。并且 Python SDK 实现了 fsspec 的接口规范，可方便的对接 Ray 等框架。\n\n## 快速上手视频\n\n
\n \n
\n\n## 编译\n\n你可以在当前工作环境中直接编译 Python SDK，也可以使用 Docker 容器进行编译。两种方式都需要先克隆仓库并进入 SDK 所在目录。\n\n```bash\n# 克隆 JuiceFS 仓库\ngit clone https://github.com/juicedata/juicefs.git\n# 进入 JuiceFS 目录\ncd juicefs/sdk/python\n```\n\n### 直接编译\n\n直接编译需要 `go1.20+` 和 `python3` 环境。\n\n#### 第一步：编译 libjfs.so\n\n```bash\ngo build -buildmode c-shared -ldflags=\"-s -w\" -o juicefs/juicefs/libjfs.so ../java/libjfs\n```\n\n编译产生的 `libjfs.so` 和 `libjfs.h` 文件在 `sdk/python/juicefs/juicefs` 目录下。\n\n#### 第二步：编译 Python SDK\n\n```bash\ncd juicefs && python3 -m build -w\n```\n\n编译好的 Python SDK 会在 `juicefs/sdk/python/dist` 目录下，文件名为 `juicefs-1.3.0-py3-none-any.whl`。\n\n### Docker 编译\n\n使用 Docker 容器编译需要当前系统安装了 `Docker`、`make` 和 `go1.20+` 环境。\n\n#### 第一步：构建 Docker 镜像\n\n```bash\n# For arm64\nmake arm-builder\n\n# For amd64\nmake builder\n```\n\n#### 第二步：编译 Python SDK\n\n```bash\nmake juicefs\n```\n\n编译好的 Python SDK 会在 `juicefs/sdk/python/dist` 目录下，文件名为 `juicefs-1.3.0-py3-none-any.whl`。\n\n### 编译报错处理\n\n如果在编译时遇到 `sed: 1: \"juicefs/setup.py\": invalid command code j` 的错误，可以尝试将 `Makefile` 中 `sed` 相关的命令注释掉。\n\n## 安装与使用\n\n### 安装 SDK\n\n将编译好的 `juicefs-1.3.0-py3-none-any.whl` 文件拷贝到目标机器上，使用 `pip` 安装：\n\n```bash\npip install juicefs-1.3.0-py3-none-any.whl\n```\n\n### 准备文件系统\n\n:::tip\nJuiceFS 的 Python SDK 暂不支持格式化文件系统，因此在使用之前请确保已经预先创建了 JuiceFS 文件系统。\n:::\n\n假设这里已经有一个预先创建好的名称为 `myfs` 的文件系统，元数据引擎 URL 为 `redis://192.168.1.8/0`。\n\n### 使用 Client\n\n`Client` 类的实现与 Python 的 io 模块类似。\n\n可以使用以下代码实例化一个 JuiceFS 客户端，`name` 参数为文件系统名称，`meta` 参数为元数据引擎的 URL。其中，`name`参数必须存在，但允许使用空字符串或 `None`。\n\n```python\nfrom juicefs import Client\n\n# 创建 JuiceFS 客户端\njfs = Client(name='', meta='redis://192.168.1.8/0')\n\n# 列出目录中的文件\njfs.listdir('/')\n```\n\n### 使用 fsspec\n\nJuiceFS 的 Python SDK 还支持 `fsspec` 接口来操作 JuiceFS 文件系统。\n\n```bash\n# 安装 fsspec\npip install fsspec\n```\n\n`fsspec` 的使用方式与 `Client` 类类似，只是需要指定 `jfs` 或 `juicefs` 作为文件系统类型。\n\n```python\nimport fsspec\nfrom juicefs.spec import JuiceFS\n\njfs = fsspec.filesystem('jfs', name='', meta='redis://192.168.1.8/0')\n\n# 列出目录中的文件\njfs.ls('/')\n```\n\n### 获取帮助信息\n\n可以使用 `help()` 函数获取类和方法的帮助信息。\n\n```python\nimport juicefs\n\nhelp(juicefs.Client)\n```\n\n也可以使用 `dir()` 函数获取类和方法的列表。\n\n```python\nimport juicefs\n\ndir(juicefs.Client)\n```\n" }, { "path": "docs/zh_cn/deployment/samba.md", "content": "---\ntitle: 创建 Samba 共享\nsidebar_position: 8\ndescription: 本文介绍如何通过 Samba 共享 JuiceFS 文件系统中的目录。\n---\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\nSamba 是一个开源的软件套件，它实现了 SMB/CIFS（Server Message Block / Common Internet File System）协议，该协议是 Windows 系统中常用的文件共享协议。通过 Samba，可以在 Linux/Unix 服务器上创建共享目录，允许 Windows 计算机通过网络访问和使用这些共享资源。\n\n在安装了 Samba 的 Linux 系统上通过编辑 `smb.conf` 配置文件即可将本地目录创建成为共享文件夹，Windows 和 macOS 系统使用文件管理器就可以直接访问读写，Linux 需要安装 Samba 客户端访问。\n\n当需要将 JuiceFS 文件系统中的目录通过 Samba 共享时，只需使用 `juicefs mount` 命令挂载，然后使用 JuiceFS 挂载点或子目录创建 Samba 共享即可。\n\n:::note\n`juicefs mount` 以 FUSE 接口的形式挂载为本地的用户态文件系统，与本地文件系统在形态和用法上无异，因此可以直接被用于创建 Samba 共享。\n:::\n\n## 第 1 步：安装 Samba\n\n主流 Linux 发行版的包管理器都会提供 Samba：\n\n\n\n\n```shell\nsudo apt install samba\n```\n\n\n \n\n```shell\nsudo dnf install samba\n```\n\n\n\n\n如果需要配置 AD/DC，还需要安装其他的软件包，详情参考 [Samba 官方安装指南](https://wiki.samba.org/index.php/Distribution-specific_Package_Installation)。\n\n## 第 2 步：启用 JuiceFS 的扩展属性支持\n\n根据 [Samba 官方文档](https://wiki.samba.org/index.php/File_System_Support#File_systems_without_xattr_support)，建议使用支持扩展属性（xattr）的文件系统，JuiceFS 文件系统需要在挂载时使用 `--enable-xattr` 选项来启用扩展属性，例如：\n\n```shell\nsudo juicefs mount -d --enable-xattr sqlite3://myjfs.db /mnt/myjfs\n```\n\n对于通过 `/etc/fstab` 配置自动挂载的情况，可以在挂载选项部分添加 `enable-xattr` 选项，例如：\n\n```ini\n# <元数据引擎 URL> <挂载点> <文件系统类型> <挂载选项>\nredis://127.0.0.1:6379/0 /mnt/myjfs juicefs _netdev,max-uploads=50,writeback,cache-size=1024000,enable-xattr 0 0\n```\n\n### 知识拓展：Samba 为什么需要文件系统支持扩展属性？\n\nSamba 是一个基于 Linux/Unix 的软件，用途是面向 Windows 系统提供文件共享。由于 Windows 系统中很多文件和目录具有附加元数据（文件作者、关键字、图标位置等），这些信息通常是 POSIX 文件系统之外，需要以 xattr 的形式存储在 Windows 中的。为了保证这类文件可以正确的保存在 Linux 系统中，因此 Samba 建议使用支持扩展属性的文件系统创建共享。\n\n## 第 3 步：创建 Samba 共享\n\n假设 JuiceFS 的挂载点是 `/mnt/myjfs`，比如要把其中的 `media` 目录创建成为 Samba 共享，可以这样配置：\n\n```ini\n[Media]\n path = /mnt/myjfs/media\n guest ok = no\n read only = no\n browseable = yes\n```\n\n## 面向 macOS 的共享\n\n苹果 macOS 系统支持直接访问 Samba 共享，与 Windows 类似，macOS 也存在一些额外的元数据（图标位置、Spotlight 搜索等）需要通过 xattr 来保存，Samba 4.9 及以上版本默认开启了对苹果系统的扩展属性支持。\n\n如果 [Samba 版本低于 4.9](https://wiki.samba.org/index.php/Configure_Samba_to_Work_Better_with_Mac_OS_X)，需要在 Samba 的 [global] 全局配置部分添加 `ea support = yes` 选项来启用面向苹果系统的扩展属性支持，编辑配置文件 `/etc/samba/smb.conf`，例如：\n\n```ini\n[global]\n workgroup = SAMBA\n security = user\n passdb backend = tdbsam\n ea support = yes\n```\n\n## Samba 的用户管理\n\nSamba 有一套自己的用户数据库，它与操作系统用户之间是独立的，但是 Samba 共享的是系统中的目录，因此必须有恰当的用户权限才能读写。\n\n### 创建 Samba 用户\n\n在为 Samba 创建用户时，要求该用户必须是系统中已经存在的用户，系统会自动进行映射，从而让 Samba 用户具有同名系统用户的权限。\n\n- 如果系统中已存在该用户，假设该账户是 herald，则这样创建 Samba 账户：\n\n ```shell\n sudo smbpasswd -a herald\n ```\n\n 根据命令提示设置密码即可，Samba 账户可以设置与系统用户不同的密码。\n\n- 如果需要创建一个新的用户，以创建一个名为 `abc` 的用户为例，则这样操作：\n 1. 创建用户：\n\n ```shell\n sudo adduser abc\n ```\n\n 2. 创建同名的 Samba 用户：\n\n ```shell\n sudo smbpasswd -a abc\n ```\n\n### 查看已创建的 Samba 用户\n\n`pdbedit` 是一个 Samba 自带的用于管理 Samba 用户数据库的工具，可以使用该工具来列出所有已创建的 Samba 用户：\n\n```shell\nsudo pdbedit -L\n```\n\n它会列出所有已创建的 Samba 用户列表，包括用户名、用户的 SID（Security Identifier）和所属的组等信息。\n\n## 扩展阅读\n\n[《如何基于 JuiceFS 配置 Samba 和 NFS 共享》](https://juicefs.com/zh-cn/blog/usage-tips/configure-samba-and-nfs-shares-based-juicefs)，这篇文章介绍了如何使用 Cockpit 在浏览器中以图形化界面方式来管理 Samba 和 NFS 共享。\n" }, { "path": "docs/zh_cn/deployment/webdav.md", "content": "---\ntitle: 配置 WebDAV 服务\nsidebar_position: 5\n---\n\nWebDAV 是 HTTP 协议的扩展，是一种便于多用户间协同编辑和管理网络上的文档的共享协议。很多涉及文件编辑和同步的工具、macOS Finder 以及一些 Linux 发行版的文件管理器都内置了 WebDAV 客户端支持。\n\nJuiceFS 支持通过 WebDAV 协议挂载访问，对于 macOS 以及其他没有原生 FUSE 支持的操作系统，通过 WebDAV 协议访问 JuiceFS 文件系统是非常方便的。\n\n## 前置条件\n\n在配置 WebDAV 服务之前，你需要预先[创建一个 JuiceFS 文件系统](../getting-started/standalone.md#juicefs-format)。\n\n## 匿名 WebDAV\n\n对于单机或内网等安全不敏感的环境中，可以配置不带身份认证的匿名 WebDAV，命令格式如下：\n\n```shell\njuicefs webdav META-URL LISTENING-ADDRESS:PORT\n```\n\n例如，为一个 JuiceFS 文件系统启用 WebDAV 协议访问：\n\n```shell\nsudo juicefs webdav sqlite3://myjfs.db 192.168.1.8:80\n```\n\nWebDAV 服务需要通过设定的监听地址和端口进行访问，如上例中使用了内网的 IP 地址 `192.168.1.8`，以及标准的 Web 端口号 `80`，访问时无需指定端口，直接访问 `http://192.168.1.8` 即可。\n\n如果使用了其他端口号，则需要在地址中明确指定，例如，监听 `9007` 端口，访问地址则应该用 `http://192.168.1.8:9007`。\n\n:::tip 提示\n当使用 macOS 的 Finder 访问匿名 WebDAV 时，不要使用「客人」身份。请使用「注册用户」身份，用户名可以输入任意字符，密码可以为空，然后直接连接即可。\n:::\n\n## 带身份认证的 WebDAV\n\n:::info 说明\nJuiceFS v1.0.3 及之前的版本不支持身份认证功能\n:::\n\nJuiceFS 的 WebDAV 身份认证功能需要通过环境变量设置用户名（`WEBDAV_USER`）和密码（`WEBDAV_PASSWORD`），例如：\n\n```shell\nexport WEBDAV_USER=user\nexport WEBDAV_PASSWORD=mypassword\nsudo juicefs webdav sqlite3://myjfs.db 192.168.1.8:80\n```\n\n## 启用 HTTPS 支持\n\nJuiceFS 支持配置通过 HTTPS 协议保护的 WebDAV 服务，通过 `--cert-file` 和 `--key-file` 选项指定证书和私钥，既可以使用受信任的数字证书颁发机构 CA 签发的证书，也可以使用 OpenSSL 创建自签名证书。\n\n### 自签名证书\n\n这里使用 OpenSSL 创建私钥和证书：\n\n1. 生成服务器私钥\n\n ```shell\n openssl genrsa -out client.key 4096\n ```\n\n2. 生成证书签名请求（CSR）\n\n ```shell\n openssl req -new -key client.key -out client.csr\n ```\n\n3. 使用 CSR 签发证书\n\n ```shell\n openssl x509 -req -days 365 -in client.csr -signkey client.key -out client.crt\n ```\n\n以上三条命令会在当前目录产生以下文件：\n\n- `client.key`：服务器私钥\n- `client.csr`：证书签名请求文件\n- `client.crt`：自签名证书\n\n创建 WebDAV 服务时需要使用 `client.key` 和 `client.crt`，例如：\n\n```shell\nsudo juicefs webdav \\\n --cert-file ./client.crt \\\n --key-file ./client.key \\\n sqlite3://myjfs.db 192.168.1.8:443\n```\n\n启用了 HTTPS 支持，监听的端口号可以改为 HTTPS 的标准端口号 `443`，然后改用 `https://` 协议头，访问时无需指定端口号，例如：`https://192.168.1.8`。\n\n同样地，设置了非 HTTPS 标准端口号，应该在访问地址中明确指定，例如，设置了监听 `9999` 端口，访问地址应使用 `https://192.168.1.8:9999`。\n" }, { "path": "docs/zh_cn/development/contributing_guide.md", "content": "---\ntitle: 贡献指南\nsidebar_position: 1\ndescription: JuiceFS 是开源软件，代码由全球开发者共同贡献和维护，您可以参考本文了解参与开发的流程和注意事项。\n---\n\n## 基本准则 {#guidelines}\n\n- 在开始修复功能或错误之前，请先通过 GitHub、Slack 等渠道与我们沟通。此步骤的目的是确保没有其他人已经在处理它，如有必要，我们将要求您创建一个 GitHub issue。\n- 在开始贡献前，使用 GitHub issue 来讨论功能实现并与核心开发者达成一致。\n- 如果这是一个重大的特性更新，写一份设计文档来帮助社区理解你的动机和解决方案。\n- 对于首次贡献者来说，找到合适 issue 的好方法是使用标签 [\"kind/good-first-issue\"](https://github.com/juicedata/juicefs/labels/kind%2Fgood-first-issue) 或 [\"kind/help-wanted\"](https://github.com/juicedata/juicefs/labels/kind%2Fhelp-wanted) 搜索未解决的问题。\n\n## 代码风格 {#coding-style}\n\n- 我们遵循 [\"Effective Go\"](https://go.dev/doc/effective_go) 和 [\"Go Code Review Comments\"](https://github.com/golang/go/wiki/CodeReviewComments)。\n- 在提交前使用 `go fmt` 格式化你的代码。你可以在 [Go 的编辑器和 IDE](https://github.com/golang/go/wiki/IDEsAndTextEditorPlugins) 中找到支持 Go 的相关工具的信息。\n- 每个新的源文件都必须以许可证头开始。\n- 安装 [pre-commit](https://pre-commit.com) 并使用它来设置一个预提交钩子来进行静态分析。只需在仓库根目录下运行 `pre-commit install` 即可。\n\n## 签署 CLA {#sign-the-cla}\n\n在您为 JuiceFS 进行贡献之前，您需要签署[贡献者许可协议](https://cla-assistant.io/juicedata/juicefs)。当你第一次提交 PR 的时候，将有一个 CLA 助手指导你。\n\n## 什么是好的 PR {#what-is-a-good-pr}\n\n- 足够的单元测试\n- 遵循编码风格\n- 足够的行内注释\n- 简要解释的提交内容\n\n## 贡献流程 {#contribution-flow}\n\n1. 基于主分支创建一个要贡献的主题分支。这个主分支通常是 `main` 分支；\n1. 提交代码；\n1. 确保提交消息的格式正确；\n1. 将主题分支中的更改推到个人 fork 的仓库；\n1. 提交一个 PR 到 [`juicedata/juicefs`](https://github.com/juicedata/juicefs/compare) 仓库。这个 PR 应该链接到你或其他人创建的一个 issue；\n1. PR 在合并之前必须得到至少一个维护者的批准。\n" }, { "path": "docs/zh_cn/development/internals.md", "content": "---\ntitle: 内部实现\nsidebar_position: 3\nslug: /internals\n---\n\n本文介绍 JuiceFS 的实现细节，用来为开发者了解和贡献开源代码作参考。其中内容对应的 JuiceFS 代码版本为 v1.0.0，元数据版本为 v1。\n\n在深入学习源码前，我们还推荐阅读：\n\n* [JuiceFS 读写请求处理流程](../introduction/io_processing.md)\n* 网易存储团队的工程师写的这几篇博客（注意文章内容可能与最新版本代码有出入，一切请以代码为准）：[JuiceFS 调研（基于开源版本代码）](https://aspirer.wang/?p=1560)、[JuiceFS 源码阅读 - 上](https://mp.weixin.qq.com/s/mdqFJLpaJ249rUUEnRiP3Q)、[JuiceFS 源码阅读 - 中](https://mp.weixin.qq.com/s/CLQbQ-cLLGFsShPKUrCUJg)。\n\n## 关键词定义\n\n高层概念：\n\n- 文件系统（File System）：即 JuiceFS Volume，代表一个独立的命名空间。文件在同文件系统内可自由移动，不同文件系统之间则需要数据拷贝；\n- 元数据引擎（Metadata Engine）：用来存储和管理文件系统元数据的组件，通常由支持事务的数据库担任。目前已支持的元数据引擎共有三大类：\n - Redis：Redis 及各种协议兼容的服务；\n - SQL：MySQL、PostgreSQL、SQLite 等；\n - TKV：TiKV、BadgerDB、etcd 等。\n- 数据存储：用来存储和管理文件系统数据的组件，通常由对象存储担任，如 Amazon S3、Aliyun OSS 等；也可由能兼容对象存储语义的其他存储系统担任，如本地文件系统、Ceph RADOS、TiKV 等；\n- JuiceFS 客户端（JuiceFS Client）：有多种形式，如挂载进程、S3 网关、WebDAV 服务器、Java SDK 等；\n- 文件：本文中泛指所有类型的文件，包括普通文件、目录文件、链接文件、设备文件等；\n- 目录：一种特殊的文件，用来组织文件树型结构，其内容是一组其他文件的索引。\n\n底层概念（详见 [JuiceFS 读写请求处理流程](../introduction/io_processing.md)）：\n\n- Chunk：对文件分割的逻辑单位，大小 64MiB。Chunk 的存在让 JuiceFS 在读取大文件时能快速定位，提升读取性能；\n- Slice：数据写入的逻辑单位，每一次写入都会分配一个已有或新的 Slice，而在元数据中则在 Chunk 下维护着 Chunk Slice 列表。\n- Block：文件分割后的实际最小存储单位，默认大小 4MiB。一个 Chunk 包含一个或多个 Slice，而一个 Slice 又包含一个或多个 Block。\n\n## 代码结构 {#source-code-structure}\n\n[JuiceFS 源码](https://github.com/juicedata/juicefs)的大体结构如下：\n\n* [`cmd`](https://github.com/juicedata/juicefs/tree/main/cmd) 是代码结构总入口，所有相关功能都能在此找到入口，如 `juicefs format` 命令对应着 `cmd/format.go`；\n* [`pkg`](https://github.com/juicedata/juicefs/tree/main/pkg) 是具体实现，核心逻辑都在其中：\n * `pkg/fuse/fuse.go` 是 FUSE 实现的入口，提供抽象 FUSE 接口；\n * `pkg/vfs` 是具体的 FUSE 接口实现，元数据请求会调用 `pkg/meta` 中的实现，读请求会调用 `pkg/vfs/reader.go`，写请求会调用 `pkg/vfs/writer.go`；\n * `pkg/meta` 目录中是所有元数据引擎的实现，其中：\n * `pkg/meta/interface.go` 是所有类型元数据引擎的接口定义\n * `pkg/meta/redis.go` 是 Redis 数据库的接口实现\n * `pkg/meta/sql.go` 是关系型数据库的接口定义及通用接口实现，特定数据库的实现在单独文件中（如 MySQL 的实现在 `pkg/meta/sql_mysql.go`）\n * `pkg/meta/tkv.go` 是 KV 类数据库的接口定义及通用接口实现，特定数据库的实现在单独文件中（如 TiKV 的实现在 `pkg/meta/tkv_tikv.go`）\n * `pkg/object` 是与各种对象存储对接的实现。\n* [`sdk/java`](https://github.com/juicedata/juicefs/tree/main/sdk/java) 是 Hadoop Java SDK 的实现，底层依赖 `sdk/java/libjfs` 这个库（通过 JNI 调用）。\n\n## FUSE 接口实现 {#fuse-interface-implementation}\n\nJuiceFS 基于 [FUSE](https://en.wikipedia.org/wiki/Filesystem_in_Userspace)（Filesystem in Userspace）实现了一个用户态文件系统，FUSE 接口在 Linux 系统中的实现库 [`libfuse`](https://github.com/libfuse/libfuse) 提供两种 API：high-level API 和 low-level API，其中 high-level API 基于文件名和路径，low-level API 基于 inode。\n\nJuiceFS 基于 low-level API 实现（事实上 JuiceFS 不依赖 `libfuse`，而是 [`go-fuse`](https://github.com/hanwen/go-fuse)），这是因为内核的 VFS 跟 FUSE 库交互就使用 low-level API。如果使用 high-level API 的话，其实是在 `libfuse` 内部做了 VFS 树的模拟，然后对外暴露基于路径的 API，这种模式适合元数据本身是基于路径提供的 API 的系统，比如 HDFS 或者 S3 之类。而如果元数据本身也是基于 inode 的目录树，这种 inode → path → inode 的反复转换就会影响性能（所以 HDFS 的 FUSE 接口实现性能都不好）。JuiceFS 的元数据是按照 inode 组织的，也直接提供基于 inode 的 API，那么使用 FUSE 的 low-level API 就非常简单和自然，性能也很好。\n\n## 元数据结构\n\n文件系统通常组织成树型结构，其中节点代表文件，边代表目录的包含关系。文件无法悬空停留，其（根目录除外）必然属于某个目录；目录可以包含一个或多个子文件。JuiceFS 中一共有十多种元数据结构，其中大部分用来维护文件树的组织关系和各个节点的属性，其余的用来管理系统配置，客户端会话和异步任务等。以下具体介绍所有的元数据结构。\n\n### 通用结构\n\n#### Setting\n\n保存文件系统的格式化信息，在执行 `juicefs format` 命令时创建，后续可通过 `juicefs config` 命令修改其中的部分字段。结构具体如下：\n\n```go\ntype Format struct {\n Name string\n UUID string\n Storage string\n Bucket string\n AccessKey string `json:\",omitempty\"`\n SecretKey string `json:\",omitempty\"`\n SessionToken string `json:\",omitempty\"`\n BlockSize int\n Compression string `json:\",omitempty\"`\n Shards int `json:\",omitempty\"`\n HashPrefix bool `json:\",omitempty\"`\n Capacity uint64 `json:\",omitempty\"`\n Inodes uint64 `json:\",omitempty\"`\n EncryptKey string `json:\",omitempty\"`\n KeyEncrypted bool `json:\",omitempty\"`\n TrashDays int `json:\",omitempty\"`\n MetaVersion int `json:\",omitempty\"`\n MinClientVersion string `json:\",omitempty\"`\n MaxClientVersion string `json:\",omitempty\"`\n EnableACL bool\n}\n```\n\n- Name：文件系统名称，在格式化时由用户指定\n- UUID：文件系统的唯一 ID，在格式化时由系统自动生成\n- Storage：用来保存数据的对象存储简称，如 `s3`、`oss` 等\n- Bucket：对象存储的桶路径\n- AccessKey：用来访问对象存储的 access key\n- SecretKey：用来访问对象存储的 secret key\n- SessionToken：用来访问对象存储的 session token，部分对象存储支持使用临时的 token 以获得有限时间的权限\n- BlockSize：存储文件时拆分成的数据块大小，默认为 4 MiB\n- Compression：数据块上传到对象存储前执行的压缩算法，默认为不压缩\n- Shards：对象存储中分片桶的个数，默认为只有一个桶；当 Shards > 1 时，数据对象会随机哈希到 Shards 个桶中\n- HashPrefix：是否为对象名称设置一个散列的前缀，默认为不设置\n- Capacity：文件系统的总容量配额限制\n- Inodes：文件系统的总文件数配额限制\n- EncryptKey：数据对象的加密私钥，只要在开启了数据加密功能后才有用\n- KeyEncrypted：保存的密钥是否处于加密状态，默认会将 SecretKey、EncryptKey 和 SessionToken 加密保存\n- TrashDays：文件在回收站中被保留的天数，默认为 1 天\n- MetaVersion：元数据结构的版本，目前为 V1（V0 和 V1 相同）\n- MinClientVersion：允许连接的最小客户端版本，早于此版本的客户端会被拒绝连接\n- MaxClientVersion：允许连接的最大客户端版本\n- EnableACL: 是否开启 ACL 功能\n\n此结构会序列化成 JSON 格式保存在元数据引擎中。\n\n#### Counter\n\n维护系统中的各个计数器值和一些后台任务的启动时间戳，具体有：\n\n- usedSpace：文件系统的已使用容量\n- totalInodes：文件系统的已使用文件数\n- nextInode：下一个可用的 inode 号（Redis 中为当前已用的最大 inode 号）\n- nextChunk：下一个可用的 sliceId（Redis 中为当前已用的最大 sliceId）\n- nextSession：当前已用的最大 SID（sessionID）\n- nextTrash：当前已用的最大 trash inode 号\n- nextCleanupSlices：上一次检查清理残留 slices 的时间点\n- lastCleanupSessions：上一次检查清理残留 stale sessions 的时间点\n- lastCleanupFiles：上一次检查清理残留文件的时间点\n- lastCleanupTrash：上一次检查清理回收站的时间点\n\n#### Session\n\n记录连接到此文件系统的客户端会话 ID 和其超时时间。每个客户端会定时发送心跳消息以更新超时时间，长时间未更新者会被其他客户端自动清理。\n\n:::tip 注意\n只读客户端无法写入元数据引擎，因此其会话**不会**被记录。\n:::\n\n#### SessionInfo\n\n记录客户端会话的具体元信息，使其可以通过 `juicefs status` 命令查看。具体为：\n\n```go\ntype SessionInfo struct {\n Version string // JuiceFS 版本\n HostName string // 主机名称\n MountPoint string // 挂载点路径。S3 网关和 WebDAV 服务分别为 \"s3gateway\" 和 \"webdav\"\n ProcessID int // 进程 ID\n}\n```\n\n此结构会序列化成 JSON 格式保存在元数据引擎中。\n\n#### Node\n\n记录每个文件的属性信息，具体为：\n\n```go\ntype Attr struct {\n Flags uint8 // reserved flags\n Typ uint8 // type of a node\n Mode uint16 // permission mode\n Uid uint32 // owner id\n Gid uint32 // group id of owner\n Rdev uint32 // device number\n Atime int64 // last access time\n Mtime int64 // last modified time\n Ctime int64 // last change time for meta\n Atimensec uint32 // nanosecond part of atime\n Mtimensec uint32 // nanosecond part of mtime\n Ctimensec uint32 // nanosecond part of ctime\n Nlink uint32 // number of links (sub-directories or hardlinks)\n Length uint64 // length of regular file\n\n Parent Ino // inode of parent; 0 means tracked by parentKey (for hardlinks)\n Full bool // the attributes are completed or not\n KeepCache bool // whether to keep the cached page or not\n\n AccessACL uint32 // access ACL id (identical ACL rules share the same access ACL ID.)\n DefaultACL uint32 // default ACL id (default ACL and the access ACL share the same cache and store)\n}\n```\n\n其中几个需要说明的字段：\n\n- Atime/Atimensec：参考 [`--atime-mode`](../reference/command_reference.mdx#mount-metadata-options)\n- Nlink：\n - 目录文件：初始值为 2（'.' 和 '..'），每有一个子目录 Nlink 值加 1\n - 其他文件：初始值为 1，每创建一个硬链接 Nlink 值加 1\n- Length：\n - 目录文件：固定为 4096\n - 软链接（symbolic link）文件：为链接指向路径的字符串长度\n - 其他文件：为文件实际内容的长度\n\n此结构一般会编码成二进制格式保存在元数据引擎中。\n\n#### Edge\n\n记录文件树中每条边的信息，具体为：\n\n```\nparentInode, name -> type, inode\n```\n\n其中 parentInode 是父目录的 inode 号，其他分别为子文件的名称、类型和 inode 号。\n\n#### LinkParent\n\n记录部分文件的父目录。绝大部分文件的父目录记在其属性的 Parent 字段中；但对于创建过硬链接的文件，其父目录可能有多个，此时会将 Parent 字段置 0，同时独立记录其所有父目录 inodes，具体为：\n\n```\ninode -> parentInode, links\n```\n\n其中 links 是 parentInode 的计数，因为一个目录中可以创建多个硬链接，这些硬连接共享 inode。\n\n#### Chunk\n\n记录每个 Chunk 的信息，具体为：\n\n```\ninode, index -> []Slices\n```\n\n其中 inode 是此 Chunk 所属文件的 inode 号，index 是其在这个文件所有 Chunks 中序号，从 0 开始。Chunk 值内容为一个 Slices 数组，每个 Slice 代表一段客户端写入的数据，并且按写入时间顺序 append 到这个数组中。当不同 Slices 之间有重叠时，以后加入的 Slice 为准。Slice 的具体结构为：\n\n```go\ntype Slice struct {\n Pos uint32 // Slice 在 Chunk 中的偏移位置\n ID uint64 // Slice 的 ID，全局唯一\n Size uint32 // Slice 的总大小\n Off uint32 // 有效数据在此 Slice 中的偏移位置\n Len uint32 // 有效数据在此 Slice 中的大小\n}\n```\n\n此结构会编码成二进制格式保存，占 24 个字节。\n\n#### SliceRef\n\n记录 Slice 的引用计数，具体为：\n\n```\nsliceId, size -> refs\n```\n\n由于绝大部分 Slice 的引用计数均为 1，为减少数据库中相关 entry 数量，在 Redis 和 TKV 中以实际值减 1 作为存储的计数值。这样，大部分的 Slice 对应 refs 值为 0，则不必在数据库中创建相关 entry。\n\n#### Symlink\n\n记录软链接文件的指向位置，具体为：\n\n```\ninode -> target\n```\n\n#### Xattr\n\n记录文件相关的扩展属性（Key-Value 对），具体为：\n\n```\ninode, key -> value\n```\n\n#### Flock\n\n记录文件相关的 BSD locks（flock），具体为：\n\n```\ninode, sid, owner -> ltype\n```\n\n其中 `sid` 为客户端会话 ID，`owner` 为一串数字，通常与进程相关联；`ltype` 为锁类型，可以为 'R' 或者 'W'。\n\n#### Plock\n\n记录文件相关的 POSIX record locks（fcntl），具体为：\n\n```\ninode, sid, owner -> []plockRecord\n```\n\n这里 plock 是一种更细粒度的锁，可以只锁定文件中的某一片段：\n\n```go\ntype plockRecord struct {\n ltype uint32 // 锁类型\n pid uint32 // 进程 ID\n start uint64 // 锁起始位置\n end uint64 // 锁结束位置\n}\n```\n\n此结构会编码成二进制格式保存，占 24 个字节。\n\n#### DelFiles\n\n记录待清理的文件列表。由于文件的数据清理是一个异步且可能长耗时的操作，可能被其他因素中断，因此会由此列表进行跟踪：\n\n```\ninode, length -> expire\n```\n\n其中 length 为文件长度，expire 为文件被删除的时间。\n\n#### DelSlices\n\n记录延迟删除的 Slices。当回收站功能开启时，因 Slice Compaction 功能删除的旧 Slices 会被保留与回收站配置相同的时间，以被在必要时可用来恢复数据。其内容为：\n\n```\nsliceId, deleted -> []slice\n```\n\n其中 sliceId 为 compact 后新 Slice 的 ID，deleted 为 compact 完成的时间戳，映射值为被 compacted 的所有旧 slice 列表，每个 slice 仅编码了 ID 和 size 信息：\n\n```go\ntype slice struct {\n ID uint64\n Size uint32\n}\n```\n\n此结构会编码成二进制格式保存，占 12 个字节。\n\n#### Sustained\n\n记录会话中需临时保留的文件列表。当文件被删除时若其仍处于打开状态，则不能立即清理数据，而需要暂时保留直至其被关闭。\n\n```\nsid -> []inode\n```\n\n其中 `sid` 为会话 ID，映射值为暂时未删除的文件 inodes 列表。\n\n### Redis\n\nRedis 中 Key 的通用格式为 `${prefix}${JFSKey}`，其中：\n\n- 在 Redis 非集群模式下 prefix 为空字符串，在集群模式中是一个大括号括起来的数据库编号，如 \"{10}\"\n- JFSKey 是指 JuiceFS 不同数据结构的 Key，具体列举在后续小节中\n\n在 Redis 的 Keys 中，如无特殊说明整数（包括 inode 号）都以十进制字符串表示。\n\n#### Setting {#redis-setting}\n\n- Key：`setting`\n- Value Type：String\n- Value：JSON 格式的文件系统格式化信息\n\n#### Counter\n\n- Key：计数器名称\n- Value Type：String\n- Value：计数器的值，实际均为整数\n\n#### Session\n\n- Key：`allSessions`\n- Value Type：Sorted Set\n- Value：所有连接此文件系统的非只读会话。在 Set 中：\n - Member：会话 ID\n - Score：此会话超时的时间点\n\n#### SessionInfo\n\n- Key：`sessionInfos`\n- Value Type：Hash\n- Value：所有非只读会话的基本元信息。在 Hash 中：\n - Key：会话 ID\n - Value：JSON 格式的会话信息\n\n#### Node {#redis-node}\n\n- Key：`i${inode}`\n- Value Type：String\n- Value：二进制编码的文件属性\n\n#### Edge {#redis-edge}\n\n- Key：`d${inode}`\n- Value Type：Hash\n- Value：此目录下的所有目录项。在 Hash 中：\n - Key：文件名称\n - Value：二进制编码的文件类型和 inode 号\n\n#### LinkParent\n\n- Key：`p${inode}`\n- Value Type：Hash\n- Value：此文件的所有父目录 inodes。在 Hash 中：\n - Key：父目录 inode\n - Value：此父目录 inode 的计数\n\n#### Chunk {#redis-chunk}\n\n- Key：`c${inode}_${index}`\n- Value Type：list\n- Value：Slices 列表，每个 Slice 均以二进制编码，各占 24 个字节\n\n#### SliceRef {#sliceref}\n\n- Key：`sliceRef`\n- Value Type：Hash\n- Value：所有需记录的 Slices 的计数值。在 Hash 中：\n - Key：`k${sliceId}_${size}`\n - Value：此 Slice 的引用计数值减 1（若引用计数为 1，则一般不创建对应 entry）\n\n#### Symlink\n\n- Key：`s${inode}`\n- Value Type：String\n- Value：符号链接指向的路径\n\n#### Xattr\n\n- Key：`x${inode}`\n- Value Type：Hash\n- Value：此文件的所有扩展属性。在 Hash 中：\n - Key：扩展属性名称\n - Value：扩展属性值\n\n#### Flock\n\n- Key：`lockf${inode}`\n- Value Type：Hash\n- Value：此文件的所有 flocks。在 Hash 中：\n - Key：`${sid}_${owner}`，owner 以十六进制表示\n - Value：锁类型，可能为 'R' 或者 'W'\n\n#### Plock {#redis-plock}\n\n- Key：`lockp${inode}`\n- Value Type：Hash\n- Value：此文件的所有 plocks。在 Hash 中：\n - Key：`${sid}_${owner}`，owner 以十六进制表示\n - Value：字节数组，其中每 24 字节对应一个 [plockRecord](#plock)\n\n#### DelFiles\n\n- Key：`delfiles`\n- Value Type：Sorted Set\n- Value：所有待清理的文件列表。在 Set 中：\n - Member：`${inode}:${length}`\n - Score：此文件加入集合的时间点\n\n#### DelSlices {#redis-delslices}\n\n- Key：`delSlices`\n- Value Type：Hash\n- Value：所有待清理的 Slices。在 Hash 中：\n - Key：`${sliceId}_${deleted}`\n - Value：字节数组，其中每 12 字节对应一个 [slice](#delslices)\n\n#### Sustained\n\n- Key：`session${sid}`\n- Value Type：List\n- Value：此会话中临时保留的文件列表。在 List 中：\n - Member：文件的 inode 号\n\n### SQL\n\n元数据按类型存储在不同的表中，每张表命名时以 `jfs_` 开头，跟上其具体的结构体名称组成表名，如 `jfs_node`。部分表中加入了 `bigserial` 类型的 `Id` 列作为主键，其仅用来确保每张表中都有主键，并不包含实际信息。\n\n#### Setting {#sql-setting}\n\n```go\ntype setting struct {\n Name string `xorm:\"pk\"`\n Value string `xorm:\"varchar(4096) notnull\"`\n}\n```\n\n固定只有一条 entry，Name 为 \"format\"，Value 为 JSON 格式的文件系统格式化信息。\n\n#### Counter\n\n```go\ntype counter struct {\n Name string `xorm:\"pk\"`\n Value int64 `xorm:\"notnull\"`\n}\n```\n\n#### Session\n\n```go\ntype session2 struct {\n Sid uint64 `xorm:\"pk\"`\n Expire int64 `xorm:\"notnull\"`\n Info []byte `xorm:\"blob\"`\n}\n```\n\n#### SessionInfo\n\n没有独立的表，而是记在 `session2` 的 `Info` 列中。\n\n#### Node {#sql-node}\n\n```go\ntype node struct {\n Inode Ino `xorm:\"pk\"`\n Type uint8 `xorm:\"notnull\"`\n Flags uint8 `xorm:\"notnull\"`\n Mode uint16 `xorm:\"notnull\"`\n Uid uint32 `xorm:\"notnull\"`\n Gid uint32 `xorm:\"notnull\"`\n Atime int64 `xorm:\"notnull\"`\n Mtime int64 `xorm:\"notnull\"`\n Ctime int64 `xorm:\"notnull\"`\n Nlink uint32 `xorm:\"notnull\"`\n Length uint64 `xorm:\"notnull\"`\n Rdev uint32\n Parent Ino\n AccessACLId uint32 `xorm:\"'access_acl_id'\"`\n DefaultACLId uint32 `xorm:\"'default_acl_id'\"`\n}\n```\n\n大部分字段与 [Attr](#node) 相同，但时间戳使用了较低精度，其中 Atime/Mtime/Ctime 的单位为微秒。\n\n#### Edge {#sql-edge}\n\n```go\ntype edge struct {\n Id int64 `xorm:\"pk bigserial\"`\n Parent Ino `xorm:\"unique(edge) notnull\"`\n Name []byte `xorm:\"unique(edge) varbinary(255) notnull\"`\n Inode Ino `xorm:\"index notnull\"`\n Type uint8 `xorm:\"notnull\"`\n}\n```\n\n#### LinkParent\n\n没有独立的表，而是根据 `edge` 中的 `Inode` 索引找到所有 `Parent`。\n\n#### Chunk {#sql-chunk}\n\n```go\ntype chunk struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"unique(chunk) notnull\"`\n Indx uint32 `xorm:\"unique(chunk) notnull\"`\n Slices []byte `xorm:\"blob notnull\"`\n}\n```\n\nSlices 是一段字节数组，每 24 字节对应一个 [Slice](#chunk)。\n\n#### SliceRef\n\n```go\ntype sliceRef struct {\n Id uint64 `xorm:\"pk chunkid\"`\n Size uint32 `xorm:\"notnull\"`\n Refs int `xorm:\"notnull\"`\n}\n```\n\n#### Symlink\n\n```go\ntype symlink struct {\n Inode Ino `xorm:\"pk\"`\n Target []byte `xorm:\"varbinary(4096) notnull\"`\n}\n```\n\n#### Xattr\n\n```go\ntype xattr struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"unique(name) notnull\"`\n Name string `xorm:\"unique(name) notnull\"`\n Value []byte `xorm:\"blob notnull\"`\n}\n```\n\n#### Flock\n\n```go\ntype flock struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"notnull unique(flock)\"`\n Sid uint64 `xorm:\"notnull unique(flock)\"`\n Owner int64 `xorm:\"notnull unique(flock)\"`\n Ltype byte `xorm:\"notnull\"`\n}\n```\n\n#### Plock {#sql-plock}\n\n```go\ntype plock struct {\n Id int64 `xorm:\"pk bigserial\"`\n Inode Ino `xorm:\"notnull unique(plock)\"`\n Sid uint64 `xorm:\"notnull unique(plock)\"`\n Owner int64 `xorm:\"notnull unique(plock)\"`\n Records []byte `xorm:\"blob notnull\"`\n}\n```\n\nRecords 是一段字节数组，每 24 字节对应一个 [plockRecord](#plock)。\n\n#### DelFiles\n\n```go\ntype delfile struct {\n Inode Ino `xorm:\"pk notnull\"`\n Length uint64 `xorm:\"notnull\"`\n Expire int64 `xorm:\"notnull\"`\n}\n```\n\n#### DelSlices {#sql-delslices}\n\n```go\ntype delslices struct {\n Id uint64 `xorm:\"pk chunkid\"`\n Deleted int64 `xorm:\"notnull\"`\n Slices []byte `xorm:\"blob notnull\"`\n}\n```\n\nSlices 是一段字节数组，每 12 字节对应一个 [slice](#delslices)。\n\n#### Sustained\n\n```go\ntype sustained struct {\n Id int64 `xorm:\"pk bigserial\"`\n Sid uint64 `xorm:\"unique(sustained) notnull\"`\n Inode Ino `xorm:\"unique(sustained) notnull\"`\n}\n```\n\n### TKV\n\nTKV（Transactional Key-Value Database）中 Key 的通用格式为 `${prefix}${JFSKey}`，其中：\n\n- prefix 用来区分不同的文件系统，通常是 `${VolumeName}0xFD`，其中的 `0xFD` 作为特殊字节用来处理不同文件系统名称间存在包含关系的情况。此外，对于无法公用的数据库（如 BadgerDB）则直接使用空字符串作前缀\n- JFSKey 是指 JuiceFS 为不同数据类型设计的 Key，具体列举在后续小节中\n\n在 TKV 的 Keys 中，所有整数都以编码后的二进制形式存储：\n\n- inode 和 counter value 占 8 个字节，使用**小端**编码\n- SID、sliceId 和 timestamp 占 8 个字节，使用**大端**编码\n\n#### Setting {#tkv-setting}\n\n```\nsetting -> JSON 格式的文件系统格式化信息\n```\n\n#### Counter\n\n```\nC${name} -> counter value\n```\n\n#### Session\n\n```\nSE${sid} -> timestamp\n```\n\n#### SessionInfo\n\n```\nSI${sid} -> JSON 格式的会话信息\n```\n\n#### Node {#tkv-node}\n\n```\nA${inode}I -> encoded Attr\n```\n\n#### Edge {#tkv-edge}\n\n```\nA${inode}D${name} -> encoded {type, inode}\n```\n\n#### LinkParent\n\n```\nA${inode}P${parentInode} -> counter value\n```\n\n#### Chunk {#tkv-chunk}\n\n```\nA${inode}C${index} -> Slices\n```\n\n其中 index 占 4 个字节，使用**大端**编码。Slices 是一段字节数组，每 24 字节对应一个 [Slice](#chunk)。\n\n#### SliceRef\n\n```\nK${sliceId}${size} -> counter value\n```\n\n其中 size 占 4 个字节，使用**大端**编码。\n\n#### Symlink\n\n```\nA${inode}S -> target\n```\n\n#### Xattr\n\n```\nA${inode}X${name} -> xattr value\n```\n\n#### Flock\n\n```\nF${inode} -> flocks\n```\n\n其中 flocks 是一段字节数组，每 17 字节对应一个 flock：\n\n```go\ntype flock struct {\n sid uint64\n owner uint64\n ltype uint8\n}\n```\n\n#### Plock {#tkv-plock}\n\n```\nP${inode} -> plocks\n```\n\n其中 plocks 是一段字节数组，对应的 plock 是变长的：\n\n```go\ntype plock struct {\n sid uint64\n owner uint64\n size uint32\n records []byte\n}\n```\n\n其中 size 是 records 数组的长度，records 中每 24 字节对应一个 [plockRecord](#plock)。\n\n#### DelFiles\n\n```\nD${inode}${length} -> timestamp\n```\n\n其中 length 占 8 个字节，使用**大端**编码。\n\n#### DelSlices {#tkv-delslices}\n\n```\nL${timestamp}${sliceId} -> slices\n```\n\n其中 slices 是一段字节数组，每 12 字节对应一个 [slice](#delslices)。\n\n#### Sustained\n\n```\nSS${sid}${inode} -> 1\n```\n\n这里 Value 值仅用来占位。\n\n## 4 文件数据格式\n\n### 根据路径查找文件\n\n根据 [Edge](#edge) 的设计，元数据引擎中只记录了每个目录的直接子节点。当应用提供一个路径来访问文件时，JuiceFS 需要逐级查找。现在假设应用想打开文件 `/dir1/dir2/testfile`，则需要：\n\n1. 在根目录（Inode 号固定为 1）的 Edge 结构中搜寻 name 为 \"dir1\" 的 entry，得到其 inode 号 N1\n2. 在 N1 的 Edge 结构中搜寻 name 为 \"dir2\" 的 entry，得到其 inode 号 N2\n3. 在 N2 的 Edge 结构中搜寻 name 为 \"testfile\" 的 entry，得到其 inode 号 N3\n4. 根据 N3 搜寻其对应的 [Node](#node) 结构，得到该文件的相关属性\n\n在以上步骤中，任何一步搜寻失败都会导致该路径指向的文件未找到。\n\n### 文件数据拆分\n\n上一节中，我们已经可以根据文件的路径找到此文件，并获取到其属性。根据文件属性中的 inode 和 size 字段，即可找到跟文件内容相关的元数据。现在假设有个文件的 inode 为 100，size 为 160 MiB，那么该文件一共有 `(size-1) / 64 MiB + 1 = 3` 个 Chunks，如下：\n\n```\n File: |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _|\nChunk: |<--- Chunk 0 --->|<--- Chunk 1 --->|<-- Chunk 2 -->|\n```\n\n在单机 Redis 中，这意味着有 3 个 [Chunk Keys](#chunk)，分别为 `c100_0`， `c100_1` 和 `c100_2`，每个 Key 对应一个 Slices 列表。这些 Slices 主要在数据写入时生成，可能互相之间有覆盖，也可能未完全填充满 Chunk。因此，在使用前需要顺序遍历这个 Slices 列表，并重新构建出最新版的数据分布，做到：\n\n1. 有多个 Slice 覆盖的部分以最后加入的 Slice 为准\n2. 没有被 Slice 覆盖的部分自动补零，用 sliceId = 0 来表示\n3. 根据文件 size 截断 Chunk\n\n现假设 Chunk 0 中有 3 个 Slices，分别为：\n\n```go\nSlice{pos: 10M, id: 10, size: 30M, off: 0, len: 30M}\nSlice{pos: 20M, id: 11, size: 16M, off: 0, len: 16M}\nSlice{pos: 16M, id: 12, size: 10M, off: 0, len: 10M}\n```\n\n图示如下（每个 '_' 表示 2 MiB）：\n\n```\n Chunk: |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|\nSlice 10: |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _|\nSlice 11: |_ _ _ _ _ _ _ _|\nSlice 12: |_ _ _ _ _|\n\nNew List: |_ _ _ _ _|_ _ _|_ _ _ _ _|_ _ _ _ _|_ _|_ _ _ _ _ _ _ _ _ _ _ _|\n 0 10 12 11 10 0\n```\n\n重构后的新列表包含且仅包含了此 Chunk 的最新数据分布，具体如下：\n\n```go\nSlice{pos: 0, id: 0, size: 10M, off: 0, len: 10M}\nSlice{pos: 10M, id: 10, size: 30M, off: 0, len: 6M}\nSlice{pos: 16M, id: 12, size: 10M, off: 0, len: 10M}\nSlice{pos: 26M, id: 11, size: 16M, off: 6M, len: 10M}\nSlice{pos: 36M, id: 10, size: 30M, off: 26M, len: 4M}\nSlice{pos: 40M, id: 0, size: 24M, off: 0, len: 24M} // 实际这一段也会省去\n```\n\n### 数据对象\n\n#### 对象命名 {#object-storage-naming-format}\n\nBlock 是 JuiceFS 管理数据的基本单元，其大小默认为 4 MiB，且可在文件系统格式化时配置，允许调整的区间范围为 [64 KiB, 16 MiB]。每个 Block 上传后即为对象存储中的一个对象，其命名格式为 `${fsname}/chunks/${hash}/${basename}`，其中：\n\n- fsname 是文件系统名称\n- “chunks”为固定字符串，代表 JuiceFS 的数据对象\n- hash 是根据 basename 算出来的哈希值，起到一定的隔离管理的作用\n- basename 是对象的有效名称，格式为 `${sliceId}_${index}_${size}`，其中：\n - sliceId 为该对象所属 Slice 的 ID，JuiceFS 中每个 Slice 都有一个全局唯一的 ID\n - index 是该对象在所属 Slice 中的序号，默认一个 Slice 最多能拆成 16 个 Blocks，因此其取值范围为 [0, 16)\n - size 是该 Block 的大小，默认情况下其取值范围为 (0, 4 MiB]\n\n目前使用的 hash 算法有两种，以 basename 中的 sliceId 为参数，根据文件系统格式化时的 [HashPrefix](#setting) 配置选择：\n\n```go\nfunc hash(sliceId int) string {\n if HashPrefix {\n return fmt.Sprintf(\"%02X/%d\", sliceId%256, sliceId/1000/1000)\n }\n return fmt.Sprintf(\"%d/%d\", sliceId/1000/1000, sliceId/1000)\n}\n```\n\n假设一个名为 `jfstest` 的文件系统中写入了一段连续的 10 MiB 数据，内部赋予的 SliceID 为 1，且未开启 HashPrefix，那么在对象存储中则会产生以下三个对象：\n\n```\njfstest/chunks/0/0/1_0_4194304\njfstest/chunks/0/0/1_1_4194304\njfstest/chunks/0/0/1_2_2097152\n```\n\n类似地，现在以上一节的 64 MiB 的 Chunk 为例，它的实际数据分布如下：\n\n```\n 0 ~ 10M: 补零\n10 ~ 16M: 10_0_4194304, 10_1_4194304(0 ~ 2M)\n16 ~ 26M: 12_0_4194304, 12_1_4194304, 12_2_2097152\n26 ~ 36M: 11_1_4194304(2 ~ 4M), 11_2_4194304, 11_3_4194304\n36 ~ 40M: 10_6_4194304(2 ~ 4M), 10_7_2097152\n40 ~ 64M: 补零\n```\n\n据此，客户端可以快速找到应用所需数据。例如，在 offset 为 10MiB 位置读取 8MiB 数据，会涉及 3 个对象，具体为：\n\n- 从 `10_0_4194304` 读取整个对象，对应读取数据的 0 ～ 4 MiB\n- 从 `10_1_4194304` 读取 0 ～ 2 MiB，对应读取数据的 4 ～ 6 MiB\n- 从 `12_0_4194304` 读取 0 ～ 2 MiB，对应读取数据的 6 ～ 8 MiB\n\n为方便直接查看文件内容对应的对象列表，JuiceFS 提供了 `info` 命令，如 `juicefs info /mnt/jfs/test.tmp`：\n\n```bash\nobjects:\n+------------+---------------------------------+----------+---------+----------+\n| chunkIndex | objectName | size | offset | length |\n+------------+---------------------------------+----------+---------+----------+\n| 0 | | 10485760 | 0 | 10485760 |\n| 0 | jfstest/chunks/0/0/10_0_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/10_1_4194304 | 4194304 | 0 | 2097152 |\n| 0 | jfstest/chunks/0/0/12_0_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/12_1_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/12_2_2097152 | 2097152 | 0 | 2097152 |\n| 0 | jfstest/chunks/0/0/11_1_4194304 | 4194304 | 2097152 | 2097152 |\n| 0 | jfstest/chunks/0/0/11_2_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/11_3_4194304 | 4194304 | 0 | 4194304 |\n| 0 | jfstest/chunks/0/0/10_6_4194304 | 4194304 | 2097152 | 2097152 |\n| 0 | jfstest/chunks/0/0/10_7_2097152 | 2097152 | 0 | 2097152 |\n| ... | ... | ... | ... | ... |\n+------------+---------------------------------+----------+---------+----------+\n```\n\n表中空的 objectName 表示文件空洞，读取时均为 0。可以看到，输出结果与之前分析一致。\n\n值得一提的是，这里的 size 是 Block 中原始数据的大小，而不是对象存储中实际对象的大小。默认情况下，原始数据拆分后直接写到对象存储，此时 size 与对象大小是相等的。但当开启了数据压缩或数据加密功能后，实际对象的大小会发生变化，此时其与 size 很可能不再相同。\n\n#### 数据压缩\n\n在文件系统格式化时可以通过 `--compress ` 参数配置压缩算法（支持 LZ4 和 zstd），使得此文件系统的所有数据 Block 会经过压缩后再上传到对象存储。此时对象名称仍与默认配置相同，且内容为原始数据经压缩算法后的结果，不携带任何其它元信息。因此，文件[文统格式化信息](#setting)中的压缩算法不允许修改，否则会导致读取已有数据失败。\n\n#### 数据加密\n\n在文件系统格式化时可以通过 `--encrypt-rsa-key ` 参数配置 RSA 私钥以开启[静态数据加密](../security/encryption.md)功能，使得此文件系统的所有数据 Block 会经过加密后再上传到对象存储。此时对象名称仍与默认配置相同，内容为一段 header 加上数据经加密算法后的结果。这段 header 里记录了用来解密的对称密钥以及随机种子，而对称密钥本身又经过 RSA 私钥加密。因此，文件[文统格式化信息](#setting)中的 RSA 私钥目前不允许修改，否则会导致读取已有数据失败。\n\n:::note 备注\n若同时开启压缩和加密，原始数据会先压缩再加密后上传到对象存储。\n:::\n" }, { "path": "docs/zh_cn/faq.md", "content": "---\ntitle: 常见问题（FAQ）\nslug: /faq\n---\n\n## 文档没能解答我的疑问\n\n请首先尝试使用「Ask AI」功能（右下角），如果 AI 助手的回答有帮到你或者给了你错误的回答，欢迎在回答里给出你的反馈。或者使用文档搜索功能（右上角），尝试用不同的关键词进行检索。\n\n如果以上方法依然未能解决你的疑问，可以加入 [JuiceFS 开源社区](https://juicefs.com/zh-cn/community)以寻求帮助。\n\n## 一般问题\n\n### JuiceFS 与 XXX 的区别是什么？\n\n请查看[「同类技术对比」](introduction/comparison/juicefs_vs_alluxio.md)文档了解更多信息。\n\n### 怎么升级 JuiceFS 客户端？\n\n首先请卸载 JuiceFS 文件系统，然后使用新版本的客户端重新挂载。\n\n### JuiceFS 的日志在哪里？\n\n不同类型的 JuiceFS 客户端获取日志的方式也不同，详情请参考[「客户端日志」](administration/fault_diagnosis_and_analysis.md#client-log)文档。\n\n### JuiceFS 是否可以直接读取对象存储中已有的文件？\n\n不可以，JuiceFS 是一个用户态文件系统，虽然它通常使用对象存储作为数据存储层，但它并不是一般意义上的对象存储访问工具。可以查看[技术架构](introduction/architecture.md)文档了解详情。\n\n如果你希望把对象存储 Bucket 中已有数据迁移到 JuiceFS，可以使用 [`juiceFS sync`](guide/sync.md)。\n\n### 如何将多台服务器组合成一个 JuiceFS 文件系统来使用？\n\n不可以，虽然 JuiceFS 支持使用本地磁盘或 SFTP 作为底层存储，但是它并不干预底层存储的逻辑结构管理。如果你希望把多台服务器的存储空间整合起来，可以考虑使用 MinIO 或 Ceph 创建对象存储集群，然后在其之上创建 JuiceFS 文件系统。\n\n## 元数据相关问题\n\n### 支持哨兵或者集群模式的 Redis 作为 JuiceFS 的元数据引擎吗？\n\n支持，另外这里还有一篇 Redis 作为 JuiceFS 元数据引擎的[最佳实践文档](administration/metadata/redis_best_practices.md)可供参考。\n\n## 对象存储相关问题\n\n### 为什么不支持某个对象存储？\n\n已经支持了绝大部分对象存储，参考这个[列表](reference/how_to_set_up_object_storage.md#supported-object-storage)。如果它跟 S3 兼容的话，也可以当成 S3 来使用。否则，请创建一个 issue 来增加支持。\n\n### 为什么我在挂载点删除了文件，但是对象存储占用空间没有变化或者变化很小？\n\n第一个原因是你可能开启了回收站特性。为了保证数据安全回收站默认开启，删除的文件其实被放到了回收站，实际并没有被删除，所以对象存储大小不会变化。回收站的保留时间可以通过 `juicefs format` 指定或者通过 `juicefs config` 修改。请参考[「回收站」](security/trash.md)文档了解更多信息。\n\n第二个原因是 JuiceFS 是异步删除对象存储中的数据，所以对象存储的空间变化会慢一点。如果你需要立即清理对象存储中需要被删除的数据，可以尝试运行 [`juicefs gc`](reference/command_reference.mdx#gc) 命令。\n\n### 关于 JuiceFS 的异步删除，具体流程是怎样的？\n\n* ​**当未开启回收站时：​**\n - 系统会检查文件是否被其他程序使用：\n * 如果文件正在被使用，会标记为**”暂缓删除 (`sustained`)**\"，等程序关闭文件后再处理\n * 如果文件没有被使用，会标记为**待删除 (`delfile`)**，尝试将其放入**删除队列 (`maxDeleting`)**\n \n* ​**当开启回收站时：​**\n - 系统会在回收站中按照**当前时间（精确到小时）​** 创建子目录（如`2024-01-15-14`）\n - 待删除文件移动到对应时间目录中：\n * ​**所有 chunk 和 slice 数据均保持完整**\n * ​**仅元数据**中的父目录指向发生变化\n * 文件名会被**重新编码**以避免冲突\n - 后台任务根据保留天数清理过期文件：\n * 从**最老的目录开始**逐个清理\n * 方法：打上**待删除 (`delfile`)**，放入**删除队列 (`maxDeleting`)**\n \n* ​**删除队列处理流程（异步清理）：​**\n 1. ​**查找文件对应的所有 chunk 并删除**\n 2. 删除 chunk 时会**减少其 slice 的引用计数**\n 3. 当 slice 的引用计数减为零，成为 ​**`Pending Deleted Slices`**\n 4. 后台清理对象存储中的这些数据片段\n\n![JuiceFS-delete-file](./images/juicefs-delete-file.svg)\n\n* 删除队列是有容量限制的，如果同时删除的文件过多，队列满后删除请求会先返回。然后由一个每小时工作一次的后台清理任务继续清理。它查找所有标记为**待删除 (`delfile`)**的文件，对其进行清理，清理方法和删除队列中的文件一致。\n* 如果配置了 NoBGJob，每小时间隔的后台定时清理任务和回收站清理任务都会被禁用，删除文件后需要手动去回收站中进一步清理。\n* 一种特殊情况是，当你手动直接删除回收站里的文件时，它可以确保以同步的方式插入到删除队列中，相对较快地回收对象存储空间，但是后续清理 chunk 的动作仍然是异步的。\n* 关于 slice 引用计数：删除 chunk 和碎片整理 compact 会减少相关 slice 的引用计数，clone 和 copyFileRange 则会增加相关 slice 的引用计数。\n\n### 为什么文件系统数据量与对象存储占用空间存在差异？ {#size-inconsistency}\n\n* [JuiceFS 随机写](#random-write)会产生文件碎片，因此对象存储的占用空间大部分情况下是大于等于实际大小的，尤其是短时间内进行大量的覆盖写产生许多文件碎片后，这些碎片仍旧占用着对象存储的空间。不过也不必担心，因为在每次读／写文件的时候都会检查，并在后台任务进行该文件相关碎片的整理工作。你可以通过 [`juicefs gc —-compact -—delete`](./reference/command_reference.mdx#gc) 命令手动触发合并与回收。\n* 如果开启了[「回收站」](./security/trash.md)功能，被删除的文件不会立刻清理，而是在回收站内保留指定时间后，才进行清理删除。\n* 碎片被合并以后，失效的旧碎片也会在回收站中进行保留（但对用户不可见），过期时间也遵循回收站的设置。如果想要清理这些碎片，阅读[回收站和文件碎片](./security/trash.md#gc)。\n* 如果文件系统开启了压缩功能（也就是 [`format`](./reference/command_reference.mdx#format) 命令的 `--compress` 参数，默认不开启），那么对象存储上存储的对象有可能比实际文件大小更小（取决于不同类型文件的压缩比）。\n* 根据所使用对象存储的[存储类型](reference/how_to_set_up_object_storage.md#storage-class)不同，云服务商可能会针对某些存储类型设置最小计量单位。例如阿里云 OSS 低频访问存储的[最小计量单位](https://help.aliyun.com/document_detail/173534.html)是 64KB，如果单个文件小于 64KB 也会按照 64KB 计算。\n* 对于自建对象存储，例如 MinIO，实际占用大小也受到[存储级别](https://github.com/minio/minio/blob/master/docs/erasure/storage-class/README.md)设置的影响。\n\n### JuiceFS 支持使用对象存储中的某个目录作为 `--bucket` 选项的值吗？\n\n到 JuiceFS 1.0 为止，还不支持该功能。\n\n### JuiceFS 支持访问对象存储中已经存在的数据吗？\n\n到 JuiceFS 1.0 为止，还不支持该功能。\n\n### 一个文件系统可以绑定多个不同的对象存储吗（比如同时用 Amazon S3、GCS 和 OSS 组成一个文件系统）？\n\n不支持。但在创建文件系统时可以设定关联同一个对象存储的多个 bucket，从而解决单个 bucket 对象数量限制的问题，例如，可以为一个文件系统关联多个 S3 Bucket。具体请参考 [`--shards`](./reference/command_reference.mdx#format) 选项的说明。\n\n## 性能相关问题\n\n### JuiceFS 的性能如何？\n\nJuiceFS 是一个分布式文件系统，元数据访问的延时取决于挂载点到服务端之间 1 到 2 个网络来回（通常 1-3 ms），数据访问的延时取决于对象存储的延时 (通常 20-100 ms)。顺序读写的吞吐量可以到 50MiB/s 至 2800MiB/s（查看 [fio 测试结果](benchmark/fio.md)），取决于网络带宽以及数据是否容易被压缩。\n\nJuiceFS 内置多级缓存（主动失效），一旦缓存预热好，访问的延时和吞吐量非常接近单机文件系统的性能（FUSE 会带来少量的开销）。\n\n### JuiceFS 支持随机读写吗？原理如何？ {#random-write}\n\n支持，包括通过 mmap 等进行的随机读写。目前 JuiceFS 主要是对顺序读写进行了大量优化，对随机读写的优化也在进行中。如果想要更好的随机读性能，建议关闭压缩（[`--compress none`](reference/command_reference.mdx#format)）。\n\nJuiceFS 不将原始文件存入对象存储，而是将其按照某个大小（默认为 4MiB）拆分为 N 个数据块（Block）后，上传到对象存储，然后将数据块的 ID 存入元数据引擎。随机写的时候，逻辑上是要覆盖原本的内容，实际上是把**要覆盖的数据块**的元数据标记为旧数据，同时只上传随机写时产生的**新数据块**到对象存储，并将**新数据块**对应的元数据更新到元数据引擎中。\n\n当读取被覆盖部分的数据时，根据**最新的元数据**，从随机写时上传的**新数据块**读取即可，同时**旧数据块**可能会被后台运行的垃圾回收任务自动清理。这样就将随机写的复杂度转移到读的复杂度上。\n\n详见[「内部实现」](development/internals.md)与[「读写请求处理流程」](introduction/io_processing.md)。\n\n### 怎么快速地拷贝大量小文件到 JuiceFS？\n\n请在挂载时加上 [`--writeback` 选项](reference/command_reference.mdx#mount-data-cache-options)，它会先把数据写入本机的缓存，然后再异步上传到对象存储，会比直接上传到对象存储快很多倍。\n\n请查看[「客户端写缓存」](guide/cache.md#client-write-cache)了解更多信息。\n\n### JuiceFS 支持分布式缓存吗？\n\n企业版支持，详见[「分布式缓存」](https://juicefs.com/docs/zh/cloud/guide/distributed-cache)。\n\n## 访问相关问题\n\n### 为什么同名用户在主机 X 上有权限访问 JuiceFS 的文件，在主机 Y 上访问该文件却没有权限？\n\n虽然用户在主机 X 和主机 Y 上的用户名相同，但各自对应的 UID 或 GID 不相同。你可以使用 `id` 命令来查看用户的具体 UID 和 GID：\n\n```bash\n$ id alice\nuid=1201(alice) gid=500(staff) groups=500(staff)\n```\n\n阅读文档[「多主机间同步账户」](administration/sync_accounts_between_multiple_hosts.md)解决这个问题。\n\n### JuiceFS 除了挂载外还支持哪些方式访问数据？\n\n除了挂载外，还支持以下几种方式：\n\n- Kubernetes CSI 驱动：通过 Kubernetes CSI 驱动的方式将 JuiceFS 作为 Kubernetes 集群的存储层，详情请参考[「JuiceFS CSI 驱动」](deployment/how_to_use_on_kubernetes.md)。\n- Hadoop Java SDK：方便在 Hadoop 体系中使用兼容 HDFS 接口的 Java 客户端访问 JuiceFS。详情请参考[「Hadoop 使用 JuiceFS」](deployment/hadoop_java_sdk.md)。\n- S3 网关：通过 S3 协议访问 JuiceFS，详情请参考[「配置 JuiceFS S3 网关」](./guide/gateway.md)。\n- Docker Volume 插件：在 Docker 中方便使用 JuiceFS 的方式，详情请参考[「Docker 使用 JuiceFS」](deployment/juicefs_on_docker.md)。\n- WebDAV 网关：通过 WebDAV 协议访问 JuiceFS\n\n### JuiceFS S3 网关支持多用户管理等高级功能吗？\n\nJuiceFS 内置的 `gateway` 从 1.2 版本开始支持多用户管理等高级功能。\n\n### JuiceFS 目前有 SDK 可以使用吗？\n\n截止到 JuiceFS 1.0 发布，社区有两个 SDK，一个是 Juicedata 官方维护的 HDFS 接口高度兼容的 [Java SDK](deployment/hadoop_java_sdk.md)，另一个是由社区用户维护的 [Python SDK](https://github.com/megvii-research/juicefs-python)。\n" }, { "path": "docs/zh_cn/getting-started/for_distributed.md", "content": "---\nsidebar_position: 3\ndescription: 本文将指导你使用基于云的对象存储和数据库，构建一个具有分布式和共享访问能力的 JuiceFS 文件系统。\n---\n\n# 分布式模式\n\n上一篇文档[「JuiceFS 单机模式快速上手指南」](./standalone.md)通过采用「对象存储」和「SQLite」数据库的组合，实现了一个可以在任意主机上挂载的文件系统。得益于对象存储是可以被网络上任何有权限的计算机访问的特点，我们只需要把 SQLite 数据库文件复制到任何想要访问该存储的计算机，就可以实现在不同计算机上访问同一个 JuiceFS 文件系统。\n\n很显然，想要依靠在计算机之间复制 SQLite 数据库的方式进行文件系统共享，虽然可行，但文件的实时性是得不到保证的。受限于 SQLite 这种单文件数据库无法被多个计算机同时读写访问的情况，为了能够让一个文件系统可以在分布式环境中被多个计算机同时挂载读写，我们需要采用支持通过网络访问的数据库，比如 Redis、PostgreSQL、MySQL 等。\n\n本文以上一篇文档为基础，进一步将数据库从单用户的「SQLite」替换成多用户的「云数据库」，从而实现可以在网络上任何一台计算机上进行挂载读写的分布式文件系统。\n\n## 快速上手视频\n\n
\n \n
\n\n## 基于网络的数据库\n\n这里所谓的「基于网络的数据库」是指允许多个用户通过网络同时访问的数据库，从这个角度出发，可以简单的把数据库分成：\n\n1. **单机数据库**：数据库是单个文件，通常只能单机访问，如 SQLite，Microsoft Access 等；\n2. **基于网络的数据库**：数据库通常是复杂的多文件结构，提供基于网络的访问接口，支持多用户同时访问，如 Redis、PostgreSQL 等。\n\nJuiceFS 目前支持的基于网络的数据库有：\n\n- **键值数据库**：Redis、TiKV、etcd、FoundationDB\n- **关系型数据库**：PostgreSQL、MySQL、MariaDB\n\n不同的数据库性能和稳定性表现也各不相同，比如 Redis 是内存型键值数据库，性能极为出色，但可靠性相对较弱。PostgreSQL 是关系型数据库，相比之下性能没有内存型强悍，但它的可靠性要更强。\n\n有关数据库选择方面的内容，我们会专门编写文档进行介绍。\n\n## 云数据库\n\n云计算平台通常都有种类丰富的云数据库提供，比如 Amazon RDS 提供各类关系型数据库的版本，Amazon ElastiCache 提供兼容 Redis 的内存型数据库产品。经过简单的初始化设置就可以创建出多副本、高可用的数据库集群。\n\n当然，如果愿意，你可以自己在服务器上搭建数据库。\n\n简单起见，这里以阿里云数据库 Redis 版为例介绍。对于基于网络的数据库来说，最基本的是以下 2 项信息：\n\n1. **数据库地址**：数据库的访问地址，云平台可能会针对内外网提供不同的链接；\n2. **用户名和密码**：用于访问数据库时的身份验证信息。\n\n## 上手实践\n\n### 1. 安装客户端\n\n在所有需要挂载文件系统的计算机上安装 JuiceFS 客户端，详情参照[「安装」](installation.md)。\n\n### 2. 准备对象存储\n\n以下是以阿里云 OSS 为例的伪样本，你可以改用其他对象存储，详情参考 [JuiceFS 支持的存储](../reference/how_to_set_up_object_storage.md#supported-object-storage)。\n\n- **Bucket Endpoint**：`https://myjfs.oss-cn-shanghai.aliyuncs.com`\n- **Access Key ID**：`ABCDEFGHIJKLMNopqXYZ`\n- **Access Key Secret**：`ZYXwvutsrqpoNMLkJiHgfeDCBA`\n\n### 3. 准备数据库\n\n以下是以阿里云数据库 Redis 版为例的伪样本，你可以改用其他类型的数据库，详情参考 [JuiceFS 支持的数据库](../reference/how_to_set_up_metadata_engine.md)。\n\n- **数据库地址**：`myjfs-sh-abc.redis.rds.aliyuncs.com:6379`\n- **数据库用户名**：`tom`\n- **数据库密码**：`mypassword`\n\n在 JuiceFS 中使用 Redis 数据库的格式如下：\n\n```\nredis://:@:6379/1\n```\n\n:::tip 提示\nRedis 6.0 之前的版本没有用户名，请省略 URL 中的 `` 部分，例如 `redis://:mypassword@myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1`（请注意密码前面的冒号是分隔符，需要保留）。\n:::\n\n### 4. 创建文件系统\n\n以下命令使用「对象存储」和「Redis」数据库的组合创建了一个支持跨网络、多机同时挂载、共享读写的文件系统。\n\n```shell\njuicefs format \\\n --storage oss \\\n --bucket https://myjfs.oss-cn-shanghai.aliyuncs.com \\\n --access-key ABCDEFGHIJKLMNopqXYZ \\\n --secret-key ZYXwvutsrqpoNMLkJiHgfeDCBA \\\n redis://tom:mypassword@myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n myjfs\n```\n\n文件系统创建完成后，终端将返回类似下面的内容：\n\n```shell\n2021/12/16 16:37:14.264445 juicefs[22290] : Meta address: redis://@myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1\n2021/12/16 16:37:14.277632 juicefs[22290] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/12/16 16:37:14.281432 juicefs[22290] : Ping redis: 3.609453ms\n2021/12/16 16:37:14.527879 juicefs[22290] : Data uses oss://myjfs/myjfs/\n2021/12/16 16:37:14.593450 juicefs[22290] : Volume is formatted as {Name:myjfs UUID:4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b Storage:oss Bucket:https://myjfs AccessKey:ABCDEFGHIJKLMNopqXYZ SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n:::info\n文件系统创建完毕以后，包含对象存储密钥等信息会完整的记录到数据库中。JuiceFS 客户端只要拥有数据库地址、用户名和密码信息，就可以挂载读写该文件系统。也正因此，JuiceFS 客户端没有本地配置文件（作为对比，JuiceFS 云服务用 [`juicefs auth`](https://juicefs.com/docs/zh/cloud/reference/commands_reference/#auth) 命令进行认证、获取配置文件）。\n:::\n\n### 5. 挂载文件系统\n\n由于这个文件系统的「数据」和「元数据」都存储在基于网络的云服务中，因此在任何安装了 JuiceFS 客户端的计算机上都可以同时挂载该文件系统进行共享读写。例如：\n\n```shell\njuicefs mount redis://tom:mypassword@myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1 ~/jfs\n```\n\n#### 数据强一致性保证\n\n对于多客户端同时挂载读写同一个文件系统的情况，JuiceFS 提供「关闭再打开（close-to-open）」一致性保证，即当两个及以上客户端同时读写相同的文件时，客户端 A 的修改在客户端 B 不一定能立即看到。但是，一旦这个文件在客户端 A 写入完成并关闭，之后在任何一个客户端重新打开该文件都可以保证能访问到最新写入的数据，不论是否在同一个节点。\n\n#### 调大缓存提升性能\n\n由于「对象存储」是基于网络的存储服务，不可避免会产生访问延时。为了解决这个问题，JuiceFS 提供并默认启用了缓存机制，即划拨一部分本地存储作为数据与对象存储之间的一个缓冲层，读取文件时会异步地将数据缓存到本地存储，详情请查阅[「缓存」](../guide/cache.md)。\n\n缓存机制让 JuiceFS 可以高效处理海量数据的读写任务，默认情况下，JuiceFS 会在 `$HOME/.juicefs/cache` 或 `/var/jfsCache` 目录设置 100GiB 的缓存。在速度更快的 SSD 上设置更大的缓存空间可以有效提升 JuiceFS 的读写性能。\n\n你可以使用 `--cache-dir` 调整缓存目录的位置，使用 `--cache-size` 调整缓存空间的大小，例如：\n\n```shell\njuicefs mount\n --background \\\n --cache-dir /mycache \\\n --cache-size 512000 \\\n redis://tom:mypassword@myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n ~/jfs\n```\n\n:::note 注意\nJuiceFS 进程需要具有读写 `--cache-dir` 目录的权限。\n:::\n\n上述命令将缓存目录设置在了 `/mycache` 目录，并指定缓存空间为 500GiB。\n\n#### 开机自动挂载\n\n在 Linux 环境中，可以在挂载文件系统时通过 `--update-fstab` 选项设置自动挂载，这个选项会将挂载 JuiceFS 所需的选项添加到 `/etc/fstab` 中。例如：\n\n:::note 注意\n此特性需要使用 1.1.0 及以上版本的 JuiceFS\n:::\n\n```bash\n$ sudo juicefs mount --update-fstab --max-uploads=50 --writeback --cache-size 204800 redis://tom:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1 \n$ grep /etc/fstab\nredis://tom:mypassword@myjfs-sh-abc.apse1.cache.amazonaws.com:6379/1 juicefs _netdev,max-uploads=50,writeback,cache-size=204800 0 0\n$ ls -l /sbin/mount.juicefs\nlrwxrwxrwx 1 root root 29 Aug 11 16:43 /sbin/mount.juicefs -> /usr/local/bin/juicefs\n```\n\n更多请参考[「启动时自动挂载 JuiceFS」](../administration/mount_at_boot.md)。\n\n### 6. 验证文件系统\n\n当挂载好文件系统以后可以通过 `juicefs bench` 命令对文件系统进行基础的性能测试和功能验证，确保 JuiceFS 文件系统能够正常访问且性能符合预期。\n\n:::info 说明\n`juicefs bench` 命令只能完成基础的性能测试，如果需要对 JuiceFS 进行更完整的评估，请参考[「JuiceFS 性能评估指南」](../benchmark/performance_evaluation_guide.md)。\n:::\n\n```shell\njuicefs bench ~/jfs\n```\n\n运行 `juicefs bench` 命令以后会根据指定的并发度（默认为 1）往 JuiceFS 文件系统中写入及读取 N 个大文件（默认为 1）及 N 个小文件（默认为 100），并统计读写的吞吐和单次操作的延迟，以及访问元数据引擎的延迟。\n\n如果在验证文件系统的过程中遇到任何问题，请先参考[「故障诊断和分析」](../administration/fault_diagnosis_and_analysis.md)文档进行问题排查。\n\n### 7. 卸载文件系统\n\n你可以通过 `juicefs umount` 命令卸载 JuiceFS 文件系统（假设挂载点路径是 `~/jfs`）：\n\n```shell\njuicefs umount ~/jfs\n```\n\n如果执行命令后，文件系统卸载失败，提示 `Device or resource busy`：\n\n```shell\n2021-05-09 22:42:55.757097 I | fusermount: failed to unmount ~/jfs: Device or resource busy\nexit status 1\n```\n\n发生这种情况，可能是因为某些程序正在读写文件系统中的文件。为了确保数据安全，你应该首先排查是哪些程序正在与文件系统中的文件进行交互（例如通过 `lsof` 命令），并尝试结束它们之间的交互动作，然后再重新执行卸载命令。\n\n:::caution 注意\n以下内容包含的命令可能会导致文件损坏、丢失，请务必谨慎操作！\n:::\n\n当然，在你能够确保数据安全的前提下，也可以在卸载命令中添加 `--force` 或 `-f` 参数，强制卸载文件系统：\n\n```shell\njuicefs umount --force ~/jfs\n```\n" }, { "path": "docs/zh_cn/getting-started/installation.md", "content": "---\ntitle: 安装\nsidebar_position: 1\ndescription: 本文介绍 JuiceFS 在 Linux、macOS 和 Windows 上的安装方法，包括一键安装、编译安装和容器化安装。\n---\n\nJuiceFS 有良好的跨平台能力，支持在几乎所有主流架构的各类操作系统上运行，包括且不限于 Linux、macOS、Windows 等。\n\nJuiceFS 客户端只有一个二进制文件，你可以下载预编译的版本直接解压使用，也可以用源代码手动编译。\n\n## 一键安装 {#one-click-installation}\n\n一键安装脚本适用于 Linux 和 macOS 系统，会根据你的硬件架构自动下载安装最新版 JuiceFS 客户端。\n\n**方式一（推荐）：** 默认安装到 `/usr/local/bin`：\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n**方式二：** 如需安装到自定义位置，例如安装到 `/tmp` 目录下：\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -s /tmp\n```\n\n:::tip 提示\n大多数用户应该选择**方式一**进行默认安装。只有在对安装目录有特殊要求时才使用**方式二**。\n:::\n\n## 安装预编译客户端 {#install-the-pre-compiled-client}\n\n你可以在 [GitHub](https://github.com/juicedata/juicefs/releases) 找到最新版客户端下载地址，每个版本的下载列表中都提供了面向不同 CPU 架构和操作系统的预编译版本，请注意识别选择，例如：\n\n| 文件名 | 说明 |\n|--------------------------------------|---------------------------------------------------------------------------------|\n| `juicefs-x.y.z-darwin-amd64.tar.gz` | 面向 Intel 芯片的 macOS 系统 |\n| `juicefs-x.y.z-darwin-arm64.tar.gz` | 面向 M1 系列芯片的 macOS 系统 |\n| `juicefs-x.y.z-linux-amd64.tar.gz` | 面向 x86 架构 Linux 发行版 |\n| `juicefs-x.y.z-linux-arm64.tar.gz` | 面向 ARM 架构的 Linux 发行版 |\n| `juicefs-x.y.z-windows-amd64.tar.gz` | 面向 x86 架构的 Windows 系统 |\n| `juicefs-hadoop-x.y.z.jar` | 面向 x86 和 ARM 架构的 Hadoop Java SDK（同时支持 Linux、macOS 及 Windows 系统） |\n\n### Linux 发行版 {#linux}\n\n以 x86 架构的 Linux 系统为例，下载文件名包含 `linux-amd64` 的压缩包，在终端依次执行以下命令。\n\n1. 获取最新的版本号\n\n ```shell\n JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '\"' -f 4 | tr -d 'v')\n ```\n\n2. 下载客户端到当前目录\n\n ```shell\n wget \"https://github.com/juicedata/juicefs/releases/download/v${JFS_LATEST_TAG}/juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\"\n ```\n\n3. 解压安装包\n\n ```shell\n tar -zxf \"juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\"\n ```\n\n4. 安装客户端\n\n ```shell\n sudo install juicefs /usr/local/bin\n ```\n\n完成上述 4 个步骤，在终端执行 `juicefs` 命令，返回帮助信息，则说明客户端安装成功。\n\n:::info 说明\n如果终端提示 `command not found`，可能是因为 `/usr/local/bin` 不在你的系统 `PATH` 环境变量中，可以执行 `echo $PATH` 查看系统设置了哪些可执行路径，根据返回结果选择一个恰当的路径，调整并重新执行第 4 步的安装命令。\n:::\n\n#### Ubuntu PPA\n\nJuiceFS 也提供 [PPA](https://launchpad.net/~juicefs) 仓库，可以方便地在 Ubuntu 系统上安装最新版的客户端。根据你的 CPU 架构选择对应的 PPA 仓库：\n\n- **x86 架构**：`ppa:juicefs/ppa`\n- **ARM 架构**：`ppa:juicefs/arm64`\n\n以 x86 架构的 Ubuntu 22.04 系统为例，执行以下命令。\n\n1. 添加 PPA 仓库：\n\n ```shell\n sudo add-apt-repository ppa:juicefs/ppa\n ```\n\n2. 更新包列表：\n\n ```shell\n sudo apt-get update\n ```\n\n3. 安装 JuiceFS 客户端：\n\n ```shell\n sudo apt-get install juicefs\n ```\n\n#### Fedora Copr\n\nJuiceFS 也提供 [Copr](https://copr.fedorainfracloud.org/coprs/juicedata/juicefs) 仓库，可以方便地在 Red Hat 及其衍生系统上安装最新版的客户端，目前支持的系统有：\n\n- **Amazonlinux 2023**\n- **CentOS 8, 9**\n- **Fedora 37, 38, 39, rawhide**\n- **RHEL 7, 8, 9**\n\n以 Fedora 38 系统为例，执行以下命令安装客户端：\n\n启用 Copr 仓库：\n\n```shell\nsudo dnf copr enable -y juicedata/juicefs\n```\n\n安装客户端：\n\n```shell\nsudo dnf install juicefs\n```\n\n#### Snapcraft\n\n我们也在 [Canonical Snapcraft](https://snapcraft.io) 平台打包并发布了 [Snap 版本的 JuiceFS 客户端](https://github.com/juicedata/juicefs-snapcraft)，对于 Ubuntu 16.04 及以上版本和其他支持 Snap 的操作系统，可以直接使用以下命令安装：\n\n```shell\nsudo snap install juicefs\n```\n\n由于 Snap 是一个封闭的沙箱环境，它会影响客户端的 FUSE 挂载，执行以下命令可以解除限制。如果只需使用 WebDAV 和 Gateway 则不必执行以下命令：\n\n```shell\nsudo ln -s -f /snap/juicefs/current/juicefs /snap/bin/juicefs\n```\n\n当有新版本时，执行以下命令更新客户端：\n\n```shell\nsudo snap refresh juicefs\n```\n\n#### AUR (Arch User Repository) {#aur}\n\nJuiceFS 也提供 [AUR](https://aur.archlinux.org/packages/juicefs) 仓库，可以方便地在 Arch Linux 及其衍生系统上安装最新版的客户端。\n\n对于使用 Yay 包管理器的系统，执行以下命令安装客户端：\n\n```shell\nyay -S juicefs\n```\n\n:::info 说明\nAUR 上存在多个 JuiceFS 客户端的打包，以下是 JuiceFS 官方维护的版本：\n\n- [`aur/juicefs`](https://aur.archlinux.org/packages/juicefs)：是稳定编译版，安装时会拉取最新的稳定版源码并编译安装；\n- [`aur/juicefs-bin`](https://aur.archlinux.org/packages/juicefs-bin)：是稳定预编译版，安装时会直接下载最新的稳定版预编译程序并安装；\n- [`aur/juicefs-git`](https://aur.archlinux.org/packages/juicefs-git)：是开发版，安装时会拉取最新的开发版源码并编译安装；\n\n:::\n\n另外，你也可以使用 `makepkg` 手动编译安装，以 Arch Linux 系统为例：\n\n安装依赖：\n\n```shell\nsudo pacman -S base-devel git go\n```\n\n克隆要打包的 AUR 仓库：\n\n```shell\ngit clone https://aur.archlinux.org/juicefs.git\n```\n\n进入仓库目录：\n\n```shell\ncd juicefs\n```\n\n编译安装：\n\n```shell\nmakepkg -si\n```\n\n### Windows 系统 {#windows}\n\n由于 Windows 没有原生支持 FUSE 接口，首先需要下载安装 [WinFsp](https://winfsp.dev) 才能实现对 FUSE 的支持。\n\n :::tip 提示\n **[WinFsp](https://github.com/winfsp/winfsp)** 是一个开源的 Windows 文件系统代理，它提供了一个 FUSE 仿真层，使得 JuiceFS 客户端可以将文件系统挂载到 Windows 系统中使用。\n :::\n\n#### 快速上手视频\n\n
\n \n
\n\n在 Windows 系统安装 JuiceFS 有以下几种方法：\n\n- [使用预编译的 Windows 客户端](#预编译的-windows-客户端)\n- [使用 Scoop 安装](#scoop)\n- [在 WSL 中使用 Linux 版客户端](#在-wsl-中使用-linux-版客户端)\n\n#### 预编译的 Windows 客户端\n\nJuiceFS 的 Windows 客户端为独立的二进制文件，下载并解压后即可直接运行。\n\n以 Windows 10 为例，下载包含 `windows-amd64` 的压缩包，解压后获得 `juicefs.exe`，即为 JuiceFS 客户端。\n\n为方便使用，可将 `juicefs.exe` 移动到 `C:\\Windows\\System32`，这样可在任意目录下通过命令行直接运行 `juicefs`。\n\n如需更灵活地管理 JuiceFS 客户端，可以在 `C:\\` 盘下新建 `juicefs` 文件夹，将 `juicefs.exe` 放入其中，并将 `C:\\juicefs` 添加到系统环境变量 PATH。重启系统后，即可在「命令提示符」或「PowerShell」等终端中直接使用 `juicefs` 命令。\n\n![Windows ENV path](../images/windows-path.png)\n\n#### 使用 Scoop 安装 {#scoop}\n\n如果你的 Windows 系统中安装了 [Scoop](https://scoop.sh)，可以使用以下命令安装最新版的 JuiceFS 客户端：\n\n```shell\nscoop install juicefs\n```\n\n#### 在 WSL 中使用 Linux 版客户端\n\n[WSL](https://docs.microsoft.com/zh-cn/windows/wsl/about) 全称 Windows Subsystem for Linux，即 Windows 的 Linux 子系统，从 Windows 10 版本 2004 以上或 Windows 11 开始支持该功能。它可以让你在 Windows 系统中运行原生的 GNU/Linux 的大多数命令行工具、实用工具和应用程序且不会产生传统虚拟机或双启动设置开销。\n\n详情查看「[在 WSL 中使用 JuiceFS](../tutorials/juicefs_on_wsl.md)」\n\n### macOS 系统 {#macos}\n\n由于 macOS 默认不支持 FUSE 接口，需要安装 [macFUSE](https://osxfuse.github.io) 才能实现 FUSE 挂载。如果 FUSE 挂载不是你的主要使用场景，则无需安装 macFUSE。通过使用 JuiceFS 的 [WebDAV](../deployment/webdav.md)、[Gateway](../guide/gateway.md)、[Python SDK](../deployment/python_sdk.md) 等访问方式也能方便地读写数据。\n\n:::tip 提示\n[macFUSE](https://github.com/osxfuse/osxfuse) 是一个开源的文件系统增强工具，它让 macOS 可以挂载第三方的文件系统，使得 JuiceFS 客户端可以将文件系统挂载到 macOS 系统中使用。\n:::\n\n#### Homebrew 安装\n\n如果你的系统安装了 [Homebrew](https://brew.sh) 包管理器，可以执行以下命令安装 JuiceFS 客户端：\n\n```shell\nbrew install juicefs\n```\n\n*请参考 [Homebrew Formulae](https://formulae.brew.sh/formula/juicefs#default) 页面了解命令详情。*\n\n#### 预编译二进制程序\n\n你也可以下载文件名包含 `darwin-amd64` 的二进制程序，解压后使用 `install` 命令将程序安装到系统的任意可执行路径，例如：\n\n```shell\nsudo install juicefs /usr/local/bin\n```\n\n### Docker 容器 {#docker}\n\n对于要在 Docker 容器中使用 JuiceFS 的情况，这里提供一份构建 JuiceFS 客户端镜像的 `Dockerfile`，可以以此为基础单独构建 JuiceFS 客户端镜像或与其他应用打包在一起使用。\n\n```dockerfile\nFROM ubuntu:20.04\n\nRUN apt update && apt install -y curl fuse && \\\n apt-get autoremove && \\\n apt-get clean && \\\n rm -rf \\\n /tmp/* \\\n /var/lib/apt/lists/* \\\n /var/tmp/*\n\nRUN set -x && \\\n mkdir /juicefs && \\\n cd /juicefs && \\\n JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '\"' -f 4 | tr -d 'v') && \\\n curl -s -L \"https://github.com/juicedata/juicefs/releases/download/v${JFS_LATEST_TAG}/juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz\" \\\n | tar -zx && \\\n install juicefs /usr/bin && \\\n cd .. && \\\n rm -rf /juicefs\n\nCMD [ \"juicefs\" ]\n```\n\n## 手动编译客户端 {#manually-compiling}\n\n如果预编译的客户端中没有适用于你的版本（比如 FreeBSD），这时可以采用手动编译的方式编译适合你的 JuiceFS 客户端。\n\n另外，手动编译客户端可以让你优先体验到 JuiceFS 开发中的各种新功能，但这需要你具备一定的软件编译相关的基础知识。\n\n:::tip 提示\n对于中国地区用户，为了加快获取 Go 模块的速度，建议通过执行 `go env -w GOPROXY=https://goproxy.cn,direct` 来将 `GOPROXY` 环境变量设置国内的镜像服务器。详情请参考：[Goproxy China](https://github.com/goproxy/goproxy.cn)。\n:::\n\n### 类 Unix 客户端\n\n编译面向 Linux、macOS、BSD 等类 Unix 系统的客户端需要满足以下依赖：\n\n- [Go](https://golang.org) 1.20+\n- GCC 5.4+\n\n1. 克隆源码\n\n ```shell\n git clone https://github.com/juicedata/juicefs.git\n ```\n\n2. 进入源代码目录\n\n ```shell\n cd juicefs\n ```\n\n3. 切换分支\n\n 源代码默认使用 `main` 分支，你可以切换到任何正式发布的版本，比如切换到 `v1.0.0` 版本：\n\n ```shell\n git checkout v1.0.0\n ```\n\n :::caution 注意\n 开发分支经常涉及较大的变化，请不要将「开发分支」编译的客户端用于生产环境。\n :::\n\n4. 执行编译\n\n ```shell\n make\n ```\n\n 编译好的 `juicefs` 二进制程序位于当前目录。\n\n### 在 Windows 下编译\n\n在 Windows 系统编译 JuiceFS 客户端需要安装以下依赖：\n\n- [WinFsp](https://github.com/winfsp/winfsp)\n- [Go](https://golang.org) 1.20+\n- GCC 5.4+\n\n其中，WinFsp 和 Go 直接下载安装即可。GCC 需要使用第三方提供的版本，可以使用 [MinGW-w64](https://www.mingw-w64.org) 或 [Cygwin](https://www.cygwin.com)，这里以 MinGW-w64 为例介绍。\n\n在 [MinGW-w64 的下载页面](https://www.mingw-w64.org/downloads) 选择一个适用于 Windows 的预编译版本，比如 [mingw-builds-binaries](https://github.com/niXman/mingw-builds-binaries/releases)。下载完成后，将其解压到 `C` 盘根目录，然后在系统环境变量设置中找到 PATH 并添加 `C:\\mingw64\\bin` 目录，重启系统后在命令行或 PowerShell 中执行 `gcc -v` 命令，如果能看到版本信息则说明 MingGW-w64 安装成功，接下来就可以开始编译了。\n\n1. 克隆并进入项目目录\n\n ```shell\n git clone https://github.com/juicedata/juicefs.git && cd juicefs\n ```\n\n2. 复制 WinFsp 头文件\n\n ```shell\n mkdir \"C:\\WinFsp\\inc\\fuse\"\n ```\n\n ```shell\n copy .\\hack\\winfsp_headers\\* C:\\WinFsp\\inc\\fuse\\\n ```\n\n ```shell\n dir \"C:\\WinFsp\\inc\\fuse\"\n ```\n\n ```shell\n set CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n ```\n\n ```shell\n go env -w CGO_CFLAGS=-IC:/WinFsp/inc/fuse\n ```\n\n3. 编译客户端\n\n ```shell\n go build -ldflags=\"-s -w\" -o juicefs.exe .\n ```\n\n编译好的 `juicefs.exe` 二进制程序位于当前目录。为了方便使用，可以将其移动到 `C:\\Windows\\System32` 目录下，这样就可以在任何地方直接使用 `juicefs.exe` 命令了。\n\n### 在 Linux 中交叉编译 Windows 客户端\n\n为 Windows 编译特定版本客户端的过程与[类 Unix 客户端](#类-unix-客户端)基本一致，可以直接在 Linux 系统中进行编译，但除了 `go` 和 `gcc` 必须安装以外，还需要安装 [MinGW-w64](https://www.mingw-w64.org/downloads)\n\n安装 Linux 发行版包管理器提供的最新版本即可，例如 Ubuntu 20.04+ 可以直接安装：\n\n```shell\nsudo apt install mingw-w64\n```\n\n编译 Windows 客户端：\n\n```shell\nmake juicefs.exe\n```\n\n编译好的客户端是一个名为 `juicefs.exe` 的二进制文件，位于当前目录。\n\n### 在 macOS 中交叉编译 Linux 客户端\n\n1. 克隆并进入项目目录\n\n ```shell\n git clone https://github.com/juicedata/juicefs.git && cd juicefs\n ```\n\n2. 安装依赖\n\n ```shell\n brew install FiloSottile/musl-cross/musl-cross\n ```\n\n3. 编译客户端\n\n ```shell\n make juicefs.linux\n ```\n\n## 卸载客户端 {#uninstall}\n\nJuiceFS 客户端只有一个二进制文件，只需找到程序所在位置删除即可。例如，参照本文档 Linux 系统安装的客户端，执行以下命令卸载客户端：\n\n```shell\nsudo rm /usr/local/bin/juicefs\n```\n\n你还可以通过 `which` 命令查看程序所在位置：\n\n```shell\nwhich juicefs\n```\n\n命令返回的路径即 JuiceFS 客户端在你系统上的安装位置。其他操作系统卸载方法依此类推。\n" }, { "path": "docs/zh_cn/getting-started/standalone.md", "content": "---\nsidebar_position: 2\npagination_next: getting-started/for_distributed\n---\n\n# 单机模式\n\nJuiceFS 文件系统由[「对象存储」](../reference/how_to_set_up_object_storage.md)和[「数据库」](../reference/how_to_set_up_metadata_engine.md)共同驱动。除了对象存储，还支持使用本地磁盘、WebDAV 和 HDFS 等作为底层存储。因此，可以使用本地磁盘和 SQLite 数据库快速创建一个单机文件系统用以了解和体验 JuiceFS。\n\n## 安装客户端\n\n对于 Linux 发行版和 macOS 系统用户，可以使用一键安装脚本快速安装 JuiceFS 客户端：\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n其他操作系统及安装方式请查阅[「安装」](installation.md)。\n\n不论你使用什么操作系统，当在终端输入并执行 `juicefs` 并返回了程序的帮助信息，就说明你成功安装了 JuiceFS 客户端。\n\n## 创建文件系统 {#juicefs-format}\n\n### 基本概念\n\n创建文件系统使用客户端提供的 [`format`](../reference/command_reference.mdx#format) 命令，一般格式为：\n\n```shell\njuicefs format [command options] META-URL NAME\n```\n\n可见，格式化文件系统需要提供 3 种信息：\n\n- **[command options]**：设定文件系统的存储介质，留空则**默认使用本地磁盘**作为存储介质，路径为 `\"$HOME/.juicefs/local\"`(darwin/macOS)，`\"/var/jfs\"`(Linux) 或 `\"C:/jfs/local\"`(Windows)；\n- **META-URL**：用来设置元数据存储，即数据库相关的信息，通常是数据库的 URL 或文件路径；\n- **NAME**：是文件系统的名称。\n\n:::tip 提示\nJuiceFS 支持丰富的存储介质和元数据存储引擎，查看 [JuiceFS 支持的存储介质](../reference/how_to_set_up_object_storage.md) 和 [JuiceFS 支持的元数据存储引擎](../reference/how_to_set_up_metadata_engine.md)。\n:::\n\n### 上手实践\n\n以 Linux 系统为例，以下命令创建了一个名为 `myjfs` 的文件系统。\n\n```shell\njuicefs format sqlite3://myjfs.db myjfs\n```\n\n创建完成将返回类似下面的输出：\n\n```shell {1,4}\n2021/12/14 18:26:37.666618 juicefs[40362] : Meta address: sqlite3://myjfs.db\n[xorm] [info] 2021/12/14 18:26:37.667504 PING DATABASE sqlite3\n2021/12/14 18:26:37.674147 juicefs[40362] : The latency to database is too high: 7.257333ms\n2021/12/14 18:26:37.675713 juicefs[40362] : Data use file:///Users/herald/.juicefs/local/myjfs/\n2021/12/14 18:26:37.689683 juicefs[40362] : Volume is formatted as {Name:myjfs UUID:d5bdf7ea-472c-4640-98a6-6f56aea13982 Storage:file Bucket:/Users/herald/.juicefs/local/ AccessKey: SecretKey: BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n从返回的信息中可以看到，该文件系统使用 SQLite 作为元数据存储引擎，数据库文件位于当前目录，文件名为 `myjfs.db`，保存了 `myjfs` 文件系统的所有信息。它构建了完善的表结构，将用作所有数据的元信息的存储。\n\n![SQLite-info](../images/sqlite-info.png)\n\n由于没有指定任何存储相关的选项，客户端默认使用本地磁盘作为存储介质，根据返回的信息， `myjfs` 的存储路径为 `file:///Users/herald/.juicefs/local/myjfs/`，即当前用户家目录下的 `.juicefs/local/myjfs/`。\n\n## 挂载文件系统\n\n### 基本概念\n\n挂载文件系统使用客户端提供的 [`mount`](../reference/command_reference.mdx#mount) 命令，一般格式为：\n\n```shell\njuicefs mount [command options] META-URL MOUNTPOINT\n```\n\n与创建文件系统的命令类似，挂载文件系统需要提供以下信息：\n\n- `[command options]`：用来指定文件系统相关的选项，例如：`-d` 可以实现后台挂载；\n- `META-URL`：用来设置元数据存储。即数据库相关的信息，通常是数据库的 URL 或文件路径；\n- `MOUNTPOINT`：指定文件系统的挂载点。\n\n:::tip 提示\nWindows 系统的挂载点（`MOUNTPOINT`）应该使用尚未占用的盘符，比如：`Z:`、`Y:`。\n:::\n\n### 上手实践\n\n:::note 注意\n由于 SQLite 是单文件数据库，挂载时要注意数据库文件的的路径，JuiceFS 同时支持相对路径和绝对路径。\n:::\n\n以下命令将 `myjfs` 文件系统挂载到 `~/jfs` 文件夹：\n\n```shell\njuicefs mount sqlite3://myjfs.db ~/jfs\n```\n\n![SQLite-mount-local](../images/sqlite-mount-local.png)\n\n默认情况下，客户端会在前台挂载文件系统。就像你在上图中看到的那样，程序会一直运行在当前终端进程中，使用 Ctrl + C 组合键或关闭终端窗口，文件系统会被卸载。\n\n为了让文件系统可以在后台保持挂载，你可以在挂载时指定 `-d` 或 `--background` 选项，即让客户端在守护进程中挂载文件系统：\n\n```shell\njuicefs mount sqlite3://myjfs.db ~/jfs -d\n```\n\n接下来，任何存入挂载点 `~/jfs` 的文件，都会按照 [JuiceFS 的文件存储格式](../introduction/architecture.md#how-juicefs-store-files)被拆分成特定的「数据块」并存入 `$HOME/.juicefs/local/myjfs` 目录中，相对应的「元数据」会全部存储在 `myjfs.db` 数据库中。\n\n最后执行以下命令可以将挂载点 `~/jfs` 卸载：\n\n```shell\njuicefs umount ~/jfs\n```\n\n## 更进一步\n\n前面介绍的内容通常只适用于快速在本地体验和了解，帮助你对 JuiceFS 的工作方式建立基本的认识。我们可以在前面内容的基础上更进一步，仍然使用 SQLite 存储元数据，把本地存储换成「对象存储」，做一个更有实用价值的方案。\n\n### 对象存储\n\n对象存储是一种基于 HTTP 协议的，提供简单访问 API 的网络存储服务。它的结构扁平，易于扩展，价格相对低廉，非常适合存储海量的非结构化数据。几乎所有主流的云计算平台都有提供对象存储服务，如亚马逊 S3、阿里云 OSS、Backblaze B2 等。\n\nJuiceFS 支持几乎所有的对象存储服务，查看「[JuiceFS 支持的存储介质](../reference/how_to_set_up_object_storage.md)」。\n\n一般来说，创建对象存储通常只需要 2 个环节：\n\n1. 创建 **Bucket** 存储桶，拿到 Endpoint 地址；\n2. 创建 **Access Key ID** 和 **Access Key Secret**，即对象存储 API 的访问密钥。\n\n以阿里云 OSS 为例，创建好的资源大概像下面这样：\n\n- **Bucket Endpoint**：`https://myjfs.oss-cn-shanghai.aliyuncs.com`\n- **Access Key ID**：`ABCDEFGHIJKLMNopqXYZ`\n- **Access Key Secret**：`ZYXwvutsrqpoNMLkJiHgfeDCBA`\n\n:::note 注意\n创建对象存储的过程各个平台会略有差别，建议查看云平台的帮助手册操作。另外，有些平台可能会针对内外网提供不同的 Endpoint 地址，由于本文要从本地访问对象存储，因此请选择使用面向外网访问的地址。\n:::\n\n### 上手实践\n\n接下来使用 SQLite 和阿里云 OSS 对象存储创建一个 JuiceFS 文件系统：\n\n:::note 注意\n如果 `myjfs.db` 文件已经存在，请先删除它再执行以下命令。\n:::\n\n```shell\n# 使用你自己所使用的对象存储信息替换下方相关参数\njuicefs format --storage oss \\\n --bucket https://myjfs.oss-cn-shanghai.aliyuncs.com \\\n --access-key ABCDEFGHIJKLMNopqXYZ \\\n --secret-key ZYXwvutsrqpoNMLkJiHgfeDCBA \\\n sqlite3://myjfs.db myjfs\n```\n\n在上述命令中，数据库和文件系统名称保持不变，增加了对象存储相关的信息：\n\n- `--storage`：设置存储类型，比如 `oss`、`s3` 等；\n- `--bucket`：设置对象存储的 Endpoint 地址；\n- `--access-key`：设置对象存储 API 访问密钥 Access Key ID；\n- `--secret-key`：设置对象存储 API 访问密钥 Access Key Secret。\n\n创建完成即可进行挂载：\n\n```shell\njuicefs mount sqlite3://myjfs.db ~/jfs\n```\n\n挂载命令与使用本地存储时完全一样，这是因为创建文件系统时，对象存储相关的信息已经写入了 `myjfs.db` 数据库，因此客户端不需要额外提供对象存储认证信息，也没有本地配置文件（作为对比，JuiceFS 云服务用 [`juicefs auth`](https://juicefs.com/docs/zh/cloud/reference/commands_reference/#auth) 命令进行认证、获取配置文件）。\n\n相比使用本地磁盘，SQLite 和对象存储的组合实用价值更高。从应用的角度看，这种形式等同于将容量几乎无限的对象存储接入到了本地计算机，让你可以像使用本地磁盘那样使用云存储。\n\n进一步的，该文件系统的所有数据都存储在云端的对象存储，因此可以把 `myjfs.db` 数据库复制到其他安装了 JuiceFS 客户端的计算机上进行挂载和读写。也就是说，任何一台计算机只要能够读取到存储了元数据的数据库，那么它就能够挂载读写该文件系统。\n\n很显然，SQLite 这种单文件数据库很难实现被多台计算机同时访问。如果把 SQLite 改为 Redis、PostgreSQL、MySQL 等能够通过网络被多台计算机同时读写访问的数据库，那么就可以实现 JuiceFS 文件系统的分布式挂载读写。\n" }, { "path": "docs/zh_cn/guide/cache.md", "content": "---\ntitle: 缓存\nsidebar_position: 3\n---\n\n对于一个由对象存储和数据库组合驱动的文件系统，缓存是本地客户端与远端服务之间高效交互的重要纽带。读写的数据可以提前或者异步载入缓存，再由客户端在后台与远端服务交互执行异步上传或预取数据。相比直接与远端服务交互，采用缓存技术可以大大降低存储操作的延时并提高数据吞吐量。\n\nJuiceFS 提供包括元数据缓存、数据读写缓存等多种缓存机制。\n\n:::tip 我的场景真的需要缓存吗？\n数据缓存可以有效地提高随机读的性能，对于像 Elasticsearch、ClickHouse 等对随机读性能要求更高的应用，建议将缓存路径设置在速度更快的存储介质上并分配更大的缓存空间。\n\n然而缓存能提升性能的前提是，你的应用需要反复读取同一批文件。如果你确定你的应用对数据是「读取一次，然后再也不需要」的访问模式（比如大数据的数据清洗常常就是这样），可以关闭缓存功能，省去缓存不断建立，又反复淘汰的开销。\n:::\n\n## 数据一致性 {#consistency}\n\n分布式系统，往往需要在缓存和一致性之间进行取舍。JuiceFS 由于其元数据分离架构，需要从元数据、文件数据（对象存储）、文件数据本地缓存三方面来思考一致性问题：\n\n对于[元数据缓存](#metadata-cache)，JuiceFS 默认的挂载设置满足「关闭再打开（close-to-open）」一致性，也就是说一个客户端修改并关闭文件之后，其他客户端重新打开这个文件都会看到最新的修改。与此同时，默认的挂载参数设置了 1 秒的内核元数据缓存，满足了一般场景的需要。但如果你的应用需要更激进的缓存设置以提升性能，可以阅读下方章节，对元数据缓存进行针对性的调优。特别地，发起修改的客户端（挂载点）能享受到更强的一致性，阅读[一致性例外](#consistency-exceptions)详细了解。\n\n对于对象存储，JuiceFS 将文件分成一个个数据块（默认 4MiB），赋予唯一 ID 并上传至对象存储服务。文件的任何修改操作都将生成新的数据块，原有块保持不变，所以不用担心数据缓存的一致性问题，因为一旦文件被修改过了，JuiceFS 会从对象存储读取新的数据块。而老的失效数据块，也会随着[回收站](../security/trash.md)或碎片合并机制被删除，避免对象存储泄露。\n\n[本地数据缓存](#client-read-cache)缓存也是以对象存储的数据块做为最小单元。一旦文件数据被下载到缓存盘，一致性就和缓存盘可靠性相关，如果磁盘数据发生了篡改，客户端也会读取到错误的数据。对于这种担忧，可以配置合适的 [`--verify-cache-checksum`](../reference/command_reference.mdx#mount-data-cache-options) 策略，确保缓存盘数据完整性。\n\n## 元数据缓存 {#metadata-cache}\n\n作为用户态文件系统，JuiceFS 元数据缓存既通过 FUSE API，以内核元数据缓存的形式进行管理，同时也直接在客户端内存中维护。\n\n### 内核元数据缓存 {#kernel-metadata-cache}\n\nJuiceFS 客户端可以控制这些内核元数据缓存：文件属性（attribute，包含文件名、大小、权限、修改时间等信息）、文件项（entry 和 direntry，用来区分文件和目录类型的文件），在挂载时，可以使用下方参数，通过 FUSE 控制这些元数据的缓存时间：\n\n```shell\n# 文件属性缓存时间（秒），默认为 1，提升 getattr 性能\n--attr-cache=1\n\n# 文件类型的缓存时间（秒），默认为 1，提升文件 lookup 性能\n--entry-cache=1\n\n# 目录类型文件的缓存时间（秒），默认为 1，提升目录的 lookup 性能\n--dir-entry-cache=1\n\n# 失败查询 (lookup 返回 ENOENT) 的缓存时间（秒），默认为 0，提升不存在文件或目录的 lookup 性能\n--negative-entry-cache=1\n```\n\n让以上元数据默认在内核中缓存 1 秒，能显著提高 `lookup` 和 `getattr` 的性能。\n\n需要注意，`entry` 缓存是随着文件访问逐渐建立起来的，不是一个完整列表，因此不能被 `readdir` 调用或者 `ls` 命令使用，而只对 `lookup` 调用有加速效果。这里的 `dir-entry` 含义也不等同于[「目录项」](https://www.kernel.org/doc/html/latest/filesystems/ext4/directory.html)的概念，他并不用来描述「一个目录下包含哪些文件」，而是和 `entry` 一样，都是文件，只不过对文件是否目录类型做了区分。\n\n在实际场景中，也很少需要对 `--entry-cache` 和 `--dir-entry-cache` 进行区分设置，如果确实要精细化调优，在目录极少变动、而文件频繁变动的场景，可以令 `--dir-entry-cache` 大于 `--entry-cache`。\n\n### 客户端内存元数据缓存 {#client-memory-metadata-cache}\n\nJuiceFS 客户端在 `open` 操作即打开一个文件时，其文件属性会被自动缓存在客户端内存中，这里的属性缓存，不仅包含内核元数据中的文件属性比如文件大小、修改时间信息，还包含 JuiceFS 特有的属性，如[文件和 chunk、slice 的对应关系](../introduction/architecture.md#how-juicefs-store-files)。\n\n为保证「关闭再打开（close-to-open）」一致性，`open` 操作默认需要直接访问元数据引擎，不会利用缓存。也就是说，客户端 A 的修改在客户端 B 不一定能立即看到。但是，一旦这个文件在 A 写入完成并关闭，之后在任何一个客户端重新打开该文件都可以保证能访问到最新写入的数据，不论是否在同一个节点。文件的属性缓存也不一定要通过 `open` 操作建立，比如 `tail -f` 会不断查询文件属性，在这种情况下无需重新打开文件，也能获得最新文件变动。\n\n如果要利用上客户端内存的元数据缓存，需要设置 [`--open-cache`](../reference/command_reference.mdx#mount-metadata-cache-options)，指定缓存的有效时长。在缓存有效期间执行的 `getattr` 和 `open` 操作会从内存缓存中立即返回 slice 信息。有了这些信息，就能省去每次打开文件都重新访问元数据服务的开销。\n\n使用 `--open-cache` 选项设置了缓存时间以后，文件系统就不再满足 close-to-open 一致性了，不过与内核元数据类似，发起修改的客户端同样能享受到客户端内存元数据缓存主动失效，其他客户端就只能等待缓存自然过期。因此为了保证文件系统语义，`--open-cache` 默认关闭。如果文件很少发生修改，或者只读场景下（例如 AI 模型训练），则推荐根据情况设置 `--open-cache`，进一步提高读性能。\n\n作为对比，JuiceFS 商业版提供更丰富的客户端内存的元数据缓存功能，并且支持主动失效，阅读[商业版文档](https://juicefs.com/docs/zh/cloud/guide/cache/#client-memory-metadata-cache)以了解。\n\n### 一致性例外 {#consistency-exceptions}\n\n当文件发生变动时，发起修改的挂载点能够享受到更强的一致性，具体而言：\n\n* 发起修改的挂载点，自身的内核元数据缓存能够主动失效。但对于多个挂载点访问、修改同一文件的情况，只有发起修改的客户端能享受到内核元数据缓存主动失效，其他客户端就只能等待缓存自然过期。\n* 存在多挂载点的并发操作时，如果某个客户端删除并重新创建了同名文件，其他客户端因为内核 entry cache 可能继续使用旧文件的 inode，找不到文件或者读取到旧文件内容（开启了回收站功能），需要等待 entry cache 过期。因为已经不是相同文件，也不属于传统的 close-to-open 一致性范畴。\n* 调用 `write` 成功后，挂载点自身立刻就能看到文件长度的变化（比如用 `ls -al` 查看文件大小，可能会注意到文件不断变大）——但这并不意味着修改已经成功提交，在 `flush` 成功前，是不会将这些改动同步到对象存储的，其他挂载点也看不到文件的变动。调用 `fsync, fdatasync, close` 都能触发 `flush`，让修改得以持久化、对其他客户端可见。\n* 作为上一点的极端情况，如果调用 `write` 写入，并在当前挂载点观察到文件长度不断增长，但最后的 `flush` 因为某种原因失败了，比方说到达了文件系统配额上限，文件长度会立刻发生回退，比如从 10M 变为 0。这是一个容易引人误会的情况——并不是 JuiceFS 清空了你的数据，而是写入自始至终就没有成功，只是由于发起修改的挂载点能够提前预览文件长度的变化，让人误以为写入已经成功提交。\n* 发起修改的挂载点，能够监听对应的文件变动（比如使用 [`fswatch`](https://emcrisostomo.github.io/fswatch/) 或者 [`Watchdog`](https://python-watchdog.readthedocs.io/en/stable)）。但范畴也仅限于该挂载点发起修改的文件，也就是说 A 修改的文件，无法在 B 挂载点进行监听。\n* 目前而言，由于 FUSE 尚不支持 inotify API，所以如果你希望监听 JuiceFS 特定目录下的文件变化，请使用轮询的方式（比如 [`PollingObserver`](https://python-watchdog.readthedocs.io/en/stable/_modules/watchdog/observers/polling.html#PollingObserver)）。\n\n## 读写缓冲区 {#buffer-size}\n\n读写缓冲区是分配给 JuiceFS 客户端进程的一块内存，通过 [`--buffer-size`](../reference/command_reference.mdx#mount-data-cache-options) 控制着大小，默认 300（单位 MiB）。读和写产生的数据，都会途经这个缓冲区。所以缓冲区的作用非常重要，在大规模场景下遇到性能不足时，提升缓冲区大小也是常见的优化方式。\n\n### 预读和预取 {#readahead-prefetch}\n\n:::tip\n为了准确描述 JuiceFS 客户端的工作机制，文档中会用「预读」和「预取」来特指客户端的两种不同提前下载数据、优化读性能的行为。\n:::\n\n顺序读文件时，JuiceFS 客户端会进行预读（readahead），也就是提前将文件后续的内容下载下来。事实上同样的行为也早已存在于[内核](https://www.halolinux.us/kernel-architecture/page-cache-readahead.html)：读取文件时，内核也会根据具体的读行为和预读窗口算法，来提前将文件读取到内核页缓存。考虑到 JuiceFS 是个网络文件系统，内核的预读窗口对他来说太小，无法有效提升顺序读的性能，因此在内核的预读之上，JuiceFS 客户端也会发起自己的预读，根据更激进的算法来“猜测”应用接下来要读取的数据范围，然后提前将对象存储对应的数据块下载下来。预读的窗口大小可以通过`max-readahead`参数来控制，在随机读场景中可以考虑将其设置为 0 来禁用预读。\n\n![readahead](../images/buffer-readahead.svg)\n\n由于 readahead 只能优化顺序读场景，因此在 JuiceFS 客户端还存在着另一种相似的机制，称作预取（prefetch）：随机读取文件某个块（Block）的一小段，客户端会异步将整个对象存储块下载下来。\n\n![prefetch](../images/buffer-prefetch.svg)\n\n预取的设计是基于「假如文件的某一小段被应用读取，那么文件附近的区域也很可能会被读取」的假设，对于不同的应用场景，这样的假设未必成立——如果应用对大文件进行偏移极大的、稀疏的随机读，那么不难想象，prefetch 会带来明显的读放大。因此如果你已经对应用场景的读取模式有深入了解，确认并不需要 prefetch，可以通过 [`--prefetch=0`](../reference/command_reference.mdx#mount-data-cache-options) 禁用该行为。\n\n预读和预取分别优化了顺序读、随机读性能，也会带来一定程度的读放大，阅读[「读放大」](../administration/troubleshooting.md#read-amplification)了解更多信息。\n\n### 写入 {#buffer-write}\n\n调用 `write` 成功，并不代表数据被持久化，持久化是 `flush` 的工作。这一点不论对于本地文件系统，还是 JuiceFS 文件系统，都是一样的。在 JuiceFS 中，`write` 会将数据写入缓冲区，写入完毕以后，你甚至会注意到，当前挂载点已经看到文件长度有所变化，不要误会，这并不代表写入已经持久化（这点也在[一致性例外](#consistency-exceptions)话题上有更详细介绍）。总而言之，在 `flush` 来临之前，改动只存在于客户端缓冲区。应用可以显式调用 `flush`，但就算不这样做，当写入超过块大小（默认 4M），或者在缓冲区停留超过一定时间，都会触发自动 `flush`。\n\n结合上方已经介绍过的预读，缓冲区的总体作用可以一张图表示：\n\n![read write buffer](../images/buffer-read-write.svg)\n\n缓冲区是读写共用的，显然「写」具有更高的优先级，这隐含着「写会影响读」的可能性。举例说明，如果对象存储的上传速度不足以支撑写入负载，会发生缓冲区拥堵：\n\n![buffer congestion](../images/buffer-congestion.svg)\n\n如上图所示，写入负载过大，在缓冲区中积攒了太多待写入的 Slice，侵占了缓冲区用于预读的空间，因此读文件会变慢。不仅如此，由于对象存储上传速度不足，写也可能会因为 `flush` 超时而最终失败。\n\n### 观测和调优 {#buffer-observation}\n\n上方小节介绍了缓冲区对读、写都有关键作用，因此在面对高并发读写场景的时候，对 `--buffer-size` 进行相应的扩容，能有效提升性能。但一味地扩大缓冲区大小，也可能产生其他的问题，比如 `--buffer-size` 过大，但对象存储上传速度不足，导致上方小节中介绍的缓冲区拥堵的情况。因此，缓冲区的大小需要结合其他性能参数一起科学地设置。\n\n在调整缓冲区大小前，我们推荐使用 [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) 来观察当前的缓冲区用量大小，这个命令能直观反映出当前的读写性能问题。\n\n如果希望增加顺序读速度，可以增加 `--max-readahead` 和 `--buffer-size`，来放大预读窗口，窗口内尚未下载到本地的数据块，会并发地异步下载。同时注意，单个文件的预读不会把整个缓冲区用完，限制为 1/4 到 1/2。因此如果在优化单个大文件的顺序读时发现 `juicefs stats` 中 `buf` 用量已经接近一半，说明该文件的预读额度已满，此时虽然缓冲区还有空闲，但也需要继续增加 `--buffer-size` 才能进一步提升单个大文件的预读性能。\n\n如果你希望增加写入速度，通过调整 [`--max-uploads`](../reference/command_reference.mdx#mount-data-storage-options) 增大了上传并发度，但并没有观察到上行带宽用量有明显增加，那么此时可能就需要相应地调大 `--buffer-size`，让并发线程更容易申请到内存来工作。这个排查原理反之亦然：如果增大 `--buffer-size` 却没有观察到上行带宽占用提升，也可以考虑增大 `--max-uploads` 来提升上传并发度。\n\n可想而知，`--buffer-size` 也控制着每次 `flush` 操作的上传数据量大小，因此如果客户端处在一个低带宽的网络环境下，可能反而需要降低 `--buffer-size` 来避免 `flush` 超时。关于低带宽场景排查请详见[「与对象存储通信不畅」](../administration/troubleshooting.md#io-error-object-storage)。\n\n## 数据缓存 {#data-cache}\n\nJuiceFS 对数据也提供多种缓存机制来提高性能，包括内核中的页缓存和客户端所在机器的本地缓存，以及客户端自身的内存读写缓冲区。读请求会依次尝试内核分页缓存、JuiceFS 进程的预读缓冲区、本地磁盘缓存，当缓存中没找到对应数据时才会从对象存储读取，并且会异步写入各级缓存保证下一次访问的性能。\n\n![JuiceFS-cache](../images/juicefs-cache.png)\n\n### 内核页缓存 {#kernel-data-cache}\n\n对于已经读过的文件，内核会为其建立页缓存（Page Cache），下次再打开的时候，如果文件没有被更新，就可以直接从内核页缓存读取，获得最好的性能。\n\nJuiceFS 客户端会跟踪所有最近被打开的文件，要重复打开相同文件时，它会根据该文件是否被修改决定是否可以使用内核页数据，如果文件被修改过，则对应的页缓存也将在再次打开时失效，这样保证了客户端能够读到最新的数据。\n\n当重复读 JuiceFS 中的同一个文件时，速度会非常快，延时可低至微秒，吞吐量可以到每秒几 GiB。\n\n### 内核回写模式 {#fuse-writeback-cache}\n\n从 Linux 内核 3.15 开始，FUSE 支持[内核回写（writeback-cache）](https://www.kernel.org/doc/Documentation/filesystems/fuse-io.txt)模式，内核会把高频随机小 IO（例如 10-100 字节）的写请求合并起来，显著提升随机写入的性能。但其副作用是会将顺序写变为随机写，严重降低顺序写的性能。开启前请考虑使用场景是否匹配。\n\n在挂载命令通过 [`-o writeback_cache`](../reference/fuse_mount_options.md) 选项来开启内核回写模式。注意，内核回写与[「客户端写缓存」](#client-write-cache)并不一样，前者是内核中的实现，后者则发生在 JuiceFS 客户端，二者适用场景也不一样，详读对应章节以了解。\n\n### 客户端读缓存 {#client-read-cache}\n\n客户端会根据应用读数据的模式，自动做预读和缓存操作以提高顺序读的性能。数据会缓存到本地文件系统中，可以是基于硬盘、SSD 或者内存的任意本地文件系统。\n\nJuiceFS 客户端会把从对象存储下载的数据，以及新上传的小于 1 个 block 大小的数据写入到缓存目录中，不做压缩和加密。如果希望保证应用程序首次访问数据的时候就能获得已缓存的性能，可以使用 [`juicefs warmup`](../reference/command_reference.mdx#warmup) 命令来对缓存数据进行预热。\n\n在未开启 `--writeback` 时，如果缓存目录所在的文件系统无法正常工作时 JuiceFS 客户端能立刻返回错误，剔除缓存盘并降级成直接访问对象存储。但在开启 `--writeback` 的情况下，如果缓存目录所在的文件系统异常时体现为读操作卡死（如某些内核态的网络文件系统），那么 JuiceFS 也会随之一起卡住，这就要求你对缓存目录底层的文件系统行为进行调优，做到快速失败。\n\n以下是缓存配置的关键参数（完整参数列表见 [`juicefs mount`](../reference/command_reference.mdx#mount)）：\n\n* `--prefetch`\n\n 并发预取 N 个块（默认 1）。所谓预取（prefetch），就是随机读取文件某个块（block）的一小段，客户端会异步将整个对象存储块下载下来。预取往往能改善随机读性能，但如果你的场景的文件访问模式无法利用到预取数据（比如 offset 跨度极大的大文件随机访问），预取会带来比较明显的读放大，可以考虑设为 0 以禁用预取特性。\n\n JuiceFS 还内置着另一种类似的预读机制：在顺序读时，会提前下载临近的对象存储块，这在 JuiceFS 内称为 readahead 机制，能有效提高顺序读性能。Readahead 的并发度受[「读写缓冲区」](#buffer-size)的大小影响，读写缓冲区越大并发度越高。\n\n* `--cache-dir`\n\n 缓存目录，默认为 `/var/jfsCache` 或 `$HOME/.juicefs/cache`。请阅读[「缓存位置」](#cache-dir)了解更多信息。\n\n 如果急需释放磁盘空间，你可以手动清理缓存目录下的文件，缓存路径为 `//raw/`。\n\n* `--cache-size` 与 `--free-space-ratio`\n\n 缓存空间大小（单位 MiB，默认 102400）与缓存盘的最少剩余空间占比（默认 0.1）。这两个参数任意一个达到阈值，均会自动触发缓存淘汰，使用的是类似于 LRU 的策略，即尽量清理较早且较少使用的缓存。\n\n 实际缓存数据占用空间大小可能会略微超过设置值，这是因为对同样一批缓存数据，很难精确计算它们在不同的本地文件系统上所占用的存储空间，JuiceFS 累加所有被缓存对象大小时会按照 4KiB 的最小值来计算，因此与 `du` 得到的数值往往不一致。\n\n* `--cache-partial-only`\n\n 只缓存小文件和随机读的部分，适合对象存储的吞吐比缓存盘还高的情况。默认为 false。\n\n 读一般有两种模式，连续读和随机读。对于连续读，一般需要较高的吞吐。对于随机读，一般需要较低的时延。当本地磁盘的吞吐反而比不上对象存储时，可以考虑启用 `--cache-partial-only`，这样一来，连续读虽然会将一整个对象块读取下来，但并不会被缓存。而随机读（例如读 Parquet 或者 ORC 文件的 footer）所读取的字节数比较小，不会读取整个对象块，此类读取就会被缓存。充分地利用了本地磁盘低时延和网络高吞吐的优势。\n\n### 客户端写缓存 {#client-write-cache}\n\n开启客户端写缓存能提升特定场景下的大量小文件写入性能，请详读本节了解。\n\n客户端写缓存默认关闭，写入的数据会首先进入 JuiceFS 客户端的内存[读写缓冲区](#buffer-size)，当一个 Chunk 被写满，或者应用强制写入（调用 `close()` 或者 `fsync()`）时，才会触发数据上传对象存储。为了确保数据安全性，客户端会等数据上传完成，才提交到元数据服务。\n\n由于默认的写入流程是「先上传，再提交」，可想而知，大量小文件写入时，这样的流程将影响写入性能。启用客户端写缓存以后，写入流程将改为「先提交，再异步上传」，写文件不会等待数据上传到对象存储，而是写入到本地缓存目录并提交到元数据服务后就立即返回，本地缓存目录中的文件数据会在后台异步上传至对象存储。\n\n如果你的场景需要写入大量临时文件，不需要持久化和分布式访问，也可以用 [`--upload-delay`](../reference/command_reference.mdx#mount-data-cache-options) 参数来设置延缓数据上传到对象存储，如果在等待的时间内数据被应用删除，则无需再上传到对象存储，既提升了性能也节省了成本。相较于本地硬盘而言，JuiceFS 提供了后端保障，在缓存目录容量不足时依然会自动将数据上传，确保在应用侧不会因此而感知到错误。\n\n挂载时加入 `--writeback` 参数，便能开启客户端写缓存，但在该模式下请注意：\n\n* 本地缓存本身的可靠性与缓存盘的可靠性直接相关，如果在上传完成前本地数据遭受损害，意味着数据丢失。因此对数据安全性要求越高，越应谨慎使用。\n* 待上传的文件默认存储在 `/var/jfsCache//rawstaging/`，只要该目录不为空，就表示还有待上传的文件。务必注意不要删除该目录下的文件，否则将造成数据丢失。\n* 写缓存大小由 [`--free-space-ratio`](#client-read-cache) 控制。默认情况下，如果未开启写缓存，JuiceFS 客户端最多使用缓存目录 90% 的磁盘空间（计算规则是 `(1 - ) * 100`）。开启写缓存后会超额使用一定比例的磁盘空间，计算规则是 `(1 - ( / 2)) * 100`，即默认情况下最多会使用缓存目录 95% 的磁盘空间。\n* 写缓存和读缓存共享缓存盘空间，因此会互相影响。例如写缓存占用过多磁盘空间，那么将导致读缓存的大小受到限制，反之亦然。\n* 如果本地盘写性能太差，带宽甚至比不上对象存储，那么 `--writeback` 会带来更差的写性能。\n* 如果缓存目录的文件系统出错，客户端则降级为同步写入对象存储，情况类似[客户端读缓存](#client-read-cache)。\n* 如果节点到对象存储的上行带宽不足（网速太差），本地写缓存迟迟无法上传完毕，此时如果在其他节点访问这些文件，则会出现读错误。低带宽场景的排查请详见[「与对象存储通信不畅」](../administration/troubleshooting.md#io-error-object-storage)。\n\n也正由于写缓存的使用注意事项较多，使用不当极易出问题，我们推荐仅在大量写入小文件时临时开启，比如：\n\n* 解压包含大量小文件的压缩文件\n* 软件编译\n* 大数据任务的临时存储场景，比如 Spark shuffle\n\n启用 `--writeback` 模式后，除了直接查看 `/var/jfsCache//rawstaging/` 目录，还可以通过以下命令确定文件上传进度：\n\n```shell\n# 假设挂载点为 /jfs\n$ cd /jfs\n$ cat .stats | grep \"staging\"\njuicefs_staging_block_bytes 1621127168 # 待上传的数据块大小\njuicefs_staging_block_delay_seconds 46116860185.95535\njuicefs_staging_blocks 394 # 待上传的数据块数量\n```\n\n### 缓存位置 {#cache-dir}\n\n取决于操作系统，JuiceFS 的默认缓存路径如下：\n\n- **Linux**：`/var/jfsCache`\n- **macOS**：`$HOME/.juicefs/cache`\n- **Windows**：`%USERPROFILE%\\.juicefs\\cache`\n\n对于 Linux 系统，要注意默认缓存路径要求管理员权限，普通用户需要有权使用 `sudo` 才能设置成功，例如：\n\n```shell\nsudo juicefs mount redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\n另外，可以在挂载文件系统时通过 `--cache-dir` 选项设置在当前系统可以访问的任何存储路径上。对于没有访问 `/var` 目录权限的普通用户，可以把缓存设置在用户的 `HOME` 目录中，例如：\n\n```shell\njuicefs mount --cache-dir ~/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\n:::tip 提示\n建议缓存目录尽量使用独立的高性能盘，不要用系统盘，也不要和其它应用共用。共用不仅会相互影响性能，还可能导致其它应用出错（例如磁盘剩余空间不足）。如果无法避免必须共用那一定要预估好其它应用所需的磁盘容量，限制缓存空间大小（`--cache-size`），避免 JuiceFS 的读缓存或者写缓存占用过多空间。\n:::\n\n#### 内存盘\n\n如果对文件的读性能有更高要求，可以把缓存设置在内存盘上。对于 Linux 系统，通过 `df` 命令查看 `tmpfs` 类型的文件系统：\n\n```shell\n$ df -Th | grep tmpfs\n文件系统 类型 容量 已用 可用 已用% 挂载点\ntmpfs tmpfs 362M 2.0M 360M 1% /run\ntmpfs tmpfs 3.8G 0 3.8G 0% /dev/shm\ntmpfs tmpfs 5.0M 4.0K 5.0M 1% /run/lock\n```\n\n其中 `/dev/shm` 是典型的内存盘，可以作为 JuiceFS 的缓存路径使用，它的容量一般是内存的一半，可以根据需要手动调整容量，例如，将缓存盘的容量调整为 32GB：\n\n```shell\nsudo mount -o size=32000M -o remount /dev/shm\n```\n\n然后使用该路径作为缓存，挂载文件系统：\n\n```shell\njuicefs mount --cache-dir /dev/shm/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\n除此之外，还可以将 `--cache-dir` 选项设置为 `memory` 来直接使用进程内存作为缓存，与 `/dev/shm` 相比，好处是简单不依赖外部设备，但相应地也无法持久化，一般在测试评估的时候使用。\n\n#### 共享目录\n\nSMB、NFS 等共享目录也可以用作 JuiceFS 的缓存，对于局域网有多个设备挂载了相同 JuiceFS 文件系统的情况，将局域网中的共享目录作为缓存路径，可以有效缓解多个设备重复预热缓存的带宽压力。\n\n以 SMB/CIFS 共享为例，使用 `cifs-utils` 包提供的工具挂载局域网中的共享目录：\n\n```shell\nsudo mount.cifs //192.168.1.18/public /mnt/jfscache\n```\n\n将共享目录作为 JuiceFS 缓存：\n\n```shell\nsudo juicefs mount --cache-dir /mnt/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\n#### 多缓存目录\n\nJuiceFS 支持同时设置多个缓存目录，从而解决缓存空间不足的问题，使用 `:`（Linux、macOS）或 `;`（Windows）字符分隔多个路径，例如：\n\n```shell\nsudo juicefs mount --cache-dir ~/jfscache:/mnt/jfscache:/dev/shm/jfscache redis://127.0.0.1:6379/1 /mnt/myjfs\n```\n\n当设置了多个缓存目录，或者使用多块设备作为缓存盘，`--cache-size` 选项表示所有缓存目录中的数据总大小。客户端会采用哈希策略向各个缓存路径中均匀地写入数据，无法对多块容量或性能不同的缓存盘进行特殊调优。\n\n因此建议不同缓存目录／缓存盘的可用空间保持一致，否则可能造成不能充分利用某个缓存目录空间的情况。例如 `--cache-dir` 为 `/data1:/data2`，其中 `/data1` 的可用空间为 1GiB，`/data2` 的可用空间为 2GiB，`--cache-size` 为 3GiB，`--free-space-ratio` 为 0.1。因为缓存的写入策略是均匀写入，所以分配给每个缓存目录的最大空间是 `3GiB / 2 = 1.5GiB`，会造成 `/data2` 目录的缓存空间最大为 1.5GiB，而不是 `2GiB * 0.9 = 1.8GiB`。\n" }, { "path": "docs/zh_cn/guide/clone.md", "content": "---\ntitle: 克隆文件或目录\nsidebar_position: 6\n---\n\n对指定数据进行克隆，创建克隆时不会实际拷贝对象存储数据，而是仅拷贝元数据，因此不论对多大的文件或目录进行克隆，都非常快。对于 JuiceFS，这个命令是 `cp` 更好的替代，甚至对于 Linux 客户端来说，如果所使用的内核支持 [`copy_file_range`](https://man7.org/linux/man-pages/man2/copy_file_range.2.html)，那么调用 `cp` 时，实际发生的也是同样的元数据拷贝，调用将会格外迅速。\n\n![clone](../images/juicefs-clone.svg)\n\n克隆结果是纯粹的元数据拷贝，实际引用的对象存储块和源文件相同，因此在各方面都和源文件一样，可以正常读写。有任何一方文件数据被实际修改时，对应的数据块变更会以写入时重定向（ROW，Redirect on Write）的方式，写入到新的数据块，并将指针指向新的数据块。而其他未经修改的文件区域，由于对象存储数据块仍然相同，所以引用关系依然保持不变。如果对快照文件进行随机写、覆盖写等操作，和普通的 JuiceFS 文件一样，快照文件也会产生文件碎片，你可以通过执行`juicefs compact`来对文件的碎片进行合并，来提升读取效率。\n\n需要注意的是，**克隆产生的元数据，也同样占用文件系统存储空间，以及元数据引擎的存储空间**，因此对庞大的目录进行克隆操作时请格外谨慎。\n\n```shell\njuicefs clone SRC DST\n\n# 克隆文件\njuicefs clone /mnt/jfs/file1 /mnt/jfs/file2\n\n# 克隆目录\njuicefs clone /mnt/jfs/dir1 /mnt/jfs/dir2\n```\n\n## 一致性 {#consistency}\n\n在事务一致性方面，克隆的行为如下：\n\n- 在 `clone` 命令完成前，目标文件不可见。\n- 对于文件：`clone` 命令确保原子性，即克隆后的文件始终处于正确和一致的状态。\n- 对于目录：`clone` 命令对目录的原子性没有保证。在克隆过程中，如果源目录发生变化，则目标目录与源目录可能不一致。\n- 同时往同一个位置创建克隆时，只会有一个成功，失败请求的会清理掉临时创建的目录树。\n\n克隆操作是在挂载进程中进行，如果克隆命令意外退出，克隆操作可能完成或者被中断。失败或者被中断的克隆操作，`mount` 进程会尝试清理已创建好的子树，如果清理子树也失败（元数据不可用或者`mount`进程意外退出），则会导致元数据泄露和可能的对象存储泄露。此时如果源对象被删除了，则会导致其对象存储上的数据不会被释放（因为被未挂载的的子树所引用），直到使用 [`juicefs gc --delete`](../reference/command_reference.mdx#gc) 命令清理。\n" }, { "path": "docs/zh_cn/guide/dir-stats.md", "content": "---\ntitle: 目录用量统计\nsidebar_position: 5\n---\n\nJuiceFS 在 v1.1.0 开始支持目录用量统计并在文件系统格式化时默认开启，旧版本 volume 迁移到新版本后默认关闭（需要[手动开启](#enable-directory-stats)）。目录用量统计可以加速 `quota`、`info` 和 `summary` 等子命令，但由于客户端会异步更新统计信息，启用后也会带来少量开销。\n\n:::tip 提示\n由于用量统计需要挂载客户端支持，请确保除所有可写入客户端已升级到 v1.1.0 以上版本再启用此特性。\n:::\n\n## 启用目录用量统计 {#enable-directory-stats}\n\n运行 `juicefs config $URL --dir-stats` 来启用目录统计，启用以后，使用 `juicefs config $URL` 命令确认生效：\n\n```shell\n$ juicefs config redis://localhost\n2023/05/31 15:56:39.721188 juicefs[30626] : Meta address: redis://localhost [interface.go:494]\n2023/05/31 15:56:39.723284 juicefs[30626] : Ping redis latency: 159.226µs [redis.go:3566]\n{\n \"Name\": \"myjfs\",\n \"UUID\": \"82db28de-bf5f-43bf-bba3-eb3535a86c48\",\n \"Storage\": \"file\",\n \"Bucket\": \"/root/.juicefs/local/\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"EncryptAlgo\": \"aes256gcm-rsa\",\n \"TrashDays\": 1,\n \"MetaVersion\": 1,\n \"DirStats\": true\n}\n```\n\n可以看到 `\"DirStats\": true` 代表目录用量统计已启用，我们可以尝试禁用它：\n\n```shell\n$ juicefs config redis://localhost --dir-stats=false\n2023/05/31 15:59:39.046134 juicefs[30752] : Meta address: redis://localhost [interface.go:494]\n2023/05/31 15:59:39.048301 juicefs[30752] : Ping redis latency: 171.308µs [redis.go:3566]\n dir-stats: true -> false\n```\n\n:::tip 提示\n[目录配额](./quota.md#directory-quota)功能依赖目录用量统计，为目录设置配额后会自动开启目录用量统计，并且需要删除所有目录配额后才能禁用目录用量统计。\n:::\n\n## 查看目录统计 {#check-directory-stats}\n\n运行 `juicefs info $PATH` 查看单层目录的统计用量：\n\n```shell\n$ juicefs info /mnt/jfs/pjdfstest/\n/mnt/jfs/pjdfstest/ :\n inode: 2\n files: 10\n dirs: 4\n length: 43.74 KiB (44794 Bytes)\n size: 92.00 KiB (94208 Bytes)\n path: /pjdfstest\n```\n\n也可以使用 `juicefs info -r $PATH` 递归查看目录统计并汇总：\n\n```shell\n/mnt/jfs/pjdfstest/: 278 921.0/s\n/mnt/jfs/pjdfstest/: 1.6 MiB (1642496 Bytes) 5.2 MiB/s\n/mnt/jfs/pjdfstest/ :\n inode: 2\n files: 278\n dirs: 37\n length: 592.42 KiB (606638 Bytes)\n size: 1.57 MiB (1642496 Bytes)\n path: /pjdfstest\n```\n\n另外你可以使用 `juicefs summary $PATH` 命令来查看各层级的目录用量：\n\n```shell\n$ ./juicefs summary /mnt/jfs/pjdfstest/\n/mnt/jfs/pjdfstest/: 315 1044.4/s\n/mnt/jfs/pjdfstest/: 1.6 MiB (1642496 Bytes) 5.2 MiB/s\n+------------------+---------+------+-------+\n| PATH | SIZE | DIRS | FILES |\n+------------------+---------+------+-------+\n| / | 1.6 MiB | 37 | 278 |\n| tests/ | 1.1 MiB | 18 | 240 |\n| tests/open/ | 112 KiB | 1 | 26 |\n| tests/... | 328 KiB | 7 | 71 |\n| .git/ | 432 KiB | 17 | 26 |\n| .git/objects/ | 252 KiB | 3 | 2 |\n| ... | 12 KiB | 0 | 3 |\n+------------------+---------+------+-------+\n```\n\n:::note 说明\n目录统计只计算每个目录的单层用量，如果要查看递归统计用量，需要使用 `juicefs info -r`，对于大目录，遍历汇总可能带来很大的开销。如需持续查看某些特定目录的总用量，可参考目录配额通过[设置空配额](./quota.md#limit-capacity-and-inodes-of-directory)的方式统计目录总用量。\n\n与社区版不同，JuiceFS 企业版的目录大小已经进行了[递归统计](/docs/zh/cloud/guide/quota#file-directory-size)，可以直接用 `ls -lh` 看到递归统计的目录总大小。\n:::\n\n## 故障和修复 {#troubleshooting}\n\n由于目录用量是异步统计，当客户端发生异常时可能会丢失部分统计值导致结果不准确。`juicefs info`、`juicefs summary` 和 `juicefs quota` 命令均配有 `--strict` 选项在严苛模式下运行以绕过目录统计（默认模式一般称为快速模式，fast mode）。\n\n如果发现严格模式和快速模式结果不一致，考虑使用 `juicefs fsck` 命令进行诊断和修复：\n\n```shell\n$ juicefs info -r /jfs/d\n/jfs/d: 1 3.3/s\n/jfs/d: 448.0 MiB (469766144 Bytes) 1.4 GiB/s\n/jfs/d :\n inode: 2\n files: 1\n dirs: 1\n length: 448.00 MiB (469762048 Bytes)\n size: 448.00 MiB (469766144 Bytes)\n path: /d\n\n$ juicefs info -r --strict /jfs/d\n/jfs/d: 1 3.3/s\n/jfs/d: 1.0 GiB (1073745920 Bytes) 3.3 GiB/s\n/jfs/d :\n inode: 2\n files: 1\n dirs: 1\n length: 1.00 GiB (1073741824 Bytes)\n size: 1.00 GiB (1073745920 Bytes)\n path: /d\n\n# 检查目录 /d 的用量统计\n$ juicefs fsck sqlite3://test.db --path /d --sync-dir-stat\n2023/05/31 17:14:34.700239 juicefs[32667] : Meta address: sqlite3://test.db [interface.go:494]\n[xorm] [info] 2023/05/31 17:14:34.700291 PING DATABASE sqlite3\n2023/05/31 17:14:34.701553 juicefs[32667] : usage stat of /d should be &{1073741824 1073741824 1}, but got &{469762048 469762048 1} [base.go:2010]\n2023/05/31 17:14:34.701577 juicefs[32667] : Stat of path /d (inode 2) should be synced, please re-run with '--path /d --repair --sync-dir-stat' to fix it [base.go:2025]\n2023/05/31 17:14:34.701615 juicefs[32667] : some errors occurred, please check the log of fsck [main.go:31]\n\n# 修复目录 /d 的用量统计\n$ juicefs fsck -v sqlite3://test.db --path /d --sync-dir-stat --repair\n2023/05/31 17:14:43.445153 juicefs[32721] : maxprocs: Leaving GOMAXPROCS=8: CPU quota undefined [maxprocs.go:47]\n2023/05/31 17:14:43.445289 juicefs[32721] : Meta address: sqlite3://test.db [interface.go:494]\n[xorm] [info] 2023/05/31 17:14:43.445350 PING DATABASE sqlite3\n2023/05/31 17:14:43.462374 juicefs[32721] : Stat of path /d (inode 2) is successfully synced [base.go:2018]\n\n# 验证\n$ juicefs info -r /jfs/d\n/jfs/d: 1 3.3/s\n/jfs/d: 1.0 GiB (1073745920 Bytes) 3.3 GiB/s\n/jfs/d :\n inode: 2\n files: 1\n dirs: 1\n length: 1.00 GiB (1073741824 Bytes)\n size: 1.00 GiB (1073745920 Bytes)\n path: /d\n```\n" }, { "path": "docs/zh_cn/guide/gateway.md", "content": "---\ntitle: S3 网关\ndescription: JuiceFS S3 网关是 JuiceFS 的一个功能，它将 JuiceFS 文件系统以 S3 协议对外提供服务，使得应用可以通过 S3 SDK 访问 JuiceFS 上存储的文件。\nsidebar_position: 5\n---\n\nJuiceFS S3 网关是 JuiceFS 支持的多种访问方式之一，它可以将 JuiceFS 文件系统以 S3 协议对外提供服务，使得应用可以通过 S3 SDK 访问 JuiceFS 上存储的文件。\n\n## 架构与原理\n\n在 JuiceFS 中，文件是以对象的形式[分块存储到底层的对象存储中](../introduction/architecture.md#how-juicefs-store-files)。JuiceFS 提供了 FUSE POSIX、WebDAV、S3 网关、CSI 驱动等多种访问方式，其中 S3 网关是较为常用的一种，其架构图如下：\n\n![JuiceFS S3 Gateway architecture](../images/juicefs-s3-gateway-arch.png)\n\nJuiceFS S3 网关功能是通过 [MinIO S3 网关](https://github.com/minio/minio/tree/ea1803417f80a743fc6c7bb261d864c38628cf8d/docs/gateway)实现的。我们利用 MinIO 的 [`object 接口`](https://github.com/minio/minio/blob/d46386246fb6db5f823df54d932b6f7274d46059/cmd/object-api-interface.go#L88)将 JuiceFS 文件系统作为 MinIO 服务器的后端存储，提供接近原生 MinIO 的使用体验，同时继承 MinIO 的许多高级功能。在这种架构中，JuiceFS 就相当于 MinIO 实例的一块本地磁盘，原理与 `minio server /data1` 命令类似。\n\nJuiceFS S3 网关的常见的使用场景有：\n\n- **为 JuiceFS 开放 S3 接口**：应用可以通过 S3 SDK 访问 JuiceFS 上存储的文件；\n- **使用 S3 客户端**：使用 s3cmd、AWS CLI、MinIO 客户端来方便地访问和操作 JuiceFS 上存储的文件；\n- **管理 JuiceFS 中的文件**：S3 网关提供了一个基于网页的文件管理器，可以在浏览器中管理 JuiceFS 中的文件；\n- **集群复制**：在跨集群复制数据的场景下，作为集群的统一数据出口，避免跨区访问元数据以提升数据传输性能，详见[「使用 S3 网关进行跨区域数据同步」](../guide/sync.md#sync-across-region)\n\n## 快速开始\n\n启动 S3 网关需要一个已经创建完毕的 JuiceFS 文件系统，如果尚不存在，请参考[文档](../getting-started/standalone.md)来创建。下方假定元数据引擎 URL 为 `redis://localhost:6379/1`。\n\n由于网关基于 MinIO 开发，因此需要先设置 `MINIO_ROOT_USER` 和 `MINIO_ROOT_PASSWORD` 两个环境变量，他们会成为访问 S3 API 时认证身份用的 Access Key 和 Secret Key，是拥有最高权限的管理员凭证。\n\n```shell\nexport MINIO_ROOT_USER=admin\nexport MINIO_ROOT_PASSWORD=12345678\n\n# Windows 用户请改用 set 命令设置环境变量\nset MINIO_ROOT_USER=admin\n```\n\n注意，`MINIO_ROOT_USER` 的长度至少 3 个字符， `MINIO_ROOT_PASSWORD` 的长度至少 8 个字符，如果未能正确设置，将会遭遇类似 `MINIO_ROOT_USER should be specified as an environment variable with at least 3 characters` 的报错，注意排查。\n\n启动 S3 网关：\n\n```shell\n# 第一个参数是元数据引擎的 URL，第二个是 S3 网关监听的地址和端口\njuicefs gateway redis://localhost:6379/1 localhost:9000\n\n# 从 v1.2 开始，S3 网关支持后台启动，追加 --background 或 -d 参数均可\n# 后台运行场景下，使用 --log 指定日志输出文件路径\njuicefs gateway redis://localhost:6379 localhost:9000 -d --log=/var/log/juicefs-s3-gateway.log\n```\n\nS3 Gateway 默认没有启用[多桶支持](#多桶支持)，可以添加 `--multi-buckets` 选项开启。还可以添加[其他选项](../reference/command_reference.mdx#gateway)优化 S3 网关，比如，可以将默认的本地缓存设置为 20 GiB。\n\n```shell\njuicefs gateway --cache-size 20480 redis://localhost:6379/1 localhost:9000\n```\n\n在这个例子中，我们假设 JuiceFS 文件系统使用的是本地的 Redis 数据库。当 S3 网关启用时，在**当前主机**上可以使用 `http://localhost:9000` 这个地址访问到 S3 网关的管理界面。\n\n![S3-gateway-file-manager](../images/s3-gateway-file-manager.jpg)\n\n如果你希望通过局域网或互联网上的其他主机访问 S3 网关，则需要调整监听地址，例如：\n\n```shell\njuicefs gateway redis://localhost:6379/1 0.0.0.0:9000\n```\n\n这样一来，S3 网关将会默认接受所有网络请求。不同的位置的 S3 客户端可以使用不同的地址访问 S3 网关，例如：\n\n- S3 网关所在主机中的第三方客户端可以使用 `http://127.0.0.1:9000` 或 `http://localhost:9000` 进行访问；\n- 与 S3 网关所在主机处于同一局域网的第三方客户端可以使用 `http://192.168.1.8:9000` 访问（假设启用 S3 网关的主机内网 IP 地址为 192.168.1.8）；\n- 通过互联网访问 S3 网关可以使用 `http://110.220.110.220:9000` 访问（假设启用 S3 网关的主机公网 IP 地址为 110.220.110.220）。\n\n
\n \n
\n\n## 访问 S3 网关\n\n各类支持 S3 API 的客户端、桌面程序、Web 程序等都可以访问 JuiceFS S3 网关。使用时请注意 S3 网关监听的地址和端口。\n\n:::tip 提示\n以下示例均为使用第三方客户端访问本地主机上运行的 S3 网关。在具体场景下，请根据实际情况调整访问 S3 网关的地址。\n:::\n\n### 使用 AWS CLI\n\n从 [https://aws.amazon.com/cli](https://aws.amazon.com/cli) 下载并安装 AWS CLI，然后进行配置：\n\n```bash\n$ aws configure\nAWS Access Key ID [None]: admin\nAWS Secret Access Key [None]: 12345678\nDefault region name [None]:\nDefault output format [None]:\n```\n\n程序会通过交互式的方式引导你完成新配置的添加，其中 `Access Key ID` 与 `MINIO_ROOT_USER` 相同，`Secret Access Key` 与 `MINIO_ROOT_PASSWORD` 相同，区域名称和输出格式请留空。\n\n之后，即可使用 `aws s3` 命令访问 JuiceFS 存储，例如：\n\n```bash\n# List buckets\n$ aws --endpoint-url http://localhost:9000 s3 ls\n\n# List objects in bucket\n$ aws --endpoint-url http://localhost:9000 s3 ls s3://\n```\n\n### 使用 MinIO 客户端\n\n为避免兼容性问题，我们推荐采用的 mc 的版本为 RELEASE.2021-04-22T17-40-00Z，你可以在这个[地址](https://dl.min.io/client/mc/release)找到历史版本和不同架构的 mc，比如这是 amd64 架构 RELEASE.2021-04-22T17-40-00Z 版本的 mc 的[下载地址](https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z)\n\n下载安装完成 mc 后添加一个新的 alias：\n\n```bash\nmc alias set juicefs http://localhost:9000 admin 12345678\n```\n\n然后，你可以通过 mc 客户端自由的在本地磁盘与 JuiceFS 存储以及其他云存储之间进行文件和文件夹的复制、移动、增删等管理操作。\n\n```shell\n$ mc ls juicefs/jfs\n[2021-10-20 11:59:00 CST] 130KiB avatar-2191932_1920.png\n[2021-10-20 11:59:00 CST] 4.9KiB box-1297327.svg\n[2021-10-20 11:59:00 CST] 21KiB cloud-4273197.svg\n[2021-10-20 11:59:05 CST] 17KiB hero.svg\n[2021-10-20 11:59:06 CST] 1.7MiB hugo-rocha-qFpnvZ_j9HU-unsplash.jpg\n[2021-10-20 11:59:06 CST] 16KiB man-1352025.svg\n[2021-10-20 11:59:06 CST] 1.3MiB man-1459246.ai\n[2021-10-20 11:59:08 CST] 19KiB sign-up-accent-left.07ab168.svg\n[2021-10-20 11:59:10 CST] 11MiB work-4997565.svg\n```\n\n## 常用功能\n\n### 多桶支持\n\n默认情况下，`juicefs gateway` 只允许一个 bucket，bucket 名字为文件系统名字，如果需要多个桶，可以在启动时添加 `--multi-buckets`开启多桶支持，该参数将会把 JuiceFS 文件系统顶级目录下的每个子目录都导出为一个 bucket。创建 bucket 的行为在文件系统上的反映是顶级目录下创建了一个同名的子目录。\n\n```shell\njuicefs gateway redis://localhost:6379/1 localhost:9000 --multi-buckets\n```\n\n
\n \n
\n\n### 保留 etag\n\n默认 S3 网关不会保存和返回对象的 etag 信息，可以通过`--keep-etag` 开启：\n\n```shell\njuicefs gateway myjfs localhost:9000 --keep-etag\n```\n\n然后通过网关上传到 JuiceFS 的文件你就可以用 s3API 的 `head-object` 来获取 etag 了：\n\n```shell\naws s3api --endpoint=http://localhost:9000 head-object --bucket myjfs --key test123/test.etag\n{\n \"AcceptRanges\": \"bytes\",\n \"LastModified\": \"Wed, 23 Apr 2025 00:17:16 GMT\",\n \"ContentLength\": 7,\n \"ETag\": \"\\\"d2fde576f44a6601b73201234b491904\\\"\",\n \"ContentType\": \"application/octet-stream\",\n \"Metadata\": {}\n}\n```\n\n这个 etag 是通过 MD5 算法生成的，并且通过 `setXattr` 设置了 key 为 `s3-tag` 的扩展属性到文件中，如果你使用 `--enable-xattr` 挂载 JuiceFS 的话也可以用 `getfattr` 来获取这个 etag：\n\n```shell\ngetfattr -n s3-etag test.etag\n# file: test.etag\ns3-etag=\"d2fde576f44a6601b73201234b491904\"\n```\n\n### 开启对象标签\n\n默认不支持对象标签，可以通过`--object-tag` 开启\n\n### 开启对象元数据 1.3\n\n默认不支持对象元数据，可以通过 `--object-meta` 开启，[参考文档](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html)\n\n### 启用虚拟主机风格请求\n\n默认情况下，S3 网关支持格式为 `http://mydomain.com/bucket/object` 的路径类型请求。`MINIO_DOMAIN` 环境变量被用来启用虚拟主机类型请求。如果请求的 `Host` 头信息匹配 `(.+).mydomain.com`，则匹配的模式 `$1` 被用作 bucket，并且路径被用作 object.\n\n示例：\n\n```shell\nexport MINIO_DOMAIN=mydomain.com\n```\n\n### 调整 IAM 刷新时间\n\n默认 IAM 缓存的刷新时间为 5 分钟，可以通过 `--refresh-iam-interval` 调整，该参数的值是一个带单位的时间字符串，例如 \"300ms\", \"-1.5h\" 或者 \"2h45m\"，有效的时间单位是 \"ns\"、\"us\" (或 \"µs\")、\"ms\"、\"s\"、\"m\"、\"h\"。\n\n例如设置 1 分钟刷新：\n\n```sh\njuicefs gateway xxxx xxxx --refresh-iam-interval 1m\n```\n\n### 多 Gateway 实例\n\nJuiceFS 的分布式特性使得可以在多个节点上同时启动多个 S3 网关实例，这样可以提高 S3 网关的可用性和性能。在这种情况下，每个 S3 网关实例都会独立地处理请求，但是它们都会访问同一个 JuiceFS 文件系统。在这种情况下，需要注意以下几点：\n\n1. 需要保证所有实例在启动时使用相同的用户，其 UID 和 GID 相同；\n2. 节点之间 IAM 刷新时间可以不同，但是需要保证 IAM 刷新时间不要太短，以免对 JuiceFS 造成过大的压力；\n3. 每个实例的监听的地址和端口可以自由设置，如果在同一台机器上启动多个实例，需要确保端口不冲突。\n\n### 以守护进程的形式运行\n\nS3 网关 可以通过以下配置以 Linux 守护进程的形式在后台运行。\n\n```shell\ncat > /lib/systemd/system/juicefs-gateway.service<\"\n metaurl: \"\"\n storage: \"\"\n accessKey: \"\"\n secretKey: \"\"\n bucket: \"\"\n```\n\n:::tip\n别忘了把上方的 `values-mycluster.yaml` 纳入 Git 项目（或者其他的源码管理方式）管理起来，这样一来，就算 values 的配置不断变化，也能对其进行追溯和回滚。\n:::\n\n填写完毕保存，就可以使用下方命令部署了：\n\n```shell\n# 不论是初次安装，还是后续调整配置重新上线，都可以使用下方命令\nhelm upgrade --install -f values-mycluster.yaml s3-gateway juicefs/juicefs-s3-gateway\n```\n\n部署完毕以后，按照输出文本的提示，获取 Kubernetes Service 的地址，并测试是否可以正常访问。\n\n```shell\n$ kubectl -n kube-system get svc -l app.kubernetes.io/name=juicefs-s3-gateway\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\njuicefs-s3-gateway ClusterIP 10.101.108.42 9000/TCP 142m\n```\n\n部署完成后，会启动一个名为 `juicefs-s3-gateway` 的 Deploy。用下方命令查看部署的 Pod：\n\n```sh\n$ kubectl -n kube-system get po -l app.kubernetes.io/name=juicefs-s3-gateway\nNAME READY STATUS RESTARTS AGE\njuicefs-s3-gateway-5c69d574cc-t92b6 1/1 Running 0 136m\n```\n\n## 高级功能\n\nJuiceFS S3 网关的核心功能是对外提供 S3 接口，目前对 S3 协议的支持已经比较完善。在 v1.2 版本中，我们又添加了对身份和访问控制（IAM）和桶事件通知的支持。\n\n这些高级功能要求 mc 客户端的版本为 `RELEASE.2021-04-22T17-40-00Z`，使用方法可以参考当时 MinIO [相关文档](https://github.com/minio/minio/tree/e0d3a8c1f4e52bb4a7d82f7f369b6796103740b3/docs)，也可以直接参考 mc 的命令行帮助信息。如果你不知道有哪些功能或者不知道某个功能如何使用，你可以直接在子命令后加 `-h` 查看帮助说明。\n\n### 身份和访问控制\n\n#### 普通用户\n\n在 v1.2 版本之前，`juicefs gateway` 只有在启动时创建一个超级用户，这个超级用户只属于这个进程，即使多个 gateway 的背后是同一个文件系统，其用户也都是进程间隔离的（你可以为每个 gateway 进程设置不同的超级用户，他们相互独立，互不影响）。\n\n在 v1.2 版本之后，`juicefs gateway` 启动时仍需要设置超级用户，该超级用户仍旧是进程隔离的，但是允许使用 `mc admin user add` 添加新的用户，新添加的用户将是同文件系统共享的。可以使用 `mc admin user` 进行管理，支持添加，关闭，启用，删除用户，也支持查看所有用户以及展示用户信息和查看用户的策略。\n\n```Shell\n$ mc admin user -h\nNAME:\n mc admin user - manage users\n\nUSAGE:\n mc admin user COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add a new user\n disable disable user\n enable enable user\n remove remove user\n list list all users\n info display info of a user\n policy export user policies in JSON format\n svcacct manage service accounts\n```\n\n例如，添加用户：\n\n```Shell\n# 添加新用户\n$ mc admin user add myjfs user1 admin123\n\n# 查看当前用户\n$ mc admin user list myjfs\nenabled user1\n\n# 查看当前用户\n$ mc admin user list myjfs --json\n{\n \"status\": \"success\",\n \"accessKey\": \"user1\",\n \"userStatus\": \"enabled\"\n}\n```\n\n#### 服务账户\n\n服务账户（service accounts）的作用是为现有用户创建一个相同权限的副本，让不同的应用可以使用独立的访问密钥。服务账户的权限继承自父用户，可以通过 `mc admin user svcacct` 命令管理。\n\n```\n$ mc admin user svcacct -h\nNAME:\n mc admin user svcacct - manage service accounts\n\nUSAGE:\n mc admin user svcacct COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add a new service account\n ls List services accounts\n rm Remove a service account\n info Get a service account info\n set edit an existing service account\n enable Enable a service account\n disable Disable a services account\n```\n\n:::tip 提示\n服务账户会从主账户继承权限并保持与主账户权限一致，而且服务账户不可以直接附加权限策略。\n:::\n\n比如，现在有一个名为 `user1` 的用户，通过以下命令为它创建一个名为 `svcacct1` 的服务账户：\n\n```Shell\nmc admin user svcacct add myjfs user1 --access-key svcacct1 --secret-key 123456abc\n```\n\n如果 `user1` 用户为只读权限，那么 `svcacct1` 也是只读权限。如果想让 `svcacct1` 拥有其他权限，则需要调整 `user1` 的权限。\n\n#### AssumeRole 安全令牌服务\n\nS3 网关安全令牌服务（STS）是一种服务，可让客户端请求 MinIO 资源的临时凭证。临时凭证的工作原理与默认管理员凭证几乎相同，但有一些不同之处：\n\n- **临时凭据顾名思义是短期的。**它们可以配置为持续几分钟到几小时不等。证书过期后，S3 网关将不再识别它们，也不允许使用它们进行任何形式的 API 请求访问。\n\n- **临时凭据不需要与应用程序一起存储，而是动态生成并在请求时提供给应用程序。**当（甚至在）临时凭据过期时，应用程序可以请求新的凭据。\n\n`AssumeRole` 会返回一组临时安全凭证，您可以使用这些凭证访问网关资源。`AssumeRole` 需要现有网关用户的授权凭据，返回的临时安全凭证包括访问密钥、秘密密钥和安全令牌。应用程序可以使用这些临时安全凭证对网关 API 操作进行签名调用，应用于这些临时凭据的策略略继承自网关用户凭据。\n\n默认情况下，`AssumeRole` 创建的临时安全凭证有效期为一个小时，可以通过可选参数 DurationSeconds 指定凭据的有效期，该值范围从 900（15 分钟）到 604800（7 天）。\n\n##### API 请求参数\n\n1. Version\n\n 指示 STS API 版本信息，唯一支持的值是 '2011-06-15'。出于兼容性原因，此值借用自 AWS STS API 文档。\n\n | Params | Value |\n | ------- | ------ |\n | Type | String |\n | Require | Yes |\n\n2. AUTHPARAMS\n\n 指示 STS API 授权信息。如果您熟悉 AWS Signature V4 授权头部，此 STS API 支持如[此处](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html)所述的签名 V4 授权。\n\n3. DurationSeconds\n\n 持续时间，以秒为单位。该值可以在 900 秒（15 分钟）至 7 天之间变化。如果值高于此设置，则操作失败。默认情况下，该值设置为 3600 秒。\n\n | Params | Value |\n | ----------- | ------------------------------- |\n | _Type_ | Integer |\n | Valid Range | 最小值为 900，最大值为 604800。 |\n | Required | No |\n\n4. Policy\n\n 您希望将其用作内联会话策略的 JSON 格式的 IAM 策略。此参数是可选的。将策略传递给此操作会返回新的临时凭证。生成会话的权限是预设策略名称和此处设置的策略集合的交集。您不能使用该策略授予比被假定预设策略名称允许的更多权限。\n\n | Params | Value |\n | ----------- | ------------------------------- |\n | Type | String |\n | Valid Range | 最小长度为 1。最大长度为 2048。 |\n | Required | No |\n\n##### 响应元素\n\n此 API 的 XML 响应类似于 [AWS STS AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_ResponseElements)\n\n##### 错误\n\n此 API 的 XML 错误响应类似于 [AWS STS AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_Errors)\n\n##### `POST`请求示例\n\n```\nhttp://minio:9000/?Action=AssumeRole&DurationSeconds=3600&Version=2011-06-15&Policy={\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Stmt1\",\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}]}&AUTHPARAMS\n```\n\n##### 响应示例\n\n```\n\n\n \n \n \n \n \n \n Y4RJU1RNFGK48LGO9I2S\n sYLRKS1Z7hSjluf6gEbb9066hnx315wHTiACPAjg\n 2019-08-08T20:26:12Z\n eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZNFJKVTFSTkZHSzQ4TEdPOUkyUyIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQxODExMDcxLCJpYXQiOjE1NDE4MDc0NzEsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiYTBiMjc2MjktZWUxYS00M2JmLTg3MzktZjMzNzRhNGNkYmMwIn0.ewHqKVFTaP-j_kgZrcOEKroNUjk10GEp8bqQjxBbYVovV0nHO985VnRESFbcT6XMDDKHZiWqN2vi_ETX_u3Q-w\n \n \n \n c6104cbe-af31-11e0-8154-cbc7ccf896c7\n \n\n```\n\n##### AWS cli 使用 AssumeRole API\n\n1. 启动 S3 网关并创建名为 foobar 的用户\n2. 配置 AWS cli\n\n ```\n [foobar]\n region = us-east-1\n aws_access_key_id = foobar\n aws_secret_access_key = foo12345\n ```\n\n3. 使用 AWS cli 请求 AssumeRole API\n\n :::note 注意\n 在以下命令中，“--role-arn”和“--role-session-name”对 S3 网关没有意义，可以设置为满足命令行要求的任何值。\n :::\n\n ```sh\n $ aws --profile foobar --endpoint-url http://localhost:9000 sts assume-role --policy '{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Stmt1\",\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}]}' --role-arn arn:xxx:xxx:xxx:xxxx --role-session-name anything\n {\n \"AssumedRoleUser\": {\n \"Arn\": \"\"\n },\n \"Credentials\": {\n \"SecretAccessKey\": \"xbnWUoNKgFxi+uv3RI9UgqP3tULQMdI+Hj+4psd4\",\n \"SessionToken\": \"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJLOURUSU1VVlpYRVhKTDNBVFVPWSIsImV4cCI6MzYwMDAwMDAwMDAwMCwicG9saWN5IjoidGVzdCJ9.PetK5wWUcnCJkMYv6TEs7HqlA4x_vViykQ8b2T_6hapFGJTO34sfTwqBnHF6lAiWxRoZXco11B0R7y58WAsrQw\",\n \"Expiration\": \"2019-02-20T19:56:59-08:00\",\n \"AccessKeyId\": \"K9DTIMUVZXEXJL3ATUOY\"\n }\n }\n ```\n\n##### go 应用程序访问 AssumeRole API\n\n请参考 MinIO 官方[示例程序](https://github.com/minio/minio/blob/master/docs/sts/assume-role.go)\n\n:::note 注意\n环境变量设置的超级用户无法使用 AssumeRole API，只有通过 `mc admin user add` 添加的用户才能使用 AssumeRole API。\n:::\n\n#### 权限管理\n\n默认新创建的用户是没有任何权限的，需要使用 `mc admin policy` 为其赋权后才可使用。该命令支持权限的增删改查以及为用户添加删除更新权限。\n\n```Shell\n$ mc admin policy -h\nNAME:\n mc admin policy - manage policies defined in the MinIO server\n\nUSAGE:\n mc admin policy COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add new policy\n remove remove policy\n list list all policies\n info show info on a policy\n set set IAM policy on a user or group\n unset unset an IAM policy for a user or group\n update Attach new IAM policy to a user or group\n```\n\nS3 网关内置了以下 4 种常用的策略：\n\n- **readonly**：只读用户\n- **readwrite**：可读写用户\n- **writeonly**：只写用户\n- **consoleAdmin**：可读可写可管理，可管理指可以调用管理 API，比如创建用户等等。\n\n例如，设置某个用户为只读：\n\n```Shell\n# 设置 user1 为只读\n$ mc admin policy set myjfs readonly user=user1\n\n# 查看用户策略\n$ mc admin user list myjfs\nenabled user1 readonly\n```\n\n以上是简单的策略，如需设置自定义的策略，可以使用 `mc admin policy add`。\n\n```Shell\n$ mc admin policy add -h\nNAME:\n mc admin policy add - add new policy\n\nUSAGE:\n mc admin policy add TARGET POLICYNAME POLICYFILE\n\nPOLICYNAME:\n Name of the canned policy on MinIO server.\n\nPOLICYFILE:\n Name of the policy file associated with the policy name.\n\nEXAMPLES:\n 1. Add a new canned policy 'writeonly'.\n $ mc admin policy add myjfs writeonly /tmp/writeonly.json\n```\n\n这里要添加的策略文件必须是一个 JSON 格式的文件，具有[IAM 兼容](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies.html)的语法，且不超过 2048 个字符。该语法可以实现更为精细化的访问控制，如果不熟悉，可以先用下面的命令查看内置的简单策略并在此基础上加以更改。\n\n```Shell\n$ mc admin policy info myjfs readonly\n{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetBucketLocation\",\n \"s3:GetObject\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::*\"\n ]\n }\n ]\n}\n```\n\n#### 用户组管理\n\nJuiceFS S3 Gateway 支持创建用户组，类似于 Linux 用户组的概念，使用 `mc admin group` 管理。你可以把一个或者多个用户设置为一个组，然后为组统一赋权，该用法与用户管理类似，此处不再赘述。\n\n```Shell\n$ mc admin group -h\nNAME:\n mc admin group - manage groups\n\nUSAGE:\n mc admin group COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]\n\nCOMMANDS:\n add add users to a new or existing group\n remove remove group or members from a group\n info display group info\n list display list of groups\n enable enable a group\n disable disable a group\n```\n\n#### 匿名访问管理\n\n以上是针对有用户记录的管理，如果希望特定的对象或桶可以被任何人访问，可以使用 `mc policy` 命令配置匿名访问策略。\n\n```Shell\nName:\n mc policy - manage anonymous access to buckets and objects\n\nUSAGE:\n mc policy [FLAGS] set PERMISSION TARGET\n mc policy [FLAGS] set-json FILE TARGET\n mc policy [FLAGS] get TARGET\n mc policy [FLAGS] get-json TARGET\n mc policy [FLAGS] list TARGET\n\nPERMISSION:\n Allowed policies are: [none, download, upload, public].\n\nFILE:\n A valid S3 policy JSON filepath.\n\nEXAMPLES:\n 1. Set bucket to \"download\" on Amazon S3 cloud storage.\n $ mc policy set download s3/burningman2011\n\n 2. Set bucket to \"public\" on Amazon S3 cloud storage.\n $ mc policy set public s3/shared\n\n 3. Set bucket to \"upload\" on Amazon S3 cloud storage.\n $ mc policy set upload s3/incoming\n\n 4. Set policy to \"public\" for bucket with prefix on Amazon S3 cloud storage.\n $ mc policy set public s3/public-commons/images\n\n 5. Set a custom prefix based bucket policy on Amazon S3 cloud storage using a JSON file.\n $ mc policy set-json /path/to/policy.json s3/public-commons/images\n\n 6. Get bucket permissions.\n $ mc policy get s3/shared\n\n 7. Get bucket permissions in JSON format.\n $ mc policy get-json s3/shared\n\n 8. List policies set to a specified bucket.\n $ mc policy list s3/shared\n\n 9. List public object URLs recursively.\n $ mc policy --recursive links s3/shared/\n```\n\nS3 网关默认内置了 4 种匿名权限\n\n- **none**：不允许匿名访问（一般用来清除已有的权限）\n- **download**：允许任何人读取\n- **upload**：允许任何人写入\n- **public**：允许任何人读写\n\n例如，设置一个允许匿名下载对象：\n\n```\n# 设置 testbucket1/afile 为匿名访问\nmc policy set download useradmin/testbucket1/afile\n\n# 查看具体权限\nmc policy get-json useradmin/testbucket1/afile\n\n$ mc policy --recursive links useradmin/testbucket1/\nhttp://127.0.0.1:9001/testbucket1/afile\n\n# 直接下载该对象\nwget http://127.0.0.1:9001/testbucket1/afile\n\n# 清除 afile 的 download 权限\nmc policy set none useradmin/testbucket1/afile\n```\n\n#### 配置生效时间\n\nJuiceFS S3 网关的所有管理 API 的更新操作都会立即生效并且持久化到 JuiceFS 文件系统中，而且接受该 API 请求的客户端也会立即生效。但是，当 S3 网关多机运行时，情况会有所不同，因为 S3 网关在处理请求鉴权时会直接采用内存缓存信息作为校验基准，避免每次请求都读取配置文件内容作为校验基准将带来不可接受的性能问题。\n\n目前 JuiceFS S3 网关的缓存刷新策略是每 5 分钟强制更新内存缓存（部分操作也会触发缓存更新操作），这样保证多机情况下配置生效最长不会超过 5 分钟，可以通过 `--refresh-iam-interval` 参数来调整该时间。如果希望某个 S3 网关立即生效，可以尝试手动将其重启。\n\n### 生成预签名 URL\n\nJuiceFS S3 网关支持使用 `mc share` 命令来管理 MinIO 存储桶上对象的预签名 URL，用于下载和上传对象。\n\n`mc share` 使用详情请参考 [这里](https://minio.org.cn/docs/minio/linux/reference/minio-mc/mc-share.html#)\n\n### 桶事件通知\n\n桶事件通知功能可以用来监视存储桶中对象上发生的事件，从而触发一些行为。\n\n目前支持的对象事件类型有：\n\n- `s3:ObjectCreated:Put`\n- `s3:ObjectCreated:CompleteMultipartUpload`\n- `s3:ObjectAccessed:Head`\n- `s3:ObjectCreated:Post`\n- `s3:ObjectRemoved:Delete`\n- `s3:ObjectCreated:Copy`\n- `s3:ObjectAccessed:Get`\n\n支持的全局事件有：\n\n- `s3:BucketCreated`\n- `s3:BucketRemoved`\n\n可以使用 mc 客户端工具通过 event 子命令设置和监听事件通知。MinIO 发送的用于发布事件的通知消息是 JSON 格式的，JSON 结构参考[这里](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html)。\n\nJuiceFS S3 网关为了减少依赖，裁剪了部分支持的事件目标类型。目前存储桶事件可以支持发布到以下目标：\n\n- Redis\n- MySQL\n- PostgreSQL\n- WebHooks\n\n```Shell\n$ mc admin config get myjfs | grep notify\nnotify_webhook publish bucket notifications to webhook endpoints\nnotify_mysql publish bucket notifications to MySQL databases\nnotify_postgres publish bucket notifications to Postgres databases\nnotify_redis publish bucket notifications to Redis datastores\n```\n\n:::note\n这里假设 JuiceFS 文件系统名为 `images`，启用 S3 Gateway 服务，在 mc 中定义它的别名为 `myjfs`。对于 S3 Gateway 而言，JuiceFS 文件系统名 `images` 就是一个存储桶名。\n:::\n\n#### 使用 Redis 发布事件\n\nRedis 事件目标支持两种格式：`namespace` 和 `access`。\n\n如果用的是 `namespacee` 格式，S3 网关将存储桶里的对象同步成 Redis hash 中的条目。对于每一个条目，对应一个存储桶里的对象，其 key 都被设为\"存储桶名称/对象名称\"，value 都是一个有关这个网关对象的 JSON 格式的事件数据。如果对象更新或者删除，hash 中对象的条目也会相应的更新或者删除。\n\n如果使用的是 access , 网关使用[RPUSH](https://redis.io/commands/rpush)将事件添加到 list 中。这个 list 中每一个元素都是一个 JSON 格式的 list，这个 list 中又有两个元素，第一个元素是时间戳的字符串，第二个元素是一个含有在这个存储桶上进行操作的事件数据的 JSON 对象。在这种格式下，list 中的元素不会更新或者删除。\n\n下面的步骤展示如何在 namespace 和 access 格式下使用通知目标。\n\n1. 配置 Redis 到 S3 网关\n\n 使用 mc admin config set 命令配置 Redis 为 事件通知的目标\n\n ```Shell\n # 命令行参数\n # mc admin config set myjfs notify_redis[:name] address=\"xxx\" format=\"namespace|access\" key=\"xxxx\" password=\"xxxx\" queue_dir=\"\" queue_limit=\"0\"\n # 具体举例\n $ mc admin config set myjfs notify_redis:1 address=\"127.0.0.1:6379/1\" format=\"namespace\" key=\"bucketevents\" password=\"yoursecret\" queue_dir=\"\" queue_limit=\"0\"\n ```\n\n 你可以通过 `mc admin config get myjfs notify_redis` 来查看有哪些配置项，不同类型的目标其配置项也不同，针对 Redis 类型，其有以下配置项：\n\n ```Shell\n $ mc admin config get myjfs notify_redis\n notify_redis enable=off format=namespace address= key= password= queue_dir= queue_limit=0\n ```\n\n 每个配置项的含义\n\n ```Shell\n notify_redis[:name] 支持设置多个 redis，只需要其 name 不同即可\n address* (address) Redis 服务器的地址。例如：localhost:6379\n key* (string) 存储/更新事件的 Redis key, key 会自动创建\n format* (namespace*|access) 是 namespace 还是 access，默认是 'namespace'\n password (string) Redis 服务器的密码\n queue_dir (path) 未发送消息的暂存目录 例如 '/home/events'\n queue_limit (number) 未发送消息的最大限制，默认是'100000'\n comment (sentence) 可选的注释说明\n ```\n\n S3 网关支持持久事件存储。持久存储将在 Redis broker 离线时备份事件，并在 broker 恢复在线时重播事件。事件存储的目录可以通过 queue_dir 字段设置，存储的最大限制可以通过 queue_limit 设置。例如，queue_dir 可以设置为/home/events, 并且 queue_limit 可以设置为 1000. 默认情况下 queue_limit 是 100000。在更新配置前，可以通过 mc admin config get 命令获取当前配置。\n\n ```Shell\n $ mc admin config get myjfs notify_redis\n notify_redis:1 address=\"127.0.0.1:6379/1\" format=\"namespace\" key=\"bucketevents\" password=\"yoursecret\" queue_dir=\"\" queue_limit=\"0\"\n\n # 重启后生效\n $ mc admin config set myjfs notify_redis:1 queue_limit=\"1000\"\n Successfully applied new settings.\n Please restart your server 'mc admin service restart myjfs'.\n # 注意这里无法使用 mc admin service restart myjfs 重启，JuiceFS S3 网关暂不支持该功能，当使用 mc 配置后出现该提醒时需要手动重启 JuiceFS Gateway\n ```\n\n 使用 mc admin config set 命令更新配置后，重启 JuiceFS S3 网关让配置生效。如果一切顺利，JuiceFS S3 网关会在启动时输出一行信息，类似 `SQS ARNs: arn:minio:sqs::1:redis`\n\n 根据你的需要，你可以添加任意多个 Redis 目标，只要提供 Redis 实例的标识符（如上例“notify_redis:1”中的“1”）和每个实例配置参数的信息即可。\n\n2. 启用 bucket 通知\n\n 我们现在可以在一个叫 images 的存储桶上开启事件通知。当一个 JPEG 文件被创建或者覆盖，一个新的 key 会被创建，或者一个已经存在的 key 就会被更新到之前配置好的 Redis hash 里。如果一个已经存在的对象被删除，这个对应的 key 也会从 hash 中删除。因此，这个 Redis hash 里的行，就映射着 images 存储桶里的.jpg 对象。\n\n 要配置这种存储桶通知，我们需要用到前面步骤 S3 网关输出的 ARN 信息。更多有关 ARN 的资料，请参考[这里](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)。\n\n 使用 mc 为文件系统启用事件通知：\n\n ```Shell\n mc event add myjfs/images arn:minio:sqs::1:redis --suffix .jpg\n mc event list myjfs/images\n arn:minio:sqs::1:redis s3:ObjectCreated:*,s3:ObjectRemoved:*,s3:ObjectAccessed:* Filter: suffix=\".jpg\"\n ```\n\n3. 验证 Redis\n\n 启动 `redis-cli` 这个 Redis 客户端程序来检查 Redis 中的内容。运行 monitor Redis 命令将会输出在 Redis 上执行的每个命令的。\n\n ```Shell\n redis-cli -a yoursecret\n 127.0.0.1:6379> monitor\n OK\n ```\n\n 上传一个名为 myphoto.jpg 的文件到 images 存储桶。\n\n ```Shell\n mc cp myphoto.jpg myjfs/images\n ```\n\n 在上一个终端中，你将看到 S3 网关在 Redis 上执行的操作：\n\n ```Shell\n 127.0.0.1:6379> monitor\n OK\n 1712562516.867831 [1 192.168.65.1:59280] \"hset\" \"bucketevents\" \"images/myphoto.jpg\" \"{\\\"Records\\\":[{\\\"eventVersion\\\":\\\"2.0\\\",\\\"eventSource\\\":\\\"minio:s3\\\",\\\"awsRegion\\\":\\\"\\\",\\\"eventTime\\\":\\\"2024-04-08T07:48:36.865Z\\\",\\\"eventName\\\":\\\"s3:ObjectCreated:Put\\\",\\\"userIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"requestParameters\\\":{\\\"principalId\\\":\\\"admin\\\",\\\"region\\\":\\\"\\\",\\\"sourceIPAddress\\\":\\\"127.0.0.1\\\"},\\\"responseElements\\\":{\\\"content-length\\\":\\\"0\\\",\\\"x-amz-request-id\\\":\\\"17C43E891887BA48\\\",\\\"x-minio-origin-endpoint\\\":\\\"http://127.0.0.1:9001\\\"},\\\"s3\\\":{\\\"s3SchemaVersion\\\":\\\"1.0\\\",\\\"configurationId\\\":\\\"Config\\\",\\\"bucket\\\":{\\\"name\\\":\\\"images\\\",\\\"ownerIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"arn\\\":\\\"arn:aws:s3:::images\\\"},\\\"object\\\":{\\\"key\\\":\\\"myphoto.jpg\\\",\\\"size\\\":4,\\\"eTag\\\":\\\"40b134ab8a3dee5dd9760a7805fd495c\\\",\\\"userMetadata\\\":{\\\"content-type\\\":\\\"image/jpeg\\\"},\\\"sequencer\\\":\\\"17C43E89196AE2A0\\\"}},\\\"source\\\":{\\\"host\\\":\\\"127.0.0.1\\\",\\\"port\\\":\\\"\\\",\\\"userAgent\\\":\\\"MinIO (darwin; arm64) minio-go/v7.0.11 mc/RELEASE.2021-04-22T17-40-00Z\\\"}}]}\"\n ```\n\n 在这我们可以看到 S3 网关在 minio_events 这个 key 上执行了 HSET 命令。\n\n 如果用的是 access 格式，那么 minio_events 就是一个 list，S3 网关就会调用 RPUSH 添加到 list 中，在 monitor 命令中将看到：\n\n ```Shell\n 127.0.0.1:6379> monitor\n OK\n 1712562751.922469 [1 192.168.65.1:61102] \"rpush\" \"aceesseventskey\" \"[{\\\"Event\\\":[{\\\"eventVersion\\\":\\\"2.0\\\",\\\"eventSource\\\":\\\"minio:s3\\\",\\\"awsRegion\\\":\\\"\\\",\\\"eventTime\\\":\\\"2024-04-08T07:52:31.921Z\\\",\\\"eventName\\\":\\\"s3:ObjectCreated:Put\\\",\\\"userIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"requestParameters\\\":{\\\"principalId\\\":\\\"admin\\\",\\\"region\\\":\\\"\\\",\\\"sourceIPAddress\\\":\\\"127.0.0.1\\\"},\\\"responseElements\\\":{\\\"content-length\\\":\\\"0\\\",\\\"x-amz-request-id\\\":\\\"17C43EBFD35A53B8\\\",\\\"x-minio-origin-endpoint\\\":\\\"http://127.0.0.1:9001\\\"},\\\"s3\\\":{\\\"s3SchemaVersion\\\":\\\"1.0\\\",\\\"configurationId\\\":\\\"Config\\\",\\\"bucket\\\":{\\\"name\\\":\\\"images\\\",\\\"ownerIdentity\\\":{\\\"principalId\\\":\\\"admin\\\"},\\\"arn\\\":\\\"arn:aws:s3:::images\\\"},\\\"object\\\":{\\\"key\\\":\\\"myphoto.jpg\\\",\\\"size\\\":4,\\\"eTag\\\":\\\"40b134ab8a3dee5dd9760a7805fd495c\\\",\\\"userMetadata\\\":{\\\"content-type\\\":\\\"image/jpeg\\\"},\\\"sequencer\\\":\\\"17C43EBFD3DACA70\\\"}},\\\"source\\\":{\\\"host\\\":\\\"127.0.0.1\\\",\\\"port\\\":\\\"\\\",\\\"userAgent\\\":\\\"MinIO (darwin; arm64) minio-go/v7.0.11 mc/RELEASE.2021-04-22T17-40-00Z\\\"}}],\\\"EventTime\\\":\\\"2024-04-08T07:52:31.921Z\\\"}]\"\n ```\n\n#### 使用 MySQL 发布事件\n\nMySQL 通知目标支持两种格式：`namespace` 和 `access`。\n\n如果使用的是 `namespace` 格式，S3 网关将存储桶里的对象同步成数据库表中的行。每一行有两列：key_name 和 value。key_name 是这个对象的存储桶名字加上对象名，value 都是一个有关这个 S3 网关对象的 JSON 格式的事件数据。如果对象更新或者删除，表中相应的行也会相应的更新或者删除。\n\n如果使用的是 `access`，S3 网关将将事件添加到表里，行有两列：event_time 和 event_data。event_time 是事件在 S3 网关 server 里发生的时间，event_data 是有关这个 S3 网关对象的 JSON 格式的事件数据。在这种格式下，不会有行会被删除或者修改。\n\n下面的步骤展示的是如何在 `namespace` 格式下使用通知目标，与 `access` 类似，不再赘述。\n\n1. 确保 MySQL 版本至少满足最低要求\n\n JuiceFS S3 网关要求 MySQL 版本 5.7.8 及以上，因为使用了 MySQL5.7.8 版本才引入的[JSON](https://dev.mysql.com/doc/refman/5.7/en/json.html) 数据类型。\n\n2. 配置 MySQL 到 S3 网关\n\n 使用 `mc admin config set` 命令配置 MySQL 为事件通知的目标\n\n ```Shell\n mc admin config set myjfs notify_mysql:myinstance table=\"minio_images\" dsn_string=\"root:123456@tcp(172.17.0.1:3306)/miniodb\"\n ```\n\n 你可以通过 `mc admin config get myjfs notify_mysql` 来查看有哪些配置项，不同类型的目标其配置项也不同，针对 MySQL 类型，其有以下配置项：\n\n ```shell\n $ mc admin config get myjfs notify_mysql\n format=namespace dsn_string= table= queue_dir= queue_limit=0 max_open_connections=2\n ```\n\n 每个配置项的含义\n\n ```Shell\n KEY:\n notify_mysql[:name] 发布存储桶通知到 MySQL 数据库。当需要多个 MySQL server endpoint 时，可以为每个配置添加用户指定的“name”（例如\"notify_mysql:myinstance\"）.\n\n ARGS:\n dsn_string* (string) MySQL 数据源名称连接字符串，例如 \":@tcp(:)/\"\n table* (string) 存储/更新事件的数据库表名，表会自动被创建\n format* (namespace*|access) 'namespace' 或者 'access', 默认是 'namespace'\n queue_dir (path) 未发送消息的暂存目录 例如 '/home/events'\n queue_limit (number) 未发送消息的最大限制，默认是 '100000'\n comment (sentence) 可选的注释说明\n ```\n\n dsn_string 是必须的，并且格式为 `:@tcp(:)/`\n\n MinIO 支持持久事件存储。持久存储将在 MySQL 连接离线时备份事件，并在 broker 恢复在线时重播事件。事件存储的目录可以通过 queue_dir 字段设置，存储的最大限制可以通过 queue_limit 设置。例如，queue_dir 可以设置为 /home/events, 并且 queue_limit 可以设置为 1000。默认情况下 queue_limit 是 100000。\n\n 更新配置前，可以使用 `mc admin config get` 命令获取当前配置：\n\n ```Shell\n $ mc admin config get myjfs/ notify_mysql\n notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0\n ```\n\n 使用带有 dsn_string 参数的 `mc admin config set` 的命令更新 MySQL 的通知配置：\n\n ```Shell\n mc admin config set myjfs notify_mysql:myinstance table=\"minio_images\" dsn_string=\"root:xxxx@tcp(127.0.0.1:3306)/miniodb\"\n ```\n\n 请注意，根据你的需要，你可以添加任意多个 MySQL server endpoint，只要提供 MySQL 实例的标识符（如上例中的\"myinstance\"）和每个实例配置参数的信息即可。\n\n 使用`mc admin config set`命令更新配置后，重启 S3 网关让配置生效。如果一切顺利，S3 网关 Server 会在启动时输出一行信息，类似 `SQS ARNs: arn:minio:sqs::myinstance:mysql`\n\n3. 启用 bucket 通知\n\n 我们现在可以在一个叫 images 的存储桶上开启事件通知，一旦上有文件上传到存储桶中，MySQL 中会 insert 一条新的记录或者一条已经存在的记录会被 update，如果一个存在对象被删除，一条对应的记录也会从 MySQL 表中删除。因此，MySQL 表中的行，对应的就是存储桶里的一个对象。\n\n 要配置这种存储桶通知，我们需要用到前面步骤 MinIO 输出的 ARN 信息。更多有关 ARN 的资料，请参考[这里](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)。\n\n 假设 S3 网关服务别名叫 myjfs，可执行下列脚本：\n\n ```Shell\n # 使用 MySQL ARN 在“images”存储桶上添加通知配置。--suffix 参数用于过滤事件。\n mc event add myjfs/images arn:minio:sqs::myinstance:mysql --suffix .jpg\n # 在“images”存储桶上打印出通知配置。\n mc event list myjfs/images\n arn:minio:sqs::myinstance:mysql s3:ObjectCreated:*,s3:ObjectRemoved:*,s3:ObjectAccessed:* Filter: suffix=”.jpg”\n ```\n\n4. 验证 MySQL\n\n 打开一个新的 terminal 终端并上传一张 JPEG 图片到 images 存储桶。\n\n ```Shell\n mc cp myphoto.jpg myjfs/images\n ```\n\n 打开一个 MySQL 终端列出表 minio_images 中所有的记录，将会发现一条刚插入的记录。\n\n#### 使用 PostgreSQL 发布事件\n\n整体方法与使用 MySQL 发布 MinIO 事件相同，这里不再赘述。\n\n需要注意的是，该功能要求 PostgreSQL 9.5 版本及以上。S3 网关用了 PostgreSQL 9.5 引入的[INSERT ON CONFLICT](https://www.postgresql.org/docs/9.5/static/sql-insert.html#SQL-ON-CONFLICT) (aka UPSERT) 特性，以及 9.4 引入的[JSONB](https://www.postgresql.org/docs/9.4/static/datatype-json.html) 数据类型。\n\n#### 使用 Webhook 发布事件\n\n[Webhooks](https://en.wikipedia.org/wiki/Webhook) 采用推的方式获取数据，而不是一直去拉取。\n\n1. 配置 webhook 到 S3 网关\n\n S3 网关支持持久事件存储。持久存储将在 webhook 离线时备份事件，并在 broker 恢复在线时重播事件。事件存储的目录可以通过 `queue_dir` 字段设置，存储的最大限制可以通过 `queue_limit` 设置。例如，`/home/events`，并且 `queue_limit` 可以设置为 1000。默认情况下 `queue_limit` 是 100000。\n\n ```Shell\n KEY:\n notify_webhook[:name] 发布存储桶通知到 webhook endpoints\n\n ARGS:\n endpoint* (url) webhook server endpoint，例如 http://localhost:8080/minio/events\n auth_token (string) opaque token 或者 JWT authorization token\n queue_dir (path) 未发送消息的暂存目录 例如 '/home/events'\n queue_limit (number) 未发送消息的最大限制，默认是 '100000'\n client_cert (string) Webhook 的 mTLS 身份验证的客户端证书\n client_key (string) Webhook 的 mTLS 身份验证的客户端证书密钥\n comment (sentence) 可选的注释说明\n ```\n\n 用 `mc admin config set` 命令更新配置，这里的 endpoint 是监听 webhook 通知的服务地址。保存配置文件并重启 MinIO 服务让配配置生效。注意，在重启 MinIO 时，这个 endpoint 必须是启动并且可访问到。\n\n ```Shell\n mc admin config set myjfs notify_webhook:1 queue_limit=\"0\" endpoint=\"http://localhost:3000\" queue_dir=\"\"\n ```\n\n2. 启用 bucket 通知\n\n 在这里，ARN 的值是 `arn:minio:sqs::1:webhook`。更多有关 ARN 的资料，请参考[这里](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)。\n\n ```Shell\n mc mb myjfs/images-thumbnail\n mc event add myjfs/images arn:minio:sqs::1:webhook --event put --suffix .jpg\n ```\n\n 如果 mc 报告无法创建 Bucket，请检查 S3 Gateway 是否启用了[多桶支持](#多桶支持)。\n\n3. 采用 Thumbnailer 进行验证\n\n [Thumbnailer](https://github.com/minio/thumbnailer) 项目是一个使用 MinIO 的 listenBucketNotification API 的缩略图生成器示例，我们使用 [Thumbnailer](https://github.com/minio/thumbnailer) 来监听 S3 网关通知。如果有文件上传于是 S3 网关服务，Thumnailer 监听到该通知，生成一个缩略图并上传到 S3 网关服务。安装 Thumbnailer:\n\n ```Shell\n git clone https://github.com/minio/thumbnailer/\n npm install\n ```\n\n 然后打开 Thumbnailer 的 `config/webhook.json` 配置文件，添加有关 MinIO server 的配置，使用下面的方式启动 Thumbnailer:\n\n ```Shell\n NODE_ENV=webhook node thumbnail-webhook.js\n ```\n\n Thumbnailer 运行在 `http://localhost:3000/`\n\n 下一步，配置 MinIO server，让其发送消息到这个 URL（第一步提到的），并使用 mc 来设置存储桶通知（第二步提到的）。然后上传一张图片到 S3 网关 server：\n\n ```Shell\n mc cp ~/images.jpg myjfs/images\n .../images.jpg: 8.31 KB / 8.31 KB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 100.00% 59.42 KB/s 0s\n ```\n\n 稍等片刻，然后使用 mc ls 检查存储桶的内容，你将看到有个缩略图出现了。\n\n ```Shell\n mc ls myjfs/images-thumbnail\n [2017-02-08 11:39:40 IST] 992B images-thumbnail.jpg\n ```\n" }, { "path": "docs/zh_cn/guide/quota.md", "content": "---\ntitle: 存储配额\nsidebar_position: 4\n---\n\nJuiceFS 同时支持文件系统总配额和子目录配额，均可用于限制可用容量和可用 inode 数量。文件系统配额和目录配额均是硬限制，当文件系统总配额用尽时，后续写入会返回 `ENOSPC`（No space left）错误；而当目录配额用尽时，后续写入会返回 `EDQUOT`（Disk quota exceeded）错误。\n\n:::tip 提示\n存储限额设置会保存在元数据引擎中以供所有挂载点读取，每个挂载点的客户端也会缓存自己的已用容量和 inodes 数，周期性地向元数据引擎同步。与此同时，客户端也会周期性地从元数据引擎读取最新的用量值，从而实现用量信息在每个挂载点之间同步，但这种信息同步机制并不能保证用量数据被精确统计，可能会存在十秒级延迟。\n:::\n\n## 文件系统配额 {#file-system-quota}\n\nJuiceFS v1.0 支持文件系统级别的存储配额。以 Linux 环境为例，使用系统自带的 `df` 命令可以看到，一个 JuiceFS 类型的文件系统默认的容量标识为 `1.0P` ：\n\n```shell\n$ df -Th | grep juicefs\nJuiceFS:ujfs fuse.juicefs 1.0P 682M 1.0P 1% /mnt\n```\n\n:::note 说明\nJuiceFS 通过 FUSE 实现对 POSIX 接口的支持，因为底层通常是容量能够无限扩展的对象存储，所以标识容量只是一个估值（也代表无限制）并非实际容量，它会随着实际用量动态变化。\n:::\n\n通过客户端自带的 `config` 命令可以查看一个文件系统的详细信息：\n\n```shell\n$ juicefs config $METAURL\n{\n \"Name\": \"ujfs\",\n \"UUID\": \"1aa6d290-279b-432f-b9b5-9d7fd597dec2\",\n \"Storage\": \"minio\",\n \"Bucket\": \"127.0.0.1:9000/jfs1\",\n \"AccessKey\": \"herald\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0,\n \"TrashDays\": 0\n}\n```\n\n### 限制总容量 {#limit-total-capacity}\n\n可以在创建文件系统时通过 `--capacity` 设置容量限额，单位 GiB，例如创建一个可用容量为 100 GiB 文件系统的：\n\n```shell\njuicefs format --storage minio \\\n --bucket 127.0.0.1:9000/jfs1 \\\n ... \\\n --capacity 100 \\\n $METAURL myjfs\n```\n\n也可以通过 `config` 命令，为一个已创建的文件系统设置容量限额：\n\n```shell\n$ juicefs config $METAURL --capacity 100\n2022/01/27 12:31:39.506322 juicefs[16259] : Meta address: postgres://herald@127.0.0.1:5432/jfs1\n2022/01/27 12:31:39.521232 juicefs[16259] : The latency to database is too high: 14.771783ms\n capacity: 0 GiB -> 100 GiB\n```\n\n设置了存储限额的文件系统，标识容量会变成限制容量：\n\n```shell\n$ df -Th | grep juicefs\nJuiceFS:ujfs fuse.juicefs 100G 682M 100G 1% /mnt\n```\n\n### 限制 inode 总量 {#limit-total-number-of-inodes}\n\n在 Linux 系统中，每个文件（文件夹也是文件的一种）不论大小都有一个 inode，因此限制 inode 数量等同于限制文件数量。\n\n可以在创建文件系统时通过 `--inodes` 设置限额，例如：\n\n```shell\njuicefs format --storage minio \\\n --bucket 127.0.0.1:9000/jfs1 \\\n ... \\\n --inodes 100 \\\n $METAURL myjfs\n```\n\n以上命令创建的文件系统仅允许存储 100 个文件，但不限制单个文件的大小，比如单个文件 1TB 甚至更大也没有问题，只要文件总数不超过 100 个即可。\n\n也可以通过 `config` 命令，为一个已创建的文件系统设置容量限额：\n\n```shell\n$ juicefs config $METAURL --inodes 100\n2022/01/27 12:35:37.311465 juicefs[16407] : Meta address: postgres://herald@127.0.0.1:5432/jfs1\n2022/01/27 12:35:37.322991 juicefs[16407] : The latency to database is too high: 11.413961ms\n inodes: 0 -> 100\n```\n\n### 组合使用 {#limit-total-capacity-and-inodes}\n\n你可以结合 `--capacity` 和 `--inodes` 更灵活的设置文件系统的容量限额，比如，创建一个文件系统，限制总容量为 100TiB 且仅允许存储 100000 文件：\n\n```shell\njuicefs format --storage minio \\\n --bucket 127.0.0.1:9000/jfs1 \\\n ... \\\n --capacity 102400 \\\n --inodes 100000 \\\n $METAURL myjfs\n```\n\n同样地，对于已创建的文件系统，可分别进行设置：\n\n```shell\njuicefs config $METAURL --capacity 102400\n```\n\n```shell\njuicefs config $METAURL --inodes 100000\n```\n\n:::tip 提示\n客户端会定期从元数据引擎读取最新的文件系统存储限额设置来更新本地的设置。刷新间隔由 `--heartbeat` 参数控制（默认值：12 秒）。其他挂载点可能需要等待最多一个 heartbeat 间隔时间才能完成限额设置的更新。\n:::\n\n## 目录配额 {#directory-quota}\n\nJuiceFS v1.1 开始支持目录级别的存储配额，可以使用 `juicefs quota` 子命令进行目录配额管理和查询。\n\n:::tip 提示\n由于用量统计需要挂载客户端支持，请确保除所有可写入客户端已升级到 v1.1.0 以上版本再使用此特性。\n:::\n\n### 限制目录容量 {#limit-directory-capacity}\n\n可以使用 `juicefs quota set $METAURL --path $DIR --capacity $N` 设置目录容量限额，单位 GiB。例如给目录`/test`设置 1GiB 的容量配额：\n\n```shell\n$ juicefs quota set $METAURL --path /test --capacity 1\n+-------+---------+---------+------+-----------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+---------+---------+------+-----------+-------+-------+\n| /test | 1.0 GiB | 1.6 MiB | 0% | unlimited | 314 | |\n+-------+---------+---------+------+-----------+-------+-------+\n```\n\n设置成功后你可以看到有一个表格描述当前设置配额的目录、配额大小、当前用量等信息。\n\n:::tip 提示\n`quota` 子命令的使用无需本地挂载点，期望输入的目录路径为相对 JuiceFS 根目录的路径而非本地挂载路径。给大目录设置配额可能需要等待较长时间，因为需要计算目录当前用量。\n:::\n\n如果需要查询某个目录的配额和当前用量，可以使用 `juicefs quota get $METAURL --path $DIR` 命令：\n\n```shell\n$ juicefs quota get $METAURL --path /test\n+-------+---------+---------+------+-----------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+---------+---------+------+-----------+-------+-------+\n| /test | 1.0 GiB | 1.6 MiB | 0% | unlimited | 314 | |\n+-------+---------+---------+------+-----------+-------+-------+\n```\n\n也可以使用 `juicefs quota ls $METAURL` 命令列出所有的目录配额。\n\n### 限制目录的 inode 总量 {#limit-total-number-of-directory-inodes}\n\n可以使用 `juicefs quota set $METAURL --path $DIR --inodes $N` 设置目录 inode 限额，单位为个。例如给目录`/test`设置 400 个 inode 的配额：\n\n```shell\n$ juicefs quota set $METAURL --path /test --inodes 400\n+-------+---------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+---------+---------+------+--------+-------+-------+\n| /test | 1.0 GiB | 1.6 MiB | 0% | 400 | 314 | 78% |\n+-------+---------+---------+------+--------+-------+-------+\n```\n\n### 组合使用 {#limit-capacity-and-inodes-of-directory}\n\n可以结合 `--capacity` 和 `--inodes` 更灵活地设置目录的容量限额。比如，给`/test`目录设置 10GiB 和 1000 个 inode 的配额：\n\n```shell\n$ juicefs quota set $METAURL --path /test --capacity 10 --inodes 1000\n+-------+--------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+--------+---------+------+--------+-------+-------+\n| /test | 10 GiB | 1.6 MiB | 0% | 1,000 | 314 | 31% |\n+-------+--------+---------+------+--------+-------+-------+\n```\n\n另外，你也可以不限制目录的容量和 inode 数（设为 `0` 表示不限制），只通过 `quota` 命令统计目录的当前用量：\n\n```shell\n$ juicefs quota set $METAURL --path /test --capacity 0 --inodes 0\n+-------+-----------+---------+------+-----------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+-----------+---------+------+-----------+-------+-------+\n| /test | unlimited | 1.6 MiB | | unlimited | 314 | |\n+-------+-----------+---------+------+-----------+-------+-------+\n```\n\n### 配额嵌套 {#nested-quota}\n\nJuiceFS 允许自由地设置各级目录配额，实际使用的时候会递归地向上查询，确保当前目录用量满足每一级目录的配额设置。也就是说，就算父目录设置了一个较小的配额，也不影响子目录可以设置更大配额。\n\n### 子目录挂载 {#subdirectory-mount}\n\nJuiceFS 支持使用 [`--subdir`](../reference/command_reference.mdx#mount-metadata-options) 挂载任意子目录。如果挂载的子目录设置了目录配额，则可以使用系统自带的 `df` 命令查看目录配额和当前使用量。比如文件系统配额为 1PiB 和 10M 个 inode，而 `/test` 目录的配额为 1GiB 和 400 个 inode。使用根目录挂载时 `df` 命令的输出为：\n\n```shell\n$ df -h\nFilesystem Size Used Avail Use% Mounted on\n...\nJuiceFS:myjfs 1.0P 1.6M 1.0P 1% /mnt/jfs\n\n$ df -i -h\nFilesystem Inodes IUsed IFree IUse% Mounted on\n...\nJuiceFS:myjfs 11M 315 10M 1% /mnt/jfs\n```\n\n而使用 `/test` 子目录挂载时，`df` 命令的输出为：\n\n```shell\n$ df -h\nFilesystem Size Used Avail Use% Mounted on\n...\nJuiceFS:myjfs 1.0G 1.6M 1023M 1% /mnt/jfs\n\n$ df -i -h\nFilesystem Inodes IUsed IFree IUse% Mounted on\n...\nJuiceFS:myjfs 400 314 86 79% /mnt/jfs\n```\n\n:::note 说明\n当挂载的子目录没有设置配额，JuiceFS 会逐级往上查询知道找到最近的目录配额再返回给 `df`。如果有多级父目录均设置目录配额，JuiceFS 会在计算后返回最小的可用容量和 inode 数量。\n:::\n\n### 用量检查与修复 {#usage-check-and-fix}\n\n由于目录用量的更新是滞后且异步的，在异常情况下可能会发生丢失（比如客户端意外退出）。我们可以使用 `juicefs quota check $METAURL --path $DIR` 命令进行检查或修复：\n\n```shell\n$ juicefs quota check $METAURL --path /test\n2023/05/23 15:40:12.704576 juicefs[1638846] : quota of /test is consistent [base.go:839]\n+-------+--------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+--------+---------+------+--------+-------+-------+\n| /test | 10 GiB | 1.6 MiB | 0% | 1,000 | 314 | 31% |\n+-------+--------+---------+------+--------+-------+-------+\n```\n\n目录用量正确时会输出当前的目录配额用量；失败时候则会输出错误日志：\n\n```shell\n$ juicefs quota check $METAURL --path /test\n2023/05/23 15:48:17.494604 juicefs[1639997] : /test: quota(314, 4.0 KiB) != summary(314, 1.6 MiB) [base.go:843]\n2023/05/23 15:48:17.494644 juicefs[1639997] : quota of /test is inconsistent, please repair it with --repair flag [main.go:31]\n```\n\n这时你可以使用 `--repair` 选项来修复目录用量：\n\n```shell\n$ juicefs quota check $METAURL --path /test --repair\n2023/05/23 15:50:08.737086 juicefs[1640281] : /test: quota(314, 4.0 KiB) != summary(314, 1.6 MiB) [base.go:843]\n2023/05/23 15:50:08.737123 juicefs[1640281] : repairing... [base.go:852]\n+-------+--------+---------+------+--------+-------+-------+\n| Path | Size | Used | Use% | Inodes | IUsed | IUse% |\n+-------+--------+---------+------+--------+-------+-------+\n| /test | 10 GiB | 1.6 MiB | 0% | 1,000 | 314 | 31% |\n+-------+--------+---------+------+--------+-------+-------+\n```\n" }, { "path": "docs/zh_cn/guide/sync.md", "content": "---\ntitle: 数据同步\nsidebar_position: 7\ndescription: 了解如何使用 JuiceFS 中的数据同步工具。\n---\n\n[`juicefs sync`](../reference/command_reference.mdx#sync) 是强大的数据同步工具，可以在所有支持的存储之间并发同步或迁移数据，包括对象存储、JuiceFS、本地文件系统，你可以在这三者之间以任意方向和搭配进行数据同步。除此之外，还支持同步通过 SSH 访问远程目录、HDFS、WebDAV 等，同时提供增量同步、模式匹配（类似 rsync）、分布式同步等高级功能。\n\n:::tip 混用社区版和企业版客户端\n`juicefs sync` 功能的代码在社区版和企业版之间共享代码，因此即便交叉混用不同版本的 JuiceFS 客户端，`sync` 命令也能正常工作——除了一个特例，就是使用 [`jfs://`](#sync-without-mount-point) 协议头的情况。社区版和企业版客户端有着不同的元数据引擎实现，因此如果用到了 `jfs://` 协议头，则不能混用不同版本的客户端。\n:::\n\n`juicefs sync` 用法以及常见示范如下：\n\n```shell\njuicefs sync [command options] SRC DST\n\n# 从 OSS 同步到 S3\njuicefs sync oss://mybucket.oss-cn-shanghai.aliyuncs.com s3://mybucket.s3.us-east-2.amazonaws.com\n\n# 拷贝所有以 .gz 结尾的文件\njuicefs sync --match-full-path --include='**.gz' --exclude='*' s3://xxx s3://xxx\n\n# 拷贝不以 .gz 结尾的所有文件\njuicefs sync --match-full-path --exclude='**.gz' s3://xxx/ s3://xxx\n\n# 拷贝所有文件，但忽略名为 tmpdir 的子目录\njuicefs sync --match-full-path --exclude='**/tmpdir/**' s3://xxx/ s3://xxx\n```\n\n## 快速上手视频\n\n
\n \n
\n\n## 模式匹配 {#pattern-matching}\n\n你可以通过 `--exclude` 和 `--include` 来包含或排除要同步的文件路径。如果不提供任何规则，默认会同步所有扫描到的文件（默认就是 `--include='*'`）。但如果需要使用 `--include` 实现只包含特定命名模式的文件，则**必须同时使用 `--exclude` 来排除其他文件**，具体请参考上方的示范命令。\n\n:::tip\n当提供多个匹配模式时，取决于你具体使用的「过滤模式」，对于判断是否要同步某个文件可能会变得很困难。此时建议加上 `--dry --debug` 选项提前查看要同步的具体文件是否符合预期，如果不符合预期则需要调整匹配模式。\n:::\n\n### 匹配规则 {#matching-rules}\n\n匹配规则指的是给定一个路径与一个模式，然后确定该路径能否匹配上该模式。模式可以包含一些特殊字符（类似 shell 通配符）：\n\n+ 单个 `*` 匹配任意字符，但在遇到 `/` 时终止匹配；\n+ `**` 匹配任意字符，包括 `/`；\n+ `?` 匹配任意非 `/` 的单个字符；\n+ `[...]` 匹配一组字符，例如 `[a-z]` 匹配任意小写字母；\n+ `[^...]` 不匹配一组字符，例如 `[^abc]` 匹配除 `a`、`b`、`c` 外的任意字符。\n\n此外，还有一些匹配规则需要注意：\n\n- 如果匹配模式中不包含特殊字符，将会完整匹配路径中的文件名。比如 `foo` 可以匹配 `foo` 和 `xx/foo`，但不匹配 `foo1`（无法前缀匹配）、`2foo`（无法后缀匹配）和 `foo/xx`（`foo` 不是目录）；\n- 如果匹配模式以 `/` 结尾，将只匹配目录，而不匹配普通文件；\n- 如果匹配模式以 `/` 开头，则表示匹配完整路径（路径不需要以 `/` 开头），因此 `/foo` 匹配的是传输中根目录的 `foo` 文件。\n\n以下是一些匹配模式的例子：\n\n+ `--exclude '*.o'` 将排除所有文件名能匹配 `*.o` 的文件；\n+ `--exclude '/foo/*/bar'` 将排除根目录中名为 `foo` 的目录向下「两层」的目录中名为 `bar` 的文件；\n+ `--exclude '/foo/**/bar'` 将排除根目录中名为 `foo` 的目录向下「任意层级」的目录中名为 `bar` 的文件。\n\n`sync` 命令支持「完整路径过滤」和「逐层过滤」两种模式，这两种模式都支持使用 `--include` 和 `--exclude` 来过滤文件，但是解析的行为并不一样：默认情况下，`sync` 命令使用逐层过滤模式，这种模式的过滤行为无论是理解还是使用都较为复杂，但是基本兼容 rsync 的 `--include/--exclude` 选项，所以只推荐已经习惯了 rsync 过滤行为的用户使用。对于大多数 JuiceFS 用户，推荐通过 `--match-full-path` 选项来使用完整路径过滤模式，他的工作流程更容易理解。\n\n### 完整路径过滤模式（推荐） 1.2.0 {#full-path-filtering-mode}\n\n从 v1.2.0 开始，sync 命令支持 `--match-full-path` 选项。完整路径过滤模式是指对于待匹配的对象，直接将其「全路径」与多个模式依次进行匹配，一旦某个匹配模式匹配成功将会直接返回结果（「同步」或者「排除」），忽略后续的匹配模式。\n\n下面是完整路径过滤模式的工作流程图：\n\n![完整路径过滤模式流程图](../images/sync-full-path-filtering-mode-flow-chart.svg)\n\n例如有一个路径为 `a1/b1/c1.txt` 的文件，以及 3 个匹配模式 `--include 'a*.txt' --inlude 'c1.txt' --exclude 'c*.txt'`。在完整路径过滤模式下，会直接将 `a1/b1/c1.txt` 这个字符串与匹配模式依次进行匹配。具体步骤为：\n\n1. 尝试将 `a1/b1/c1.txt` 与 `--include 'a*.txt'` 匹配，结果是不匹配。因为 `*` 不能匹配 `/` 字符，参见[「匹配规则」](#matching-rules)；\n2. 尝试将 `a1/b1/c1.txt` 与 `--inlude 'c1.txt'` 匹配，此时根据匹配规则将会匹配成功。后续的 `--exclude 'c*.txt'` 虽然根据匹配规则也能匹配上，但是根据完整路径过滤模式的逻辑，一旦匹配上某个模式，后续的模式将不再尝试匹配。所以最终的匹配结果是「同步」。\n\n以下是更多示例：\n\n+ `--exclude '/foo**'` 将排除所有根目录名为 `foo` 的文件或目录；\n+ `--exclude '**foo/**'` 将排除所有以 `foo` 结尾的目录；\n+ `--include '*/' --include '*.c' --exclude '*'` 将只包含所有目录和后缀名为 `.c` 的文件，除此之外的所有文件和目录都会被排除；\n+ `--include 'foo/bar.c' --exclude '*'` 将只包含 `foo` 目录和 `foo/bar.c` 文件。\n\n### 逐层过滤模式 {#layer-by-layer-filtering-mode}\n\n逐层过滤模式的核心是先将完整路径按照目录层级拆分，并逐层组合成多个字符串序列。比如完整路径为 `a1/b1/c1.txt`，组成的序列就是 `a1`、`a1/b1`、`a1/b1/c1.txt`。然后将这个序列中的每个元素都当成完整路径过滤模式中的路径，依次执行[「完整路径过滤」](#full-path-filtering-mode)。\n\n如果某个元素匹配上了某个模式，则会有两种处理逻辑：\n\n- 如果该模式是 exclude 模式，则直接返回「排除」行为，作为最终的匹配结果；\n- 如果该模式是 include 模式，则跳过本层级的后续待匹配的模式，直接进入下一层级。\n\n如果某层的所有模式都未匹配，则进入下一层级。**如果所有层级匹配完毕后都没有返回「排除」，则返回默认的行为——即「同步」。**\n\n下面是逐层过滤模式的工作流程图：\n\n![逐层过滤模式流程图](../images/sync-layer-by-layer-filtering-mode-flow-chart.svg)\n\n例如有一个路径为 `a1/b1/c1.txt` 的文件，以及 3 个匹配模式 `--include 'a*.txt' --inlude 'c1.txt' --exclude 'c*.txt'`。在逐层过滤模式中，组成的序列就是 `a1`、`a1/b1`、`a1/b1/c1.txt`。具体匹配步骤为：\n\n1. 第一层级的路径为 `a1`，根据匹配模式，结果是全部未匹配。进入下一层级；\n2. 第二层级的路径为 `a1/b1`，根据匹配模式，结果是全部未匹配。进入下一层级；\n3. 第三层级的路径为 `a1/b1/c1.txt`，根据匹配模式，将会匹配上 `--inlude 'c1.txt'` 模式。该模式的行为是「同步」，进入下一层级；\n4. 由于没有下一层级了，所以最终返回的行为是「同步」。\n\n上面的例子是到最后一层才匹配成功，除此之外可能还有两种情况：\n\n- 在最后一层之前匹配成功，且匹配模式是 exclude 模式，则直接返回「排除」行为作为最终结果，跳过后续的所有层级；\n- 所有层级都已经匹配完毕，但都未匹配上，此时也将会返回「同步」行为。\n\n如果你已经熟悉上一小节的“完整路径过滤模式”，那么逐层过滤其实就是按路径层级由高到低依次执行完整路径过滤，每层过滤只有两种结果：要么直接得到「排除」作为最终结果，要么进入下一层级。得到「同步」结果的唯一方式就是执行完所有过滤层级。\n\n以下是更多示例：\n\n+ `--exclude /foo` 将排除所有根目录名为 `foo` 的文件或目录；\n+ `--exclude foo/` 将排除所有名为 `foo` 的目录；\n+ 对于 `dir_name/.../.../...` 这种多级目录来说，将按照目录层级匹配 `dir_name` 下的所有路径。如果某个文件的父目录被「排除」了，那即使加上了这个文件的 include 规则，也不会同步这个文件。如果想要同步这个文件就必须保证它的「所有父目录」都不要被排除。例如，下面的例子中 `/some/path/this-file-will-not-be-synced` 文件将不会被同步，因为它的父目录 `some` 已经被规则 `--exclude '*'` 所排除：\n\n ```shell\n --include '/some/path/this-file-will-not-be-synced' \\\n --exclude '*'\n ```\n\n 一种解决方式是包含目录层级中的所有目录，也就是使用 `--include '*/'` 规则（需放在 `--exclude '*'` 规则的前面）；另一种解决方式是为所有父目录增加 include 规则，例如：\n\n ```shell\n --include '/some/' \\\n --include '/some/path/' \\\n --include '/some/path/this-file-will-be-synced' \\\n --exclude '*'\n ```\n\n## 存储协议 {#storage-protocols}\n\n凡是 JuiceFS 支持的[存储系统](../reference/how_to_set_up_object_storage.md)，都可以使用 sync 命令来同步数据。特别一提，如果其中一端是 JuiceFS 文件系统，那么建议优先使用[无挂载点同步](#sync-without-mount-point)方式。\n\n### 无挂载点同步 1.1 {#sync-without-mount-point}\n\n在两个存储系统之间同步数据，如果其中一方是 JuiceFS，推荐直接使用 `jfs://` 协议头，而不是先挂载 JuiceFS，再访问本地目录。这样便能跳过挂载点，直接读取或写入数据，在大规模场景下，绕过 FUSE 挂载点将能节约资源开销以及提升数据同步性能。\n\n```shell\nmyfs=redis://10.10.0.8:6379/1 juicefs sync s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/movies/ jfs://myfs/movies/\n```\n\n### 对象存储与 JuiceFS 之间同步 {#synchronize-between-object-storage-and-juicefs}\n\n将对象存储的 `movies` 目录同步到 JuiceFS 文件系统：\n\n```shell\n# 挂载 JuiceFS\njuicefs mount -d redis://10.10.0.8:6379/1 /mnt/jfs\n# 执行同步\njuicefs sync s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/movies/ /mnt/jfs/movies/\n```\n\n将 JuiceFS 文件系统的 `images` 目录同步到对象存储：\n\n```shell\n# 挂载 JuiceFS\njuicefs mount -d redis://10.10.0.8:6379/1 /mnt/jfs\n# 执行同步\njuicefs sync /mnt/jfs/images/ s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/images/\n```\n\n### 对象存储与对象存储之间同步 {#synchronize-between-object-storages}\n\n将对象存储的全部数据同步到另一个对象存储桶：\n\n```shell\njuicefs sync s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com oss://ABCDEFG:HIJKLMN@bbb.oss-cn-hangzhou.aliyuncs.com\n```\n\n### 本地及服务器之间同步 {#synchronize-between-local-and-remote-servers}\n\n对于本地计算机上的目录之间拷贝文件，直接指定数据源与目标端的路径即可，比如将 `/media/` 目录同步到 `/backup/` 目录：\n\n```shell\njuicefs sync /media/ /backup/\n```\n\n如果需要在服务器之间同步，可以通过 SFTP/SSH 协议访问目标服务器，例如，将本地的 `/media/` 目录同步到另一台服务器的 `/backup/` 目录：\n\n```shell\njuicefs sync /media/ username@192.168.1.100:/backup/\n# 指定密码（可选）\njuicefs sync /media/ \"username:password\"@192.168.1.100:/backup/\n```\n\n当使用 SFTP/SSH 协议时，如果没有指定密码，执行 sync 任务时会提示输入密码。如果希望显式指定用户名和密码，则需要用半角引号把用户名和密码括起来，用户名和密码之间用半角冒号分隔。\n\n## 同步行为\n\n### 增量同步与全量同步 {#incremental-and-full-synchronization}\n\n`juicefs sync` 默认以增量同步方式工作，对于已存在的文件，仅在文件大小不一样时，才再次同步进行覆盖。在此基础上，还可以指定 [`--update`](../reference/command_reference.mdx#sync)，在源文件 `mtime` 更新时进行覆盖。如果你的场景对正确性有着极致要求，可以指定 [`--check-new`](../reference/command_reference.mdx#sync) 或 [`--check-all`](../reference/command_reference.mdx#sync)，来对两边的文件进行字节流比对，确保数据一致。\n\n如需全量同步，即不论目标路径上是否存在相同的文件都重新同步，可以使用 `--force-update` 或 `-f`。例如，将对象存储的 `movies` 目录全量同步到 JuiceFS 文件系统：\n\n```shell\n# 挂载 JuiceFS\njuicefs mount -d redis://10.10.0.8:6379/1 /mnt/jfs\n# 执行全量同步\njuicefs sync --force-update s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/movies/ /mnt/jfs/movies/\n```\n\n### 目录结构与文件权限 {#directory-structure-and-file-permissions}\n\n默认情况下，sync 命令只同步文件对象以及包含文件对象的目录，空目录不会被同步。如需同步空目录，可以使用 `--dirs` 选项。\n\n另外，在 local、SFTP、HDFS 等文件系统之间同步时，如需保持文件权限，可以使用 `--perms` 选项。\n\n### 拷贝符号链接 {#copy-symbolic-links}\n\nJuiceFS `sync` 在**本地目录之间**同步时，支持通过设置 `--links` 选项开启遇到符号链时同步其自身而不是其指向的对象的功能。同步后的符号链接指向的路径为源符号链接中存储的原始路径，无论该路径在同步前后是否可达都不会被转换。\n\n另外需要注意的几个细节\n\n1. 符号链接自身的 `mtime` 不会被拷贝；\n1. `--check-new` 和 `--perms` 选项的行为在遇到符号链接时会被忽略。\n\n### 数据同步与碎片合并 {#sync-and-compaction}\n\n对于顺序写场景，一定要尽力保证每个文件的写入都有最少 4M（默认块大小）的缓冲区可用，如果写并发太高，或者缓冲区设置太小，都会导致原本高效的“大块写”退化为“碎片化缓慢写”。叠加上 JuiceFS 的碎片合并，可能会带来严重的写放大问题。\n\n碎片合并情况可以通过 `juicefs_compact_size_histogram_bytes` 这个指标来观测。如果在 `sync` 期间碎片合并流量很高，说明需要进行相关调优。推荐实践和调优思路如下：\n\n* 如果对象存储的写带宽不足，慎用高并发（`--threads`），最好从默认值甚至更低的并发开始测起，谨慎增加到满意的速度；\n* 如果目的地是 JuiceFS 文件系统，确保该文件系统的 JuiceFS 客户端有着充足的[读写缓冲区](./cache.md#buffer-size)，按照每个文件的写入都必须起码预留 4M 的写入空间，那么 `--buffer-size` 起码要大于等于 `--threads` 参数的 4 倍，如果希望进一步提高写入并发，那么建议使用 8 或 12 倍的并发量来设置缓冲区。特别注意，根据写入目的地使用的协议头不同，设置缓冲区的方法也不同：\n * 目的地是 `jfs://` 协议头的文件系统，客户端进程就是 `juicefs sync` 命令本身，此时 `--buffer-size` 参数需要追加到 `juicefs sync` 命令里；\n * 目的地是本地的 FUSE 挂载点，那么客户端进程是宿主机上运行的 `juicefs mount` 命令，此时 `--buffer-size` 参数追加到该挂载点的 `juicefs mount` 命令里。\n* 如果需要施加限速，那么加上了 `--bwlimit` 参数后，需要降低 `--threads`，避免过高的并发争抢带宽，产生类似的碎片化问题。每个对象存储的延迟和吞吐不尽相同，再次无法给出细致的调优计算流程，建议从更低的并发开始重新测试。\n\n### 删除特定文件\n\n模式匹配还可以实现删除存储系统中特定文件。诀窍是在本地创建一个空目录，将其作为 `SRC`。\n\n示范如下，谨慎起见，所有示范均添加了 `--dry --debug` 选项来空运行，不会实际删除任何文件，而是打印执行计划。验证成功后，去掉这两个选项便能实际执行。\n\n```shell\nmkdir empty-dir\n# 删除 mybucket 中所有对象，但保留后缀名为 .gz 的文件\njuicefs sync ./empty-dir/ s3://mybucket.s3.us-east-2.amazonaws.com/ --match-full-path --delete-dst --exclude='**.gz' --include='*' --dry --debug\n# 删除 mybucket 中所有后缀名为 .gz 的文件\njuicefs sync ./empty-dir/ s3://mybucket.s3.us-east-2.amazonaws.com/ --match-full-path --delete-dst --include='**.gz' --exclude='*' --dry --debug\n```\n\n## 加速同步 {#accelerate-sync}\n\n`juicefs sync` 默认启用 10 个线程执行同步任务，可以根据需要设置 `--threads` 选项调大或减少线程数。但也要注意，受限于有限的单机资源，一味增加 `--threads` 未必能持续提升同步速度，反而可能会导致 OOM。因此如果同步速度不足，还需要考虑：\n\n* `SRC` 和 `DST` 的存储系统是否已经达到了带宽上限，如果其中一个存储已经到达带宽限制，同步的瓶颈就在这里，增加并发度也不会继续提升同步速度；\n* 单机资源是否吃紧，比如 CPU、网卡拥堵。如果同步受限于单机资源，那么可以考虑：\n * 如果运行环境有硬件条件更好的节点（CPU、网络出口带宽等），可以换用该节点来运行 `juicefs sync`，通过 SSH 访问源数据，例如 `juicefs sync root@src:/data /jfs/data`；\n * 使用[分布式同步](#distributed-sync)，在下方相关小节介绍。\n* 如果同步的数据以小文件为主，并且 `SRC` 的存储系统的 `list` API 性能极佳，那么 `juicefs sync` 默认的单线程 `list` 可能会成为瓶颈。此时考虑启用[并发 `list`](#concurrent-list) 操作，在下一小节介绍。\n\n### 并发 `list` {#concurrent-list}\n\n在 `juicefs sync` 命令的输出中，关注 `Pending objects` 的数量，如果该值持续为 0，说明消费速度大于生产，可以增大 `--list-threads` 来启用并发 `list`，以及用 `--list-depth` 来控制并发 `list` 的目录深度。\n\n比方说，如果你面对的是 JuiceFS 所使用的对象存储服务，那么目录结构为 `//chunks/xxx/xxx/...`，对于这样的目录结构，使用 `--list-depth=2` 来实现对于 `//chunks` 的并发列表操作，是比较合适的选择。\n\n### 分布式同步 {#distributed-sync}\n\n在两个对象存储之间同步数据，就是从一端拉取数据再推送到另一端，同步的效率取决于客户端与云之间的带宽：\n\n![JuiceFS-sync-single](../images/juicefs-sync-single.png)\n\n在同步大量数据时，单机带宽往往会被占满出现瓶颈，针对这种情况，考虑使用多机并发同步：\n\n![JuiceFS-sync-worker](../images/juicefs-sync-worker.png)\n\nManager 作为主控执行 `sync` 命令，通过 `--worker` 参数定义多个 Worker 节点（Manager 自身也参与同步），JuiceFS 会根据 Worker 的总数量，动态拆分同步任务并分发给各个节点并发执行，单位时间内能处理的数据量更大，总带宽也成倍增加。\n\n在配置多机并发同步任务时，需要提前配置好 Manager 节点到 Worker 节点的 SSH 免密登录，如果 Worker 节点的 SSH 端口不是默认的 22，请在 Manager 节点的 `~/.ssh/config` 设置其端口号。Manager 会将 JuiceFS 客户端程序分发到 Worker 节点，为避免兼容性问题，Manager 和 Worker 应使用相同类型和架构的操作系统。\n\n举例说明，用分布式同步的方式进行对象存储间的数据同步：\n\n```shell\njuicefs sync --worker bob@192.168.1.21,tom@192.168.1.22 s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com oss://ABCDEFG:HIJKLMN@bbb.oss-cn-hangzhou.aliyuncs.com\n```\n\n运行这个命令后，当前节点与两个 Worker 节点 `bob@192.168.1.21` 和 `tom@192.168.1.22` 将共同分担数据同步任务。\n\n上方的示范中是对象存储 → 对象存储的数据同步，如果需要基于 FUSE 挂载点做数据同步，那么可以在所有节点挂载 JuiceFS，然后用类似下方的命令来进行分布式同步：\n\n```shell\n# 源文件系统需要更好的读性能，因此增大 buffer-size\nparallel-ssh -h hosts.txt -i juicefs mount -d redis://10.10.0.8:6379/1 /jfs-src --buffer-size=1024 --cache-size=0\n\n# 目标文件系统需要更好的写性能\nparallel-ssh -h hosts.txt -i juicefs mount -d redis://10.10.0.8:6379/1 /jfs-dst --buffer-size=1024 --cache-size=0 --max-uploads=50\n\n# 挂载完毕后，用下方命令拷贝数据\njuicefs sync --worker host1,host2 /jfs-src /jfs-dst\n```\n\n## 观测和监控 {#observation}\n\n简单来说，用 `sync` 命令拷贝大文件时，进度条可能会迟迟不更新，如果担心命令未能正常工作，可以用其他手段对传输情况进行观测。\n\n`sync` 假定了使用场景是拷贝大量文件，因此进度的计算也是针对多个文件设计的：每一个文件完成了拷贝后，进度会更新一次。因此如果面对的都是大文件，单个文件的拷贝速度太慢，进度条就会变化缓慢，或者呈现卡死的状态。如果目的地端协议不支持 multipart upload（比如 `file`、`sftp`、`jfs`、`gluster` 协议头），单个文件会单线程进行复制，无法对大文件进行并发上传。可想而知，大文件 + 不支持 multipart upload，将会更容易出现进度条卡死的情况。\n\n如果你观察到进度不再变化，参考下列手段进行观测和排查：\n\n* 为 `juicefs sync` 添加 [`--verbose` 或 `--debug`](../reference/command_reference.mdx#global-options) 参数，打印 debug 日志；\n* 如果数据同步的两方有任何一个是 JuiceFS 宿主机挂载点：\n * 用 [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) 快速查看文件系统是否正在写入（或读出）；\n * 阅读[客户端日志](../administration/fault_diagnosis_and_analysis.md#client-log)（默认 `/var/log/juicefs.log`），观察是否有[慢请求或者超时错误日志](../administration/troubleshooting.md#io-error-object-storage)。\n* 如果数据同步的目的地是宿主机本地盘，可以直接观察目录下是否存在名称中带 `.jfs.xxx.tmp.xxx` 后缀的临时文件，`sync` 过程中会将传输结果写入临时文件，待传输完成后进行重命名，才完成最终的写入。观察临时文件大小是否变化，就能确定当前的写入状况；\n* 如果传输目的地均为对象存储，可以通过类似 `nethogs` 的命令，查看出入网流量，来判断传输进展；\n* 以上手段均未能获得有效排查信息，则需要对 `sync` 进程采集 goroutine，结合源码分析排查：\n\n ```shell\n # 将 替换为卡死的 sync 命令的 PID，记下 pprof 监听端口\n lsof -p | grep TCP | grep LISTEN\n # pprof 端口一般是 6061，如果已经被占用，则需要递增，需要根据实际情况修改\n curl -s localhost:6061/debug/pprof/goroutine?debug=1\n ```\n\n如果需要监控 `sync` 命令的进度，可以使用 [`--metrics`](../reference/command_reference.mdx#sync-metrics-related-options) 参数指定监控指标地址，默认为 `127.0.0.1:9567`。用 Prometheus 抓取这些指标，就能进行监控。\n\n## 场景应用 {#application-scenarios}\n\n### 数据异地容灾备份 {#geo-disaster-recovery-backup}\n\n异地容灾备份针对的是文件本身，因此应将 JuiceFS 中存储的文件同步到其他的对象存储，例如，将 JuiceFS 文件系统中的文件同步到对象存储：\n\n```shell\n# 挂载 JuiceFS\njuicefs mount -d redis://10.10.0.8:6379/1 /mnt/jfs\n# 执行同步\njuicefs sync /mnt/jfs/ s3://ABCDEFG:HIJKLMN@aaa.s3.us-west-1.amazonaws.com/\n```\n\n### 建立 JuiceFS 数据副本 {#build-a-juicefs-data-copy}\n\n与面向文件本身的容灾备份不同，建立 JuiceFS 数据副本的目的是为 JuiceFS 的数据存储建立一个内容和结构完全相同的镜像，当使用中的对象存储发生了故障，可以通过修改配置切换到数据副本继续工作。需要注意这里仅复制了 JuiceFS 文件系统的数据，并没有复制元数据，元数据引擎的数据备份依然需要。\n\n这需要直接操作 JuiceFS 底层的对象存储，将它与目标对象存储之间进行同步。例如，要把对象存储作为 JuiceFS 文件系统的数据副本：\n\n```shell\njuicefs sync cos://ABCDEFG:HIJKLMN@ccc-125000.cos.ap-beijing.myqcloud.com oss://ABCDEFG:HIJKLMN@bbb.oss-cn-hangzhou.aliyuncs.com\n```\n\n### 使用 S3 网关进行跨区域数据同步 {#sync-across-region}\n\n通过 POSIX 方式访问 JuiceFS 时，会有频繁的元数据访问，跨区域访问元数据的延迟比较高会影响访问性能。如果需要跨区域传输大量小文件，这时元数据服务延迟高对性能影响更严重。\n\n![sync via public metadata service](../images/sync-public-metadata.svg)\n\n在这种情况下，可以通过跨区访问部署在源区域的 S3 网关来提升性能，它可以大幅减少跨区域访问的请求数。\n\n![sync via gateway](../images/sync-via-gateway.svg)\n\n阅读[「S3 网关」](./gateway.md)学习如何使用和部署 S3 网关。\n" }, { "path": "docs/zh_cn/introduction/README.md", "content": "---\ntitle: JuiceFS 简介\nsidebar_position: 1\nslug: .\npagination_next: introduction/architecture\n---\n\n**JuiceFS** 是一款面向云原生设计的高性能分布式文件系统，在 Apache 2.0 开源协议下发布。提供完备的 [POSIX](https://en.wikipedia.org/wiki/POSIX) 兼容性，可将几乎所有对象存储接入本地作为海量本地磁盘使用，亦可同时在跨平台、跨地区的不同主机上挂载读写。\n\nJuiceFS 采用「数据」与「元数据」分离存储的架构，从而实现文件系统的分布式设计。文件数据本身会被切分保存在[对象存储](../reference/how_to_set_up_object_storage.md#supported-object-storage)（例如 Amazon S3），而元数据则可以保存在 Redis、MySQL、TiKV、SQLite 等多种[数据库](../reference/how_to_set_up_metadata_engine.md)中，你可以根据场景与性能要求进行选择。\n\nJuiceFS 提供了丰富的 API，适用于各种形式数据的管理、分析、归档、备份，可以在不修改代码的前提下无缝对接大数据、机器学习、人工智能等应用平台，为其提供海量、弹性、低价的高性能存储。运维人员不用再为可用性、灾难恢复、监控、扩容等工作烦恼，专注于业务开发，提升研发效率。同时运维细节的简化，对 DevOps 极其友好。\n\n
\n \n
\n\n## 核心特性 {#features}\n\n1. **POSIX 兼容**：像本地文件系统一样使用，无缝对接已有应用，无业务侵入性；\n2. **HDFS 兼容**：完整兼容 [HDFS API](../deployment/hadoop_java_sdk.md)，提供更强的元数据性能；\n3. **S3 兼容**：提供 [S3 网关](../guide/gateway.md) 实现 S3 协议兼容的访问接口；\n4. **云原生**：通过 [Kubernetes CSI 驱动](../deployment/how_to_use_on_kubernetes.md) 轻松地在 Kubernetes 中使用 JuiceFS；\n5. **分布式设计**：同一文件系统可在上千台服务器同时挂载，高性能并发读写，共享数据；\n6. **强一致性**：确认的文件修改会在所有服务器上立即可见，保证强一致性；\n7. **强悍性能**：毫秒级延迟，近乎无限的吞吐量（取决于对象存储规模），查看[性能测试结果](../benchmark/benchmark.md)；\n8. **数据安全**：支持传输中加密（encryption in transit）和静态加密（encryption at rest），[查看详情](../security/encryption.md)；\n9. **文件锁**：支持 BSD 锁（flock）和 POSIX 锁（fcntl）；\n10. **数据压缩**：支持 [LZ4](https://lz4.github.io/lz4) 和 [Zstandard](https://facebook.github.io/zstd) 压缩算法，节省存储空间。\n\n## 应用场景 {#scenarios}\n\nJuiceFS 为海量数据存储设计，可以作为很多分布式文件系统和网络文件系统的替代，特别是以下场景：\n\n- **大数据分析**：HDFS 兼容；与主流计算引擎（Spark、Presto、Hive 等）无缝衔接；无限扩展的存储空间；运维成本几乎为 0；性能远好于直接对接对象存储。\n- **机器学习**：POSIX 兼容，可以支持所有机器学习、深度学习框架；方便的文件共享还能提升团队管理、使用数据效率。\n- **Kubernetes**：JuiceFS 支持 Kubernetes CSI；为容器提供解耦的文件存储，令应用服务可以无状态化；方便地在容器间共享数据。\n- **共享工作区**：可以在任意主机挂载；没有客户端并发读写限制；POSIX 兼容已有的数据流和脚本操作。\n- **数据备份**：在无限平滑扩展的存储空间备份各种数据，结合共享挂载功能，可以将多主机数据汇总至一处，做统一备份。\n\n## 数据隐私 {#data-privacy}\n\nJuiceFS 是开源软件，你可以在 [GitHub](https://github.com/juicedata/juicefs) 找到完整的源代码。在使用 JuiceFS 存储数据时，数据会按照一定的规则被拆分成数据块并保存在你自己定义的对象存储或其它存储介质中，数据所对应的元数据则存储在你自己定义的数据库中。\n\n## 更多相关信息 {#more-info}\n\n* **案例**：想了解更多相似场景的实践案例，请访问[用户案例](https://juicefs.com/zh-cn/blog/user-stories)。\n* **视频**：我们在 [Bilibili 频道](https://space.bilibili.com/1206844881)提供了丰富的视频教程。\n* **加入社群**：欢迎加入我们的[微信用户组](https://juicefs.com/zh-cn/wechat-user-group)（中文）或者 [Slack](https://go.juicefs.com/slack)（英文），与 JuiceFS 用户共同探讨。\n* **Office Hours**：每月第 2 周的星期三 16:00-17:00（UTC+8）在线上举行，Juicedata 工程师将为你实时答疑解惑。请加入微信用户组获取最新活动信息。\n* **AI 助手**：如果你遇到了任何问题，欢迎使用「Ask AI」功能（右下角）求助 AI 助手。AI 助手的知识库来源于文档以及 GitHub 中的相关内容。\n" }, { "path": "docs/zh_cn/introduction/architecture.md", "content": "---\ntitle: 技术架构\nsidebar_position: 2\nslug: /architecture\ndescription: 本文介绍 JuiceFS 的技术架构以及由此带来的技术优势，同时介绍 JuiceFS 的文件存储原理。\n---\n\nJuiceFS 文件系统由三个部分组成：\n\n![JuiceFS-arch](../images/juicefs-arch.svg)\n\n**JuiceFS 客户端（Client）**：所有文件读写，以及碎片合并、回收站文件过期删除等后台任务，均在客户端中发生。客户端需要同时与对象存储和元数据引擎打交道。客户端支持多种接入方式：\n\n- 通过 **FUSE**，JuiceFS 文件系统能够以 POSIX 兼容的方式挂载到服务器，将海量云端存储直接当做本地存储来使用。点击[此处](https://juicefs.com/docs/zh/community/getting-started/installation)查看使用详情。\n- 通过 **Python SDK**，在无法通过 FUSE 挂载，或需要在 Python 进程中直接访问文件系统的场景，可以使用 Python SDK 直接读写文件系统。此外，Python SDK 原生实现了 fsspec 便于接入 Ray 等框架。点击[此处](https://juicefs.com/docs/zh/community/deployment/python_sdk)查看使用详情。\n- 通过 **Windows 客户端**，获得接近本地的文件系统体验。点击[此处](https://juicefs.com/docs/zh/community/tutorials/windows)查看使用详情。\n- 通过 **Hadoop Java SDK**，JuiceFS 文件系统能够直接替代 HDFS，为 Hadoop 提供低成本的海量存储。点击[此处](https://juicefs.com/docs/zh/community/hadoop_java_sdk)查看使用细节。\n- 通过 **Kubernetes CSI 驱动**，JuiceFS 文件系统能够直接为 Kubernetes 提供海量存储。点击[此处](https://juicefs.com/docs/zh/csi/introduction)查看 JuiceFS CSI 文档。\n- 通过 **S3 网关**，使用 S3 作为存储层的应用可直接接入，同时可使用 AWS CLI、s3cmd、MinIO client 等工具访问 JuiceFS 文件系统。点击[此处](https://juicefs.com/docs/zh/community/guide/gateway)查看使用详情。\n- 通过 **WebDAV 服务**，以 HTTP 协议，以类似 RESTful API 的方式接入 JuiceFS 并直接操作其中的文件。\n\n**数据存储（Data Storage）**：文件将会被切分上传至对象存储服务。JuiceFS 支持几乎所有的公有云对象存储，同时也支持 OpenStack Swift、Ceph、MinIO 等私有化的对象存储。\n\n**元数据引擎（Metadata Engine）**：用于存储文件元数据（metadata），包含以下内容：\n\n- 常规文件系统的元数据：文件名、文件大小、权限信息、创建修改时间、目录结构、文件属性、符号链接、文件锁等。\n- 文件数据的索引：文件的数据分配和引用计数、客户端会话等。\n\nJuiceFS 采用多引擎设计，目前已支持 Redis、TiKV、MySQL/MariaDB、PostgreSQL、SQLite 等作为元数据服务引擎，也将陆续实现更多元数据存储引擎。欢迎[提交 Issue](https://github.com/juicedata/juicefs/issues) 反馈你的需求。\n\n## JuiceFS 如何存储文件 {#how-juicefs-store-files}\n\n与传统文件系统只能使用本地磁盘存储数据和对应的元数据的模式不同，JuiceFS 会将数据格式化以后存储在对象存储，同时会将文件的元数据存储在元数据引擎。在这个过程中，Chunk、Slice、Block 是三个重要的概念：\n\n对于 JuiceFS，每一个文件都由 1 或多个「Chunk」组成，每个 Chunk 最大 64M。不论文件有多大，所有的读写都会根据其偏移量（也就是产生读写操作的文件位置）来定位到对应的 Chunk。正是这种分而治之的设计，让 JuiceFS 面对大文件也有优秀的性能。只要文件总长度没有变化，不论经历多少修改写入，文件的 Chunk 切分都是固定的。\n\n![file-and-chunks](../images/file-and-chunks.svg)\n\nChunk 的存在是为了优化查找定位，实际的文件写入则在「Slice」上进行。在 JuiceFS 中，一个 Slice 代表一次连续写入，隶属于某个 Chunk，并且不能跨越 Chunk 边界，因此 Slice 长度也不会超 64M。\n\n举例说明，如果一个文件是由一次连贯的顺序写生成，那么每个 Chunk 中只将会仅包含一个 Slice。上方的示意图就属于这种情况：顺序写入一个 160M 文件，最终会产生 3 个 Chunk，而每个 Chunk 仅包含一个 Slice。\n\n文件写入会产生 Slice，而调用 `flush` 则会将这些 Slice 持久化。`flush` 可以被用户显式调用，就算不调用，JuiceFS 客户端也会自动在恰当的时机进行 `flush`，防止[缓冲区](../guide/cache.md#buffer-size)被写满。持久化到对象存储时，为了能够尽快写入，会对 Slice 进行进一步拆分成一个个「Block」（默认最大 4M），多线程并发写入以提升写性能。上边介绍的 Chunk、Slice，其实都是逻辑数据结构，Block 则是最终的物理存储形式，是对象存储和磁盘缓存的最小存储单元。\n\n![slice-to-block](../images/slice-to-block.svg)\n\n因此，文件写入 JuiceFS 后，你不会在对象存储中找到原始文件，存储桶中只有一个 `chunks` 目录和一堆数字编号的目录和文件，让人不禁疑惑「我的文件到底去了哪儿」？但事实上，这些数字编号的对象存储文件正是经过 JuiceFS 拆分存储的 Block，而这些 Block 与 Chunk、Slice 的对应关系，以及其他元数据信息（比如文件名、大小等属性）则存储在元数据引擎中，这样的分离设计，让 JuiceFS 文件系统得以高性能运作。\n\n![how-JuiceFS-stores-files](../images/how-juicefs-stores-files.svg)\n\n回到逻辑数据结构的话题，如果文件并不是由连贯的顺序写生成，而是多次追加写，每次追加均调用 `flush` 触发写入上传，就会产生多个 Slice。如果每次追加写入的数据量不足 4M，那么最终存入对象存储的数据块，也会是一个个小于 4M 的 Block。\n\n![small-append](../images/small-append.svg)\n\n取决于写入模式，Slice 的排列模式可以是多种多样的：如果文件在相同区域被反复修改，Slice 之间会发生重叠。如果在互不重合的区域进行写入，Slice 中间会有间隔。但不论 Slice 的排列有多复杂，当读文件发生时，对于每一处文件位置，都会读到该位置最新写入的 Slice，用下图可以更加直观地理解：Slice 虽然会相互堆叠，但读文件一定是“从上往下看”，因此一定会看到该文件的最新状态。\n\n![complicate-pattern](../images/complicate-pattern.svg)\n\n正是由于 Slice 会相互覆盖，JuiceFS 在 Chunk 与 Slice 的引用关系中，[标记了各个 Slice 的有效数据偏移范围](../development/internals.md#sliceref)，用这种方式告诉文件系统，每一个 Slice 中的哪些部分是有效的数据。\n\n但也不难想象，读取文件需要查找「当前读取范围内最新写入的 Slice」，在上图所示的大量堆叠 Slice 的情况下，这样的反复查找将会显著影响读性能，我们称之为文件「碎片化」。碎片化不仅影响读性能，还会在各个层面（对象存储、元数据）增加空间占用。因此每当写入发生时，客户端都会判断文件的碎片化情况，并异步地运行碎片合并，将同一个 Chunk 内的所有 Slice 合并为一。\n\n![compaction](../images/compaction.svg)\n\n最后，JuiceFS 的存储设计，还有着以下值得一提的技术特点：\n\n* 对于任意大小的文件，JuiceFS 都不进行合并存储，这也是为了性能考虑，避免读放大。\n* 提供强一致性保证，但也可以根据场景需要与缓存功能一起调优，比如通过设置出更激进的元数据缓存，牺牲一部分一致性，换取更好的性能。详见[「元数据缓存」](../guide/cache.md#metadata-cache)。\n* 支持并默认开启[「回收站」](../security/trash.md)功能，删除文件后保留一段时间才彻底清理，最大程度避免误删文件导致事故。\n" }, { "path": "docs/zh_cn/introduction/comparison/_category_.yml", "content": "position: 4\nlabel: \"Comparing with Others\"\n# collapsible: true \n# collapsed: true " }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_3fs.md", "content": "---\nslug: /comparison/juicefs_vs_3fs\ndescription: 本文对比了 DeepSeek 3FS 和 JuiceFS 在 AI 存储场景中的架构、特性和创新技术。\n---\n\n# JuiceFS 对比 3FS\n\n3FS (Fire-Flyer File System) 是一款分布式文件系统，针对 AI 训练和推理工作负载设计，由 DeepSeek 开源。该系统使用 NVMe SSD 和 RDMA 网络提供共享存储层，面向大规模 AI 应用的 I/O 需求。\n\nJuiceFS 是一个云原生分布式文件系统，其数据存储在对象存储中。社区版可与多种元数据服务集成，适用场景广泛，于 2021 年在 GitHub 开源。企业版专为高性能场景设计，广泛应用于大规模 AI 任务，涵盖生成式 AI、自动驾驶、量化金融和生物科技等。\n\n本文从架构设计、文件分布、RPC 框架和功能特性等方面对 3FS 和 JuiceFS 进行全面对比。\n\n## 架构对比\n\n### 3FS\n\n3FS 采用针对 AI 工作负载设计的架构，包含以下关键组件：\n\n- **集群管理服务（Cluster Manager）**：处理成员变更，并将集群配置分发给其他服务和客户端。为了提高系统可靠性和避免单点故障，会部署多个集群管理服务，其中一个被选为主节点。\n- **元数据服务（Metadata Service）**：无状态服务，处理文件元数据操作，依靠支持事务的键值数据库 FoundationDB 来存储元数据。\n- **存储服务（Storage Service）**：使用本地 NVMe SSD 管理数据存储，采用 CRAQ（Chain Replication with Apportioned Queries）算法确保数据一致性。\n- **客户端（Clients）**：提供 FUSE Client 以实现 POSIX 兼容性，以及 Native Client API 用于高性能零拷贝操作。\n\n所有组件通过 RDMA 进行高性能网络通信。集群配置通常存储在可靠的分布式服务中，例如 ZooKeeper 或 etcd。\n\n![3FS architecture](https://static1.juicefs.com/images/3FS_JiaGou.original.png)\n\n### JuiceFS\n\nJuiceFS 采用模块化的云原生架构，包含三个核心组件：\n\n- **元数据引擎**：用于存储文件元数据，包括常规文件系统的元数据和文件数据的索引。社区版支持 Redis、TiKV、MySQL、PostgreSQL、FoundationDB 等多种数据库。企业版使用自研高性能元数据服务。\n- **数据存储**：一般是对象存储服务，可以是公有云的对象存储也可以是私有部署的对象存储服务。支持与各种存储后端集成。\n- **JuiceFS 客户端**：提供 POSIX（FUSE）、Hadoop SDK、CSI Driver、S3 网关等不同的接入方式。\n\n![JuiceFS Community Edition architecture](../../images/juicefs-arch.svg)\n\n### 架构差异\n\n#### 存储模块\n\n3FS 使用本地 NVMe SSD 进行数据存储，为了保证数据存储的一致性，采用 CRAQ（Chain Replication with Apportioned Queries）算法。几个副本被组成一个 Chain，写请求从 Chain 的 Head 开始，一直到达 Chain 的 Tail 时返回写成功应答。读请求可以发送到 Chain 的所有副本，如果读到脏节点的数据，该节点会联系 Tail 节点检查状态。\n\n![CRAQ consistency algorithm](https://static1.juicefs.com/images/CRAQ_YiZhiXingSuanFa.original.png)\n\n数据的写入是按顺序逐节点传递，因此会带来比较高的延时，但这种设计优先考虑读性能，这对于读密集型的 AI 工作负载至关重要。\n\n相比之下，JuiceFS 利用对象存储作为数据存储解决方案，从而可享有对象存储带来的若干优势，如数据可靠性、一致性等。存储模块提供了一组用于对象操作的标准接口（GET/PUT/HEAD/LIST），可以与各种存储后端无缝集成。社区版 JuiceFS 提供本地缓存来应对 AI 场景下的带宽需求，企业版使用分布式缓存满足更大的聚合读带宽的需求。\n\n#### 元数据模块\n\n在 3FS 中，文件的属性以 KV 的形式存储在元数据服务中。该服务是一个无状态的高可用服务，依靠 FoundationDB 做支撑。FoundationDB 所有键值使用 Key 做全局排序，然后均匀拆分到不同的节点上。为了优化 list 目录的效率，3FS 使用字符 \"DENT\" 前缀加父目录 inode 号和名字作为 dentry 的 Key。\n\nJuiceFS 社区版的元数据模块提供一组操作元数据的接口，可以接入不同的元数据服务，比如 Redis、TiKV 等 KV 数据库，MySQL、PostgreSQL 等关系型数据库，也可以使用 FoundationDB。JuiceFS 企业版使用自研高性能元数据服务，可根据负载情况来平衡数据和热点操作，以避免大规模训练中元数据服务热点集中在某些节点的问题。\n\n#### 客户端\n\n3FS 的客户端除了提供 FUSE 操作外，还提供了一组 API 用于绕过 FUSE 直接操作数据，也就是 Native Client。这组 API 的作用是避免使用 FUSE 模块带来的数据拷贝，从而减少 I/O 延迟和对内存带宽的占用，通过共享内存和信号量实现零拷贝通信。\n\n![3FS native client API](https://static1.juicefs.com/images/3FS_NATIVE_Client_API.original.png)\n\n3FS 通过 `hf3fs_iov` 保存共享内存的大小、地址和其他一些属性，使用 `IoRing` 在两个进程间通信。系统创建虚拟文件并使用信号量来促进用户进程和 FUSE 进程之间的通信。\n\nJuiceFS 的 FUSE 客户端实现更加全面，提供以下功能：\n\n- 在每次成功上传对象后会立即更新文件长度\n- 支持 BSD 锁（flock）和 POSIX 锁（fcntl）\n- 支持高级接口如 `file_copy_range`、`readdirplus` 和 `fallocate`\n\n除了 FUSE 客户端，JuiceFS 社区版还提供 Java SDK、Python SDK、S3 网关、CSI Driver 等用于用户空间执行的功能，企业版在此基础上提供了更多企业级特性。\n\n## 文件分布对比\n\n### 3FS 文件分布\n\n3FS 将每个文件分成固定长度的 chunk，每个 chunk 位于一个链上（CRAQ 算法）。因为 3FS 中的 chunk 是固定的，客户端只需要获取一次 inode 的 chain 信息，就可以根据文件 inode 和 I/O 请求的 offset、length 计算出这个请求位于哪些 chunk 上，从而避免了每个 I/O 都从数据库查询的需求。可以通过 `offset/chunk_size` 得到 chunk 的索引，而 chunk 所在的 chain 的索引就是 `chunk_id%stripe`。\n\n为了应对数据不平衡问题，每个文件的第一个 chain 按照轮询（round robin）的方式选择。创建文件时，系统会将选择的 chain 做随机排序，然后存储到元数据中。\n\n![3FS file distribution](https://static1.juicefs.com/images/3FS_WenJianFenBu.original.png)\n\n### JuiceFS 文件分布\n\nJuiceFS 按照 Chunk、Slice、Block 的规则进行数据块管理。每个 Chunk 的大小固定为 64M，主要用于优化数据的查找和定位。实际的文件写入操作则在 Slice 上执行，Slice 代表块内连续的写入过程。Block（默认大小为 4M）则是物理存储的基本单位，用于在对象存储和磁盘缓存中实现数据的最终存储。\n\n![JuiceFS file distribution](../../images/file-and-chunks.svg)\n\nJuiceFS 中的 Slice 是在其他文件系统中不常见的一个结构。主要功能是记录文件的写入操作，并在对象存储中进行持久化。由于对象存储不支持原地文件修改，JuiceFS 通过引入 Slice 结构允许更新文件内容，而无需重写整个文件。JuiceFS 的所有 Slice 均为一次性写入，这减少了对底层对象存储一致性的依赖，并大大简化了缓存系统的复杂度。\n\n## 3FS RPC 框架\n\n3FS 使用 RDMA 作为底层网络通信协议，目前 JuiceFS 尚未支持。3FS 通过实现一个 RPC 框架，来完成对底层 IB 网络的操作。除了网络操作外，RPC 框架还提供序列化、小包合并等能力，使用模版实现了一个反射库，用于序列化 RPC 使用的 request、response 等数据结构。\n\n![3FS FUSE client RPC process](https://static1.juicefs.com/images/3FS_FUSE_Client_DiaoYong_MetadataFuWuDe_RPC_Guo.original.png)\n\n3FS 的缓存有两部份组成，一个 TLS（Thread-Local Storage）队列和一个全局队列。从 TLS 队列获取缓存时不需要加锁；当 TLS 缓存为空时就得加锁，从全局队列中获取缓存。多个 RPC 请求可能被合并为一个 InfiniBand 请求以提高效率。\n\n## 功能特性对比\n\n| 功能特性 | 3FS | JuiceFS 社区版 | JuiceFS 企业版 |\n|----------|-----|---------------|---------------|\n| 元数据 | 无状态元数据服务+FoundationDB | 独立数据库服务 | 自研高性能分布式元数据引擎（可横向扩展） |\n| 数据存储 | 自主管理 | 使用对象存储 | 使用对象存储 |\n| 冗余保护 | 多副本 | 对象存储提供 | 对象存储提供 |\n| 数据缓存 | 无缓存 | 本地缓存 | 自研高性能多副本分布式缓存 |\n| 数据加密 | 不支持 | 支持 | 支持 |\n| 数据压缩 | 不支持 | 支持 | 支持 |\n| 配额管理 | 不支持 | 支持 | 支持 |\n| 网络协议 | RDMA | TCP | TCP |\n| 快照 | 不支持 | 支持克隆 | 支持克隆 |\n| POSIX ACL | 不支持 | 支持 | 支持 |\n| POSIX 兼容性 | 少量子集 | 完全兼容 | 完全兼容 |\n| CSI 驱动 | 没有官方支持 | 支持 | 支持 |\n| 客户端 | FUSE + Native Client | POSIX（FUSE）、Java SDK、Python SDK、S3 网关 | POSIX（FUSE）、Java SDK、S3 网关、Python SDK |\n| 多云镜像 | 不支持 | 不支持 | 支持 |\n| 跨云和跨区数据复制 | 不支持 | 不支持 | 支持 |\n| 主要维护者 | DeepSeek | Juicedata | Juicedata |\n| 开发语言 | C++, Rust (本地存储引擎) | Go | Go |\n| 开源协议 | MIT | Apache License 2.0 | 商业软件 |\n\n## 总结\n\n大规模 AI 训练中最主要的需求是高读带宽，为此 3FS 采用了性能优先的设计策略：\n\n- **本地存储**：将数据存储在本地 NVMe SSD 上，用户需要自行管理底层数据存储基础设施\n- **零拷贝优化**：实现了客户端到网卡的零拷贝，利用共享内存和信号量减少 I/O 延迟和内存带宽占用\n- **RDMA 网络**：引入了 RDMA 技术，提供更好的网络性能\n- **优化的 I/O**：通过带 TLS 的 I/O buffer pool 和合并网络请求，增强了小 I/O 和文件元数据操作的能力\n\n这种方法提升了性能，但成本较高，维护也更繁重。\n\nJuiceFS 使用对象存储作为底层数据存储，用户因此可大幅降低存储成本并简化维护工作。为了满足 AI 场景的对读性能的需求：\n\n- **企业版功能**：分布式缓存、分布式元数据服务和 Python SDK\n- **即将推出的优化**：v5.2 企业版中，在 TCP 网络中实现了零拷贝，进一步提升数据传输效率\n- **云原生优势**：提供完整的 POSIX 兼容性和成熟活跃的开源生态，支持 Kubernetes CSI\n- **企业级能力**：Quota、安全管理和数据灾备等多项企业级管理功能\n" }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_alluxio.md", "content": "---\nslug: /comparison/juicefs_vs_alluxio\n---\n\n# JuiceFS 对比 Alluxio\n\n用户在存算分离的数据平台和 AI 训练加速场景中经常会比较 Alluxio 和 JuiceFS 两个产品。除了使用场景有相似之处，更多的原因是这两个产品都与对象存储相结合，都提供了访问数据时的缓存加速能力，看上去像是同一个场景中的「替代产品」，但两个产品又有很多的不同。本章详细介绍二者的功能区别，两个系统都是开源项目，但也都各自提供功能更为强大的企业版，本文的对比也会考虑到不同版本的区别，帮助你的团队进行技术选型。\n\nAlluxio 的定位是在现有的存储系统之上提供缓存加速层，在实际项目中存储系统大多是对象存储系统。JuiceFS 的定位是为云环境设计的分布式文件系统，可以通过缓存机制加速数据访问。\n\n从架构设计角度讲，Alluxio 与对象存储是两套系统，Alluxio 是业务应用于对象存储之间的中间件，维护多个节点中的存储空间形成一个缓存系统，存储应用访问过的热数据。\n\nJuiceFS 用对象存储做数据持久层，对象存储可以看做是 JuiceFS 的一个内部组件，打比方说对象存储好像一块容量无限大的硬盘，JuiceFS 对这块「硬盘」进行格式化，JuiceFS 的元数据服务就是分区表，结合在一起形成完整的「文件系统」概念。\n\nAlluxio 和 JuiceFS 虽然都能提供文件系统服务，但架构以及使用场景存在很大差异：Alluxio 的主要作用是为各个数据存储系统提供统一接入平台（你的数据仍存储在外部系统），为应用提供高速缓存层。而 JuiceFS 则是一个分布式高性能文件系统，你可以将其作为大数据存储平台，也可以用他来替换当前的存储系统，为你的业务增效降本。\n\n考虑到二者的定位有很大不同，下方表格只能呈现各自作为文件系统角色时的功能特性，并不是一个「公平的对比」。JuiceFS 并不提供多数据源聚合功能，因此也无法与 Alluxio 进行比较。如果你对两个产品都感兴趣，请继续阅读表格下方的章节。\n\n| 特性 | Alluxio | JuiceFS |\n|:---:|:---:|:---:|\n| 多级缓存 | 支持 | 支持 |\n| Hadoop 兼容 | 支持 | 支持 |\n| S3 兼容 | 支持 | 支持 |\n| Kubernetes CSI 驱动 | 支持 | 支持 |\n| WebDAV 协议 | 不支持 | 支持 |\n| Hadoop 数据本地性 | 支持 | 支持 |\n| 完全兼容 POSIX | 不支持 | 支持 |\n| 一致性 | 不一定 | 强一致性|\n| 数据压缩 | 不支持 | 支持 |\n| 数据加密 | 不支持 | 支持 |\n| 服务端运维 | 复杂 | 推荐直接使用云服务商托管服务，实现零运维 |\n| 开发语言 | Java | Go |\n| 开源协议 | Apache License 2.0 | Apache License 2.0 |\n| 开源时间 | 2014 | 2021.1 |\n\n## 架构与核心特性 {#architecture-and-key-features}\n\n### 存储与缓存 {#storage-and-cache}\n\nAlluxio 自身不是一个存储系统，而是一个强大的聚合层，来为不同的存储系统（比如 HDFS、NFS）提供统一接入和缓存服务。这也是为什么我们无法将存储和缓存拆开来讨论与对比，因为 Alluxio 自己的存储层，作用实际上就是提供缓存服务（更多关于架构信息请阅读其[官方文档](https://docs.alluxio.io/os/user/stable/cn/core-services/Caching.html)）。\n\n在 Alluxio 的架构中，背后的存储系统称作「UFS」（Under File Storage），可想而知，这些存储系统都是外部系统，不受 Alluxio 管辖，他们各自的存储格式与 Alluxio 无关。\n\nUFS 层让 Alluxio 能够聚合不同的文件系统，但 Alluxio 的重要作用是为这些存储系统提供缓存服务，因此 Alluxio 也有自己的数据存储，称作 Alluxio storage，会被部署成 Alluxio workers，用来提供缓存服务。\n\n在 Alluxio 存储层，默认使用 64MB 作为缓存块大小，并且在缓存盘之上优先使用内存，为热数据提供更加高速的缓存服务。新版实验功能中也引入了[可调节缓存块大小的设计](https://docs.alluxio.io/os/user/stable/en/core-services/Caching.html#experimental-paging-worker-storage)，来调节缓存粒度，优化性能。\n\nJuiceFS 是一个分布式文件系统，实现了自己的存储格式，文件会被视作一个个最大 64MB 的逻辑数据块（Chunk），再拆成 4MB 的 Block 上传至对象存储，作为最基本的物理存储单位。Block 也是本地缓存的粒度，相比 Alluxio 的 64MB 缓存块，JuiceFS 的粒度更小，更适合随机读取（例如 Parquet 和 ORC）工作负载，缓存管理也更有效率。JuiceFS 的存储设计，在[架构文档](../architecture.md#how-juicefs-store-files)中有更详细的介绍。\n\nAlluxio 和 JuiceFS 都支持多级缓存，设计上各有特色，但都能够支持用硬盘、SSD、内存来灵活配置大容量或者高性能缓存，详见：\n\n* [Alluxio 缓存](https://docs.alluxio.io/os/user/stable/cn/core-services/Caching.html)\n* [JuiceFS 缓存](../../guide/cache.md)\n* JuiceFS 企业版在社区版的基础上，支持更为强大的[分布式缓存](/docs/zh/cloud/guide/distributed-cache)\n\n### 一致性 {#consistency}\n\nJuiceFS 是一个强一致性的分布式文件系统，它的原子性依赖底层元数据引擎的事务支持（比如 [Redis 事务](https://redis.io/topics/transactions)），因此大部分元数据操作都具有原子性，例如重命名文件、删除文件、重命名目录。\n\nAlluxio 自身并不是一个存储系统，但你依然可以通过 Alluxio 进行写入，不过原子性肯定就无法支持了，因为 Alluxio 依赖 UFS 来实现元数据操作，比如重命名文件操作会变成复制和删除操作。\n\n继续讨论一致性之前，必须先简单了解 Alluxio 的写入是如何实现的。上一小节已经介绍过，Alluxio 存储层和 UFS 是分离的——你可以写存储层，也可以写 UFS，具体文件写入要如何在两个层之间协调，通过以下几种[写入策略](https://docs.alluxio.io/os/user/stable/cn/overview/Architecture.html#%E6%95%B0%E6%8D%AE%E6%B5%81%E5%86%99%E6%93%8D%E4%BD%9C)来控制：\n\n* `MUST_CACHE`：写入 Alluxio worker 内存，性能最好，但 worker 异常会导致数据丢失。适合用来写入临时数据。\n* `THROUGH`：直接写入 UFS，性能取决于底层存储。适合用来写入需要持久化，但最近不需要用到的数据。\n* `CACHE_THROUGH`：同时写入 Alluxio worker 内存和底层 UFS\n* `ASYNC_THROUGH`：先写入 Alluxio worker 内存，再异步提交给 UFS。\n\n可想而知，任何在 Alluxio 中进行的数据写入，都面临着写入性能和一致性之间的取舍。为了达到最理想的性能，用户需要仔细研究写入场景，并为其分配合适的写入策略。显而易见的是，使用 `MUST_CACHE` 或 `ASYNC_THROUGH` 策略一定没有一致性保证，如果写入操作过程中发生故障，其状态是不可预测的。\n\n以上是两个系统在写入一致性方面的对比，至于读数据，Alluxio 会按需从 UFS 加载元数据，并且它在启动时没有关于 UFS 的信息。默认情况下，Alluxio 期望对 UFS 的所有修改都通过 Alluxio 进行。如果直接对 UFS 进行更改，则需要手动或定期在 Alluxio 和 UFS 之间同步元数据，这也容易成为成为不一致的来源。\n\nJuiceFS 则不存在这方面的问题，这是因为 JuiceFS 以元数据服务作为唯一的真实来源（single source of truth），对象存储在这个架构下，只作为数据存储使用，不管理任何元数据。\n\n### 数据压缩 {#data-compression}\n\nJuiceFS 支持使用 [LZ4](https://lz4.github.io/lz4) 或 [Zstandard](https://facebook.github.io/zstd) 来压缩数据。\n\nAlluxio 本质上并不是一个存储系统，虽然你也可以通过 Alluxio 进行数据写入，但[并不支持压缩](https://alluxio.atlassian.net/browse/ALLUXIO-31)。\n\n### 数据加密 {#data-encryption}\n\nAlluxio 仅在[企业版](https://docs.alluxio.io/ee/user/stable/en/security/Security.html#encryption)支持数据加密。\n\nJuiceFS 支持[传输中加密以及静态加密](../../security/encryption.md)。\n\n## 客户端协议对比 {#client-protocol-comparison}\n\n### POSIX\n\nJuiceFS[完全兼容 POSIX](../../reference/posix_compatibility.md)，完整通过用于检验 POSIX 兼容性的 [pjdfstest](https://github.com/pjd/pjdfstest)，并以 99% 以上的成功率通过用于检验 Linux 软件可靠性的 [Linux Test Project](https://github.com/linux-test-project/ltp)，无缝对接已有应用。\n\n除了 pjdfstest 的兼容性测试外，JuiceFS 支持 mmap、fallocate 文件打洞、xattr、BSD 锁（flock）和 POSIX 记录锁（fcntl）。\n\nAlluxio 没有通过 POSIX 兼容性测试。[京东](https://www.slideshare.net/Alluxio/using-alluxio-posix-fuse-api-in-jdcom)的 pjdfstest 测试表明 Alluxio 不支持符号链接、truncate、fallocate、append、xattr、mkfifo、mknod 和 utimes。\n\n### HDFS\n\n二者均兼容 HDFS，包括 Hadoop 2.x 和 Hadoop 3.x，以及 Hadoop 生态系统中的各种组件。详见：\n\n* [JuiceFS Hadoop SDK](../../deployment/hadoop_java_sdk.md)\n* [Alluxio 集成 HDFS 作为底层存储](https://docs.alluxio.io/os/user/stable/cn/ufs/HDFS.html)\n\n### S3\n\nJuiceFS 实现了 [S3 网关](../../guide/gateway.md)，因此如果有需要，可以通过 S3 API 直接访问文件系统，也能使用 s3cmd、AWS CLI、MinIO Client（mc）等工具直接管理文件系统。\n\nAlluxio 也支持大部分 S3 API，详见[文档](https://docs.alluxio.io/os/user/stable/cn/api/S3-API.html)。\n\n### Kubernetes CSI Driver\n\n二者均提供 Kubernetes CSI 驱动：\n\n* [JuiceFS CSI Driver](/docs/zh/csi/introduction) 由 Juicedata 团队持续维护\n* [Alluxio CSI Driver](https://github.com/Alluxio/alluxio/tree/master-2.x/integration/docker/csi) 由 Alluxio 团队持续维护，相对来说迭代速度较慢。\n\n### WebDAV\n\nJuiceFS 实现了 [WebDAV 服务](../../deployment/webdav.md)，用户可以通过 WebDAV 协议管理文件系统中的数据。\n\nAlluxio 不支持 WebDAV 协议。\n\n## 云上部署和运维 {#deployment-and-operation}\n\n这一小节只讨论两个产品的社区版，两个产品的企业版都能获取技术支持服务，因此不作讨论。\n\nAlluxio 的架构可以分为 3 个组件：master、worker 和客户端。一个典型的集群由一个主节点（master）、多个备用主节点（standby master）、一个作业主节点（job master）、多个备用作业主节点（standby job master）、多个 worker 和 job worker 组成。需要自己部署及运维这些节点，详见[文档](https://docs.alluxio.io/os/user/stable/cn/overview/Getting-Started.html#%E9%83%A8%E7%BD%B2-alluxio)。\n\nJuiceFS 使用 Redis 或者其它流行的数据库作为[元数据引擎](../../reference/how_to_set_up_metadata_engine.md)，大部分公有云服务商都提供这些数据库的全托管服务，你可以直接将其作为 JuiceFS 元数据引擎，没有任何运维负担。\n" }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_cephfs.md", "content": "---\nslug: /comparison/juicefs_vs_cephfs\ndescription: Ceph 是一套提供对象存储、块存储和文件存储的统一系统，本文从架构和特性两个维度来对比 JuiceFS 与 Ceph 的异同。\n---\n\n# JuiceFS 对比 CephFS\n\n## 共同点\n\n两者都是高可靠，高性能的弹性分布式文件系统，且均有良好的 POSIX 兼容性，在各种文件系统使用场景都可一试。\n\n## 不同点\n\n### 系统架构\n\n两者都采用了数据和元数据分离的架构，但在组件实现上有很大区别。\n\n#### CephFS\n\n是一套完整且独立的系统，倾向于私有云部署；所有数据和元数据都会持久化在 Ceph 自己的存储池（RADOS Pool）中。\n\n- 元数据\n - 服务进程（MDS）：无状态且理论可水平扩展。目前已有成熟的主备机制，但多主部署依然有性能和稳定性隐患；生产环境通常采用一主多备或者多主静态隔离\n - 持久化：独立的 RADOS 存储池，通常采用 SSD 或更高性能的硬件存储\n- 数据：一个或多个 RADOS 存储池，支持通过 Layout 指定不同的配置，如分块大小（默认 4 MiB），冗余方式（多副本，EC）等\n- 客户端：支持内核客户端（`kcephfs`），用户态客户端（`ceph-fuse`）以及基于 libcephfs 实现的 C++、Python 等 SDK；近来社区也提供了 Windows 客户端（`ceph-dokan`）。同时生态中也有与 Samba 对接的 VFS object 和与 NFS-Ganesha 对接的 FSAL 模块可供考虑。\n\n#### JuiceFS\n\nJuiceFS 主要实现一个 libjfs 库和 FUSE 客户端程序、Java SDK 等，支持对接多种元数据引擎和对象存储，适合在公有云、私有云或混合云环境下部署。\n\n- 元数据：支持多种已有的[数据库实现](../../reference/how_to_set_up_metadata_engine.md)，包括：\n - Redis 及各种兼容 Redis 协议的变种（需要支持事务）；\n - SQL 系列：MySQL，PostgreSQL，SQLite 等；\n - 分布式 K/V 存储：TiKV，FoundationDB，etcd；\n - 自研引擎：用于公有云上的 JuiceFS 全托管服务；\n- 数据：支持超过 30 种公有云上的[对象存储](../../reference/how_to_set_up_object_storage.md)，也可以和 MinIO，Ceph RADOS，Ceph RGW 等对接；\n- 客户端：支持 Unix 用户态挂载，Windows 挂载，完整兼容 HDFS 语义的 Java SDK，[Python SDK](https://github.com/megvii-research/juicefs-python) 以及内置的 S3 网关。\n\n### 功能特性\n\n| | CephFS | JuiceFS |\n| ----------------------- | ----------------- | ------------------ |\n| 文件分块 [1] | ✓ | ✓ |\n| 元数据事务 | ✓ | ✓ |\n| 强一致性 | ✓ | ✓ |\n| Kubernetes CSI Driver | ✓ | ✓ |\n| Hadoop 兼容 | ✓ | ✓ |\n| 数据压缩 [2] | ✓ | ✓ |\n| 数据加密 [3] | ✓ | ✓ |\n| 快照 | ✓ | ✕ |\n| 客户端数据缓存 | ✕ | ✓ |\n| Hadoop 数据本地性 | ✕ | ✓ |\n| S3 兼容 | ✕ | ✓ |\n| 配额 | 目录级配额 | 目录级配额 |\n| 开发语言 | C++ | Go |\n| 开源协议 | LGPLv2.1 & LGPLv3 | Apache License 2.0 |\n\n#### 注 1：文件分块\n\n虽然两者都做了大文件的分块，但在实现原理上有本质区别。CephFS 会将文件按 [`object_size`](https://docs.ceph.com/en/latest/cephfs/file-layouts/#reading-layouts-with-getfattr)（默认为 4MiB）拆分，每个分块对应一个 RADOS object。而 JuiceFS 则将文件先按 64MiB Chunk 拆分，每个 Chunk 在写入时根据实际情况进一步拆分成一个或多个逻辑 Slice，每个 Slice 在写入对象存储时再拆分成默认 4MiB 的 Block，Block 与对象存储中 object 一一对应。在处理覆盖写时，CephFS 需要直接修改对应的 objects，流程较为复杂；尤其是冗余策略为 EC 或者开启数据压缩时，往往需要先读取部分 object 内容，在内存中修改后再写入，这个流程会带来很大的性能开销。而 JuiceFS 在覆盖写时将更新数据作为新 objects 写入并修改元数据即可，性能大幅提升；此外，过程中出现的冗余数据会异步完成垃圾回收。\n\n#### 注 2：数据压缩\n\n严格来讲，CephFS 本身并未提供数据压缩功能，其实际依赖的是 RADOS 层 BlueStore 的压缩。而 JuiceFS 则可以在 Block 上传到对象存储之前就进行一次数据压缩，以减少对象存储中的容量使用。换言之，如果用 JuiceFS 对接 RADOS，是能做到在 Block 进 RADOS 前后各进行一次压缩。另外，就像在**文件分块**中提到的，出于对覆盖写的性能保障，CephFS 一般不会开启 BlueStore 的压缩功能。\n\n#### 注 3：数据加密\n\nCeph **Messenger v2** 支持网络传输层的数据加密，存储层则与压缩类似，依赖于 OSD 创建时提供的加密功能。JuiceFS 是在上传对象前和下载后执行加解密，在对象存储侧完全透明。\n" }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_glusterfs.md", "content": "---\ntitle: JuiceFS 对比 GlusterFS\nslug: /comparison/juicefs_vs_glusterfs\ndescription: 本文对比 JuiceFS 和 GlusterFS 的架构、元数据管理、数据管理、访问协议及扩展功能。\n---\n\n[GlusterFS](https://github.com/gluster/glusterfs) 是一款开源的软件定义分布式存储解决方案，能够在单个集群中支持高达 PiB 级别的数据存储。\n\nJuiceFS 是一款专为云端设计的开源、高性能分布式文件系统，以较低的成本提供了大规模、弹性和高性能的存储能力。\n\n本文先通过一份表格简要对比 JuiceFS 和 GlusterFS 的主要特点，然后进行详细探讨。你可以通过下表速查二者的关键特性对比，然后在本文中选取感兴趣的话题详细阅读。\n\n## JuiceFS 和 GlusterFS 对比一览 {#a-quick-summary-of-glusterfs-vs-juicefs}\n\n下表快速概述了 GlusterFS 和 JuiceFS 之间的差异：\n\n| 对比项 | GlusterFS | JuiceFS |\n| :--- | :--- | :--- |\n| 元数据 | 纯分布式 | 独立数据库服务 |\n| 数据存储 | 自主管理 | 依赖对象存储服务 |\n| 大文件拆分 | 不拆分 | 拆分 |\n| 冗余保护 | 副本、纠删码 | 依赖对象存储服务 |\n| 数据压缩 | 部分支持 | 支持 |\n| 数据加密 | 部分支持 | 支持 |\n| POSIX 兼容性 | 完整 | 完整 |\n| NFS 协议 | 不直接支持 | 不直接支持 |\n| CIFS 协议 | 不直接支持 | 不直接支持 |\n| S3 协议 | 支持（久未更新） | 支持 |\n| HDFS 兼容性 | 支持（久未更新） | 支持 |\n| CSI 驱动 | 支持 | 支持 |\n| POSIX ACLs | 支持 | 支持 |\n| 跨域复制 | 支持 | 依赖外部服务 |\n| 目录配额 | 支持 | 支持 |\n| 快照 | 支持 | 不支持（但支持克隆） |\n| 回收站 | 支持 | 支持 |\n| 主要维护者 | Red Hat, Inc | Juicedata, Inc |\n| 开发语言 | C | Go |\n| 开源协议 | GPLv2 and LGPLv3+ | Apache License 2.0 |\n\n## 系统架构对比 {#system-architecture-comparison}\n\n### GlusterFS 的架构 {#glusterfs-architectire}\n\nGlusterFS 采用的是全分布式的架构，没有中心化节点。GlusterFS 集群主要由服务端和客户端两大部分组成。其中服务端负责管理和存储数据，通常被称为可信存储池（Trusted Storage Pool）。这个存储池由一系列对等的 Server 节点组成，一般会运行两类进程：\n\n* glusterd：每个节点一个，负责配置管理和分发等。\n* glusterfsd：每个 [Brick](https://docs.gluster.org/en/latest/glossary/#Brick) 一个，负责处理数据请求和对接底层文件系统。\n\n每个 Brick 上的所有文件可以看成是 GlusterFS 的一个子集，就文件内容而言，通过 Brick 直接访问和通过 GlusterFS 客户端访问看到的结果通常是一致的。因此，在 GlusterFS 异常情况下，用户通过整合多个 Bricks 内容就能一定程度上恢复出原有数据。另外在部署时，为了确保某台机器故障时，整个文件系统的访问不受影响，通常会对数据做冗余保护。在 GlusterFS 中，多个 Bricks 会组成一个冗余组，互相之间通过副本或纠删码的方式实现数据保护。当某个节点故障时，只能在冗余组内做恢复，恢复的时间会比较长。在 GlusterFS 集群扩容时，需要以冗余组为单位整体扩容。\n\n客户端是挂载了 GlusterFS 的节点，负责对应用程序展示统一的命名空间。其架构图如下（出自 [Gluster 架构](https://docs.gluster.org/en/latest/Quick-Start-Guide/Architecture)）：\n\n![Gluster 架构](../../images/glusterfs-architecture.jpg)\n\n### JuiceFS 的架构 {#juicefs-architecture}\n\nJuiceFS 采用「数据」与「元数据」分离存储的架构，文件数据本身会被切分保存在对象存储（如 Amazon S3）当中，而元数据则是会被保存在用户自行选择的数据库里（如 Redis、MySQL）。通过共享同一个份数据库与对象存储，JuiceFS 实现了一个强一致性保证的分布式文件系统，同时还具有「POSIX 完全兼容」、「高性能」等诸多特性。更详细的介绍参见[文档](../architecture.md)。\n\n![JuiceFS 架构](../../images/juicefs-arch-new.png)\n\n## 元数据管理对比 {#metadata-management-comparison}\n\n### GlusterFS {#glusterfs}\n\nGlusterFS 元数据是纯分布式的，没有集中的元数据服务。客户端通过对文件名哈希确定其所属的 Brick；当请求需要跨多个 Bricks 访问（如 mv，ls 等）时，由客户端负责协调。这种设计架构上比较简单，但当系统规模扩大时，往往会带来性能瓶颈。比如，ls 一个大目录时可能会需要访问多个 Bricks 来获得完整的结果，其中任何一个的卡顿都会导致整个请求变慢。另外，跨 Bricks 修改操作在途中遇到故障时，元数据一致性也比较难保证。在严重故障时，还可能出现脑裂，需要[手动恢复](https://docs.gluster.org/en/latest/Troubleshooting/resolving-splitbrain)数据到统一版本。\n\n### JuiceFS {#juicefs}\n\nJuiceFS 的元数据存储在一个独立的数据库（称为元数据引擎）中，客户端会将文件元数据操作转换成此数据库的一个事务，借助数据库的事务能力来保证操作的原子性。这种设计使得 JuiceFS 的实现变得简单，但对元数据引擎提出了较高的要求。目前 JuiceFS 支持三大类 10 种事务型数据库，具体可参见[元数据引擎文档](../../reference/how_to_set_up_metadata_engine.md)。\n\n## 数据管理对比 {#data-management-comparison}\n\nGlusterFS 通过整合多个服务端节点的 Bricks（一般构建在本地文件系统之上，如 XFS）来存储数据。因此，它本身提供了一定的数据管理功能，如分布管理、冗余保护、故障切换、静默错误检测等。\n\nJuiceFS 则不直接使用硬盘，而是通过对接各种对象存储来管理数据，大部分特性都依赖于对象存储自身的实现。\n\n### 大文件拆分 {#large-file-splitting}\n\n在分布式系统中，将大文件拆分成多个小块散列存储在不同节点中是一种常见的优化手段。这往往能让应用在访问此文件时有更高的并发度和整体带宽。\n\n* GlusterFS：不拆分（曾有过 Striped Volume 会拆分大文件，现已不再支持）。\n* JuiceFS：文件先按大小拆成 64 MiB 的 Chunks，每个 Chunk 再根据写入模式进一步拆成默认 4 MiB 的 Blocks；具体可参见[架构文档](../architecture.md#how-juicefs-store-files)。\n\n### 冗余保护 {#redundancy-protection}\n\nGlusterFS 支持副本（Replicated Volume）和纠删码（Dispersed Volume）两种类型。\n\nJuiceFS 依赖于使用的对象存储。\n\n### 数据压缩 {#data-compression}\n\nGlusterFS：\n\n* 仅支持传输层压缩，文件由客户端执行压缩，传输到服务端后再由 Brick 负责解压缩。\n* 不直接实现存储层压缩，而是依赖于 Brick 使用的底层文件系统，如 [ZFS](https://docs.gluster.org/en/latest/Administrator-Guide/Gluster-On-ZFS)。\n\nJuiceFS 同时支持传输层压缩和存储层压缩，数据的压缩和解压缩都在客户端执行。\n\n### 数据加密 {#data-encryption}\n\nGlusterFS：\n\n* 仅支持[传输层加密](https://docs.gluster.org/en/latest/Administrator-Guide/SSL)，依赖于 SSL/TLS。\n* 曾支持过[存储层加密](https://github.com/gluster/glusterfs-specs/blob/master/done/GlusterFS%203.5/Disk%20Encryption.md)，但现已不再支持。\n\nJuiceFS 同时支持[传输层加密和存储层加密](../../security/encryption.md)，数据的加密和解密都在客户端进行。\n\n## 访问协议 {#access-protocols}\n\n### POSIX 兼容性 {#posix-compatibility}\n\n[GlusterFS](https://docs.gluster.org/en/latest/glossary) 和 [JuiceFS](../../reference/posix_compatibility.md) 都提供 POSIX 兼容性。\n\n### NFS 协议 {#nfs-protocol}\n\nGlusterFS 曾有内嵌服务来支持 NFSv3，但现已[不再推荐使用](https://github.com/gluster/glusterfs-specs/blob/master/done/GlusterFS%203.8/gluster-nfs-off.md)，而是建议用 NFS server 将挂载点导出。\n\nJuiceFS 不直接支持，需要挂载后[通过其他 NFS server 导出](../../deployment/nfs.md)。\n\n### CIFS 协议 {#cifs-protocol}\n\nGlusterFS 内嵌支持 Windows，Linux Samba client 和 macOS 的 CLI 访问，不支持 macOS Finder。然而，文档中建议用[通过 Samba 将挂载点导出](https://docs.gluster.org/en/latest/Administrator-Guide/Setting-Up-Clients/#testing-mounted-volumes)的方式使用。\n\nJuiceFS 不直接支持，需要挂载后[通过 Samba 导出](../../deployment/samba.md)。\n\n### S3 协议 {#s3-protocol}\n\nGlusterFS 通过 [`gluster-swift`](https://github.com/gluster/gluster-swift) 项目支持，但其最近更新停留在 2017 年 11 月。\n\nJuiceFS 通过 [S3 网关](../../guide/gateway.md)支持。\n\n### HDFS 兼容性 {#hdfs-compatibility}\n\nGlusterFS 通过 [`glusterfs-hadoop`](https://github.com/gluster/glusterfs-hadoop) 项目支持，但其最近更新停留在 2015 年 5 月。\n\nJuiceFS 完整[兼容 HDFS API](../../deployment/hadoop_java_sdk.md)。\n\n### CSI 驱动 {#csi-driver}\n\nGlusterFS 曾[支持过](https://github.com/gluster/gluster-csi-driver)，但最近版本发布于 2018 年 11 月，且仓库已被标记 DEPRECATED。\n\nJuiceFS 支持，具体可参见 [JuiceFS CSI 驱动文档](https://juicefs.com/docs/zh/csi/introduction)。\n\n## 扩展功能 {#extended-features}\n\n### POSIX ACLs {#posix-acls}\n\nLinux 下对文件的访问权限控制一般有三类实体，即文件拥有者（owner）、拥有组（group）和其他（other）。当我们有更复杂的需求，比如要给本属于 other 的某个特定用户单独赋予权限时，这套机制就做不到了。POSIX Access Control Lists (ACLs) 提供增强的权限管理功能，可用来为任意用户/用户组指定权限。\n\nGlusterFS [支持](https://docs.gluster.org/en/main/Administrator-Guide/Access-Control-Lists)，且支持 access ACLs 和 default ACLs。\n\nJuiceFS 从 v1.2 版本开始支持 [POSIX ACLs](../../security/posix_acl.md) 特性。\n\n### 跨域复制 {#cross-cluster-replication}\n\n跨域复制是指在两套独立的集群间进行数据复制，一般被用来实现异地灾备。\n\nGlusterFS [支持单向的异步增量复制](https://docs.gluster.org/en/main/Administrator-Guide/Geo-Replication)，但需要两边是同版本的 Gluster 集群。\n\nJuiceFS 依赖元数据引擎和对象存储自身的复制能力，可以做单向复制。\n\n### 目录配额 {#directory-quotas}\n\n[GlusterFS](https://docs.gluster.org/en/main/Administrator-Guide/Directory-Quota) 和 [JuiceFS](../../guide/quota.md#directory-quota) 都支持目录配额，包括容量和/或文件数限制。\n\n### 快照 {#snapshots}\n\nGlusterFS 仅[支持存储卷级别的快照](https://docs.gluster.org/en/main/Administrator-Guide/Managing-Snapshots)，而且需要所有 Bricks 部署在 LVM 精简卷（Thinly-Provisioned LVM）上。\n\nJuiceFS 不支持快照，但支持[目录级别的克隆](../../guide/clone.md)。\n\n### 回收站 {#trash}\n\nGlusterFS [支持](https://docs.gluster.org/en/main/Administrator-Guide/Trash)，且默认关闭。\n\nJuiceFS [支持](../../security/trash.md)，且默认打开。\n" }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_lustre.md", "content": "---\nslug: /comparison/juicefs_vs_lustre\ndescription: 本文对比了 Lustre 和 JuiceFS 在架构设计、文件分布和功能特性方面的差异。\n---\n\n# JuiceFS 对比 Lustre\n\nLustre 是一款专为高性能计算（HPC）环境设计的并行分布式文件系统，最初在美国政府资助下，由多个国家实验室联合开发，旨在支持大规模科学研究和工程计算任务。当前，Lustre 的主要开发与维护由 DDN（DataDirect Networks）负责，广泛应用于超算中心、科研机构及企业级 HPC 集群中。\n\nJuiceFS 是一个云原生分布式文件系统，其数据存储在对象存储中。社区版可与多种元数据服务集成，适用场景广泛，于 2021 年在 GitHub 开源。企业版专为高性能场景设计，广泛应用于大规模 AI 任务，涵盖生成式 AI、自动驾驶、量化金融和生物科技等。\n\n本文从架构设计、文件分布和功能特性等方面对 Lustre 和 JuiceFS 进行全面对比。\n\n## 架构对比\n\n### Lustre\n\nLustre 采用传统的客户端 - 服务器架构，由以下几个核心模块组成：\n\n- **元数据服务器（MDS）**：负责处理命名空间相关操作，如文件创建、删除、权限检查等。自 2.4 版本起引入了分布式命名空间（DNE）功能，支持将单个文件系统的不同目录分布在多个元数据服务器上，实现元数据访问负载的横向扩展。\n- **对象存储服务器（OSS）**：负责实际的数据读写，提供高性能的大规模 I/O 服务。\n- **管理服务器（MGS）**：作为全局配置注册中心，负责存储和分发 Lustre 文件系统的配置信息。MGS 在功能上独立于具体的 Lustre 实例。\n- **客户端（Client）**：为用户应用程序提供访问 Lustre 文件系统的接口，实现标准的 POSIX 文件操作语义。\n\n各组件通过 Lustre 专用的网络协议 LNet 连接，构成一个统一高效的文件系统整体。\n\n![Lustre architecture](https://static1.juicefs.com/images/Lustre_JiaGouTu_SWMlRaK.original.png)\n\n### JuiceFS\n\nJuiceFS 采用模块化架构，包括三个核心组件：\n\n- **元数据引擎**：用于存储文件元数据，包括常规文件系统的元数据和文件数据的索引。社区版支持 Redis、TiKV、MySQL、PostgreSQL、FoundationDB 等多种数据库。企业版使用自研高性能元数据服务。\n- **数据存储**：一般是对象存储服务，可以是公有云的对象存储也可以是私有部署的对象存储服务。支持 30 多种对象存储，包括 AWS S3、Azure Blob、Google Cloud Storage、MinIO、Ceph RADOS 等。\n- **客户端**：提供 POSIX（FUSE）、Hadoop SDK、CSI Driver、S3 网关、Python SDK 等不同的接入方式。\n\n![JuiceFS Community Edition architecture](../../images/juicefs-arch.svg)\n\n### 架构差异\n\n#### 客户端实现\n\nLustre 采用 C 语言实现，其客户端模块运行在内核态；而 JuiceFS 使用 Go 语言开发，客户端通过 FUSE（Filesystem in Userspace）暴露文件系统接口，运行在用户态。由于 Lustre 客户端运行于内核空间，访问元数据服务器（MDS）或对象存储服务器（OSS）时无需进行用户态与内核态的上下文切换或额外的内存拷贝，从而显著减少了系统调用所带来的性能开销，在吞吐和延迟方面具备一定优势。\n\n然而，内核态实现也带来了运维和调试的复杂性。相比用户态的开发环境和调试工具，内核态工具门槛更高，不易为普通开发者所掌握。同时，与 C 语言相比，Go 语言更易于学习、维护和开发，具备更高的开发效率和可维护性。\n\n#### 存储模块\n\nLustre 在部署时通常需要配置一块或多块共享磁盘来存储文件数据。这一设计源于其早期版本尚不支持文件级冗余（File Level Redundancy，FLR）。为了实现高可用性（HA），当某个节点下线时，必须将其文件系统挂载到对等节点，否则该节点上的数据块将不可访问。因此，数据的可靠性需依赖于共享存储本身的高可用机制，或用户自行配置的软件 RAID 实现。\n\nJuiceFS 利用对象存储作为数据存储解决方案，从而可享有对象存储带来的若干优势，如数据可靠性、一致性等。用户可以根据自己的需求对接具体的存储系统，既包括主流云厂商的对象存储，也支持如 MinIO、Ceph RADOS 等私有部署的对象存储系统。社区版 JuiceFS 提供本地缓存来应对 AI 场景下的带宽需求，企业版使用分布式缓存满足更大的聚合读带宽的需求。\n\n#### 元数据模块\n\nLustre 的 MDS 高可用性依赖于软硬件协同实现：\n\n- **硬件层面**：MDS 使用的磁盘需配置 RAID，以避免因单点磁盘故障导致服务不可用；磁盘也需具备共享能力，以便当主节点宕机时，备节点能接管磁盘资源。\n- **软件层面**：使用 Pacemaker 与 Corosync 构建高可用集群，确保任一时刻仅有一个 MDS 实例处于活动状态。\n\nJuiceFS 社区版的元数据模块提供一组操作元数据的接口，可以接入不同的元数据服务，包括 Redis、TiKV、MySQL、PostgreSQL、FoundationDB 等不同类型的数据库。JuiceFS 企业版使用自研高性能元数据服务，可根据负载情况来平衡数据和热点操作，以避免大规模训练中元数据服务热点集中在某些节点的问题。\n\n## 文件分布对比\n\n### Lustre 文件分布\n\n#### Normal File Layout (NFL)\n\nLustre 早期采用的文件分布方式被称为 Normal File Layout。在该模式下，文件被切分为多个数据块，并分别存储在多个对象存储目标（OSTs）上，其策略类似于 RAID 0。\n\n文件分布策略主要由以下两个参数控制：\n\n- **Stripe Count**：指定文件可以同时分布到多少个 OST 上。该值越大，文件并行访问能力越强，但也可能带来额外的调度和管理开销。\n- **Stripe Size**：定义在切换到下一个 OST 之前，每个数据块的大小。也就是说，写入达到设定的 Stripe Size 后，数据将被写入下一个 OST，这也决定了每个 Chunk 的粒度。\n\n![Lustre NFL file distribution](https://static1.juicefs.com/images/Lustre_NFL_WenJianFenBuShiLi.original.png)\n\n上图展示了一个 Stripe Count 为 3、Stripe Size 为 1 MB 的文件在多个 OST 上的分布方式。每个数据块（Stripe）采用轮询（Round-Robin）方式依次分布到不同的 OST 上。\n\n主要限制包括：\n\n- 一旦文件创建，配置参数不可变\n- 如果任何目标 OST 空间耗尽，可能导致 ENOSPC（空间不足）错误\n- 随时间推移可能导致存储不均衡\n\n#### Progressive File Layout (PFL)\n\n为了解决 NFL 在应对动态数据增长和资源分配方面存在的局限，Lustre 引入了一种新的文件分布机制，称为 Progressive File Layout (PFL)。\n\n![Lustre PFL file distribution](https://static1.juicefs.com/images/Lustre_PFL_WenJianFenBuShiLi.original.png)\n\nPFL 支持为同一个文件的不同区段定义不同的布局策略，具备以下优势：\n\n- 动态适应文件增长\n- 减缓磁盘不均衡问题\n- 提高空间利用率和灵活性\n\n虽然 PFL 引入了更具弹性的布局策略，但 Lustre 进一步结合 Lazy Initialization 技术，以实现更高效的资源调度。\n\n#### File Level Redundancy (FLR)\n\nLustre 引入了文件级冗余来简化 HA 架构并提升系统容错能力。FLR 允许为每个文件配置一个或多个副本，实现文件级别的冗余保护。在写入操作发生时，数据仅写入其中一个副本，其余副本会被标记为 STALE（过期）。随后，系统通过一个称为 Resync 的同步过程，确保数据一致性。\n\n### JuiceFS 文件分布\n\nJuiceFS 按照 Chunk、Slice、Block 的规则进行数据块管理。每个 Chunk 的大小固定为 64M，主要用于优化数据的查找和定位。实际的文件写入操作则在 Slice 上执行，每个 Slice 代表一次连续的写入过程，属于特定的 Chunk，并且不会跨越 Chunk 的边界，因此长度不超过 64M。Block（默认大小为 4M）则是物理存储的基本单位，用于在对象存储和磁盘缓存中实现数据的最终存储。\n\n![JuiceFS file distribution](../../images/file-and-chunks.svg)\n\nJuiceFS 中的 Slice 是在其他文件系统中不常见的一个结构。主要功能是记录文件的写入操作，并在对象存储中进行持久化。对象存储不支持原地文件修改，因此，JuiceFS 通过引入 Slice 结构允许更新文件内容，而无需重写整个文件。当修改文件时，系统会创建新的 Slice，并在该 Slice 上传完毕后更新元数据，从而将文件内容指向新的 Slice。\n\nJuiceFS 的所有 Slice 均为一次性写入，这减少了对底层对象存储一致性的依赖，并大大简化了缓存系统的复杂度，使数据一致性更易于保证。\n\n## 功能特性对比\n\n| 功能特性 | Lustre | JuiceFS 社区版 | JuiceFS 企业版 |\n|----------|--------|---------------|---------------|\n| 元数据 | 分布式元数据服务 | 独立数据库服务 | 自研高性能分布式元数据引擎（可横向扩展） |\n| 元数据冗余保护 | 需要存储设备提供 | 取决于使用的数据库 | 三副本 |\n| 数据存储 | 自主管理 | 使用对象存储 | 使用对象存储 |\n| 数据冗余保护 | 存储设备提供或异步复制 | 对象存储提供 | 对象存储提供 |\n| 数据缓存 | 客户端本地缓存 | 客户端本地缓存 | 自研高性能多副本分布式缓存 |\n| 数据加密 | 支持 | 支持 | 支持 |\n| 数据压缩 | 支持 | 支持 | 支持 |\n| 配额管理 | 支持 | 支持 | 支持 |\n| 网络协议 | 支持多种网络协议 | TCP | TCP |\n| 快照 | 文件系统级别快照 | 文件级别快照 | 文件级别快照 |\n| POSIX ACL | 支持 | 支持 | 支持 |\n| POSIX 兼容性 | 兼容 | 完全兼容 | 完全兼容 |\n| CSI 驱动 | 非官方支持 | 支持 | 支持 |\n| 客户端 | POSIX | POSIX（FUSE）、Java SDK、S3 网关、Python SDK | POSIX（FUSE）、Java SDK、S3 网关、Python SDK |\n| 多云镜像 | 不支持 | 不支持 | 支持 |\n| 跨云和跨区数据复制 | 不支持 | 不支持 | 支持 |\n| 主要维护者 | DDN | Juicedata | Juicedata |\n| 开发语言 | C | Go | Go |\n| 开源协议 | GPL 2.0 | Apache License 2.0 | 商业软件 |\n\n## 小结\n\nLustre 是一款高性能并行分布式文件系统，客户端运行于内核态，直接与元数据服务器（MDS）和对象存储服务器（OSS）交互，避免了用户态与内核态之间的上下文切换。结合高性能存储设备，Lustre 在高带宽 I/O 场景下展现出卓越的性能。\n\n然而，由于客户端运行在内核态，这使得运维过程更具挑战性，运维团队需具备深入的内核调试经验和底层系统故障排查能力。此外，由于 Lustre 使用固定容量的存储方案，文件分布设计相对复杂，需要精细的规划与配置来实现资源的高效利用。因此，Lustre 的部署和运维门槛较高。\n\nJuiceFS 是一款云原生、用户态分布式文件系统，紧密集成对象存储，并原生支持 Kubernetes CSI，从而简化了在云平台上的部署和运维。用户无需深入关注底层存储设备和复杂的存储调度机制，即可在容器化环境中实现弹性扩展、高可用数据服务。在性能方面，JuiceFS 企业版通过分布式缓存，有效降低对象存储的访问延迟，提升文件操作的响应速度。\n\n从成本角度看，Lustre 需要依赖高性能的专用存储设备，初始投资和长期维护成本较高。对象存储则更加经济，具备天然的可扩展性以及按需付费的灵活性。\n\n两个系统各有优势：Lustre 在传统 HPC 环境中追求极致性能方面表现卓越，而 JuiceFS 在云原生和 AI 工作负载方面提供了更好的灵活性、更容易的管理和更高的性价比。\n" }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_s3fs.md", "content": "---\nslug: /comparison/juicefs_vs_s3fs\n---\n\n# JuiceFS 对比 S3FS\n\n[S3FS](https://github.com/s3fs-fuse/s3fs-fuse) 是一个 C++ 开发的开源工具，可以将 S3 对象存储通过 FUSE 挂载到本地，像本地磁盘一样进行读写访问。除了 Amazon S3，它还支持所有兼容 S3 API 的对象存储。\n\n在基本功能方面，S3FS 与 JuiceFS 都能通过 FUSE 将对象存储 Bucket 挂载到本地并以 POSIX 接口使用。但在功能细节和技术实现上，二者有着本质的不同。\n\n## 产品定位\n\nS3FS 是一种实用工具，可以方便地将对象存储 Bucket 挂载到本地，以用户熟悉的方式进行读写，面向那些对性能和网络延迟不敏感的一般使用场景。\n\nJuiceFS 是分布式文件系统，具有独特的数据管理方式以及一系列针对高性能、可靠性和安全性等方面的技术优化，主要解决海量数据的存储需求。\n\n## 系统架构\n\nS3FS 没有针对文件做特别的优化处理，它就像一个本地与对象存储之间的访问通道，本地挂载点看到的内容与对象存储浏览器上看到的一致，这样可以很方便地实现在本地使用云端存储。但从另一个角来看，正是因为这种简单的架构，使得 S3FS 对文件的检索和读写都需要与对象存储直接交互，网络延迟对性能和用户体验都会有较大的影响。\n\nJuiceFS 采用数据和元数据分离的技术架构，任何文件都会先按照特定规则拆分成数据块再上传到对象存储，相应的元数据会存储在独立的数据库中。这样带来的好处是对文件的检索以及文件名等元信息的修改可以直接与响应速度更快的数据库交互，避开了与对象存储交互的网络延迟影响。\n\n另外，在大文件的处理方面，虽然 S3FS 可以通过分块上传解决大文件的传输问题，但对象存储的特性决定了追加和改写文件需要重写整个对象。对于几十几百 GB 甚至 TB 级的大文件来说，重复上传势必会浪费大量的时间和带宽资源。\n\nJuiceFS 则规避了此类问题，不论单个文件尺寸多大，在上传之前都会预先在本地按照特定规则拆分成数据块（默认 4MiB）。对任何文件的改写和追加最终都会变成生成新的数据块，而不是修改已生成的数据块，大大减少了时间和带宽资源的浪费。\n\n有关 JuiceFS 的详细架构介绍请参考[文档](../../introduction/architecture.md)。\n\n## 缓存机制\n\nS3FS 支持磁盘缓存，但默认不启用。可以通过 `-o use_cache` 指定一个缓存路径来启用本地缓存。启用缓存后，任何文件的读写都会先写入缓存，然后再执行操作。S3FS 通过 MD5 来检测数据变化，确保数据正确性，同时降低文件的重复下载。由于 S3FS 涉及的所有操作都需要与 S3 交互，因此是否启用缓存对其应用体验有显著的影响。\n\nS3FS 默认不限制缓存空间上限，对于较大的 Buket 可能导致缓存把磁盘写满，需要通过 `-o ensure_diskfree` 定义为磁盘保留的空间。另外，S3FS 没有缓存过期和清理机制，用户需要定期手动清理缓存，一旦缓存空间被存满，未缓存文件操作则需要直接与对象存储交互，处理大规模文件会有一定影响。\n\n在缓存方面，JuiceFS 与 S3FS 完全不同，首先，JuiceFS 是保证数据一致性的。其次，JuiceFS 默认定义了 100GiB 的磁盘缓存使用上限，用户可以根据需要自由调整该值，而且默认会确保磁盘剩余空间低于 10% 时不再使用更多空间。当缓存用量达到上限，JuiceFS 会采用类似 LRU 的算法自动进行清理，确保后续的读写操作始终有缓存可用。\n\n有关 JuiceFS 缓存的更多内容请参考[文档](../../guide/cache.md)。\n\n## 功能特性\n\n| | S3FS | JuiceFS |\n|----------------|----------------------------------|-----------------------------------------|\n| 数据存储 | S3 | S3、其他对象存储、WebDAV、本地磁盘 |\n| 元数据存储 | 无 | 独立数据库 |\n| 系统 | Linux、macOS | Linux、macOS、Windows |\n| 访问接口 | POSIX | POSIX、HDFS API、S3 Gateway、CSI Driver |\n| POSIX 兼容 | 部分兼容 | 完全兼容 |\n| 共享挂载 | 支持但不保证数据的完整性和一致性 | 保证强一致性 |\n| 本地缓存 | ✓ | ✓ |\n| 符号链接 | ✓ | ✓ |\n| 标准 Unix 权限 | ✓ | ✓ |\n| 强一致性 | ✕ | ✓ |\n| 扩展属性 | ✕ | ✓ |\n| 硬链接 | ✕ | ✓ |\n| 文件分块 | ✕ | ✓ |\n| 原子操作 | ✕ | ✓ |\n| 数据压缩 | ✕ | ✓ |\n| 客户端加密 | ✕ | ✓ |\n| 开发语言 | C++ | Go |\n| 开源协议 | GPL v2.0 | Apache License 2.0 |\n\n## 补充说明\n\n[OSSFS](https://github.com/aliyun/ossfs)、[COSFS](https://github.com/tencentyun/cosfs)、[OBSFS](https://github.com/huaweicloud/huaweicloud-obs-obsfs) 等都是基于 S3FS 开发的衍生品，功能特性和用法与 S3FS 基本一致。\n" }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_s3ql.md", "content": "---\nslug: /comparison/juicefs_vs_s3ql\n---\n\n# JuiceFS 对比 S3QL\n\n与 JuiceFS 类似，S3QL 也是一款由对象存储和数据库组合驱动的开源网络文件系统，所有存入的数据会被分块后存储到亚马逊 S3、Backblaze B2、OpenStack Swift 等主流的对象存储中，相应的元数据会存储在数据库中。\n\n## 共同点\n\n- 都是通过 FUSE 模块实现对标准 POSIX 文件系统接口的支持，从而可以将海量的云端存储挂载到本地，像本地存储一样使用。\n- 都能提供标准的文件系统功能：硬链接、符号链接、扩展属性、文件权限。\n- 都支持数据压缩和加密，但二者采用的算法各有不同。\n- 都支持元数据库备份，S3QL 自动备份 SQLite 数据库到对象存储。JuiceFS 每小时自动将元数据导出为 JSON 格式文件并备份到对象存储，便于恢复以及在各种元数据引擎间迁移。\n\n## 不同点\n\n- S3QL 仅支持 SQLite 一种数据库，而 JuiceFS 除了支持 SQLite 以外还支持 Redis、TiKV、MySQL、PostgreSQL 等数据库。\n- S3QL 没有分布式能力，**不支持**多主机同时挂载。JuiceFS 是典型的分布式文件系统，在使用基于网络的数据库时，支持多主机分布式挂载读写。\n- S3QL 在一个数据块几秒内未被访问时将其上传到对象存储。文件被关闭甚者 fsync 后其仍仅保证在系统内存中，节点故障时可能丢失数据。JuiceFS 确保了数据的高可靠性，在文件关闭时会将其同步上传到对象存储。\n- S3QL 提供数据去重，相同数据只存储一份，可以降低对象存储的用量，但也会加重系统的性能开销。相比之下，JuiceFS 更注重性能，对大规模数据去重代价过高，暂不提供该功能。\n\n| | **S3QL** | **JuiceFS** |\n| :-------------------- | :----------------- | :------------------------- |\n| 项目状态 | 活跃维护 | 活跃开发 |\n| 元数据引擎 | SQLite | Redis、MySQL、SQLite、TiKV |\n| 存储引擎 | 对象存储、本地磁盘 | 对象存储、WebDAV、本地磁盘 |\n| 操作系统 | Unix-like | Linux、macOS、Windows |\n| 压缩算法 | LZMA, bzip2, gzip | LZ4, zstd |\n| 加密算法 | AES-256 | AES-GCM, RSA |\n| POSIX 兼容 | ✓ | ✓ |\n| 硬链接 | ✓ | ✓ |\n| 符号链接 | ✓ | ✓ |\n| 扩展属性 | ✓ | ✓ |\n| 标准 Unix 权限 | ✓ | ✓ |\n| 数据分块 | ✓ | ✓ |\n| 本地缓存 | ✓ | ✓ |\n| 空间弹性伸缩 | ✓ | ✓ |\n| 元数据备份 | ✓ | ✓ |\n| 数据去重 | ✓ | ✕ |\n| 只读目录 | ✓ | ✕ |\n| 快照 | ✓ | ✕ |\n| 共享挂载 | ✕ | ✓ |\n| Hadoop SDK | ✕ | ✓ |\n| Kubernetes CSI Driver | ✕ | ✓ |\n| S3 网关 | ✕ | ✓ |\n| 开发语言 | Python | Go |\n| 开源协议 | GPLv3 | Apache License 2.0 |\n| 开源时间 | 2011 | 2021.1 |\n\n## 易用性\n\n这部分主要评估两个产品在安装和使用上的的易用程度。\n\n### 安装\n\n在安装过程中，我们使用 Rocky Linux 8.4 操作系统（内核版本 4.18.0-305.12.1.el8_4.x86_64）。\n\n#### S3QL\n\nS3QL 采用 Python 开发，在安装时需要依赖 `python-devel` 3.7 及以上版本。另外，还需要至少满足以下依赖：`fuse3-devel`、`gcc`、`pyfuse3`、`sqlite-devel`、`cryptography`、`defusedxml`、`apsw`、`dugong`。另外，需要特别注意 Python 的包依赖和位置问题。\n\nS3QL 会在系统中安装 12 个二进制程序，每个程序都提供一个独立的功能，如下图。\n\n![S3QL-bin](../../images/s3ql-bin.jpg)\n\n#### JuiceFS\n\nJuiceFS 客户端采用 Go 语言开发，直接下载预编译的二进制文件即可直接使用。JuiceFS 客户端只有一个二进制程序 `juicefs`，将其拷贝到系统的任何一个可执行路径下即可，比如：`/usr/local/bin`。\n\n### 使用\n\nS3QL 和 JuiceFS 都使用数据库保存元数据，S3QL 仅支持 SQLite 数据库，JuiceFS 支持 Redis、TiKV、MySQL、MariaDB、PostgreSQL 和 SQLite 等数据库。\n\n这里使用本地创建的 MinIO 对象存储，使用两款工具分别创建文件系统：\n\n#### S3QL\n\nS3QL 使用 `mkfs.s3ql` 工具创建文件系统：\n\n```shell\nmkfs.s3ql --plain --backend-options no-ssl -L s3ql s3c://127.0.0.1:9000/s3ql/\n```\n\n挂载文件系统使用 `mount.s3ql`：\n\n```shell\nmount.s3ql --compress none --backend-options no-ssl s3c://127.0.0.1:9000/s3ql/ mnt-s3ql\n```\n\nS3QL 在创建和挂载文件系统时都需要通过命令行交互式的提供对象存储 API 的访问密钥。\n\n#### JuiceFS\n\nJuiceFS 使用 `format` 子命令创建文件系统：\n\n```shell\njuicefs format --storage minio \\\n --bucket http://127.0.0.1:9000/myjfs \\\n --access-key minioadmin \\\n --secret-key minioadmin \\\n sqlite3://myjfs.db \\\n myjfs\n```\n\n挂载文件系统使用 `mount` 子命令：\n\n```shell\nsudo juicefs mount -d sqlite3://myjfs.db mnt-juicefs\n```\n\nJuiceFS 只在创建文件系统时设置对象存储 API 访问密钥，相关信息会写入元数据引擎，之后挂载使用无需重复提供对象存储地址、密钥等信息。\n\n## 对比总结\n\n**S3QL** 采用对象存储 + SQLite 的存储结构，数据分块存储既能提高文件的读写效率，也能降低文件修改时的资源开销。贴心的提供了快照、数据去重、数据保持等高级功能，加之默认的数据压缩和数据加密，让 S3QL 非常适合个人在云存储上用较低的成本、更安全的存储文件。\n\n**JuiceFS** 支持对象存储、HDFS、WebDAV、本地磁盘作为数据存储引擎，支持 Redis、TiKV、MySQL、MariaDB、PostgreSQL、SQLite 等流行的数据作为元数据存储引擎。除了通过 FUSE 提供标准的 POSIX 文件系统接口以外，JuiceFS 还提供 Java API，可以直接替代 HDFS 为 Hadoop 提供存储。同时还提供 [Kubernetes CSI Driver](https://github.com/juicedata/juicefs-csi-driver)，可以作为 Kubernetes 的存储层做数据持久化存储。JuiceFS 是为企业级分布式数据存储场景设计的文件系统，广泛应用于大数据分析、机器学习、容器共享存储、数据共享及备份等多种场景。\n" }, { "path": "docs/zh_cn/introduction/comparison/juicefs_vs_seaweedfs.md", "content": "---\ntitle: JuiceFS 对比 SeaweedFS\nslug: /comparison/juicefs_vs_seaweedfs\ndescription: 本文对比 JuiceFS 和 SeaweedFS 的架构、存储机制、客户端协议及其他高级功能。\n---\n\n[SeaweedFS](https://github.com/seaweedfs/seaweedfs) 与 [JuiceFS](https://github.com/juicedata/juicefs) 皆是开源的高性能分布式文件存储系统，但二者存在诸多设计区别与功能差异，本章将会详述他们的区别和各自适用场景，帮助你的团队进行技术选型。\n\nSeaweedFS 和 JuiceFS 都采用了对商用更友好的 Apache License 2.0，但 JuiceFS 分为[社区版](https://juicefs.com/docs/zh/community/introduction)和[企业版](https://juicefs.com/zh-cn/blog/solutions/juicefs-enterprise-edition-features-vs-community-edition)，企业版提供多种交付形式，例如私有部署和[云服务](https://juicefs.com/docs/zh/cloud)。JuiceFS 企业版使用自研的闭源元数据引擎，其客户端则与[社区版](https://github.com/juicedata/juicefs)大量共享代码。你可以通过下表速查两者的关键特性对比，然后在本文中选取感兴趣的话题详细阅读。\n\n## JuiceFS 和 SeaweedFS 对比一览\n\n| 对比项 | SeaweedFS | JuiceFS |\n| :--- | :--- | :--- |\n| 元数据引擎 | 支持多种数据库 | 社区版支持多种数据库、企业版使用自研高性能元数据引擎 |\n| 元数据操作原子性 | 未保证 | 社区版通过数据库事务保证、企业版元数据引擎自身保证操作原子性 |\n| 变更日志 | 支持 | 仅企业版支持 |\n| 数据存储 | 自包含 | 依赖对象存储 |\n| 纠删码 | 支持 | 依赖对象存储 |\n| 数据合并 | 支持 | 依赖对象存储 |\n| 文件拆分 | 8MB | 64MB 逻辑块 + 4MB 物理存储块 |\n| 分层存储 | 支持 | 依赖对象存储 |\n| 数据压缩 | 支持（基于扩展名） | 支持（全局设置） |\n| 存储加密 | 支持 | 支持 |\n| POSIX 兼容性 | 基本 | 完整 |\n| S3 协议 | 基本 | 基本 |\n| WebDAV 协议 | 支持 | 支持 |\n| HDFS 兼容性 | 基本 | 完整 |\n| CSI 驱动 | 支持 | 支持 |\n| 客户端缓存 | 支持 | 支持 |\n| 集群数据复制 | 支持单向、双向复制模式 | 仅企业版支持单向复制 |\n| 云上数据缓存 | 支持（手动同步） | 仅企业版支持 |\n| 回收站 | 不支持 | 支持 |\n| 运维与监控 | 支持 | 支持 |\n| 发布时间 | 2015.4 | 2021.1 |\n| 主要维护者 | 个人（Chris Lu） | 公司（Juicedata Inc） |\n| 语言 | Go | Go |\n| 开源协议 | Apache License 2.0 | Apache License 2.0 |\n\n## SeaweedFS 技术架构\n\n系统由 3 部分组成：\n\n- **Volume Server**，用于底层存储文件\n- **Master Server**，用于管理集群\n- **Filer**，一个向上提供更多特性的可选组件\n\n![SeaweedFS 系统架构](../../images/seaweedfs_arch_intro.png)\n\nVolume Server 与 Master Server 一并构成文件存储服务：\n\n- Volume Server 专注于数据的写入与读取\n- Master Server 负责管理集群与 Volumes\n\n在读写数据时，SeaweedFS 的实现与 Haystack 相似，用户创建的文件系统（Volume）实际上是一个大磁盘文件，也就是下图的 Superblock。在此 Volume 中，用户写入的所有文件都会被合并到该大磁盘文件中，借用 Haystack 的术语，每一个文件都是“一根针”，needle。\n\n![SeaweedFS Superblock](../../images/seaweedfs_superblock.png)\n\nSeaweedFS 中数据写入和读取流程：\n\n1. 在开始写入数据之前，客户端向 Master Server 发起写入申请。\n2. SeaweedFS 根据当前的数据量返回一个 File ID，这个 ID 由 \\ 三部分构成。在写入的过程中，一并被写入的还有基础的元数据信息（文件长度与 Chunk 等信息）。\n3. 当写入完成之后，调用者需要在一个外部系统（例如 MySQL）中对该文件与返回的 File ID 进行关联保存。\n4. 在读取数据时，由于 Volume 的索引信息已被加载入内存，可以通过 File ID 直接获取文件位置（偏移）的所有信息，因此可以高效地将文件的内容读取出来。\n\n在上述的底层存储服务之上，SeaweedFS 提供了一个名为 Filer 的组件，他对接 Volume Server 与 Master Server，对外提供丰富的功能与特性，如 POSIX 支持、WebDAV、S3 API。与 JuiceFS 相同，Filer 也需要对接一个外部数据库以保存元数据信息。\n\n## JuiceFS 技术架构\n\nJuiceFS 采用元数据与数据分离存储的架构：\n\n- 文件数据本身会被切分保存在对象存储（如 S3）当中\n- 元数据被保存在元数据引擎中，元数据引擎是一个由用户自行选择数据库，如 Redis、MySQL。\n\n客户端连接元数据引擎获取元数据服务，然后将实际数据写入对象存储，实现强一致性分布式文件系统。\n\n![JuiceFS Arch](../../images/juicefs-arch-new.png)\n\nJuiceFS 的架构在[「技术架构」](../architecture.md)有更详细的介绍。\n\n## 架构对比\n\n### 元数据\n\nSeaweedFS 与 JuiceFS 都支持通过外部数据库以存储文件系统的元数据信息：\n\n- SeaweedFS 支持多达 [24](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Stores) 种数据库。\n- JuiceFS 对数据库事务能力要求更高（下方会详细介绍），当前支持了 [3 类共 10 种事务型数据库](../../reference/how_to_set_up_metadata_engine.md)。\n\n### 原子性操作\n\n* JuiceFS 严格确保每一项操作的原子性，因此对于元数据引擎（例如 Redis、MySQL）的事务能力有着较强的要求，因此支持的数据库更少。\n* SeaweedFS 则对操作的原子性保证较弱，目前而言 SeaweedFS 仅在执行 `rename` 操作时启用了部分数据库（SQL、ArangoDB 和 TiKV）的事务，因此对于数据库的事务能力要求较低。同时，由于 SeaweedFS 在 `rename` 操作中拷贝元数据时，未对原目录或文件进行加锁，高负载下可能造成数据丢失。\n\n### 变更日志以及相关功能\n\nSeaweedFS 会为所有的元数据操作生成变更日志（changelog），日志可以被传输、重放，保证数据安全的同时，还能用来实现文件系统数据复制、操作审计等功能。\n\nSeaweedFS 支持在多个集群之间进行文件系统数据复制，存在两种异步数据复制模式：\n\n- 「Active-Active」：此模式中，两个集群都能够参与文件写入并双向同步。如果集群节点数量超过 2，SeaweedFS 的一些操作（如重命名目录）会受到一些限制。\n- 「Active-Passive」：此模式是主从关系，Passive 一方只读。\n\n这两种模式都是通过传递 changelog 再应用的机制实现了不同集群数据间的一致性，对于每一条 changelog，其中会有一个签名信息以保证同一个修改不会被循环多次。\n\nJuiceFS 社区版没有实现变更日志，但可以自行使用元数据引擎和对象存储自身的数据复制能力实现文件系统镜像功能，比方说 [MySQL](https://dev.mysql.com/doc/refman/8.0/en/replication.html) 或 [Redis](https://redis.io/docs/management/replication) 仅支持数据复制，配合上 [S3 的复制对象功能](https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/userguide/replication.html)，就能脱离 JuiceFS 实现类似 SeaweedFS 的 Active-Passive 模式。\n\n顺带一提，JuiceFS 企业版的元数据引擎也是基于变更日志实现，支持[数据复制](https://juicefs.com/docs/zh/cloud/guide/replication)、[镜像文件](https://juicefs.com/docs/zh/cloud/guide/mirror)系统，可以点击对应文档链接以了解更多。\n\n## 存储对比\n\n如前文所述，SeaweedFS 的数据存储由 Volume Server + Master Server 实现，支持小数据块的合并存储、纠删码等特性。而 JuiceFS 的数据存储则是依托于对象存储服务服务，相关的特性也都由对象存储提供。\n\n### 文件拆分\n\nSeaweedFS 与 JuiceFS 都会将文件拆分成若干个小块再持久化到底层的数据系统中：\n\n- SeaweedFS 将文件拆分成 8MB 的块，对于超大文件（超过 8GB），它会将 Chunk 索引也保存到底层的数据系统中。\n- JuiceFS 内部会使用 64MB 的逻辑数据块（Chunk），再拆成 4MB 的 Block 上传至对象存储，这点在[架构文档](../architecture.md#how-juicefs-store-files)中有更详细的介绍。\n\n### 分层存储\n\n对于新创建的 Volume，SeaweedFS 会把数据存储在本地，而对于较旧的 Volume，SeaweedFS 支持将他们上传至云端以达到[冷热数据的分离](https://github.com/seaweedfs/seaweedfs/wiki/Tiered-Storage)。JuiceFS 自身并没有实现分层存储的功能，而是直接使用对象存储提供的分层管理服务，比如 [S3 存储类](https://aws.amazon.com/cn/s3/storage-classes/glacier/?nc1=h_ls)。\n\n### 数据压缩\n\nJuiceFS 支持使用 LZ4 或者 Zstandard 来为所有写入的数据进行压缩，而 SeaweedFS 则是根据写入文件的扩展名、文件类型等信息来选择是否进行压缩。\n\n### 加密\n\n二者均支持加密，包括传输中加密及静态加密：\n\n* SeaweedFS 支持传输中加密与静态加密。在开启了数据加密后，所有写入 Volume Server 的数据都会使用随机的密钥进行加密，而这些对应的随机密钥信息则由维护文件元数据的 Filer 进行管理，详见 [Wiki](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Data-Encryption)。\n* JuiceFS 的加密功能详见[文档](../../security/encryption.md)。\n\n## 客户端协议对比\n\n### POSIX\n\nJuiceFS [完全兼容 POSIX](../../reference/posix_compatibility.md)，而 SeaweedFS 目前[实现了部分的 POSIX 兼容](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount)），功能还持续完善中。\n\n### S3\n\nJuiceFS 实现了 [S3 网关](https://juicefs.com/docs/zh/community/s3_gateway)，因此如果有需要，可以通过 S3 API 直接访问文件系统，也能使用 s3cmd、AWS CLI、MinIO Client（mc）等工具直接管理文件系统。\n\nSeaweedFS 当前[支持部分 S3 API](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API)，覆盖了常用的读写查删等请求，对一些特定的请求（如 Read）还做了功能上的扩展。\n\n### HDFS\n\nJuiceFS [完整兼容 HDFS API](../../deployment/hadoop_java_sdk.md)。包括 Hadoop 2.x 和 Hadoop 3.x，以及 Hadoop 生态系统中的各种组件。SeaweedFS 则是提供了对 HDFS API 的[基础兼容](https://github.com/seaweedfs/seaweedfs/wiki/Hadoop-Compatible-File-System)，一些更加高级的操作如如 truncate、concat、checksum 和扩展属性等则尚未支持。\n\n### CSI 驱动\n\n二者均支持 CSI 驱动，详见：\n\n* [SeaweedFS CSI 驱动](https://github.com/seaweedfs/seaweedfs-csi-driver)\n* [JuiceFS CSI 驱动](https://github.com/juicedata/juicefs-csi-driver)\n\n### WebDAV\n\n二者均支持 WebDAV 协议，详见：\n\n* [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki/WebDAV)\n* [JuiceFS 文档](../../deployment/webdav.md)\n\n## 其他高级功能\n\n### 客户端缓存\n\nSeaweedFS 客户端[具备简单客户端缓存能力](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount)，由于在写作期间未能找到具体文档，可以直接在其[源码](https://github.com/seaweedfs/seaweedfs/blob/master/weed/command/mount.go)中搜索 `cache` 相关字样。\n\nJuiceFS 客户端支持[元数据以及数据缓存](../../guide/cache.md)，提供更丰富的定制空间，允许用户根据自己的应用场景进行调优。\n\n### 对象存储网关\n\nSeaweedFS 可以作为[对象存储的网关（Gateway）](https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage)来使用，可以将对象存储中指定的数据预热到本地，在本地发生的数据修改也会异步同步到对象存储中。\n\n由于 JuiceFS 使用对象存储的方式是将文件进行切分存储，因架构所限，不支持直接作为对象存储的网关或者缓存层。但是在 JuiceFS 企业版，我们开发了单独的功能，为对象存储中已有的数据提供缓存服务，功能类似 SeaweedFS。\n\n### 回收站\n\nJuiceFS 支持并默认开启[回收站](../../security/trash.md)功能，删除的文件会保留指定的时间才删除，避免数据误删，保证数据安全。SeaweedFS 暂不支持此功能。\n\n### 运维\n\n二者均提供完善的运维和排查调优方案：\n\n* JuiceFS 可以通过 [`juicefs stats`](../../administration/fault_diagnosis_and_analysis.md#stats)，[`juicefs profile`](../../administration/fault_diagnosis_and_analysis.md#profile) 来实时观测文件系统性能。也可以通过 [`metrics API`](../../administration/monitoring.md#collect-metrics) 将监控数据接入到 Prometheus，用 Grafana 进行可视化和监控告警。\n* SeaweedFS 可以通过 [`weed shell`](https://github.com/seaweedfs/seaweedfs/wiki/weed-shell) 来交互式执行运维工作，例如查看当前集群状态、列举文件列表。SeaweedFS 同时支持 [push 和 pull 方式](https://github.com/seaweedfs/seaweedfs/wiki/System-Metrics) 对接 Prometheus。\n" }, { "path": "docs/zh_cn/introduction/io_processing.md", "content": "---\ntitle: 读写请求处理流程\nsidebar_position: 3\nslug: /internals/io_processing\ndescription: 本文分别介绍 JuiceFS 的读和写的流程，更进一步的介绍 JuiceFS 读写分块技术在操作系统上的实现过程。\n---\n\n## 写入流程 {#workflow-of-write}\n\nJuiceFS 对大文件会做多级拆分（[JuiceFS 如何存储文件](../introduction/architecture.md#how-juicefs-store-files)），以提高读写效率。在处理写请求时，JuiceFS 先将数据写入 Client 的内存缓冲区，并在其中按 Chunk/Slice 的形式进行管理。Chunk 是根据文件内 offset 按 64 MiB 大小拆分的连续逻辑单元，不同 Chunk 之间完全隔离。每个 Chunk 内会根据应用写请求的实际情况进一步拆分成 Slice；当新的写请求与已有的 Slice 连续或有重叠时，会直接在该 Slice 上进行更新，否则就创建新的 Slice。Slice 是启动数据持久化的逻辑单元，其在 flush 时会先将数据按照默认 4 MiB 大小拆分成一个或多个连续的 Block，并作为最小单元上传到对象存储；然后再更新一次元数据，写入新的 Slice 信息。\n\n显然，在应用顺序写情况下，只需要一个不停增长的 Slice，最后仅 `flush` 一次即可；此时能最大化发挥出对象存储的写入性能。以一次简单的 [JuiceFS 基准测试](../benchmark/performance_evaluation_guide.md)为例，使用 1 MiB IO 顺序写 1 GiB 文件，在不考虑压缩和加密的前提下，数据在各个组件中的形式如下图所示：\n\n![internals-write](../images/internals-write.png)\n\n用 [`juicefs stats`](../reference/command_reference.mdx#stats) 命令记录的指标图，可以直观地看到实时性能数据：\n\n![internals-stats](../images/internals-stats.png)\n\n图中第 1 阶段：\n\n- 对象存储写入的平均 IO 大小为 `object.put / object.put_c = 4 MiB`，等于 Block 的默认大小\n- 元数据事务数与对象存储写入数比例大概为 `meta.txn : object.put_c ~= 1 : 16`，对应 Slice flush 需要的 1 次元数据修改和 16 次对象存储上传，同时也说明了每次 flush 写入的数据量为 4 MiB * 16 = 64 MiB，即 Chunk 的默认大小\n- FUSE 层的平均请求大小为约 `fuse.write / fuse.ops ~= 128 KiB`，与其默认的请求大小限制一致\n\n小文件的写入通常是在文件关闭时被上传到对象存储，对应 IO 大小一般就是文件大小。指标图的第 3 阶段是创建 128 KiB 小文件，可以发现：\n\n- 对象存储 PUT 的大小就是 128 KiB\n- 元数据事务数大致是 PUT 计数的两倍，对应每个文件的一次 Create 和一次 Write\n\n对于这种不足一个 Block Size 的对象，JuiceFS 在上传的同时还会尝试写入到本地[缓存](../guide/cache.md)，来提升后续可能的读请求速度。因此从图中第 3 阶段也可以看到，创建小文件时，本地缓存（blockcache）与对象存储有着同等的写入带宽，而在读取时（第 4 阶段）大部分均在缓存命中，这使得小文件的读取速度看起来特别快。\n\n由于写请求写入客户端内存缓冲区即可返回，因此通常来说 JuiceFS 的 Write 时延非常低（几十微秒级别），真正上传到对象存储的动作由内部自动触发，比如单个 Slice 过大，Slice 数量过多，或者仅仅是在缓冲区停留时间过长等，或应用主动触发，比如关闭文件、调用 `fsync` 等。\n\n缓冲区中的数据只有在被持久化后才能释放，因此当写入并发较大时，如果缓冲区大小不足（默认 300MiB，通过 [`--buffer-size`](../reference/command_reference.mdx#mount-data-cache-options) 调节），或者对象存储性能不佳，读写缓冲区将持续被占用而导致写阻塞。缓冲区大小可以在指标图的 usage.buf 一列中看到。当使用量超过阈值时，JuiceFS Client 会主动为 Write 添加约 10ms 等待时间以减缓写入速度；若已用量超过阈值两倍，则会导致写入暂停直至缓冲区得到释放。因此，在观察到 Write 时延上升以及 Buffer 长时间超过阈值时，通常需要尝试设置更大的 `--buffer-size`。另外，增大上传并发度（[`--max-uploads`](../reference/command_reference.mdx#mount-data-storage-options)，默认 20）也能提升写入到对象存储的带宽，从而加快缓冲区的释放。\n\n### 随机写 {#random-write}\n\nJuiceFS 支持随机写，包括通过 mmap 等进行的随机写。\n\n要知道，Block 是一个不可变对象，这也是因为大部分对象存储服务并不支持修改对象，只能重新上传覆盖。因此发生覆盖写、大文件随机写时，并不会将 Block 重新下载、修改、重新上传（这样会带来严重的读写放大！），而是在新分配或者已有 Slice 中进行写入，以新 Block 的形式上传至对象存储，然后修改对应文件的元数据，在 Chunk 的 Slice 列表中追加新 Slice。后续读取文件时，其实在读取通过合并 Slice 得到的视图。\n\n因此相较于顺序写来说，大文件随机写的情况更复杂：每个 Chunk 内可能存在多个不连续的 Slice，使得一方面数据对象难以达到 4 MiB 大小，另一方面元数据需要多次更新。因此，JuiceFS 在大文件随机写有明显的性能下降。当一个 Chunk 内已写入的 Slice 过多时，会触发碎片清理（Compaction）来尝试合并与清理这些 Slice，来提升读性能。碎片清理以后台任务形式发生，除了系统自动运行，还能通过 [`juicefs gc`](../administration/status_check_and_maintenance.md#gc) 命令手动触发。\n\n### 客户端写缓存 {#client-write-cache}\n\n客户端写缓存，也称为「回写模式」。\n\n如果对数据一致性和可靠性没有极致要求，可以在挂载时添加 `--writeback` 以进一步提写性能。客户端缓存开启后，Slice flush 仅需写到本地缓存目录即可返回，数据由后台线程异步上传到对象存储。换个角度理解，此时本地目录就是对象存储的缓存层。\n\n更详细的介绍请见[「客户端写缓存」](../guide/cache.md#client-write-cache)。\n\n## 读取流程 {#workflow-of-read}\n\nJuiceFS 支持顺序读和随机读（包括基于 mmap 的随机读），在处理读请求时会通过对象存储的 `GetObject` 接口完整读取 Block 对应的对象，也有可能仅仅读取对象中一定范围的数据（比如通过 [S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) 的 `Range` 参数限定读取范围）。与此同时异步地进行预读（通过 [`--prefetch`](../reference/command_reference.mdx#mount) 参数控制预读并发度），预读会将整个对象存储块下载到本地缓存目录，以备后用（如指标图中的第 2 阶段，blockcache 有很高的写入带宽）。显然，在顺序读时，这些提前获取的数据都会被后续的请求访问到，缓存命中率非常高，因此也能充分发挥出对象存储的读取性能。数据流如下图所示：\n\n![internals-read](../images/internals-read.png)\n\n但是对于大文件随机读场景，预读的用途可能不大，反而容易因为读放大和本地缓存的频繁写入与驱逐使得系统资源的实际利用率降低，此时可以考虑用 `--prefetch=0` 禁用预读。考虑到此类场景下，一般的缓存策略很难有足够高的收益，可考虑尽可能提升缓存的整体容量，达到能几乎完全缓存所需数据的效果；或者直接禁用缓存（`--cache-size=0`），并尽可能提高对象存储的读取性能。\n\n小文件的读取则比较简单，通常就是在一次请求里读取完整个文件。由于小文件写入时会直接被缓存起来，因此，之后的读性能非常可观。\n" }, { "path": "docs/zh_cn/reference/_common_options.mdx", "content": "#### 元数据相关参数 {#mount-metadata-options}\n\n|项 | 说明|\n|-|-|\n|`--subdir=value`|挂载指定的子目录，默认挂载整个文件系统。|\n|`--backup-meta=3600`|自动备份元数据到对象存储的间隔时间；单位秒，默认 3600，设为 0 表示不备份。|\n|`--backup-skip-trash` 1.2|备份元数据时跳过回收站中的文件和目录。|\n|`--heartbeat=12`|发送心跳的间隔（单位秒），建议所有客户端使用相同的心跳值 (默认：12)|\n|`--read-only`|只读模式，只允许 lookup 和 read 请求。注意，只读模式隐含了 `--no-bgjob`，因此只读客户端不会运行后台任务。|\n|`--no-bgjob`|禁用后台任务，默认为 false，也就是说客户端会默认运行后台任务。后台任务包含：
  • 清理回收站中过期的文件（在 [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/base.go) 中搜索 `cleanupDeletedFiles` 和 `cleanupTrash`）
  • 清理引用计数为 0 的 Slice（在 [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/base.go) 中搜索 `cleanupSlices`）
  • 清理过期的客户端会话（在 [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/base.go) 中搜索 `CleanStaleSessions`）
特别地，与[企业版](https://juicefs.com/docs/zh/cloud/guide/background-job)不同，社区版碎片合并（Compaction）不受该选项的影响，而是随着文件读写操作，自动判断是否需要合并，然后异步执行（以 Redis 为例，在 [`pkg/meta/base.go`](https://github.com/juicedata/juicefs/blob/main/pkg/meta/redis.go) 中搜索 `compactChunk`）|\n|`--atime-mode=noatime` 1.1|控制如何更新 atime（文件最后被访问的时间）。支持以下模式：
  • `noatime`（默认）：仅在文件创建和主动调用 `SetAttr` 时设置，平时访问与修改文件不影响 atime 值。考虑到更新 atime 需要运行额外的事务，对性能有影响，因此默认关闭。
  • `relatime`：仅在 mtime（文件内容修改时间）或 ctime（文件元数据修改时间）比 atime 新，或者 atime 超过 24 小时没有更新时进行更新。
  • `strictatime`：持续更新 atime
|\n|`--skip-dir-nlink=20` 1.1|跳过更新目录 nlink 前的重试次数 (仅用于 TKV, 0 代表永不跳过) (默认：20)|\n|`--skip-dir-mtime=100ms` 1.2|如果 mtime 差异小于该值（默认值：100ms），则跳过更新目录的属性。|\n|`--sort-dir` 1.3|按名称对目录中的条目进行排序|\n|`--fast-statfs` 1.3|通过使用本地缓存减少元数据访问提升`statfs`性能，准确性会降低（默认：false）|\n\n#### 元数据缓存参数 {#mount-metadata-cache-options}\n\n元数据缓存的介绍和使用，详见[「内核元数据缓存」](../guide/cache.md#kernel-metadata-cache)及[「客户端内存元数据缓存」](../guide/cache.md#client-memory-metadata-cache)。\n\n|项 | 说明|\n|-|-|\n|`--attr-cache=1`|属性缓存过期时间；单位为秒，默认为 1。|\n|`--entry-cache=1`|文件项缓存过期时间；单位为秒，默认为 1。|\n|`--dir-entry-cache=1`|目录项缓存过期时间；单位为秒，默认为 1。|\n|`--open-cache=0`|打开的文件的缓存过期时间，单位为秒，默认为 0，代表关闭该特性。|\n|`--open-cache-limit=value` 1.1|允许缓存的最大文件个数 (软限制，0 代表不限制) (默认：10000)|\n|`--readdir-cache=false` 1.3, only for mount|开启目录项缓存，默认为 false，代表不开启|\n|`--negative-entry-cache=0` 1.3, only for mount|失败 lookup 查询 (返回 ENOENT) 缓存过期时间，默认为 0，代表不缓存|\n\n#### 数据存储参数 {#mount-data-storage-options}\n\n|项 | 说明|\n|-|-|\n|`--storage=file`|对象存储类型 (例如 `s3`、`gs`、`oss`、`cos`) (默认：`\"file\"`，参考[文档](../reference/how_to_set_up_object_storage.md#supported-object-storage)查看所有支持的对象存储类型)|\n|`--bucket=value`|为当前挂载点指定访问对象存储的 Endpoint。|\n|`--storage-class value` 1.1|当前客户端写入数据的存储类型|\n|`--get-timeout=60`|下载一个对象的超时时间；单位为秒 (默认：60)|\n|`--put-timeout=60`|上传一个对象的超时时间；单位为秒 (默认：60)|\n|`--io-retries=10`|网络异常时的重试次数，元数据请求的重试次数也由这个选项控制。如果超过重试次数将会返回 `EIO Input/output error` 错误。（默认：10）|\n|`--max-uploads=20`|上传并发度，默认为 20。对于粒度为 4M 的写入模式，20 并发已经是很高的默认值，在这样的写入模式下，提高写并发往往需要伴随增大 `--buffer-size`, 详见「[读写缓冲区](../guide/cache.md#buffer-size)」。但面对百 K 级别的小随机写，并发量大的时候很容易产生阻塞等待，造成写入速度恶化。如果无法改善应用写模式，对其进行合并，那么需要考虑采用更高的写并发，避免排队等待。|\n|`--max-stage-write=0` 1.2|异步写入数据块到缓存盘的最大并发数，如果达到最大并发数则会直接上传对象存储（此选项仅在启用[「客户端写缓存」](../guide/cache.md#client-write-cache)时有效）（默认值：0，即没有并发限制）|\n|`--max-deletes=10`|删除对象的连接数 (默认：10)|\n|`--upload-limit=0`|上传带宽限制，单位为 Mbps (默认：0)|\n|`--download-limit=0`|下载带宽限制，单位为 Mbps (默认：0)|\n|`--check-storage`1.3|在挂载前测试存储以提前暴露访问问题|\n\n#### 数据缓存相关参数 {#mount-data-cache-options}\n\n|项 | 说明|\n|-|-|\n|`--buffer-size=300`|读写缓冲区的总大小；单位为 MiB (默认：300)。阅读[「读写缓冲区」](../guide/cache.md#buffer-size)了解更多。|\n|`--prefetch=1`|并发预读 N 个块 (默认：1)。阅读[「客户端读缓存」](../guide/cache.md#client-read-cache)了解更多。|\n|`--writeback`|后台异步上传对象，默认为 false。阅读[「客户端写缓存」](../guide/cache.md#client-write-cache)了解更多。|\n|`--upload-delay=0`|启用 `--writeback` 后，可以使用该选项控制数据延迟上传到对象存储，默认为 0 秒，相当于写入后立刻上传。该选项也支持 `s`（秒）、`m`（分）、`h`（时）这些单位。如果在等待的时间内数据被应用删除，则无需再上传到对象存储。如果数据只是临时落盘，可以考虑用该选项节约资源。阅读[「客户端写缓存」](../guide/cache.md#client-write-cache)了解更多。|\n|`--upload-hours` 1.2|启用 `--writeback` 后，只在一天中指定的时间段上传数据块。参数的格式为 `<起始小时>,<结束小时>`（含「起始小时」，但是不含「结束小时」，「起始小时」必须小于或者大于「结束小时」），其中 `<小时>` 的取值范围为 0 到 23。例如 `0,6` 表示只在每天 0:00 至 5:59 之间上传数据块、`23,3` 表示只在每天 23:00 至第二天 2:59 之间上传数据块。|\n|`--cache-dir=value`|本地缓存目录路径；使用 `:`（Linux、macOS）或 `;`（Windows）隔离多个路径 (默认：`$HOME/.juicefs/cache` 或 `/var/jfsCache`)。阅读[「客户端读缓存」](../guide/cache.md#client-read-cache)了解更多。|\n|`--cache-mode value` 1.1|缓存块的文件权限 (默认：\"0600\")|\n|`--cache-size=102400`|缓存对象的总大小；单位为 MiB (默认：102400)。阅读[「客户端读缓存」](../guide/cache.md#client-read-cache)了解更多。|\n|`--cache-items=0` 1.3 |最大缓存项目数 (默认会根据`free-space-ratio`计算最大值)|\n|`--free-space-ratio=0.1`|最小剩余空间比例，默认为 0.1。如果启用了[「客户端写缓存」](../guide/cache.md#client-write-cache)，则该参数还控制着写缓存占用空间。阅读[「客户端读缓存」](../guide/cache.md#client-read-cache)了解更多。|\n|`--cache-partial-only`|仅缓存随机小块读，默认为 false。阅读[「客户端读缓存」](../guide/cache.md#client-read-cache)了解更多。|\n|`--cache-large-write` 1.3|在上传后缓存完整数据块|\n|`--verify-cache-checksum=extend` 1.1|缓存数据一致性检查级别，启用 Checksum 校验后，生成缓存文件时会对数据切分做 Checksum 并记录于文件末尾，供读缓存时进行校验。支持以下级别：
  • `none`：禁用一致性检查，如果本地数据被篡改，将会读到错误数据；
  • `full`（1.3 以前的默认值）：读完整数据块时才校验，适合顺序读场景；
  • `shrink`：对读范围内的切片数据进行校验，校验范围不包含读边界所在的切片（可以理解为开区间），适合随机读场景；
  • `extend`（1.3+ 的默认值）：对读范围内的切片数据进行校验，校验范围同时包含读边界所在的切片（可以理解为闭区间），因此将带来一定程度的读放大，适合对正确性有极致要求的随机读场景。
|\n|`--cache-eviction=2-random` 1.1|缓存逐出策略（`none` 或 `2-random`）（默认值：2-random）|\n|`--cache-scan-interval=1h` 1.1|扫描缓存目录重建内存索引的间隔（以秒为单位）（默认值：1h）|\n|`--cache-expire=0` 1.2|超过设置的时间未被访问的缓存块将会被自动清除（即使 `--cache-eviction` 的值为 `none`，这些缓存块也会被删除），单位为秒，值为 0 表示永不过期（默认值：0）|\n|`--max-readahead` 1.3|最大预读缓冲区大小，单位为 MiB |\n\n#### 监控相关参数 {#mount-metrics-options}\n\n|项 | 说明|\n|-|-|\n|`--metrics=127.0.0.1:9567`|监控数据导出地址，默认为 `127.0.0.1:9567`。|\n|`--custom-labels`|监控指标自定义标签，格式为 `key1:value1;key2:value2` (默认：\"\")|\n|`--consul=127.0.0.1:8500`|Consul 注册中心地址，默认为 `127.0.0.1:8500`。|\n|`--no-usage-report`|不发送使用量信息 (默认：false)|\n\n#### Windows 相关参数 {#mount-windows-options}\n\n|项 | 说明|\n|-|-|\n|`--o=`|可以用于指定 FUSE 的额外挂载参数，具体支持情况由 WinFsp 决定。|\n|`--log=c:/juicefs.log` 1.3|JuiceFS 日志保存路径（仅对后台运行生效）。|\n|`-d` 1.3|是否后台运行。在 Windows 系统下，指定了后台运行后，JuiceFS 将以系统服务运行。（注：运行此参数时，需要有管理员权限，并且同一时间只能挂载一个文件系统）|\n|`--fuse-trace-log=c:/fuse.log` 1.3|用于指定 WinFsp FUSE 层的回调日志。（默认：\"\"）|\n|`--as-root`|这是一个兼容选项，开启此设置后，将会把所有文件 uid,gid 以及写入的身份都映射为 root(uid=0) 用户。|\n|`--show-dot-files` 1.3 |显示`.`开头的文件。默认情况下，这些文件会被设置为隐藏文件。|\n|`--winfsp-threads=16` 1.3|设置 WinFsp 用于处理内核事件的线程数量，默认为 min(CPU 核数 * 2, 16)。|\n|`--report-case` 1.3|配置 JuiceFS 在处理文件名时，是否尽可能上报精确的大小写信息。例如在使用 aaa.txt 打开一个实际为 AAA.txt 的文件名时，JuiceFS 是否向 Windows 内核汇报实际的文件名。（打开此选项可能会对性能有影响）|\n" }, { "path": "docs/zh_cn/reference/command_reference.mdx", "content": "---\ntitle: 命令参考\nsidebar_position: 1\nslug: /command_reference\ndescription: JuiceFS 客户端的所有命令及选项的说明、用法和示例。\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n\nimport CommonOptions from './_common_options.mdx';\n\n在终端输入 `juicefs` 并执行，就能看到所有可用的命令。在每个子命令后面添加 `-h/--help` 并运行，就能获得该命令的详细帮助信息，例如 `juicefs format -h`。\n\n```\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 1.2.0\n\nCOMMANDS:\n ADMIN:\n format Format a volume\n config Change configuration of a volume\n quota Manage directory quotas\n destroy Destroy an existing volume\n gc Garbage collector of objects in data storage\n fsck Check consistency of a volume\n restore restore files from trash\n dump Dump metadata into a JSON file\n load Load metadata from a previously dumped JSON file\n version Show version\n INSPECTOR:\n status Show status of a volume\n stats Show real time performance statistics of JuiceFS\n profile Show profiling of operations completed in JuiceFS\n info Show internal information of a path or inode\n debug Collect and display system static and runtime information\n summary Show tree summary of a directory\n SERVICE:\n mount Mount a volume\n umount Unmount a volume\n gateway Start an S3-compatible gateway\n webdav Start a WebDAV server\n TOOL:\n bench Run benchmarks on a path\n objbench Run benchmarks on an object storage\n warmup Build cache for target directories/files\n rmr Remove directories recursively\n sync Sync between two storages\n clone clone a file or directory without copying the underlying data\n compact Trigger compaction of chunks\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q show warning and errors only (default: false)\n --trace enable trace log (default: false)\n --log-id value append the given log id in log, use \"random\" to use random uuid\n --no-agent disable pprof (:6060) agent (default: false)\n --pyroscope value pyroscope address\n --no-color disable colors (default: false)\n --help, -h show help (default: false)\n --version, -V print version only (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\n## 自动补全 {#auto-completion}\n\n通过加载 [`hack/autocomplete`](https://github.com/juicedata/juicefs/tree/main/hack/autocomplete) 目录下的对应脚本可以启用命令的自动补全，例如：\n\n\n \n\n```shell\nsource hack/autocomplete/bash_autocomplete\n```\n\n \n \n\n```shell\nsource hack/autocomplete/zsh_autocomplete\n```\n\n \n\n\n请注意自动补全功能仅对当前会话有效。如果你希望对所有新会话都启用此功能，请将 `source` 命令添加到 `.bashrc` 或 `.zshrc` 中：\n\n\n \n\n```shell\necho \"source path/to/bash_autocomplete\" >> ~/.bashrc\n```\n\n \n \n\n```shell\necho \"source path/to/zsh_autocomplete\" >> ~/.zshrc\n```\n\n \n\n\n另外，如果你是在 Linux 系统上使用 bash，也可以直接将脚本拷贝到 `/etc/bash_completion.d` 目录并将其重命名为 `juicefs`：\n\n```shell\ncp hack/autocomplete/bash_autocomplete /etc/bash_completion.d/juicefs\nsource /etc/bash_completion.d/juicefs\n```\n\n## 全局选项 {#global-options}\n\n|项 | 说明|\n|-|-|\n|`-q` `--quiet`|仅显示警告及错误日志。|\n|`-v` `--verbose` `--debug`|开启调试日志。|\n|`--trace`|开启比 `--debug` 选项更详细的调试日志。|\n|`--no-agent`|关闭 pprof 代理。|\n|`--pyroscope`|配置 [Pyroscope](https://github.com/pyroscope-io/pyroscope) 地址，如 `http://localhost:4040`。|\n|`--no-color`|关闭日志的颜色。|\n\n## 管理 {#admin}\n\n### `juicefs format` {#format}\n\n创建并格式化文件系统，如果 `META-URL` 中已经存在一个文件系统，不会再次进行格式化。如果文件系统创建后需要调整配置，请使用 [`juicefs config`](#config)。\n\n#### 概览\n\n```shell\njuicefs format [command options] META-URL NAME\n\n# 创建一个简单的测试卷（数据将存储在本地目录中）\njuicefs format sqlite3://myjfs.db myjfs\n\n# 使用 Redis 和 S3 创建卷\njuicefs format redis://localhost myjfs --storage=s3 --bucket=https://mybucket.s3.us-east-2.amazonaws.com\n\n# 使用带有密码的 MySQL 创建卷\njuicefs format mysql://jfs:mypassword@(127.0.0.1:3306)/juicefs myjfs\n# 更安全的方法\nMETA_PASSWORD=mypassword juicefs format mysql://jfs:@(127.0.0.1:3306)/juicefs myjfs\n\n# 创建一个开启配额设置的卷\njuicefs format sqlite3://myjfs.db myjfs --inodes=1000000 --capacity=102400\n\n# 创建一个关闭了回收站的卷\njuicefs format sqlite3://myjfs.db myjfs --trash-days=0\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`META-URL`|用于元数据存储的数据库 URL，详情查看[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)。|\n|`NAME`|文件系统名称。|\n|`--force`|强制覆盖当前的格式化配置，默认为 false。|\n|`--no-update`|不要修改已有的格式化配置，默认为 false。|\n\n#### 数据存储参数 {#format-data-storage-options}\n\n|项 | 说明|\n|-|-|\n|`--storage=file`|对象存储类型，例如 `s3`、`gs`、`oss`、`cos`。默认为 `file`，参考[文档](../reference/how_to_set_up_object_storage.md#supported-object-storage)查看所有支持的对象存储类型。|\n|`--bucket=path`|存储数据的桶路径（默认：`$HOME/.juicefs/local` 或 `/var/jfs`）。|\n|`--access-key=value`|对象存储的 Access Key，也可通过环境变量 `ACCESS_KEY` 设置。查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#aksk)以了解更多。|\n|`--secret-key=value`|对象存储的 Secret Key，也可通过环境变量 `SECRET_KEY` 设置。查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#aksk)以了解更多。|\n|`--session-token=value`|对象存储的临时访问凭证（Session Token），查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#session-token)以了解更多。|\n|`--storage-class=value` 1.1|默认存储类型。|\n\n#### 数据格式参数 {#format-data-format-options}\n\n|项 | 说明|\n|-|-|\n|`--block-size=4M`|块大小，单位为 KiB，默认 4M。4M 是一个较好的默认值，不少对象存储（比如 S3）都将 4M 设为内部的块大小，因此将 JuiceFS block size 设为相同大小，往往也能获得更好的性能。|\n|`--compress=none`|压缩算法，支持 `lz4`、`zstd`、`none`（默认），启用压缩将不可避免地对性能产生一定影响。这两种压缩算法中，`lz4` 提供更好的性能，但压缩比要逊于 `zstd`，他们的具体性能差别具体需要读者自行搜索了解。|\n|`--encrypt-rsa-key=value`|RSA 私钥的路径，查看[数据加密](../security/encryption.md)以了解更多。|\n|`--encrypt-algo=aes256gcm-rsa`|加密算法 (aes256gcm-rsa, chacha20-rsa) (默认：\"aes256gcm-rsa\")|\n|`--hash-prefix`|对于部分对象存储服务，如果对象存储命名路径的键值（key）是连续的，那么坐落在对象存储上的物理数据也将是连续的。在大规模顺序读场景下，这样会带来数据访问热点，让对象存储服务的部分区域访问压力过大。

启用 `--hash-prefix` 将会给每个对象路径命名添加 hash 前缀（用 slice ID 对 256 取模，详见[内部实现](../development/internals.md#object-storage-naming-format)），相当于“打散”对象存储键值，避免在对象存储服务层面创造请求热点。显而易见，由于影响着对象存储块的命名规则，该选项**必须在创建文件系统之初就指定好、不能动态修改。**

目前而言，[AWS S3](https://aws.amazon.com/about-aws/whats-new/2018/07/amazon-s3-announces-increased-request-rate-performance) 已经做了优化，不再需要应用侧的随机对象前缀。而对于其他对象对象存储服务（比如 [COS 就在文档里推荐随机化前缀](https://cloud.tencent.com/document/product/436/13653#.E6.B7.BB.E5.8A.A0.E5.8D.81.E5.85.AD.E8.BF.9B.E5.88.B6.E5.93.88.E5.B8.8C.E5.89.8D.E7.BC.80)），因此，对于这些对象存储，如果文件系统规模庞大，建议启用该选项以提升性能。|\n|`--shards=0`|如果对象存储服务在桶级别设置了限速（或者你使用自建的对象存储服务，单个桶的性能有限），可以将数据块根据名字哈希分散存入 N 个桶中。该值默认为 0，也就是所有数据存入单个桶。当 N 大于 0 时，`bucket` 需要包含 `%d` 占位符，例如 `--bucket=juicefs-%d`。`--shards` 设置无法动态修改，需要提前规划好用量。|\n\n#### 管理参数 {#format-management-options}\n\n|项 | 说明|\n|-|-|\n|`--capacity=0`|容量配额，单位为 GiB，默认为 0 代表不限制。如果启用了[回收站](../security/trash.md)，那么配额大小也将包含回收站文件。|\n|`--inodes=0`|文件数配额，默认为 0 代表不限制。|\n|`--trash-days=1`|文件被删除后，默认会进入[回收站](../security/trash.md)，该选项控制已删除文件在回收站内保留的天数，默认为 1，设为 0 以禁用回收站。|\n|`--enable-acl=true` 1.2|启用[POSIX ACL](../security/posix_acl.md)，该选项启用后暂不支持关闭。|\n\n### `juicefs config` {#config}\n\n修改指定文件系统的配置项。注意更新某些设置以后，客户端未必能立刻生效，需要等待一定时间，具体的等待时间可以通过 [`--heartbeat`](#mount) 选项控制。\n\n#### 概览\n\n```shell\njuicefs config [command options] META-URL\n\n# 显示当前配置\njuicefs config redis://localhost\n\n# 改变目录的配额\njuicefs config redis://localhost --inodes 10000000 --capacity 1048576\n\n# 更改回收站中文件可被保留的最长天数\njuicefs config redis://localhost --trash-days 7\n\n# 限制允许连接的客户端版本\njuicefs config redis://localhost --min-client-version 1.0.0 --max-client-version 1.1.0\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--yes, -y`|对所有提示自动回答 \"yes\" 并以非交互方式运行 (默认值：false)|\n|`--force`|跳过合理性检查并强制更新指定配置项 (默认：false)|\n\n#### 数据存储参数 {#config-data-storage-options}\n\n|项 | 说明|\n|-|-|\n|`--storage=file` 1.1|对象存储类型，例如 `s3`、`gs`、`oss`、`cos`。默认为 `file`，参考[文档](../reference/how_to_set_up_object_storage.md#supported-object-storage)查看所有支持的对象存储类型。|\n|`--bucket=path`|存储数据的桶路径（默认：`$HOME/.juicefs/local` 或 `/var/jfs`）。|\n|`--access-key=value`|对象存储的 Access Key，也可通过环境变量 `ACCESS_KEY` 设置。查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#aksk)以了解更多。|\n|`--secret-key=value`|对象存储的 Secret Key，也可通过环境变量 `SECRET_KEY` 设置。查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#aksk)以了解更多。|\n|`--session-token=value`|对象存储的临时访问凭证（Session Token），查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#session-token)以了解更多。|\n|`--storage-class=value` 1.1|默认存储类型。|\n|`--upload-limit=0`|上传带宽限制，单位为 Mbps (默认：0)|\n|`--download-limit=0`|下载带宽限制，单位为 Mbps (默认：0)|\n\n#### 管理参数 {#config-management-options}\n\n|项 | 说明|\n|-|-|\n|`--capacity value`|容量配额，单位为 GiB|\n|`--inodes value`|文件数配额|\n|`--trash-days value`|文件被自动清理前在回收站内保留的天数|\n|`--enable-acl` 1.2|开启 [POSIX ACL](../security/posix_acl.md)（不支持关闭），同时允许连接的最小客户端版本会提升到 v1.2|\n|`--encrypt-secret`|如果密钥之前以原格式存储，则加密密钥 (默认值：false)|\n|`--min-client-version value` 1.1|允许连接的最小客户端版本|\n|`--max-client-version value` 1.1|允许连接的最大客户端版本|\n|`--dir-stats` 1.1|开启目录统计，这是快速汇总和目录配额所必需的 (默认值：false)|\n\n### `juicefs quota` 1.1 {#quota}\n\n管理目录配额\n\n#### 概览\n\n```shell\njuicefs quota command [command options] META-URL\n\n# 为目录设置配额\njuicefs quota set redis://localhost --path /dir1 --capacity 1 --inodes 100\n\n# 获取目录配额信息\njuicefs quota get redis://localhost --path /dir1\n\n# 列出所有目录配额\njuicefs quota list redis://localhost\n\n# 删除目录配额\njuicefs quota delete redis://localhost --path /dir1\n\n# 检查目录配额的一致性\njuicefs quota check redis://localhost\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`META-URL`|用于元数据存储的数据库 URL，详情查看[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)。|\n|`--path value`|卷中目录的全路径|\n|`--capacity value`|目录空间硬限制，单位 GiB (默认：0)|\n|`--inodes value`|用于硬限制目录 inode 数 (默认：0)|\n|`--repair`|修复不一致配额 (默认：false)|\n|`--strict`|在严格模式下计算目录的总使用量 (注意：对于大目录可能很慢) (默认：false)|\n\n### `juicefs destroy` {#destroy}\n\n销毁一个已经存在的文件系统，将会清空元数据引擎与对象存储中的相关数据。详见[「如何销毁文件系统」](../administration/destroy.md)。\n\n#### 概览\n\n```shell\njuicefs destroy [command options] META-URL UUID\n\njuicefs destroy redis://localhost e94d66a8-2339-4abd-b8d8-6812df737892\n```\n\n#### 参数\n\n| 项 | 说明|\n|-------------------------------------------|-|\n| `--yes, -y` 1.1 |对所有提示自动回答 \"yes\" 并以非交互方式运行 (默认值：false)|\n| `--force` |跳过合理性检查并强制销毁文件系统 (默认：false)|\n\n### `juicefs gc` {#gc}\n\n如果对象存储块因为某种原因，完全脱离了 JuiceFS 的管理，也就是对象存储上依然还存在，但在 JuiceFS 元数据已经不复存在，无法被回收释放，这种现象称作「对象泄漏」。如果你并没有进行任何特殊操作，那么对象泄露通常昭示着 bug，建议提交 [GitHub Issue](https://github.com/juicedata/juicefs/issues/new/choose)。\n\n与此同时，你可以用该命令清理泄漏对象。顺带一提，该命令还能够清理失效的文件碎片。详见[「状态检查 & 维护」](../administration/status_check_and_maintenance.md#gc)。\n\n#### 概览\n\n```shell\njuicefs gc [command options] META-URL\n\n# 只检查，没有更改的能力\njuicefs gc redis://localhost\n\n# 触发所有 slices 的压缩\njuicefs gc redis://localhost --compact\n\n# 删除泄露的对象\njuicefs gc redis://localhost --delete\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--compact`|对所有文件执行碎片合并。|\n|`--delete`|删除泄漏的对象，以及因不完整的 `clone` 命令而产生泄漏的元数据。|\n|`--threads=10`|并发线程数，默认为 10。|\n\n### `juicefs fsck` {#fsck}\n\n检查文件系统一致性。\n\n#### 概览\n\n```shell\njuicefs fsck [command options] META-URL\n\njuicefs fsck redis://localhost\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--path value` 1.1|待检查的 JuiceFS 中的绝对路径|\n|`--repair` 1.1|发现损坏后尽可能修复 (默认：false)|\n|`--recursive, -r` 1.1|递归检查或修复 (默认值：false)|\n|`--sync-dir-stat` 1.1|同步所有目录的状态，即使他们没有损坏 (注意：巨大的文件树可能会花费很长时间) (默认：false)|\n\n### `juicefs restore` 1.1 {#restore}\n\n重新构建回收站文件的树结构，并将它们放回原始目录。如果需要恢复文件存在命名冲突，程序会直接跳过，不会覆盖新创建的文件（注意日志中会有提示）。\n\n#### 概览\n\n```shell\njuicefs restore [command options] META HOUR ...\n\njuicefs restore redis://localhost/1 2023-05-10-01\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--put-back`|将恢复的文件移动到原始目录，面对命名冲突时会直接跳过，不会覆盖已有文件。|\n|`--threads=10`|线程数，默认 10，如果恢复速度慢，增加并发以提速。|\n\n### `juicefs dump` {#dump}\n\n导出元数据。阅读[「元数据备份」](../administration/metadata_dump_load.md#backup)以了解更多。\n\n#### 概览\n\n```shell\njuicefs dump [command options] META-URL [FILE]\n\n# 导出元数据至 meta-dump.json\njuicefs dump redis://localhost meta-dump.json\n\n# 只导出文件系统的一个子目录的元数据\njuicefs dump redis://localhost sub-meta-dump.json --subdir /dir/in/jfs\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`META-URL`|用于元数据存储的数据库 URL，详情查看[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)。|\n|`FILE`|导出文件路径，如果不指定，则会导出到标准输出。如果文件名以 `.gz` 结尾，将会自动压缩。|\n|`--subdir=path`|只导出指定子目录的元数据。|\n|`--keep-secret-key` 1.1|导出对象存储认证信息，默认为 `false`。由于是明文导出，使用时注意数据安全。如果导出文件不包含对象存储认证信息，后续的导入完成后，需要用 [`juicefs config`](#config) 重新配置对象存储认证信息。|\n|`--threads=10` 1.2|并发线程数，默认 10。|\n|`--fast` 1.2|使用更多内存来加速导出。|\n|`--skip-trash` 1.2|跳过回收站中的文件和目录。|\n\n### `juicefs load` {#load}\n\n将元数据导入一个空的文件系统。阅读[「元数据恢复与迁移」](../administration/metadata_dump_load.md#recovery-and-migration)以了解更多。\n\n#### 概览\n\n```shell\njuicefs load [command options] META-URL [FILE]\n\n# 将元数据备份文件 meta-dump.json 导入数据库\njuicefs load redis://127.0.0.1:6379/1 meta-dump.json\n```\n\n#### 参数\n\n| 项 | 说明|\n|------------------------------------------------------------|-|\n| `META-URL` |用于元数据存储的数据库 URL，详情查看[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)。|\n| `FILE` |导入文件路径，如果不指定，则会从标准输入导入。如果文件名以 `.gz` 结尾，将会自动解压。|\n| `--encrypt-rsa-key=path` 1.0.4 |加密所使用的 RSA 私钥文件路径。|\n| `--encrypt-algo=aes256gcm-rsa` 1.0.4 |加密算法，默认为 `aes256gcm-rsa`。|\n\n## 检视 {#inspector}\n\n### `juicefs status` {#status}\n\n显示 JuiceFS 的状态。\n\n#### 概览\n\n```shell\njuicefs status [command options] META-URL\n\njuicefs status redis://localhost\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--session=0, -s 0`|展示指定会话 (SID) 的具体信息 (默认：0)|\n|`--more, -m` 1.1|显示更多的统计信息，可能需要很长时间 (默认值：false)|\n\n### `juicefs stats` {#stats}\n\n展示实时的性能统计信息，阅读[「实时性能监控」](../administration/fault_diagnosis_and_analysis.md#performance-monitor)以了解更多。\n\n#### 概览\n\n```shell\njuicefs stats [command options] MOUNTPOINT\n\njuicefs stats /mnt/jfs\n\n# 更多的指标\njuicefs stats /mnt/jfs -l 1\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--schema=ufmco`|控制输出内容的标题字符串，默认为 `ufmco`，含义如下：
`u`：usage
`f`：FUSE
`m`：metadata
`c`：block cache
`o`：object storage
`g`：Go|\n|`--interval=1`|更新间隔；单位为秒 (默认：1)|\n|`--verbosity=0`|详细级别；通常 0 或 1 已足够 (默认：0)|\n\n### `juicefs profile` {#profile}\n\n展示基于[文件系统访问日志](../administration/fault_diagnosis_and_analysis.md#access-log)的实时监控数据，阅读[「实时性能监控」](../administration/fault_diagnosis_and_analysis.md#performance-monitor)以了解更多。\n\n#### 概览\n\n```shell\njuicefs profile [command options] MOUNTPOINT/LOGFILE\n\n# 监控实时操作\njuicefs profile /mnt/jfs\n\n# 重放访问日志\ncat /mnt/jfs/.accesslog > /tmp/jfs.alog\n# 一段时间后按 Ctrl-C 停止 \"cat\" 命令\njuicefs profile /tmp/jfs.alog\n\n# 分析访问日志并立即打印总统计数据\njuicefs profile /tmp/jfs.alog --interval 0\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--uid=value, -u value`|仅跟踪指定 UIDs (用逗号分隔)|\n|`--gid=value, -g value`|仅跟踪指定 GIDs (用逗号分隔)|\n|`--pid=value, -p value`|仅跟踪指定 PIDs (用逗号分隔)|\n|`--interval=2`|显示间隔；在回放模式中将其设置为 0 可以立即得到整体的统计结果；单位为秒 (默认：2)|\n\n### `juicefs info` {#info}\n\n显示指定路径或 inode 的内部信息。\n\n#### 概览\n\n```shell\njuicefs info [command options] PATH or INODE\n\n# 检查路径\njuicefs info /mnt/jfs/foo\n\n# 检查 inode\ncd /mnt/jfs\njuicefs info -i 100\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--inode, -i`|使用 inode 号而不是路径 (当前目录必须在 JuiceFS 挂载点内) (默认：false)|\n|`--recursive, -r`|递归获取所有子目录的概要信息，当指定一个目录结构很复杂的路径时可能会耗时很长） (默认：false)|\n|`--strict` 1.1|获取准确的目录概要 (注意：巨大的文件树可能会花费很长的时间) (默认：false)|\n|`--raw`|显示内部原始信息 (默认：false)|\n\n### `juicefs debug` 1.1 {#debug}\n\n从运行环境、系统日志等多个维度收集和展示信息，帮助更好地定位错误\n\n#### 概览\n\n```shell\njuicefs debug [command options] MOUNTPOINT\n\n# 收集并展示挂载点 /mnt/jfs 的各类信息\njuicefs debug /mnt/jfs\n\n# 指定输出目录为 /var/log\njuicefs debug --out-dir=/var/log /mnt/jfs\n\n# 收集最后 1000 条日志条目\njuicefs debug --out-dir=/var/log --limit=1000 /mnt/jfs\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--out-dir=./debug/`|结果输出目录，若目录不存在则自动创建，默认为 `./debug/`。|\n|`--limit=value`|收集的日志条目数，从新到旧，若不指定则收集全部条目|\n|`--stats-sec=5`|.stats 文件采样秒数 (默认：5)|\n|`--trace-sec=5`|trace 指标采样秒数 (默认：5)|\n|`--profile-sec=30`|profile 指标采样秒数 (默认：30)|\n\n### `juicefs summary` 1.1 {#summary}\n\n显示目标目录树摘要。\n\n#### 概览\n\n```shell\njuicefs summary [command options] PATH\n\njuicefs summary /mnt/jfs/foo\n\n# 显示最大深度为 5\njuicefs summary --depth 5 /mnt/jfs/foo\n\n# 显示前 20 个 entry\njuicefs summary --entries 20 /mnt/jfs/foo\n\n# 显示准确的结果\njuicefs summary --strict /mnt/jfs/foo\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--depth value, -d value`|显示树的深度 (0 表示只显示根) (默认：2)|\n|`--entries value, -e value`|显示前 N 个 entry (按大小排序)(默认：10)|\n|`--strict`|显示准确的摘要，包括目录和文件 (可能很慢) (默认值：false)|\n|`--csv`|以 CSV 格式打印摘要 (默认：false)|\n\n## 服务 {#service}\n\n### `juicefs mount` {#mount}\n\n挂载一个已经创建的文件系统。\n\nJuiceFS 支持用 root 以及普通用户挂载，但由于权限不同，挂载时所使用的的缓存目录和日志文件等路径会有所区别，详见下方参数说明。\n\n#### 概要\n\n```shell\njuicefs mount [command options] META-URL MOUNTPOINT\n\n# 前台挂载\njuicefs mount redis://localhost /mnt/jfs\n\n# 使用带密码的 redis 后台挂载\njuicefs mount redis://:mypassword@localhost /mnt/jfs -d\n# 更安全的方式\nMETA_PASSWORD=mypassword juicefs mount redis://localhost /mnt/jfs -d\n\n# 将一个子目录挂载为根目录\njuicefs mount redis://localhost /mnt/jfs --subdir /dir/in/jfs\n\n# 开启写缓存（writeback）模式，可以提升写入性能但同时有数据丢失风险\njuicefs mount redis://localhost /mnt/jfs -d --writeback\n\n# 开启只读模式\njuicefs mount redis://localhost /mnt/jfs -d --read-only\n\n# 关闭元数据自动备份\njuicefs mount redis://localhost /mnt/jfs --backup-meta 0\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`META-URL`|用于元数据存储的数据库 URL，详情查看[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)。|\n|`MOUNTPOINT`|文件系统挂载点，例如：`/mnt/jfs`、`Z:`。|\n|`-d, --background`|后台运行，默认为 false。|\n|`--no-syslog`|禁用系统日志，默认为 false。|\n|`--log=path`|后台运行时日志文件的位置 (默认：`$HOME/.juicefs/juicefs.log` 或 `/var/log/juicefs.log`)|\n|`--force`|强制挂载即使挂载点已经被相同的文件系统挂载 (默认值:false)|\n|`--update-fstab` 1.1|新增／更新 `/etc/fstab` 中的条目，如果不存在将会创建一个从 `/sbin/mount.juicefs` 到 JuiceFS 可执行文件的软链接，默认为 false。|\n|`--disable-transparent-hugepage` 1.3| 关闭内核的透明大页（THP），内存紧张时，THP 可能导致进程挂起等待，默认为 false。 |\n\n#### FUSE 相关参数 {#mount-fuse-options}\n\n|项 | 说明|\n|-|-|\n|`--enable-xattr`|启用扩展属性 (xattr) 功能，默认为 false。|\n|`--enable-cap` 1.3|启用 security.capability 扩展属性 (xattr) ，默认为 false。|\n|`--enable-selinux` 1.3|启用 security.selinux 扩展属性 (xattr) ，默认为 false。|\n|`--enable-ioctl` 1.1|启用 ioctl (仅支持 GETFLAGS/SETFLAGS) (默认：false)|\n|`--root-squash value` 1.1|将本地 root 用户 (UID=0) 映射到一个指定用户，如 UID:GID|\n|`--all-squash value` 1.3|将所有用户映射到一个指定用户，如 UID:GID|\n|`--umask value` 1.3 |新文件和新目录的 umask 的八进制格式|\n|`--prefix-internal` 1.1|挂载 JuiceFS 后，挂载点下默认创建 `.stats`, `.accesslog` 等虚拟文件。如果这些内部文件和你的应用发生冲突，可以启用该选项，添加 `.jfs` 前缀到所有内部文件。|\n|`--max-fuse-io=128K` 1.3| fuse 请求最大大小 (默认：128K)|\n|`-o value`|其他 FUSE 选项，详见 [FUSE 挂载选项](../reference/fuse_mount_options.md)|\n\n\n\n\n
\n\n#### {#mount-metadata-options}\n#### {#mount-metadata-cache-options}\n#### {#mount-data-storage-options}\n#### {#mount-data-cache-options}\n#### {#mount-metrics-options}\n\n
\n\n### `juicefs umount` {#umount}\n\n卸载 JuiceFS 文件系统。\n\n#### 概要\n\n```shell\njuicefs umount [command options] MOUNTPOINT\n\njuicefs umount /mnt/jfs\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`-f, --force`|强制卸载一个忙碌的文件系统 (默认：false)|\n|`--flush` 1.1|等待所有暂存块被刷新 (默认值：false)|\n\n### `juicefs gateway` {#gateway}\n\n启动一个 S3 兼容的网关，详见[「配置 JuiceFS S3 网关」](../guide/gateway.md)。\n\n#### 概览\n\n```shell\njuicefs gateway [command options] META-URL ADDRESS\n\nexport MINIO_ROOT_USER=admin\nexport MINIO_ROOT_PASSWORD=12345678\njuicefs gateway redis://localhost localhost:9000\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`META-URL`|用于元数据存储的数据库 URL，详情查看[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)。|\n|`ADDRESS`|S3 网关地址和监听的端口，例如：`localhost:9000`|\n|`--log value` 1.2|网关日志路径|\n|`--access-log=path`|访问日志的路径|\n|`--background, -d` 1.2|后台运行（默认：false）|\n|`--no-banner`| 禁用 MinIO 的启动信息（默认：false）|\n|`--multi-buckets`|使用第一级目录作为存储桶（默认：false）|\n|`--keep-etag`|保留对象上传时的 ETag（默认：false）|\n|`--umask=022`|新文件和新目录的 umask 的八进制格式（默认值：022）|\n|`--object-tag` 1.2|启用对象标签 API|\n|`--domain value` 1.2|虚拟主机样式请求的域|\n|`--refresh-iam-interval=5m` 1.2|重新加载网关 IAM 配置的间隔时间（默认值：5 分钟）|\n\n\n\n### `juicefs webdav` {#webdav}\n\n启动一个 WebDAV 服务，阅读[「配置 WebDAV 服务」](../deployment/webdav.md)以了解更多。\n\n#### 概览\n\n```shell\njuicefs webdav [command options] META-URL ADDRESS\n\njuicefs webdav redis://localhost localhost:9007\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`META-URL`|用于元数据存储的数据库 URL，详情查看[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)。|\n|`ADDRESS`|WebDAV 服务监听的地址与端口，例如：`localhost:9007`|\n|`--cert-file` 1.1|HTTPS 证书文件|\n|`--key-file` 1.1|HTTPS 密钥文件|\n|`--gzip`|通过 gzip 压缩提供的文件（默认值：false）|\n|`--disallowList`|禁止列出目录（默认值：false）|\n|`--enable-proppatch` 1.3|启用 proppatch 方法支持|\n|`--log value` 1.2|WebDAV 日志路径|\n|`--access-log=path`|访问日志的路径|\n|`--background, -d` 1.2|后台运行（默认：false）|\n|`--threads=50, -p 50`1.3|用于删除作业的最大线程数（最大 255 个）|\n\n\n\n## 工具 {#tool}\n\n### `juicefs bench` {#bench}\n\n对指定的路径做基准测试，包括对大文件和小文件的读/写/获取属性操作。详细介绍参考[文档](../benchmark/performance_evaluation_guide.md#juicefs-bench)。\n\n#### 概览\n\n```shell\njuicefs bench [command options] PATH\n\n# 使用 4 个线程运行基准测试\njuicefs bench /mnt/jfs -p 4\n\n# 只运行小文件的基准测试\njuicefs bench /mnt/jfs --big-file-size 0\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--block-size=1`|块大小；单位为 MiB (默认：1)|\n|`--big-file-size=1024`|大文件大小；单位为 MiB (默认：1024)|\n|`--small-file-size=128`|小文件大小；单位为 KiB (默认：128)|\n|`--small-file-count=100`|小文件数量 (默认：100)|\n|`--threads=1, -p 1`|并发线程数 (默认：1)|\n\n### `juicefs objbench` {#objbench}\n\n测试对象存储接口的正确性与基本性能，详细介绍参考[文档](../benchmark/performance_evaluation_guide.md#juicefs-objbench)。\n\n#### 概览\n\n```shell\njuicefs objbench [command options] BUCKET\n\n# 测试 S3 对象存储的基准性能\nACCESS_KEY=myAccessKey SECRET_KEY=mySecretKey juicefs objbench --storage=s3 https://mybucket.s3.us-east-2.amazonaws.com -p 6\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--storage=file`|对象存储类型 (例如 `s3`、`gs`、`oss`、`cos`) (默认：`file`，参考[文档](../reference/how_to_set_up_object_storage.md#supported-object-storage)查看所有支持的对象存储类型)|\n|`--access-key=value`|对象存储的 Access Key，也可通过环境变量 `ACCESS_KEY` 设置。查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#aksk)以了解更多。|\n|`--secret-key=value`|对象存储的 Secret Key，也可通过环境变量 `SECRET_KEY` 设置。查看[如何设置对象存储](../reference/how_to_set_up_object_storage.md#aksk)以了解更多。|\n|`--session-token value` 1.0|对象存储的会话令牌|\n|`--shards`1.3|如果对象存储服务在桶级别设置了限速（或者你使用自建的对象存储服务，单个桶的性能有限），可以将数据块根据名字哈希分散存入 N 个桶中。该值默认为 0，也就是所有数据存入单个桶。当 N 大于 0 时，`bucket` 需要包含 `%d` 占位符，例如 `--bucket=juicefs-%d`。`--shards` 设置无法动态修改，需要提前规划好用量。|\n|`--block-size=4096`|每个 IO 块的大小（以 KiB 为单位）（默认值：4096）|\n|`--big-object-size=1024`|大文件的大小（以 MiB 为单位）（默认值：1024）|\n|`--small-object-size=128`|每个小文件的大小（以 KiB 为单位）（默认值：128）|\n|`--small-objects=100`|小文件的数量（默认值：100）|\n|`--skip-functional-tests`|跳过功能测试（默认值：false）|\n|`--threads=4, -p 4`|上传下载等操作的并发数（默认值：4）|\n\n### `juicefs warmup` {#warmup}\n\n将文件提前下载到缓存，提升后续本地访问的速度。可以指定某个挂载点路径，递归对这个路径下的所有文件进行缓存预热；也可以通过 `--file` 选项指定文本文件，在文本文件中指定需要预热的文件名。\n\n如果需要预热的文件分布在许多不同的目录，推荐将这些文件名保存到文本文件中并用 `--file` 参数传给预热命令，这样做能利用 `warmup` 的并发功能，速度会显著优于多次调用 `juicefs warmup`，在每次调用里传入单个文件。\n\n#### 概览\n\n```shell\njuicefs warmup [command options] [PATH ...]\n\n# 预热目录中的所有文件\njuicefs warmup /mnt/jfs/datadir\n\n# 只预热指定文件\necho '/jfs/f1\n/jfs/f2\n/jfs/f3' > /tmp/filelist.txt\njuicefs warmup -f /tmp/filelist.txt\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--file=value, -f value`|指定一个包含一组路径的文件（每一行为一个文件路径）。|\n|`--threads=50, -p 50`|并发的工作线程数，默认 50。如果带宽不足导致下载失败，需要减少并发度，控制下载速度。|\n|`--background, -b`|后台运行（默认：false）|\n|`--evict` 1.2|逐出已缓存的块|\n|`--check` 1.2|检查数据块是否已缓存|\n\n### `juicefs rmr` {#rmr}\n\n快速删除目录里的所有文件和子目录，效果等同于 `rm -rf`，但该命令直接操纵元数据，不经过内核，所以速度更快。\n\n如果文件系统启用了回收站功能，被删除的文件会进入回收站。详见[「回收站」](../security/trash.md)。\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--skip-trash`1.3|跳过垃圾并直接删除文件（需要 root 权限）|\n|`--threads=50, -p 50`1.3|用于删除作业的最大线程数（最大 255 个）|\n\n#### 概览\n\n```shell\njuicefs rmr PATH ...\n\njuicefs rmr /mnt/jfs/foo\n```\n\n### `juicefs sync` {#sync}\n\n在两个存储之间同步数据，阅读[「数据同步」](../guide/sync.md)以了解更多。\n\n#### 概览\n\n```shell\njuicefs sync [command options] SRC DST\n\n# 从 OSS 同步到 S3\njuicefs sync oss://mybucket.oss-cn-shanghai.aliyuncs.com s3://mybucket.s3.us-east-2.amazonaws.com\n\n# 从 S3 直接同步到 JuiceFS\njuicefs sync s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n\n# 源端：a1/b1, a2/b2, aaa/b1 目标端：empty 同步结果：aaa/b1\njuicefs sync --exclude='a?/b*' s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n\n# 源端：a1/b1, a2/b2, aaa/b1 目标端：empty 同步结果：a1/b1, aaa/b1\njuicefs sync --include='a1/b1' --exclude='a[1-9]/b*' s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n\n# 源端：a1/b1, a2/b2, aaa/b1, b1, b2 目标端：empty 同步结果：b2\njuicefs sync --include='a1/b1' --exclude='a*' --include='b2' --exclude='b?' s3://mybucket.s3.us-east-2.amazonaws.com/ jfs://META-URL/\n```\n\n源路径（`SRC`）和目标路径（`DST`）的格式均为：\n\n```\n[NAME://][ACCESS_KEY:SECRET_KEY[:SESSIONTOKEN]@]BUCKET[.ENDPOINT][/PREFIX]\n```\n\n其中：\n\n- `NAME`：JuiceFS 支持的数据存储类型，比如 `s3`、`oss`，完整列表见[文档](../reference/how_to_set_up_object_storage.md#supported-object-storage)。\n- `ACCESS_KEY` 和 `SECRET_KEY`：访问数据存储所需的密钥信息，特殊字符需要进行 [URL 转义](https://www.w3schools.com/tags/ref_urlencode.ASP)并替换，比如 `/` 需要被替换为 `%2F`。另外，如果不清楚如何获取对象存储的 AKSK，参考[这里](../reference/how_to_set_up_object_storage.md#aksk)。\n- `TOKEN` 用来访问对象存储的 token，部分对象存储支持使用临时的 token 以获得有限时间的权限。\n- `BUCKET[.ENDPOINT]`：数据存储服务的访问地址，不同存储类型格式可能不同（MinIO 目前仅支持路径风格），详见[文档](../reference/how_to_set_up_object_storage.md#supported-object-storage)。\n- `[/PREFIX]`：可选，源路径和目标路径的前缀，可用于限定只同步某些路径中的数据。\n\n#### 选择条件相关参数 {#sync-selection-related-options}\n\n|项 | 说明|\n|-|-|\n|`--files-from` 1.3|仅同步给定文件中记录的对象，其每行内容都是待同步对象的相对路径，如果对象是目录建议以 / 结尾|\n|`--start=KEY, -s KEY, --end=KEY, -e KEY`|提供 KEY 范围，来指定对象存储的 List 范围。|\n|`--end KEY, -e KEY`|同步的最后一个 `KEY`|\n|`--exclude=PATTERN`|排除匹配 `PATTERN` 的 Key。参考[「过滤」](../guide/sync.md#filtering)文档了解如何使用。|\n|`--include=PATTERN`|不排除匹配 `PATTERN` 的 Key，需要与 `--exclude` 选项配合使用。参考[「过滤」](../guide/sync.md#filtering)文档了解如何使用。|\n|`--match-full-path` 1.2|使用「完整路径过滤模式」，默认为 false。参考[「过滤模式」](../guide/sync.md#filtering-mode)文档了解如何使用。|\n|`--max-size=SIZE` 1.2|跳过大小大于 `SIZE` 的文件，单位字节|\n|`--min-size=SIZE` 1.2|跳过大小小于 `SIZE` 的文件，单位字节|\n|`--max-age=DURATION` 1.2|跳过最后修改时间超过 `DURATION` 的文件，单位秒。例如 `--max-age=3600` 表示仅同步在 1 小时内被修改过的文件。|\n|`--min-age=DURATION` 1.2|跳过最后修改时间不超过 `DURATION` 的文件，单位秒。例如 `--min-age=3600` 表示仅同步最后修改时间距离当前时间已经超过 1 小时的文件。|\n|`--start-time=DURATION` 1.3|跳过开始时间之前修改的文件。例如 `2006-01-02 15:04:05`|\n|`--end-time=DURATION` 1.3|跳过结束时间之后修改的文件。例如 `2006-01-02 15:04:05`|\n|`--limit=-1`|限制将要处理的对象的数量，默认为 -1 表示不限制|\n|`--update, -u`|当源文件更新时（`mtime` 更新），覆盖已存在的文件，默认为 false。|\n|`--force-update, -f`|强制覆盖已存在的文件，默认为 false。|\n|`--existing, --ignore-non-existing` 1.1|不创建任何新文件，默认为 false。|\n|`--ignore-existing` 1.1|不更新任何已经存在的文件，默认为 false。|\n\n#### 行为相关参数 {#sync-action-related-options}\n\n|项 | 说明|\n|-|-|\n|`--dirs`|同步目录（包括空目录）。|\n|`--perms`|保留权限设置，默认为 false。|\n|`--links, -l`|将符号链接复制为符号链接，默认为 false，此时会查找并同步符号链接所指向的文件。|\n|`--inplace` 1.2|当源路径的文件被修改时，直接修改目标路径中的同名文件，而不是先在目标路径中写一个临时文件，再将这个临时文件原子重命名到真实的文件名。这个选项只有当 `--update` 选项开启，以及目标路径的存储系统支持原地修改文件（如 JuiceFS、HDFS、NFS）时才有意义，也就是说如果目标路径的存储系统是对象存储开启这个选项是无效的。（默认值：false）|\n|`--delete-src, --deleteSrc`|如果目标存储已经存在，删除源存储的对象。与 rsync 不同，为保数据安全，首次执行时不会删除源存储文件，只有拷贝成功后再次运行时，扫描确认目标存储已经存在相关文件，才会删除源存储文件。|\n|`--delete-dst, --deleteDst`|删除目标存储下的不相关对象。|\n|`--check-all`|校验源路径和目标路径中所有文件的数据完整性，默认为 false。校验方式是基于字节流对比，因此也将带来相应的开销。|\n|`--check-new`|校验新拷贝文件的数据完整性，默认为 false。校验方式是基于字节流对比，因此也将带来相应的开销。|\n|`--check-change` 1.3|校验同步前后的数据是否有变更，默认为 false。校验方式基于文件大小和 mtime，比较轻量。|\n|`--max-failure` 1.3|最大允许失败的文件数（-1 表示无限）|\n|`--dry`|仅打印执行计划，不实际拷贝文件。|\n\n#### 对象存储相关参数 {#sync-storage-related-options}\n\n|项 | 说明|\n|-|-|\n|`--threads=10, -p 10`|并发线程数，默认为 10。|\n|`--list-threads=1` 1.1|并发 `list` 线程数，默认为 1。阅读[并发 `list`](../guide/sync.md#concurrent-list)以了解如何使用。|\n|`--list-depth=1` 1.1|并发 `list` 目录深度，默认为 1。阅读[并发 `list`](../guide/sync.md#concurrent-list)以了解如何使用。|\n|`--no-https`|不要使用 HTTPS，默认为 false。|\n|`--storage-class value` 1.1|目标端的新建文件的存储类型。|\n|`--bwlimit=0`|限制最大带宽，单位 Mbps，默认为 0 表示不限制。|\n\n#### 分布式相关参数 {#sync-cluster-related-options}\n\n|项 | 说明 |\n|-|-|\n|`--manager-addr=ADDR`| 分布式同步模式中，Manager 节点的监听地址，格式：`:[port]`，如果不写端口，则监听随机端口。如果没有该参数，则监听本机随机的 IPv4 地址与随机端口。|\n|`--worker=ADDR,ADDR`| 分布式同步模式中，工作节点列表，使用逗号分隔。|\n\n#### 监控相关参数 {#sync-metrics-related-options}\n\n|项 | 说明|\n|-|-|\n|`--metrics value` 1.2|导出监控指标的地址（默认值：\"127.0.0.1:9567\"）|\n|`--consul value` 1.2|用于注册的 Consul 地址（默认值：\"127.0.0.1:8500\"）|\n\n### `juicefs clone` 1.1 {#clone}\n\n快速在同一挂载点下克隆目录或者文件，只拷贝元数据但不拷贝数据块，因此拷贝速度非常快。更多介绍详见[「克隆文件或目录」](../guide/clone.md)。\n\n#### 概览\n\n```shell\njuicefs clone [command options] SRC DST\n\n# 克隆文件\njuicefs clone /mnt/jfs/file1 /mnt/jfs/file2\n\n# 克隆目录\njuicefs clone /mnt/jfs/dir1 /mnt/jfs/dir2\n\n# 克隆时保留文件的 UID、GID 和 mode\njuicefs clone -p /mnt/jfs/file1 /mnt/jfs/file2\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--preserve, -p`|克隆时默认使用当前用户的 UID 和 GID，而 mode 则使用当前用户的 umask 重新计算获得。如果启用该选项，则保留文件的 UID、GID 和 mode。|\n\n### `juicefs compact` 1.2 {#compact}\n\n对给定的目录执行碎片整理，合并或清理不连续的 slice，从而提升读性能。详细介绍参考[「状态检查和维护」](../administration/status_check_and_maintenance.md)。\n\n#### 概览\n\n```shell\njuicefs compact [command options] PATH\n\n# 对给定目录执行碎片整理\njuicefs compact /mnt/jfs\n```\n\n#### 参数\n\n|项 | 说明|\n|-|-|\n|`--threads, -p`| 并发执行任务的线程数（默认：10） |\n" }, { "path": "docs/zh_cn/reference/fuse_mount_options.md", "content": "---\ntitle: FUSE 挂载选项\nsidebar_position: 5\nslug: /fuse_mount_options\n---\n\nJuiceFS 文件系统为用户提供多种访问方式，FUSE 是其中较为常用的一种，即使用 `juicefs mount` 命令将文件系统挂载到本地的方式。用户可以根据需要添加 FUSE 支持的挂载选项，从而实现更细粒度的控制。\n\n本指南介绍 JuiceFS 常用的 FUSE 挂载选项，有两种添加挂载选项的方式：\n\n1. 手动执行 [`juicefs mount`](../reference/command_reference.mdx#mount) 命令时，通过 `-o` 选项指定，多个选项使用半角逗号分隔。\n\n ```bash\n juicefs mount -d -o allow_other,writeback_cache sqlite3://myjfs.db ~/jfs\n ```\n\n2. Linux 发行版通过 `/etc/fstab` 定义自动挂载时，在 `options` 字段处直接添加选项，多个选项使用半角逗号分隔。\n\n ```\n # \n redis://localhost:6379/1 /jfs juicefs _netdev,writeback_cache 0 0\n ```\n\n## default_permissions\n\nJuiceFS 在挂载时会自动启用该选项，无需显式指定。该选项将启用内核的文件访问权限检查，它会在文件系统之外进行，启用后，内核检查和文件系统检查必须全部成功才允许进一步操作，该选项通常与 `allow_other` 一起使用。\n\n:::tip\n内核执行的是标准的 Unix 权限检查，基于 mode bits、UID/GID、目录所有权。\n:::\n\n## allow_other\n\nFUSE 默认只有挂载文件系统的用户可以访问挂载点中的文件，`allow_other` 选项可以让其他用户也可以访问挂载点上的文件。当 root 用户挂载时，该选项会自动启用（在 [`fuse.go`](https://github.com/juicedata/juicefs/blob/main/pkg/fuse/fuse.go) 搜索 `AllowOther` 字样），无需显式指定。而如果是普通用户挂载，则需要修改 `/etc/fuse.conf`，在该配置文件中开启 `user_allow_other` 配置选项，才能在普通用户挂载时启用 `allow_other`。\n\n## writeback_cache\n\n:::note 注意\n该挂载选项仅在 Linux 3.15 及以上版本内核上支持\n:::\n\nFUSE 支持[「writeback-cache 模式」](https://www.kernel.org/doc/Documentation/filesystems/fuse-io.txt)，这意味着 `write()` 系统调用通常可以非常快速地完成。当频繁写入非常小的数据（如 100 字节左右）时，建议启用此挂载选项。\n\n## user_id 和 group_id\n\n这两个选项用来指定文件系统的所有者 ID 和所有者组 ID（不同于文件或目录的 UID、GID），用以做更高层级的权限校验。如果指定了 allow_other 选项，此选项将失效。用法如 `sudo juicefs mount -o user_id=100,group_id=100`。\n\n## debug\n\n该选项会将低层类库（`go-fuse`）的 Debug 信息输出到 `juicefs.log` 中。\n\n:::note 注意\n该选项会将低层类库（`go-fuse`）的 Debug 信息输出到 `juicefs.log` 中，需要注意的是，该选项与 JuiceFS 客户端的全局 `--debug` 选项不同，前者是输出 `go-fuse` 类库的调试信息，后者是输出 JuiceFS 客户端的调试信息。详情参考文档[故障诊断和分析](../administration/fault_diagnosis_and_analysis.md)。\n:::\n" }, { "path": "docs/zh_cn/reference/how_to_set_up_metadata_engine.md", "content": "---\ntitle: 如何设置元数据引擎\nsidebar_position: 2\nslug: /databases_for_metadata\ndescription: JuiceFS 支持 Redis、TiKV、PostgreSQL、MySQL 等多种数据库作为元数据引擎，本文分别介绍相应的设置和使用方法。\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n:::tip\n`META_PASSWORD` 是 JuiceFS v1.0 新增功能，旧版客户端需要[升级](../administration/upgrade.md)后才能使用。\n:::\n\nJuiceFS 采用数据和元数据分离的存储架构，元数据可以存储在任意支持的数据库中，称为「元数据存储引擎」。JuiceFS 支持众多元数据存储引擎，各个数据库性能、易用性、场景均有区别，具体性能对比可参考[该文档](../benchmark/metadata_engines_benchmark.md)。\n\n## 元数据存储用量 {#storage-usage}\n\n元数据所需的存储空间跟文件名的长度、文件的类型和长度以及扩展属性等相关，无法准确地估计一个文件系统的元数据存空间需求。简单起见，我们可以根据没有扩展属性的单个小文件所需的存储空间来做近似：\n\n- **键值（Key-Value）数据库**（如 Redis、TiKV）：300 字节／文件\n- **关系型数据库**（如 SQLite、MySQL、PostgreSQL）：600 字节／文件\n\n当平均文件更大（超过 64MB），或者文件被频繁修改导致有很多碎片，或者有很多扩展属性，或者平均文件名很长（超过 50 字节），都会导致需要更多的存储空间。\n\n当你需要在两种类型的元数据引擎之间迁移时，就可以据此来估算所需的存储空间。例如，假设你希望将元数据引擎从一个关系型数据库（MySQL）迁移到键值数据库（Redis），如果当前 MySQL 的用量为 30GB，那么目标 Redis 至少需要准备 15GB 以上的内存。反之亦然。\n\n## Redis 兼容数据库\n\n### Redis\n\nJuiceFS 要求使用 4.0 及以上版本的 Redis。JuiceFS 也支持使用 Redis Cluster 作为元数据引擎，但为了避免在 Redis 集群中执行跨节点事务，同一个文件系统的元数据总会坐落于单个 Redis 实例中。\n\n:::tip Redis Cluster 键前缀\n使用 Redis Cluster 时，URL 中的数据库编号会被用作**键前缀**，而不是用于实际的数据库选择（因为 Redis Cluster 仅支持数据库 0）。前缀格式为 `{N}`（例如 `{1}`、`{2}`），使用 Redis 哈希标签（hash tag）确保同一个卷的所有键都被路由到同一个槽（slot）。这使得多个 JuiceFS 文件系统可以共享同一个 Redis Cluster：\n\n```shell\n# 不同的卷使用不同的数据库编号作为键前缀\njuicefs format redis://cluster:6379/1 volume1 # 键前缀为 {1}\njuicefs format redis://cluster:6379/2 volume2 # 键前缀为 {2}\n```\n\n可以使用以下命令在 Redis Cluster 中验证键：\n\n```shell\nredis-cli -c -h -p 6379 keys '{1}*' # 列出前缀为 {1} 的所有键\n```\n\n:::\n\n为了保证元数据安全，JuiceFS 需要 [`maxmemory-policy noeviction`](https://redis.io/docs/reference/eviction/)，否则在启动 JuiceFS 的时候将会尝试将其设置为 `noeviction`，如果设置失败将会打印告警日志。更多可以参考 [Redis 最佳实践](../administration/metadata/redis_best_practices.md)。\n\n#### 创建文件系统\n\n使用 Redis 作为元数据存储引擎时，通常使用以下格式访问数据库：\n\n\n \n\n```\nredis[s]://[:@][:]/\n```\n\n \n \n\n```\nunix://[:@]?db=\n```\n\n \n\n\n其中，`[]` 括起来的是可选项，其它部分为必选项。\n\n- 如果开启了 Redis 的 [TLS](https://redis.io/docs/manual/security/encryption) 特性，协议头需要使用 `rediss://`，否则使用 `redis://`。\n- `` 是 Redis 6.0 之后引入的，如果没有用户名可以忽略，但密码前面的 `:` 冒号需要保留，如 `redis://:@:6379/1`。\n- Redis 监听的默认端口号为 `6379`，如果没有改变默认端口号可以不用填写，如 `redis://:@/1`，否则需要显式指定端口号。\n- Redis 支持多个[逻辑数据库](https://redis.io/commands/select)，请将 `` 替换为实际使用的数据库编号。\n- 如果需要连接 Redis 哨兵（Sentinel），元数据 URL 的格式会稍有不同，具体请参考[「Redis 最佳实践」](../administration/metadata/redis_best_practices.md#数据可用性)。\n- 如果 Redis 的用户名或者密码中包含特殊字符，需要使用单引号进行封闭，避免 shell 进行解释。或者使用环境变量 `REDIS_PASSWORD` 进行传递。\n\n:::tip 提示\n一个 Redis 实例默认可以创建 16 个逻辑数据库，而一个逻辑数据库可以创建一个 JuiceFS 文件系统。也就是说，在默认情况下，你可以使用一个 Redis 实例创建 16 个 JuiceFS 文件系统。需要注意，用于 JuiceFS 的逻辑数据库不要和其他应用共享，否则可能会造成数据混乱。\n:::\n\n例如，创建名为 `pics` 的文件系统，使用 Redis 的 `1` 号数据库存储元数据：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"redis://:mypassword@192.168.1.6:6379/1\" \\\n pics\n```\n\n安全起见，建议使用环境变量 `META_PASSWORD` 或 `REDIS_PASSWORD` 传递数据库密码，例如：\n\n```shell\nexport META_PASSWORD=mypassword\n```\n\n然后就无需在元数据 URL 中设置密码了：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"redis://192.168.1.6:6379/1\" \\\n pics\n```\n\n#### 挂载文件系统\n\n如果需要在多台服务器上共享同一个文件系统，必须确保每台服务器都能访问到存储元数据的数据库。\n\n```shell\njuicefs mount -d \"redis://:mypassword@192.168.1.6:6379/1\" /mnt/jfs\n```\n\n挂载文件系统也支持用 `META_PASSWORD` 或 `REDIS_PASSWORD` 环境变量传递密码：\n\n```shell\nexport META_PASSWORD=mypassword\njuicefs mount -d \"redis://192.168.1.6:6379/1\" /mnt/jfs\n```\n\n#### 设置 TLS\n\nJuiceFS 同时支持 Redis 的 TLS 单向加密认证和 mTLS 双向加密认证连接。通过 TLS 或 mTLS 连接到 Redis 时均使用 `rediss://` 协议头，但是在使用 TLS 单向加密认证时，不需要指定客户端证书和私钥。\n\n:::note\n对 Redis mTLS 功能的支持需要使用 1.1.0 及以上版本的 JuiceFS\n:::\n\n当通过 mTLS 连接 Redis 时，需要提供客户端证书和私钥，以及签发客户端证书的 CA 证书进行连接。在 JuiceFS 中，可以通过以下方式设置 mTLS 需要的客户端证书：\n\n```shell\njuicefs format --storage s3 \\\n ... \\\n \"rediss://192.168.1.6:6379/1?tls-cert-file=/etc/certs/client.crt&tls-key-file=/etc/certs/client.key&tls-ca-cert-file=/etc/certs/ca.crt\"\n pics\n```\n\n上面的示例代码使用 `rediss://` 协议头来开启 mTLS 功能，然后使用以下选项来指定客户端证书的路径：\n\n- `tls-cert-file=` 指定客户端证书的路径\n- `tls-key-file=` 指定客户端密钥的路径\n- `tls-ca-cert-file=` 指定签发客户端证书的 CA 证书路径，它是可选的，如果不指定，客户端会使用系统默认的 CA 证书进行验证。\n- `insecure-skip-verify=true` 可以用来跳过对服务端证书的验证\n\n在 URL 指定选项时，以 `?` 符号开头，使用 `&` 符号来分隔多个选项，例如：`?tls-cert-file=client.crt&tls-key-file=client.key`。\n\n上例中的 `/etc/certs` 只是一个目录，实际使用时请替换为你的证书目录，可以使用相对路径或绝对路径。\n\n### KeyDB\n\n[KeyDB](https://keydb.dev) 是 Redis 的开源分支，在开发上保持与 Redis 主线对齐。KeyDB 在 Redis 的基础上实现了多线程支持、更好的内存利用率和更大的吞吐量，另外还支持 [Active Replication](https://github.com/JohnSully/KeyDB/wiki/Active-Replication)，即 Active Active（双活）功能。\n\n:::note 注意\nKeyDB 的数据复制是异步的，使用 Active Active（双活）功能可能导致数据一致性问题，请务必充分验证、谨慎使用！\n:::\n\n在用于 JuiceFS 元数据存储时，KeyDB 与 Redis 的用法完全一致，这里不再赘述，请参考 [Redis](#redis) 部分使用。\n\n## 键值数据库\n\n### BadgerDB\n\n[BadgerDB](https://github.com/dgraph-io/badger) 是一个 Go 语言开发的嵌入式、持久化的单机 Key-Value 数据库，它的数据库文件存储在本地你指定的目录中。\n\n使用 BadgerDB 作为 JuiceFS 元数据存储引擎时，使用 `badger://` 协议头指定数据库路径。\n\n#### 创建文件系统\n\n无需提前创建 BadgerDB 数据库，直接创建文件系统即可：\n\n```shell\njuicefs format badger://$HOME/badger-data myjfs\n```\n\n上述命令在当前用户的 `home` 目录创建 `badger-data` 作为数据库目录，并以此作为 JuiceFS 的元数据存储。\n\n#### 挂载文件系统\n\n挂载文件系统时需要指定数据库路径：\n\n```shell\njuicefs mount -d badger://$HOME/badger-data /mnt/jfs\n```\n\n:::tip 提示\nBadgerDB 只允许单进程访问，如果需要执行 `gc`、`fsck`、`dump`、`load` 等操作，需要先卸载文件系统。\n:::\n\n### TiKV\n\n[TiKV](https://tikv.org) 是一个分布式事务型的键值数据库，最初作为 PingCAP 旗舰产品 TiDB 的存储层而研发，现已独立开源并从 CNCF 毕业。\n\nTiKV 的测试环境搭建非常简单，使用官方提供的 TiUP 工具即可实现一键部署，具体可参见[这里](https://tikv.org/docs/latest/concepts/tikv-in-5-minutes)。生产环境一般需要至少三个节点来存储三份数据副本，部署步骤可以参考[官方文档](https://tikv.org/docs/latest/deploy/install/install)。\n\n:::note 注意\n建议使用独立部署的 TiKV 5.0+ 集群作为 JuiceFS 的元数据引擎\n:::\n\n#### 创建文件系统\n\n使用 TiKV 作为元数据引擎时，需要使用如下格式来指定参数：\n\n```shell\ntikv://[,...]/\n```\n\n其中 `prefix` 是一个用户自定义的字符串，当多个文件系统或者应用共用一个 TiKV 集群时，设置前缀可以避免混淆和冲突。示例如下：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"tikv://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs\" \\\n pics\n```\n\n#### 设置 TLS\n\n如果需要开启 TLS，可以通过在元数据 URL 后以添加 query 参数的形式设置 TLS 的配置项，目前支持的配置项：\n\n| 配置项 | 值 |\n|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `ca` | CA 根证书，用于用 TLS 连接 TiKV/PD |\n| `cert` | 证书文件路径，用于用 TLS 连接 TiKV/PD |\n| `key` | 私钥文件路径，用于用 TLS 连接 TiKV/PD |\n| `verify-cn` | 证书通用名称，用于验证调用者身份，[详情](https://docs.pingcap.com/zh/tidb/stable/enable-tls-between-components#%E8%AE%A4%E8%AF%81%E7%BB%84%E4%BB%B6%E8%B0%83%E7%94%A8%E8%80%85%E8%BA%AB%E4%BB%BD) |\n\n例子：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"tikv://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs?ca=/path/to/ca.pem&cert=/path/to/tikv-server.pem&key=/path/to/tikv-server-key.pem&verify-cn=CN1,CN2\" \\\n pics\n```\n\n#### 挂载文件系统\n\n```shell\njuicefs mount -d \"tikv://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs\" /mnt/jfs\n```\n\n### etcd\n\n[etcd](https://etcd.io) 是一个高可用高可靠的小规模键值数据库，可以用作 JuiceFS 的元数据存储。\n\n#### 创建文件系统\n\n使用 etcd 作为元数据引擎时，需要使用如下格式来指定 `Meta-URL` 参数：\n\n```\netcd://[user:password@][,...]/\n```\n\n其中 `user` 和 `password` 是当 etcd 开启了用户认证时需要。`prefix` 是一个用户自定义的字符串，当多个文件系统或者应用共用一个 etcd 集群时，设置前缀可以避免混淆和冲突。示例如下：\n\n```shell\njuicefs format etcd://user:password@192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs pics\n```\n\n#### 设置 TLS\n\n如果需要开启 TLS，可以通过在元数据 URL 后以添加 query 参数的形式设置 TLS 的配置项，注意证书文件请使用绝对路径，避免后台挂载时找不到文件。\n\n| 配置项 | 值 |\n|----------------------|--------------|\n| cacert | CA 根证书 |\n| cert | 证书文件路径 |\n| key | 私钥文件路径 |\n| server-name | 服务器名称 |\n| insecure-skip-verify | 1 |\n\n例子：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"etcd://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs?cert=/path/to/ca.pem&cacert=/path/to/etcd-server.pem&key=/path/to/etcd-key.pem&server-name=etcd\" \\\n pics\n```\n\n#### 挂载文件系统\n\n```shell\njuicefs mount -d \"etcd://192.168.1.6:2379,192.168.1.7:2379,192.168.1.8:2379/jfs\" /mnt/jfs\n```\n\n### FoundationDB 1.1\n\n[FoundationDB](https://www.foundationdb.org) 是一个能在多集群服务器上存放大规模结构化数据的分布式数据库。该数据库系统专注于高性能、高可扩展性，且具有不错的容错能力。由于对接 FoundationDB 需要先安装其客户端库，因此 JuiceFS 的发布版本默认不支持，使用前需要自行编译。\n\n#### 编译 JuiceFS\n\n首先安装 FoundationDB 客户端（参考[官方文档](https://apple.github.io/foundationdb/api-general.html#installing-client-binaries)）：\n\n\n \n\n```shell\ncurl -O https://github.com/apple/foundationdb/releases/download/6.3.25/foundationdb-clients_6.3.25-1_amd64.deb\nsudo dpkg -i foundationdb-clients_6.3.25-1_amd64.deb\n```\n\n \n \n\n```shell\ncurl -O https://github.com/apple/foundationdb/releases/download/6.3.25/foundationdb-clients-6.3.25-1.el7.x86_64.rpm\nsudo rpm -Uvh foundationdb-clients-6.3.25-1.el7.x86_64.rpm\n```\n\n \n\n\n然后编译支持 FoundationDB 的 JuiceFS：\n\n```shell\nmake juicefs.fdb\n```\n\n#### 创建文件系统\n\n使用 FoundationDB 作为元数据引擎时，需要使用如下格式来指定 `Meta-URL` 参数：\n\n```uri\nfdb://?prefix=\n```\n\n其中 `` 为 FoundationDB 的配置文件路径，用来连接 FoundationDB 服务端。`` 是一个用户自定义的字符串，当多个文件系统或者应用共用一个 FoundationDB 集群时，设置前缀可以避免混淆和冲突。示例如下：\n\n```shell\njuicefs.fdb format \\\n --storage s3 \\\n ... \\\n \"fdb:///etc/foundationdb/fdb.cluster?prefix=jfs\" \\\n pics\n```\n\n#### 设置 TLS\n\n如果需要开启 TLS，大体步骤如下，详细信息请参考[官方文档](https://apple.github.io/foundationdb/tls.html)。\n\n##### 使用 OpenSSL 生成 CA 证书\n\n```shell\nopenssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout private.key -out cert.crt\ncat cert.crt private.key > fdb.pem\n```\n\n##### 配置 TLS\n\n| 命令行选项 | 客户端选项 | 环境变量 | 目的 |\n|------------------------|--------------------|----------------------------|------------------------------------|\n| `tls_certificate_file` | `TLS_cert_path` | `FDB_TLS_CERTIFICATE_FILE` | 可以从中加载本地证书的文件的路径 |\n| `tls_key_file` | `TLS_key_path` | `FDB_TLS_KEY_FILE` | 从中加载私钥的文件的路径 |\n| `tls_verify_peers` | `TLS_verify_peers` | `FDB_TLS_VERIFY_PEERS` | 用于验证对等证书和会话的字节字符串 |\n| `tls_password` | `TLS_password` | `FDB_TLS_PASSWORD` | 表示用于解密私钥的密码的字节字符串 |\n| `tls_ca_file` | `TLS_ca_path` | `FDB_TLS_CA_FILE` | 包含要信任的 CA 证书的文件的路径 |\n\n##### 配置服务端 TLS\n\n可以在 `foundationdb.conf` 或者环境变量中配置 TLS 参数，配置文件如下（重点在 `[foundationdb.4500]` 配置中）。\n\n```ini title=\"foundationdb.conf\"\n[fdbmonitor]\nuser = foundationdb\ngroup = foundationdb\n\n[general]\nrestart-delay = 60\n## by default, restart-backoff = restart-delay-reset-interval = restart-delay\n# initial-restart-delay = 0\n# restart-backoff = 60\n# restart-delay-reset-interval = 60\ncluster-file = /etc/foundationdb/fdb.cluster\n# delete-envvars =\n# kill-on-configuration-change = true\n\n## Default parameters for individual fdbserver processes\n[fdbserver]\ncommand = /usr/sbin/fdbserver\n#public-address = auto:$ID\n#listen-address = public\ndatadir = /var/lib/foundationdb/data/$ID\nlogdir = /var/log/foundationdb\n# logsize = 10MiB\n# maxlogssize = 100MiB\n# machine-id =\n# datacenter-id =\n# class =\n# memory = 8GiB\n# storage-memory = 1GiB\n# cache-memory = 2GiB\n# metrics-cluster =\n# metrics-prefix =\n\n[fdbserver.4500]\npublic-address = 127.0.0.1:4500:tls\nlisten-address = public\ntls_certificate_file = /etc/foundationdb/fdb.pem\ntls_ca_file = /etc/foundationdb/cert.crt\ntls_key_file = /etc/foundationdb/private.key\ntls_verify_peers= Check.Valid=0\n\n[backup_agent]\ncommand = /usr/lib/foundationdb/backup_agent/backup_agent\nlogdir = /var/log/foundationdb\n\n[backup_agent.1]\n```\n\n除此之外还需将 `fdb.cluster` 中的地址加上 `:tls` 后缀，`fdb.cluster` 示例如下：\n\n```uri title=\"fdb.cluster\"\nu6pT9Jhl:ClZfjAWM@127.0.0.1:4500:tls\n```\n\n##### 配置客户端\n\n在客户端机器上需要配置 TLS 参数以及 `fdb.cluster`，`fdbcli` 同理。\n\n通过 `fdbcli` 连接时：\n\n```shell\nfdbcli --tls_certificate_file=/etc/foundationdb/fdb.pem \\\n --tls_ca_file=/etc/foundationdb/cert.crt \\\n --tls_key_file=/etc/foundationdb/private.key \\\n --tls_verify_peers=Check.Valid=0\n```\n\n通过 API 连接时（`fdbcli` 也适用）：\n\n```shell\nexport FDB_TLS_CERTIFICATE_FILE=/etc/foundationdb/fdb.pem \\\nexport FDB_TLS_CA_FILE=/etc/foundationdb/cert.crt \\\nexport FDB_TLS_KEY_FILE=/etc/foundationdb/private.key \\\nexport FDB_TLS_VERIFY_PEERS=Check.Valid=0\n```\n\n#### 挂载文件系统\n\n```shell\njuicefs.fdb mount -d \\\n \"fdb:///etc/foundationdb/fdb.cluster?prefix=jfs\" \\\n /mnt/jfs\n```\n\n## SQL 数据库\n\n每个数据库默认只能被一个 JuiceFS 文件系统所使用，如果想要多个文件系统共享一个数据库，可以通过在 META-URL 中添加 `table_prefix` 1.3 query 参数\n为不同的文件系统添加不同的表名来前缀实现。例如：`mysql://user:mypassword@(192.168.1.6:3306)/juicefs?table_prefix=volume1`\n\n### MySQL\n\n[MySQL](https://www.mysql.com) 是受欢迎的开源关系型数据库之一，常被作为 Web 应用程序的首选数据库。\n\n>[MariaDB](https://mariadb.org) 是 MySQL 的一个开源分支，由 MySQL 原始开发者维护并保持开源，与 MySQL 高度兼容，在设置元数据引擎方法上也没有任何差别。\n>\n>[OceanBase](https://www.oceanbase.com)是一款自主研发的分布式关系型数据库，专为处理海量数据和高并发事务而设计，具备高性能、强一致性和高可用性的特点。同时，OceanBase 与 MySQL 高度兼容，在设置元数据引擎方法上也没有任何差别。\n\n#### 创建文件系统\n\n使用 MySQL 作为元数据存储引擎时，需要提前手动创建数据库，通常使用以下格式访问数据库：\n\n\n \n\n```\nmysql://[:]@(:3306)/\n```\n\n \n \n\n```\nmysql://[:]@unix()/\n```\n\n \n\n\n:::note 注意\n\n1. 不要漏掉 URL 两边的 `()` 括号\n2. 密码中的特殊字符不需要进行 url 编码\n\n:::\n\n例如：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs\" \\\n pics\n```\n\n更安全的做法是可以通过环境变量 `META_PASSWORD` 传递数据库密码：\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user@(192.168.1.6:3306)/juicefs\" \\\n pics\n```\n\n要连接到启用 TLS 的 MySQL 服务器，请传递 `tls=true` 参数（或 `tls=skip-verify` 如果使用自签名证书）：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs?tls=true\" \\\n pics\n```\n\n要启用 JuiceFS 到 MySQL 服务器建立连接的超时控制，请传递 `timeout=5s` 参数（时间可自定义）：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs?timeout=5s\" \\\n pics\n```\n\n:::note 注意\n\n设置建立连接超时，在 JuiceFS 和 MySQL 间出现网络故障场景时，能明确控制对 JuiceFS 文件系统进行读写的阻塞时间，从而可控的对网络故障进行响应。\n\n:::\n\n#### 挂载文件系统\n\n```shell\njuicefs mount -d \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\n挂载文件系统也支持用 `META_PASSWORD` 环境变量传递密码：\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs mount -d \"mysql://user@(192.168.1.6:3306)/juicefs\" /mnt/jfs\n```\n\n要连接到启用 TLS 的 MySQL 服务器，请传递 `tls=true` 参数（或 `tls=skip-verify` 如果使用自签名证书）：\n\n```shell\njuicefs mount -d \"mysql://user:mypassword@(192.168.1.6:3306)/juicefs?tls=true\" /mnt/jfs\n```\n\n更多 MySQL 数据库的地址格式示例，[点此查看](https://github.com/Go-SQL-Driver/MySQL/#examples)。\n\n### PostgreSQL\n\n[PostgreSQL](https://www.postgresql.org) 是功能强大的开源关系型数据库，有完善的生态和丰富的应用场景，也可以用来作为 JuiceFS 的元数据引擎。\n\n许多云计算平台都提供托管的 PostgreSQL 数据库服务，也可以按照[使用向导](https://www.postgresqltutorial.com/postgresql-getting-started)自己部署一个。\n\n其他跟 PostgreSQL 协议兼容的数据库（比如 CockroachDB 等) 也可以这样使用。\n\n#### 创建文件系统\n\n使用 PostgreSQL 作为元数据引擎时，需要提前手动创建数据库，使用如下的格式来指定参数：\n\n\n \n\n```\npostgres://[username][:]@[:5432]/[?parameters]\n```\n\n \n \n\n```\npostgres://[username][:]@/?host=[¶meters]\n```\n\n \n\n\n其中，`[]` 括起来的是可选项，其它部分为必选项。\n\n例如：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"postgres://user:mypassword@192.168.1.6:5432/juicefs\" \\\n pics\n```\n\n更安全的做法是可以通过环境变量 `META_PASSWORD` 传递数据库密码：\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs format \\\n --storage s3 \\\n ... \\\n \"postgres://user@192.168.1.6:5432/juicefs\" \\\n pics\n```\n\n:::note 说明\n\n1. JuiceFS 默认使用的 public [schema](https://www.postgresql.org/docs/current/ddl-schemas.html) ，如果要使用非 `public schema`，需要在连接字符串中指定 `search_path` 参数，例如 `postgres://user:mypassword@192.168.1.6:5432/juicefs?search_path=pguser1`\n2. 如果 `public schema` 并非是 PostgreSQL 服务端配置的 `search_path` 中第一个命中的，则必须在连接字符串中明确设置 `search_path` 参数\n3. `search_path` 连接参数原生可以设置为多个 schema，但是目前 JuiceFS 仅支持设置一个。`postgres://user:mypassword@192.168.1.6:5432/juicefs?search_path=pguser1,public` 将被认为不合法\n4. 密码中的特殊字符需要进行 url 编码，例如 `|` 需要编码为`%7C`。\n\n:::\n\n#### 挂载文件系统\n\n```shell\njuicefs mount -d \"postgres://user:mypassword@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\n挂载文件系统也支持用 `META_PASSWORD` 环境变量传递密码：\n\n```shell\nexport META_PASSWORD=\"mypassword\"\njuicefs mount -d \"postgres://user@192.168.1.6:5432/juicefs\" /mnt/jfs\n```\n\n#### 故障排除\n\nJuiceFS 客户端默认采用 SSL 加密连接 PostgreSQL，如果连接时报错 `pq: SSL is not enabled on the server` 说明数据库没有启用 SSL。可以根据业务场景为 PostgreSQL 启用 SSL 加密，也可以在元数据 URL 中添加参数禁用加密验证：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"postgres://user@192.168.1.6:5432/juicefs?sslmode=disable\" \\\n pics\n```\n\n元数据 URL 中还可以附加更多参数，[查看详情](https://pkg.go.dev/github.com/lib/pq#hdr-Connection_String_Parameters)。\n\n### SQLite\n\n[SQLite](https://sqlite.org) 是全球广泛使用的小巧、快速、单文件、可靠、全功能的单文件 SQL 数据库引擎。\n\nSQLite 数据库只有一个文件，创建和使用都非常灵活，用它作为 JuiceFS 元数据存储引擎时无需提前创建数据库文件，可以直接创建文件系统：\n\n```shell\njuicefs format \\\n --storage s3 \\\n ... \\\n \"sqlite3://my-jfs.db\" \\\n pics\n```\n\n以上命令会在当前目录创建名为 `my-jfs.db` 的数据库文件，请 **务必妥善保管** 这个数据库文件！\n\n挂载文件系统：\n\n```shell\njuicefs mount -d \"sqlite3://my-jfs.db\" /mnt/jfs/\n```\n\n请注意数据库文件的位置，如果不在当前目录，则需要指定数据库文件的绝对路径，比如：\n\n```shell\njuicefs mount -d \"sqlite3:///home/herald/my-jfs.db\" /mnt/jfs/\n```\n\n也可以在连接字符串中添加参数来支持 [PRAGMA 语句](https://www.sqlite.org/pragma.html)：\n\n```shell\n\"sqlite3://my-jfs.db?cache=shared&_busy_timeout=5000\"\n```\n\n更多 SQLite 数据库的地址格式示例，请参考 [Go-SQLite3 Driver](https://github.com/mattn/go-sqlite3#connection-string)。\n\n:::note 注意\n由于 SQLite 是一款单文件数据库，在不做特殊共享设置的情况下，只有数据库所在的主机可以访问它。对于多台服务器共享同一文件系统的情况，需要使用 Redis 或 MySQL 等数据库。\n:::\n" }, { "path": "docs/zh_cn/reference/how_to_set_up_object_storage.md", "content": "---\ntitle: 如何设置对象存储\nsidebar_position: 3\ndescription: JuiceFS 以对象存储作为数据存储，本文介绍 JuiceFS 支持的对象存储以及相应的配置和使用方法。\n---\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n通过阅读 [JuiceFS 的技术架构](../introduction/architecture.md)可以了解到，JuiceFS 是一个数据与元数据分离的分布式文件系统，以对象存储作为主要的数据存储，以 Redis、PostgreSQL、MySQL 等数据库作为元数据存储。\n\n## 存储选项 {#storage-options}\n\n在创建 JuiceFS 文件系统时，设置数据存储一般涉及以下几个选项：\n\n- `--storage` 指定文件系统要使用的存储类型，例如：`--storage s3`。\n- `--bucket` 指定存储访问地址，例如：`--bucket https://myjuicefs.s3.us-east-2.amazonaws.com`。\n- `--access-key` 和 `--secret-key` 指定访问存储时的身份认证信息。\n\n例如，以下命令使用 Amazon S3 对象存储创建文件系统：\n\n```shell\njuicefs format --storage s3 \\\n --bucket https://myjuicefs.s3.us-east-2.amazonaws.com \\\n --access-key abcdefghijklmn \\\n --secret-key nmlkjihgfedAcBdEfg \\\n redis://192.168.1.6/1 \\\n myjfs\n```\n\n## 其他选项 {#other-options}\n\n在执行 `juicefs format` 或 `juicefs mount` 命令时，可以在 `--bucket` 选项中以 URL 参数的形式设置一些特别的选项，比如 `https://myjuicefs.s3.us-east-2.amazonaws.com?tls-insecure-skip-verify=true` 中的 `tls-insecure-skip-verify=true` 即为跳过 HTTPS 请求的证书验证环节。\n\n客户端证书也受支持，因为它们通常用于 mTLS 连接，例如：\n`https://myjuicefs.s3.us-east-2.amazonaws.com?ca-certs=./path/to/ca&ssl-cert=./path/to/cert&ssl-key=./path/to/privatekey`\n\n## 配置数据分片（Sharding） {#enable-data-sharding}\n\n创建文件系统时，可以通过 [`--shards`](../reference/command_reference.mdx#format-data-format-options) 选项定义多个 Bucket 作为文件系统的底层存储。这样一来，系统会根据文件名哈希值将文件分散到多个 Bucket 中。数据分片技术可以将大规模数据并发写的负载分散到多个 Bucket 中，从而提高写入性能。\n\n启用数据分片功能需要注意以下事项：\n\n- 只能使用同一种对象存储下的多个 bucket\n- `--shards` 选项接受一个 0～256 之间的整数，表示将文件分散到多少个 Bucket 中。默认值为 0，表示不启用数据分片功能。\n- 需要使用整型数字通配符 `%d` 或许 `%x` 之类指定用户生成 bucket 的 endpoint 的字符串，例如 `\"http://192.168.1.18:9000/myjfs-%d\"`，可以按照这样的格式预先创建 bucket，也可以在创建文件系统时由 JuiceFS 客户端自动创建；\n- 数据分片在创建时设定，创建完毕不允许修改。不可增加或减少 bucket，也不可以取消 shards 功能。\n\n例如，以下命令创建了一个数据分片为 4 的文件系统：\n\n```shell\njuicefs format --storage s3 \\\n --shards 4 \\\n --bucket \"https://myjfs-%d.s3.us-east-2.amazonaws.com\" \\\n ...\n```\n\n执行上述命令后，JuiceFS 客户端会创建 4 个 bucket，分别为 `myjfs-0`、`myjfs-1`、`myjfs-2` 和 `myjfs-3`。\n\n## Access Key 和 Secret Key {#aksk}\n\n一般而言，对象存储通过 Access Key ID 和 Access Key Secret 验证用户身份，对应到 JuiceFS 文件系统就是 `--access-key` 和 `--secret-key` 这两个选项（或者简称为 AK、SK）。\n\n创建文件系统时除了使用 `--access-key` 和 `--secret-key` 两个选项显式指定，更安全的做法是通过 `ACCESS_KEY` 和 `SECRET_KEY` 环境变量传递密钥信息，例如：\n\n```shell\nexport ACCESS_KEY=abcdefghijklmn\nexport SECRET_KEY=nmlkjihgfedAcBdEfg\njuicefs format --storage s3 \\\n --bucket https://myjuicefs.s3.us-east-2.amazonaws.com \\\n redis://192.168.1.6/1 \\\n myjfs\n```\n\n公有云通常允许用户创建 IAM（Identity and Access Management）角色，例如：[AWS IAM 角色](https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/id_roles.html) 或 [阿里云 RAM 角色](https://help.aliyun.com/document_detail/93689.html)，可将角色分配给 VM 实例。如果云服务器实例已经拥有读写对象存储的权限，则无需再指定 `--access-key` 和 `--secret-key`。\n\n## 使用临时访问凭证 {#session-token}\n\n永久访问凭证一般有两个部分：Access Key 和 Secret Key，而临时访问凭证一般包括 3 个部分：Access Key、Secret Key 与 token，并且临时访问凭证具有过期时间，一般在几分钟到几个小时之间。\n\n### 如何获取临时凭证 {#how-to-get-temporary-credentials}\n\n不同云厂商的获取方式不同，一般是需要已具有相应权限用户的 Access Key、Secret Key 以及代表临时访问凭证的权限边界的 ARN 作为参数请求访问云服务厂商的 STS 服务器来获取临时访问凭证。这个过程一般可以由云厂商提供的 SDK 简化操作。比如 Amazon S3 获取临时凭证方式可以参考这个[链接](https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/id_credentials_temp_request.html)，阿里云 OSS 获取临时凭证方式可以参考这个[链接](https://help.aliyun.com/document_detail/100624.html)。\n\n### 如何使用临时访问凭证设置对象存储 {#how-to-set-up-object-storage-with-temporary-access-credentials}\n\n使用临时凭证的方式与使用永久凭证差异不大，在格式化文件系统时，将临时凭证的 Access Key、Secret Key、token 分别通过 `--access-key`、`--secret-key`、`--session-token` 设置值即可。例如：\n\n```bash\njuicefs format \\\n --storage oss \\\n --access-key xxxx \\\n --secret-key xxxx \\\n --session-token xxxx \\\n --bucket https://bucketName.oss-cn-hangzhou.aliyuncs.com \\\n redis://localhost:6379/1 \\\n test1\n```\n\n由于临时凭证很快就会过期，所以关键在于格式化文件系统以后，如何在临时凭证过期前更新 JuiceFS 正在使用的临时凭证。一次凭证更新过程分为两步：\n\n1. 在临时凭证过期前，申请好新的临时凭证；\n2. 无需停止正在运行的 JuiceFS，直接使用 `juicefs config Meta-URL --access-key xxxx --secret-key xxxx --session-token xxxx` 命令热更新访问凭证。\n\n新挂载的客户端会直接使用新的凭证，已经在运行的所有客户端也会在一分钟内更新自己的凭证。整个更新过程不会影响正在运行的业务。由于临时凭证过期时间较短，所以以上步骤需要**长期循环执行**才能保证 JuiceFS 服务可以正常访问到对象存储。\n\n## 内网和外网 Endpoint {#internal-and-public-endpoint}\n\n通常情况下，对象存储服务提供统一的 URL 进行访问，但云平台会同时提供内网和外网通信线路，比如满足条件的同平台云服务会自动解析通过内网线路访问对象存储，这样不但时延更低，而且内网通信产生的流量是免费的。\n\n另外，一些云计算平台也区分内外网线路，但没有提供统一访问 URL，而是分别提供内网 Endpoint 和外网 Endpoint 地址。\n\nJuiceFS 对这种区分内网外地址的对象存储服务也做了灵活的支持，对于共享同一个文件系统的场景，在满足条件的服务器上通过内网 Endpoint 访问对象存储，其他计算机通过外网 Endpoint 访问，可以这样使用：\n\n- **创建文件系统时**：`--bucket` 建议使用内网 Endpoint 地址；\n- **挂载文件系统时**：对于不满足内网线路的客户端，可以通过 `--bucket` 指定外网 Endpoint 地址。\n\n使用内网 Endpoint 创建文件系统可以确保性能更好、延时更低，对于无法通过内网访问的客户端，可以在挂载文件系统时通过 `--bucket` 指定外网 Endpoint 进行挂载访问。\n\n## 存储类 1.1 {#storage-class}\n\n对象存储通常支持多种存储类，如标准存储、低频访问存储、归档存储。不同的存储类会有不同的价格及服务可用性，你可以在创建 JuiceFS 文件系统时通过 [`--storage-class`](../reference/command_reference.mdx#format-data-storage-options) 选项设置默认的存储类，或者在挂载 JuiceFS 文件系统时通过 [`--storage-class`](../reference/command_reference.mdx#mount-data-storage-options) 选项设置一个新的存储类。请查阅你所使用的对象存储的用户手册了解应该如何设置 `--storage-class` 选项的值（如 [Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-StorageClass)）。\n\n:::note 注意\n当使用某些存储类（如归档、深度归档）时，数据无法立即访问，需要提前恢复数据并等待一段时间之后才能访问。\n:::\n\n:::note 注意\n当使用某些存储类（如低频访问）时，会有最小计费单位，读取数据也可能会产生额外的费用，请查阅你所使用的对象存储的用户手册了解详细信息。\n:::\n\n## 使用代理 {#using-proxy}\n\n如果客户端所在的网络环境受防火墙策略或其他因素影响需要通过代理访问外部的对象存储服务，使用的操作系统不同，相应的代理设置方法也不同，请参考相应的用户手册进行设置。\n\n以 Linux 为例，可以通过创建 `http_proxy` 和 `https_proxy` 环境变量设置代理：\n\n```shell\nexport http_proxy=http://localhost:8035/\nexport https_proxy=http://localhost:8035/\njuicefs format \\\n --storage s3 \\\n ... \\\n myjfs\n```\n\n## 支持的存储服务 {#supported-object-storage}\n\n如果你希望使用的存储类型不在列表中，欢迎提交需求 [issue](https://github.com/juicedata/juicefs/issues)。\n\n| 名称 | 值 |\n|:-------------------------------------------:|:----------:|\n| [Amazon S3](#amazon-s3) | `s3` |\n| [Google 云存储](#google-cloud) | `gs` |\n| [Azure Blob 存储](#azure-blob-存储) | `wasb` |\n| [Backblaze B2](#backblaze-b2) | `b2` |\n| [IBM 云对象存储](#ibm-云对象存储) | `ibmcos` |\n| [Oracle 云对象存储](#oracle-云对象存储) | `s3` |\n| [Scaleway](#scaleway) | `scw` |\n| [DigitalOcean Spaces](#digitalocean-spaces) | `space` |\n| [Wasabi](#wasabi) | `wasabi` |\n| [Storj DCS](#storj-dcs) | `s3` |\n| [Vultr 对象存储](#vultr-对象存储) | `s3` |\n| [Cloudflare R2](#r2) | `s3` |\n| [阿里云 OSS](#阿里云-oss) | `oss` |\n| [腾讯云 COS](#腾讯云-cos) | `cos` |\n| [华为云 OBS](#华为云-obs) | `obs` |\n| [百度云 BOS](#百度-bos) | `bos` |\n| [火山引擎 TOS](#volcano-engine-tos) | `tos` |\n| [金山云 KS3](#金山云-ks3) | `ks3` |\n| [青云 QingStor](#青云-qingstor) | `qingstor` |\n| [七牛云 Kodo](#七牛云-kodo) | `qiniu` |\n| [天翼云 OOS](#天翼云-oos) | `oos` |\n| [移动云 EOS](#移动云-eos) | `eos` |\n| [京东云 OSS](#京东云-oss) | `s3` |\n| [优刻得 US3](#优刻得-us3) | `ufile` |\n| [Ceph RADOS](#ceph-rados) | `ceph` |\n| [Ceph RGW](#ceph-rgw) | `s3` |\n| [Gluster](#gluster) | `gluster` |\n| [Swift](#swift) | `swift` |\n| [MinIO](#minio) | `minio` |\n| [WebDAV](#webdav) | `webdav` |\n| [HDFS](#hdfs) | `hdfs` |\n| [Apache Ozone](#apache-ozone) | `s3` |\n| [Redis](#redis) | `redis` |\n| [TiKV](#tikv) | `tikv` |\n| [etcd](#etcd) | `etcd` |\n| [SQLite](#sqlite) | `sqlite3` |\n| [MySQL](#mysql) | `mysql` |\n| [PostgreSQL](#postgresql) | `postgres` |\n| [本地磁盘](#本地磁盘) | `file` |\n| [SFTP/SSH](#sftp) | `sftp` |\n| [NFS](#nfs) | `nfs` |\n\n### Amazon S3\n\nS3 支持[两种风格的 endpoint URI](https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/userguide/VirtualHosting.html)：「虚拟托管类型」和「路径类型」。区别如下：\n\n- 虚拟托管类型：`https://.s3..amazonaws.com`\n- 路径类型：`https://s3..amazonaws.com/`\n\n其中 `` 要替换成实际的区域代码，比如：美国西部（俄勒冈）的区域代码为 `us-west-2`。[点此查看](https://docs.aws.amazon.com/zh_cn/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions)所有的区域代码。\n\n:::note 注意\nAWS 中国的用户，应使用 `amazonaws.com.cn` 域名。相应的区域代码信息[点此查看](https://docs.amazonaws.cn/aws/latest/userguide/endpoints-arns.html)。\n:::\n\n:::note 注意\n如果 S3 的桶具有公共访问权限（支持匿名访问），请将 `--access-key` 设置为 `anonymous`。\n:::\n\nJuiceFS 中可选择任意一种风格来指定存储桶的地址，例如：\n\n\n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://.s3..amazonaws.com \\\n ... \\\n myjfs\n```\n\n \n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://s3..amazonaws.com/ \\\n ... \\\n myjfs\n```\n\n \n\n\n你也可以将 `--storage` 设置为 `s3` 用来连接 S3 兼容的对象存储，比如：\n\n\n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n \n \n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https:/// \\\n ... \\\n myjfs\n```\n\n \n\n\n:::tip 提示\n所有 S3 兼容的对象存储服务其 `--bucket` 选项的格式为 `https://.` 或者 `https:///`，默认的 `region` 为 `us-east-1`，当需要不同的 `region` 的时候，可以通过环境变量 `AWS_REGION` 或者 `AWS_DEFAULT_REGION` 手动设置。\n:::\n\n### Google 云存储 {#google-cloud}\n\nGoogle 云采用 [IAM](https://cloud.google.com/iam/docs/overview) 管理资源的访问权限，通过对[服务账号](https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-gcloud)授权，可以对云服务器、对象存储的访问权限进行精细化的控制。\n\n对于归属于同一服务账号的云服务器和对象存储，只要该账号赋予了相关资源的访问权限，创建 JuiceFS 文件系统时无需提供身份验证信息，云平台会自行完成鉴权。\n\n对于要从谷歌云平台外部访问对象存储的情况，比如要在本地计算机上使用 Google 云存储创建 JuiceFS 文件系统，则需要配置认证信息。由于 Google 云存储并不使用 Access Key ID 和 Access Key Secret，而是通过服务账号的 JSON 密钥文件验证身份。\n\n请参考[「以服务帐号身份进行身份验证」](https://cloud.google.com/docs/authentication/production)为服务账号创建 JSON 密钥文件并下载到本地计算机，通过 `GOOGLE_APPLICATION_CREDENTIALS` 环境变量定义密钥文件的路径，例如：\n\n```shell\nexport GOOGLE_APPLICATION_CREDENTIALS=\"$HOME/service-account-file.json\"\n```\n\n可以把创建环境变量的命令写入 `~/.bashrc` 或 `~/.profile` 让 Shell 在每次启动时自动设置。\n\n配置了传递密钥信息的环境变量以后，在本地和在 Google 云服务器上创建文件系统的命令是完全相同的。例如：\n\n```bash\njuicefs format \\\n --storage gs \\\n --bucket [.region] \\\n ... \\\n myjfs\n```\n\n可以看到，命令中无需包含身份验证信息，客户端会通过前面环境变量设置的 JSON 密钥文件完成对象存储的访问鉴权。同时，由于 bucket 名称是 [全局唯一](https://cloud.google.com/storage/docs/naming-buckets#considerations) 的，创建文件系统时，`--bucket` 选项中只需指定 bucket 名称即可。\n\n### Azure Blob 存储\n\n使用 Azure Blob 存储作为 JuiceFS 的数据存储，请先 [查看文档](https://docs.microsoft.com/zh-cn/azure/storage/common/storage-account-keys-manage) 了解如何查看存储帐户的名称和密钥，它们分别对应 `--access-key` 和 `--secret-key` 选项的值。\n\n`--bucket` 选项的设置格式为 `https://.`，请将其中的 `` 替换为实际的 Blob 容器的名称，将 `` 替换为 `core.windows.net`（Azure 全球）或 `core.chinacloudapi.cn`（Azure 中国）。例如：\n\n```bash\njuicefs format \\\n --storage wasb \\\n --bucket https://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n除了使用 `--access-key` 和 `--secret-key` 选项之外，你也可以使用 [连接字符串](https://docs.microsoft.com/zh-cn/azure/storage/common/storage-configure-connection-string) 并通过 `AZURE_STORAGE_CONNECTION_STRING` 环境变量进行设定。例如：\n\n```bash\n# Use connection string\nexport AZURE_STORAGE_CONNECTION_STRING=\"DefaultEndpointsProtocol=https;AccountName=XXX;AccountKey=XXX;EndpointSuffix=core.windows.net\"\njuicefs format \\\n --storage wasb \\\n --bucket https:// \\\n ... \\\n myjfs\n```\n\n:::note 注意\n对于 Azure 中国用户，`EndpointSuffix` 的值为 `core.chinacloudapi.cn`。\n:::\n\n### Backblaze B2\n\n使用 Backblaze B2 作为 JuiceFS 的数据存储，需要先创建 [application key](https://www.backblaze.com/b2/docs/application_keys.html)，**Application Key ID** 和 **Application Key** 分别对应 Access Key 和 Secret Key。\n\nBackblaze B2 支持两种访问接口：B2 原生 API 和 S3 兼容 API。\n\n#### B2 原生 API\n\n存储类型应设置为 `b2`，`--bucket` 只需设置 bucket 名称。例如：\n\n```bash\njuicefs format \\\n --storage b2 \\\n --bucket \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n#### S3 兼容 API\n\n存储类型应设置为 `s3`，`--bucket` 应指定完整的 bucket 地址。例如：\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://s3.eu-central-003.backblazeb2.com/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### IBM 云对象存储\n\n使用 IBM 云对象存储创建 JuiceFS 文件系统，你首先需要创建 [API key](https://cloud.ibm.com/docs/account?topic=account-manapikey) 和 [instance ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID)。**API key** 和 **instance ID** 分别对应 Access Key 和 Secret Key。\n\nIBM 云对象存储为每一个区域提供了 `公网` 和 `内网` 两种 [endpoint 地址](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-endpoints)，你可以根据实际需要选用。例如：\n\n```bash\njuicefs format \\\n --storage ibmcos \\\n --bucket https://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Oracle 云对象存储\n\nOracle 云对象存储支持 S3 兼容的形式进行访问，详细请参考[官方文档](https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm)。\n\n该对象存储的 `endpoint` 格式为：`${namespace}.compat.objectstorage.${region}.oraclecloud.com`，例如：\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Scaleway\n\n使用 Scaleway 对象存储作为 JuiceFS 数据存储，请先 [查看文档](https://www.scaleway.com/en/docs/generate-api-keys) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的设置格式为 `https://.s3..scw.cloud`，请将其中的 `` 替换成实际的区域代码，例如：荷兰阿姆斯特丹的区域代码是 `nl-ams`。[点此查看](https://www.scaleway.com/en/docs/object-storage-feature/#-Core-Concepts) 所有可用的区域代码。\n\n```bash\njuicefs format \\\n --storage scw \\\n --bucket https://.s3..scw.cloud \\\n ... \\\n myjfs\n```\n\n### DigitalOcean Spaces\n\n使用 DigitalOcean Spaces 作为 JuiceFS 数据存储，请先 [查看文档](https://www.digitalocean.com/community/tutorials/how-to-create-a-digitalocean-space-and-api-key) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的设置格式为 `https://..digitaloceanspaces.com`，请将其中的 `` 替换成实际的区域代码，例如：`nyc3`。[点此查看](https://www.digitalocean.com/docs/spaces/#regional-availability) 所有可用的区域代码。\n\n```bash\njuicefs format \\\n --storage space \\\n --bucket https://..digitaloceanspaces.com \\\n ... \\\n myjfs\n```\n\n### Wasabi\n\n使用 Wasabi 作为 JuiceFS 数据存储，请先 [查看文档](https://wasabi-support.zendesk.com/hc/en-us/articles/360019677192-Creating-a-Root-Access-Key-and-Secret-Key) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的设置格式为 `https://.s3..wasabisys.com`，请将其中的 `` 替换成实际的区域代码，例如：US East 1 (N. Virginia) 的区域代码为 `us-east-1`。[点此查看](https://wasabi-support.zendesk.com/hc/en-us/articles/360.15.26031-What-are-the-service-URLs-for-Wasabi-s-different-regions-) 所有可用的区域代码。\n\n```bash\njuicefs format \\\n --storage wasabi \\\n --bucket https://.s3..wasabisys.com \\\n ... \\\n myjfs\n```\n\n:::note 注意\nTokyo (ap-northeast-1) 区域的用户，查看 [这篇文档](https://wasabi-support.zendesk.com/hc/en-us/articles/360039372392-How-do-I-access-the-Wasabi-Tokyo-ap-northeast-1-storage-region-) 了解 endpoint URI 的设置方法。\n:::\n\n### Storj DCS\n\n使用 Storj DCS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://docs.storj.io/api-reference/s3-compatible-gateway) 了解如何创建 Access Key 和 Secret Key。\n\nStorj DCS 兼容 AWS S3，存储类型使用 `s3` ，`--bucket` 格式为 `https://gateway..storjshare.io/`。`` 为存储区域，目前 DCS 有三个可用存储区域：us1、ap1 和 eu1。\n\n```shell\njuicefs format \\\n --storage s3 \\\n --bucket https://gateway..storjshare.io/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n:::caution 特别提示\n因为 Storj DCS 的 [ListObjects](https://github.com/storj/gateway-st/blob/main/docs/s3-compatibility.md#listobjects) API 并非完全 S3 兼容（返回结果没有实现排序功能），所以 JuiceFS 的部分功能无法使用，比如 `juicefs gc`，`juicefs fsck`，`juicefs sync`，`juicefs destroy`。另外，使用 `juicefs mount` 时需要关闭[元数据自动备份](../administration/metadata_dump_load.md#backup-automatically)功能，即加上 `--backup-meta 0`。\n:::\n\n### Vultr 对象存储\n\nVultr 的对象存储兼容 S3 API，存储类型使用 `s3`，`--bucket` 格式为 `https://..vultrobjects.com/`。例如：\n\n```shell\njuicefs format \\\n --storage s3 \\\n --bucket https://.ewr1.vultrobjects.com/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n访问对象存储的 API 密钥可以在 [管理控制台](https://my.vultr.com/objectstorage) 中找到。\n\n### Cloudflare R2 {#r2}\n\nR2 是 Cloudflare 的对象存储服务，提供 S3 兼容的 API，因此用法与 Amazon S3 基本一致。请参照[文档](https://developers.cloudflare.com/r2/data-access/s3-api/tokens)了解如何创建 Access Key 和 Secret Key。\n\n```shell\njuicefs format \\\n --storage s3 \\\n --bucket https://.r2.cloudflarestorage.com/myjfs \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n对于生产环境，建议通过 `ACCESS_KEY` 和 `SECRET_KEY` 环境变量传递密钥信息，例如：\n\n```shell\nexport ACCESS_KEY=\nexport SECRET_KEY=\njuicefs format \\\n --storage s3 \\\n --bucket https://.r2.cloudflarestorage.com/myjfs \\\n ... \\\n myjfs\n```\n\n:::caution 特别提示\n因为 Cloudflare R2 的 `ListObjects` API 并非完全 S3 兼容（返回结果没有实现排序功能），所以 JuiceFS 的部分功能无法使用，比如 `juicefs gc`、`juicefs fsck`、`juicefs sync`、`juicefs destroy`。另外，使用 `juicefs mount` 时需要关闭[元数据自动备份](../administration/metadata_dump_load.md#backup-automatically)功能，即加上 `--backup-meta 0`。\n:::\n\n### 阿里云 OSS\n\n使用阿里云 OSS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://help.aliyun.com/document_detail/38738.html) 了解如何创建 Access Key 和 Secret Key。如果你已经创建了 [RAM 角色](https://help.aliyun.com/document_detail/93689.html) 并指派给了云服务器实例，则在创建文件系统时可以忽略 `--access-key` 和 `--secret-key` 选项。\n\n阿里云也支持使用 [Security Token Service (STS)](https://help.aliyun.com/document_detail/100624.html) 作为 OSS 的临时访问身份验证。如果你要使用 STS，请设置 `ALICLOUD_ACCESS_KEY_ID`、`ALICLOUD_ACCESS_KEY_SECRET` 和 `SECURITY_TOKEN` 环境变量，不要设置 `--access-key` and `--secret-key` 选项。例如：\n\n```bash\n# Use Security Token Service (STS)\nexport ALICLOUD_ACCESS_KEY_ID=XXX\nexport ALICLOUD_ACCESS_KEY_SECRET=XXX\nexport SECURITY_TOKEN=XXX\njuicefs format \\\n --storage oss \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n阿里云 OSS 为每个区域都提供了 `公网` 和 `内网` [endpoint 链接](https://help.aliyun.com/document_detail/31834.html)，你可以根据实际的场景选用。\n\n如果你是在阿里云的服务器上创建文件系统，可以在 `--bucket` 选项中直接指定 bucket 名称。例如：\n\n```bash\n# 在阿里云中运行\njuicefs format \\\n --storage oss \\\n --bucket \\\n ... \\\n myjfs\n```\n\n### 腾讯云 COS\n\n使用腾讯云 COS 作为 JuiceFS 数据存储，Bucket 名称格式为 `-`，即需要在 bucket 名称后面指定 `APPID`，[点此查看](https://cloud.tencent.com/document/product/436/13312) 如何获取 `APPID` 。\n\n`--bucket` 选项的完整格式为 `https://-.cos..myqcloud.com`，请将 `` 替换成你实际使用的存储区域，例如：上海的区域代码为 `ap-shanghai`。[点此查看](https://cloud.tencent.com/document/product/436/6224) 所有可用的区域代码。例如：\n\n```bash\njuicefs format \\\n --storage cos \\\n --bucket https://-.cos..myqcloud.com \\\n ... \\\n myjfs\n```\n\n如果你是在腾讯云的服务器上创建文件系统，可以在 `--bucket` 选项中直接指定 bucket 名称。例如：\n\n```bash\n# 在腾讯云中运行\njuicefs format \\\n --storage cos \\\n --bucket - \\\n ... \\\n myjfs\n```\n\n### 华为云 OBS\n\n使用华为云 OBS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://support.huaweicloud.com/usermanual-ca/zh-cn_topic_0046606340.html) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的格式为 `https://.obs..myhuaweicloud.com`，请将 `` 替换成你实际使用的存储区域，例如：北京一的区域代码为 `cn-north-1`。[点此查看](https://developer.huaweicloud.com/endpoint?OBS) 所有可用的区域代码。例如：\n\n```bash\njuicefs format \\\n --storage obs \\\n --bucket https://.obs..myhuaweicloud.com \\\n ... \\\n myjfs\n```\n\n如果是你在华为云的服务器上创建文件系统，可以在 `--bucket` 直接指定 bucket 名称。例如：\n\n```bash\n# 在华为云中运行\njuicefs format \\\n --storage obs \\\n --bucket \\\n ... \\\n myjfs\n```\n\n### 百度 BOS\n\n使用百度云 BOS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://cloud.baidu.com/doc/Reference/s/9jwvz2egb) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的格式为 `https://..bcebos.com`，请将 `` 替换成你实际使用的存储区域，例如：北京的区域代码为 `bj`。[点此查看](https://cloud.baidu.com/doc/BOS/s/Ck1rk80hn#%E8%AE%BF%E9%97%AE%E5%9F%9F%E5%90%8D%EF%BC%88endpoint%EF%BC%89) 所有可用的区域代码。例如：\n\n```bash\njuicefs format \\\n --storage bos \\\n --bucket https://..bcebos.com \\\n ... \\\n myjfs\n```\n\n如果你是在百度云的服务器上创建文件系统，可以在 `--bucket` 直接指定 bucket 名称。例如：\n\n```bash\n# 在百度云中运行\njuicefs format \\\n --storage bos \\\n --bucket \\\n ... \\\n myjfs\n```\n\n### 火山引擎 TOS 1.0.3 {#volcano-engine-tos}\n\n使用火山引擎 TOS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://www.volcengine.com/docs/6291/65568) 了解如何创建 Access Key 和 Secret Key。\n\n火山引擎 TOS 为每个区域都提供了公网和内网 [endpoint 链接](https://www.volcengine.com/docs/6349/107356)，你可以根据实际的场景选用。\n\n```bash\njuicefs format \\\n --storage tos \\\n --bucket https://.\\\n ... \\\n myjfs\n```\n\n### 金山云 KS3\n\n使用金山云 KS3 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://docs.ksyun.com/documents/1386) 了解如何创建 Access Key 和 Secret Key。\n\n金山云 KS3 为每个区域都提供了公网和内网 [endpoint 链接](https://docs.ksyun.com/documents/6761)，你可以根据实际的场景选用。\n\n```bash\njuicefs format \\\n --storage ks3 \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### 青云 QingStor\n\n使用青云 QingStor 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://docsv3.qingcloud.com/storage/object-storage/api/practices/signature/#%E8%8E%B7%E5%8F%96-access-key) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的格式为 `https://..qingstor.com`，请将 `` 替换成你实际使用的存储区域，例如：北京 3-A 的区域代码为 `pek3a`。[点此查看](https://docs.qingcloud.com/qingstor/#%E5%8C%BA%E5%9F%9F%E5%8F%8A%E8%AE%BF%E9%97%AE%E5%9F%9F%E5%90%8D) 所有可用的区域代码。例如：\n\n```bash\njuicefs format \\\n --storage qingstor \\\n --bucket https://..qingstor.com \\\n ... \\\n myjfs\n```\n\n:::note 注意\n所有 QingStor 兼容的对象存储服务其 `--bucket` 选项的格式为 `http://.`。\n:::\n\n### 七牛云 Kodo\n\n使用七牛云 Kodo 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://developer.qiniu.com/af/kb/1479/how-to-access-or-locate-the-access-key-and-secret-key) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的格式为 `https://.s3-.qiniucs.com`，请将 `` 替换成你实际使用的存储区域，例如：中国东部的区域代码为 `cn-east-1`。[点此查看](https://developer.qiniu.com/kodo/4088/s3-access-domainname) 所有可用的区域代码。例如：\n\n```bash\njuicefs format \\\n --storage qiniu \\\n --bucket https://.s3-.qiniucs.com \\\n ... \\\n myjfs\n```\n\n### 天翼云 OOS\n\n使用天翼云 OOS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://www.ctyun.cn/help2/10000101/10473683) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的格式为 `https://.`，例如：\n\n```bash\njuicefs format \\\n --storage oos \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### 移动云 EOS\n\n使用移动云 EOS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://ecloud.10086.cn/op-help-center/doc/article/24501) 了解如何创建 Access Key 和 Secret Key。\n\n移动云 EOS 为每个区域都提供了 `公网` 和 `内网` [endpoint 链接](https://ecloud.10086.cn/op-help-center/doc/article/40956)，你可以根据实际的场景选用。例如：\n\n```bash\njuicefs format \\\n --storage eos \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### 京东云 OSS\n\n使用京东云 OSS 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://docs.jdcloud.com/cn/account-management/accesskey-management) 了解如何创建 Access Key 和 Secret Key。\n\n`--bucket` 选项的格式为 `https://..jdcloud-oss.com`，请将 `` 替换成你实际使用的存储区域，区域代码[点此查看](https://docs.jdcloud.com/cn/object-storage-service/oss-endpont-list) 。例如：\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket https://..jdcloud-oss.com \\\n ... \\\n myjfs\n```\n\n### 优刻得 US3\n\n使用优刻得 US3 作为 JuiceFS 数据存储，请先参照 [这篇文档](https://docs.ucloud.cn/uai-censor/access/key) 了解如何创建 Access Key 和 Secret Key。\n\n优刻得 US3（原名 UFile）为每个区域都提供了 `公网` 和 `内网` [endpoint 链接](https://docs.ucloud.cn/ufile/introduction/region)，你可以根据实际的场景选用。例如：\n\n```bash\njuicefs format \\\n --storage ufile \\\n --bucket https://. \\\n ... \\\n myjfs\n```\n\n### Ceph RADOS\n\n:::note\nJuiceFS v1.0 使用的 `go-ceph` 库版本为 v0.4.0，其支持的 Ceph 最低版本为 Luminous（v12.2.*）。\nJuiceFS v1.1 使用的 `go-ceph` 库版本为 v0.18.0，其支持的 Ceph 最低版本为 Octopus（v15.2.*）。\n使用前请确认 JuiceFS 与使用的 Ceph 和 `librados` 版本是否匹配，详见 [`go-ceph`](https://github.com/ceph/go-ceph#supported-ceph-versions)、[`librados`](https://docs.ceph.com/en/quincy/rados/api/librados-intro/)。\n:::\n\n[Ceph 存储集群](https://docs.ceph.com/en/latest/rados) 具有消息传递层协议，该协议使客户端能够与 Ceph Monitor 和 Ceph OSD 守护程序进行交互。[`librados`](https://docs.ceph.com/en/latest/rados/api/librados-intro) API 使您可以与这两种类型的守护程序进行交互：\n\n- [Ceph Monitor](https://docs.ceph.com/en/latest/rados/configuration/common/#monitors) 维护群集映射的主副本\n- [Ceph OSD Daemon (OSD)](https://docs.ceph.com/en/latest/rados/configuration/common/#osds) 将数据作为对象存储在存储节点上\n\nJuiceFS 支持使用基于 `librados` 的本地 Ceph API。您需要分别安装 `librados` 库并重新编译 `juicefs` 二进制文件。\n\n首先安装 `librados`，建议使用匹配你的 Ceph 版本的 `librados`，例如 Ceph 版本是 Octopus（v15.2.x），那么 `librados` 也建议使用 v15.2.x 版本。\n\n\n \n\n```bash\nsudo apt-get install librados-dev\n```\n\n \n \n\n```bash\nsudo yum install librados2-devel\n```\n\n \n\n\n然后为 Ceph 编译 JuiceFS（要求 Go 1.20+ 和 GCC 5.4+）：\n\n```bash\nmake juicefs.ceph\n```\n\n在使用 Ceph 时，原本 JuiceFS 客户端的对象存储参数的含义不太相同：\n\n* `--bucket` 是 Ceph 存储池，格式为 `ceph://`，[存储池](https://docs.ceph.com/zh_CN/latest/rados/operations/pools)是用于存储对象的逻辑分区，使用前需要先创建好\n* `--access-key` 选项的值是 Ceph 集群名称，默认集群名称是 `ceph`。\n* `--secret-key` 选项的值是 [Ceph 客户端用户名](https://docs.ceph.com/en/latest/rados/operations/user-management)，默认用户名是 `client.admin`。\n\n为了连接到 Ceph Monitor，`librados` 将通过搜索默认位置读取 Ceph 的配置文件，并使用找到的第一个。这些位置是：\n\n- `CEPH_CONF` 环境变量\n- `/etc/ceph/ceph.conf`\n- `~/.ceph/config`\n- 在当前工作目录中的 `ceph.conf`\n\n创建一个文件系统：\n\n```bash\njuicefs.ceph format \\\n --storage ceph \\\n --bucket ceph:// \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Ceph RGW\n\n[Ceph Object Gateway](https://ceph.io/ceph-storage/object-storage) 是在 `librados` 之上构建的对象存储接口，旨在为应用程序提供访问 Ceph 存储集群的 RESTful 网关。Ceph 对象网关支持 S3 兼容的接口，因此我们可以将 `--storage` 设置为 `s3`。\n\n`--bucket` 选项的格式为 `http://.`（虚拟托管类型），例如：\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket http://. \\\n ... \\\n myjfs\n```\n\n### Gluster\n\n[Gluster](https://github.com/gluster/glusterfs) 是一款开源的软件定义分布式存储，单集群能支持 PiB 级别的数据。JuiceFS 通过 `libgfapi` 库与 Gluster 集群交互，使用前需要单独编译。\n\n首先安装 `libgfapi`（版本范围 6.0 - 10.1, [10.4+ 暂不支持](https://github.com/juicedata/juicefs/issues/4043))：\n\n\n \n\n```bash\nsudo apt-get install uuid-dev libglusterfs-dev glusterfs-common\n```\n\n \n \n\n```bash\nsudo yum install glusterfs glusterfs-api-devel glusterfs-libs\n```\n\n \n\n\n然后编译支持 Gluster 的 JuiceFS：\n\n```bash\nmake juicefs.gluster\n```\n\n现在我们可以创建出基于 Gluster 的 JuiceFS volume：\n\n```bash\njuicefs format \\\n --storage gluster \\\n --bucket host1,host2,host3/gv0 \\\n ... \\\n myjfs\n```\n\n其中 `--bucket` 选项格式为 `/`。注意这里的 `volume_name` 为 Gluster 中的卷名称，与 JuiceFS volume 自身的名字没有直接关系。\n\n### Swift\n\n[OpenStack Swift](https://github.com/openstack/swift) 是一种分布式对象存储系统，旨在从一台计算机扩展到数千台服务器。Swift 已针对多租户和高并发进行了优化。Swift 广泛适用于备份、Web 和移动内容的理想选择，可以无限量存储任何非结构化数据。\n\n`--bucket` 选项格式为 `http://.`，`container` 用来设定对象的命名空间。\n\n**当前，JuiceFS 仅支持 [Swift V1 authentication](https://www.swiftstack.com/docs/cookbooks/swift_usage/auth.html)。**\n\n`--access-key` 选项的值是用户名，`--secret-key` 选项的值是密码。例如：\n\n```bash\njuicefs format \\\n --storage swift \\\n --bucket http://. \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### MinIO\n\n[MinIO](https://min.io) 是开源的轻量级对象存储，兼容 Amazon S3 API。\n\n使用 Docker 可以很容易地在本地运行一个 MinIO 实例。例如，以下命令通过 `--console-address \":9900\"` 为控制台设置并映射了 `9900` 端口，还将 MinIO 的数据路径映射到了当前目录下的 `minio-data` 文件夹中，你可以按需修改这些参数：\n\n```shell\n$ sudo docker run -d --name minio \\\n -p 9000:9000 \\\n -p 9900:9900 \\\n -e \"MINIO_ROOT_USER=minioadmin\" \\\n -e \"MINIO_ROOT_PASSWORD=minioadmin\" \\\n -v $PWD/minio-data:/data \\\n --restart unless-stopped \\\n minio/minio server /data --console-address \":9900\"\n```\n\n容器创建成功以后使用以下地址访问：\n\n- **MinIO API**：[http://127.0.0.1:9000](http://127.0.0.1:9000)，这也是 JuiceFS 访问对象存储时所使用的的 API\n- **MinIO 管理界面**：[http://127.0.0.1:9900](http://127.0.0.1:9900)，用于管理对象存储本身，与 JuiceFS 无关\n\n对象存储初始的 Access Key 和 Secret Key 均为 `minioadmin`。\n\n使用 MinIO 作为 JuiceFS 的数据存储，`--storage` 选项设置为 `minio`。\n\n```bash\njuicefs format \\\n --storage minio \\\n --bucket http://127.0.0.1:9000/ \\\n --access-key minioadmin \\\n --secret-key minioadmin \\\n ... \\\n myjfs\n```\n\n:::note\n\n1. 当前，JuiceFS 仅支持路径风格的 MinIO URI 地址，例如：`http://127.0.0.1:9000/myjfs`\n1. `MINIO_REGION` 环境变量可以用于设置 MinIO 的 region，如果不设置，默认为 `us-east-1`\n1. 面对多节点 MinIO 集群，考虑在 Endpoint 中使用 DNS 域名，解析到各个 MinIO 节点，作为简易负载均衡，比如 `http://minio.example.com:9000/myjfs`\n\n:::\n\n### WebDAV\n\n[WebDAV](https://en.wikipedia.org/wiki/WebDAV) 是 HTTP 的扩展协议，有利于用户间协同编辑和管理存储在万维网服务器的文档。JuiceFS 0.15+ 支持使用 WebDAV 协议的存储系统作为后端数据存储。\n\n你需要将 `--storage` 设置为 `webdav`，并通过 `--bucket` 来指定访问 WebDAV 的地址。如果存储系统启用了用户验证，用户名和密码可以通过 `--access-key` 和 `--secret-key` 来指定，例如：\n\n```bash\njuicefs format \\\n --storage webdav \\\n --bucket http:/// \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### HDFS\n\nHadoop 的文件系统 [HDFS](https://hadoop.apache.org) 也可以作为对象存储供 JuiceFS 使用。\n\n当使用 HDFS 作为 JuiceFS 数据存储，`--access-key` 的值设置为用户名，默认的超级用户通常是 `hdfs`。例如：\n\n```bash\njuicefs format \\\n --storage hdfs \\\n --bucket namenode1:8020 \\\n --access-key hdfs \\\n ... \\\n myjfs\n```\n\n如果在创建文件系统时不指定 `--access-key`，JuiceFS 会使用执行 `juicefs mount` 命令的用户身份或通过 Hadoop SDK 访问 HDFS 的用户身份。如果该用户没有 HDFS 的读写权限，则程序会失败挂起，发生 IO 错误。\n\nJuiceFS 会尝试基于 `$HADOOP_CONF_DIR` 或 `$HADOOP_HOME` 为 HDFS 客户端加载配置。如果 `--bucket` 选项留空，将使用在 Hadoop 配置中找到的默认 HDFS。\n\nbucket 参数支持格式如下：\n\n- `[hdfs://]namenode:port[/path]`\n\n对于 HA 集群，bucket 参数可以：\n\n- `[hdfs://]namenode1:port,namenode2:port[/path]`\n- `[hdfs://]nameservice[/path]`\n\n对于启用 Kerberos 的 HDFS，可以通过 `KRB5KEYTAB` 和 `KRB5PRINCIPAL` 环境变量来指定 keytab 和 principal。\n\n### Apache Ozone\n\nApache Ozone 是 Hadoop 的分布式对象存储系统，提供了 S3 兼容的 API。所以可以通过 S3 兼容的模式作为对象存储供 JuiceFS 使用。例如：\n\n```bash\njuicefs format \\\n --storage s3 \\\n --bucket http:///\\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### Redis\n\nRedis 既可以作为 JuiceFS 的元数据存储，也可以作为数据存储，但当使用 Redis 作为数据存储时，建议不要存储大规模数据。\n\n#### 单机模式\n\n`--bucket` 选项格式为 `redis://:/`。`--access-key` 选项的值是用户名，`--secret-key` 选项的值是密码。例如：\n\n```bash\njuicefs format \\\n --storage redis \\\n --bucket redis://:/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n#### Redis Sentinel\n\nRedis Sentinel 模式下，`--bucket` 选项格式为 `redis[s]://MASTER_NAME,SENTINEL_ADDR[,SENTINEL_ADDR]:SENTINEL_PORT[/DB]`。Sentinel 的密码则需要通过 `SENTINEL_PASSWORD_FOR_OBJ` 环境变量来声明。例如：\n\n```bash\nexport SENTINEL_PASSWORD_FOR_OBJ=sentinel_password\njuicefs format \\\n --storage redis \\\n --bucket redis://masterName,1.2.3.4,1.2.5.6:26379/2 \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n#### Redis 集群\n\nRedis 集群模式下，`--bucket` 选项格式为 `redis[s]://ADDR:PORT,[ADDR:PORT],[ADDR:PORT]`。例如：\n\n```bash\njuicefs format \\\n --storage redis \\\n --bucket redis://127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002 \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n### TiKV\n\n[TiKV](https://tikv.org) 是一个高度可扩展、低延迟且易于使用的键值数据库。它提供原始和符合 ACID 的事务键值 API。\n\nTiKV 既可以用作 JuiceFS 的元数据存储，也可以用于 JuiceFS 的数据存储。\n\n:::note 注意\n建议使用独立部署的 TiKV 5.0+ 集群作为 JuiceFS 的数据存储\n:::\n\n`--bucket` 选项格式类似 `:,:,:`，其中 `` 是 Placement Driver（PD）的地址。`--access-key` 和 `--secret-key` 选项没有作用，可以省略。例如：\n\n```bash\njuicefs format \\\n --storage tikv \\\n --bucket \":,:,:\" \\\n ... \\\n myjfs\n```\n\n:::note 注意\n不要使用同一个 TiKV 集群来存储元数据和数据，因为 JuiceFS 是使用不同的协议来存储元数据（支持事务的 TxnKV) 和数据 (不支持事务的 RawKV)，TxnKV 的对象名会被编码后存储，即使添加了不同的前缀也可能导致它们的名字冲突。另外，建议启用 [Titan](https://tikv.org/docs/latest/deploy/configure/titan) 来提升存储数据的集群的性能。\n:::\n\n#### 设置 TLS\n\n如果需要开启 TLS，可以通过在 Bucket URL 后以添加 query 参数的形式设置 TLS 的配置项，目前支持的配置项：\n\n| 配置项 | 值 |\n|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| `ca` | CA 根证书，用于用 TLS 连接 TiKV/PD |\n| `cert` | 证书文件路径，用于用 TLS 连接 TiKV/PD |\n| `key` | 私钥文件路径，用于用 TLS 连接 TiKV/PD |\n| `verify-cn` | 证书通用名称，用于验证调用者身份，[详情](https://docs.pingcap.com/zh/tidb/dev/enable-tls-between-components#%E8%AE%A4%E8%AF%81%E7%BB%84%E4%BB%B6%E8%B0%83%E7%94%A8%E8%80%85%E8%BA%AB%E4%BB%BD) |\n\n例子：\n\n```bash\njuicefs format \\\n --storage tikv \\\n --bucket \":,:,:?ca=/path/to/ca.pem&cert=/path/to/tikv-server.pem&key=/path/to/tikv-server-key.pem&verify-cn=CN1,CN2\" \\\n ... \\\n myjfs\n```\n\n### etcd\n\n[etcd](https://etcd.io) 是一个高可用高可靠的小规模键值数据库，既可以用作 JuiceFS 的元数据存储，也可以用于 JuiceFS 的数据存储。\n\netcd 默认会[限制](https://etcd.io/docs/latest/dev-guide/limit)单个请求不能超过 1.5MB，需要将 JuiceFS 的分块大小（`--block-size` 选项）改成 1MB 甚至更低。\n\n`--bucket` 选项需要填 etcd 的地址，格式类似 `:,:,:`。`--access-key` 和 `--secret-key` 选项填用户名和密码，当 etcd 没有启用用户认证时可以省略。例如：\n\n```bash\njuicefs format \\\n --storage etcd \\\n --block-size 1024 \\ # 这个选项非常重要\n --bucket \":,:,:/prefix\" \\\n --access-key myname \\\n --secret-key mypass \\\n ... \\\n myjfs\n```\n\n#### 设置 TLS\n\n如果需要开启 TLS，可以通过在 Bucket URL 后以添加 query 参数的形式设置 TLS 的配置项，目前支持的配置项：\n\n| 配置项 | 值 |\n|------------------------|--------------|\n| `cacert` | CA 根证书 |\n| `cert` | 证书文件路径 |\n| `key` | 私钥文件路径 |\n| `server-name` | 服务器名称 |\n| `insecure-skip-verify` | 1 |\n\n例子：\n\n```bash\njuicefs format \\\n --storage etcd \\\n --bucket \":,:,:?cacert=/path/to/ca.pem&cert=/path/to/server.pem&key=/path/to/key.pem&server-name=etcd\" \\\n ... \\\n myjfs\n```\n\n:::note 注意\n证书的路径需要使用绝对路径，并且确保所有需要挂载的机器上能用该路径访问到它们。\n:::\n\n### SQLite\n\n[SQLite](https://sqlite.org) 是全球广泛使用的小巧、快速、单文件、可靠、全功能的单文件 SQL 数据库引擎。\n\n使用 SQLite 作为数据存储时只需要指定它的绝对路径即可。\n\n```shell\njuicefs format \\\n --storage sqlite3 \\\n --bucket /path/to/sqlite3.db \\\n ... \\\n myjfs\n```\n\n:::note 注意\n由于 SQLite 是一款嵌入式数据库，只有数据库所在的主机可以访问它，不能用于多机共享场景。如果格式化时使用的是相对路径，会导致挂载时出问题，请使用绝对路径。\n:::\n\n### MySQL\n\n[MySQL](https://www.mysql.com) 是受欢迎的开源关系型数据库之一，常被作为 Web 应用程序的首选数据库，既可以作为 JuiceFS 的元数据引擎也可以用来存储文件数据。跟 MySQL 兼容的 [MariaDB](https://mariadb.org)、[TiDB](https://github.com/pingcap/tidb) 等都可以用来作为数据存储。\n\n使用 MySQL 作为数据存储时，需要提前创建数据库并添加想要权限，通过 `--bucket` 选项指定访问地址，通过 `--access-key` 选项指定用户名，通过 `--secret-key` 选项指定密码，示例如下：\n\n```shell\njuicefs format \\\n --storage mysql \\\n --bucket (:3306)/ \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n创建文件系统后，JuiceFS 会在该数据库中创建名为 `jfs_blob` 的表用来存储数据。\n\n:::note 注意\n不要漏掉 `--bucket` 参数里的括号 `()`。\n:::\n\n### PostgreSQL\n\n[PostgreSQL](https://www.postgresql.org) 是功能强大的开源关系型数据库，有完善的生态和丰富的应用场景，既可以作为 JuiceFS 的元数据引擎也可以作为数据存储。其他跟 PostgreSQL 协议兼容的数据库（比如 [CockroachDB](https://github.com/cockroachdb/cockroach) 等) 也可以用来作为数据存储。\n\n创建文件系统时需要先创建好数据库并添加相应读写权限，使用 `--bucket` 选项来指定数据的地址，使用 `--access-key` 选项指定用户名，使用 `--secret-key` 选项指定密码，示例如下：\n\n```shell\njuicefs format \\\n --storage postgres \\\n --bucket :/[?parameters] \\\n --access-key \\\n --secret-key \\\n ... \\\n myjfs\n```\n\n创建文件系统后，JuiceFS 会在该数据库中创建名为 `jfs_blob` 的表用来存储数据。\n\n#### 故障排除\n\nJuiceFS 客户端默认采用 SSL 加密连接 PostgreSQL，如果连接时报错 `pq: SSL is not enabled on the server` 说明数据库没有启用 SSL。可以根据业务场景为 PostgreSQL 启用 SSL 加密，也可以在 bucket URL 中添加参数 `sslmode=disable` 禁用加密验证。\n\n### 本地磁盘\n\n在创建 JuiceFS 文件系统时，如果没有指定任何存储类型，会默认使用本地磁盘作为数据存储，root 用户默认存储路径为 `/var/jfs`，普通用户默认存储路径为 `~/.juicefs/local`。\n\n例如，以下命令使用本地的 Redis 数据库和本地磁盘创建了一个名为 `myfs` 的文件系统：\n\n```shell\njuicefs format redis://localhost:6379/1 myjfs\n```\n\n本地存储通常仅用于了解和体验 JuiceFS 的基本功能，创建的 JuiceFS 存储无法被网络内的其他客户端挂载，只能单机使用。\n\n### SFTP/SSH {#sftp}\n\nSFTP 全称 Secure File Transfer Protocol 即安全文件传输协议，它并不是文件存储。准确来说，JuiceFS 是通过 SFTP/SSH 这种文件传输协议对远程主机上的磁盘进行连接和读写，从而让任何启用了 SSH 服务的操作系统都可以作为 JuiceFS 的数据存储来使用。\n\n例如，以下命令使用 SFTP 协议连接远程服务器 `192.168.1.11` ，在用户 `tom` 的 `$HOME` 目录下创建 `myjfs/` 文件夹作为文件系统的数据存储。\n\n```shell\njuicefs format \\\n --storage sftp \\\n --bucket 192.168.1.11:myjfs/ \\\n --access-key tom \\\n --secret-key 123456 \\\n ...\n redis://localhost:6379/1 myjfs\n```\n\n#### 注意事项\n\n- `--bucket` 用来设置服务器的地址及存储路径，格式为 `[sftp://]:[port]:`。注意，目录名应该以 `/` 结尾，端口号为可选项默认为 `22`，例如 `192.168.1.11:22:myjfs/`。\n- `--access-key` 用来设置远程服务器的用户名\n- `--secret-key` 用来设置远程服务器的密码\n\n### NFS {#nfs}\n\nNFS - Network File System，即网络文件系统，是类 Unix 操作系统中很常用的文件共享服务，它可以让网络内的计算机能够像访问本地文件一样访问远程文件。\n\nJuiceFS 支持使用 NFS 作为底层存储来构建文件系统，提供两种使用方式：本地挂载和直连模式。\n\n#### 本地挂载\n\nJuiceFS v1.1 及之前的版本仅支持本地挂载的方式使用 NFS 作为底层存储，这种方式需要先在本地挂载 NFS 服务器上的目录，然后以本地磁盘的方式使用它来创建 JuiceFS 文件系统。\n\n例如，先把远程 NFS 服务器 `192.168.1.11` 上的 `/srv/data` 目录挂载到本地的 `/mnt/data` 目录，然后再使用 `file` 模式访问。\n\n```shell\n$ sudo mount -t nfs 192.168.1.11:/srv/data /mnt/data\n$ sudo juicefs format \\\n --storage file \\\n --bucket /mnt/data \\\n ...\n redis://localhost:6379/1 myjfs\n```\n\n从 JuiceFS 的角度来看，本地挂载的 NFS 仍然是本地磁盘，所以 `--storage` 选项设置为 `file`。\n\n同理，由于底层存储只能在挂载的设备上访问，所以要在多台设备上共享访问，则需要在每台设备上分别挂载 NFS 共享，或通过 WebDAV、S3 Gateway 等基于网络的方式来提供外部访问。\n\n#### 直连模式\n\nJuiceFS v1.2 及以上版本支持直连模式使用 NFS 作为底层存储，这种方式不需要在本地挂载预先挂载 NFS 目录，而是直接通过 JuiceFS 客户端内置的 NFS 协议访问共享目录。\n\n例如，远程服务器 `/etc/exports` 配置文件导出了下面的 NFS 共享：\n\n```\n/srv/data 192.168.1.0/24(rw,sync,no_subtree_check)\n```\n\n可以直接使用 JuiceFS 客户端连接 NFS 服务器上的 `/srv/data` 目录来创建文件系统：\n\n```shell\n$ sudo juicefs format \\\n --storage nfs \\\n --bucket 192.168.1.11:/srv/data \\\n ...\n redis://localhost:6379/1 myjfs\n```\n\n在直连模式下，`--storage` 选项设置为 `nfs`，`--bucket` 选项设置为 NFS 服务器的地址和共享目录，JuiceFS 客户端会直接连接 NFS 服务器上的目录来读写数据。\n\n**几个注意事项：**\n\n1. JuiceFS 直连 NFS 模式目前仅支持 NFSv3 协议\n2. JuiceFS 客户端需要有访问 NFS 共享目录的权限\n3. NFS 默认会启用 `root_squash` 功能，当以 root 身份访问 NFS 共享时默认会被挤压成 nobody 用户。为了避免无权 NFS 共享的问题，可以将共享目录的所有者设置为 `nobody:nogroup`，或者为 NFS 共享配置 `no_root_squash` 选项来关闭权限挤压。\n" }, { "path": "docs/zh_cn/reference/p8s_metrics.md", "content": "---\ntitle: JuiceFS 监控指标\nsidebar_position: 4\n---\n\n如果你尚未搭建监控系统、收集 JuiceFS 客户端指标，阅读[「监控」](../administration/monitoring.md)文档了解如何收集这些指标以及可视化。\n\n## 全局标签 {#global-labels}\n\n| 名称 | 描述 |\n| ---- | ----------- |\n| `vol_name` | Volume 名称 |\n| `instance` | 客户端主机名，格式为 `:`。详见[官方文档](https://prometheus.io/docs/concepts/jobs_instances) |\n| `mp` | 挂载点路径，如果是通过 [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) 上报，例如 [JuiceFS Hadoop Java SDK](../administration/monitoring.md#hadoop)，那么 `mp` 标签的值为 `sdk-` |\n\n## 文件系统 {#file-system}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n|-------------------------------|---------------|----|\n| `juicefs_used_space` | 总使用空间 | 字节 |\n| `juicefs_used_inodes` | 总 inodes 数量 | |\n\n## 操作系统 {#operating-system}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n| ---- | ----------- | ---- |\n| `juicefs_uptime` | 总运行时间 | 秒 |\n| `juicefs_cpu_usage` | CPU 使用量 | 秒 |\n| `juicefs_memory` | 内存使用量 | 字节 |\n\n## 元数据引擎 {#metadata-engine}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n| ---- | ----------- | ---- |\n| `juicefs_transaction_durations_histogram_seconds` | 事务的延时分布 | 秒 |\n| `juicefs_transaction_restart` | 事务重启的次数 | |\n\n## FUSE {#fuse}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n| ---- | ----------- | ---- |\n| `juicefs_fuse_read_size_bytes` | 读请求的大小分布 | 字节 |\n| `juicefs_fuse_written_size_bytes` | 写请求的大小分布 | 字节 |\n| `juicefs_fuse_ops_durations_histogram_seconds` | 所有请求的延时分布 | 秒 |\n| `juicefs_fuse_open_handlers` | 打开的文件和目录数量 | |\n\n## SDK {#sdk}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n| ---- | ----------- | ---- |\n| `juicefs_sdk_read_size_bytes` | 读请求的大小分布 | 字节 |\n| `juicefs_sdk_written_size_bytes` | 写请求的大小分布 | 字节 |\n| `juicefs_sdk_ops_durations_histogram_seconds` | 所有请求的延时分布 | 秒 |\n\n## 缓存 {#cache}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n|-----------------------------------------|-------------|----|\n| `juicefs_blockcache_blocks` | 缓存块的总个数 | |\n| `juicefs_blockcache_bytes` | 缓存块的总大小 | 字节 |\n| `juicefs_blockcache_hits` | 命中缓存块的总次数 | |\n| `juicefs_blockcache_miss` | 没有命中缓存块的总次数 | |\n| `juicefs_blockcache_writes` | 写入缓存块的总次数 | |\n| `juicefs_blockcache_drops` | 丢弃缓存块的总次数 | |\n| `juicefs_blockcache_evicts` | 淘汰缓存块的总次数 | |\n| `juicefs_blockcache_hit_bytes` | 命中缓存块的总大小 | 字节 |\n| `juicefs_blockcache_miss_bytes` | 没有命中缓存块的总大小 | 字节 |\n| `juicefs_blockcache_write_bytes` | 写入缓存块的总大小 | 字节 |\n| `juicefs_blockcache_read_hist_seconds` | 读缓存块的延时分布 | 秒 |\n| `juicefs_blockcache_write_hist_seconds` | 写缓存块的延时分布 | 秒 |\n| `juicefs_staging_blocks` | 暂存路径中的块数 | |\n| `juicefs_staging_block_bytes` | 暂存路径中块的总字节数 | 秒 |\n| `juicefs_staging_block_delay_seconds` | 暂存块延迟的总秒数 | 秒 |\n\n## 对象存储 {#object-storage}\n\n### 标签\n\n| 名称 | 描述 |\n| ---- | ----------- |\n| `method` | 请求对象存储的方法（例如 GET、PUT、HEAD、DELETE） |\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n| ---- | ----------- | ---- |\n| `juicefs_object_request_durations_histogram_seconds` | 请求对象存储的延时分布 | 秒 |\n| `juicefs_object_request_errors` | 请求失败的总次数 | |\n| `juicefs_object_request_data_bytes` | 请求对象存储的总数据大小 | 字节 |\n\n## 内部特性 {#internal}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n|----------------------------------------| ----------- | ---- |\n| `juicefs_compact_size_histogram_bytes` | 合并数据的大小分布 | 字节 |\n| `juicefs_used_read_buffer_size_bytes` | 当前用于读取的缓冲区的大小 | |\n\n## 数据同步 {#sync}\n\n### 指标\n\n| 名称 | 描述 | 单位 |\n|-|-|-|\n| `juicefs_sync_scanned` | 从源端扫描的所有对象数量 | |\n| `juicefs_sync_handled` | 已经处理过的来自源端的对象数量 | |\n| `juicefs_sync_pending` | 等待同步的对象数量 | |\n| `juicefs_sync_copied` | 已经同步过的对象数量 | |\n| `juicefs_sync_copied_bytes` | 已经同步过的数据总大小 | 字节 |\n| `juicefs_sync_skipped` | 同步时被跳过的对象数量 | |\n| `juicefs_sync_failed` | 同步时失败的对象数量 | |\n| `juicefs_sync_deleted` | 同步时被删除的对象数量 | |\n| `juicefs_sync_checked` | 同步时校验过 checksum 的对象数量 | |\n| `juicefs_sync_checked_bytes` | 同步时校验过 checksum 的数据总大小 | 字节 |\n" }, { "path": "docs/zh_cn/reference/posix_compatibility.md", "content": "---\ntitle: POSIX 兼容性\nsidebar_position: 6\nslug: /posix_compatibility\n---\n\nJuiceFS 借助于 pjdfstest 和 LTP 来验证其对 POSIX 的兼容性。\n\n## Pjdfstest\n\n[Pjdfstest](https://github.com/pjd/pjdfstest) 是一个用来帮助验证 POSIX 系统调用的测试集，JuiceFS 通过了其最新的 8813 项测试：\n\n```\nAll tests successful.\n\nTest Summary Report\n-------------------\n/root/soft/pjdfstest/tests/chown/00.t (Wstat: 0 Tests: 1323 Failed: 0)\n TODO passed: 693, 697, 708-709, 714-715, 729, 733\nFiles=235, Tests=8813, 233 wallclock secs ( 2.77 usr 0.38 sys + 2.57 cusr 3.93 csys = 9.65 CPU)\nResult: PASS\n```\n\n:::note 注意\n测试 pjdfstest 时，需要将 JuiceFS 的回收站关闭，因为 pjdfstest 测试的删除行为是直接删除而非进入回收站，而 JuiceFS 回收站是默认开启的。\n关闭回收站命令：`juicefs config --trash-days 0`\n:::\n\n此外，JuiceFS 还提供：\n\n- 关闭再打开（close-to-open）一致性。一旦一个文件写入完成并关闭，之后的打开和读操作保证可以访问之前写入的数据。如果是在同一个挂载点，所有写入的数据都可以立即读。\n- 重命名以及所有其他元数据操作都是原子的，由元数据引擎的事务机制保证。\n- 当文件被删除后，同一个挂载点上如果已经打开了，文件还可以继续访问。\n- 支持 mmap\n- 支持 fallocate 以及空洞\n- 支持扩展属性\n- 支持 BSD 锁（flock）\n- 支持传统 POSIX 记录锁（fcntl）\n\n:::note 注意\nPOSIX 记录锁分为**传统锁**和 **OFD 锁**（Open file description locks）两类，它们的加锁操作命令分别为 `F_SETLK` 和 `F_OFD_SETLK`。受限于 FUSE 内核模块的实现，目前 JuiceFS 只支持传统类型的记录锁。更多细节可参见：[https://man7.org/linux/man-pages/man2/fcntl.2.html](https://man7.org/linux/man-pages/man2/fcntl.2.html)。\n:::\n\n## LTP\n\n[LTP](https://github.com/linux-test-project/ltp)（Linux Test Project）是一个由 IBM，Cisco 等多家公司联合开发维护的项目，旨在为开源社区提供一个验证 Linux 可靠性和稳定性的测试集。LTP 中包含了各种工具来检验 Linux 内核和相关特性；JuiceFS 通过了其中与文件系统相关的大部分测试例。\n\n### 测试环境\n\n- 测试主机：Amazon EC2: c5d.xlarge (4C 8G)\n- 操作系统：Ubuntu 20.04.1 LTS (Kernel `5.4.0-1029-aws`)\n- 对象存储：Amazon S3\n- JuiceFS 版本：0.17-dev (2021-09-16 292f2b65)\n\n### 测试步骤\n\n1. 在 GitHub 下载 LTP [源码包](https://github.com/linux-test-project/ltp/releases/download/20210524/ltp-full-20210524.tar.bz2)\n2. 解压后编译安装：\n\n ```bash\n tar -jvxf ltp-full-20210524.tar.bz2\n cd ltp-full-20210524\n ./configure\n make all\n make install\n ```\n\n3. 测试工具安装在 `/opt/ltp`，需先切换到此目录：\n\n ```bash\n cd /opt/ltp\n ```\n\n 测试配置文件在 `runtest` 目录下；为方便测试，删去了 `fs` 和 `syscalls` 中部分压力测试和与文件系统不相关的条目（参见[附录](#附录)，修改后保存到文件 `fs-jfs` 和 `syscalls-jfs`），然后执行命令：\n\n ```bash\n ./runltp -d /mnt/jfs -f fs_bind,fs_perms_simple,fsx,io,smoketest,fs-jfs,syscalls-jfs\n ```\n\n### 测试结果\n\n```bash\nTestcase Result Exit Value\n-------- ------ ----------\nfcntl17 FAIL 7\nfcntl17_64 FAIL 7\ngetxattr05 CONF 32\nioctl_loop05 FAIL 4\nioctl_ns07 FAIL 1\nlseek11 CONF 32\nopen14 CONF 32\nopenat03 CONF 32\nsetxattr03 FAIL 6\n\n-----------------------------------------------\nTotal Tests: 1270\nTotal Skipped Tests: 4\nTotal Failures: 5\nKernel Version: 5.4.0-1029-aws\nMachine Architecture: x86_64\n```\n\n其中跳过和失败的测试例原因如下：\n\n- fcntl17，fcntl17_64：在 POSIX locks 加锁时需要文件系统自动检测死锁，目前 JuiceFS 尚不支持\n- getxattr05：需要设置文件扩展权限 ACL，目前 JuiceFS 尚不支持\n- ioctl_loop05，ioctl_ns07，setxattr03：需要调用 `ioctl`，目前 JuiceFS 尚不支持\n- lseek11：需要 `lseek` 处理 SEEK_DATA 和 SEEK_HOLE 标记位，目前 JuiceFS 用的是内核通用实现，尚不支持这两个 flags\n- open14，openat03：需要 `open` 处理 O_TMPFILE 标记位，由于 FUSE 不支持，JuiceFS 也无法实现\n\n### 附录\n\n在 `fs` 和 `syscalls` 文件中删去的测试例：\n\n```bash\n# fs --> fs-jfs\ngf01 growfiles -W gf01 -b -e 1 -u -i 0 -L 20 -w -C 1 -l -I r -T 10 -f glseek20 -S 2 -d $TMPDIR\ngf02 growfiles -W gf02 -b -e 1 -L 10 -i 100 -I p -S 2 -u -f gf03_ -d $TMPDIR\ngf03 growfiles -W gf03 -b -e 1 -g 1 -i 1 -S 150 -u -f gf05_ -d $TMPDIR\ngf04 growfiles -W gf04 -b -e 1 -g 4090 -i 500 -t 39000 -u -f gf06_ -d $TMPDIR\ngf05 growfiles -W gf05 -b -e 1 -g 5000 -i 500 -t 49900 -T10 -c9 -I p -u -f gf07_ -d $TMPDIR\ngf06 growfiles -W gf06 -b -e 1 -u -r 1-5000 -R 0--1 -i 0 -L 30 -C 1 -f g_rand10 -S 2 -d $TMPDIR\ngf07 growfiles -W gf07 -b -e 1 -u -r 1-5000 -R 0--2 -i 0 -L 30 -C 1 -I p -f g_rand13 -S 2 -d $TMPDIR\ngf08 growfiles -W gf08 -b -e 1 -u -r 1-5000 -R 0--2 -i 0 -L 30 -C 1 -f g_rand11 -S 2 -d $TMPDIR\ngf09 growfiles -W gf09 -b -e 1 -u -r 1-5000 -R 0--1 -i 0 -L 30 -C 1 -I p -f g_rand12 -S 2 -d $TMPDIR\ngf10 growfiles -W gf10 -b -e 1 -u -r 1-5000 -i 0 -L 30 -C 1 -I l -f g_lio14 -S 2 -d $TMPDIR\ngf11 growfiles -W gf11 -b -e 1 -u -r 1-5000 -i 0 -L 30 -C 1 -I L -f g_lio15 -S 2 -d $TMPDIR\ngf12 mkfifo $TMPDIR/gffifo17; growfiles -b -W gf12 -e 1 -u -i 0 -L 30 $TMPDIR/gffifo17\ngf13 mkfifo $TMPDIR/gffifo18; growfiles -b -W gf13 -e 1 -u -i 0 -L 30 -I r -r 1-4096 $TMPDIR/gffifo18\ngf14 growfiles -W gf14 -b -e 1 -u -i 0 -L 20 -w -l -C 1 -T 10 -f glseek19 -S 2 -d $TMPDIR\ngf15 growfiles -W gf15 -b -e 1 -u -r 1-49600 -I r -u -i 0 -L 120 -f Lgfile1 -d $TMPDIR\ngf16 growfiles -W gf16 -b -e 1 -i 0 -L 120 -u -g 4090 -T 101 -t 408990 -l -C 10 -c 1000 -S 10 -f Lgf02_ -d $TMPDIR\ngf17 growfiles -W gf17 -b -e 1 -i 0 -L 120 -u -g 5000 -T 101 -t 499990 -l -C 10 -c 1000 -S 10 -f Lgf03_ -d $TMPDIR\ngf18 growfiles -W gf18 -b -e 1 -i 0 -L 120 -w -u -r 10-5000 -I r -l -S 2 -f Lgf04_ -d $TMPDIR\ngf19 growfiles -W gf19 -b -e 1 -g 5000 -i 500 -t 49900 -T10 -c9 -I p -o O_RDWR,O_CREAT,O_TRUNC -u -f gf08i_ -d $TMPDIR\ngf20 growfiles -W gf20 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 1-256000:512 -R 512-256000 -T 4 -f gfbigio-$$ -d $TMPDIR\ngf21 growfiles -W gf21 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -T 10 -t 20480 -f gf-bld-$$ -d $TMPDIR\ngf22 growfiles -W gf22 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -T 10 -t 20480 -f gf-bldf-$$ -d $TMPDIR\ngf23 growfiles -W gf23 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 512-64000:1024 -R 1-384000 -T 4 -f gf-inf-$$ -d $TMPDIR\ngf24 growfiles -W gf24 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -g 20480 -f gf-jbld-$$ -d $TMPDIR\ngf25 growfiles -W gf25 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 1024000-2048000:2048 -R 4095-2048000 -T 1 -f gf-large-gs-$$ -d $TMPDIR\ngf26 growfiles -W gf26 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -r 128-32768:128 -R 512-64000 -T 4 -f gfsmallio-$$ -d $TMPDIR\ngf27 growfiles -W gf27 -b -D 0 -w -g 8b -C 1 -b -i 1000 -u -f gfsparse-1-$$ -d $TMPDIR\ngf28 growfiles -W gf28 -b -D 0 -w -g 16b -C 1 -b -i 1000 -u -f gfsparse-2-$$ -d $TMPDIR\ngf29 growfiles -W gf29 -b -D 0 -r 1-4096 -R 0-33554432 -i 0 -L 60 -C 1 -u -f gfsparse-3-$$ -d $TMPDIR\ngf30 growfiles -W gf30 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -o O_RDWR,O_CREAT,O_SYNC -g 20480 -T 10 -t 20480 -f gf-sync-$$ -d $TMPDIR\nrwtest01 export LTPROOT; rwtest -N rwtest01 -c -q -i 60s -f sync 10%25000:$TMPDIR/rw-sync-$$\nrwtest02 export LTPROOT; rwtest -N rwtest02 -c -q -i 60s -f buffered 10%25000:$TMPDIR/rw-buffered-$$\nrwtest03 export LTPROOT; rwtest -N rwtest03 -c -q -i 60s -n 2 -f buffered -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-buff-$$\nrwtest04 export LTPROOT; rwtest -N rwtest04 -c -q -i 60s -n 2 -f sync -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-sync-$$\nrwtest05 export LTPROOT; rwtest -N rwtest05 -c -q -i 50 -T 64b 500b:$TMPDIR/rwtest01%f\niogen01 export LTPROOT; rwtest -N iogen01 -i 120s -s read,write -Da -Dv -n 2 500b:$TMPDIR/doio.f1.$$ 1000b:$TMPDIR/doio.f2.$$\nquota_remount_test01 quota_remount_test01.sh\nisofs isofs.sh\n\n# syscalls --> syscalls-jfs\nbpf_prog05 bpf_prog05\ncacheflush01 cacheflush01\nchown01_16 chown01_16\nchown02_16 chown02_16\nchown03_16 chown03_16\nchown04_16 chown04_16\nchown05_16 chown05_16\nclock_nanosleep03 clock_nanosleep03\nclock_gettime03 clock_gettime03\nleapsec01 leapsec01\nclose_range01 close_range01\nclose_range02 close_range02\nfallocate06 fallocate06\nfchown01_16 fchown01_16\nfchown02_16 fchown02_16\nfchown03_16 fchown03_16\nfchown04_16 fchown04_16\nfchown05_16 fchown05_16\nfcntl06 fcntl06\nfcntl06_64 fcntl06_64\ngetegid01_16 getegid01_16\ngetegid02_16 getegid02_16\ngeteuid01_16 geteuid01_16\ngeteuid02_16 geteuid02_16\ngetgid01_16 getgid01_16\ngetgid03_16 getgid03_16\ngetgroups01_16 getgroups01_16\ngetgroups03_16 getgroups03_16\ngetresgid01_16 getresgid01_16\ngetresgid02_16 getresgid02_16\ngetresgid03_16 getresgid03_16\ngetresuid01_16 getresuid01_16\ngetresuid02_16 getresuid02_16\ngetresuid03_16 getresuid03_16\ngetrusage04 getrusage04\ngetuid01_16 getuid01_16\ngetuid03_16 getuid03_16\nioctl_sg01 ioctl_sg01\nfanotify16 fanotify16\nfanotify18 fanotify18\nfanotify19 fanotify19\nlchown01_16 lchown01_16\nlchown02_16 lchown02_16\nlchown03_16 lchown03_16\nmbind02 mbind02\nmbind03 mbind03\nmbind04 mbind04\nmigrate_pages02 migrate_pages02\nmigrate_pages03 migrate_pages03\nmodify_ldt01 modify_ldt01\nmodify_ldt02 modify_ldt02\nmodify_ldt03 modify_ldt03\nmove_pages01 move_pages01\nmove_pages02 move_pages02\nmove_pages03 move_pages03\nmove_pages04 move_pages04\nmove_pages05 move_pages05\nmove_pages06 move_pages06\nmove_pages07 move_pages07\nmove_pages09 move_pages09\nmove_pages10 move_pages10\nmove_pages11 move_pages11\nmove_pages12 move_pages12\nmsgctl05 msgctl05\nmsgstress04 msgstress04\nopenat201 openat201\nopenat202 openat202\nopenat203 openat203\nmadvise06 madvise06\nmadvise09 madvise09\nptrace04 ptrace04\nquotactl01 quotactl01\nquotactl04 quotactl04\nquotactl06 quotactl06\nreaddir21 readdir21\nrecvmsg03 recvmsg03\nsbrk03 sbrk03\nsemctl08 semctl08\nsemctl09 semctl09\nset_mempolicy01 set_mempolicy01\nset_mempolicy02 set_mempolicy02\nset_mempolicy03 set_mempolicy03\nset_mempolicy04 set_mempolicy04\nset_thread_area01 set_thread_area01\nsetfsgid01_16 setfsgid01_16\nsetfsgid02_16 setfsgid02_16\nsetfsgid03_16 setfsgid03_16\nsetfsuid01_16 setfsuid01_16\nsetfsuid02_16 setfsuid02_16\nsetfsuid03_16 setfsuid03_16\nsetfsuid04_16 setfsuid04_16\nsetgid01_16 setgid01_16\nsetgid02_16 setgid02_16\nsetgid03_16 setgid03_16\nsgetmask01 sgetmask01\nsetgroups01_16 setgroups01_16\nsetgroups02_16 setgroups02_16\nsetgroups03_16 setgroups03_16\nsetgroups04_16 setgroups04_16\nsetregid01_16 setregid01_16\nsetregid02_16 setregid02_16\nsetregid03_16 setregid03_16\nsetregid04_16 setregid04_16\nsetresgid01_16 setresgid01_16\nsetresgid02_16 setresgid02_16\nsetresgid03_16 setresgid03_16\nsetresgid04_16 setresgid04_16\nsetresuid01_16 setresuid01_16\nsetresuid02_16 setresuid02_16\nsetresuid03_16 setresuid03_16\nsetresuid04_16 setresuid04_16\nsetresuid05_16 setresuid05_16\nsetreuid01_16 setreuid01_16\nsetreuid02_16 setreuid02_16\nsetreuid03_16 setreuid03_16\nsetreuid04_16 setreuid04_16\nsetreuid05_16 setreuid05_16\nsetreuid06_16 setreuid06_16\nsetreuid07_16 setreuid07_16\nsetuid01_16 setuid01_16\nsetuid03_16 setuid03_16\nsetuid04_16 setuid04_16\nshmctl06 shmctl06\nsocketcall01 socketcall01\nsocketcall02 socketcall02\nsocketcall03 socketcall03\nssetmask01 ssetmask01\nswapoff01 swapoff01\nswapoff02 swapoff02\nswapon01 swapon01\nswapon02 swapon02\nswapon03 swapon03\nswitch01 endian_switch01\nsysinfo03 sysinfo03\ntimerfd04 timerfd04\nperf_event_open02 perf_event_open02\nstatx07 statx07\nio_uring02 io_uring02\n```\n" }, { "path": "docs/zh_cn/reference/spec-limits.md", "content": "---\nsidebar_position: 7\n---\n\n# 规格限制\n\n## 文件系统限制\n\n以下为 JuiceFS 文件系统的理论限制，实际性能和文件系统规模会受到所采用的的元数据引擎以及对象存储的限制。\n\n* 目录层级：无限制\n* 文件名长度：255 字节\n* 符号链接长度：4096 字节\n* 硬链接个数：2^31\n* 单目录的文件数：2^31\n* 单个文件系统文件数：无限制\n* 单文件长度：2^(26+31)\n* 数据量：4EiB\n" }, { "path": "docs/zh_cn/release_notes.md", "content": "# 版本更新\n\n:::tip 提示\n所有历史版本请查看 [GitHub Releases](https://github.com/juicedata/juicefs/releases) 页面\n:::\n\n## 版本号 {#version-number}\n\nJuiceFS 社区版采用[语义化版本号](https://semver.org/lang/zh-CN)标记方式，每个版本号都由三个数字组成 `x.y.z`，分别是主版本号（x）、次版本号（y）和修订号（z）。\n\n1. **主版本号（x）**：主版本号大于等于 `1` 时，表示该版本已经适用于生产环境。当主版本号发生变化时，表明这个版本可能增加了不能向后兼容的重大功能、架构变化或数据格式变化。例如，`v0.8.3` → `v1.0.0` 代表生产就绪，`v1.0.0` → `v2.0.0` 代表架构或功能变化。\n2. **次版本号（y）**：次版本号表示该版本增加了一些能够向后兼容的新功能、性能优化和 bug 修复等。例如，`v1.0.0` → `v1.1.0`。\n3. **修订号（z）**：修订号表示软件的小更新或者 bug 修复，只是对现有功能的一些小的改动或者修复，不会影响软件兼容性。例如，`v1.0.3` → `v1.0.4`。\n\n## 版本升级 {#changes}\n\nJuiceFS 的客户端只有一个二进制文件，一般情况下升级时只需要用新版本软件替换旧版即可。\n\n### JuiceFS v1.1\n\n:::tip 提示\n若您正在使用的版本小于 v1.0，请先[升级到 v1.0 版本](#juicefs-v10)。\n:::\n\nJuiceFS 在 v1.1（具体而言，是 v1.1.0-beta2）版本中新增了[**目录用量统计**](https://juicefs.com/docs/zh/community/guide/dir-stats)和[**目录配额**](https://juicefs.com/docs/zh/community/guide/quota#directory-quota)两个功能，且目录配额依赖于用量统计。这两项功能在旧版本客户端中没有，当它们被开启的情况下使用旧客户端写入会导致统计数值出现较大偏差。在升级到 v1.1 时，若您不打算启用这两项新功能，可以直接使用新版本客户端替换升级，无需额外操作。若您打算使用，则建议您在升级前了解以下内容。\n\n#### 默认配置\n\n目前这两项功能的默认配置为：\n\n- 新创建的文件系统，会自动启用\n\n- 已有的文件系统，默认均不启用\n - 目录用量统计可以通过 `juicefs config` 命令单独开启\n - 设置目录配额时，用量统计会自动开启\n\n#### 推荐升级步骤\n\n1. 升级所有客户端软件到 v1.1 版本\n2. 拒绝 v1.1 之前的版本再次连接：`juicefs config META-URL --min-client-version 1.1.0-A`\n3. 在合适的时间重启服务（重新挂载，重启 gateway 等）\n4. 确保所有在线客户端版本都在 v1.1 或以上：`juicefs status META-URL | grep -w Version`\n5. 启用新特性，具体参见[目录用量统计](https://juicefs.com/docs/zh/community/guide/dir-stats)和[目录配额](https://juicefs.com/docs/zh/community/guide/quota#directory-quota)\n\n### JuiceFS v1.0\n\nJuiceFS 在 v1.0（具体而言，是 v1.0.0-beta3）版本中有两项兼容性修改。若您原来使用的客户端版本较低，建议您在升级前先了解以下内容。\n\n#### 调整 SQL 表结构以支持非 UTF-8 字符\n\nJuiceFS v1.0 改进了 SQL 元数据引擎对非 UTF-8 字符集的支持。对于已有的文件系统，需要手动调整表结构才能支持非 UTF-8 字符集，建议在升级完所有客户端后再选择访问压力比较低的时候进行操作。\n\n:::note 注意\n调整 SQL 表结构时数据库性能可能会下降，影响正在运行的服务。\n:::\n\n##### MySQL/MariaDB\n\n```sql\nalter table jfs_edge\n modify name varbinary(255) not null;\nalter table jfs_symlink\n modify target varbinary(4096) not null;\n```\n\n##### PostgreSQL\n\n```sql\nalter table jfs_edge\n alter column name type bytea using name::bytea;\nalter table jfs_symlink\n alter column target type bytea using target::bytea;\n```\n\n##### SQLite\n\n由于 SQLite 不支持修改字段，可以通过 dump 和 load 命令进行迁移，详情参考：[JuiceFS 元数据备份和恢复](administration/metadata_dump_load.md)。\n\n#### 会话管理格式变更\n\nJuiceFS v1.0 使用了新的会话管理格式，历史版本客户端通过 `juicefs status` 或者 `juicefs destroy` 将无法看到 v1.0 客户端产生的会话，新版客户端可以看到所有会话。\n" }, { "path": "docs/zh_cn/security/encryption.md", "content": "---\nsidebar_position: 1\n---\n# 数据加密\n\n在数据安全方面，JuiceFS 提供两个方面的数据加密保护：\n\n1. 传输加密\n2. 静态数据加密\n\n## 传输加密 {#in-transit}\n\nJuiceFS 的架构决定了它的运行通常涉及与数据库和对象存储之间的网络连接，只要这些服务支持加密连接，JuiceFS 就可以通过其提供的加密通道进行访问。\n\n### 通过 HTTPS 与对象存储连接\n\n公有云对象存储一般会同时支持 HTTP 和 HTTPS，在创建文件系统时如果没有指定协议头，JuiceFS 会默认使用 HTTPS 协议头。例如：\n\n```shell {2}\njuicefs format --storage s3 \\\n --bucket myjfs.s3.ap-southeast-1.amazonaws.com \\\n ...\n```\n\n以上命令，客户端会默认将 bucket 识别为 `https://myjfs.s3.ap-southeast-1.amazonaws.com`。\n\n对于服务器和对象存储运行在相同 VPC 网络的情况，如果不需要加密连接，可以明确指定要使用的协议头，例如：`--bucket http://myjfs.s3.ap-southeast-1.amazonaws.com`。\n\n### 通过 TLS/SSL 加密连接到数据库\n\n对于所有[支持的元数据引擎](../reference/how_to_set_up_metadata_engine.md)，只要数据库本身支持并配置了 TLS/SSL 等加密链接，JuiceFS 即可通过其加密通道进行连接。例如，配置了 TLS 加密的 Redis 数据库可以使用 `rediss://` 协议头进行链接：\n\n```shell {3}\njuicefs format --storage s3 \\\n --bucket myjfs.s3.ap-southeast-1.amazonaws.com \\\n \"rediss://myredis.ap-southeast-1.amazonaws.com:6379/1\" myjfs\n```\n\n## 静态数据加密 {#at-rest}\n\nJuiceFS 提供静态数据加密支持，即先加密，再上传。所有存入 JuiceFS 的文件都会在本地完成加密后再上传到对象存储，这可以在对象存储本身被破坏时有效地防止数据泄露。\n\nJuiceFS 的静态数据加密采用混合加密架构：对称加密负责数据加密，非对称加密负责密钥保护。只需在创建文件系统时提供一个私钥即可启用数据加密功能，通过 `JFS_RSA_PASSPHRASE` 环境变量提供私钥密码。在使用上，挂载点对应用程序完全透明，即加密和解密过程对文件系统的访问不会产生影响。\n\n### 加密原理\n\n#### 加密架构设计\n\nJuiceFS 采用**混合加密架构**，包含两个加密层次：\n\n1. **数据加密层**（对称加密 - AES-256-GCM 或 ChaCha20-Poly1305 或 SM4-GCM）\n - **作用**：实际加密用户数据内容\n - **机制**：每个 block 生成唯一的对称密钥 `S` + 随机种子 `N`（均使用 256 位密钥）\n - **优势**：AES-256-GCM 和 ChaCha20-Poly1305 都提供高速加密和完整性验证（AEAD）\n - **标准**：256 位密钥强度符合 NIST 安全标准，ChaCha20-Poly1305 是 RFC 8439 标准算法\n\n2. **密钥保护层**（非对称加密）\n - **作用**：保护对称密钥的安全分发和存储\n - **机制**：使用私钥 `M` 加密每个数据块的对称密钥 `S`\n - **优势**：解决密钥分发难题，避免密钥重用风险\n - **方案**：支持PKCS#1、PKCS#8格式的私钥\n\n需要用户预先为文件系统创建一个全局私钥 `M`。在对象存储中保存的每个对象都将有自己的随机对称密钥 `S`。\n\n符号说明：\n\n- `M` 代表用户自行创建的私钥\n- `S` 代表 JuiceFS 客户端为每个文件对象生成的 256 位对称密钥\n- `N` 代表 JuiceFS 客户端为每个文件对象生成的随机种子\n- `K` 代表 `M` 加密 `S` 得到的密文\n\n![Encryption At-rest](../images/encryption.png)\n\n#### 数据加密过程\n\n- 在写入对象存储之前，数据块会使用 LZ4 或 Zstandard 进行压缩。\n- 为每个数据块生成一个随机的 256 位对称密钥 `S` 和一个随机种子 `N`。\n- 基于 AES-256-GCM 或 ChaCha20-Poly1305 或 SM4-GCM 使用 `S` 和 `N` 对每个数据块进行加密得到 `encrypted_data`。\n- 为了避免对称密钥 `S` 在网络上明文传输，使用 RSA 私钥 `M` 对对称密钥 `S` 进行加密得到密文 `K` 。\n- 将加密后的数据 `encrypted_data`、密文 `K` 和随机种子 `N` 组合成对象，然后写入对象存储。\n\n#### 数据解密过程\n\n- 读取整个加密对象（它可能比 4MB 大一点）。\n- 解析对象数据得到密文 `K`、随机种子 `N` 和被加密的数据 `encrypted_data`。\n- 用私钥解密 `K`，得到对称密钥 `S`。\n- 基于 AES-256-GCM 或 ChaCha20-Poly1305 或 SM4-GCM 使用 `S` 和 `N` 解密数据 `encrypted_data` 得到数据块明文。\n- 对数据块解压缩。\n\n### 启用静态加密\n\n:::note 注意\n静态数据加密功能必须在创建文件系统时启用，已创建的文件系统无法再启用数据加密。\n:::\n\n启用静态加密功能的步骤为：\n\n1. 创建私钥\n2. 使用私钥创建加密的文件系统\n3. 挂载文件系统\n\n#### 第一步 创建私钥\n\n私钥是静态数据加密的关键，一般使用 OpenSSL 手动生成。以下命令将使用 aes256 算法在当前目录生成长度为 2048 位，文件名为 `my-priv-key.pem` 的 RSA 私钥：\n\n```shell\nopenssl genrsa -out my-priv-key.pem -aes256 2048\n```\n\n由于使用了 `aes256` 加密算法，命令行会要求必须为该私钥提供一个至少 4 位的 `Passphrase`，可以简单地把它理解为一个用于加密 RSA 私钥文件本身的密码，它也是 RSA 私钥文件的最后一道安全保障。\n\n:::caution 特别注意\n私钥的安全极其重要，需要特别注意以下几点：\n\n- **Passphrase 泄露风险**：如果私钥的 Passphrase 泄露，攻击者可能解密存储在元数据引擎中的私钥，从而危及所有加密数据的安全\n- **私钥文件泄露**：如果加密的私钥文件本身泄露，同时 Passphrase 也被获取，将导致严重的安全风险\n- **数据不可恢复性**：如果无法提供正确的 Passphrase 来访问存储在元数据引擎中的私钥，**所有的加密数据将永久丢失且无法恢复**\n\n建议专注于保护 Passphrase 的安全，并通过环境变量方式传递，避免在命令行历史中泄露。\n:::\n\n#### 第二步 创建加密的文件系统\n\n创建加密的文件系统需要使用 `--encrypt-rsa-key` 选项指定私钥，提供的私钥内容将写入元数据引擎。需要用环境变量 `JFS_RSA_PASSPHRASE` 来指定私钥的 Passphrase。\n\nJuiceFS 支持三种加密算法组合，可以通过 `--encrypt-algo` 选项指定：\n\n- `aes256gcm-rsa`（默认）：使用 AES-256-GCM + RSA（或其他私钥）\n- `chacha20-rsa`：使用 ChaCha20-Poly1305 + RSA（或其他私钥）\n- `sm4gcm`: 使用 SM4-GCM + SM2（或其他私钥）\n\n1. 用环境变量设置 Passphrase\n\n ```shell\n export JFS_RSA_PASSPHRASE=the-passwd-for-rsa\n ```\n\n2. 创建文件系统（使用默认的 AES-256-GCM 加密）\n\n ```shell {2}\n juicefs format --storage s3 \\\n --encrypt-rsa-key my-priv-key.pem \\\n ...\n ```\n\n 或者明确指定使用 ChaCha20-Poly1305 加密：\n\n ```shell {2,3}\n juicefs format --storage s3 \\\n --encrypt-rsa-key my-priv-key.pem \\\n --encrypt-algo chacha20-rsa \\\n ...\n ```\n\n3. （可选）删除本地私钥文件\n\n JuiceFS 在格式化文件系统时会将私钥的内容安全地存储在元数据引擎中。因此，在完成文件系统创建后（除非有特殊的合规性要求），建议您删除本地的私钥文件：\n\n ```shell\n rm my-priv-key.pem\n ```\n\n 这样只需确保 `JFS_RSA_PASSPHRASE` 环境变量的安全，后续的文件系统挂载和访问只需要提供正确的 Passphrase 即可。\n\n 如果由于合规性要求或其他原因需要保留私钥文件，请务必将私钥文件存储在安全位置，设置严格的访问权限，并确保私钥文件和 Passphrase 分开保管。\n\n#### 第三步 挂载文件系统\n\n挂载加密的文件系统无需指定额外的选项，但在挂载之前需要通先过环境变量设置私钥的 Passphrase。\n\n1. 用环境变量设置 Passphrase\n\n ```shell\n export JFS_RSA_PASSPHRASE=the-passwd-for-rsa\n ```\n\n2. 挂载文件系统\n\n ```shell\n juicefs mount redis://127.0.0.1:6379/1 /mnt/myjfs\n ```\n\n### 性能考量\n\n启用加密功能确实会带来一定的性能开销，但现代硬件技术已经让这种影响变得相当可控。具体的性能影响取决于工作负载类型、硬件配置（特别是 CPU 的加密指令集支持）和数据访问模式。\n\n现代 CPU 中 TLS、HTTPS 和 AES-256 这些加密技术都有专门的硬件优化。特别是 Intel 和 AMD 的现代处理器都内置了 AES-NI 指令集，能够以接近原生的速度执行 AES 加密操作，这让数据加密的性能损耗大大降低。\n\n#### 加密算法选择建议\n\n**AES-256-GCM**（默认选择）：\n\n- 在支持 AES-NI 指令集的现代 CPU 上性能优异\n- 广泛的行业标准支持和验证\n- 适合大多数生产环境\n\n**ChaCha20-Poly1305**：\n\n- 在不支持 AES-NI 的 CPU 上可能提供更好的性能\n- 适合 ARM 架构或较旧的 x86 处理器\n- 对抗时序攻击具有更好的抗性\n- Google 等公司在移动设备和某些服务器环境中的首选算法\n\n在选择加密密钥时，推荐使用 RSA-2048 密钥，它在安全强度和性能表现之间有较好的平衡。RSA-4096 提供更高的安全性，但其解密操作会更慢，在高并发读取场景下可能影响性能。\n\n值得一提的是，加密后的数据会比原始数据稍大一些，主要是因为 AES-256-GCM 和 ChaCha20-Poly1305 加密算法需要添加认证标签（16 字节）和其他加密元数据。\n\n### 安全实践指南\n\n加密方案的安全性不仅取决于算法本身，更在于如何正确地管理和使用加密密钥。以下是一些重要的安全实践建议：\n\n**密钥管理是安全的核心**。私钥的密码应该足够强大——建议使用至少 16 个字符的组合，包含大小写字母、数字和特殊符号。建议通过环境变量传递密码，避免在命令行历史中泄露。\n\n虽然定期更换私钥是个好习惯，但需要注意的是，更换私钥意味着需要重新格式化整个文件系统。因此，在规划私钥轮换策略时，要权衡安全需求和业务连续性。\n\n**访问控制同样重要**。确保您的元数据引擎（无论是 Redis、MySQL 还是其他数据库）都配置了适当的认证和授权机制。对象存储的访问权限也应该遵循最小权限原则，只授予必要的操作权限。\n\n在网络层面，尽量使用 VPC 或私有网络来隔离元数据引擎和对象存储之间的通信流量，减少被中间人攻击的风险。\n\n**监控和审计**能帮助您及时发现异常情况。建议记录所有与加密相关的操作日志，定期检查密钥的使用模式，建立异常访问的检测机制。这样即使发生安全事件，您也能快速响应并采取应对措施。\n\n### 重要注意事项\n\n在使用 JuiceFS 加密功能时，有几个重要的技术限制需要了解：\n\n首先，客户端本地缓存的数据是**不加密的**。虽然只有 root 用户或文件所有者能够访问这些缓存数据，但如果您的使用场景要求端到端的完全加密，就需要考虑额外的保护措施，比如将缓存目录放在加密的文件系统或块存储上。\n\n其次，加密功能有一些固有的限制。文件元数据（如文件名、大小、权限等信息）是不加密的，解密后的数据在内存中也是明文状态。最重要的是，一旦为文件系统启用了加密，就无法再关闭这个功能——加密是不可逆的操作。\n\n在部署规划时，请考虑到加密会带来额外的 CPU 和内存开销。为了确保最佳的兼容性和稳定性，建议所有访问加密文件系统的客户端使用相同或兼容版本的 JuiceFS。\n\n### 适用场景分析\n\nJuiceFS 的加密功能特别适合这些场景：保护云端对象存储中的敏感数据、满足 GDPR、HIPAA 等合规性要求、长期安全存储重要业务数据，以及在多租户环境中实现数据隔离。\n\n不过，如果您需要客户端本地缓存也加密，或者想要为现有的文件系统后期添加加密功能，这个方案可能就不太适合。同样，对于性能要求极其苛刻的应用，或者需要频繁更换密钥但又不能接受重新格式化的场景，也需要慎重考虑。\n" }, { "path": "docs/zh_cn/security/posix_acl.md", "content": "---\ntitle: POSIX ACL\ndescription: 本文介绍了 JuiceFS 支持的 POSIX ACL 功能，以及如何启用和使用 ACL 权限。\nsidebar_position: 3\n---\n\nPOSIX ACL（Portable Operating System Interface for Unix - Access Control List）是 Unix-like 操作系统中的一种访问控制机制，可以对文件和目录的访问权限进行更细粒度的控制。\n\n## 版本及兼容性要求\n\n- JuiceFS 从 v1.2 版本开始支持 POSIX ACL；\n- 所有版本客户端都可以挂载没有开启 ACL 的卷，不论这些卷是由新版本客户端创建的还是由旧版本客户端创建的；\n- ACL 开启后暂不支持取消，因此 `--enable-acl` 选项是关联到卷的。\n\n:::caution 提示\n如果计划使用 ACL 功能，建议将所有客户端升级的最新版，避免旧版本客户端影响 ACL 的正确性。\n:::\n\n## 快速上手视频\n\n
\n \n
\n\n## 启用 ACL\n\n如前所述，可以用新版客户端在创建新卷时开启 ACL，也可以用新版客户端在已创建的卷上开启 ACL。\n\n### 创建新卷并开启 ACL\n\n```shell\njuicefs format --enable-acl sqlite3://myjfs.db myjfs\n```\n\n### 在已有卷上开启 ACL\n\n使用 `config` 命令为一个已创建的卷开启 ACL 功能：\n\n```shell\njuicefs config --enable-acl sqlite3://myjfs.db\n```\n\n## 使用方法\n\n为一个文件或目录设置 ACL 权限，可以使用 `setfacl` 命令，例如：\n\n```shell\nsetfacl -m u:alice:rw- /mnt/jfs/file\n```\n\n更多关于 POSIX ACL 的详细规则，请参考：\n\n- [POSIX Access Control Lists on Linux](https://www.usenix.org/legacy/publications/library/proceedings/usenix03/tech/freenix03/full_papers/gruenbacher/gruenbacher_html/main.html)\n- [setfacl](https://linux.die.net/man/1/setfacl)\n- [JuiceFS ACL 功能全解析，更精细的权限控制](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v12-beta-1-acl)\n\n## 注意事项\n\n- ACL 权限检测需要 [Linux kernel 4.9](https://lkml.iu.edu/hypermail/linux/kernel/1610.0/01531.html) 及以上版本；\n- 启用 ACL 会有额外的性能影响。但因为有内存缓存优化，大部分使用场景性能损耗都较低，可参考[压测结果](https://juicefs.com/zh-cn/blog/release-notes/juicefs-v12-beta-1-acl#03-%E6%80%A7%E8%83%BD)。\n" }, { "path": "docs/zh_cn/security/trash.md", "content": "---\nsidebar_position: 2\n---\n\n# 回收站\n\n:::note 注意\n此特性需要使用 1.0.0 及以上版本的 JuiceFS。旧版本 JuiceFS 欲使用回收站，需要在升级所有挂载点后通过 `config` 命令手动设置回收站，详见下方示范。\n:::\n\nJuiceFS 默认开启回收站功能，你删除的文件会被保存在文件系统根目录下的 `.trash` 目录内，保留指定时间后才将数据真正清理。在清理到来之前，通过 `df -h` 命令看到的文件系统使用量并不会减少，对象存储中的对象也会依然存在。\n\n不论你正在用 `format` 命令初始化文件系统，还是用 `config` 命令调整已有的文件系统，都可以用 [`--trash-days`](../reference/command_reference.mdx#format) 参数来指定回收站保留时长：\n\n```shell\n# 初始化新的文件系统\njuicefs format META-URL myjfs --trash-days=7\n\n# 修改已有文件系统\njuicefs config META-URL --trash-days=7\n\n# 设置为 0 以禁用回收站\njuicefs config META-URL --trash-days=0\n```\n\n另外，回收站自动清理依赖 JuiceFS 客户端的后台任务，为了保证后台任务能够正常执行，需要至少 1 个在线的挂载点，并且在挂载文件系统时不可以使用 [`--no-bgjob`](../reference/command_reference.mdx#mount-metadata-options) 参数。\n\n## 快速上手视频\n\n
\n \n
\n\n## 恢复文件 {#recover}\n\n文件被删除时，会根据删除时间，被保存在格式为 `.trash/YYYY-MM-DD-HH/[parent inode]-[file inode]-[file name]` 的目录，其中 `YYYY-MM-DD-HH` 就是删除操作的 UTC 时间。因此只需要确定文件的删除时间，就能在对应的目录中找到他们，来进行恢复操作。\n\n如果已经顺利找到想要恢复的文件，只需将其 `mv` 出来即可：\n\n```shells\nmv .trash/2022-11-30-10/[parent inode]-[file inode]-[file name] .\n```\n\n被删除的文件会完全丢失其目录结构，在回收站中“平铺”存储，但会在文件名保留父目录的 inode，如果你确实忘记了被误删的文件名，可以使用 [`juicefs info`](../reference/command_reference.mdx#info) 命令先找出父目录的 inode，然后顺藤摸瓜地定位到误删文件。\n\n假设挂载点为 `/jfs`，你误删了 `/jfs/data/config.json`，但无法直接通过 `config.json` 文件名来操作恢复文件（因为你忘了），可以用下方流程反查父目录 inode，然后在回收站中定位文件：\n\n```shell\n# 用 info 命令确定父目录 inode\njuicefs info /jfs/data\n\n# 在上方的输出中，关注 inode 字段，假设 /jfs/data 这个目录的 inode 为 3\n# 使用 find 命令，就能找出该目录下所有被删除的文件\nfind /jfs/.trash -name '3-*'\n\n# 将该目录下所有文件进行恢复\nmv /jfs/.trash/2022-11-30-10/3-* /jfs/data\n```\n\n需要注意，只有 root 用户具有回收站目录的写权限，因此只能使用 root 用户能用 `mv` 进行上述恢复操作。普通用户如果有这些文件的读权限，也可以用 `cp` 的方式读取文件，再写到新文件，虽然产生了存储空间浪费，但也能实现恢复文件的效果。\n\n如果误删了结构复杂的目录，用 `mv` 命令手动恢复原样会非常艰难，比方说：\n\n```shell\n$ tree data\ndata\n├── app1\n│   └── config\n│   └── config.json\n└── app2\n └── config\n └── config.json\n\n# 删除上方的复杂目录\n$ juicefs rmr data\n\n# 文件会在回收站内平铺存储，丢失目录结构\n$ tree .trash/2023-08-14-05\n.trash/2023-08-14-05\n├── 1-12-data\n├── 12-13-app1\n├── 12-15-app2\n├── 13-14-config\n├── 14-17-config.json\n├── 15-16-config\n└── 16-18-config.json\n```\n\n正因如此，JuiceFS v1.1 提供了 [`restore`](../reference/command_reference.mdx#restore) 子命令来快速恢复大量误删的文件，以上方目录结构为例，恢复操作如下：\n\n```shell\n# 先运行 restore 命令，在回收站内重建目录结构\n$ juicefs restore $META_URL 2023-08-14-05\n\n# 预览恢复完毕的目录结构，确定需要恢复的范畴\n# 既可以直接用下方命令完整恢复整个目录，也可以单独用 mv 命令恢复某一部分\n$ tree .trash/2023-08-14-05\n.trash/2023-08-14-05\n└── 1-12-data\n ├── app1\n │   └── config\n │   └── config.json\n └── app2\n └── config\n └── config.json\n\n# 增加 --put-back 参数将文件恢复至原位\njuicefs restore $META_URL 2023-08-14-05 --put-back\n```\n\n## 彻底删除文件 {#purge}\n\n当回收站中的文件到了过期时间，会被自动清理。需要注意的是，文件清理由 JuiceFS 客户端的后台任务（background job，也称 bgjob）执行，默认每小时清理一次，因此面对大量文件过期时，对象存储的清理速度未必和你期望的一样快，可能需要一些时间才能看到存储容量变化。\n\n如果你希望在过期时间到来之前彻底删除文件，需要使用 root 用户身份，用 [`juicefs rmr`](../reference/command_reference.mdx#rmr) 或系统自带的 `rm` 命令来删除回收站目录 `.trash` 中的文件，这样就能立刻释放存储空间。\n\n例如，彻底删除回收站中某个目录：\n\n```shell\njuicefs rmr .trash/2022-11-30-10/\n```\n\n如果希望更快速删除过期文件，可以挂载多个挂载点来突破单个客户端的删除速度上限。\n\n## 选择性跳过回收站 {#skip}\n\n开启回收站功能后，可以通过 chattr 命令为文件或目录设置's'属性，带有's'属性的文件或目录在被删除时不会进入回收站，而是直接从文件系统中移除。如果父目录设置了's'属性，则该目录下新创建的文件和子目录都会继承该属性，但是已存在的和之后转移过来的文件或目录不会继承这个属性。\n\n需要在挂载时启用`--enable-ioctl`选项，才能使用 chattr 命令修改文件属性。\n\n## 回收站和文件碎片 {#gc}\n\n在回收站里，除了因用户操作而产生的文件，还存在另一类对用户不可见的数据——覆写产生的文件碎片。关于文件碎片是怎么产生的，可以详细阅读[「JuiceFS 如何存储文件」](../introduction/architecture.md#how-juicefs-store-files)。总而言之，如果应用经常删除文件或者频繁覆盖写文件，会导致对象存储使用量远大于文件系统用量。\n\n虽然失效的文件碎片不能直接浏览、操作，但你可以通过 [`juicefs status`](../reference/command_reference.mdx#status) 命令来简单观测其规模：\n\n```shell\n# 下方 Trash Slices 就是失效的文件碎片统计\n$ juicefs status META-URL --more\n...\n Trash Files: 0 0.0/s\n Trash Files: 0.0 b (0 Bytes) 0.0 b/s\n Pending Deleted Files: 0 0.0/s\n Pending Deleted Files: 0.0 b (0 Bytes) 0.0 b/s\n Trash Slices: 27 26322.2/s\n Trash Slices: 783.0 b (783 Bytes) 753.1 KiB/s\nPending Deleted Slices: 0 0.0/s\nPending Deleted Slices: 0.0 b (0 Bytes) 0.0 b/s\n...\n```\n\n文件碎片也按照回收站设置的时间进行保留，这对数据安全同样具有重要意义：如果你不小心对文件进行了错误修改，或者覆盖写，一样可以通过元数据备份，把数据找回来（当然，前提是误操作之前已经设置好了元数据备份）。如果确实需要对误修改的文件进行恢复，则需要找回旧版元数据，挂载后手动将文件拷贝出来进行恢复，详见[备份与恢复](../administration/metadata_dump_load.md)。\n\n由于对用户不可见，这些失效的文件碎片无法轻易删除。如果规模巨大，确实需要主动清理它们，可以用以下操作手动处理：\n\n```shell\n# 临时禁用回收站\njuicefs config META-URL --trash-days 0\n\n# 如果有需要，可以手动触发再次运行碎片合并\njuicefs gc --compact\n\n# 运行 gc 命令删除泄露对象\njuicefs gc --delete\n\n# 操作完成后，记得重新开启回收站\n```\n\n## 访问权限 {#permission}\n\n所有用户均有权限浏览回收站，可以看到所有被删除的文件。然而 `.trash` 目录只有 root 具备写权限，但就算文件被移入回收站，也会保留原先的文件权限，因此在操作回收站内的文件时，注意权限问题并根据情况调整操作用户。\n\n关于回收站的权限问题，还需要注意：\n\n* 当 JuiceFS 客户端由非 root 用户启动时，需要在 mount 时指定 `-o allow_root` 参数，允许 root 用户访问文件系统，否则将无法正常清空回收站。\n* `.trash` 目录只能通过文件系统根目录访问，子目录挂载点无法访问。\n* 回收站内不允许用户自行创建新的文件，只有 root 才能删除或移动其中的文件。\n" }, { "path": "docs/zh_cn/tutorials/aliyun.md", "content": "---\ntitle: 在阿里云使用 JuiceFS\nsidebar_position: 7\nslug: /clouds/aliyun\n---\n\n如下图所示，JuiceFS 存储由数据库和对象存储共同驱动。存入 JuiceFS 的文件会按照一定的规则被拆分成固定大小的数据块存储在对象存储中，数据对应的元数据则会存储在数据库中。\n\n元数据完全独立存储，对文件的检索和处理并不会直接操作对象存储中的数据，而是先在数据库中操作元数据，只有当数据发生变化的时候，才会与对象存储交互。\n\n这样的设计可以有效缩减对象存储在请求数量上的费用，同时也能让我们显著感受到 JuiceFS 带来的性能提升。\n\n![JuiceFS-aliyun](../images/juicefs-aliyun.png)\n\n## 准备\n\n通过前面的架构描述，可以知道 JuiceFS 需要搭配数据库和对象存储一起使用。这里我们直接使用阿里云的 ECS 云服务器，结合云数据库和 OSS 对象存储。\n\n在创建云计算资源时，尽量选择在相同的区域，这样可以让资源之间通过内网线路相互访问，避免使用公网线路产生额外的流量费用。\n\n### 一、云服务器 ECS\n\nJuiceFS 对服务器硬件没有特殊要求，一般来说，云平台上最低配的云服务器也能稳定使用 JuiceFS，通常你只需要选择能够满足自身业务的配置即可。\n\n需要特别说明的是，你不需要为使用 JuiceFS 重新购买服务器或是重装系统，JuiceFS 没有业务入侵性，不会对你现有的系统和程序造成任何的干扰，你完全可以在正在运行的服务器上安装和使用 JuiceFS。\n\nJuiceFS 默认会占用不超过 1GB 的硬盘空间作为缓存，可以根据需要调整缓存空间的大小。该缓存是客户端与对象存储之间的一个数据缓冲层，选择性能更好的云盘，可以获得更好的性能表现。\n\n在操作系统方面，阿里云 ECS 提供的所有操作系统都可以安装 JuiceFS。\n\n**本文使用的 ECS 配置如下：**\n\n| **实例规格** | ecs.t5-lc1m1.small |\n| ---------------- | ------------------------ |\n| **CPU** | 1 核 |\n| **内存** | 1 GB |\n| **存储** | 40 GB |\n| **操作系统** | Ubuntu Server 20.04 64 位 |\n| **地域及可用区** | 华东 2（上海） |\n\n### 二、云数据库\n\nJuiceFS 会将数据对应的元数据全部存储在独立的数据库中，目前已开放支持的数据库有 Redis、MySQL、PostgreSQL、SQLite，以及 OceanBase。\n\n根据数据库类型的不同，带来的元数据性能和可靠性表现也各不相同。比如 Redis 是完全运行在内存上的，它能提供极致的性能，但运维难度较高，可靠性相对低。而 MySQL、PostgreSQL 是关系型数据库，性能不如 Redis，但运维难度不高，可靠性也有一定的保障。SQLite 是单机单文件关系型数据库，性能较低，也不适合用于大规模数据存储，但它免配置，适合单机少量数据存储的场景。相比之下，OceanBase 是一款分布式关系型数据库，能够在提供高性能的同时，确保数据的一致性和高可靠性（RTO < 8s）。它特别适合金融、零售、电信等对事务一致性和分布式能力要求较高的场景，使 JuiceFS 在处理海量元数据时可以实现更高效率、更低延迟和更强稳定性，从而满足现代分布式存储系统对底层数据库的苛刻要求。\n\n如果只是为了评估 JuiceFS 的功能，你可以在 ECS 云服务器手动搭建数据库使用。当你要在生产环境使用 JuiceFS 时，如果没有专业的数据库运维团队，阿里云的云数据库服务通常是更好的选择。\n\n当然，如果你愿意，也可以使用其他云平台上提供的云数据库服务。但在这种情况下，你只能通过公网访问云数据库，也就是说，你必须向公网暴露数据库的端口，这存在极大的安全风险，最好不要这样使用。\n\n如果必须通过公网访问数据库，可以通过云数据库控台提供的白名单功能，严格限制允许访问数据库的 IP 地址，从而提升数据的安全性。从另一个角度说，如果你通过公网无法成功连接云数据库，那么可以检查数据库的白名单，检查是不是该设置限制了你的访问。\n\n| 数据库 | Redis | MySQL、PostgreSQL | SQLite | OceanBase |\n| :----------: | :----------------------: | :------------------------: | :--------------------: | :---------------------: |\n| **性能** | 强 | 适中 | 弱 | 强 |\n| **运维门槛** | 高 | 适中 | 低 | 适中 |\n| **可靠性** | 低 | 适中 | 低 | 高 |\n| **应用场景** | 海量数据、分布式高频读写 | 海量数据、分布式中低频读写 | 少量数据单机中低频读写 | 分布式场景、强事务一致性、高可靠性要求 |\n\n**本文使用了[云数据 Redis 版](https://www.aliyun.com/product/kvstore)，以下连接地址只是为了演示目的编制的伪地址：**\n\n| Redis 版本 | 5.0 社区版 |\n|--------------|----------------------------------------|\n| **实例规格** | 256M 标准版 - 单副本 |\n| **连接地址** | `herald-sh-abc.redis.rds.aliyuncs.com` |\n| **可用区** | 上海 |\n\n### 三、对象存储 OSS\n\nJuiceFS 会将所有的数据都存储到对象存储中，它支持几乎所有的对象存储服务。但为了获得最佳的性能，当使用阿里云 ECS 时，搭配阿里云 OSS 对象存储通常是最优选择。不过请注意，将 ECS 和 OSS Bucket 选择在相同的地区，这样才能通过阿里云的内网线路进行访问，不但延时低，而且不需要额外的流量费用。\n\n当然，如果你愿意，也可以使用其他云平台提供的对象存储服务，但不推荐这样做。首先，通过阿里云 ECS 访问其他云平台的对象存储要走公网线路，对象存储会产生流量费用，而且这样的访问延时相比也会更高，可能会影响 JuiceFS 的性能发挥。\n\n阿里云 OSS 有不同的存储级别，由于 JuiceFS 需要与对象存储频繁交互，建议使用标准存储。你可以搭配 OSS 资源包使用，降低对象存储的使用成本。\n\n### API 访问秘钥\n\n阿里云 OSS 需要通过 API 进行访问，你需要准备访问秘钥，包括 `Access Key ID` 和 `Access Key Secret` ，[点此查看](https://help.aliyun.com/document_detail/38738.html)获取方式。\n\n> **安全建议**：显式使用 API 访问秘钥可能导致密钥泄露，推荐为云服务器分配 [RAM 服务角色](https://help.aliyun.com/document_detail/93689.htm)。当一台 ECS 被授予 OSS 操作权限以后，无需使用 API 访问秘钥即可访问 OSS。\n\n## 安装\n\n当前使用的是 Ubuntu Server 20.04 64 位系统，依次执行以下命令可以安装最新版本客户端。\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n你也可以访问 [JuiceFS GitHub Releases](https://github.com/juicedata/juicefs/releases) 页面选择其他版本。\n\n执行命令，看到返回 `juicefs` 的命令帮助信息，代表客户端安装成功。\n\n```shell\n$ juicefs\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 0.15.2 (2021-07-07T05:51:36Z 4c16847)\n\nCOMMANDS:\n format format a volume\n mount mount a volume\n umount unmount a volume\n gateway S3-compatible gateway\n sync sync between two storage\n rmr remove directories recursively\n info show internal information for paths or inodes\n bench run benchmark to read/write/stat big/small files\n gc collect any leaked objects\n fsck Check consistency of file system\n profile analyze access log\n status show status of JuiceFS\n warmup build cache for target directories/files\n dump dump metadata into a JSON file\n load load metadata from a previously dumped JSON file\n help, h Shows a list of commands or help for one command\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q only warning and errors (default: false)\n --trace enable trace log (default: false)\n --no-agent disable pprof (:6060) agent (default: false)\n --help, -h show help (default: false)\n --version, -V print only the version (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\nJuiceFS 具有良好的跨平台兼容性，同时支持在 Linux、Windows 和 macOS 上使用。本文着重介绍 JuiceFS 在 Linux 系统上的安装和使用，如果你需要了解其他系统上的安装方法，请[查阅文档](../getting-started/installation.md)。\n\n## 创建 JuiceFS 存储\n\nJuiceFS 客户端安装好以后，现在就可以使用前面准备好的 Redis 数据库和 OSS 对象存储来创建 JuiceFS 存储了。\n\n严格意义上说，这一步操作应该叫做“Format a volume”，即格式化一个卷。但考虑到有很多用户可能不了解或者不关心文件系统的标准术语，所以简单起见，我们就直白的把这个过程叫做“创建 JuiceFS 存储”。\n\n以下命令使用 JuiceFS 客户端提供的 `format` 子命令创建了一个名为 `mystor` 的存储，即文件系统：\n\n```shell\n$ juicefs format \\\n --storage oss \\\n --bucket https:// \\\n --access-key \\\n --secret-key \\\n redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n mystor\n```\n\n**选项说明：**\n\n- `--storage`：指定对象存储类型，[点此查看](../reference/how_to_set_up_object_storage.md#supported-object-storage) JuiceFS 支持的对象存储。\n- `--bucket`：对象存储的 Bucket 域名。当使用阿里云 OSS 时，只需填写 bucket 名称即可，无需填写完整的域名，JuiceFS 会自动识别并补全地址。\n- `--access-key` 和 `--secret-key`：访问对象存储 API 的秘钥对，[点此查看](https://help.aliyun.com/document_detail/38738.html)获取方式。\n\n> Redis 6.0 身份认证需要用户名和密码两个参数，地址格式为 `redis://username:password@redis-server-url:6379/1`。目前阿里云数据库 Redis 版只提供 Reids 4.0 和 5.0 两个版本，认证身份只需要密码，在设置 Redis 服务器地址时只需留空用户名即可，例如：`redis://:password@redis-server-url:6379/1`\n\n使用 RAM 角色绑定 ECS 时，创建 JuiceFS 存储只需指定 `--storage` 和 `--bucket` 两个选项，无需提供 API 访问秘钥。命令可以改写成：\n\n```shell\n$ juicefs format \\\n --storage oss \\\n --bucket https://mytest.oss-cn-shanghai.aliyuncs.com \\\n redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n mystor\n```\n\n看到类似下面的输出，代表文件系统创建成功了。\n\n```shell\n2021/07/13 16:37:14.264445 juicefs[22290] : Meta address: redis://@herald-sh-abc.redis.rds.aliyuncs.com:6379/1\n2021/07/13 16:37:14.277632 juicefs[22290] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/07/13 16:37:14.281432 juicefs[22290] : Ping redis: 3.609453ms\n2021/07/13 16:37:14.527879 juicefs[22290] : Data uses oss://mytest/mystor/\n2021/07/13 16:37:14.593450 juicefs[22290] : Volume is formatted as {Name:mystor UUID:4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b Storage:oss Bucket:https://mytest340 AccessKey:LTAI4G4v6ioGzQXy56m3XDkG SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n## 挂载 JuiceFS 存储\n\n文件系统创建完成，对象存储相关的信息会被存入数据库，挂载时无需再输入对象存储的 Bucket 和秘钥等信息。\n\n使用 `mount` 子命令，将文件系统挂载到 `/mnt/jfs` 目录：\n\n```shell\nsudo juicefs mount -d redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1 /mnt/jfs\n```\n\n> **注意**：挂载文件系统时，只需填写 Redis 数据库地址，不需要文件系统名称。默认的缓存路径为 `/var/jfsCache`，请确保当前用户有足够的读写权限。\n\n看到类似下面的输出，代表文件系统挂载成功。\n\n```shell\n2021/07/13 16:40:37.088847 juicefs[22307] : Meta address: redis://@herald-sh-abc.redis.rds.aliyuncs.com/1\n2021/07/13 16:40:37.101279 juicefs[22307] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/07/13 16:40:37.104870 juicefs[22307] : Ping redis: 3.408807ms\n2021/07/13 16:40:37.384977 juicefs[22307] : Data use oss://mytest/mystor/\n2021/07/13 16:40:37.387412 juicefs[22307] : Disk cache (/var/jfsCache/4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b/): capacity (1024 MB), free ratio (10%), max pending pages (15)\n.2021/07/13 16:40:38.410742 juicefs[22307] : OK, mystor is ready at /mnt/jfs\n```\n\n使用 `df` 命令，可以看到文件系统的挂载情况：\n\n```shell\n$ df -Th\n文件系统 类型 容量 已用 可用 已用% 挂载点\nJuiceFS:mystor fuse.juicefs 1.0P 64K 1.0P 1% /mnt/jfs\n```\n\n文件系统挂载成功以后，现在就可以像使用本地硬盘那样，在 `/mnt/jfs` 目录中存储数据了。\n\n> **多主机共享**：JuiceFS 存储支持被多台云服务器同时挂载使用，你可以在其他 ECS 上安装 JuiceFS 客户端，然后使用 `redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1` 数据库地址挂载文件系统到每一台主机上。\n\n## 查看文件系统状态\n\n使用 JuiceFS 客户端的 `status` 子命令可以查看一个文件系统的基本信息和连接状态。\n\n```shell\n$ juicefs status redis://:@herald-sh-abc.redis.rds.aliyuncs.com:6379/1\n\n2021/07/13 16:56:17.143503 juicefs[22415] : Meta address: redis://@herald-sh-abc.redis.rds.aliyuncs.com:6379/1\n2021/07/13 16:56:17.157972 juicefs[22415] : maxmemory_policy is \"volatile-lru\", please set it to 'noeviction'.\n2021/07/13 16:56:17.161533 juicefs[22415] : Ping redis: 3.392906ms\n{\n \"Setting\": {\n \"Name\": \"mystor\",\n \"UUID\": \"4ad0bb86-6ef5-4861-9ce2-a16ac5dea81b\",\n \"Storage\": \"oss\",\n \"Bucket\": \"https://mytest\",\n \"AccessKey\": \"\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0\n },\n \"Sessions\": [\n {\n \"Sid\": 3,\n \"Heartbeat\": \"2021-07-13T16:55:38+08:00\",\n \"Version\": \"0.15.2 (2021-07-07T05:51:36Z 4c16847)\",\n \"Hostname\": \"demo-test-sh\",\n \"MountPoint\": \"/mnt/jfs\",\n \"ProcessID\": 22330\n }\n ]\n}\n```\n\n## 卸载 JuiceFS 存储\n\n使用 JuiceFS 客户端提供的 `umount` 命令即可卸载文件系统，比如：\n\n```shell\nsudo juicefs umount /mnt/jfs\n```\n\n> **注意**：强制卸载使用中的文件系统可能导致数据损坏或丢失，请务必谨慎操作。\n\n## 开机自动挂载\n\n请参考[「启动时自动挂载 JuiceFS」](../administration/mount_at_boot.md)\n" }, { "path": "docs/zh_cn/tutorials/aws.md", "content": "---\ntitle: 在 AWS 上使用 JuiceFS\nsidebar_position: 4\nslug: /clouds/aws\n---\n\n亚马逊云（AWS）是全球领先的云计算平台，提供几乎所有类型的云计算服务。AWS 丰富的产品线，为创建和使用 JuiceFS 文件系统提供了灵活的选择。\n\n## 可以在哪里使用 JuiceFS {#where-can-juicefs-be-used}\n\nJuiceFS 具有丰富的 API 接口，对 AWS 而言，通常可以在以下产品中使用：\n\n- **Amazon EC2**：通过挂载 JuiceFS 文件系统来使用\n- **Amazon Elastic Kubernetes Service（EKS）**：通过 JuiceFS CSI 驱动使用\n- **Amazon EMR**：通过 JuiceFS Hadoop Java SDK 使用\n\n## 准备 {#preparation}\n\n一个 JuiceFS 文件系统由两部分组成：\n\n1. **对象存储**：用于数据存储\n2. **元数据引擎**：用于元数据存储的数据库\n\n可以根据具体需求，选择在 AWS 上使用全托管的数据库和 S3 对象存储，或者在 EC2、EKS 上自行部署。\n\n:::tip\n本文着重介绍使用 AWS 全托管的服务创建 JuiceFS 文件系统的方法，对于自托管的情况，请查阅[「JuiceFS 支持的元数据引擎」](../reference/how_to_set_up_metadata_engine.md)和[「JuiceFS 支持的对象存储」](../reference/how_to_set_up_object_storage.md)以及相应程序文档。\n:::\n\n### 对象存储 {#object-storage}\n\nS3 是 AWS 提供的对象存储服务，可以根据需要在相应地区创建 bucket，也可以通过 [IAM 角色授权](../reference/how_to_set_up_object_storage.md#aksk)让 JuiceFS 客户端自动创建 bucket。\n\nAmazon S3 提供多种[存储类](https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/userguide/storage-class-intro.html)，例如：\n\n- **S3 Standard**：标准存储，适用于频繁访问数据的通用型存储，实时访问，无取回费用。\n- **S3 Standard-IA**：低频存储，适用于长期需要但访问频率不太高的数据，实时访问，有取回费用。\n- **S3 Glacier**：归档存储，适用于长期存档几乎不访问的数据，访问前需解冻。\n\n你可以在创建或者挂载 JuiceFS 文件系统时设置存储类，具体请参考[文档](../reference/how_to_set_up_object_storage.md#storage-class)。建议优先选择标准的存储类，其他的存储类虽然有更低的单位存储价格，但会涉及最低存储时长要求和检索（取回）费用。\n\n另外，访问对象存储服务需要通过 Access Key（也叫 access key ID）和 Secret Key（也叫 secret access key）验证用户身份，可以参照文档[「管理 IAM 用户的访问密钥」](https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/id_credentials_access-keys.html)进行创建。当通过 EC2 云服务器访问 S3 时，还可以为 EC2 分配 [IAM 角色](https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/id_roles.html)，实现在 EC2 上免密钥调用 S3 API。\n\n### 数据库 {#database}\n\nAWS 提供了多种基于网络的全托管数据库，可以用于构建 JuiceFS 的元数据引擎，主要有：\n\n- **Amazon MemoryDB for Redis**（以下简称 MemoryDB）：持久的 Redis 内存数据库服务，可提供超快的性能。\n- **Amazon RDS**：全托管的 MariaDB、MySQL、PostgreSQL 等数据库。\n\n:::note 注意\n虽然 Amazon ElastiCache for Redis（以下简称 ElastiCache）也提供兼容 Redis 协议的服务，但是相比 MemoryDB 来说，ElastiCache 无法提供「强一致性保证」，因此更推荐使用 MemoryDB。\n:::\n\n## 在 EC2 上使用 JuiceFS {#using-juicefs-on-ec2}\n\n### 安装 JuiceFS 客户端 {#installing-the-juicefs-client}\n\n请根据 EC2 所使用的操作系统，参考[安装](../getting-started/installation.md)文档安装最新的 JuiceFS 社区版客户端。\n\n这里以 Linux 系统为例，使用一键安装脚本自动安装客户端：\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n### 创建文件系统 {#creating-a-file-system}\n\n#### 准备对象存储 {#preparing-object-storage}\n\n可以通过创建一个拥有 [AmazonS3FullAccess](https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-amazons3fullaccess) 权限的 IAM 角色分配给 EC2，从而无需使用 Access Key 和 Secret Key 即可直接在 EC2 上创建和使用 S3 Bucket。\n\n#### 准备数据库 {#preparing-the-database}\n\n这里以 MemoryDB 为例，请参考[「Redis 最佳实践」](../administration/metadata/redis_best_practices.md)及 AWS 文档创建数据库。\n\n为了让 EC2 能够访问 Redis 集群，需要将它们创建在相同的 VPC，或者为 Redis 集群的安全组添加规则允许 EC2 实例访问。\n\n:::note 注意\n如果创建的是 Redis 7.0 版本集群，需要安装 JuiceFS v1.1 及以上版本客户端。\n:::\n\n#### 格式化文件系统 {#formatting-file-system}\n\n```shell\njuicefs format --storage s3 \\\n --bucket https://s3.ap-east-1.amazonaws.com/myjfs \\\n rediss://clustercfg.myredis.hc79sw.memorydb.ap-east-1.amazonaws.com:6379/1 \\\n myjfs\n```\n\n### 挂载文件系统 {#mounting-file-system}\n\n```shell\nsudo juicefs mount -d \\\n rediss://clustercfg.myredis.hc79sw.memorydb.ap-east-1.amazonaws.com:6379/1 \\\n /mnt/myjfs\n```\n\n对于通过 IAM 角色授权 S3 访问创建的文件系统，如果需要在 AWS 外部挂载使用，需要使用 `juicefs config` 为文件系统添加 Access Key 和 Secret Key：\n\n```shell\njuicefs config \\\n --access-key= \\\n --secret-key= \\\n rediss://clustercfg.myredis.hc79sw.memorydb.ap-east-1.amazonaws.com:6379/1\n```\n\n### 开机自动挂载 {#mounting-at-boot}\n\n请参考文档[启动时自动挂载 JuiceFS](../administration/mount_at_boot.md)。\n\n## 在 Amazon EKS 上使用 JuiceFS {#using-juicefs-on-amazon-eks}\n\nAmazon EKS 支持[三种节点类型](https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/eks-compute.html)：\n\n- **EKS 托管节点组**：使用 Amazon EC2 作为计算节点\n- **自行管理的节点**：使用 Amazon EC2 作为计算节点\n- **Fargate**：一个无服务器的计算引擎\n\nFargate 类型节点暂不支持安装 JuiceFS CSI 驱动，请使用「EKS 托管节点组」或者「自行管理的节点」类型。\n\nAmazon EKS 是标准的 Kubernetes 集群，可以使用 `eksctl`、`kubectl`、`helm` 等工具进行管理，请查阅 [JuiceFS CSI 驱动文档](/docs/zh/csi/introduction)了解如何安装和使用。\n\n## 在 Amazon EMR 上使用 JuiceFS {#using-juicefs-on-amazon-emr}\n\n请参考文档[「在 Hadoop 生态使用 JuiceFS」](../deployment/hadoop_java_sdk.md)。\n" }, { "path": "docs/zh_cn/tutorials/digitalocean.md", "content": "---\ntitle: 在 DigitalOcean 使用 JuiceFS\nsidebar_position: 6\nslug: /clouds/digitalocean\n---\n\nJuiceFS 是面向云设计的，使用云平台开箱即用的存储和数据库服务，最快几分钟就能完成配置投入使用，本文以 DigitalOcean 平台为例，介绍如何在云计算平台上快速简单的安装和使用 JuiceFS。\n\n## 准备工作\n\nJuiceFS 由存储和数据库组合驱动，因此你需要准备的东西应该包括：\n\n### 1. 云服务器\n\nDigitalOcean 上的云服务器被称为 Droplet。你不需要为使用 JuiceFS 而单独购买新的 Droplet，哪个云服务器上需要使用 JuiceFS 存储，就在它上面安装 JuiceFS 客户端即可。\n\n#### 硬件配置\n\nJuiceFS 对硬件配置没有特殊的要求，任何规格的 Droplet 都能稳定的使用。但建议选择性能更好的 SSD 并预留至少 1GB 的容量提供给 JuiceFS 作为本地缓存使用。\n\n#### 操作系统\n\nJuiceFS 支持 Linux、BSD、macOS 和 Windows，在本文中，我们会以 Ubuntu Server 20.04 为例进行介绍。\n\n### 2. 对象存储\n\nJuiceFS 使用对象存储来存储所有的数据，在 DigitalOcean 上使用 Spaces 是最简便的方案。Spaces 是一个 S3 兼容的对象存储服务，开箱即用。在创建时建议选择与 Droplet 相同的区域，这样可以获得最佳的访问速度，同时也能避免额外的流量开销。\n\n当然，你也可以使用其他平台的对象存储服务，或是在 Droplet 上使用 Ceph 或 MinIO 手动搭建。总之，你可以自由选择要使用的对象存储，只要确保 JuiceFS 客户端能够访问到对象存储的 API 就可以。\n\n这里，我们创建了一个名为 `juicefs` 的 Spaces 存储桶，区域为新加坡 `sgp1`，它的访问地址为：\n\n- `https://juicefs.sgp1.digitaloceanspaces.com`\n\n另外，还需要在 API 菜单创建 `Spaces access keys`，JuiceFS 需要用它访问 Spaces 的 API。\n\n### 3. 数据库\n\n与一般的文件系统不同，JuiceFS 将数据所对应的所有元数据都存储在独立的数据库，存储的数据规模越大性能越出色。目前，JuiceFS 支持 Redis、TiKV、MySQL/MariaDB、PostgreSQL、SQLite 等常见数据库，同时也在持续开发对其他数据库的支持。如果你需要的数据库暂未支持，请提交 [Issue](https://github.com/juicedata/juicefs/issues) 反馈。\n\n在性能、规模和可靠性等方面，每种数据库都有各自的优缺点，你应该根据实际的场景需要进行选择。\n\n在数据库的选择方面请不要有顾虑，JuiceFS 客户端提供了元数据迁移功能，你可以将元数据从一种数据库中轻松的导出并迁移到其他的数据库中。\n\n本文我们使用 DigitalOcean 的 Redis 6 数据库托管服务，区域选择 `新加坡`，选择与已存在的 Droplet 相同的 VPC 私有网络。创建 Redis 大概需要 5 分钟左右的时间，我们跟随设置向导对数据库进行初始化设置。\n\n![DigitalOcean-Redis-guide](../images/digitalocean-redis-guide.png)\n\n默认情况下 Redis 允许所有入站连接，出于安全考虑，应该在设置向导的安全设置环节，在 `Add trusted sources` 中选中有权访问 Redis 的 Droplet，即仅允许选中的主机访问 Redis。\n\n在数据回收策略的设置环节，建议选择 `noeviction`，即当内存耗尽时，仅报告错误，不回收任何数据。\n\n> **注意**：为了确保元数据的安全和完整，回收策略请不要选择 `allkeys-lru` 和 `allkey-random`。\n\nRedis 的访问地址可以从控制台的 `Connection Details` 中找到，如果所有计算资源都在 DigitalOcean，则建议优先使用 VPC 私有网络进行连接，这样能最大程度的提升安全性。\n\n![DigitalOcean-Redis-url](../images/digitalocean-redis-url.png)\n\n## 安装和使用\n\n### 1. 安装 JuiceFS 客户端\n\n我们当前使用的是 Ubuntu Server 20.04，执行以下命令即可安装最新版本客户端。\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n执行命令，看到返回 `juicefs` 的命令帮助信息，代表客户端安装成功。\n\n```shell\n$ juicefs\n\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 0.16.2 (2021-08-25T04:01:15Z 29d6fee)\n\nCOMMANDS:\n format format a volume\n mount mount a volume\n umount unmount a volume\n gateway S3-compatible gateway\n sync sync between two storage\n rmr remove directories recursively\n info show internal information for paths or inodes\n bench run benchmark to read/write/stat big/small files\n gc collect any leaked objects\n fsck Check consistency of file system\n profile analyze access log\n stats show runtime stats\n status show status of JuiceFS\n warmup build cache for target directories/files\n dump dump metadata into a JSON file\n load load metadata from a previously dumped JSON file\n help, h Shows a list of commands or help for one command\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q only warning and errors (default: false)\n --trace enable trace log (default: false)\n --no-agent disable pprof (:6060) agent (default: false)\n --help, -h show help (default: false)\n --version, -V print only the version (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\n另外，你也可以访问 [JuiceFS GitHub Releases](https://github.com/juicedata/juicefs/releases) 页面选择其他版本进行手动安装。\n\n### 2. 创建文件系统\n\n创建文件系统使用 `format` 子命令，格式为：\n\n```shell\njuicefs format [command options] META-URL NAME\n```\n\n以下命令创建了一个名为 `mystor` 的文件系统：\n\n```shell\n$ juicefs format \\\n --storage space \\\n --bucket https://juicefs.sgp1.digitaloceanspaces.com \\\n --access-key \\\n --secret-key \\\n rediss://default:your-password@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1 \\\n mystor\n```\n\n**参数说明：**\n\n- `--storage`：指定数据存储引擎，这里使用的是 `space`，点此查看所有[支持的存储](../reference/how_to_set_up_object_storage.md)。\n- `--bucket`：指定存储桶访问地址。\n- `--access-key` 和 `--secret-key`：指定访问对象存储 API 的秘钥。\n- DigitalOcean 托管的 Redis 需要使用 TLS/SSL 加密访问，因此需要使用 `rediss://` 协议头，链接最后添加的 `/1` 代表使用 Redis 的 1 号数据库。\n\n看到类似下面的输出，代表文件系统创建成功。\n\n```shell\n2021/08/23 16:36:28.450686 juicefs[2869028] : Meta address: rediss://default@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:36:28.481251 juicefs[2869028] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/08/23 16:36:28.481763 juicefs[2869028] : Ping redis: 331.706µs\n2021/08/23 16:36:28.482266 juicefs[2869028] : Data uses space://juicefs/mystor/\n2021/08/23 16:36:28.534677 juicefs[2869028] : Volume is formatted as {Name:mystor UUID:6b0452fc-0502-404c-b163-c9ab577ec766 Storage:space Bucket:https://juicefs.sgp1.digitaloceanspaces.com AccessKey:7G7WQBY2QUCBQC5H2DGK SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n### 3. 挂载文件系统\n\n挂载文件系统使用 `mount` 子命令，使用 `-d` 参数以守护进程的形式挂载。以下命令将刚刚创建的文件系统挂载到当前目录下的 `mnt` 目录：\n\n```shell\n$ sudo juicefs mount -d \\\n rediss://default:your-password@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1 mnt\n```\n\n使用 sudo 执行挂载操作的目的是为了让 JuiceFS 能够有权限在 `/var/` 下创建缓存目录。值得注意的是，在挂载文件系统时，只需要指定`数据库地址`和`挂载点`，并不需要指定文件系统的名称。\n\n看到类似下面的输出，代表文件系统挂载成功。\n\n```shell\n2021/08/23 16:39:14.202151 juicefs[2869081] : Meta address: rediss://default@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:39:14.234925 juicefs[2869081] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/08/23 16:39:14.235536 juicefs[2869081] : Ping redis: 446.247µs\n2021/08/23 16:39:14.236231 juicefs[2869081] : Data use space://juicefs/mystor/\n2021/08/23 16:39:14.236540 juicefs[2869081] : Disk cache (/var/jfsCache/6b0452fc-0502-404c-b163-c9ab577ec766/): capacity (1024 MB), free ratio (10%), max pending pages (15)\n2021/08/23 16:39:14.738416 juicefs[2869081] : OK, mystor is ready at mnt\n```\n\n使用 `df` 命令，可以看到文件系统的挂载情况：\n\n```shell\n$ df -Th\n文件系统 类型 容量 已用 可用 已用% 挂载点\nJuiceFS:mystor fuse.juicefs 1.0P 64K 1.0P 1% /home/herald/mnt\n```\n\n从挂载命令的输出信息中可以看到，JuiceFS 默认设置了 1024 MB 的作为本地缓存。设置更大的缓存，可以让 JuiceFS 有更好的性能表现，可以在挂载文件系统时通过 `--cache-size` 选项设置缓存（单位 MiB），例如，设置 20GB 的本地缓存：\n\n```shell\n$ sudo juicefs mount -d --cache-size 20000 \\\n rediss://default:your-password@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1 mnt\n```\n\n文件系统挂载成功以后，就可以像使用本地硬盘那样，在 `~/mnt` 目录中存储数据了。\n\n### 4. 查看文件系统\n\n使用 `status` 子命令可以查看一个文件系统的基本信息和连接状态，只需指定数据库访问地址即可。\n\n```shell\n$ juicefs status rediss://default:bn8l7ui2cun4iaji@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:48:48.567046 juicefs[2869156] : Meta address: rediss://default@private-db-redis-sgp1-03138-do-user-2500071-0.b.db.ondigitalocean.com:25061/1\n2021/08/23 16:48:48.597513 juicefs[2869156] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/08/23 16:48:48.598193 juicefs[2869156] : Ping redis: 491.003µs\n{\n \"Setting\": {\n \"Name\": \"mystor\",\n \"UUID\": \"6b0452fc-0502-404c-b163-c9ab577ec766\",\n \"Storage\": \"space\",\n \"Bucket\": \"https://juicefs.sgp1.digitaloceanspaces.com\",\n \"AccessKey\": \"7G7WQBY2QUCBQC5H2DGK\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0\n },\n \"Sessions\": [\n {\n \"Sid\": 1,\n \"Heartbeat\": \"2021-08-23T16:46:14+08:00\",\n \"Version\": \"0.16.2 (2021-08-25T04:01:15Z 29d6fee)\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869091\n },\n {\n \"Sid\": 2,\n \"Heartbeat\": \"2021-08-23T16:47:59+08:00\",\n \"Version\": \"0.16.2 (2021-08-25T04:01:15Z 29d6fee)\",\n \"Hostname\": \"ubuntu-s-1vcpu-1gb-sgp1-01\",\n \"MountPoint\": \"/home/herald/mnt\",\n \"ProcessID\": 2869146\n }\n ]\n}\n```\n\n### 5. 卸载文件系统\n\n使用 `umount` 子命令卸载文件系统，比如：\n\n```shell\nsudo juicefs umount ~/mnt\n```\n\n> **注意**：强制卸载使用中的文件系统可能导致数据损坏或丢失，请务必谨慎操作。\n\n### 6. 开机自动挂载\n\n请参考[「启动时自动挂载 JuiceFS」](../administration/mount_at_boot.md)\n\n### 7. 多主机共享挂载\n\nJuiceFS 文件系统支持被多台云服务器同时挂载，而且对云服务器的地理位置没有要求，可以很容的实现同平台之间、跨云平台之间、公有云和私有云之间服务器的数据实时共享。\n\n不单如此，JuiceFS 的共享挂载功能还能提供数据的强一致性保证，在多台服务器挂载了同一个文件系统时，文件系统上确认的写入会在所有主机上实时可见。\n\n使用共享挂载功能，务必要确保组成文件系统的数据库和对象存储服务，能够被每一台要挂载它的主机正常访问。在本文的演示环境中，Spaces 对象存储是对整个互联网开放访问的，只要使用正确的秘钥就能够通过 API 进行读写。但对于平台托管的 Redis 数据库，你需要合理的配置访问策略，确保平台外的主机有访问权限。\n\n在使用多主机共享挂载功能时，首先在任何一台主机上创建文件系统，然后在其他主机上安装 JuiceFS 客户端，使用同一个数据库地址通过 `mount` 命令挂载即可。特别注意，文件系统只需创建一次，不应该也不需要在其他主机上重复执行文件系统创建操作。\n" }, { "path": "docs/zh_cn/tutorials/juicefs_on_colab.md", "content": "---\ntitle: 在 Colab 上通过 Google CloudSQL 和 GCS 使用 JuiceFS\nsidebar_position: 5\nslug: /juicefs_on_colab\n---\n\n[Colaboratory](https://colab.research.google.com), 或者简称“Colab”, 是 Google Research 的产品，它允许任何人通过浏览器编写和执行 Python 代码，特别适合机器学习、数据分析和教育。\nColab 支持从 Google Drive 将文件上传到 Colab 实例或从 Colab 实例下载文件。然而在某些情况下，Google Drive 可能不太方便与 Colab 一起使用，在这种情况下，JuiceFS 是一个很有用的工具，因为他允许在 Colab 实例之间，或在 Colab 实例与本地或本地机器之间轻松的同步文件。[这里是一个使用了 JuiceFS 的 Colab 笔记本示例](https://colab.research.google.com/drive/1wA8vRwqiihXkI6ViDU8Ud868UeYtmCo5)\n\n说明下在 Colab 环境中使用 JuiceFS 的必要步骤。我们使用 Google CloudSQL 作为 JuiceFS 的元数据引擎，使用 Google Cloud Storage (GCS) 作为 JuiceFS 的对象存储。其他类型的元数据引擎与对象存储可以参考 [如何设置元数据引擎](../reference/how_to_set_up_metadata_engine.md) 和 [如何设置对象存储](../reference/how_to_set_up_object_storage.md)。\n\n下面将要提到的很多步骤你可以也参考 [快速上手指南](../getting-started/for_distributed.md)。\n\n## 步骤\n\n1. 在任何一个可以访问 Google Cloud 资源的机器或者实例上格式化一个 JuiceFS 文件系统\n2. 挂载 JuiceFS 文件系统到 Colab Notebook 上\n3. 愉快的跨平台跨机器分享存储的文件\n\n## 先决条件\n\n在这个示例中，我们使用了 Google Cloud 平台的 CloudSQL 和 Google Cloud Storage (GCS) 来创建一个高性能的 JuiceFS 文件系统。因此它需要你有一个 Google Cloud 平台的账户才能按照文档操作下去。\n或者如果你有其他云平台的资源（比如 AWS 的 RDBS 和 S3），您也可以根据本指南和其他参考文档，以实现类似的解决方案。\n\n您可能还希望 Colab 实例位于同一区域或靠近部署 CloudSQL 和 GCS 的区域使 JuiceFS 达到最佳性能。该教程适用于随机托管的 Colab 实例，所以您或许注意到了由于 Colab 实例和 CloudSQL/GCS 区域之间的延迟而导致 JuiceFS 性能缓慢。如果想要实例在特定地区去启动 Colab，可以参考[通过 GCP Marketplace 在 Colab 上启动 GCE 虚拟机](https://research.google.com/colaboratory/marketplace.html)\n\n按照本指南操作前，您需要准备好以下资源：\n\n* 谷歌云平台账户需要准备就绪，还要创建了一个 *project* 。就这个示例而言，我们将创建 `juicefs-learning` GCP 项目作为演示项目\n* 准备使用的 CloudSQL（Postgres）。在本演示中使用实例 `juicefs-learning:europe-west1:juicefs-sql-example-1` 作为元数据服务\n* 创建的 GCS 桶作为对象存储服务。在这个演示中，我们将使用`gs://juicefs-bucket-example-1`作为存储文件的桶。\n* 对 Postgres 服务器和 GCS 存储桶具有写入访问权限的服务账户或授权用户帐户\n\n## 详细步骤\n\n### 步骤 1 - 创建并挂载一个 JuiceFS 文件系统\n\n这个步骤只需要操作一次，你可以在任何可以访问你的 Google Cloud 资源的机器或者实例上执行。\n在这里例子中，我将在我的本地机器上操作，首先你可以使用 `gcloud auth application-default login` 获取本地的凭证，或者使用 `GOOGLE_APPLICATION_CREDENTIALS` 设置 JSON 凭证文件。\n然后你可以使用 [Cloud SQL 代理功能](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy) 将你的 Postgres 云服务暴露在你本地机器上的一个端口上（这里是 5432）。\n\n```shell\ngcloud auth application-default login\n\n# 或者设置 JSON 凭证文件 GOOGLE_APPLICATION_CREDENTIALS=/path/to/key\n\ncloud_sql_proxy -instances=juicefs-learning:europe-west1:juicefs-sql-example-1=tcp:0.0.0.0:5432\n```\n\n然后使用 `juicefs format` 命令创建一个名为“myvolume”的新文件系统。之后将此文件系统挂载到您可以访问云资源的任何其他机器/实例中。\n你可以在[这里](https://github.com/juicedata/juicefs/releases)下载 JuiceFS。\n\n```shell\njuicefs format \\\n --storage gs \\\n --bucket gs://juicefs-bucket-example-1 \\\n \"postgres://postgres:mushroom1@localhost:5432/juicefs?sslmode=disable\" \\\n myvolume\n```\n\n再次提醒：这个步骤只需要被执行一次。\n\n### 步骤 2 - 挂载 JuiceFS 到 Colab\n\n完成上述步骤 1 后，这意味着您已经有一个 JuiceFS 文件系统（此案例中为“myvolume”）并准备就绪可以使用了。\n因此，在这里，我们打开一个 Colab 页面并运行这些命令，将我们的文件系统挂载到一个名为“mnt”的文件夹中。\n首先我们下载 JuiceFS 二进制然后按照步骤一操作获取 GCP 的凭证和打开 Cloud SQL 代理。\n请注意，以下命令在 Colab 环境中运行，一个 `!` 在开头意味着开始运行 shell 命令。\n\n1. 下载 `JuiceFS`到 Colab 实例上\n\n ```shell\n ! curl -sSL https://d.juicefs.com/install | sh -\n ```\n\n2. 设置 Google Cloud 凭证\n\n ```shell\n ! gcloud auth application-default login\n ```\n\n3. 打开 cloud_sql 代理\n\n ```shell\n ! wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O cloud_sql_proxy\n ! chmod +x cloud_sql_proxy\n ! GOOGLE_APPLICATION_CREDENTIALS=/content/.config/application_default_credentials.json nohup ./cloud_sql_proxy -instances=juicefs-learning:europe-west1:juicefs-sql-example-1=tcp:0.0.0.0:5432 >> cloud_sql_proxy.log &\n ```\n\n4. 挂载 JuiceFS file system `myvolumn` 到 `mnt` 目录上。\n\n ```shell\n ! GOOGLE_APPLICATION_CREDENTIALS=/content/.config/application_default_credentials.json nohup juicefs mount \"postgres://postgres:mushroom1@localhost:5432/juicefs?sslmode=disable\" mnt > juicefs.log &\n ```\n\n现在你应该可以像使用本地文件系统一样使用 `mnt` 目录了。\n\n### 步骤 3 - 在任意时间从其他实例加载数据\n\n现在，由于您在 JuiceFS 文件系统中的第 2 步中存储了数据，因此您可以随时在任何其他机器中重复第 2 步中提到的所有操作，以便再次访问之前存储的数据或存储更多数据。\n\n恭喜！现在您已经学会了如何使用 JuiceFS，特别是如何将其与 Google Colab 一起以分布式的方式共享和存储数据文件。\n[一个使用了 JuiceFS 的 Colab 笔记本示例](https://colab.research.google.com/drive/1wA8vRwqiihXkI6ViDU8Ud868UeYtmCo5)\n\n愉快的编码吧 :）\n" }, { "path": "docs/zh_cn/tutorials/juicefs_on_k3s.md", "content": "---\ntitle: 在 K3s 上使用 JuiceFS\nsidebar_position: 2\nslug: /juicefs_on_k3s\n---\n\n[K3s](https://k3s.io) 是一个经过功能优化的 Kubernetes 发行版，它与 Kubernetes 完全兼容，即几乎所有在 Kubernetes 的操作都可以在 K3s 上执行。K3s 将整个容器编排系统打包进了一个容量不足 100MB 的二进制程序，减少了部署 Kubernetes 生产集群的环境依赖，大大降低了安装难度，对系统硬件的性能要求也更低。\n\n在本文中，我们会建立一个包含两个节点的 K3s 集群，为集群安装并配置使用 [JuiceFS CSI Driver](https://github.com/juicedata/juicefs-csi-driver)，最后会创建一个 NGINX 容器进行验证。\n\n## 部署 K3s 集群\n\nK3s 对硬件的**最低要求**很低：\n\n- **内存**：512MB+（建议 1GB+）\n- **CPU**：1 核\n\n在部署生产集群时，通常可以将 4 核 CPU 和 8G 内存作为一个节点的硬件配置起点，详情查看[硬件需求](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#hardware)。\n\n### K3s server 节点\n\n运行 server 节点的服务器 IP 地址为：`192.168.1.35`\n\n使用 K3s 官方提供的脚本，即可将常规的 Linux 发行版自动部署成为 server 节点。\n\n```shell\ncurl -sfL https://get.k3s.io | sh -\n```\n\n部署成功后，K3s 服务会自动启动，kubectl 等工具也会一并安装。\n\n执行命令查看节点状态：\n\n```shell\nsudo kubectl get nodes\n```\n\n```output\nNAME STATUS ROLES AGE VERSION\nk3s-s1 Ready control-plane,master 28h v1.21.4+k3s1\n```\n\n获取 `node-token`：\n\n```shell\nsudo -u root cat /var/lib/rancher/k3s/server/node-token\n```\n\n### K3s worker 节点\n\n运行 worker 节点的服务器 IP 地址为：`192.168.1.36`\n\n执行以下命令，将其中 `K3S_URL` 的值改成 server 节点的 IP 或域名，默认端口 `6443`。将 `K3S_TOKEN` 的值替换成从 server 节点获取的 `node-token`。\n\n```shell\ncurl -sfL https://get.k3s.io | K3S_URL=http://192.168.1.35:6443 K3S_TOKEN=K1041f7c4fabcdefghijklmnopqrste2ec338b7300674f::server:3d0ab12800000000000000006328bbd80 sh -\n```\n\n部署成功以后，回到 server 节点查看节点状态：\n\n```shell\nsudo kubectl get nodes\n```\n\n```output\nNAME STATUS ROLES AGE VERSION\nk3s-s1 Ready control-plane,master 28h v1.21.4+k3s1\nk3s-n1 Ready 28h v1.21.4+k3s1\n```\n\n## 安装 CSI Driver\n\n与在 [Kubernetes 上安装 JuiceFS CSI Driver](../deployment/how_to_use_on_kubernetes.md) 的方法一致，你可以通过 Helm 安装，也可以通过 kubectl 安装。\n\n这里我们用 kubectl 安装，执行以下命令安装 JuiceFS CSI Driver：\n\n```shell\nkubectl apply -f https://raw.githubusercontent.com/juicedata/juicefs-csi-driver/master/deploy/k8s.yaml\n```\n\n### 创建存储类\n\n复制并修改以下代码创建一个配置文件，例如：`juicefs-sc.yaml`\n\n```yaml\napiVersion: v1\nkind: Secret\nmetadata:\n name: juicefs-sc-secret\n namespace: kube-system\ntype: Opaque\nstringData:\n name: \"test\"\n metaurl: \"redis://juicefs.afyq4z.0001.use1.cache.amazonaws.com/3\"\n storage: \"s3\"\n bucket: \"https://juicefs-test.s3.us-east-1.amazonaws.com\"\n access-key: \"\"\n secret-key: \"\"\n---\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: juicefs-sc\nprovisioner: csi.juicefs.com\nreclaimPolicy: Retain\nvolumeBindingMode: Immediate\nparameters:\n csi.storage.k8s.io/node-publish-secret-name: juicefs-sc-secret\n csi.storage.k8s.io/node-publish-secret-namespace: kube-system\n csi.storage.k8s.io/provisioner-secret-name: juicefs-sc-secret\n csi.storage.k8s.io/provisioner-secret-namespace: kube-system\n```\n\n配置文件中 `stringData` 部分用来设置 JuiceFS 文件系统相关的信息，系统会根据你指定的信息创建文件系统。当需要在存储类中使用已经预先创建好的文件系统时，则只需要填写 `name` 和 `metaurl` 两项即可，其他项可以删除或将值留空。\n\n执行命令，部署存储类：\n\n```shell\nkubectl apply -f juicefs-sc.yaml\n```\n\n查看存储类状态：\n\n```shell\nsudo kubectl get sc\n```\n\n```output\nNAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE\nlocal-path (default) rancher.io/local-path Delete WaitForFirstConsumer false 28h\njuicefs-sc csi.juicefs.com Retain Immediate false 28h\n```\n\n> **注意**：一个存储类与一个 JuiceFS 文件系统相关联，你可以根据需要创建任意数量的存储类。但需要注意修改配置文件中的存储类名称，避免同名冲突。\n\n## 使用 JuiceFS 持久化 NGINX 数据\n\n接下来部署一个 NGINX Pod，使用 JuiceFS 存储类声明的持久化存储。\n\n### Deployment\n\n创建一个配置文件，例如：`depolyment.yaml`\n\n```yaml\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: web-pvc\nspec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n storage: 10Pi\n storageClassName: juicefs-sc\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-run\n labels:\n app: nginx\nspec:\n replicas: 2\n selector:\n matchLabels:\n app: nginx\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: linuxserver/nginx\n ports:\n - containerPort: 80\n volumeMounts:\n - mountPath: /config\n name: web-data\n volumes:\n - name: web-data\n persistentVolumeClaim:\n claimName: web-pvc\n```\n\n执行部署：\n\n```\nsudo kubectl apply -f depolyment.yaml\n```\n\n### Service\n\n创建一个配置文件，例如：`service.yaml`\n\n```yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: nginx-run-service\nspec:\n selector:\n app: nginx\n ports:\n - name: http\n port: 80\n```\n\n执行部署：\n\n```shell\nsudo kubectl apply -f service.yaml\n```\n\n### Ingress\n\nK3s 默认预置了 traefik-ingress，通过以下配置为 NGINX 创建一个 ingress。例如：`ingress.yaml`\n\n```yaml\napiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: nginx-run-ingress\n annotations:\n traefik.ingress.kubernetes.io/router.entrypoints: web\nspec:\n rules:\n - http:\n paths:\n - pathType: Prefix\n path: \"/web\"\n backend:\n service:\n name: nginx-run-service\n port:\n number: 80\n```\n\n执行部署：\n\n```shell\nsudo kubectl apply -f ingress.yaml\n```\n\n### 访问\n\n部署完成以后，使用相同局域网的主机访问任何一个集群节点，即可看到 NGINX 的欢迎页面。\n\n![K3s-NGINX-welcome](../images/k3s-nginx-welcome.png)\n\n接下来查看一下容器是否成功挂载了 JuiceFS，执行命令查看 Pod 状态：\n\n```shell\nsudo kubectl get pods\n```\n\n```output\nNAME READY STATUS RESTARTS AGE\nnginx-run-7d6fb7d6df-qhr2m 1/1 Running 0 28h\nnginx-run-7d6fb7d6df-5hpv7 1/1 Running 0 24h\n```\n\n执行命令，查看任何一个 Pod 的文件系统挂载情况：\n\n```shell\n$ sudo kubectl exec nginx-run-7d6fb7d6df-qhr2m -- df -Th\nFilesystem Type Size Used Avail Use% Mounted on\noverlay overlay 20G 3.2G 17G 17% /\ntmpfs tmpfs 64M 0 64M 0% /dev\ntmpfs tmpfs 2.0G 0 2.0G 0% /sys/fs/cgroup\nJuiceFS:jfs fuse.juicefs 1.0P 174M 1.0P 1% /config\n/dev/sda1 ext4 20G 3.2G 17G 17% /etc/hosts\nshm tmpfs 64M 0 64M 0% /dev/shm\ntmpfs tmpfs 2.0G 12K 2.0G 1% /run/secrets/kubernetes.io/serviceaccount\ntmpfs tmpfs 2.0G 0 2.0G 0% /proc/acpi\ntmpfs tmpfs 2.0G 0 2.0G 0% /proc/scsi\ntmpfs tmpfs 2.0G 0 2.0G 0% /sys/firmware\n```\n\n可以看到，名为 `jfs` 的文件系统已经挂载到了容器的 `/config` 目录，已使用空间为 174M。\n\n这就表明集群中的 Pod 已经成功配置并使用 JuiceFS 持久化数据了。\n" }, { "path": "docs/zh_cn/tutorials/juicefs_on_kubesphere.md", "content": "---\ntitle: 在 KubeSphere 上使用 JuiceFS\nsidebar_position: 3\nslug: /juicefs_on_kubesphere\n---\n\n[KubeSphere](https://kubesphere.com.cn) 是在 Kubernetes 之上构建的以应用为中心的多租户容器平台，提供全栈的 IT 自动化运维的能力，简化企业的 DevOps 工作流。\n\nKubeSphere 提供了运维友好的向导式操作界面，即便是 Kubernetes 经验并不丰富的用户，也能相对轻松的上手开始管理和使用。它提供了基于 Helm 的应用市场，可以在图形化界面下非常轻松地安装各种 Kubernetes 应用。\n\n本文将介绍如何在 KubeSphere 中一键部署 JuiceFS CSI Driver，为集群上的各种应用提供数据持久化。\n\n## 前提条件\n\n1. 安装 KubeSphere\n\n 安装 KubeSphere 有两种方法。一是在 Linux 上直接安装，可以参考文档：[在 Linux 安装 KubeSphere](https://kubesphere.com.cn/docs/quick-start/all-in-one-on-linux) ；\n二是在已有 Kubernetes 中安装，可以参考文档：[在 Kubernetes 安装 KubeSphere](https://kubesphere.com.cn/docs/quick-start/minimal-kubesphere-on-k8s) 。\n\n2. 在 KubeSphere 中启用应用商店\n\n 在 KubeSphere 中启用应用商店可以参考文档：[KubeSphere 应用商店](https://kubesphere.com.cn/docs/pluggable-components/app-store) 。\n\n## 安装 JuiceFS CSI Driver\n\n如果 KubeSphere 的版本为 v3.2.0 及以上，可以直接在应用商店中安装 CSI Driver，跳过「配置应用模板/应用仓库」步骤，直接进入「安装」步骤；如果 KubeSphere 版本低于 v3.2.0，按照以下步骤配置应用模板/应用仓库。\n\n### 配置应用模板/应用仓库\n\n安装 JuiceFS CSI Driver 首先需要创建应用模板，这里有两种方法。\n\n#### 方法一：应用仓库\n\n在企业空间中点击进去应用管理，选择「应用仓库」，点击创建按钮添加 JuiceFS CSI 仓库，填写：\n\n- 仓库名称：`juicefs-csi-driver`\n- Index URL：`https://juicedata.github.io/charts/`\n\n![kubesphere_app_shop](../images/kubesphere_app_shop.png)\n\n#### 方法二：应用模板\n\n先在 JuiceFS CSI Driver 仓库下载 chart 压缩包：[https://github.com/juicedata/juicefs-csi-driver/releases](https://github.com/juicedata/juicefs-csi-driver/releases)。\n\n在「企业空间」中点击进入「应用管理」，选择「应用模板」，点击「创建」，上传 chart 压缩包：\n\n![kubesphere_app_template](../images/kubesphere_app_template.png)\n\n### 安装\n\n在「企业空间」中选择您所需部署的「项目」（KubeSphere 中的项目即为 K8s 中的 namespace），选择「应用负载」，点击「部署新应用」按钮，选择「来自应用商店」，然后选择 `juicefs`：\n\n![kubesphere_shop_juicefs](../images/kubesphere_shop_juicefs.jpg)\n\n若 KubeSphere 版本低于 v3.2.0，根据上一步配置好的应用模板，选择部署应用「来自应用模板」：\n\n![kubesphere_install_csi](../images/kubesphere_install_csi.png)\n\n进入配置修改页面后一致，修改以下两个地方：\n\n- namespace：改成对应的项目名\n- storageClass.backend：\n `backend` 部分用来定义文件系统后端的数据库和对象存储，可以查阅[创建文件系统](../getting-started/standalone.md#juicefs-format)了解相关内容。\n\n您也可以通过 KubeSphere 的应用商店快速创建数据库（如 Redis）和对象存储（如 MinIO）。\n比如在 KubeSphere 平台搭建 Redis：在当前所在项目中选择「应用负载」，点击「部署新应用」按钮，选择「来自应用商店」，选择「Redis」，然后快速部署即可。Redis 的访问 URL 可以通过部署好的应用的服务名，如下：\n\n![kubesphere_redis](../images/kubesphere_redis.png)\n\n在 KubeSphere 平台搭建 MinIO 也是类似的流程，不过在部署 MinIO 之前可以修改 MinIO 的 accessKey 和 secretKey，并且需要记住配置的值。如下图：\n\n![kubesphere_create_minio](../images/kubesphere_create_minio.png)\n\n> 注：如果部署 MinIO 出现权限问题，可以将配置中的 `securityContext.enables` 设置为 false。\n\nMinIO 的访问 URL 可以通过部署好的应用的服务名，如下：\n\n![kubesphere_minio](../images/kubesphere_minio.png)\n\nRedis 和 MinIO 都搭建好之后，就可以填写 JuiceFS CSI Driver 的 `backend` 值了。其中：\n\n1. `metaurl` 为刚才创建的 Redis 的数据库地址，Redis 的访问地址可用 Redis 应用对应的服务名，如 `redis://redis-rzxoz6:6379/1`\n2. `storage` 为对象存储的类型，如 `minio`\n3. `bucket` 为刚才创建的 MinIO 的可用 bucket（JuiceFS 会自动创建，不需要手动创建），MinIO 的访问地址可用 MinIO 应用对应的服务名，如 `http://minio-qkp9my:9000/minio/test`\n4. `accessKey` 和 `secretKey` 用刚才创建的 MinIO 的 accessKey 和 secretKey\n\n![kubesphere_update_csi](../images/kubesphere_update_csi.png)\n\n配置修改完毕后，点击安装即可。\n\n## 使用\n\n### 部署应用\n\n按照上述方法安装好的 JuiceFS CSI Driver 已经创建好一个 `StorageClass`，名为上述 `storageClass` 的 `name`，比如上述创建的 `StorageClass` 为 `juicefs-sc`，可以直接使用。\n\n然后需要创建一个 PVC，指定使用 `juicefs-sc` 这个 `StorageClass`。在「项目」中，选择「存储管理」，再选择「存储卷」，点击「创建」按钮创建 PVC，其中「存储类型」选择 `juicefs-sc`，如下：\n\n![kubesphere_pvc](../images/kubesphere_pvc.png)\n\nPVC 创建好之后，再在「项目」的「应用负载」中，选择「工作负载」，点击「创建」按钮部署工作负载，其中「基本信息」页填写自己喜欢的名字；「容器镜像」页可以填写镜像 `centos` ；\n启动命令 `sh,-c,while true; do echo $(date -u) >> /data/out.txt; sleep 5; done` ；「存储卷来源」选择「已有存储卷」，再选择上一步创建的 PVC，容器内路径填写 `/data` 如下：\n\n![kubesphere_deployment](../images/kubesphere_deployment.png)\n\n![kubesphere_workload](../images/kubesphere_workload.png)\n\n部署完成后可以看到运行中的容器组：\n\n![kubesphere_pod](../images/kubesphere_pod.png)\n\n### 新建 StorageClass\n\n若安装 JuiceFS CSI Driver 的时候没有创建 `StorageClass`，或者需要另外新建，可以遵循以下步骤：\n\n准备好元数据服务和对象存储服务后，新建一个 `Secret`。在「平台管理」页面选择「配置中心」，选择「密钥」，点击「创建」按钮新建：\n\n![kubesphere_create_secret](../images/kubesphere_create_secret.png)\n\n「密钥设置」中填入准备好的元数据服务和对象存储信息，如下：\n\n![kubesphere_update_secret](../images/kubesphere_update_secret.png)\n\n`Secret` 新建好之后，创建 `StorageClass`，在「平台管理」页面选择「存储管理」，选择「存储类型」，点击「创建」按钮新建，其中「存储系统」选择「自定义」：\n\n![kubesphere_sc_create](../images/kubesphere_sc_create.png)\n\n设置页面信息如下，其中「存储系统」填写 `csi.juicefs.com`，另外再设置 4 个参数：\n\n- `csi.storage.k8s.io/provisioner-secret-name`: 刚刚创建好的 secret name\n- `csi.storage.k8s.io/provisioner-secret-namespace`: secret 对应的项目名\n- `csi.storage.k8s.io/node-publish-secret-name`: 刚刚创建好的 secret name\n- `csi.storage.k8s.io/node-publish-secret-namespace`: secret 对应的项目名\n\n![kubesphere_sc_update](../images/kubesphere_sc_update.png)\n\n点击「创建」按钮之后，`StorageClass` 就创建好了。\n" }, { "path": "docs/zh_cn/tutorials/juicefs_on_rancher.md", "content": "---\ntitle: 在 Rancher 上使用 JuiceFS\nsidebar_position: 2\nslug: /juicefs_on_rancher\n---\n\n简单来说，[Rancher](https://rancher.com) 是一个企业级的 Kubernetes 集群管理工具，使用它可以非常轻松的在各种云计算平台上快速的完成 Kubernetes 集群的部署。\n\nRancher 提供了基于浏览器的管理界面，即便是 Kubernetes 经验并不丰富的用户，也能相对轻松的上手开始管理和使用。它默认预置了基于 Helm 的应用市场，可以在图形化界面下非常轻松的安装各种 Kubernetes 应用。\n\n本文将介绍如何在 Linux 系统上部署 Rancher，并在上面创建 Kubernetes 集群，然后通过其内置的应用市场，一键部署 JuiceFS CSI Driver，为集群上的各种应用提供数据持久化。\n\n## 安装 Rancher\n\n几乎所有主流的现代 Linux 发行版都可以安装 Rancher，它既可以直接安装在操作系统上，也可以安装在 Docker、Kubernetes、K3s 或 RKE 上，不论在哪种环境上安装都是“Product-Ready”的。\n\n这里我们选择将 Rancher 安装在 Docker 上，配置上需要满足以下要求：\n\n- **操作系统**：x86-64 架构的 Linux 系统\n- **内存**：4GB 以上\n- **Docker**：19.03+\n\n执行以下命令安装 Rancher：\n\n```shell\nsudo docker run --privileged -d --restart=unless-stopped -p 80:80 -p 443:443 rancher/rancher\n```\n\n容器创建完成以后，通过浏览器访问主机的 IP 地址就能打开 Rancher 的管理界面。\n\n![Rancher-welcome](../images/rancher-welcome.jpeg)\n\n## 创建 Kubernetes 集群\n\nRancher 安装成功以后，可以看到它已经在当前容器中部署了一个 K3s 集群，Rancher 相关资源都运行在这个内部的 K3s 集群中，无需理会这个集群。\n\n接下来开始创建 Kubernetes 集群，在欢迎页面的 Cluster 部分点击 `Create` 创建集群。Rancher 支持在各大主流云计算平台创建 Kubernetes 集群，这里我们要在 Rancher 的宿主机上直接选择集群，因此选择 `Custom`。然后根据向导填写集群名称，选择 Kubernetes 版本即可。\n\n![Rancher-cluster-create](../images/rancher-cluster-create.jpg)\n\n在 `Cluster Options` 页面中，选择要创建的节点角色，然后复制生成命令，在目标主机上执行即可。\n\n![Rancher-cluster-options](../images/rancher-cluster-options.jpg)\n\n集群创建完成后，Rancher 的集群列表中会有状态显示。\n\n![Rancher-clusters](../images/rancher-clusters.jpg)\n\n## 一键安装 JuiceFS CSI Driver\n\n在集群列表中点击进入创建的 Kubernetes 集群，左侧导航菜单点击展开 `应用市场` → `Chart 仓库`，点击 `创建` 按钮添加 JuiceFS CSI 仓库，填写：\n\n- **仓库名称**：`juicefs`\n- **Index URL**：`https://juicedata.github.io/charts/`\n\n![Rancher-new-repo](../images/rancher-new-repo.jpg)\n\n创建以后，在仓库列表中可以看到刚刚添加的 JuiceFS CSI 仓库。\n\n![Rancher-repos](../images/rancher-repos.jpg)\n\n紧接着通过左侧菜单点击打开 `应用市场` → `Charts`，搜索栏中输入 `juicefs`，然后点击打开检索出的 `juicefs-csi-driver`。\n\n![Rancher-chart-search](../images/rancher-chart-search.jpg)\n\n在应用详情页面点击“安装”按钮，默认会安装最新版本，也可以点选切换到历史版本进行安装。\n\n![Rancher-chart-info](../images/rancher-chart-info.jpg)\n\n安装向导共有两步：\n\n### 第一步：设置应用的 `Namespace`\n\nJuiceFS CSI Driver 默认为 `kube-system`，这一步无需设置。\n\n### 第二步：调整配置参数\n\n这个页面提供了 YAML 编辑器，你可以根据需要调整 JuiceFS 相关的信息，通常只需要修改 `storageClasses` 部分，其中 `backend` 部分用来定义文件系统后端的数据库和对象存储。如果你使用的是已经预先创建的文件系统，那么只需填写 `metaurl` 和 `name` 两项即可，例如：\n\n```yaml\n...\nstorageClasses:\n - backend:\n accessKey: ''\n bucket: ''\n metaurl: 'redis://:mypasswd@efgh123.redis.rds.aliyuncs.com/1'\n name: myjfs\n secretKey: ''\n storage: ''\n enabled: true\n name: juicefs-sc\n reclaimPolicy: Retain\n...\n```\n\n> **提示**：如果你有多个 JuiceFS 文件系统，分别需要关联到 Kubernetes 集群不同的 storageClass，可以在 `storageClasses` 数组后面再加 storageClass 配置项，注意修改存储类的名称，避免冲突。\n\n点击「安装」，等待应用安装完成。\n\n![Rancher-chart-installed](../images/rancher-chart-installed.jpg)\n\n## 使用 JuiceFS 持久化数据\n\n部署应用时，在存储配置中指定 `juicefs-sc` 即可。\n\n![Rancher-PVC](../images/rancher-pvc.jpg)\n" }, { "path": "docs/zh_cn/tutorials/juicefs_on_wsl.md", "content": "---\ntitle: 在 WSL 中使用 JuiceFS\nsidebar_position: 9\n---\n\nWSL 全称 Windows Subsystem for Linux，即适用于 Linux 的 Windows 子系统。它可以让你在 Windows 系统环境下运行大多数 GNU/Linux 原生命令、工具和程序，且不必像用虚拟机或双系统那样产生额外的硬件开销。\n\n## 安装 WSL\n\n使用 WSL 要求必须是 Windows 10 2004 以上或 Windows 11。\n\n查看当前系统的版本，可以通过组合键 Win + R 唤出运行程序，输入并运行 `winver`。\n\n![WSL/winver](../images/wsl/winver.png)\n\n确认 Windows 版本以后，以管理员身份打开 PowerShell 或 Windows 命令提示符，运行安装命令：\n\n```powershell\nwsl --install\n```\n\n该命令会下载最新的 Linux 内核，安装并将 WSL 2 作为默认版本，并安装 Linux 发行版（默认为 Ubuntu）。\n\n也可以直接指定要安装的发行版：\n\n```powershell\nwsl --install -d ubuntu\n```\n\n:::tip 提示\n`wsl --list --online` 命令可以查看所有可选的发行版。\n:::\n\n## 设置 Linux 用户和密码\n\nWSL 安装完成以后，即可在开始菜单找到新安装的 Linux 发行版。\n\n![WSL/startmenu](../images/wsl/startmenu.png)\n\n点击 Ubuntu 子系统的快捷方式，WSL 会打开 Linux 子系统的终端。初次运行会要求设置管理 Linux 子系统的用户和密码，根据提示设置即可。\n\n![WSL/init](../images/wsl/init.png)\n\n这里设置的用户名和密码有以下几点需要注意：\n\n- 此用户专用于该 Linux 子系统的管理，与 Windows 系统中的用户无关；\n- 此用户将作为 Linux 子系统的默认用户，并在启动时自动登录；\n- 此用户将被视为 Linux 子系统的管理员，允许执行 `sudo` 命令；\n- WSL 中允许同时运行多个 Linux 子系统，且每个子系统都需要设置一个管理用户。\n\n## 在 WSL 中使用 JuiceFS\n\n在 WSL 中使用 JuiceFS，即是在 Linux 系统中使用 JuiceFS，这里以社区版为例进行介绍。\n\n### 安装客户端\n\n执行命令，在 Linux 子系统中安装 JuiceFS 客户端：\n\n ```shell\n curl -sSL https://d.juicefs.com/install | sh -\n ```\n\n### 创建文件系统\n\nJuiceFS 是数据与元数据分离的分布式文件系统，通常用对象存储作为数据存储，用 Redis、PostgreSQL 或 MySQL 作为元数据存储。这里假设已经准备了如下材料：\n\n#### 对象存储\n\n查看「[JuiceFS 支持的数据存储](../reference/how_to_set_up_object_storage.md)」\n\n- **Bucket Endpoint**：`https://myjfs.oss-cn-shanghai.aliyuncs.com`\n- **Access Key ID**：`ABCDEFGHIJKLMNopqXYZ`\n- **Access Key Secret**：`ZYXwvutsrqpoNMLkJiHgfeDCBA`\n\n#### 数据库\n\n查看「[JuiceFS 支持的元数据引擎](../reference/how_to_set_up_metadata_engine.md)」\n\n- **数据库地址**：`myjfs-sh-abc.redis.rds.aliyuncs.com:6379`\n- **数据库密码**：`mypassword`\n\n将私密信息写入环境变量：\n\n```shell\nexport ACCESS_KEY=ABCDEFGHIJKLMNopqXYZ\nexport SECRET_KEY=ZYXwvutsrqpoNMLkJiHgfeDCBA\nexport REDIS_PASSWORD=mypassword\n```\n\n创建名为 `myjfs` 的文件系统：\n\n```shell\njuicefs format \\\n --storage oss \\\n --bucket https://myjfs.oss-cn-shanghai.aliyuncs.com \\\n redis://myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1 \\\n myjfs\n```\n\n### 挂载和使用\n\n把数据库密码写入环境变量：\n\n```shell\nexport REDIS_PASSWORD=mypassword\n```\n\n:::note 注意\n对象存储的 API 密钥信息仅在创建文件系统时需要设置，一旦文件系统创建成功，相应的密钥信息会被写入数据库，JuiceFS 客户端会在挂载文件系统时自动从数据库中读取，无需重复设置。\n:::\n\n挂载文件系统到用户家目录下的 `mnt`：\n\n```shell\nsudo juicefs mount -d redis://myjfs-sh-abc.redis.rds.aliyuncs.com:6379/1 $HOME/mnt\n```\n\n如果需要从 Windows 系统访问 Linux 子系统中挂载的 JuiceFS 文件系统，在资源管理器左侧列表中找到 Linux 子系统，然后找到并打开挂载点路径即可。\n\n![WSL/access-jfs-from-win](../images/wsl/access-jfs-from-win.png)\n\n有关 JuiceFS 使用方面的更多内容请查阅官方文档。\n\n## WSL 文件存储性能问题\n\nWSL 打通了 Windows 与 Linux 子系统，允许二者相互访问彼此系统中存储的文件。\n\n![WSL/Windows-to-Linux](../images/wsl/windows-to-linux.png)\n\n但需要注意，从 Windows 访问 Linux 子系统或从 Linux 子系统访问 Windows 势必会因系统之间的转换而产生一定的性能开销。因此，推荐的做法是根据程序所在的系统来决定文件存储的位置，对于 Linux 子系统中的程序，它要处理的文件也应该存储在 Linux 子系统中性能才更理想。\n\n在 Linux 子系统中，WSL 将 Windows 的各个盘符挂载到了 `/mnt`，比如 C: 盘在 Linux 子系统中的挂载点是 `/mnt/c`。\n\n![WSL/mount-point](../images/wsl/mount-point.png)\n\n为了保证性能最优，在 WSL 中使用 JuiceFS 时，不论存储还是缓存路径都应设置在 Linux 子系统中。换言之，应该避免把存储或缓存设置在 `/mnt/c` 类似的 Windows 分区挂载点上。\n\n通过使用 JuiceFS 自带的 `bench` 基准测试工具，结果显示，将文件系统挂载到 Windows（如 `/mnt/c`）的性能要比挂载到 Linux 子系统内部（如 `$HOME/mnt`）低 30% 左右。\n\n## 已知问题\n\n当通过 Windows 资源管理器拷贝文件到 Linux 子系统时，WSL 会自动为每个文件附加一个带有 `Zone.Identifier` 标识的同名文件。这是 NTFS 文件系统的一种安全防护机制，意在对外部文件的来源进行跟踪，但对于 WSL 来说，这个功能应该属于 bug 且已经有人在 GitHub 上向微软开发团队反馈 [#7456](https://github.com/microsoft/WSL/issues/7456)。\n\n受此问题影响，通过 Windows 资源管理器向 Linux 子系统中挂载的 JuiceFS 文件系统存入文件时也会出现同样的问题。但在 Linux 子系统内部读写 JuiceFS 文件系统不受该 bug 的干扰。\n\n![WSL/zone-identifier](../images/wsl/zone-identifier.png)\n" }, { "path": "docs/zh_cn/tutorials/qcloud.md", "content": "---\ntitle: 在腾讯云使用 JuiceFS\nsidebar_position: 8\nslug: /clouds/qcloud\n---\n\n如下图所示，JuiceFS 存储由数据库和对象存储共同驱动。存入 JuiceFS 的文件会按照一定的规则被拆分成固定大小的数据块存储在对象存储中，数据对应的元数据则会存储在数据库中。\n\n元数据完全独立存储，对文件的检索和处理并不会直接操作对象存储中的数据，而是先在数据库中操作元数据，只有当数据发生变化的时候，才会与对象存储交互。\n\n这样的设计可以有效缩减对象存储在请求数量上的费用，同时也能让我们显著感受到 JuiceFS 带来的性能提升。\n\n![JuiceFS-qcloud](../images/juicefs-qcloud.png)\n\n## 准备\n\n通过前面的架构描述，可以知道 JuiceFS 需要搭配数据库和对象存储一起使用。这里我们直接使用腾讯云的 CVM 云服务器，结合云数据库和 COS 对象存储。\n\n在创建云计算资源时，尽量选择在相同的区域，这样可以让资源之间通过内网线路相互访问，避免使用公网线路产生额外的流量费用。\n\n### 一、云服务器 CVM\n\nJuiceFS 对服务器硬件没有特殊要求，一般来说，云平台上最低配的云服务器也能稳定使用 JuiceFS，通常你只需要选择能够满足自身业务的配置即可。\n\n需要特别说明的是，你不需要为使用 JuiceFS 重新购买服务器或是重装系统，JuiceFS 没有业务入侵性，不会对你现有的系统和程序造成任何的干扰，你完全可以在正在运行的服务器上安装和使用 JuiceFS。\n\nJuiceFS 默认会占用不超过 1GB 的硬盘空间作为缓存，可以根据需要调整缓存空间的大小。该缓存是客户端与对象存储之间的一个数据缓冲层，选择性能更好的云盘，可以获得更好的性能表现。\n\n在操作系统方面，腾讯云 CVM 提供的所有操作系统都可以安装 JuiceFS。\n\n**本文使用的 CVM 配置如下：**\n\n| 服务器配置 | |\n| ------------ | ------------------------ |\n| **CPU** | 1 核 |\n| **内存** | 2 GB |\n| **存储** | 50 GB |\n| **操作系统** | Ubuntu Server 20.04 64 位 |\n| **地域** | 上海五区 |\n\n### 二、云数据库\n\nJuiceFS 会将数据对应的元数据全部存储在独立的数据库中，目前已开放支持的数据库有 Redis、MySQL、PostgreSQL、TiKV 和 SQLite。\n\n根据数据库类型的不同，带来的元数据性能和可靠性表现也各不相同。比如 Redis 是完全运行在内存上的，它能提供极致的性能，但运维难度较高，可靠性相对低。而 MySQL、PostgreSQL 是关系型数据库，性能不如 Redis，但运维难度不高，可靠性也有一定的保障。SQLite 是单机单文件关系型数据库，性能较低，也不适合用于大规模数据存储，但它免配置，适合单机少量数据存储的场景。\n\n如果只是为了评估 JuiceFS 的功能，你可以在 CVM 云服务器手动搭建数据库使用。当你要在生产环境使用 JuiceFS 时，如果没有专业的数据库运维团队，腾讯云的云数据库服务通常是更好的选择。\n\n当然，如果你愿意，也可以使用其他云平台上提供的云数据库服务。但在这种情况下，你只能通过公网访问云数据库，也就是说，你必须向公网暴露数据库的端口，这存在极大的安全风险，最好不要这样使用。\n\n如果必须通过公网访问数据库，可以通过云数据库控台提供的白名单功能，严格限制允许访问数据库的 IP 地址，从而提升数据的安全性。从另一个角度说，如果你通过公网无法成功连接云数据库，那么可以检查数据库的白名单，检查是不是该设置限制了你的访问。\n\n| 数据库 | Redis | MySQL、PostgreSQL | SQLite |\n| :----------: | :----------------------: | :------------------------: | :--------------------: |\n| **性能** | 强 | 适中 | 弱 |\n| **运维门槛** | 高 | 适中 | 低 |\n| **可靠性** | 低 | 适中 | 低 |\n| **应用场景** | 海量数据、分布式高频读写 | 海量数据、分布式中低频读写 | 少量数据单机中低频读写 |\n\n**本文使用了云数据库 TencentDB Redis，通过 VPC 私有网络与 CVM 云服务器交互访问：**\n\n| Redis 版本 | 5.0 社区版 |\n| ------------ | ---------------- |\n| **实例规格** | 1GB 内存版（标准架构） |\n| **连接地址** | 192.168.5.5:6379 |\n| **可用区** | 上海五区 |\n\n注意，数据库的连接地址取决于你创建的 VPC 网络设置，创建 Redis 实例时会自动在你定义的网段中获取地址。\n\n![qcloud-Redis-network](../images/qcloud-redis-network.png)\n\n### 三、对象存储 COS\n\nJuiceFS 会将所有的数据都存储到对象存储中，它支持几乎所有的对象存储服务。但为了获得最佳的性能，当使用腾讯云 CVM 时，搭配腾讯云 COS 对象存储通常是最优选择。不过请注意，将 CVM 和 COS Bucket 选择在相同的地区，这样才能通过腾讯云的内网线路进行访问，不但延时低，而且不需要额外的流量费用。\n\n> **提示**：腾讯云对象存储 COS 提供的唯一访问地址同时支持内网和外网访问，当通过内网访问时，COS 会自动解析到内网 IP，此时产生的流量均为内网流量，不会产生流量费用。\n\n当然，如果你愿意，也可以使用其他云平台提供的对象存储服务，但不推荐这样做。首先，通过腾讯云 CVM 访问其他云平台的对象存储要走公网线路，对象存储会产生流量费用，而且这样的访问延时相比也会更高，可能会影响 JuiceFS 的性能发挥。\n\n腾讯云 COS 有不同的存储级别，由于 JuiceFS 需要与对象存储频繁交互，建议使用标准存储。你可以搭配 COS 资源包使用，降低对象存储的使用成本。\n\n### API 访问秘钥\n\n腾讯云 COS 需要通过 API 进行访问，你需要准备访问秘钥，包括 `Access Key ID` 和 `Access Key Secret` ，[点此查看](https://cloud.tencent.com/document/product/598/37140)获取方式。\n\n> **安全建议**：显式使用 API 访问秘钥可能导致密钥泄露，推荐为云服务器分配 [CAM 服务角色](https://cloud.tencent.com/document/product/598/19420)。当一台 CVM 被授予 COS 操作权限以后，无需使用 API 访问秘钥即可访问 COS。\n\n## 安装\n\n我当前使用的是 Ubuntu Server 20.04 64 位系统，执行以下命令可以安装最新版本客户端。\n\n```shell\ncurl -sSL https://d.juicefs.com/install | sh -\n```\n\n你也可以访问 [JuiceFS GitHub Releases](https://github.com/juicedata/juicefs/releases) 页面选择其他版本。\n\n执行命令，看到返回 `juicefs` 的命令帮助信息，代表客户端安装成功。\n\n```shell\n$ juicefs\nNAME:\n juicefs - A POSIX file system built on Redis and object storage.\n\nUSAGE:\n juicefs [global options] command [command options] [arguments...]\n\nVERSION:\n 0.15.2 (2021-07-07T05:51:36Z 4c16847)\n\nCOMMANDS:\n format format a volume\n mount mount a volume\n umount unmount a volume\n gateway S3-compatible gateway\n sync sync between two storage\n rmr remove directories recursively\n info show internal information for paths or inodes\n bench run benchmark to read/write/stat big/small files\n gc collect any leaked objects\n fsck Check consistency of file system\n profile analyze access log\n status show status of JuiceFS\n warmup build cache for target directories/files\n dump dump metadata into a JSON file\n load load metadata from a previously dumped JSON file\n help, h Shows a list of commands or help for one command\n\nGLOBAL OPTIONS:\n --verbose, --debug, -v enable debug log (default: false)\n --quiet, -q only warning and errors (default: false)\n --trace enable trace log (default: false)\n --no-agent disable pprof (:6060) agent (default: false)\n --help, -h show help (default: false)\n --version, -V print only the version (default: false)\n\nCOPYRIGHT:\n Apache License 2.0\n```\n\nJuiceFS 具有良好的跨平台兼容性，同时支持在 Linux、Windows 和 macOS 上使用。本文着重介绍 JuiceFS 在 Linux 系统上的安装和使用，如果你需要了解其他系统上的安装方法，请[查阅文档](../getting-started/installation.md)。\n\n## 创建 JuiceFS 存储\n\nJuiceFS 客户端安装好以后，现在就可以使用前面准备好的 Redis 数据库和 COS 对象存储来创建 JuiceFS 存储了。\n\n严格意义上说，这一步操作应该叫做“Format a volume”，即格式化一个卷。但考虑到有很多用户可能不了解或者不关心文件系统的标准术语，所以简单起见，我们就直白的把这个过程叫做“创建 JuiceFS 存储”。\n\n以下命令使用 JuiceFS 客户端提供的 `format` 子命令创建了一个名为 `mystor` 的存储，即文件系统：\n\n```shell\n$ juicefs format \\\n --storage cos \\\n --bucket https:// \\\n --access-key \\\n --secret-key \\\n redis://:@192.168.5.5:6379/1 \\\n mystor\n```\n\n**选项说明：**\n\n- `--storage`：指定对象存储类型，[点此查看](../reference/how_to_set_up_object_storage.md#supported-object-storage) JuiceFS 支持的对象存储。\n- `--bucket`：对象存储的 Bucket 访问域名，可以在 COS 的管理控制台找到。\n ![cos-bucket-url](../images/cos-bucket-url.png)\n- `--access-key` 和 `--secret-key`：访问对象存储 API 的秘钥对，[点此查看](https://cloud.tencent.com/document/product/598/37140)获取方式。\n\n> Redis 6.0 身份认证需要用户名和密码两个参数，地址格式为 `redis://username:password@redis-server-url:6379/1`。目前腾讯云数据库 Redis 版只提供 Reids 4.0 和 5.0 两个版本，认证身份只需要密码，在设置 Redis 服务器地址时只需留空用户名即可，例如：`redis://:password@redis-server-url:6379/1`\n\n看到类似下面的输出，代表文件系统创建成功了。\n\n```shell\n2021/07/30 11:44:31.904157 juicefs[44060] : Meta address: redis://@192.168.5.5:6379/1\n2021/07/30 11:44:31.907083 juicefs[44060] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/07/30 11:44:31.907634 juicefs[44060] : Ping redis: 474.98µs\n2021/07/30 11:44:31.907850 juicefs[44060] : Data uses cos://juice-0000000000/mystor/\n2021/07/30 11:44:32.149692 juicefs[44060] : Volume is formatted as {Name:mystor UUID:dbf05314-57af-4a2c-8ac1-19329d73170c Storage:cos Bucket:https://juice-0000000000.cos.ap-shanghai.myqcloud.com AccessKey:AKIDGLxxxxxxxxxxxxxxxxxxZ8QRBdpkOkp SecretKey:removed BlockSize:4096 Compression:none Shards:0 Partitions:0 Capacity:0 Inodes:0 EncryptKey:}\n```\n\n## 挂载 JuiceFS 存储\n\n文件系统创建完成，对象存储相关的信息会被存入数据库，挂载时无需再输入对象存储的 Bucket 和秘钥等信息。\n\n使用 `mount` 子命令，将文件系统挂载到 `/mnt/jfs` 目录：\n\n```shell\nsudo juicefs mount -d redis://:@192.168.5.5:6379/1 /mnt/jfs\n```\n\n> **注意**：挂载文件系统时，只需填写 Redis 数据库地址，不需要文件系统名称。默认的缓存路径为 `/var/jfsCache`，请确保当前用户有足够的读写权限。\n\n看到类似下面的输出，代表文件系统挂载成功。\n\n```shell\n2021/07/30 11:49:56.842211 juicefs[44175] : Meta address: redis://@192.168.5.5:6379/1\n2021/07/30 11:49:56.845100 juicefs[44175] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/07/30 11:49:56.845562 juicefs[44175] : Ping redis: 383.157µs\n2021/07/30 11:49:56.846164 juicefs[44175] : Data use cos://juice-0000000000/mystor/\n2021/07/30 11:49:56.846731 juicefs[44175] : Disk cache (/var/jfsCache/dbf05314-57af-4a2c-8ac1-19329d73170c/): capacity (1024 MB), free ratio (10%), max pending pages (15)\n2021/07/30 11:49:57.354763 juicefs[44175] : OK, mystor is ready at /mnt/jfs\n```\n\n使用 `df` 命令，可以看到文件系统的挂载情况：\n\n```shell\n$ df -Th\n文件系统 类型 容量 已用 可用 已用% 挂载点\nJuiceFS:mystor fuse.juicefs 1.0P 64K 1.0P 1% /mnt/jfs\n```\n\n文件系统挂载成功以后，现在就可以像使用本地硬盘那样，在 `/mnt/jfs` 目录中存储数据了。\n\n> **多主机共享**：JuiceFS 存储支持被多台云服务器同时挂载使用，你可以在其他 CVM 上安装 JuiceFS 客户端，然后使用 `redis://:@192.168.5.5:6379/1` 数据库地址挂载文件系统到每一台主机上。\n\n## 查看文件系统状态\n\n使用 JuiceFS 客户端的 `status` 子命令可以查看一个文件系统的基本信息和连接状态。\n\n```shell\n$ juicefs status redis://:@192.168.5.5:6379/1\n\n2021/07/30 11:51:17.864767 juicefs[44196] : Meta address: redis://@192.168.5.5:6379/1\n2021/07/30 11:51:17.866619 juicefs[44196] : AOF is not enabled, you may lose data if Redis is not shutdown properly.\n2021/07/30 11:51:17.867092 juicefs[44196] : Ping redis: 379.391µs\n{\n \"Setting\": {\n \"Name\": \"mystor\",\n \"UUID\": \"dbf05314-57af-4a2c-8ac1-19329d73170c\",\n \"Storage\": \"cos\",\n \"Bucket\": \"https://juice-0000000000.cos.ap-shanghai.myqcloud.com\",\n \"AccessKey\": \"AKIDGLxxxxxxxxxxxxxxxxx8QRBdpkOkp\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"Shards\": 0,\n \"Partitions\": 0,\n \"Capacity\": 0,\n \"Inodes\": 0\n },\n \"Sessions\": [\n {\n \"Sid\": 1,\n \"Heartbeat\": \"2021-07-30T11:49:56+08:00\",\n \"Version\": \"0.15.2 (2021-07-07T05:51:36Z 4c16847)\",\n \"Hostname\": \"VM-5-6-ubuntu\",\n \"MountPoint\": \"/mnt/jfs\",\n \"ProcessID\": 44175\n },\n {\n \"Sid\": 3,\n \"Heartbeat\": \"2021-07-30T11:50:56+08:00\",\n \"Version\": \"0.15.2 (2021-07-07T05:51:36Z 4c16847)\",\n \"Hostname\": \"VM-5-6-ubuntu\",\n \"MountPoint\": \"/mnt/jfs\",\n \"ProcessID\": 44185\n }\n ]\n}\n```\n\n## 卸载 JuiceFS 存储\n\n使用 JuiceFS 客户端提供的 `umount` 命令即可卸载文件系统，比如：\n\n```shell\nsudo juicefs umount /mnt/jfs\n```\n\n> **注意**：强制卸载使用中的文件系统可能导致数据损坏或丢失，请务必谨慎操作。\n\n## 开机自动挂载\n\n请参考[「启动时自动挂载 JuiceFS」](../administration/mount_at_boot.md)\n" }, { "path": "docs/zh_cn/tutorials/windows.md", "content": "---\ntitle: 在 Windows 上使用 JuiceFS\nsidebar_position: 1\n---\n\n## 快速上手视频\n\n
\n \n
\n\n## 安装 JuiceFS 客户端\n\n:::tip 环境依赖\n在 Windows 系统上，JuiceFS 依赖 WinFsp 实现文件系统的挂载。你可以在 [WinFsp 源码仓库](https://github.com/winfsp/winfsp) 下载最新版本，安装后建议重启计算机，以确保所有组件正常加载。\n:::\n\n[安装文档](../getting-started/installation.md#windows) 介绍了在 Windows 上安装 JuiceFS 客户端的多种方式，这里我们展开介绍手动安装方式。\n\n### 第一步 下载 JuiceFS 客户端\n\n在项目仓库的 [Release 页面](https://github.com/juicedata/juicefs/releases) 下载最新版本的 JuiceFS 客户端，例如 `juicefs-1.3.0-windows-amd64.tar.gz`。\n\n### 第二步 创建程序目录\n\n为了便于管理，建议在系统中创建一个专用的目录来存放 JuiceFS 客户端程序。例如，可以在 `C:\\` 目录下创建一个名为 `juicefs` 的文件夹，将解压后的 `juicefs.exe` 客户端程序放入该目录。\n\n### 第三步 配置环境变量\n\n为了在命令行中方便地使用 `juicefs` 命令，需要将 JuiceFS 客户端所在的目录添加到系统的环境变量中。具体操作如下：\n\n1. 右键点击“此电脑”或“计算机”，选择“属性”；\n2. 点击“高级系统设置”；\n3. 在“系统属性”窗口中，点击“环境变量”按钮；\n4. 在“系统变量”部分，找到名为 `Path` 的变量，选中后点击“编辑”；\n5. 在编辑窗口中，点击“新建”，然后输入 JuiceFS 客户端所在的目录路径，例如 `C:\\juicefs`；\n6. 点击“确定”保存更改。\n\n![Windows 环境变量设置](https://static1.juicefs.com/docs/windows-path.png)\n\n### 第四步 验证安装\n\n安装完成后，可以通过命令行验证 JuiceFS 客户端是否安装成功。打开命令提示符（CMD）或 PowerShell，输入以下命令：\n\n```bash\njuicefs version\n```\n\n如果安装成功，你应该能看到类似以下的输出：\n\n```\njuicefs version 1.3.0+2025-07-03.30190ca1094d2\n```\n\n## 创建和挂载文件系统\n\n创建和挂载 JuiceFS 文件系统的步骤与其他操作系统类似，但需要注意 Windows 上的命令行语法和路径格式。\n\n### 创建文件系统\n\n```shell\njuicefs format --storage oss `\n --bucket https://your-bucket.oss-cn-region.aliyuncs.com `\n --access-key your-access-key `\n --secret-key your-secret-key `\n redis://your-redis-host:6379/0 `\n mywinfs\n```\n\n> 与 Linux 系统不同，Windows 上的命令行需要使用反引号（`）来换行。\n\n### 挂载文件系统\n\n在 Windows 上，挂载点需要指定一个未被占用的盘符（如 X、Y、Z 等）。这与 Linux 和 macOS 上的挂载方式不同，因为这些系统是将文件系统挂载到目录中。\n\n```shell\njuicefs mount -d redis://your-redis-host:6379/0 X:\n```\n\n## 环境变量配置\n\n从安全性的角度出发，为了避免明文输入密码，可以通过设置环境变量来存储敏感信息。这样在挂载文件系统或启用 S3 Gateway 时无需填写密码，客户端会自动从环境变量中读取。\n\n以下是在 Windows 上使用 JuiceFS 时常用的环境变量：\n\n| 环境变量名 | 说明 |\n|----------------------|------------------------|\n| `META_PASSWORD` | 元数据引擎密码 |\n| `MINIO_ROOT_USER` | S3 网关 Access Key |\n| `MINIO_ROOT_PASSWORD`| S3 网关 Secret Key |\n\n可以直接在命令行设置这些环境变量：\n\n```cmd\nset META_PASSWORD=your_password\nset MINIO_ROOT_USER=your_access_key\nset MINIO_ROOT_PASSWORD=your_secret_key\n```\n\n但这样的设置方式仅在当前命令行会话中有效，关闭窗口后环境变量失效，需重新设置。\n\n### 持久化环境变量\n\n如果希望在每次启动 Windows 时都能自动加载这些环境变量，可以通过系统环境变量设置来实现。\n\n1. **打开系统环境变量设置**\n - 按下 `Win + S`，搜索并打开“编辑系统环境变量”。\n - 点击“环境变量”按钮。\n\n ![系统环境变量设置](https://static1.juicefs.com/docs/win_env_01.png)\n\n2. **新建系统级环境变量**\n - 在“系统变量”区域点击“新建”。\n - **变量名**：例如 `META_PASSWORD`\n - **变量值**：填写密码或秘钥\n - 点击“确定”保存。\n\n ![添加环境变量](https://static1.juicefs.com/docs/win_env_02.png)\n\n ![添加环境变量](https://static1.juicefs.com/docs/win_env_03.png)\n\n3. **验证环境变量**\n\n 重新打开终端，尝试不带密码挂载文件系统。如果能够成功挂载，则说明环境变量已生效。\n\n## 开机自启动挂载\n\n通过 Windows 计划任务实现开机自动挂载有多种方式，这里介绍通过“任务计划程序”设置的方法。\n\n1. 打开“任务计划程序”，点击“创建任务”。\n\n ![任务计划程序](https://static1.juicefs.com/docs/task_00.png)\n\n2. 在“常规”选项卡中，设置任务名称（如 `JuiceFS_AutoMount`），并勾选“使用最高权限运行”。\n\n ![常规设置](https://static1.juicefs.com/docs/task_01.png)\n\n3. 切换到“触发器”选项卡，点击“新建”，选择“系统启动时”作为触发条件。\n\n ![触发器设置](https://static1.juicefs.com/docs/task_02.png)\n\n4. 切换到“操作”选项卡，点击“新建”，填写以下信息：\n\n - **程序或脚本**：浏览选择 JuiceFS 客户端路径（如 `C:\\juicefs\\juicefs.exe`）。\n - **参数**：填写挂载命令参数。建议将元数据引擎密码通过系统环境变量进行设置，这样可以避免在此处明文输入密码。\n\n ![触发器设置](https://static1.juicefs.com/docs/task_03.png)\n\n5. 在“条件”选项卡中，勾选“仅当网络连接可用时”，以确保挂载操作在网络可用时执行。\n\n ![触发器设置](https://static1.juicefs.com/docs/task_04.png)\n\n6. 点击“确定”保存任务。\n\n**注意事项：**\n\n- 确保挂载命令参数正确，无需在命令中包含密码（环境变量已存储）。\n- 卸载文件系统：右键点击挂载盘符，选择“断开连接”。\n" }, { "path": "go.mod", "content": "module github.com/juicedata/juicefs\n\ngo 1.23.0\n\nrequire (\n\tcloud.google.com/go/compute/metadata v0.5.2\n\tcloud.google.com/go/storage v1.48.0\n\tgithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0\n\tgithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0\n\tgithub.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1\n\tgithub.com/DataDog/zstd v1.5.6\n\tgithub.com/IBM/ibm-cos-sdk-go v1.12.1\n\tgithub.com/agiledragon/gomonkey/v2 v2.6.0\n\tgithub.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1\n\tgithub.com/aliyun/credentials-go v1.4.5\n\tgithub.com/apple/foundationdb/bindings/go v0.0.0-20211207225159-47b9a81d1c10\n\tgithub.com/aws/aws-sdk-go-v2 v1.36.1\n\tgithub.com/aws/aws-sdk-go-v2/config v1.29.6\n\tgithub.com/aws/aws-sdk-go-v2/credentials v1.17.59\n\tgithub.com/aws/aws-sdk-go-v2/service/s3 v1.72.3\n\tgithub.com/aws/smithy-go v1.22.2\n\tgithub.com/baidubce/bce-sdk-go v0.9.221\n\tgithub.com/bytedance/mockey v1.2.14\n\tgithub.com/ceph/go-ceph v0.18.0\n\tgithub.com/charlievieth/fastwalk v1.0.14\n\tgithub.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc\n\tgithub.com/colinmarc/hdfs/v2 v2.4.0\n\tgithub.com/davies/groupcache v0.0.0-20230821031435-e4e8362f58e1\n\tgithub.com/dgraph-io/badger/v4 v4.5.1\n\tgithub.com/dustin/go-humanize v1.0.1\n\tgithub.com/emmansun/gmsm v0.34.1\n\tgithub.com/erikdubbelboer/gspt v0.0.0-20210805194459-ce36a5128377\n\tgithub.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a\n\tgithub.com/go-sql-driver/mysql v1.9.1\n\tgithub.com/goccy/go-json v0.10.5\n\tgithub.com/gofrs/flock v0.8.1\n\tgithub.com/golang/snappy v0.0.4\n\tgithub.com/google/btree v1.1.2\n\tgithub.com/google/uuid v1.6.0\n\tgithub.com/grafana/pyroscope-go v1.2.1\n\tgithub.com/grafana/pyroscope-go/godeltaprof v0.1.8\n\tgithub.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8\n\tgithub.com/hashicorp/consul/api v1.29.2\n\tgithub.com/hashicorp/go-hclog v1.6.3\n\tgithub.com/hashicorp/golang-lru/v2 v2.0.7\n\tgithub.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible\n\tgithub.com/hungys/go-lz4 v0.0.0-20170805124057-19ff7f07f099\n\tgithub.com/jackc/pgx/v5 v5.7.3\n\tgithub.com/jcmturner/gokrb5/v8 v8.4.4\n\tgithub.com/json-iterator/go v1.1.12\n\tgithub.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d\n\tgithub.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80\n\tgithub.com/juju/ratelimit v1.0.2\n\tgithub.com/ks3sdklib/aws-sdk-go v1.6.0\n\tgithub.com/l0wl3vel/bunny-storage-go-sdk v0.0.10\n\tgithub.com/mattn/go-isatty v0.0.20\n\tgithub.com/mattn/go-sqlite3 v1.14.24\n\tgithub.com/minio/cli v1.24.2\n\tgithub.com/minio/minio v0.0.0-20210206053228-97fe57bba92c\n\tgithub.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78\n\tgithub.com/ncw/swift/v2 v2.0.3\n\tgithub.com/oliverisaac/shellescape v0.0.0-20220131224704-1b6c6b87b668\n\tgithub.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3\n\tgithub.com/pkg/errors v0.9.1\n\tgithub.com/pkg/sftp v1.13.5\n\tgithub.com/pkg/xattr v0.4.9\n\tgithub.com/prometheus/client_golang v1.21.1\n\tgithub.com/prometheus/client_model v0.6.1\n\tgithub.com/prometheus/common v0.62.0\n\tgithub.com/prometheus/prometheus v0.54.1\n\tgithub.com/qingstor/qingstor-sdk-go/v4 v4.4.0\n\tgithub.com/qiniu/go-sdk/v7 v7.25.2\n\tgithub.com/redis/go-redis/v9 v9.16.0\n\tgithub.com/sirupsen/logrus v1.9.3\n\tgithub.com/smartystreets/goconvey v1.7.2\n\tgithub.com/spf13/cast v1.7.1\n\tgithub.com/stretchr/testify v1.10.0\n\tgithub.com/studio-b12/gowebdav v0.10.0\n\tgithub.com/tencentyun/cos-go-sdk-v5 v0.7.63\n\tgithub.com/tikv/client-go/v2 v2.0.7\n\tgithub.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1\n\tgithub.com/twmb/murmur3 v1.1.8\n\tgithub.com/urfave/cli/v2 v2.19.3\n\tgithub.com/vbauerster/mpb/v7 v7.0.3\n\tgithub.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8\n\tgithub.com/vimeo/go-util v1.4.1\n\tgithub.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b\n\tgithub.com/volcengine/ve-tos-golang-sdk/v2 v2.7.8\n\tgithub.com/winfsp/cgofuse v1.6.0\n\tgo.etcd.io/etcd v3.3.27+incompatible\n\tgo.etcd.io/etcd/client/v3 v3.5.9\n\tgo.uber.org/automaxprocs v1.6.0\n\tgo.uber.org/zap v1.24.0\n\tgolang.org/x/crypto v0.41.0\n\tgolang.org/x/net v0.42.0\n\tgolang.org/x/oauth2 v0.24.0\n\tgolang.org/x/sync v0.16.0\n\tgolang.org/x/sys v0.35.0\n\tgolang.org/x/term v0.34.0\n\tgolang.org/x/text v0.28.0\n\tgoogle.golang.org/api v0.210.0\n\tgoogle.golang.org/protobuf v1.36.3\n\tgopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216\n\tpgregory.net/rapid v0.5.3\n\txorm.io/xorm v1.0.7\n)\n\nrequire (\n\tcel.dev/expr v0.16.1 // indirect\n\tcloud.google.com/go v0.116.0 // indirect\n\tcloud.google.com/go/auth v0.11.0 // indirect\n\tcloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect\n\tcloud.google.com/go/iam v1.2.2 // indirect\n\tcloud.google.com/go/monitoring v1.21.2 // indirect\n\tfilippo.io/edwards25519 v1.1.0 // indirect\n\tgit.apache.org/thrift.git v0.13.0 // indirect\n\tgithub.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect\n\tgithub.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect\n\tgithub.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect\n\tgithub.com/BurntSushi/toml v1.3.2 // indirect\n\tgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect\n\tgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect\n\tgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect\n\tgithub.com/IBM/go-sdk-core/v5 v5.18.5 // indirect\n\tgithub.com/VividCortex/ewma v1.2.0 // indirect\n\tgithub.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect\n\tgithub.com/alecthomas/participle v0.2.1 // indirect\n\tgithub.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect\n\tgithub.com/alibabacloud-go/debug v1.0.1 // indirect\n\tgithub.com/alibabacloud-go/tea v1.2.2 // indirect\n\tgithub.com/andybalholm/brotli v1.1.0 // indirect\n\tgithub.com/armon/go-metrics v0.4.1 // indirect\n\tgithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect\n\tgithub.com/bcicen/jstream v1.0.1 // indirect\n\tgithub.com/beevik/ntp v0.3.0 // indirect\n\tgithub.com/benbjohnson/clock v1.3.0 // indirect\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/census-instrumentation/opencensus-proto v0.4.1 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.3.0 // indirect\n\tgithub.com/cheggaaa/pb v1.0.29 // indirect\n\tgithub.com/clbanning/mxj v1.8.4 // indirect\n\tgithub.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect\n\tgithub.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect\n\tgithub.com/coredns/coredns v1.4.0 // indirect\n\tgithub.com/coreos/etcd v3.3.27+incompatible // indirect\n\tgithub.com/coreos/go-semver v0.3.0 // indirect\n\tgithub.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect\n\tgithub.com/coreos/go-systemd/v22 v22.5.0 // indirect\n\tgithub.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect\n\tgithub.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect\n\tgithub.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect\n\tgithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect\n\tgithub.com/dchest/siphash v1.2.1 // indirect\n\tgithub.com/dgraph-io/ristretto/v2 v2.1.0 // indirect\n\tgithub.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect\n\tgithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect\n\tgithub.com/djherbis/atime v1.0.0 // indirect\n\tgithub.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d // indirect\n\tgithub.com/elastic/gosigar v0.14.2 // indirect\n\tgithub.com/envoyproxy/go-control-plane v0.13.0 // indirect\n\tgithub.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect\n\tgithub.com/fatih/color v1.16.0 // indirect\n\tgithub.com/fatih/structs v1.1.0 // indirect\n\tgithub.com/felixge/httpsnoop v1.0.4 // indirect\n\tgithub.com/gabriel-vasile/mimetype v1.4.3 // indirect\n\tgithub.com/gammazero/toposort v0.1.1 // indirect\n\tgithub.com/geoffgarside/ber v1.1.0 // indirect\n\tgithub.com/go-asn1-ber/asn1-ber v1.5.1 // indirect\n\tgithub.com/go-ldap/ldap/v3 v3.2.4 // indirect\n\tgithub.com/go-logr/logr v1.4.2 // indirect\n\tgithub.com/go-logr/stdr v1.2.2 // indirect\n\tgithub.com/go-ole/go-ole v1.2.6 // indirect\n\tgithub.com/go-openapi/errors v0.22.0 // indirect\n\tgithub.com/go-openapi/strfmt v0.23.0 // indirect\n\tgithub.com/go-playground/locales v0.14.1 // indirect\n\tgithub.com/go-playground/universal-translator v0.18.1 // indirect\n\tgithub.com/go-playground/validator/v10 v10.19.0 // indirect\n\tgithub.com/go-resty/resty/v2 v2.13.1 // indirect\n\tgithub.com/gogo/protobuf v1.3.2 // indirect\n\tgithub.com/golang-jwt/jwt/v4 v4.5.2 // indirect\n\tgithub.com/golang-jwt/jwt/v5 v5.2.2 // indirect\n\tgithub.com/golang/glog v1.2.2 // indirect\n\tgithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect\n\tgithub.com/golang/protobuf v1.5.4 // indirect\n\tgithub.com/google/flatbuffers v24.12.23+incompatible // indirect\n\tgithub.com/google/go-querystring v1.1.0 // indirect\n\tgithub.com/google/readahead v0.0.0-20161222183148-eaceba169032 // indirect\n\tgithub.com/google/s2a-go v0.1.8 // indirect\n\tgithub.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect\n\tgithub.com/googleapis/gax-go/v2 v2.14.0 // indirect\n\tgithub.com/gopherjs/gopherjs v1.12.80 // indirect\n\tgithub.com/gorilla/handlers v1.5.1 // indirect\n\tgithub.com/gorilla/mux v1.8.1 // indirect\n\tgithub.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect\n\tgithub.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect\n\tgithub.com/hashicorp/errwrap v1.1.0 // indirect\n\tgithub.com/hashicorp/go-cleanhttp v0.5.2 // indirect\n\tgithub.com/hashicorp/go-immutable-radix v1.3.1 // indirect\n\tgithub.com/hashicorp/go-multierror v1.1.1 // indirect\n\tgithub.com/hashicorp/go-retryablehttp v0.7.7 // indirect\n\tgithub.com/hashicorp/go-rootcerts v1.0.2 // indirect\n\tgithub.com/hashicorp/go-uuid v1.0.3 // indirect\n\tgithub.com/hashicorp/golang-lru v0.6.0 // indirect\n\tgithub.com/hashicorp/serf v0.10.1 // indirect\n\tgithub.com/jackc/pgpassfile v1.0.0 // indirect\n\tgithub.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect\n\tgithub.com/jackc/puddle/v2 v2.2.2 // indirect\n\tgithub.com/jcmturner/aescts/v2 v2.0.0 // indirect\n\tgithub.com/jcmturner/dnsutils/v2 v2.0.0 // indirect\n\tgithub.com/jcmturner/gofork v1.7.6 // indirect\n\tgithub.com/jcmturner/goidentity/v6 v6.0.1 // indirect\n\tgithub.com/jcmturner/rpc/v2 v2.0.3 // indirect\n\tgithub.com/jmespath/go-jmespath v0.4.0 // indirect\n\tgithub.com/jtolds/gls v4.20.0+incompatible // indirect\n\tgithub.com/klauspost/compress v1.17.11 // indirect\n\tgithub.com/klauspost/cpuid v1.3.1 // indirect\n\tgithub.com/klauspost/cpuid/v2 v2.2.3 // indirect\n\tgithub.com/klauspost/pgzip v1.2.5 // indirect\n\tgithub.com/klauspost/readahead v1.3.1 // indirect\n\tgithub.com/klauspost/reedsolomon v1.9.11 // indirect\n\tgithub.com/kr/fs v0.1.0 // indirect\n\tgithub.com/kylelemons/godebug v1.1.0 // indirect\n\tgithub.com/leodido/go-urn v1.4.0 // indirect\n\tgithub.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect\n\tgithub.com/mattn/go-colorable v0.1.13 // indirect\n\tgithub.com/mattn/go-runewidth v0.0.13 // indirect\n\tgithub.com/miekg/dns v1.1.61 // indirect\n\tgithub.com/minio/highwayhash v1.0.2 // indirect\n\tgithub.com/minio/md5-simd v1.1.1 // indirect\n\tgithub.com/minio/selfupdate v0.3.1 // indirect\n\tgithub.com/minio/sha256-simd v1.0.1 // indirect\n\tgithub.com/minio/simdjson-go v0.2.1 // indirect\n\tgithub.com/minio/sio v0.2.1 // indirect\n\tgithub.com/mitchellh/go-homedir v1.1.0 // indirect\n\tgithub.com/mitchellh/mapstructure v1.5.0 // indirect\n\tgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect\n\tgithub.com/modern-go/reflect2 v1.0.2 // indirect\n\tgithub.com/montanaflynn/stats v0.7.0 // indirect\n\tgithub.com/mozillazg/go-httpheader v0.2.1 // indirect\n\tgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect\n\tgithub.com/ncw/directio v1.0.5 // indirect\n\tgithub.com/oklog/ulid v1.3.1 // indirect\n\tgithub.com/opentracing/opentracing-go v1.2.0 // indirect\n\tgithub.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect\n\tgithub.com/philhofer/fwd v1.1.1 // indirect\n\tgithub.com/pierrec/lz4 v2.5.2+incompatible // indirect\n\tgithub.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect\n\tgithub.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect\n\tgithub.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect\n\tgithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect\n\tgithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect\n\tgithub.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect\n\tgithub.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 // indirect\n\tgithub.com/prometheus/procfs v0.15.1 // indirect\n\tgithub.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect\n\tgithub.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect\n\tgithub.com/rivo/uniseg v0.2.0 // indirect\n\tgithub.com/rjeczalik/notify v0.9.3 // indirect\n\tgithub.com/rs/cors v1.7.0 // indirect\n\tgithub.com/rs/xid v1.2.1 // indirect\n\tgithub.com/russross/blackfriday/v2 v2.1.0 // indirect\n\tgithub.com/secure-io/sio-go v0.3.1 // indirect\n\tgithub.com/shirou/gopsutil/v3 v3.23.11 // indirect\n\tgithub.com/shoenig/go-m1cpu v0.1.6 // indirect\n\tgithub.com/smartystreets/assertions v1.2.0 // indirect\n\tgithub.com/spaolacci/murmur3 v1.1.0 // indirect\n\tgithub.com/syndtr/goleveldb v1.0.0 // indirect\n\tgithub.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect\n\tgithub.com/tidwall/gjson v1.6.7 // indirect\n\tgithub.com/tidwall/match v1.0.3 // indirect\n\tgithub.com/tidwall/pretty v1.0.2 // indirect\n\tgithub.com/tidwall/sjson v1.0.4 // indirect\n\tgithub.com/tinylib/msgp v1.1.3 // indirect\n\tgithub.com/tklauser/go-sysconf v0.3.12 // indirect\n\tgithub.com/tklauser/numcpus v0.6.1 // indirect\n\tgithub.com/valyala/bytebufferpool v1.0.0 // indirect\n\tgithub.com/valyala/fasthttp v1.52.0 // indirect\n\tgithub.com/valyala/tcplisten v1.0.0 // indirect\n\tgithub.com/willf/bitset v1.1.11 // indirect\n\tgithub.com/willf/bloom v2.0.3+incompatible // indirect\n\tgithub.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect\n\tgithub.com/yusufpapurcu/wmi v1.2.3 // indirect\n\tgo.etcd.io/etcd/api/v3 v3.5.9 // indirect\n\tgo.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect\n\tgo.mongodb.org/mongo-driver v1.14.0 // indirect\n\tgo.opencensus.io v0.24.0 // indirect\n\tgo.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect\n\tgo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect\n\tgo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect\n\tgo.opentelemetry.io/otel v1.29.0 // indirect\n\tgo.opentelemetry.io/otel/metric v1.29.0 // indirect\n\tgo.opentelemetry.io/otel/sdk v1.29.0 // indirect\n\tgo.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect\n\tgo.opentelemetry.io/otel/trace v1.29.0 // indirect\n\tgo.uber.org/atomic v1.11.0 // indirect\n\tgo.uber.org/multierr v1.11.0 // indirect\n\tgolang.org/x/arch v0.11.0 // indirect\n\tgolang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect\n\tgolang.org/x/mod v0.26.0 // indirect\n\tgolang.org/x/time v0.8.0 // indirect\n\tgolang.org/x/tools v0.35.0 // indirect\n\tgolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect\n\tgoogle.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect\n\tgoogle.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f // indirect\n\tgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect\n\tgoogle.golang.org/grpc v1.67.2 // indirect\n\tgoogle.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect\n\tgopkg.in/ini.v1 v1.67.0 // indirect\n\tgopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect\n\tgopkg.in/yaml.v2 v2.4.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n\tmodernc.org/fileutil v1.0.0 // indirect\n\txorm.io/builder v0.3.7 // indirect\n)\n\nreplace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20251120043259-079fa6a601db\n\nreplace github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 => github.com/juicedata/go-fuse/v2 v2.1.1-0.20250807045235-112198daa7df\n\nreplace github.com/dgrijalva/jwt-go v3.2.0+incompatible => github.com/golang-jwt/jwt v3.2.1+incompatible\n\nreplace github.com/vbauerster/mpb/v7 v7.0.3 => github.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b\n\nreplace xorm.io/xorm v1.0.7 => gitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a\n\nreplace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible\n\nreplace github.com/urfave/cli/v2 v2.19.3 => github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83\n\nreplace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20250220101412-d3a8c1ca64a1\n\nreplace github.com/mattn/go-colorable v0.1.13 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db\n\nreplace github.com/mattn/go-colorable v0.1.12 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db\n\nreplace github.com/mattn/go-colorable v0.1.4 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db\n\nreplace github.com/mattn/go-colorable v0.1.6 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db\n\nreplace github.com/mattn/go-colorable v0.1.9 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db\n\nreplace github.com/mattn/go-colorable v0.0.9 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db\n\nreplace github.com/cloudsoda/go-smb2 => github.com/juicedata/go-smb2 v0.0.0-20260310064141-58f27d06634e\n\nreplace github.com/hashicorp/golang-lru/v2 v2.0.7 => github.com/juicedata/golang-lru/v2 v2.0.8-0.20251126062551-1b321869f904\n" }, { "path": "go.sum", "content": "cel.dev/expr v0.16.1 h1:NR0+oFYzR1CqLFhTAqg3ql59G9VfN8fKq1TCHJ6gq1g=\ncel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8=\ncloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE=\ncloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U=\ncloud.google.com/go/auth v0.11.0 h1:Ic5SZz2lsvbYcWT5dfjNWgw6tTlGi2Wc8hyQSC9BstA=\ncloud.google.com/go/auth v0.11.0/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=\ncloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU=\ncloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=\ncloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=\ncloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=\ncloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA=\ncloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY=\ncloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk=\ncloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM=\ncloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc=\ncloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI=\ncloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU=\ncloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU=\ncloud.google.com/go/storage v1.48.0 h1:FhBDHACbVtdPx7S/AbcKujPWiHvfO6F8OXGgCEbB2+o=\ncloud.google.com/go/storage v1.48.0/go.mod h1:aFoDYNMAjv67lp+xcuZqjUKv/ctmplzQ3wJgodA7b+M=\ncloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI=\ncloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io=\nfilippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=\nfilippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=\ngit.apache.org/thrift.git v0.13.0 h1:/3bz5WZ+sqYArk7MBBBbDufMxKKOA56/6JO6psDpUDY=\ngit.apache.org/thrift.git v0.13.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=\ngitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a h1:awR9qREIs6qSnKr/cmSewVwDo74/kQ32x0CDEXUtiB8=\ngitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a/go.mod h1:uF9EtbhODq5kNWxMbnBEj8hRRZnlcNSz2t2N7HW/+A4=\ngitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=\ngitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=\ngithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=\ngithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=\ngithub.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=\ngithub.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=\ngithub.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=\ngithub.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=\ngithub.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=\ngithub.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=\ngithub.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=\ngithub.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=\ngithub.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=\ngithub.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=\ngithub.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=\ngithub.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=\ngithub.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY=\ngithub.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 h1:pB2F2JKCj1Znmp2rwxxt1J0Fg0wezTMgWYk5Mpbi1kg=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE=\ngithub.com/IBM/go-sdk-core/v5 v5.18.5 h1:g0JRl3sYXJczB/yuDlrN6x22LJ6jIxhp0Sa4ARNW60c=\ngithub.com/IBM/go-sdk-core/v5 v5.18.5/go.mod h1:KonTFRR+8ZSgw5cxBSYo6E4WZoY1+7n1kfHM82VcjFU=\ngithub.com/IBM/ibm-cos-sdk-go v1.12.1 h1:pWs5c5/j9PNJE1lIQhYtzpdCxu2fpvCq9PHs6/nDjyI=\ngithub.com/IBM/ibm-cos-sdk-go v1.12.1/go.mod h1:7vmUThyAq4+AD1eEyGZi90ir06Z9YhsEzLBsdGPfcqo=\ngithub.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=\ngithub.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=\ngithub.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=\ngithub.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=\ngithub.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=\ngithub.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=\ngithub.com/agiledragon/gomonkey/v2 v2.6.0 h1:RzdlW1ibfVipfXKy9U4zYumdHTIY7RoZwyXY3tXLYd8=\ngithub.com/agiledragon/gomonkey/v2 v2.6.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=\ngithub.com/alecthomas/participle v0.2.1 h1:4AVLj1viSGa4LG5HDXKXrm5xRx19SB/rS/skPQB1Grw=\ngithub.com/alecthomas/participle v0.2.1/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk=\ngithub.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4=\ngithub.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI=\ngithub.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc=\ngithub.com/alibabacloud-go/debug v1.0.1 h1:MsW9SmUtbb1Fnt3ieC6NNZi6aEwrXfDksD4QA6GSbPg=\ngithub.com/alibabacloud-go/debug v1.0.1/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc=\ngithub.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU=\ngithub.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk=\ngithub.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 h1:sOhpJdR/+lbQniznp3cYSfwQlXbVkT0ccuiZScBrI6Y=\ngithub.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M=\ngithub.com/aliyun/credentials-go v1.4.5 h1:O76WYKgdy1oQYYiJkERjlA2dxGuvLRrzuO2ScrtGWSk=\ngithub.com/aliyun/credentials-go v1.4.5/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U=\ngithub.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=\ngithub.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=\ngithub.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=\ngithub.com/apple/foundationdb/bindings/go v0.0.0-20211207225159-47b9a81d1c10 h1:xU6bzJilZ630rLUhRsqWgJjSl2PCn5uLrehoG6ntwls=\ngithub.com/apple/foundationdb/bindings/go v0.0.0-20211207225159-47b9a81d1c10/go.mod h1:w63jdZTFCtvdjsUj5yrdKgjxaAD5uXQX6hJ7EaiLFRs=\ngithub.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=\ngithub.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=\ngithub.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=\ngithub.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=\ngithub.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=\ngithub.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=\ngithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=\ngithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=\ngithub.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E=\ngithub.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=\ngithub.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=\ngithub.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=\ngithub.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg=\ngithub.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ=\ngithub.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4=\ngithub.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08=\ngithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso=\ngithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI=\ngithub.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA=\ngithub.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU=\ngithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4=\ngithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI=\ngithub.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=\ngithub.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=\ngithub.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 h1:AmB5QxnD+fBFrg9LcqzkgF/CaYvMyU/BTlejG4t1S7Q=\ngithub.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27/go.mod h1:Sai7P3xTiyv9ZUYO3IFxMnmiIP759/67iQbU4kdmkyU=\ngithub.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=\ngithub.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=\ngithub.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 h1:iwYS40JnrBeA9e9aI5S6KKN4EB2zR4iUVYN0nwVivz4=\ngithub.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8/go.mod h1:Fm9Mi+ApqmFiknZtGpohVcBGvpTu542VC4XO9YudRi0=\ngithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw=\ngithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE=\ngithub.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 h1:/Mn7gTedG86nbpjT4QEKsN1D/fThiYe1qvq7WsBGNHg=\ngithub.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8/go.mod h1:Ae3va9LPmvjj231ukHB6UeT8nS7wTPfC3tMZSZMwNYg=\ngithub.com/aws/aws-sdk-go-v2/service/s3 v1.72.3 h1:WZOmJfCDV+4tYacLxpiojoAdT5sxTfB3nTqQNtZu+J4=\ngithub.com/aws/aws-sdk-go-v2/service/s3 v1.72.3/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4=\ngithub.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ=\ngithub.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y=\ngithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE=\ngithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ=\ngithub.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE=\ngithub.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc=\ngithub.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=\ngithub.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=\ngithub.com/baidubce/bce-sdk-go v0.9.221 h1:x5uTXND33m5TE3UBXYhlePuXcJi5rxNnBBt+bP7kPe0=\ngithub.com/baidubce/bce-sdk-go v0.9.221/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=\ngithub.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck=\ngithub.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ=\ngithub.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=\ngithub.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=\ngithub.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=\ngithub.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=\ngithub.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=\ngithub.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=\ngithub.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=\ngithub.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=\ngithub.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=\ngithub.com/bytedance/mockey v1.2.14 h1:KZaFgPdiUwW+jOWFieo3Lr7INM1P+6adO3hxZhDswY8=\ngithub.com/bytedance/mockey v1.2.14/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=\ngithub.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=\ngithub.com/ceph/go-ceph v0.18.0 h1:4WM6yAq/iqBDaeeADDiPKLqKiP0iZ4fffdgCr1lnOL4=\ngithub.com/ceph/go-ceph v0.18.0/go.mod h1:cflETVTBNAQM6jdr7hpNHHFHKYiJiWWcAeRDrRx/1ng=\ngithub.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=\ngithub.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/charlievieth/fastwalk v1.0.14 h1:3Eh5uaFGwHZd8EGwTjJnSpBkfwfsak9h6ICgnWlhAyg=\ngithub.com/charlievieth/fastwalk v1.0.14/go.mod h1:diVcUreiU1aQ4/Wu3NbxxH4/KYdKpLDojrQ1Bb2KgNY=\ngithub.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=\ngithub.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=\ngithub.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=\ngithub.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=\ngithub.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=\ngithub.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=\ngithub.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI=\ngithub.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=\ngithub.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0=\ngithub.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI=\ngithub.com/coredns/coredns v1.4.0 h1:RubBkYmkByUqZWWkjRHvNLnUHgkRVqAWgSMmRFvpE1A=\ngithub.com/coredns/coredns v1.4.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=\ngithub.com/coreos/etcd v3.3.27+incompatible h1:QIudLb9KeBsE5zyYxd1mjzRSkzLg9Wf9QlRwFgd6oTA=\ngithub.com/coreos/etcd v3.3.27+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=\ngithub.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=\ngithub.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=\ngithub.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=\ngithub.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=\ngithub.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=\ngithub.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=\ngithub.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE=\ngithub.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=\ngithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=\ngithub.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=\ngithub.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davies/groupcache v0.0.0-20230821031435-e4e8362f58e1 h1:m8crlQg+91orxncf8Xt6utYTKi9N2PqbPvOnVmb2p24=\ngithub.com/davies/groupcache v0.0.0-20230821031435-e4e8362f58e1/go.mod h1:rUkViuo3izQae5A7J4apO+ALkf5DqvVwKGzAbROmZUE=\ngithub.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4=\ngithub.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=\ngithub.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=\ngithub.com/dgraph-io/badger/v4 v4.5.1 h1:7DCIXrQjo1LKmM96YD+hLVJ2EEsyyoWxJfpdd56HLps=\ngithub.com/dgraph-io/badger/v4 v4.5.1/go.mod h1:qn3Be0j3TfV4kPbVoK0arXCD1/nr1ftth6sbL5jxdoA=\ngithub.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I=\ngithub.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4=\ngithub.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=\ngithub.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=\ngithub.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=\ngithub.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=\ngithub.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d h1:QK8IYltsNy+5QZcDFbVkyInrs98/wHy1tfUTGG91sps=\ngithub.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d/go.mod h1:apXo4PA/BgBPrt66j0N45O2stlBTRowdip2igwcUWVc=\ngithub.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=\ngithub.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=\ngithub.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=\ngithub.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=\ngithub.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=\ngithub.com/emmansun/gmsm v0.34.1 h1:7eMyHjB0AeoSZ+sB3FZE9gZOJBZFbtY0tmWJdVFkfc0=\ngithub.com/emmansun/gmsm v0.34.1/go.mod h1:NtH8X3s0ywBIICiOHD6Jj6P4brHHN6qUOI/nSK/x1jQ=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les=\ngithub.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM=\ngithub.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=\ngithub.com/erikdubbelboer/gspt v0.0.0-20210805194459-ce36a5128377 h1:gT+RM6gdTIAzMT7HUvmT5mL8SyG8Wx7iS3+L0V34Km4=\ngithub.com/erikdubbelboer/gspt v0.0.0-20210805194459-ce36a5128377/go.mod h1:v6o7m/E9bfvm79dE1iFiF+3T7zLBnrjYjkWMa1J+Hv0=\ngithub.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=\ngithub.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=\ngithub.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=\ngithub.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=\ngithub.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=\ngithub.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=\ngithub.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=\ngithub.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=\ngithub.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=\ngithub.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=\ngithub.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=\ngithub.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=\ngithub.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=\ngithub.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=\ngithub.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=\ngithub.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=\ngithub.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=\ngithub.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg=\ngithub.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw=\ngithub.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=\ngithub.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=\ngithub.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8=\ngithub.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=\ngithub.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=\ngithub.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=\ngithub.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-ldap/ldap/v3 v3.2.4 h1:PFavAq2xTgzo/loE8qNXcQaofAaqIpI4WgaLdv+1l3E=\ngithub.com/go-ldap/ldap/v3 v3.2.4/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg=\ngithub.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=\ngithub.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=\ngithub.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=\ngithub.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=\ngithub.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=\ngithub.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=\ngithub.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=\ngithub.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w=\ngithub.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE=\ngithub.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c=\ngithub.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4=\ngithub.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=\ngithub.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=\ngithub.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=\ngithub.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=\ngithub.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=\ngithub.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=\ngithub.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=\ngithub.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=\ngithub.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=\ngithub.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=\ngithub.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=\ngithub.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=\ngithub.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=\ngithub.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=\ngithub.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g=\ngithub.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0=\ngithub.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=\ngithub.com/go-sql-driver/mysql v1.9.1 h1:FrjNGn/BsJQjVRuSa8CBrM5BWA9BWoXXat3KrtSb/iI=\ngithub.com/go-sql-driver/mysql v1.9.1/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=\ngithub.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=\ngithub.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=\ngithub.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=\ngithub.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=\ngithub.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc=\ngithub.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=\ngithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=\ngithub.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=\ngithub.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=\ngithub.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=\ngithub.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=\ngithub.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=\ngithub.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=\ngithub.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=\ngithub.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY=\ngithub.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=\ngithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=\ngithub.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=\ngithub.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=\ngithub.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=\ngithub.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=\ngithub.com/google/flatbuffers v24.12.23+incompatible h1:ubBKR94NR4pXUCY/MUsRVzd9umNW7ht7EG9hHfS9FX8=\ngithub.com/google/flatbuffers v24.12.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=\ngithub.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=\ngithub.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=\ngithub.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=\ngithub.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=\ngithub.com/google/readahead v0.0.0-20161222183148-eaceba169032 h1:6Be3nkuJFyRfCgr6qTIzmRp8y9QwDIbqy/nYr9WDPos=\ngithub.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y=\ngithub.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=\ngithub.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=\ngithub.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=\ngithub.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=\ngithub.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o=\ngithub.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=\ngithub.com/gopherjs/gopherjs v1.12.80 h1:aC68NT6VK715WeUapxcPSFq/a3gZdS32HdtghdOIgAo=\ngithub.com/gopherjs/gopherjs v1.12.80/go.mod h1:d55Q4EjGQHeJVms+9LGtXul6ykz5Xzx1E1gaXQXdimY=\ngithub.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=\ngithub.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=\ngithub.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=\ngithub.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=\ngithub.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=\ngithub.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=\ngithub.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=\ngithub.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=\ngithub.com/grafana/pyroscope-go v1.2.1 h1:ewi38pE6XMnoHlZYhGxS3uH5TGKA7vDhkT1T3RVkjq0=\ngithub.com/grafana/pyroscope-go v1.2.1/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU=\ngithub.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=\ngithub.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=\ngithub.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=\ngithub.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE=\ngithub.com/hashicorp/consul/api v1.29.2 h1:aYyRn8EdE2mSfG14S1+L9Qkjtz8RzmaWh6AcNGRNwPw=\ngithub.com/hashicorp/consul/api v1.29.2/go.mod h1:0YObcaLNDSbtlgzIRtmRXI1ZkeuK0trCBxwZQ4MYnIk=\ngithub.com/hashicorp/consul/proto-public v0.6.2 h1:+DA/3g/IiKlJZb88NBn0ZgXrxJp2NlvCZdEyl+qxvL0=\ngithub.com/hashicorp/consul/proto-public v0.6.2/go.mod h1:cXXbOg74KBNGajC+o8RlA502Esf0R9prcoJgiOX/2Tg=\ngithub.com/hashicorp/consul/sdk v0.16.1 h1:V8TxTnImoPD5cj0U9Spl0TUxcytjcbbJeADFF07KdHg=\ngithub.com/hashicorp/consul/sdk v0.16.1/go.mod h1:fSXvwxB2hmh1FMZCNl6PwX0Q/1wdWtHJcZ7Ea5tns0s=\ngithub.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=\ngithub.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=\ngithub.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=\ngithub.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=\ngithub.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=\ngithub.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=\ngithub.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=\ngithub.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=\ngithub.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=\ngithub.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=\ngithub.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=\ngithub.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=\ngithub.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=\ngithub.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=\ngithub.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=\ngithub.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=\ngithub.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=\ngithub.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=\ngithub.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=\ngithub.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=\ngithub.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=\ngithub.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=\ngithub.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=\ngithub.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=\ngithub.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=\ngithub.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=\ngithub.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=\ngithub.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=\ngithub.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=\ngithub.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=\ngithub.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=\ngithub.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=\ngithub.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=\ngithub.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=\ngithub.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=\ngithub.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=\ngithub.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=\ngithub.com/hungys/go-lz4 v0.0.0-20170805124057-19ff7f07f099 h1:heHZCso/ytvpYr+hp2cDxlZfA/jTw46aHSvT9kZnJ7o=\ngithub.com/hungys/go-lz4 v0.0.0-20170805124057-19ff7f07f099/go.mod h1:h44tqw4M3GN0Woo9KBStxJxm8huNi+9+tOHoeqSvhaY=\ngithub.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=\ngithub.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=\ngithub.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=\ngithub.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=\ngithub.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=\ngithub.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=\ngithub.com/jackc/pgx/v5 v5.7.3 h1:PO1wNKj/bTAwxSJnO1Z4Ai8j4magtqg2SLNjEDzcXQo=\ngithub.com/jackc/pgx/v5 v5.7.3/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=\ngithub.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=\ngithub.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=\ngithub.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=\ngithub.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=\ngithub.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=\ngithub.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=\ngithub.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=\ngithub.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=\ngithub.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=\ngithub.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=\ngithub.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=\ngithub.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=\ngithub.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=\ngithub.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=\ngithub.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=\ngithub.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=\ngithub.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\ngithub.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=\ngithub.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=\ngithub.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=\ngithub.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=\ngithub.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnTaUqfRYjlwcQlr53aasmkvHEbYLXthqr8=\ngithub.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI=\ngithub.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db h1:esc0bVXkjEuyPLn7JXFhKBDztpM0dT0GYQn7CqaBB6w=\ngithub.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=\ngithub.com/juicedata/go-fuse/v2 v2.1.1-0.20250807045235-112198daa7df h1:H3/AM/YZGPitgptMKBn3WrvWj7UrlhJSMHx4BrjuXMo=\ngithub.com/juicedata/go-fuse/v2 v2.1.1-0.20250807045235-112198daa7df/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=\ngithub.com/juicedata/go-nfs-client v0.0.0-20250220101412-d3a8c1ca64a1 h1:GgH2ZG9inMYSme7zZb79z3QeOW70YusbJIVYjvqd508=\ngithub.com/juicedata/go-nfs-client v0.0.0-20250220101412-d3a8c1ca64a1/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs=\ngithub.com/juicedata/go-smb2 v0.0.0-20260310064141-58f27d06634e h1:M4iUt9qotJuRbZgD1TLMzy4BkVGidsOXh4YvHDNKhdY=\ngithub.com/juicedata/go-smb2 v0.0.0-20260310064141-58f27d06634e/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=\ngithub.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA=\ngithub.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw=\ngithub.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80 h1:EPg/f3lhbAOjE2M0WpVi47Fk62mEmmPejRuGVdOFQww=\ngithub.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4=\ngithub.com/juicedata/golang-lru/v2 v2.0.8-0.20251126062551-1b321869f904 h1:oNtkL1jwrNMMcBlHNW1fhdl4quK7p1EdR7o1Rja5xpM=\ngithub.com/juicedata/golang-lru/v2 v2.0.8-0.20251126062551-1b321869f904/go.mod h1:qnbgnNzfydwuHjSCApF4bdul+tZ8T3y1MkZG/OFczLA=\ngithub.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s=\ngithub.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE=\ngithub.com/juicedata/minio v0.0.0-20251120043259-079fa6a601db h1:yGKlGEz3nOD2IovjI+V4O+eY1TPgOp/T6gOxMl9/xKI=\ngithub.com/juicedata/minio v0.0.0-20251120043259-079fa6a601db/go.mod h1:1/4WHQKDOsWA1dd3ADrq9IE/jtFec9MHLy656kIXjNg=\ngithub.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b h1:0/6suPNZnrOlRlBaU/Bnitu8HiKkkLSzQhHbwQ9AysM=\ngithub.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b/go.mod h1:NXGsfPGx6G2JssqvEcULtDqUrxuuYs4llpv8W6ZUpzk=\ngithub.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI=\ngithub.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=\ngithub.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=\ngithub.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=\ngithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=\ngithub.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=\ngithub.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=\ngithub.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=\ngithub.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=\ngithub.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=\ngithub.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=\ngithub.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=\ngithub.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=\ngithub.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=\ngithub.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=\ngithub.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=\ngithub.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0=\ngithub.com/klauspost/readahead v1.3.1/go.mod h1:AH9juHzNH7xqdqFHrMRSHeH2Ps+vFf+kblDqzPFiLJg=\ngithub.com/klauspost/reedsolomon v1.9.11 h1:n2kipJFo+CPqg7fH988XJXjqEyj14RJ8BYj7UayxPNg=\ngithub.com/klauspost/reedsolomon v1.9.11/go.mod h1:nLvuzNvy1ZDNQW30IuMc2ZWCbiqrJgdLoUS2X8HAUVg=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=\ngithub.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/ks3sdklib/aws-sdk-go v1.6.0 h1:ejTeQ+l5l5mok7MM3Bz8WW4/kUVjGkPSSKqllgp1uMc=\ngithub.com/ks3sdklib/aws-sdk-go v1.6.0/go.mod h1:jGcsV0dJgMmStAyqjkKVUu6F167pAXYZAS3LqoZMmtM=\ngithub.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=\ngithub.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=\ngithub.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=\ngithub.com/l0wl3vel/bunny-storage-go-sdk v0.0.10 h1:Vy8I4nGazW1QvwdIR3b/viHmBVFBf2i4RgR0dV0wJ/c=\ngithub.com/l0wl3vel/bunny-storage-go-sdk v0.0.10/go.mod h1:2kvY9oZnsZR4QAvtkj8s7MuEl37dTARhQz7ICLpyD2M=\ngithub.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=\ngithub.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=\ngithub.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=\ngithub.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=\ngithub.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=\ngithub.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=\ngithub.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=\ngithub.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=\ngithub.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=\ngithub.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=\ngithub.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=\ngithub.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=\ngithub.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=\ngithub.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=\ngithub.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=\ngithub.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=\ngithub.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=\ngithub.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=\ngithub.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=\ngithub.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=\ngithub.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=\ngithub.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=\ngithub.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=\ngithub.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=\ngithub.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=\ngithub.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=\ngithub.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=\ngithub.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=\ngithub.com/minio/md5-simd v1.1.1 h1:9ojcLbuZ4gXbB2sX53MKn8JUZ0sB/2wfwsEcRw+I08U=\ngithub.com/minio/md5-simd v1.1.1/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=\ngithub.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 h1:v7OMbUnWkyRlO2MZ5AuYioELhwXF/BgZEznrQ1drBEM=\ngithub.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=\ngithub.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=\ngithub.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=\ngithub.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=\ngithub.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=\ngithub.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=\ngithub.com/minio/simdjson-go v0.2.1 h1:nxYlp4Qd0w2pwLlif00l5vTFL6PcNAKpyHq27/pageg=\ngithub.com/minio/simdjson-go v0.2.1/go.mod h1:JPUSkRykfSPS+AhO0YPA1h0l5vY7NqrF4zel2b12wxc=\ngithub.com/minio/sio v0.2.1 h1:NjzKiIMSMcHediVQR0AFVx2tp7Wxh9tKPfDI3kH7aHQ=\ngithub.com/minio/sio v0.2.1/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw=\ngithub.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=\ngithub.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=\ngithub.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=\ngithub.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=\ngithub.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=\ngithub.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=\ngithub.com/mmcloughlin/avo v0.0.0-20201105074841-5d2f697d268f/go.mod h1:6aKT4zZIrpGqB3RpFU14ByCSSyKY6LfJz4J/JJChHfI=\ngithub.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=\ngithub.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=\ngithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=\ngithub.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU=\ngithub.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=\ngithub.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=\ngithub.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4=\ngithub.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk=\ngithub.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg=\ngithub.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=\ngithub.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=\ngithub.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=\ngithub.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=\ngithub.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=\ngithub.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=\ngithub.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=\ngithub.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=\ngithub.com/oliverisaac/shellescape v0.0.0-20220131224704-1b6c6b87b668 h1:WUilXdVrxYH+fFkmstviAOj1o9CfoW5O/Sd0LWPIVUA=\ngithub.com/oliverisaac/shellescape v0.0.0-20220131224704-1b6c6b87b668/go.mod h1:EDgl+cvbmeOQUMTTH94gjXVtFHr8xDe5BiXhWn7Hf1E=\ngithub.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=\ngithub.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=\ngithub.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=\ngithub.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=\ngithub.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=\ngithub.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=\ngithub.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=\ngithub.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=\ngithub.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=\ngithub.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=\ngithub.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=\ngithub.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=\ngithub.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=\ngithub.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=\ngithub.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=\ngithub.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=\ngithub.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=\ngithub.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=\ngithub.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=\ngithub.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4=\ngithub.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=\ngithub.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ=\ngithub.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=\ngithub.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=\ngithub.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=\ngithub.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 h1:lOtHtTItLlc9R+Vg/hU2klOOs+pjKLT2Cq+CEJgjvIQ=\ngithub.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106/go.mod h1:guCyM5N+o+ru0TsoZ1hi9lDjUMs2sIBjW3ARTEpVbnk=\ngithub.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=\ngithub.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=\ngithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=\ngithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=\ngithub.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=\ngithub.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=\ngithub.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=\ngithub.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=\ngithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=\ngithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=\ngithub.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=\ngithub.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=\ngithub.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=\ngithub.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20=\ngithub.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=\ngithub.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=\ngithub.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=\ngithub.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=\ngithub.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=\ngithub.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk=\ngithub.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=\ngithub.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=\ngithub.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=\ngithub.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=\ngithub.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=\ngithub.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=\ngithub.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=\ngithub.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=\ngithub.com/prometheus/prometheus v0.54.1 h1:vKuwQNjnYN2/mDoWfHXDhAsz/68q/dQDb+YbcEqU7MQ=\ngithub.com/prometheus/prometheus v0.54.1/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY=\ngithub.com/qingstor/qingstor-sdk-go/v4 v4.4.0 h1:tbItWtGB1TDfYzqK8dtm6tV+xWU5iYMwL37C6AL5dDs=\ngithub.com/qingstor/qingstor-sdk-go/v4 v4.4.0/go.mod h1:mDVFtA7+bXQ5xoELTWkoFy1Ad13wtp8jtlnl/RU+zzM=\ngithub.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=\ngithub.com/qiniu/go-sdk/v7 v7.25.2 h1:URwgZpxySdiwu2yQpHk93X4LXWHyFRp1x3Vmlk/YWvo=\ngithub.com/qiniu/go-sdk/v7 v7.25.2/go.mod h1:dmKtJ2ahhPWFVi9o1D5GemmWoh/ctuB9peqTowyTO8o=\ngithub.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=\ngithub.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=\ngithub.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=\ngithub.com/redis/go-redis/v9 v9.16.0 h1:OotgqgLSRCmzfqChbQyG1PHC3tLNR89DG4jdOERSEP4=\ngithub.com/redis/go-redis/v9 v9.16.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=\ngithub.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=\ngithub.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=\ngithub.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=\ngithub.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=\ngithub.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY=\ngithub.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc=\ngithub.com/rogpeppe/go-internal v1.0.1-alpha.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=\ngithub.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=\ngithub.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=\ngithub.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=\ngithub.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=\ngithub.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=\ngithub.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=\ngithub.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=\ngithub.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=\ngithub.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=\ngithub.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=\ngithub.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=\ngithub.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=\ngithub.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=\ngithub.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc=\ngithub.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs=\ngithub.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=\ngithub.com/shirou/gopsutil/v3 v3.23.11 h1:i3jP9NjCPUz7FiZKxlMnODZkdSIp2gnzfrvsu9CuWEQ=\ngithub.com/shirou/gopsutil/v3 v3.23.11/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=\ngithub.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=\ngithub.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=\ngithub.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=\ngithub.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=\ngithub.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=\ngithub.com/shurcooL/httpfs v0.0.0-20181222201310-74dc9339e414/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=\ngithub.com/shurcooL/vfsgen v0.0.0-20180915214035-33ae1944be3f/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=\ngithub.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=\ngithub.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=\ngithub.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A=\ngithub.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=\ngithub.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=\ngithub.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=\ngithub.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=\ngithub.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=\ngithub.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=\ngithub.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=\ngithub.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=\ngithub.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=\ngithub.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=\ngithub.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=\ngithub.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=\ngithub.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=\ngithub.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=\ngithub.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=\ngithub.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=\ngithub.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=\ngithub.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=\ngithub.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=\ngithub.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=\ngithub.com/studio-b12/gowebdav v0.10.0 h1:Yewz8FFiadcGEu4hxS/AAJQlHelndqln1bns3hcJIYc=\ngithub.com/studio-b12/gowebdav v0.10.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=\ngithub.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=\ngithub.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=\ngithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=\ngithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0=\ngithub.com/tencentyun/cos-go-sdk-v5 v0.7.63 h1:A+FH9HU8a2ozcd36VkrtiCooyzDPEOupIGWKgATtGlQ=\ngithub.com/tencentyun/cos-go-sdk-v5 v0.7.63/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0=\ngithub.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=\ngithub.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=\ngithub.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE=\ngithub.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI=\ngithub.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=\ngithub.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=\ngithub.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=\ngithub.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=\ngithub.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg=\ngithub.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=\ngithub.com/tikv/client-go/v2 v2.0.7 h1:nNTx/AR6n8Ew5VtHanFPG8NkFLLXbaNs5/K43DDma04=\ngithub.com/tikv/client-go/v2 v2.0.7/go.mod h1:9JNUWtHN8cx8eynHZ9xzdPi5YY6aiN1ILQyhfPUBcMo=\ngithub.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 h1:bzlSSzw+6qTwPs8pMcPI1bt27TAOhSdAEwdPCz6eBlg=\ngithub.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GRA2H/uSttqA3LvMfMSHVBJaXk3IgkFXFVxo=\ngithub.com/tinylib/msgp v1.1.3 h1:3giwAkmtaEDLSV0MdO1lDLuPgklgPzmk8H9+So2BVfA=\ngithub.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=\ngithub.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=\ngithub.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=\ngithub.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=\ngithub.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=\ngithub.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=\ngithub.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=\ngithub.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=\ngithub.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=\ngithub.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=\ngithub.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=\ngithub.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=\ngithub.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=\ngithub.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=\ngithub.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ=\ngithub.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE=\ngithub.com/vimeo/go-util v1.4.1 h1:UbNoaYH1eHv4LqBSH6zIItj+zKqbln0i01oY3iA/QPM=\ngithub.com/vimeo/go-util v1.4.1/go.mod h1:r+yspV//C48HeMXV8nEvtUeNiIiGfVv3bbEHzOgudwE=\ngithub.com/volcengine/ve-tos-golang-sdk/v2 v2.7.8 h1:/vB6jop4i70Ys8KAzK0xZfbMzMggJsTnIp6gZYnnSFM=\ngithub.com/volcengine/ve-tos-golang-sdk/v2 v2.7.8/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM=\ngithub.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=\ngithub.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=\ngithub.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=\ngithub.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=\ngithub.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0=\ngithub.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=\ngithub.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=\ngithub.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=\ngithub.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=\ngithub.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=\ngithub.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=\ngo.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0=\ngo.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=\ngo.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs=\ngo.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k=\ngo.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE=\ngo.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4=\ngo.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E=\ngo.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA=\ngo.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=\ngo.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=\ngo.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=\ngo.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=\ngo.opentelemetry.io/contrib/detectors/gcp v1.29.0 h1:TiaiXB4DpGD3sdzNlYQxruQngn5Apwzi1X0DRhuGvDQ=\ngo.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU=\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=\ngo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=\ngo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=\ngo.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=\ngo.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=\ngo.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc=\ngo.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=\ngo.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=\ngo.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=\ngo.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo=\ngo.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok=\ngo.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY=\ngo.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=\ngo.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=\ngo.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=\ngo.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=\ngo.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=\ngo.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=\ngo.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=\ngo.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=\ngo.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=\ngo.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=\ngo.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=\ngo.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngo.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=\ngo.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=\ngo.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=\ngo.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=\ngo.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=\ngo.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=\ngo.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=\ngo.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=\ngo.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=\ngolang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=\ngolang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4=\ngolang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=\ngolang.org/x/crypto v0.0.0-20180807104621-f027049dab0a/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=\ngolang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=\ngolang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=\ngolang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=\ngolang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=\ngolang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=\ngolang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=\ngolang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=\ngolang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=\ngolang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=\ngolang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=\ngolang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ=\ngolang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=\ngolang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=\ngolang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=\ngolang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=\ngolang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=\ngolang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=\ngolang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=\ngolang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=\ngolang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=\ngolang.org/x/sys v0.0.0-20180807162357-acbc56fc7007/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=\ngolang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=\ngolang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=\ngolang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=\ngolang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=\ngolang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=\ngolang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=\ngolang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=\ngolang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=\ngolang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=\ngolang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=\ngolang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=\ngolang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=\ngolang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=\ngolang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=\ngolang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=\ngolang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190308142131-b40df0fb21c3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=\ngolang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=\ngolang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=\ngolang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngoogle.golang.org/api v0.210.0 h1:HMNffZ57OoZCRYSbdWVRoqOa8V8NIHLL0CzdBPLztWk=\ngoogle.golang.org/api v0.210.0/go.mod h1:B9XDZGnx2NtyjzVkOVTGrFSAVZgPcbedzKg/gTLwqBs=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk=\ngoogle.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f h1:M65LEviCfuZTfrfzwwEoxVtgvfkFkBUbFnRbxCXuXhU=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f/go.mod h1:Yo94eF2nj7igQt+TiJ49KxjIH8ndLYPZMIRSiRcEbg0=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 h1:LWZqQOEjDyONlF1H6afSWpAL/znlREo2tHfLoe+8LMA=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=\ngoogle.golang.org/grpc v1.67.2 h1:Lq11HW1nr5m4OYV+ZVy2BjOK78/zqnTx24vyDBP1JcQ=\ngoogle.golang.org/grpc v1.67.2/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=\ngoogle.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a h1:UIpYSuWdWHSzjwcAFRLjKcPXFZVVLXGEM23W+NWqipw=\ngoogle.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a/go.mod h1:9i1T9n4ZinTUZGgzENMi8MDDgbGC5mqTS75JAv6xN3A=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU=\ngoogle.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=\ngopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=\ngopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=\ngopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=\ngopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=\ngopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 h1:2TSTkQ8PMvGOD5eeqqRVv6Z9+BYI+bowK97RCr3W+9M=\ngopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE=\ngopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=\ngopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=\ngopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=\ngopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nmodernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w=\nmodernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8=\npgregory.net/rapid v0.5.3 h1:163N50IHFqr1phZens4FQOdPgfJscR7a562mjQqeo4M=\npgregory.net/rapid v0.5.3/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=\nrsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=\nxorm.io/builder v0.3.7 h1:2pETdKRK+2QG4mLX4oODHEhn5Z8j1m8sXa7jfu+/SZI=\nxorm.io/builder v0.3.7/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=\n" }, { "path": "hack/autocomplete/bash_autocomplete", "content": "#! /bin/bash\n\n_cli_bash_autocomplete() {\n if [[ \"${COMP_WORDS[0]}\" != \"source\" ]]; then\n local cur opts base\n COMPREPLY=()\n cur=\"${COMP_WORDS[COMP_CWORD]}\"\n if [[ \"$cur\" == \"-\"* ]]; then\n opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} ${cur} --generate-bash-completion )\n else\n opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion )\n fi\n COMPREPLY=( $(compgen -W \"${opts}\" -- ${cur}) )\n return 0\n fi\n}\n\ncomplete -o bashdefault -o default -o nospace -F _cli_bash_autocomplete juicefs\n" }, { "path": "hack/autocomplete/zsh_autocomplete", "content": "#compdef juicefs\n\n_cli_zsh_autocomplete() {\n local -a opts\n local cur\n cur=${words[-1]}\n if [[ \"$cur\" == \"-\"* ]]; then\n opts=(\"${(@f)$(_CLI_ZSH_AUTOCOMPLETE_HACK=1 ${words[@]:0:#words[@]-1} ${cur} --generate-bash-completion)}\")\n else\n opts=(\"${(@f)$(_CLI_ZSH_AUTOCOMPLETE_HACK=1 ${words[@]:0:#words[@]-1} --generate-bash-completion)}\")\n fi\n\n if [[ \"${opts[1]}\" != \"\" ]]; then\n _describe 'values' opts\n else\n _files\n fi\n\n return\n}\n\ncompdef _cli_zsh_autocomplete juicefs\n" }, { "path": "hack/builder/Dockerfile", "content": "FROM ghcr.io/gythialy/golang-cross:v1.21.9-0\n\nRUN apt-get update && apt-get install -y musl-tools && apt-get -y autoremove && \\\n apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* /tmp/* /var/tmp/* && \\\n git config --global --add safe.directory /go/src/github.com/juicedata/juicefs && \\\n curl -fsSL -o /tmp/aarch64-linux-musl-cross.tgz https://musl.cc/aarch64-linux-musl-cross.tgz && \\\n tar -xf /tmp/aarch64-linux-musl-cross.tgz -C /usr/local/ && rm -f /tmp/aarch64-linux-musl-cross.tgz\n" }, { "path": "hack/builder/sdk.Dockerfile", "content": "FROM centos:7\n\nRUN yum install -y java-1.8.0-openjdk maven git gcc make \\\n && ln -s /go/bin/go /usr/local/bin/go \\\n && rm -rf /var/cache/yum\n" }, { "path": "hack/winfsp_headers/fuse.h", "content": "/**\n * @file fuse/fuse.h\n * WinFsp FUSE compatible API.\n *\n * This file is derived from libfuse/include/fuse.h:\n * FUSE: Filesystem in Userspace\n * Copyright 2001-2007 Miklos Szeredi \n *\n * @copyright 2015-2020 Bill Zissimopoulos\n */\n/*\n * This file is part of WinFsp.\n *\n * You can redistribute it and/or modify it under the terms of the GNU\n * General Public License version 3 as published by the Free Software\n * Foundation.\n *\n * Licensees holding a valid commercial license may use this software\n * in accordance with the commercial license agreement provided in\n * conjunction with the software. The terms and conditions of any such\n * commercial license agreement shall govern, supersede, and render\n * ineffective any application of the GPLv3 license to this software,\n * notwithstanding of any reference thereto in the software or\n * associated repository.\n */\n\n#ifndef FUSE_H_\n#define FUSE_H_\n\n#include \"fuse_common.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nstruct fuse;\n\ntypedef int (*fuse_fill_dir_t)(void *buf, const char *name,\n const struct fuse_stat *stbuf, fuse_off_t off);\ntypedef struct fuse_dirhandle *fuse_dirh_t;\ntypedef int (*fuse_dirfil_t)(fuse_dirh_t h, const char *name,\n int type, fuse_ino_t ino);\n\nstruct fuse_operations\n{\n /* S - supported by WinFsp */\n /* S */ int (*getattr)(const char *path, struct fuse_stat *stbuf);\n /* S */ int (*getdir)(const char *path, fuse_dirh_t h, fuse_dirfil_t filler);\n /* S */ int (*readlink)(const char *path, char *buf, size_t size);\n /* S */ int (*mknod)(const char *path, fuse_mode_t mode, fuse_dev_t dev);\n /* S */ int (*mkdir)(const char *path, fuse_mode_t mode);\n /* S */ int (*unlink)(const char *path);\n /* S */ int (*rmdir)(const char *path);\n /* S */ int (*symlink)(const char *dstpath, const char *srcpath);\n /* S */ int (*rename)(const char *oldpath, const char *newpath);\n /* _ */ int (*link)(const char *srcpath, const char *dstpath);\n /* S */ int (*chmod)(const char *path, fuse_mode_t mode);\n /* S */ int (*chown)(const char *path, fuse_uid_t uid, fuse_gid_t gid);\n /* S */ int (*truncate)(const char *path, fuse_off_t size);\n /* S */ int (*utime)(const char *path, struct fuse_utimbuf *timbuf);\n /* S */ int (*open)(const char *path, struct fuse_file_info *fi);\n /* S */ int (*read)(const char *path, char *buf, size_t size, fuse_off_t off,\n struct fuse_file_info *fi);\n /* S */ int (*write)(const char *path, const char *buf, size_t size, fuse_off_t off,\n struct fuse_file_info *fi);\n /* S */ int (*statfs)(const char *path, struct fuse_statvfs *stbuf);\n /* S */ int (*flush)(const char *path, struct fuse_file_info *fi);\n /* S */ int (*release)(const char *path, struct fuse_file_info *fi);\n /* S */ int (*fsync)(const char *path, int datasync, struct fuse_file_info *fi);\n /* S */ int (*setxattr)(const char *path, const char *name, const char *value, size_t size,\n int flags);\n /* S */ int (*getxattr)(const char *path, const char *name, char *value, size_t size);\n /* S */ int (*listxattr)(const char *path, char *namebuf, size_t size);\n /* S */ int (*removexattr)(const char *path, const char *name);\n /* S */ int (*opendir)(const char *path, struct fuse_file_info *fi);\n /* S */ int (*readdir)(const char *path, void *buf, fuse_fill_dir_t filler, fuse_off_t off,\n struct fuse_file_info *fi);\n /* S */ int (*releasedir)(const char *path, struct fuse_file_info *fi);\n /* S */ int (*fsyncdir)(const char *path, int datasync, struct fuse_file_info *fi);\n /* S */ void *(*init)(struct fuse_conn_info *conn);\n /* S */ void (*destroy)(void *data);\n /* _ */ int (*access)(const char *path, int mask);\n /* S */ int (*create)(const char *path, fuse_mode_t mode, struct fuse_file_info *fi);\n /* S */ int (*ftruncate)(const char *path, fuse_off_t off, struct fuse_file_info *fi);\n /* S */ int (*fgetattr)(const char *path, struct fuse_stat *stbuf, struct fuse_file_info *fi);\n /* _ */ int (*lock)(const char *path,\n struct fuse_file_info *fi, int cmd, struct fuse_flock *lock);\n /* S */ int (*utimens)(const char *path, const struct fuse_timespec tv[2]);\n /* _ */ int (*bmap)(const char *path, size_t blocksize, uint64_t *idx);\n /* _ */ unsigned int flag_nullpath_ok:1;\n /* _ */ unsigned int flag_nopath:1;\n /* _ */ unsigned int flag_utime_omit_ok:1;\n /* _ */ unsigned int flag_reserved:29;\n /* S */ int (*ioctl)(const char *path, int cmd, void *arg, struct fuse_file_info *fi,\n unsigned int flags, void *data);\n /* _ */ int (*poll)(const char *path, struct fuse_file_info *fi,\n struct fuse_pollhandle *ph, unsigned *reventsp);\n /* FUSE 2.9 */\n /* _ */ int (*write_buf)(const char *path,\n struct fuse_bufvec *buf, fuse_off_t off, struct fuse_file_info *fi);\n /* _ */ int (*read_buf)(const char *path,\n struct fuse_bufvec **bufp, size_t size, fuse_off_t off, struct fuse_file_info *fi);\n /* _ */ int (*flock)(const char *path, struct fuse_file_info *, int op);\n /* _ */ int (*fallocate)(const char *path, int mode, fuse_off_t off, fuse_off_t len,\n struct fuse_file_info *fi);\n /* OSXFUSE */\n /* _ */ int (*reserved00)();\n /* _ */ int (*reserved01)();\n /* _ */ int (*reserved02)();\n /* _ */ int (*statfs_x)(const char *path, struct fuse_statfs *stbuf);\n /* _ */ int (*setvolname)(const char *volname);\n /* _ */ int (*exchange)(const char *oldpath, const char *newpath, unsigned long flags);\n /* _ */ int (*getxtimes)(const char *path,\n struct fuse_timespec *bkuptime, struct fuse_timespec *crtime);\n /* _ */ int (*setbkuptime)(const char *path, const struct fuse_timespec *tv);\n /* S */ int (*setchgtime)(const char *path, const struct fuse_timespec *tv);\n /* S */ int (*setcrtime)(const char *path, const struct fuse_timespec *tv);\n /* S */ int (*chflags)(const char *path, uint32_t flags);\n /* _ */ int (*setattr_x)(const char *path, struct fuse_setattr_x *attr);\n /* _ */ int (*fsetattr_x)(const char *path, struct fuse_setattr_x *attr,\n struct fuse_file_info *fi);\n};\n\nstruct fuse_context\n{\n struct fuse *fuse;\n fuse_uid_t uid;\n fuse_gid_t gid;\n fuse_pid_t pid;\n void *private_data;\n fuse_mode_t umask;\n};\n\n#define fuse_main(argc, argv, ops, data)\\\n fuse_main_real(argc, argv, ops, sizeof *(ops), data)\n\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_main_real)(struct fsp_fuse_env *env,\n int argc, char *argv[],\n const struct fuse_operations *ops, size_t opsize, void *data);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_is_lib_option)(struct fsp_fuse_env *env,\n const char *opt);\nFSP_FUSE_API struct fuse *FSP_FUSE_API_NAME(fsp_fuse_new)(struct fsp_fuse_env *env,\n struct fuse_chan *ch, struct fuse_args *args,\n const struct fuse_operations *ops, size_t opsize, void *data);\nFSP_FUSE_API void FSP_FUSE_API_NAME(fsp_fuse_destroy)(struct fsp_fuse_env *env,\n struct fuse *f);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_loop)(struct fsp_fuse_env *env,\n struct fuse *f);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_loop_mt)(struct fsp_fuse_env *env,\n struct fuse *f);\nFSP_FUSE_API void FSP_FUSE_API_NAME(fsp_fuse_exit)(struct fsp_fuse_env *env,\n struct fuse *f);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_exited)(struct fsp_fuse_env *env,\n struct fuse *f);\nFSP_FUSE_API struct fuse_context *FSP_FUSE_API_NAME(fsp_fuse_get_context)(struct fsp_fuse_env *env);\n\nFSP_FUSE_SYM(\nint fuse_main_real(int argc, char *argv[],\n const struct fuse_operations *ops, size_t opsize, void *data),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_main_real)\n (fsp_fuse_env(), argc, argv, ops, opsize, data);\n})\n\nFSP_FUSE_SYM(\nint fuse_is_lib_option(const char *opt),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_is_lib_option)\n (fsp_fuse_env(), opt);\n})\n\nFSP_FUSE_SYM(\nstruct fuse *fuse_new(struct fuse_chan *ch, struct fuse_args *args,\n const struct fuse_operations *ops, size_t opsize, void *data),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_new)\n (fsp_fuse_env(), ch, args, ops, opsize, data);\n})\n\nFSP_FUSE_SYM(\nvoid fuse_destroy(struct fuse *f),\n{\n FSP_FUSE_API_CALL(fsp_fuse_destroy)\n (fsp_fuse_env(), f);\n})\n\nFSP_FUSE_SYM(\nint fuse_loop(struct fuse *f),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_loop)\n (fsp_fuse_env(), f);\n})\n\nFSP_FUSE_SYM(\nint fuse_loop_mt(struct fuse *f),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_loop_mt)\n (fsp_fuse_env(), f);\n})\n\nFSP_FUSE_SYM(\nvoid fuse_exit(struct fuse *f),\n{\n FSP_FUSE_API_CALL(fsp_fuse_exit)\n (fsp_fuse_env(), f);\n})\n\nFSP_FUSE_SYM(\nint fuse_exited(struct fuse *f),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_exited)\n (fsp_fuse_env(), f);\n})\n\nFSP_FUSE_SYM(\nstruct fuse_context *fuse_get_context(void),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_get_context)\n (fsp_fuse_env());\n})\n\nFSP_FUSE_SYM(\nint fuse_getgroups(int size, fuse_gid_t list[]),\n{\n (void)size;\n (void)list;\n return -ENOSYS;\n})\n\nFSP_FUSE_SYM(\nint fuse_interrupted(void),\n{\n return 0;\n})\n\nFSP_FUSE_SYM(\nint fuse_invalidate(struct fuse *f, const char *path),\n{\n (void)f;\n (void)path;\n return -EINVAL;\n})\n\nFSP_FUSE_SYM(\nint fuse_notify_poll(struct fuse_pollhandle *ph),\n{\n (void)ph;\n return 0;\n})\n\nFSP_FUSE_SYM(\nstruct fuse_session *fuse_get_session(struct fuse *f),\n{\n return (struct fuse_session *)f;\n})\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n" }, { "path": "hack/winfsp_headers/fuse_common.h", "content": "/**\n * @file fuse/fuse_common.h\n * WinFsp FUSE compatible API.\n *\n * This file is derived from libfuse/include/fuse_common.h:\n * FUSE: Filesystem in Userspace\n * Copyright 2001-2007 Miklos Szeredi \n *\n * @copyright 2015-2020 Bill Zissimopoulos\n */\n/*\n * This file is part of WinFsp.\n *\n * You can redistribute it and/or modify it under the terms of the GNU\n * General Public License version 3 as published by the Free Software\n * Foundation.\n *\n * Licensees holding a valid commercial license may use this software\n * in accordance with the commercial license agreement provided in\n * conjunction with the software. The terms and conditions of any such\n * commercial license agreement shall govern, supersede, and render\n * ineffective any application of the GPLv3 license to this software,\n * notwithstanding of any reference thereto in the software or\n * associated repository.\n */\n\n#ifndef FUSE_COMMON_H_\n#define FUSE_COMMON_H_\n\n#include \"winfsp_fuse.h\"\n#include \"fuse_opt.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define FUSE_MAJOR_VERSION 2\n#define FUSE_MINOR_VERSION 8\n#define FUSE_MAKE_VERSION(maj, min) ((maj) * 10 + (min))\n#define FUSE_VERSION FUSE_MAKE_VERSION(FUSE_MAJOR_VERSION, FUSE_MINOR_VERSION)\n\n#define FUSE_CAP_ASYNC_READ (1 << 0)\n#define FUSE_CAP_POSIX_LOCKS (1 << 1)\n#define FUSE_CAP_ATOMIC_O_TRUNC (1 << 3)\n#define FUSE_CAP_EXPORT_SUPPORT (1 << 4)\n#define FUSE_CAP_BIG_WRITES (1 << 5)\n#define FUSE_CAP_DONT_MASK (1 << 6)\n#define FUSE_CAP_ALLOCATE (1 << 27) /* reserved (OSXFUSE) */\n#define FUSE_CAP_EXCHANGE_DATA (1 << 28) /* reserved (OSXFUSE) */\n#define FUSE_CAP_CASE_INSENSITIVE (1 << 29) /* file system is case insensitive */\n#define FUSE_CAP_VOL_RENAME (1 << 30) /* reserved (OSXFUSE) */\n#define FUSE_CAP_XTIMES (1 << 31) /* reserved (OSXFUSE) */\n\n#define FSP_FUSE_CAP_READDIR_PLUS (1 << 21) /* file system supports enhanced readdir */\n#define FSP_FUSE_CAP_READ_ONLY (1 << 22) /* file system is marked read-only */\n#define FSP_FUSE_CAP_STAT_EX (1 << 23) /* file system supports fuse_stat_ex */\n#define FSP_FUSE_CAP_CASE_INSENSITIVE FUSE_CAP_CASE_INSENSITIVE\n\n#define FUSE_IOCTL_COMPAT (1 << 0)\n#define FUSE_IOCTL_UNRESTRICTED (1 << 1)\n#define FUSE_IOCTL_RETRY (1 << 2)\n#define FUSE_IOCTL_MAX_IOV 256\n\n/* from FreeBSD */\n#define FSP_FUSE_UF_HIDDEN 0x00008000\n#define FSP_FUSE_UF_READONLY 0x00001000\n#define FSP_FUSE_UF_SYSTEM 0x00000080\n#define FSP_FUSE_UF_ARCHIVE 0x00000800\n#if !defined(UF_HIDDEN)\n#define UF_HIDDEN FSP_FUSE_UF_HIDDEN\n#endif\n#if !defined(UF_READONLY)\n#define UF_READONLY FSP_FUSE_UF_READONLY\n#endif\n#if !defined(UF_SYSTEM)\n#define UF_SYSTEM FSP_FUSE_UF_SYSTEM\n#endif\n#if !defined(UF_ARCHIVE)\n#define UF_ARCHIVE FSP_FUSE_UF_ARCHIVE\n#endif\n\nstruct fuse_file_info\n{\n int flags;\n unsigned int fh_old;\n int writepage;\n unsigned int direct_io:1;\n unsigned int keep_cache:1;\n unsigned int flush:1;\n unsigned int nonseekable:1;\n unsigned int padding:28;\n uint64_t fh;\n uint64_t lock_owner;\n};\n\nstruct fuse_conn_info\n{\n unsigned proto_major;\n unsigned proto_minor;\n unsigned async_read;\n unsigned max_write;\n unsigned max_readahead;\n unsigned capable;\n unsigned want;\n unsigned reserved[25];\n};\n\nstruct fuse_session;\nstruct fuse_chan;\nstruct fuse_pollhandle;\nstruct fuse_bufvec;\nstruct fuse_statfs;\nstruct fuse_setattr_x;\n\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_version)(struct fsp_fuse_env *env);\nFSP_FUSE_API struct fuse_chan *FSP_FUSE_API_NAME(fsp_fuse_mount)(struct fsp_fuse_env *env,\n const char *mountpoint, struct fuse_args *args);\nFSP_FUSE_API void FSP_FUSE_API_NAME(fsp_fuse_unmount)(struct fsp_fuse_env *env,\n const char *mountpoint, struct fuse_chan *ch);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_parse_cmdline)(struct fsp_fuse_env *env,\n struct fuse_args *args,\n char **mountpoint, int *multithreaded, int *foreground);\nFSP_FUSE_API int32_t FSP_FUSE_API_NAME(fsp_fuse_ntstatus_from_errno)(struct fsp_fuse_env *env,\n int err);\n\nFSP_FUSE_SYM(\nint fuse_version(void),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_version)\n (fsp_fuse_env());\n})\n\nFSP_FUSE_SYM(\nstruct fuse_chan *fuse_mount(const char *mountpoint, struct fuse_args *args),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_mount)\n (fsp_fuse_env(), mountpoint, args);\n})\n\nFSP_FUSE_SYM(\nvoid fuse_unmount(const char *mountpoint, struct fuse_chan *ch),\n{\n FSP_FUSE_API_CALL(fsp_fuse_unmount)\n (fsp_fuse_env(), mountpoint, ch);\n})\n\nFSP_FUSE_SYM(\nint fuse_parse_cmdline(struct fuse_args *args,\n char **mountpoint, int *multithreaded, int *foreground),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_parse_cmdline)\n (fsp_fuse_env(), args, mountpoint, multithreaded, foreground);\n})\n\nFSP_FUSE_SYM(\nvoid fuse_pollhandle_destroy(struct fuse_pollhandle *ph),\n{\n (void)ph;\n})\n\nFSP_FUSE_SYM(\nint fuse_daemonize(int foreground),\n{\n return fsp_fuse_daemonize(foreground);\n})\n\nFSP_FUSE_SYM(\nint fuse_set_signal_handlers(struct fuse_session *se),\n{\n return fsp_fuse_set_signal_handlers(se);\n})\n\nFSP_FUSE_SYM(\nvoid fuse_remove_signal_handlers(struct fuse_session *se),\n{\n (void)se;\n fsp_fuse_set_signal_handlers(0);\n})\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n" }, { "path": "hack/winfsp_headers/fuse_opt.h", "content": "/**\n * @file fuse/fuse_opt.h\n * WinFsp FUSE compatible API.\n *\n * This file is derived from libfuse/include/fuse_opt.h:\n * FUSE: Filesystem in Userspace\n * Copyright 2001-2007 Miklos Szeredi \n *\n * @copyright 2015-2020 Bill Zissimopoulos\n */\n/*\n * This file is part of WinFsp.\n *\n * You can redistribute it and/or modify it under the terms of the GNU\n * General Public License version 3 as published by the Free Software\n * Foundation.\n *\n * Licensees holding a valid commercial license may use this software\n * in accordance with the commercial license agreement provided in\n * conjunction with the software. The terms and conditions of any such\n * commercial license agreement shall govern, supersede, and render\n * ineffective any application of the GPLv3 license to this software,\n * notwithstanding of any reference thereto in the software or\n * associated repository.\n */\n\n#ifndef FUSE_OPT_H_\n#define FUSE_OPT_H_\n\n#include \"winfsp_fuse.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define FUSE_OPT_KEY(templ, key) { templ, -1, key }\n#define FUSE_OPT_END { NULL, 0, 0 }\n\n#define FUSE_OPT_KEY_OPT -1\n#define FUSE_OPT_KEY_NONOPT -2\n#define FUSE_OPT_KEY_KEEP -3\n#define FUSE_OPT_KEY_DISCARD -4\n\n#define FUSE_ARGS_INIT(argc, argv) { argc, argv, 0 }\n\nstruct fuse_opt\n{\n const char *templ;\n unsigned int offset;\n int value;\n};\n\nstruct fuse_args\n{\n int argc;\n char **argv;\n int allocated;\n};\n\ntypedef int (*fuse_opt_proc_t)(void *data, const char *arg, int key,\n struct fuse_args *outargs);\n\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_opt_parse)(struct fsp_fuse_env *env,\n struct fuse_args *args, void *data,\n const struct fuse_opt opts[], fuse_opt_proc_t proc);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_opt_add_arg)(struct fsp_fuse_env *env,\n struct fuse_args *args, const char *arg);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_opt_insert_arg)(struct fsp_fuse_env *env,\n struct fuse_args *args, int pos, const char *arg);\nFSP_FUSE_API void FSP_FUSE_API_NAME(fsp_fuse_opt_free_args)(struct fsp_fuse_env *env,\n struct fuse_args *args);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_opt_add_opt)(struct fsp_fuse_env *env,\n char **opts, const char *opt);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_opt_add_opt_escaped)(struct fsp_fuse_env *env,\n char **opts, const char *opt);\nFSP_FUSE_API int FSP_FUSE_API_NAME(fsp_fuse_opt_match)(struct fsp_fuse_env *env,\n const struct fuse_opt opts[], const char *opt);\n\nFSP_FUSE_SYM(\nint fuse_opt_parse(struct fuse_args *args, void *data,\n const struct fuse_opt opts[], fuse_opt_proc_t proc),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_opt_parse)\n (fsp_fuse_env(), args, data, opts, proc);\n})\n\nFSP_FUSE_SYM(\nint fuse_opt_add_arg(struct fuse_args *args, const char *arg),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_opt_add_arg)\n (fsp_fuse_env(), args, arg);\n})\n\nFSP_FUSE_SYM(\nint fuse_opt_insert_arg(struct fuse_args *args, int pos, const char *arg),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_opt_insert_arg)\n (fsp_fuse_env(), args, pos, arg);\n})\n\nFSP_FUSE_SYM(\nvoid fuse_opt_free_args(struct fuse_args *args),\n{\n FSP_FUSE_API_CALL(fsp_fuse_opt_free_args)\n (fsp_fuse_env(), args);\n})\n\nFSP_FUSE_SYM(\nint fuse_opt_add_opt(char **opts, const char *opt),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_opt_add_opt)\n (fsp_fuse_env(), opts, opt);\n})\n\nFSP_FUSE_SYM(\nint fuse_opt_add_opt_escaped(char **opts, const char *opt),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_opt_add_opt_escaped)\n (fsp_fuse_env(), opts, opt);\n})\n\nFSP_FUSE_SYM(\nint fuse_opt_match(const struct fuse_opt opts[], const char *opt),\n{\n return FSP_FUSE_API_CALL(fsp_fuse_opt_match)\n (fsp_fuse_env(), opts, opt);\n})\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n" }, { "path": "hack/winfsp_headers/winfsp_fuse.h", "content": "/**\n * @file fuse/winfsp_fuse.h\n * WinFsp FUSE compatible API.\n *\n * @copyright 2015-2020 Bill Zissimopoulos\n */\n/*\n * This file is part of WinFsp.\n *\n * You can redistribute it and/or modify it under the terms of the GNU\n * General Public License version 3 as published by the Free Software\n * Foundation.\n *\n * Licensees holding a valid commercial license may use this software\n * in accordance with the commercial license agreement provided in\n * conjunction with the software. The terms and conditions of any such\n * commercial license agreement shall govern, supersede, and render\n * ineffective any application of the GPLv3 license to this software,\n * notwithstanding of any reference thereto in the software or\n * associated repository.\n */\n\n#ifndef FUSE_WINFSP_FUSE_H_INCLUDED\n#define FUSE_WINFSP_FUSE_H_INCLUDED\n\n#include \n#include \n#if !defined(WINFSP_DLL_INTERNAL)\n#include \n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#if !defined(FSP_FUSE_API)\n#if defined(WINFSP_DLL_INTERNAL)\n#define FSP_FUSE_API __declspec(dllexport)\n#else\n#define FSP_FUSE_API __declspec(dllimport)\n#endif\n#endif\n\n#if !defined(FSP_FUSE_API_NAME)\n#define FSP_FUSE_API_NAME(n) (n)\n#endif\n\n#if !defined(FSP_FUSE_API_CALL)\n#define FSP_FUSE_API_CALL(n) (n)\n#endif\n\n#if !defined(FSP_FUSE_SYM)\n#if !defined(CYGFUSE)\n#define FSP_FUSE_SYM(proto, ...) static inline proto { __VA_ARGS__ }\n#else\n#define FSP_FUSE_SYM(proto, ...) proto;\n#endif\n#endif\n\n#define FSP_FUSE_DEVICE_TYPE (0x8000 | 'W' | 'F' * 0x100) /* DeviceIoControl -> ioctl */\n#define FSP_FUSE_CTLCODE_FROM_IOCTL(cmd)\\\n (FSP_FUSE_DEVICE_TYPE << 16) | (((cmd) & 0x0fff) << 2)\n#define FSP_FUSE_IOCTL(cmd, isiz, osiz) \\\n ( \\\n (((osiz) != 0) << 31) | \\\n (((isiz) != 0) << 30) | \\\n (((isiz) | (osiz)) << 16) | \\\n (cmd) \\\n )\n\n/*\n * FUSE uses a number of types (notably: struct stat) that are OS specific.\n * Furthermore there are sometimes multiple definitions of the same type even\n * within the same OS. This is certainly true on Windows, where these types\n * are not even native.\n *\n * For this reason we will define our own fuse_* types which represent the\n * types as the WinFsp DLL expects to see them. We will define these types\n * to be compatible with the equivalent Cygwin types as we want WinFsp-FUSE\n * to be usable from Cygwin.\n */\n\n#define FSP_FUSE_STAT_FIELD_DEFN \\\n fuse_dev_t st_dev; \\\n fuse_ino_t st_ino; \\\n fuse_mode_t st_mode; \\\n fuse_nlink_t st_nlink; \\\n fuse_uid_t st_uid; \\\n fuse_gid_t st_gid; \\\n fuse_dev_t st_rdev; \\\n fuse_off_t st_size; \\\n struct fuse_timespec st_atim; \\\n struct fuse_timespec st_mtim; \\\n struct fuse_timespec st_ctim; \\\n fuse_blksize_t st_blksize; \\\n fuse_blkcnt_t st_blocks; \\\n struct fuse_timespec st_birthtim;\n#define FSP_FUSE_STAT_EX_FIELD_DEFN \\\n FSP_FUSE_STAT_FIELD_DEFN \\\n uint32_t st_flags; \\\n uint32_t st_reserved32[3]; \\\n uint64_t st_reserved64[2];\n\n#if defined(_WIN64) || defined(_WIN32)\n\ntypedef uint32_t fuse_uid_t;\ntypedef uint32_t fuse_gid_t;\ntypedef int32_t fuse_pid_t;\n\ntypedef uint32_t fuse_dev_t;\ntypedef uint64_t fuse_ino_t;\ntypedef uint32_t fuse_mode_t;\ntypedef uint16_t fuse_nlink_t;\ntypedef int64_t fuse_off_t;\n\n#if defined(_WIN64)\ntypedef uint64_t fuse_fsblkcnt_t;\ntypedef uint64_t fuse_fsfilcnt_t;\n#else\ntypedef uint32_t fuse_fsblkcnt_t;\ntypedef uint32_t fuse_fsfilcnt_t;\n#endif\ntypedef int32_t fuse_blksize_t;\ntypedef int64_t fuse_blkcnt_t;\n\n#if defined(_WIN64)\nstruct fuse_utimbuf\n{\n int64_t actime;\n int64_t modtime;\n};\nstruct fuse_timespec\n{\n int64_t tv_sec;\n int64_t tv_nsec;\n};\n#else\nstruct fuse_utimbuf\n{\n int32_t actime;\n int32_t modtime;\n};\nstruct fuse_timespec\n{\n int32_t tv_sec;\n int32_t tv_nsec;\n};\n#endif\n\n#if !defined(FSP_FUSE_USE_STAT_EX)\nstruct fuse_stat\n{\n FSP_FUSE_STAT_FIELD_DEFN\n};\n#else\nstruct fuse_stat\n{\n FSP_FUSE_STAT_EX_FIELD_DEFN\n};\n#endif\n\n#if defined(_WIN64)\nstruct fuse_statvfs\n{\n uint64_t f_bsize;\n uint64_t f_frsize;\n fuse_fsblkcnt_t f_blocks;\n fuse_fsblkcnt_t f_bfree;\n fuse_fsblkcnt_t f_bavail;\n fuse_fsfilcnt_t f_files;\n fuse_fsfilcnt_t f_ffree;\n fuse_fsfilcnt_t f_favail;\n uint64_t f_fsid;\n uint64_t f_flag;\n uint64_t f_namemax;\n};\n#else\nstruct fuse_statvfs\n{\n uint32_t f_bsize;\n uint32_t f_frsize;\n fuse_fsblkcnt_t f_blocks;\n fuse_fsblkcnt_t f_bfree;\n fuse_fsblkcnt_t f_bavail;\n fuse_fsfilcnt_t f_files;\n fuse_fsfilcnt_t f_ffree;\n fuse_fsfilcnt_t f_favail;\n uint32_t f_fsid;\n uint32_t f_flag;\n uint32_t f_namemax;\n};\n#endif\n\nstruct fuse_flock\n{\n int16_t l_type;\n int16_t l_whence;\n fuse_off_t l_start;\n fuse_off_t l_len;\n fuse_pid_t l_pid;\n};\n\n#if defined(WINFSP_DLL_INTERNAL)\n#define FSP_FUSE_ENV_INIT \\\n { \\\n 'W', \\\n MemAlloc, MemFree, \\\n fsp_fuse_daemonize, \\\n fsp_fuse_set_signal_handlers, \\\n 0/*conv_to_win_path*/, \\\n 0/*winpid_to_pid*/, \\\n { 0 }, \\\n }\n#else\n#define FSP_FUSE_ENV_INIT \\\n { \\\n 'W', \\\n malloc, free, \\\n fsp_fuse_daemonize, \\\n fsp_fuse_set_signal_handlers, \\\n 0/*conv_to_win_path*/, \\\n 0/*winpid_to_pid*/, \\\n { 0 }, \\\n }\n#endif\n\n#elif defined(__CYGWIN__)\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#define fuse_uid_t uid_t\n#define fuse_gid_t gid_t\n#define fuse_pid_t pid_t\n\n#define fuse_dev_t dev_t\n#define fuse_ino_t ino_t\n#define fuse_mode_t mode_t\n#define fuse_nlink_t nlink_t\n#define fuse_off_t off_t\n\n#define fuse_fsblkcnt_t fsblkcnt_t\n#define fuse_fsfilcnt_t fsfilcnt_t\n#define fuse_blksize_t blksize_t\n#define fuse_blkcnt_t blkcnt_t\n\n#define fuse_utimbuf utimbuf\n#define fuse_timespec timespec\n\n#if !defined(FSP_FUSE_USE_STAT_EX)\n#define fuse_stat stat\n#else\nstruct fuse_stat\n{\n FSP_FUSE_STAT_EX_FIELD_DEFN\n};\n#endif\n#define fuse_statvfs statvfs\n#define fuse_flock flock\n\n#define FSP_FUSE_ENV_INIT \\\n { \\\n 'C', \\\n malloc, free, \\\n fsp_fuse_daemonize, \\\n fsp_fuse_set_signal_handlers, \\\n fsp_fuse_conv_to_win_path, \\\n fsp_fuse_winpid_to_pid, \\\n { 0 }, \\\n }\n\n/*\n * Note that long is 8 bytes long in Cygwin64 and 4 bytes long in Win64.\n * For this reason we avoid using long anywhere in these headers.\n */\n\n#else\n#error unsupported environment\n#endif\n\nstruct fuse_stat_ex\n{\n FSP_FUSE_STAT_EX_FIELD_DEFN\n};\n\nstruct fsp_fuse_env\n{\n unsigned environment;\n void *(*memalloc)(size_t);\n void (*memfree)(void *);\n int (*daemonize)(int);\n int (*set_signal_handlers)(void *);\n char *(*conv_to_win_path)(const char *);\n fuse_pid_t (*winpid_to_pid)(uint32_t);\n void (*reserved[2])();\n};\n\nFSP_FUSE_API void FSP_FUSE_API_NAME(fsp_fuse_signal_handler)(int sig);\n\n#if defined(_WIN64) || defined(_WIN32)\n\nstatic inline int fsp_fuse_daemonize(int foreground)\n{\n (void)foreground;\n return 0;\n}\n\nstatic inline int fsp_fuse_set_signal_handlers(void *se)\n{\n (void)se;\n return 0;\n}\n\n#elif defined(__CYGWIN__)\n\nstatic inline int fsp_fuse_daemonize(int foreground)\n{\n int daemon(int nochdir, int noclose);\n int chdir(const char *path);\n\n if (!foreground)\n {\n if (-1 == daemon(0, 0))\n return -1;\n }\n else\n chdir(\"/\");\n\n return 0;\n}\n\nstatic inline void *fsp_fuse_signal_thread(void *psigmask)\n{\n int sig;\n\n if (0 == sigwait((sigset_t *)psigmask, &sig))\n FSP_FUSE_API_CALL(fsp_fuse_signal_handler)(sig);\n\n return 0;\n}\n\nstatic inline int fsp_fuse_set_signal_handlers(void *se)\n{\n#define FSP_FUSE_SET_SIGNAL_HANDLER(sig, newha)\\\n if (-1 != sigaction((sig), 0, &oldsa) &&\\\n oldsa.sa_handler == (se ? SIG_DFL : (newha)))\\\n {\\\n newsa.sa_handler = se ? (newha) : SIG_DFL;\\\n sigaction((sig), &newsa, 0);\\\n }\n#define FSP_FUSE_SIGADDSET(sig)\\\n if (-1 != sigaction((sig), 0, &oldsa) &&\\\n oldsa.sa_handler == SIG_DFL)\\\n sigaddset(&sigmask, (sig));\n\n static sigset_t sigmask;\n static pthread_t sigthr;\n struct sigaction oldsa, newsa;\n\n // memset instead of initializer to avoid GCC -Wmissing-field-initializers warning\n memset(&newsa, 0, sizeof newsa);\n\n if (0 != se)\n {\n if (0 == sigthr)\n {\n FSP_FUSE_SET_SIGNAL_HANDLER(SIGPIPE, SIG_IGN);\n\n sigemptyset(&sigmask);\n FSP_FUSE_SIGADDSET(SIGHUP);\n FSP_FUSE_SIGADDSET(SIGINT);\n FSP_FUSE_SIGADDSET(SIGTERM);\n if (0 != pthread_sigmask(SIG_BLOCK, &sigmask, 0))\n return -1;\n\n if (0 != pthread_create(&sigthr, 0, fsp_fuse_signal_thread, &sigmask))\n return -1;\n }\n }\n else\n {\n if (0 != sigthr)\n {\n pthread_cancel(sigthr);\n pthread_join(sigthr, 0);\n sigthr = 0;\n\n if (0 != pthread_sigmask(SIG_UNBLOCK, &sigmask, 0))\n return -1;\n sigemptyset(&sigmask);\n\n FSP_FUSE_SET_SIGNAL_HANDLER(SIGPIPE, SIG_IGN);\n }\n }\n\n return 0;\n\n#undef FSP_FUSE_SIGADDSET\n#undef FSP_FUSE_SET_SIGNAL_HANDLER\n}\n\nstatic inline char *fsp_fuse_conv_to_win_path(const char *path)\n{\n void *cygwin_create_path(unsigned, const void *);\n return (char *)cygwin_create_path(\n 0/*CCP_POSIX_TO_WIN_A*/ | 0x100/*CCP_RELATIVE*/,\n path);\n}\n\nstatic inline fuse_pid_t fsp_fuse_winpid_to_pid(uint32_t winpid)\n{\n pid_t cygwin_winpid_to_pid(int winpid);\n pid_t pid = cygwin_winpid_to_pid(winpid);\n return -1 != pid ? pid : (fuse_pid_t)winpid;\n}\n#endif\n\n\nstatic inline struct fsp_fuse_env *fsp_fuse_env(void)\n{\n static struct fsp_fuse_env env = FSP_FUSE_ENV_INIT;\n return &env;\n}\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n" }, { "path": "integration/Makefile", "content": "\nall: s3test webdav ioctl\n\ns3test:\n\tpip install awscli==1.27.153\n\tbash s3gateway_test.sh\n\nwebdav:\n\tcd /home/travis/.m2/litmus-0.13 ; for i in \"basic\" \"copymove\" \"http\"; do sudo ./$${i} http://127.0.0.1:9009 root 1234; done\n\nioctl:\n\tbash ioctl_test.sh /tmp/jfs-unit-test/ioctl_test 2>/dev/null\n" }, { "path": "integration/ioctl_test.sh", "content": "#!/bin/bash\n\n# JuiceFS, Copyright 2021 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\ntest_dir=$1\nif [ ! -d \"$test_dir\" ]; then\n mkdir \"$test_dir\"\nfi\n\nfunction cleanup() {\n code=$?\n if [ $code -eq 0 ]; then\n echo \"ioctl test passed\"\n else\n echo \"ioctl test failed\"\n fi\n trap - EXIT\n sudo chattr -R \"=\" \"$test_dir\"\n rm -rf \"$test_dir\"\n exit $code\n}\n\nfunction exec_should_failed() {\n eval \"$1\"\n if [ $? -eq 0 ]; then\n echo \"$1 should fail\"\n exit 1\n fi\n}\n\nfunction exec_should_success() {\n eval \"$1\"\n if [ $? -ne 0 ]; then\n echo \"$1 should success\"\n exit 1\n fi\n}\n\na_test_dir=\"$test_dir\"/a\nsudo chattr -R \"=\" \"${test_dir:?}\"\nsudo rm -rf \"${test_dir:?}\"/*\nmkdir \"$a_test_dir\"\n\ntrap cleanup INT EXIT\n\n{\n touch \"$a_test_dir\"/afile\n exec_should_failed 'sudo chattr \"+u\" $a_test_dir/afile'\n exec_should_success 'sudo chattr \"+a\" $a_test_dir/afile'\n exec_should_success '[[ \"$(lsattr $a_test_dir/afile | awk -F \" \" \"{print \\$1}\")\" =~ \"a\" ]]'\n exec_should_failed \"echo aa > $a_test_dir/afile\"\n exec_should_failed \"rm -rf $a_test_dir/afile\"\n touch \"$a_test_dir/tmpfile\"\n exec_should_failed \"mv -f $a_test_dir/tmpfile $a_test_dir/afile\"\n exec_should_failed \"mv -f $a_test_dir/afile $a_test_dir/tmpfile\"\n exec_should_failed \"ln $a_test_dir/afile $a_test_dir/linkfile\"\n echo \"12345\" >> \"$a_test_dir\"/afile\n exec_should_success '[ \"$(cat \"$a_test_dir\"/afile)\" == \"12345\" ]'\n\n # FIXME: sudo chattr \"+a\" $a_test_dir/fallocatefile random failed\n touch \"$a_test_dir\"/fallocatefile\n exec_should_success 'sudo chattr \"+a\" $a_test_dir/fallocatefile'\n exec_should_success '[[ \"$(lsattr $a_test_dir/fallocatefile | awk -F \" \" \"{print \\$1}\")\" =~ \"a\" ]]'\n exec_should_failed 'fallocate -l 1k -n $a_test_dir/fallocatefile'\n}\n\n\n{\n mkdir -p \"$a_test_dir\"/adir/child_dir1/child_dir2\n touch \"$a_test_dir\"/adir/file\n exec_should_success 'sudo chattr \"+a\" $a_test_dir/adir'\n exec_should_success '[[ \"$(lsattr -d $a_test_dir/adir | awk -F \" \" \"{print \\$1}\")\" =~ \"a\" ]]'\n exec_should_failed 'rm -rf $a_test_dir/adir'\n exec_should_failed 'rm -rf $a_test_dir/adir/file'\n exec_should_success 'touch \"$a_test_dir\"/adir/child_dir1/child_file'\n exec_should_success 'rm -rf $a_test_dir/adir/child_dir1/child_dir2'\n exec_should_success 'rm -rf $a_test_dir/adir/child_dir1/child_file'\n exec_should_failed 'rm -rf $a_test_dir/adir/child_dir1'\n\n exec_should_success 'touch $a_test_dir/adir/tmpfile'\n exec_should_success 'echo 123 > $a_test_dir/adir/tmpfile'\n exec_should_success 'echo 123 >> $a_test_dir/adir/tmpfile'\n\n exec_should_failed 'mv -f $a_test_dir/adir/tmpfile $a_test_dir/adir/file'\n exec_should_failed 'mv -f $a_test_dir/adir/file $a_test_dir/adir/tmpfile'\n touch \"$a_test_dir\"/tfile\n exec_should_success 'mv -f $a_test_dir/tfile $a_test_dir/adir/file2'\n}\n\n\ni_test_dir=\"$test_dir\"/i\nsudo chattr -R \"=\" \"${i_test_dir:?}\"\nsudo rm -rf \"${i_test_dir:?}\"/*\nmkdir \"$i_test_dir\"\n\n{\n touch \"$i_test_dir\"/ifile\n exec_should_success 'sudo chattr \"+i\" \"$i_test_dir\"/ifile'\n exec_should_success '[[ \"$(lsattr $i_test_dir/ifile | awk -F \" \" \"{print \\$1}\")\" =~ \"i\" ]]'\n\n exec_should_failed \"echo aa > $i_test_dir/ifile\"\n exec_should_failed \"echo aa >> $i_test_dir/ifile\"\n exec_should_failed \"rm -rf $i_test_dir/ifile\"\n touch \"$i_test_dir/tmpfile\"\n exec_should_failed \"mv -f $i_test_dir/tmpfile $i_test_dir/ifile\"\n exec_should_failed \"mv -f $i_test_dir/ifile $a_test_dir/tmpfile\"\n exec_should_failed \"ln $i_test_dir/ifile $i_test_dir/linkfile\"\n\n touch \"$i_test_dir\"/fallocatefile\n exec_should_success 'sudo chattr \"+i\" $i_test_dir/fallocatefile'\n exec_should_success '[[ \"$(lsattr $i_test_dir/fallocatefile | awk -F \" \" \"{print \\$1}\")\" =~ \"i\" ]]'\n exec_should_failed 'fallocate -l 1k -n $i_test_dir/fallocatefile'\n}\n\n{\n mkdir -p \"$i_test_dir\"/idir/child_dir1/child_dir2\n touch \"$i_test_dir\"/idir/file\n\n exec_should_success 'sudo chattr \"+i\" $i_test_dir/idir'\n exec_should_success '[[ \"$(lsattr -d $i_test_dir/idir | awk -F \" \" \"{print \\$1}\")\" =~ \"i\" ]]'\n exec_should_success 'touch \"$i_test_dir\"/idir/child_dir1/child_file'\n exec_should_success 'rm -rf $i_test_dir/idir/child_dir1/child_dir2'\n exec_should_success 'rm -rf $i_test_dir/idir/child_dir1/child_file'\n exec_should_failed 'rm -rf $i_test_dir/idir'\n exec_should_failed 'rm -rf $i_test_dir/idir/file'\n exec_should_failed 'rm -rf $i_test_dir/idir/child_dir1'\n\n exec_should_failed 'touch $i_test_dir/idir/tmpfile'\n exec_should_success 'echo 123 > $i_test_dir/idir/file'\n exec_should_success 'echo 123 >> $i_test_dir/idir/file'\n\n exec_should_failed 'mv -f $i_test_dir/idir/tmpfile $i_test_dir/idir/file'\n exec_should_failed 'mv -f $i_test_dir/idir/file $i_test_dir/idir/tmpfile'\n touch \"$i_test_dir\"/tfile\n exec_should_failed 'mv -f $i_test_dir/tfile $i_test_dir/idir/file2'\n}\n" }, { "path": "integration/s3gateway_test.sh", "content": "#!/bin/bash\n\n# Mint (C) 2017-2020 Minio, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# environment\n\nos=\"linux\"\nif [[ `uname -a` =~ \"Darwin\" ]];then\n os=\"mac\"\nfi\necho \"os=$os\"\n\nset -x\n\nMINT_DATA_DIR=testdata\nSERVER_ENDPOINT=\"127.0.0.1:9008\"\nACCESS_KEY=\"testUser\"\nSECRET_KEY=\"testUserPassword\"\nENABLE_HTTPS=0\nSERVER_REGION=us-east-1\nENABLE_VIRTUAL_STYLE=0\n\n# macos need bash 4.0+\n# create testdata\ndeclare -A data_file_map\ndata_file_map[\"datafile-0-b\"]=\"0\"\ndata_file_map[\"datafile-1-b\"]=\"1\"\ndata_file_map[\"datafile-1-kB\"]=\"1K\"\ndata_file_map[\"datafile-10-kB\"]=\"10K\"\ndata_file_map[\"datafile-33-kB\"]=\"33K\"\ndata_file_map[\"datafile-100-kB\"]=\"100K\"\ndata_file_map[\"datafile-1.03-MB\"]=\"1056K\"\ndata_file_map[\"datafile-1-MB\"]=\"1M\"\ndata_file_map[\"datafile-5-MB\"]=\"5M\"\ndata_file_map[\"datafile-5243880-b\"]=\"5243880\"\ndata_file_map[\"datafile-6-MB\"]=\"6M\"\ndata_file_map[\"datafile-10-MB\"]=\"10M\"\ndata_file_map[\"datafile-11-MB\"]=\"11M\"\ndata_file_map[\"datafile-65-MB\"]=\"65M\"\ndata_file_map[\"datafile-129-MB\"]=\"129M\"\n\nmkdir -p \"$MINT_DATA_DIR\"\n\n\nif [ ! \"$(ls $MINT_DATA_DIR)\" ]; then\n for filename in \"${!data_file_map[@]}\"; do\n echo \"creating $MINT_DATA_DIR/$filename\"\n if ! shred -n 1 -s \"${data_file_map[$filename]}\" - 1>\"$MINT_DATA_DIR/$filename\" 2>/dev/null; then\n echo \"unable to create data file $MINT_DATA_DIR/$filename\"\n exit 1\n fi\n done\nfi\n\n# configuration\naws configure set aws_access_key_id \"$ACCESS_KEY\"\naws configure set aws_secret_access_key \"$SECRET_KEY\"\naws configure set default.region \"$SERVER_REGION\"\n\n# run tests for virtual style if provided\nif [ \"$ENABLE_VIRTUAL_STYLE\" -eq 1 ]; then\n # Setup endpoint scheme\n endpoint=\"http://$DOMAIN:$SERVER_PORT\"\n if [ \"$ENABLE_HTTPS\" -eq 1 ]; then\n endpoint=\"https://$DOMAIN:$SERVER_PORT\"\n fi\n dnsmasq --address=\"/$DOMAIN/$SERVER_IP\" --user=root\n echo -e \"nameserver 127.0.0.1\\n$(cat /etc/resolv.conf)\" > /etc/resolv.conf\n aws configure set default.s3.addressing_style virtual\n# ./test.sh \"$endpoint\" 1>>\"$output_log_file\" 2>\"$error_log_file\"\n ./test.sh \"$endpoint\"\n aws configure set default.s3.addressing_style path\nfi\n\nendpoint=\"http://$SERVER_ENDPOINT\"\nif [ \"$ENABLE_HTTPS\" -eq 1 ]; then\n endpoint=\"https://$SERVER_ENDPOINT\"\nfi\n# run path style tests\n# ./test.sh \"$endpoint\" 1>>\"$output_log_file\" 2>\"$error_log_file\"\n\n\n# test\nfunction get_md5() {\n if [ $os == \"mac\" ]; then\n md5rt=$(md5 \"$1\" | awk '{print $4}')\n else\n md5rt=$(md5sum \"$1\" | awk '{print $1}')\n fi\n}\n\nget_md5 \"${MINT_DATA_DIR}/datafile-1-kB\"\nHASH_1_KB=$md5rt\n\nget_md5 \"${MINT_DATA_DIR}/datafile-65-MB\"\nHASH_65_MB=$md5rt\n\n_init() {\n AWS=\"aws --endpoint-url $1\"\n}\n\n\nfunction get_time() {\n date +%s%N\n}\n\nfunction get_duration() {\n start_time=$1\n end_time=$(get_time)\n\n echo $(( (end_time - start_time) / 1000000 ))\n}\n\nfunction log_success() {\n function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<\"$2\")\n printf '{\"name\": \"awscli\", \"duration\": %d, \"function\": %s, \"status\": \"PASS\"}\\n' \"$1\" \"$function\"\n}\n\nfunction log_failure() {\n function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<\"$2\")\n err=$(echo \"$3\" | tr -d '\\n')\n printf '{\"name\": \"awscli\", \"duration\": %d, \"function\": %s, \"status\": \"FAIL\", \"error\": \"%s\"}\\n' \"$1\" \"$function\" \"$err\"\n}\n\nfunction log_alert() {\n function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<\"$2\")\n err=$(echo \"$4\" | tr -d '\\n')\n printf '{\"name\": \"awscli\", \"duration\": %d, \"function\": %s, \"status\": \"FAIL\", \"alert\": \"%s\", \"error\": \"%s\"}\\n' \"$1\" \"$function\" \"$3\" \"$err\"\n}\n\nfunction make_bucket() {\n # Make bucket\n bucket_name=\"awscli-mint-test-bucket-$RANDOM\"\n function=\"${AWS} s3api create-bucket --bucket ${bucket_name}\"\n\n # execute the test\n out=$($function 2>&1)\n rv=$?\n\n # if command is successful print bucket_name or print error\n if [ $rv -eq 0 ]; then\n echo \"${bucket_name}\"\n else\n echo \"${out}\"\n fi\n\n return $rv\n}\n\nfunction delete_bucket() {\n # Delete bucket\n function=\"${AWS} s3 rb s3://${1} --force\"\n out=$($function 2>&1)\n rv=$?\n\n # echo the output\n echo \"${out}\"\n\n return $rv\n}\n\n# Tests creating, stat and delete on a bucket.\nfunction test_create_bucket() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n\n # if make_bucket is successful stat the bucket\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api head-bucket --bucket ${bucket_name}\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket failes, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # if stat bucket is successful remove the bucket\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"${bucket_name}\")\n rv=$?\n else\n # if make bucket failes, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests creating and deleting an object.\nfunction test_upload_object() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # if upload succeeds download the file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB /tmp/datafile-1-kB\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n # calculate the md5 hash of downloaded file\n get_md5 \"/tmp/datafile-1-kB\"\n hash2=$md5rt\n fi\n\n # if download succeeds, verify downloaded file\n if [ $rv -eq 0 ]; then\n if [ \"$HASH_1_KB\" == \"$hash2\" ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove download file\n rm -f /tmp/datafile-1-kB\n else\n rv=1\n out=\"Checksum verification failed for uploaded object\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Test lookup a directory prefix.\nfunction test_lookup_object_prefix() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds create a directory.\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n\n out=$($function 2>&1)\n\n rv=$?\n else\n # if make_bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n ## Attempt an overwrite of the prefix again and should succeed as well.\n function=\"${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n # if upload succeeds lookup for the prefix.\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n # if directory create succeeds, upload the object.\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key prefix/directory/datafile-1-kB\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it.\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api delete-object --bucket ${bucket_name} --key prefix/directory/\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n # if upload succeeds lookup for the object should succeed.\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/datafile-1-kB\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n # delete bucket\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n\n if [ $rv -ne 0 ]; then\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n else\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n fi\n\n return $rv\n}\n\n# Tests listing objects for both v1 and v2 API.\nfunction test_list_objects() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # if upload objects succeeds, list objects with existing prefix\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB\"\n test_function=${function}\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"datafile-1-kB\" ]; then\n rv=1\n # since rv is 0, command passed, but didn't return expected value. In this case set the output\n out=\"list-objects with existing prefix failed\"\n fi\n fi\n\n # if upload objects succeeds, list objects with not exist prefix\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix linux\"\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"\" ]; then\n rv=1\n out=\"list-objects without existing prefix failed\"\n fi\n fi\n\n # put dir1/dir2/dir3/dir4/ listobject(prefix=dir1/) should return \"dir1/dir2/dir3/dir4/\" ...\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --bucket ${bucket_name} --key dir1/dir2/dir3/dir4/\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix dir1/dir2/dir3/dir4/\"\n test_function=${function}\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[0].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"dir1/dir2/dir3/dir4/\" ]; then\n rv=1\n # since rv is 0, command passed, but didn't return expected value. In this case set the output\n out=\"list-objects with prefix is dir failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix dir1/\"\n test_function=${function}\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[0].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"dir1/dir2/dir3/dir4/\" ]; then\n rv=1\n # since rv is 0, command passed, but didn't return expected value. In this case set the output\n out=\"list-objects with prefix is dir failed\"\n fi\n fi\n\n # put dir1/dir2/ listobject(prefix=dir2/) should return \"dir1/dir2/\"\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --bucket ${bucket_name} --key dir1/dir2/\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # if upload objects succeeds, list objects with existing prefix\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix dir1/\"\n test_function=${function}\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[0].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"dir1/dir2/\" ]; then\n rv=1\n # since rv is 0, command passed, but didn't return expected value. In this case set the output\n out=\"list-objects with prefix is dir failed\"\n fi\n fi\n\n # delete dir1/dir2/ listobject(prefix=dir1/) should return \"dir2/\"\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api delete-object --bucket ${bucket_name} --key dir1/dir2/\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix dir1/\"\n test_function=${function}\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[0].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"dir1/dir2/dir3/dir4/\" ]; then\n rv=1\n # since rv is 0, command passed, but didn't return expected value. In this case set the output\n out=\"list-objects with prefix is dir failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix dir1/ --delimiter /\"\n test_function=${function}\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .CommonPrefixes[0].Prefix)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"dir1/dir2/\" ]; then\n rv=1\n # since rv is 0, command passed, but didn't return expected value. In this case set the output\n out=\"list-objects with prefix is dir failed\"\n fi\n fi\n\n # delete dir1/dir2/dir3/dir4/ listobject(prefix=dir1/) should return nothing\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api delete-object --bucket ${bucket_name} --key dir1/dir2/dir3/dir4/\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix dir1/\"\n test_function=${function}\n out=$($function)\n rv=$?\n output=$(echo \"$out\")\n if [ $rv -eq 0 ] && [ \"$output\" != \"\" ]; then\n rv=1\n # since rv is 0, command passed, but didn't return expected value. In this case set the output\n out=\"list-objects with prefix is dir failed\"\n fi\n fi\n\n\n\n # if upload objects succeeds, list objectsv2 with existing prefix\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB\"\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"datafile-1-kB\" ]; then\n rv=1\n out=\"list-objects-v2 with existing prefix failed\"\n fi\n fi\n\n # if upload objects succeeds, list objectsv2 without existing prefix\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix linux\"\n out=$($function)\n rv=$?\n key_name=$(echo \"$out\" | jq -r .Contents[].Key)\n if [ $rv -eq 0 ] && [ \"$key_name\" != \"\" ]; then\n rv=1\n out=\"list-objects-v2 without existing prefix failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove download file\n rm -f /tmp/datafile-1-kB\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n rm -f /tmp/datafile-1-kB\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests multipart API with 0 byte part.\nfunction test_multipart_upload_0byte() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n object_name=${bucket_name}\"-object\"\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-0-b --bucket ${bucket_name} --key datafile-0-b\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n # create multipart\n function=\"${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}\"\n test_function=${function}\n out=$($function)\n rv=$?\n upload_id=$(echo \"$out\" | jq -r .UploadId)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 1\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-0-b --upload-id ${upload_id} --part-number 1\"\n out=$($function)\n rv=$?\n etag1=$(echo \"$out\" | jq -r .ETag)\n fi\n\n if [ $rv -eq 0 ]; then\n # Create a multipart struct file for completing multipart transaction\n echo \"{\n \\\"Parts\\\": [\n {\n \\\"ETag\\\": ${etag1},\n \\\"PartNumber\\\": 1\n }\n ]\n }\" >> /tmp/multipart\n fi\n\n if [ $rv -eq 0 ]; then\n # Use saved etags to complete the multipart transaction\n function=\"${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}\"\n out=$($function)\n rv=$?\n etag=$(echo \"$out\" | jq -r .ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ \"${etag}\" == \"\" ]; then\n rv=1\n out=\"complete-multipart-upload failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key ${object_name} /tmp/datafile-0-b\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n ret_etag=$(echo \"$out\" | jq -r .ETag | sed -e 's/^\"//' -e 's/\"$//')\n # match etag\n if [ \"$etag\" != \"$ret_etag\" ]; then\n rv=1\n out=\"Etag mismatch for multipart 0 byte object\"\n fi\n rm -f /tmp/datafile-0-b\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove temp file\n rm -f /tmp/multipart\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n rm -f /tmp/multipart\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests multipart API by making each individual calls.\nfunction test_multipart_upload() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n object_name=${bucket_name}\"-object\"\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n # create multipart\n function=\"${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}\"\n test_function=${function}\n out=$($function)\n rv=$?\n upload_id=$(echo \"$out\" | jq -r .UploadId)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 1\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5-MB --upload-id ${upload_id} --part-number 1\"\n out=$($function)\n rv=$?\n etag1=$(echo \"$out\" | jq -r .ETag)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 2\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-1-kB --upload-id ${upload_id} --part-number 2\"\n out=$($function)\n rv=$?\n etag2=$(echo \"$out\" | jq -r .ETag)\n # Create a multipart struct file for completing multipart transaction\n echo \"{\n \\\"Parts\\\": [\n {\n \\\"ETag\\\": ${etag1},\n \\\"PartNumber\\\": 1\n },\n {\n \\\"ETag\\\": ${etag2},\n \\\"PartNumber\\\": 2\n }\n ]\n }\" >> /tmp/multipart\n fi\n\n if [ $rv -eq 0 ]; then\n # Use saved etags to complete the multipart transaction\n function=\"${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}\"\n out=$($function)\n rv=$?\n finalETag=$(echo \"$out\" | jq -r .ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ \"${finalETag}\" == \"\" ]; then\n rv=1\n out=\"complete-multipart-upload failed\"\n fi\n fi\n\n\n for key in \"afile\" \"bfile\" \"bfile\" \"documents/report1.pdf\" \"documents/report2.pdf\" \"ebook\" \"photos/2021/a2.png\" \"photos/2021/a3.png\" \"photos/2022/a4.png\"\n do\n if [ $rv -eq 0 ]; then\n # create multipart\n function=\"${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${key}\"\n test_function=${function}\n out=$($function)\n rv=$?\n upload_id=$(echo \"$out\" | jq -r .UploadId)\n fi\n done\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-multipart-uploads --bucket ${bucket_name}\"\n test_function=${function}\n out=$($function)\n rv=$?\n keys=$(echo \"$out\" | jq -r '.Uploads | map(.Key) | join(\",\")')\n if [ $keys != \"afile,bfile,bfile,documents/report1.pdf,documents/report2.pdf,ebook,photos/2021/a2.png,photos/2021/a3.png,photos/2022/a4.png\" ]; then\n rv=1\n out=\"list-multipart-uploads failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --key-marker bfile\"\n test_function=${function}\n out=$($function)\n rv=$?\n keys=$(echo \"$out\" | jq -r '.Uploads | map(.Key) | join(\",\")')\n if [ $keys != \"bfile,bfile,documents/report1.pdf,documents/report2.pdf,ebook,photos/2021/a2.png,photos/2021/a3.png,photos/2022/a4.png\" ]; then\n rv=1\n out=\"list-multipart-upload failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --key-marker bfile --delimiter /\"\n test_function=${function}\n out=$($function)\n rv=$?\n keys=$(echo \"$out\" | jq -r '.Uploads | map(.Key) | join(\",\")')\n if [ $keys != \"bfile,bfile,ebook\" ]; then\n rv=1\n out=\"list-multipart-uploads failed\"\n fi\n keys=$(echo \"$out\" | jq -r '.CommonPrefixes | map(.Prefix) | join(\",\")')\n if [ $keys != \"documents/,photos/\" ]; then\n rv=1\n out=\"list-multipart-uploads failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --delimiter / --max-upload 5\"\n test_function=${function}\n out=$($function)\n rv=$?\n keys=$(echo \"$out\" | jq -r '.Uploads | map(.Key) | join(\",\")')\n if [ $keys != \"afile,bfile,bfile,ebook\" ]; then\n rv=1\n out=\"list-multipart-uploads failed\"\n fi\n keys=$(echo \"$out\" | jq -r '.CommonPrefixes[0].Prefix')\n if [ $keys != \"documents/\" ]; then\n rv=1\n out=\"list-multipart-uploads failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --prefix documents/\"\n test_function=${function}\n out=$($function)\n rv=$?\n keys=$(echo \"$out\" | jq -r '.Uploads | map(.Key) | join(\",\")')\n if [ $keys != \"documents/report1.pdf,documents/report2.pdf\" ]; then\n rv=1\n out=\"list-multipart-upload failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove temp file\n rm -f /tmp/multipart\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n rm -f /tmp/multipart\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# List number of objects based on the maxKey\n# value set.\nfunction test_max_key_list() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-b --bucket ${bucket_name} --key datafile-1-b\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # copy object server side\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-b-copy --copy-source ${bucket_name}/datafile-1-b\"\n out=$($function)\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api list-objects-v2 --bucket ${bucket_name} --max-keys 1\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n if [ $rv -eq 0 ]; then\n out=$(echo \"$out\" | jq '.KeyCount')\n rv=$?\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # The command passed, but the delete_bucket failed\n out=\"delete_bucket for test_max_key_list failed\"\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Copy object tests for server side copy\n# of the object, validates returned md5sum.\nfunction test_copy_object() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # copy object server side\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB\"\n test_function=${function}\n out=$($function)\n rv=$?\n hash2=$(echo \"$out\" | jq -r .CopyObjectResult.ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ $rv -eq 0 ] && [ \"$HASH_1_KB\" != \"$hash2\" ]; then\n # Verification failed\n rv=1\n out=\"Hash mismatch expected $HASH_1_KB, got $hash2\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --key /not-exist-dir/datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB\"\n test_function=${function}\n out=$($function)\n rv=$?\n hash2=$(echo \"$out\" | jq -r .CopyObjectResult.ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ $rv -eq 0 ] && [ \"$HASH_1_KB\" != \"$hash2\" ]; then\n # Verification failed\n rv=1\n out=\"Hash mismatch expected $HASH_1_KB, got $hash2\"\n fi\n fi\n\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Copy object tests for server side copy\n# of the object, validates returned md5sum.\n# validates change in storage class as well\nfunction test_copy_object_storage_class() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # copy object server side\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n # if this functionality is not implemented return right away.\n if [ $rv -ne 0 ]; then\n if echo \"$out\" | grep -q \"NotImplemented\"; then\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n return 0\n fi\n fi\n hash2=$(echo \"$out\" | jq -r .CopyObjectResult.ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ $rv -eq 0 ] && [ \"$HASH_1_KB\" != \"$hash2\" ]; then\n # Verification failed\n rv=1\n out=\"Hash mismatch expected $HASH_1_KB, got $hash2\"\n fi\n fi\n\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Copy object tests for server side copy\n# to itself by changing storage class\nfunction test_copy_object_storage_class_same() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # copy object server side\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB --copy-source ${bucket_name}/datafile-1-kB\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n # if this functionality is not implemented return right away.\n if [ $rv -ne 0 ]; then\n if echo \"$out\" | grep -q \"NotImplemented\"; then\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n return 0\n fi\n fi\n hash2=$(echo \"$out\" | jq -r .CopyObjectResult.ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ $rv -eq 0 ] && [ \"$HASH_1_KB\" != \"$hash2\" ]; then\n # Verification failed\n rv=1\n out=\"Hash mismatch expected $HASH_1_KB, got $hash2\"\n fi\n fi\n\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests for presigned URL success case, presigned URL\n# is correct and accessible - we calculate md5sum of\n# the object and validate it against a local files md5sum.\nfunction test_presigned_object() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3 presign s3://${bucket_name}/datafile-1-kB\"\n test_function=${function}\n url=$($function)\n rv=$?\n curl -sS -X GET \"${url}\" > /tmp/datafile-1-kB\n get_md5 /tmp/datafile-1-kB\n hash2=$md5rt\n if [ \"$HASH_1_KB\" == \"$hash2\" ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove download file\n rm -f /tmp/datafile-1-kB\n else\n rv=1\n out=\"Checksum verification failed for downloaded object\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests creating and deleting an object - 10MiB\nfunction test_upload_object_10() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-MB --bucket ${bucket_name} --key datafile-10-MB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests multipart API by making each individual calls with 10MiB part size.\nfunction test_multipart_upload_10() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n object_name=${bucket_name}\"-object\"\n rv=$?\n\n if [ $rv -eq 0 ]; then\n # create multipart\n function=\"${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}\"\n test_function=${function}\n out=$($function)\n rv=$?\n upload_id=$(echo \"$out\" | jq -r .UploadId)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 1\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 1\"\n out=$($function)\n rv=$?\n etag1=$(echo \"$out\" | jq -r .ETag)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 2\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 2\"\n out=$($function)\n rv=$?\n etag2=$(echo \"$out\" | jq -r .ETag)\n # Create a multipart struct file for completing multipart transaction\n echo \"{\n \\\"Parts\\\": [\n {\n \\\"ETag\\\": ${etag1},\n \\\"PartNumber\\\": 1\n },\n {\n \\\"ETag\\\": ${etag2},\n \\\"PartNumber\\\": 2\n }\n ]\n }\" >> /tmp/multipart\n fi\n\n if [ $rv -eq 0 ]; then\n # Use saved etags to complete the multipart transaction\n function=\"${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}\"\n out=$($function)\n rv=$?\n finalETag=$(echo \"$out\" | jq -r .ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ \"${finalETag}\" == \"\" ]; then\n rv=1\n out=\"complete-multipart-upload failed\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove temp file\n rm -f /tmp/multipart\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n rm -f /tmp/multipart\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests lifecycle of a bucket.\nfunction test_bucket_lifecycle() {\n # log start time\n start_time=$(get_time)\n\n echo \"{ \\\"Rules\\\": [ { \\\"Expiration\\\": { \\\"Days\\\": 365 },\\\"ID\\\": \\\"Bucketlifecycle test\\\", \\\"Filter\\\": { \\\"Prefix\\\": \\\"\\\" }, \\\"Status\\\": \\\"Enabled\\\" } ] }\" >> /tmp/lifecycle.json\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds put bucket lifecycle\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-bucket-lifecycle-configuration --bucket ${bucket_name} --lifecycle-configuration file:///tmp/lifecycle.json\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -ne 0 ]; then\n # if this functionality is not implemented return right away.\n if echo \"$out\" | grep -q \"NotImplemented\"; then\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n return 0\n fi\n fi\n\n # if put bucket lifecycle succeeds get bucket lifecycle\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-bucket-lifecycle-configuration --bucket ${bucket_name}\"\n out=$($function 2>&1)\n rv=$?\n fi\n\n # if get bucket lifecycle succeeds delete bucket lifecycle\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api delete-bucket-lifecycle --bucket ${bucket_name}\"\n out=$($function 2>&1)\n rv=$?\n fi\n\n # delete lifecycle.json\n rm -f /tmp/lifecycle.json\n\n # delete bucket\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"test_bucket_lifecycle\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests `aws s3 cp` by uploading a local file.\nfunction test_aws_s3_cp() {\n file_name=\"${MINT_DATA_DIR}/datafile-65-MB\"\n\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file using cp\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3 cp $file_name s3://${bucket_name}/$(basename \"$file_name\")\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3 rm s3://${bucket_name}/$(basename \"$file_name\")\"\n out=$($function 2>&1)\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3 rb s3://${bucket_name}/\"\n out=$($function 2>&1)\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# Tests `aws s3 sync` by mirroring all the\n# local content to remove bucket.\nfunction test_aws_s3_sync() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds sync all the files in a directory\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3 sync --no-progress $MINT_DATA_DIR s3://${bucket_name}/\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # remove files recusively\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3 rm --recursive s3://${bucket_name}/\"\n out=$($function 2>&1)\n rv=$?\n fi\n\n # delete bucket\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# list objects negative test - tests for following conditions.\n# v1 API with max-keys=-1 and max-keys=0\n# v2 API with max-keys=-1 and max-keys=0\nfunction test_list_objects_error() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n if [ $rv -eq 0 ]; then\n # Server replies an error for v1 with max-key=-1\n function=\"${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n # Server replies an error for v2 with max-keys=-1\n function=\"${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n # Server returns success with no keys when max-keys=0\n function=\"${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=0\"\n out=$($function 2>&1)\n rv=$?\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# put object negative test - tests for following conditions.\n# - invalid object name.\n# - invalid Content-Md5\n# - invalid Content-Length\nfunction test_put_object_error() {\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload an object without content-md5.\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-md5 invalid\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n fi\n\n # upload an object without content-length.\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-length -1\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n# tests server side encryption headers for get and put calls\nfunction test_serverside_encryption() {\n #skip server side encryption tests if HTTPS disabled.\n if [ \"$ENABLE_HTTPS\" != \"1\" ]; then\n return 0\n fi\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # put object with server side encryption headers\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n # now get encrypted object from server\n if [ $rv -eq 0 ]; then\n etag1=$(echo \"$out\" | jq -r .ETag)\n sse_customer_key1=$(echo \"$out\" | jq -r .SSECustomerKeyMD5)\n sse_customer_algo1=$(echo \"$out\" | jq -r .SSECustomerAlgorithm)\n\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n etag2=$(echo \"$out\" | jq -r .ETag)\n sse_customer_key2=$(echo \"$out\" | jq -r .SSECustomerKeyMD5)\n sse_customer_algo2=$(echo \"$out\" | jq -r .SSECustomerAlgorithm)\n get_md5 \"/tmp/datafile-1-kB\"\n hash2=$md5rt\n # match downloaded object's hash to original\n if [ \"$HASH_1_KB\" == \"$hash2\" ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove download file\n rm -f /tmp/datafile-1-kB\n else\n rv=1\n out=\"Checksum verification failed for downloaded object\"\n fi\n # match etag and SSE headers\n if [ \"$etag1\" != \"$etag2\" ]; then\n rv=1\n out=\"Etag mismatch for object encrypted with server side encryption\"\n fi\n if [ \"$sse_customer_algo1\" != \"$sse_customer_algo2\" ]; then\n rv=1\n out=\"sse customer algorithm mismatch\"\n fi\n if [ \"$sse_customer_key1\" != \"$sse_customer_key2\" ]; then\n rv=1\n out=\"sse customer key mismatch\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# tests server side encryption headers for multipart put\nfunction test_serverside_encryption_multipart() {\n #skip server side encryption tests if HTTPS disabled.\n if [ \"$ENABLE_HTTPS\" != \"1\" ]; then\n return 0\n fi\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # put object with server side encryption headers\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-65-MB --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n # now get encrypted object from server\n if [ $rv -eq 0 ]; then\n etag1=$(echo \"$out\" | jq -r .ETag)\n sse_customer_key1=$(echo \"$out\" | jq -r .SSECustomerKeyMD5)\n sse_customer_algo1=$(echo \"$out\" | jq -r .SSECustomerAlgorithm)\n\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-65-MB\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n etag2=$(echo \"$out\" | jq -r .ETag)\n sse_customer_key2=$(echo \"$out\" | jq -r .SSECustomerKeyMD5)\n sse_customer_algo2=$(echo \"$out\" | jq -r .SSECustomerAlgorithm)\n get_md5 \"${MINT_DATA_DIR}/datafile-65-MB\"\n hash2=$md5rt\n # match downloaded object's hash to original\n if [ \"$HASH_65_MB\" == \"$hash2\" ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n # remove download file\n rm -f /tmp/datafile-65-MB\n else\n rv=1\n out=\"Checksum verification failed for downloaded object\"\n fi\n # match etag and SSE headers\n if [ \"$etag1\" != \"$etag2\" ]; then\n rv=1\n out=\"Etag mismatch for object encrypted with server side encryption\"\n fi\n if [ \"$sse_customer_algo1\" != \"$sse_customer_algo2\" ]; then\n rv=1\n out=\"sse customer algorithm mismatch\"\n fi\n if [ \"$sse_customer_key1\" != \"$sse_customer_key2\" ]; then\n rv=1\n out=\"sse customer key mismatch\"\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n# tests encrypted copy from multipart encrypted object to\n# single part encrypted object. This test in particular checks if copy\n# succeeds for the case where encryption overhead for individually\n# encrypted parts vs encryption overhead for the original datastream\n# differs.\nfunction test_serverside_encryption_multipart_copy() {\n #skip server side encryption tests if HTTPS disabled.\n if [ \"$ENABLE_HTTPS\" != \"1\" ]; then\n return 0\n fi\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n object_name=${bucket_name}\"-object\"\n rv=$?\n\n if [ $rv -eq 0 ]; then\n # create multipart\n function=\"${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name} --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n out=$($function)\n rv=$?\n upload_id=$(echo \"$out\" | jq -r .UploadId)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 1\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 1 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n out=$($function)\n rv=$?\n etag1=$(echo \"$out\" | jq -r .ETag)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 2\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 2 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n out=$($function)\n rv=$?\n etag2=$(echo \"$out\" | jq -r .ETag)\n # Create a multipart struct file for completing multipart transaction\n echo \"{\n \\\"Parts\\\": [\n {\n \\\"ETag\\\": ${etag1},\n \\\"PartNumber\\\": 1\n },\n {\n \\\"ETag\\\": ${etag2},\n \\\"PartNumber\\\": 2\n }\n ]\n }\" >> /tmp/multipart\n fi\n\n if [ $rv -eq 0 ]; then\n # Use saved etags to complete the multipart transaction\n function=\"${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}\"\n out=$($function)\n rv=$?\n finalETag=$(echo \"$out\" | jq -r .ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ \"${finalETag}\" == \"\" ]; then\n rv=1\n out=\"complete-multipart-upload failed\"\n fi\n fi\n\n # copy object server side\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --key ${object_name}-copy --copy-source ${bucket_name}/${object_name} --copy-source-sse-customer-algorithm AES256 --copy-source-sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --copy-source-sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n test_function=${function}\n out=$($function)\n rv=$?\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n rm -f /tmp/multipart\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n# tests server side encryption headers for range get calls\nfunction test_serverside_encryption_get_range() {\n #skip server side encryption tests if HTTPS disabled.\n if [ \"$ENABLE_HTTPS\" != \"1\" ]; then\n return 0\n fi\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n # put object with server side encryption headers\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-kB --bucket ${bucket_name} --key datafile-10-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n # now get encrypted object from server for range 500-999\n if [ $rv -eq 0 ]; then\n etag1=$(echo \"$out\" | jq -r .ETag)\n sse_customer_key1=$(echo \"$out\" | jq -r .SSECustomerKeyMD5)\n sse_customer_algo1=$(echo \"$out\" | jq -r .SSECustomerAlgorithm)\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key datafile-10-kB --range bytes=500-999 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-10-kB\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n cnt=$(stat -c%s /tmp/datafile-10-kB)\n if [ \"$cnt\" -ne 500 ]; then\n rv=1\n fi\n fi\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n return $rv\n}\n\n# tests server side encryption error for get and put calls\nfunction test_serverside_encryption_error() {\n #skip server side encryption tests if HTTPS disabled.\n if [ \"$ENABLE_HTTPS\" != \"1\" ]; then\n return 0\n fi\n # log start time\n start_time=$(get_time)\n\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # put object with server side encryption headers with MD5Sum mismatch for sse-customer-key-md5 header\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n # put object with missing server side encryption header sse-customer-algorithm\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n\n # put object with server side encryption headers successfully\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n\n # now test get on encrypted object with nonmatching sse-customer-key and sse-customer-md5 headers\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB\"\n test_function=${function}\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -ne 255 ] && [ $rv -ne 254 ]; then\n rv=1\n else\n rv=0\n fi\n # delete bucket\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n log_success \"$(get_duration \"$start_time\")\" \"${test_function}\"\n else\n # clean up and log error\n ${AWS} s3 rb s3://\"${bucket_name}\" --force > /dev/null 2>&1\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n fi\n\n return $rv\n}\n\n\n# test GetObjectInfo http code is 404\nfunction test_get_object_error(){\n # log start time\n start_time=$(get_time)\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # if make bucket succeeds upload a file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key /dir1/datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # if upload succeeds download the file\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key /dir1 /tmp/datafile-1-kB\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n if [ $? -eq 255 ] || [ $? -eq 254 ];then\n rv=0\n fi\n if ! [[ \"$out\" =~ \"The specified key does not exist\" ]];then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object --bucket ${bucket_name} --key /dir1/ /tmp/datafile-1-kB\"\n # save the ref to function being tested, so it can be logged\n test_function=${function}\n out=$($function 2>&1)\n if [ $? -eq 255 ] || [ $? -eq 254 ];then\n rv=0\n fi\n if ! [[ \"$out\" =~ \"The specified key does not exist\" ]];then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n # delete bucket\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n return $rv\n}\n\nfunction test_object_tagging(){\n # log start time\n start_time=$(get_time)\n function=\"make_bucket\"\n bucket_name=$(make_bucket)\n rv=$?\n\n # put object with object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key /datafile-1-kB --tagging k1=v1&k2=v2\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # check object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object-tagging --bucket ${bucket_name} --key /datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq 'sort_by(.Key)' | jq -c)\n if [ \"$tagSet\" != '[{\"Key\":\"k1\",\"Value\":\"v1\"},{\"Key\":\"k2\",\"Value\":\"v2\"}]' ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n # overwrite object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key /datafile-1-kB --tagging key1=value1&key2=value2\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # check object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object-tagging --bucket ${bucket_name} --key /datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq 'sort_by(.Key)' | jq -c)\n if [ \"$tagSet\" != '[{\"Key\":\"key1\",\"Value\":\"value1\"},{\"Key\":\"key2\",\"Value\":\"value2\"}]' ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n # delete object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api delete-object-tagging --bucket ${bucket_name} --key /datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n fi\n # check object tagging\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq -c)\n if [ \"$tagSet\" != '' ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n # create multipart upload with object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key /datafile-1-kB --tagging k1=v1&k2=v2\"\n out=$($function)\n rv=$?\n upload_id=$(echo \"$out\" | jq -r .UploadId)\n fi\n # upload part\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 1\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key /datafile-1-kB --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 1\"\n out=$($function)\n rv=$?\n etag1=$(echo \"$out\" | jq -r .ETag)\n fi\n\n if [ $rv -eq 0 ]; then\n # Capture etag for part-number 2\n function=\"${AWS} s3api upload-part --bucket ${bucket_name} --key /datafile-1-kB --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 2\"\n out=$($function)\n rv=$?\n etag2=$(echo \"$out\" | jq -r .ETag)\n # Create a multipart struct file for completing multipart transaction\n echo \"{\n \\\"Parts\\\": [\n {\n \\\"ETag\\\": ${etag1},\n \\\"PartNumber\\\": 1\n },\n {\n \\\"ETag\\\": ${etag2},\n \\\"PartNumber\\\": 2\n }\n ]\n }\" > /tmp/multipart\n fi\n\n # complete multipart upload\n if [ $rv -eq 0 ]; then\n # Use saved etags to complete the multipart transaction\n function=\"${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key /datafile-1-kB --upload-id ${upload_id}\"\n out=$($function)\n rm -rf /tmp/multipart\n rv=$?\n finalETag=$(echo \"$out\" | jq -r .ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ \"${finalETag}\" == \"\" ]; then\n rv=1\n out=\"complete-multipart-upload failed\"\n fi\n fi\n # check object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object-tagging --bucket ${bucket_name} --key /datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq 'sort_by(.Key)' | jq -c)\n if [ \"$tagSet\" != '[{\"Key\":\"k1\",\"Value\":\"v1\"},{\"Key\":\"k2\",\"Value\":\"v2\"}]' ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n # overwrite object\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key /datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # check object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object-tagging --bucket ${bucket_name} --key /datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n fi\n # check object tagging\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq -c)\n if [ \"$tagSet\" != \"[]\" ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key /datafile-1-kB --tagging key1=value1&key2=value2\"\n out=$($function 2>&1)\n rv=$?\n else\n # if make bucket fails, $bucket_name has the error output\n out=\"${bucket_name}\"\n fi\n\n # check object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object-tagging --bucket ${bucket_name} --key /datafile-1-kB\"\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq 'sort_by(.Key)' | jq -c)\n if [ \"$tagSet\" != '[{\"Key\":\"key1\",\"Value\":\"value1\"},{\"Key\":\"key2\",\"Value\":\"value2\"}]' ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n # copy object with tagging-directive COPY\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB\"\n out=$($function)\n rv=$?\n hash2=$(echo \"$out\" | jq -r .CopyObjectResult.ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ $rv -eq 0 ] && [ \"$HASH_1_KB\" != \"$hash2\" ]; then\n # Verification failed\n rv=1\n out=\"Hash mismatch expected $HASH_1_KB, got $hash2\"\n fi\n fi\n\n # check object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object-tagging --bucket ${bucket_name} --key /datafile-1-kB-copy\"\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq 'sort_by(.Key)' | jq -c)\n if [ \"$tagSet\" != '[{\"Key\":\"key1\",\"Value\":\"value1\"},{\"Key\":\"key2\",\"Value\":\"value2\"}]' ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n\n # copy object with tagging-directive REPLACE\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB --tagging key=value --tagging-directive REPLACE\"\n out=$($function)\n rv=$?\n hash2=$(echo \"$out\" | jq -r .CopyObjectResult.ETag | sed -e 's/^\"//' -e 's/\"$//')\n if [ $rv -eq 0 ] && [ \"$HASH_1_KB\" != \"$hash2\" ]; then\n # Verification failed\n rv=1\n out=\"Hash mismatch expected $HASH_1_KB, got $hash2\"\n fi\n fi\n\n # check object tagging\n if [ $rv -eq 0 ]; then\n function=\"${AWS} s3api get-object-tagging --bucket ${bucket_name} --key datafile-1-kB-copy\"\n out=$($function 2>&1)\n rv=$?\n fi\n if [ $rv -eq 0 ]; then\n tagSet=$(echo \"$out\" | jq -r .TagSet | jq 'sort_by(.Key)' | jq -c)\n if [ \"$tagSet\" != '[{\"Key\":\"key\",\"Value\":\"value\"}]' ]; then\n log_failure \"$(get_duration \"$start_time\")\" \"${function}\" \"${out}\"\n rv=1\n fi\n fi\n\n # delete bucket\n if [ $rv -eq 0 ]; then\n function=\"delete_bucket\"\n out=$(delete_bucket \"$bucket_name\")\n rv=$?\n fi\n return $rv\n}\n# main handler for all the tests.\nmain() {\n # Success tests\n test_create_bucket && \\\n test_upload_object && \\\n test_lookup_object_prefix && \\\n test_list_objects && \\\n test_multipart_upload_0byte && \\\n test_multipart_upload && \\\n test_max_key_list && \\\n test_copy_object && \\\n test_copy_object_storage_class && \\\n test_copy_object_storage_class_same && \\\n test_presigned_object && \\\n test_upload_object_10 && \\\n test_multipart_upload_10 && \\\n# test_bucket_lifecycle && \\\n test_serverside_encryption && \\\n test_serverside_encryption_get_range && \\\n test_serverside_encryption_multipart && \\\n test_serverside_encryption_multipart_copy && \\\n # Success cli ops.\n test_aws_s3_cp && \\\n test_aws_s3_sync && \\\n # Error tests\n test_list_objects_error && \\\n test_put_object_error && \\\n test_serverside_encryption_error && \\\n # test_worm_bucket && \\\n # test_legal_hold\n test_get_object_error && \\\n test_object_tagging\n return $?\n}\n\n_init \"$endpoint\" && main\n" }, { "path": "main.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage main\n\nimport (\n\t\"os\"\n\n\t\"github.com/juicedata/juicefs/cmd\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nvar logger = utils.GetLogger(\"juicefs\")\n\nfunc main() {\n\terr := cmd.Main(os.Args)\n\tif err != nil {\n\t\tlogger.Fatal(err)\n\t}\n}\n" }, { "path": "package.json", "content": "{\n \"name\": \"juicefs\",\n \"version\": \"1.0.0\",\n \"author\": \"Juicedata\",\n \"license\": \"Apache\",\n \"repository\": \"github:juicedata/juicefs\",\n \"scripts\": {\n \"autocorrect-lint\": \"autocorrect --lint ./docs/ README*.md\",\n \"autocorrect-lint-fix\": \"autocorrect --fix ./docs/ README*.md\",\n \"check-broken-link\": \"./node_modules/.bin/remark --quiet --frail ./docs/ README*.md\",\n \"markdown-lint\": \"./node_modules/.bin/markdownlint-cli2 './docs/**/*.md' README*.md\",\n \"markdown-lint-fix\": \"./node_modules/.bin/markdownlint-cli2 --fix './docs/**/*.md' README*.md\"\n },\n \"dependencies\": {\n \"markdownlint-cli2\": \"^0.17.2\",\n \"markdownlint-rule-enhanced-proper-names\": \"^0.0.1\",\n \"markdownlint-rule-no-trailing-slash-in-links\": \"^0.0.1\",\n \"remark-cli\": \"^11.0.0\",\n \"remark-validate-links\": \"^13.0.1\",\n \"remark-validate-links-heading-id\": \"^0.0.3\"\n },\n \"remarkConfig\": {\n \"plugins\": [\n \"remark-validate-links-heading-id\",\n \"remark-validate-links\"\n ]\n }\n}\n" }, { "path": "pkg/acl/acl.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage acl\n\nimport (\n\t\"fmt\"\n\t\"hash/crc32\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nconst Version uint8 = 2\n\ntype Entry struct {\n\tId uint32\n\tPerm uint16\n}\n\ntype Entries []Entry\n\nfunc (es *Entries) Len() int { return len(*es) }\nfunc (es *Entries) Less(i, j int) bool { return (*es)[i].Id < (*es)[j].Id }\nfunc (es *Entries) Swap(i, j int) { (*es)[i], (*es)[j] = (*es)[j], (*es)[i] }\n\nfunc (es *Entries) IsEqual(other *Entries) bool {\n\tif es.Len() != other.Len() {\n\t\treturn false\n\t}\n\tfor i := 0; i < es.Len(); i++ {\n\t\tif (*es)[i].Id != (*other)[i].Id || (*es)[i].Perm != (*other)[i].Perm {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc (es *Entries) Encode() []byte {\n\tw := utils.NewBuffer(uint32(es.Len() * 6))\n\tfor _, e := range *es {\n\t\tw.Put32(e.Id)\n\t\tw.Put16(e.Perm)\n\t}\n\treturn w.Bytes()\n}\n\nfunc (es *Entries) Decode(data []byte) {\n\tr := utils.ReadBuffer(data)\n\tfor r.HasMore() {\n\t\t*es = append(*es, Entry{\n\t\t\tId: r.Get32(),\n\t\t\tPerm: r.Get16(),\n\t\t})\n\t}\n}\n\n// Rule acl rule\ntype Rule struct {\n\tOwner uint16\n\tGroup uint16\n\tMask uint16\n\tOther uint16\n\tNamedUsers Entries\n\tNamedGroups Entries\n}\n\nfunc (r *Rule) String() string {\n\treturn fmt.Sprintf(\"owner %o, group %o, mask %o, other %o, named users: %+v, named group %+v\",\n\t\tr.Owner, r.Group, r.Mask, r.Other, r.NamedUsers, r.NamedGroups)\n}\n\nfunc (r *Rule) Dup() *Rule {\n\tif r != nil {\n\t\tnewRule := *r\n\t\t// NamedUsers and NamedGroups are never modified\n\t\treturn &newRule\n\t}\n\treturn nil\n}\n\nfunc (r *Rule) Encode() []byte {\n\tw := utils.NewBuffer(uint32(16 + (len(r.NamedUsers)+len(r.NamedGroups))*6))\n\tw.Put16(r.Owner)\n\tw.Put16(r.Group)\n\tw.Put16(r.Mask)\n\tw.Put16(r.Other)\n\tw.Put32(uint32(len(r.NamedUsers)))\n\tfor _, entry := range r.NamedUsers {\n\t\tw.Put32(entry.Id)\n\t\tw.Put16(entry.Perm)\n\t}\n\tw.Put32(uint32(len(r.NamedGroups)))\n\tfor _, entry := range r.NamedGroups {\n\t\tw.Put32(entry.Id)\n\t\tw.Put16(entry.Perm)\n\t}\n\treturn w.Bytes()\n}\n\nfunc (r *Rule) Decode(buf []byte) {\n\trb := utils.ReadBuffer(buf)\n\tr.Owner = rb.Get16()\n\tr.Group = rb.Get16()\n\tr.Mask = rb.Get16()\n\tr.Other = rb.Get16()\n\tuCnt := rb.Get32()\n\tr.NamedUsers = make([]Entry, uCnt)\n\tfor i := 0; i < int(uCnt); i++ {\n\t\tr.NamedUsers[i].Id = rb.Get32()\n\t\tr.NamedUsers[i].Perm = rb.Get16()\n\t}\n\n\tgCnt := rb.Get32()\n\tr.NamedGroups = make([]Entry, gCnt)\n\tfor i := 0; i < int(gCnt); i++ {\n\t\tr.NamedGroups[i].Id = rb.Get32()\n\t\tr.NamedGroups[i].Perm = rb.Get16()\n\t}\n}\n\nfunc EmptyRule() *Rule {\n\treturn &Rule{\n\t\tOwner: 0xFFFF,\n\t\tGroup: 0xFFFF,\n\t\tOther: 0xFFFF,\n\t\tMask: 0xFFFF,\n\t}\n}\n\nfunc (r *Rule) IsEmpty() bool {\n\treturn len(r.NamedUsers)+len(r.NamedGroups) == 0 &&\n\t\tr.Owner&r.Group&r.Other&r.Mask == 0xFFFF\n}\n\n// IsMinimal just like normal permission\nfunc (r *Rule) IsMinimal() bool {\n\treturn len(r.NamedGroups)+len(r.NamedUsers) == 0 && r.Mask == 0xFFFF\n}\n\nfunc (r *Rule) IsEqual(other *Rule) bool {\n\tif r.Owner != other.Owner || r.Group != other.Group || r.Mask != other.Mask || r.Other != other.Other {\n\t\treturn false\n\t}\n\n\treturn r.NamedUsers.IsEqual(&other.NamedUsers) &&\n\t\tr.NamedGroups.IsEqual(&other.NamedGroups)\n}\n\n// InheritPerms from normal permission\nfunc (r *Rule) InheritPerms(mode uint16) {\n\tif r.Owner == 0xFFFF {\n\t\tr.Owner = (mode >> 6) & 7\n\t}\n\tif r.Group == 0xFFFF {\n\t\tr.Group = (mode >> 3) & 7\n\t}\n\tif r.Other == 0xFFFF {\n\t\tr.Other = mode & 7\n\t}\n}\n\nfunc (r *Rule) SetMode(mode uint16) {\n\tr.Owner &= 0xFFF8\n\tr.Owner |= (mode >> 6) & 7\n\n\tif r.IsMinimal() {\n\t\tr.Group &= 0xFFF8\n\t\tr.Group |= (mode >> 3) & 7\n\t} else {\n\t\tr.Mask &= 0xFFF8\n\t\tr.Mask |= (mode >> 3) & 7\n\t}\n\tr.Other &= 0xFFF8\n\tr.Other |= mode & 7\n}\n\nfunc (r *Rule) GetMode() uint16 {\n\tif r.IsMinimal() {\n\t\treturn ((r.Owner & 7) << 6) | ((r.Group & 7) << 3) | (r.Other & 7)\n\t}\n\treturn ((r.Owner & 7) << 6) | ((r.Mask & 7) << 3) | (r.Other & 7)\n}\n\n// ChildAccessACL return the child node access acl with this default acl\nfunc (r *Rule) ChildAccessACL(mode uint16) *Rule {\n\tcRule := &Rule{}\n\tcRule.Owner = (mode >> 6) & 7 & r.Owner\n\tcRule.Mask = (mode >> 3) & 7 & r.Mask\n\tcRule.Other = mode & 7 & r.Other\n\n\tcRule.Group = r.Group\n\tcRule.NamedUsers = r.NamedUsers\n\tcRule.NamedGroups = r.NamedGroups\n\treturn cRule\n}\n\nvar crc32c = crc32.MakeTable(crc32.Castagnoli)\n\nfunc (r *Rule) Checksum() uint32 {\n\treturn crc32.Checksum(r.Encode(), crc32c)\n}\n\nfunc (r *Rule) CanAccess(uid uint32, gids []uint32, fUid, fGid uint32, mMask uint8) bool {\n\tif uid == fUid {\n\t\treturn uint8(r.Owner&7)&mMask == mMask\n\t}\n\tfor _, nUser := range r.NamedUsers {\n\t\tif uid == nUser.Id {\n\t\t\treturn uint8(nUser.Perm&r.Mask&7)&mMask == mMask\n\t\t}\n\t}\n\n\tisGrpMatched := false\n\tfor _, gid := range gids {\n\t\tif gid == fGid {\n\t\t\tif uint8(r.Group&r.Mask&7)&mMask == mMask {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tisGrpMatched = true\n\t\t}\n\t}\n\tfor _, gid := range gids {\n\t\tfor _, nGrp := range r.NamedGroups {\n\t\t\tif gid == nGrp.Id {\n\t\t\t\tif uint8(nGrp.Perm&r.Mask&7)&mMask == mMask {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t\tisGrpMatched = true\n\t\t\t}\n\t\t}\n\t}\n\tif isGrpMatched {\n\t\treturn false\n\t}\n\n\treturn uint8(r.Other&7)&mMask == mMask\n}\n\nconst (\n\tTypeNone = iota\n\tTypeAccess\n\tTypeDefault\n)\n" }, { "path": "pkg/acl/cache.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage acl\n\nimport (\n\t\"sync\"\n)\n\nconst None = 0\n\n// Cache all rules\n// - cache all rules when meta init.\n// - on getfacl failure, read and cache rule from meta.\n// - on setfacl success, read and cache all missed rules from meta. (considered as a low-frequency operation)\n// - concurrent mounts may result in duplicate rules.\ntype Cache interface {\n\tPut(id uint32, r *Rule)\n\tGet(id uint32) *Rule\n\tGetAll() map[uint32]*Rule\n\tGetId(r *Rule) uint32\n\tSize() int\n\tGetMissIds() []uint32\n\tClear()\n}\n\nfunc NewCache() Cache {\n\treturn &cache{\n\t\tlock: sync.RWMutex{},\n\t\tmaxId: None,\n\t\tid2Rule: make(map[uint32]*Rule),\n\t\tcksum2Id: make(map[uint32][]uint32),\n\t}\n}\n\ntype cache struct {\n\tlock sync.RWMutex\n\tmaxId uint32\n\tid2Rule map[uint32]*Rule\n\tcksum2Id map[uint32][]uint32\n}\n\nfunc (c *cache) GetAll() map[uint32]*Rule {\n\tc.lock.RLock()\n\tdefer c.lock.RUnlock()\n\n\tcpy := make(map[uint32]*Rule, len(c.id2Rule))\n\tfor id, r := range c.id2Rule {\n\t\tcpy[id] = r\n\t}\n\treturn cpy\n}\n\nfunc (c *cache) Clear() {\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\tc.maxId = None\n\tc.id2Rule = make(map[uint32]*Rule)\n\tc.cksum2Id = make(map[uint32][]uint32)\n}\n\n// GetMissIds return all miss ids from 1 to c.maxId\nfunc (c *cache) GetMissIds() []uint32 {\n\tc.lock.RLock()\n\tdefer c.lock.RUnlock()\n\n\tif uint32(len(c.id2Rule)) == c.maxId {\n\t\treturn nil\n\t}\n\n\tn := c.maxId + 1\n\tvar ret []uint32\n\tfor i := uint32(1); i < n; i++ {\n\t\tif _, ok := c.id2Rule[i]; !ok {\n\t\t\tret = append(ret, i)\n\t\t}\n\t}\n\treturn ret\n}\n\nfunc (c *cache) Size() int {\n\tc.lock.RLock()\n\tdefer c.lock.RUnlock()\n\treturn len(c.id2Rule)\n}\n\nfunc (c *cache) Get(id uint32) *Rule {\n\tc.lock.RLock()\n\tdefer c.lock.RUnlock()\n\tif r, ok := c.id2Rule[id]; ok {\n\t\treturn r\n\t}\n\treturn nil\n}\n\nfunc (c *cache) Put(id uint32, r *Rule) {\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\n\tif _, ok := c.id2Rule[id]; ok {\n\t\treturn\n\t}\n\n\tif id > c.maxId {\n\t\tc.maxId = id\n\t}\n\n\tc.id2Rule[id] = r\n\n\t// empty slot\n\tif r == nil {\n\t\treturn\n\t}\n\n\tcksum := r.Checksum()\n\tc.cksum2Id[cksum] = append(c.cksum2Id[cksum], id)\n}\n\nfunc (c *cache) GetId(r *Rule) uint32 {\n\tif r == nil {\n\t\treturn None\n\t}\n\n\tc.lock.RLock()\n\tdefer c.lock.RUnlock()\n\n\tif ids, ok := c.cksum2Id[r.Checksum()]; ok {\n\t\tfor _, id := range ids {\n\t\t\tif r.IsEqual(c.id2Rule[id]) {\n\t\t\t\treturn id\n\t\t\t}\n\t\t}\n\t}\n\treturn None\n}\n" }, { "path": "pkg/acl/cache_test.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage acl\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestCache(t *testing.T) {\n\trule := &Rule{\n\t\tOwner: 6,\n\t\tGroup: 4,\n\t\tMask: 4,\n\t\tOther: 4,\n\t\tNamedUsers: Entries{\n\t\t\t{\n\t\t\t\tId: 2,\n\t\t\t\tPerm: 2,\n\t\t\t},\n\t\t\t{\n\t\t\t\tId: 1,\n\t\t\t\tPerm: 1,\n\t\t\t},\n\t\t},\n\t\tNamedGroups: Entries{\n\t\t\t{\n\t\t\t\tId: 4,\n\t\t\t\tPerm: 4,\n\t\t\t},\n\t\t\t{\n\t\t\t\tId: 3,\n\t\t\t\tPerm: 3,\n\t\t\t},\n\t\t},\n\t}\n\n\tc := NewCache()\n\tc.Put(1, rule)\n\tc.Put(2, rule)\n\tassert.True(t, rule.IsEqual(c.Get(1)))\n\tassert.True(t, rule.IsEqual(c.Get(2)))\n\tassert.Equal(t, uint32(1), c.GetId(rule))\n\n\trule2 := &Rule{}\n\t*rule2 = *rule\n\trule2.Owner = 4\n\n\tc.Put(3, rule2)\n\tassert.Equal(t, uint32(3), c.GetId(rule2))\n\n\tc.Put(8, rule2)\n\tassert.Equal(t, []uint32{4, 5, 6, 7}, c.GetMissIds())\n\n\tassert.NotPanics(t, func() {\n\t\tc.Put(10, nil)\n\t})\n}\n" }, { "path": "pkg/chunk/cache_eviction.go", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"container/heap\"\n\t\"fmt\"\n\t\"math\"\n\t\"time\"\n)\n\nconst (\n\tEvictionNone = \"none\"\n\tEviction2Random = \"2-random\"\n\tEvictionLRU = \"lru\"\n)\n\nconst notInLru = math.MinInt // to trigger panic when misused\n\ntype cacheItem struct {\n\tsize int32\n\tatime uint32\n}\n\ntype KeyIndex interface {\n\tname() string\n\tadd(key cacheKey, item cacheItem)\n\t// remove removes key, staging blocks will not be removed unless explicitly requested\n\tremove(key cacheKey, staging bool) *cacheItem\n\tget(key cacheKey) *cacheItem\n\tpeekAtime(key cacheKey) uint32\n\tlen() int\n\treset() KeyIndex\n\t// randomIter iterates over all items randomly\n\trandomIter() func(yield func(key cacheKey, item cacheItem) bool)\n\t// evictionIter evicts items based on different evict policies, yielding each evicted item\n\tevictionIter() func(yield func(key cacheKey, item cacheItem) bool)\n}\n\nfunc NewKeyIndex(config *Config) (KeyIndex, error) {\n\tswitch config.CacheEviction {\n\tcase EvictionNone:\n\t\treturn &noneEviction{keys: make(map[cacheKey]cacheItem)}, nil\n\tcase Eviction2Random:\n\t\treturn &randomEviction{\n\t\t\tnoneEviction: noneEviction{keys: make(map[cacheKey]cacheItem)},\n\t\t\tcacheExpire: config.CacheExpire,\n\t\t}, nil\n\tcase EvictionLRU:\n\t\treturn &lruEviction{\n\t\t\tkeys: make(map[cacheKey]*lruItem),\n\t\t\tlruHeap: atimeHeap{},\n\t\t}, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unknown cache eviction policy: %q\", config.CacheEviction)\n\t}\n}\n\n// noneEviction is a policy that does nothing.\ntype noneEviction struct {\n\tkeys map[cacheKey]cacheItem\n}\n\nfunc (p *noneEviction) name() string {\n\treturn EvictionNone\n}\n\nfunc (p *noneEviction) add(key cacheKey, item cacheItem) {\n\tp.keys[key] = item\n}\n\nfunc (p *noneEviction) remove(key cacheKey, staging bool) *cacheItem {\n\titem, ok := p.keys[key]\n\tif !ok {\n\t\treturn nil\n\t}\n\tif item.size < 0 && !staging {\n\t\treturn nil\n\t}\n\tdelete(p.keys, key)\n\treturn &item\n}\n\nfunc (p *noneEviction) get(key cacheKey) *cacheItem {\n\tif iter, ok := p.keys[key]; ok {\n\t\t// update atime\n\t\tp.keys[key] = cacheItem{iter.size, uint32(time.Now().Unix())}\n\t\treturn &iter\n\t}\n\treturn nil\n}\n\nfunc (p *noneEviction) peekAtime(key cacheKey) uint32 {\n\treturn p.keys[key].atime\n}\n\nfunc (p *noneEviction) len() int {\n\treturn len(p.keys)\n}\n\nfunc (p *noneEviction) reset() KeyIndex {\n\tsnap := &noneEviction{keys: p.keys}\n\tp.keys = make(map[cacheKey]cacheItem, len(p.keys))\n\treturn snap\n}\n\nfunc (p *noneEviction) randomIter() func(yield func(key cacheKey, item cacheItem) bool) {\n\treturn func(yield func(key cacheKey, item cacheItem) bool) {\n\t\tfor k, v := range p.keys {\n\t\t\tif !yield(k, v) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (p *noneEviction) evictionIter() func(yield func(key cacheKey, item cacheItem) bool) {\n\tpanic(\"not implemented for \" + p.name())\n}\n\n// randomEviction evicts items randomly.\ntype randomEviction struct {\n\tnoneEviction\n\tcacheExpire time.Duration\n}\n\nfunc (p *randomEviction) name() string {\n\treturn Eviction2Random\n}\n\nfunc (p *randomEviction) reset() KeyIndex {\n\tsnap := &randomEviction{\n\t\tnoneEviction: noneEviction{keys: p.keys},\n\t\tcacheExpire: p.cacheExpire,\n\t}\n\tp.keys = make(map[cacheKey]cacheItem, len(p.keys))\n\treturn snap\n}\n\nfunc (p *randomEviction) evictionIter() func(yield func(key cacheKey, item cacheItem) bool) {\n\treturn func(yield func(key cacheKey, item cacheItem) bool) {\n\t\tvar cnt int\n\t\tvar lastK cacheKey\n\t\tvar lastValue cacheItem\n\t\tvar now = uint32(time.Now().Unix())\n\t\tvar cutoff = now - uint32(p.cacheExpire/time.Second)\n\t\tfor k, value := range p.keys {\n\t\t\tif value.size < 0 {\n\t\t\t\tcontinue // staging\n\t\t\t}\n\t\t\tif p.cacheExpire > 0 && value.atime < cutoff {\n\t\t\t\tlastK = k\n\t\t\t\tlastValue = value\n\t\t\t\tcnt++\n\t\t\t} else if cnt == 0 || lastValue.atime > value.atime {\n\t\t\t\tlastK = k\n\t\t\t\tlastValue = value\n\t\t\t}\n\t\t\tcnt++\n\t\t\tif cnt > 1 {\n\t\t\t\tdelete(p.keys, lastK)\n\t\t\t\tif !yield(lastK, lastValue) {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tcnt = 0\n\t\t\t}\n\t\t}\n\t}\n}\n\ntype lruItem struct {\n\tcacheItem\n\tpos int // Item position in lru heap, needed for updates\n}\n\n// A min-heap based on atime for cache eviction\ntype atimeHeap []heapItem\n\ntype heapItem struct {\n\t*lruItem\n\tkey *cacheKey // key to cacheItem\n}\n\nfunc (h atimeHeap) Len() int { return len(h) }\n\nfunc (h atimeHeap) Less(i, j int) bool { // min-heap\n\tif h[i].atime != h[j].atime {\n\t\treturn h[i].atime < h[j].atime\n\t}\n\tif h[i].size != h[j].size {\n\t\treturn h[i].size > h[j].size // prefer deleting larger blocks\n\t}\n\treturn h[i].key.id < h[j].key.id\n}\n\nfunc (h atimeHeap) Swap(i, j int) {\n\th[i], h[j] = h[j], h[i]\n\th[i].pos = i\n\th[j].pos = j\n}\n\nfunc (h *atimeHeap) Push(x any) {\n\titem := x.(heapItem)\n\titem.pos = len(*h)\n\t*h = append(*h, item)\n}\n\nfunc (h *atimeHeap) Pop() any {\n\told := *h\n\tn := len(old)\n\titem := old[n-1]\n\titem.pos = notInLru\n\t*h = old[0 : n-1]\n\treturn item\n}\n\n// lruEviction evicts items based on least recent use (atime).\ntype lruEviction struct {\n\tkeys map[cacheKey]*lruItem\n\tlruHeap atimeHeap\n}\n\nfunc (p *lruEviction) name() string {\n\treturn EvictionLRU\n}\n\nfunc (p *lruEviction) add(key cacheKey, item cacheItem) {\n\tif iter, ok := p.keys[key]; !ok {\n\t\titer = &lruItem{cacheItem: item, pos: notInLru}\n\t\tp.keys[key] = iter\n\t\tif iter.size > 0 { // don't add staging blocks to lru as they should not be evicted in `cleanupFull`\n\t\t\theap.Push(&p.lruHeap, heapItem{iter, &key})\n\t\t}\n\t} else {\n\t\titer.cacheItem = item\n\t\tif iter.pos == notInLru {\n\t\t\tif iter.size > 0 {\n\t\t\t\theap.Push(&p.lruHeap, heapItem{iter, &key})\n\t\t\t}\n\t\t} else {\n\t\t\theap.Fix(&p.lruHeap, iter.pos)\n\t\t}\n\t}\n}\n\nfunc (p *lruEviction) remove(key cacheKey, staging bool) *cacheItem {\n\titem, ok := p.keys[key]\n\tif !ok {\n\t\treturn nil\n\t}\n\tif item.size < 0 && !staging {\n\t\treturn nil\n\t}\n\tdelete(p.keys, key)\n\tif item.pos != notInLru {\n\t\theap.Remove(&p.lruHeap, item.pos)\n\t}\n\treturn &item.cacheItem\n}\n\nfunc (p *lruEviction) get(key cacheKey) *cacheItem {\n\tif iter, ok := p.keys[key]; ok {\n\t\t// update atime\n\t\titer.atime = uint32(time.Now().Unix())\n\t\tif iter.pos != notInLru {\n\t\t\theap.Fix(&p.lruHeap, iter.pos)\n\t\t}\n\t\treturn &iter.cacheItem\n\t}\n\treturn nil\n}\n\nfunc (p *lruEviction) peekAtime(key cacheKey) uint32 {\n\tif item, ok := p.keys[key]; ok {\n\t\treturn item.cacheItem.atime\n\t}\n\treturn 0\n}\n\nfunc (p *lruEviction) len() int {\n\treturn len(p.keys)\n}\n\nfunc (p *lruEviction) reset() KeyIndex {\n\tsnap := &lruEviction{\n\t\tkeys: p.keys,\n\t\tlruHeap: p.lruHeap,\n\t}\n\tp.keys = make(map[cacheKey]*lruItem, len(p.keys))\n\tp.lruHeap = make(atimeHeap, 0, len(p.lruHeap))\n\treturn snap\n}\n\nfunc (p *lruEviction) randomIter() func(yield func(key cacheKey, item cacheItem) bool) {\n\treturn func(yield func(key cacheKey, item cacheItem) bool) {\n\t\tfor k, v := range p.keys {\n\t\t\tif !yield(k, v.cacheItem) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (p *lruEviction) evictionIter() func(yield func(key cacheKey, item cacheItem) bool) {\n\treturn func(yield func(key cacheKey, item cacheItem) bool) {\n\t\tfor p.lruHeap.Len() > 0 {\n\t\t\titem := heap.Pop(&p.lruHeap).(heapItem)\n\t\t\tif item.size < 0 {\n\t\t\t\tlogger.Warnf(\"Got a staging block in LRU: %s\", item.key) // should not happen\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdelete(p.keys, *item.key)\n\t\t\tif !yield(*item.key, item.lruItem.cacheItem) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// nolint:unused\nfunc (p *lruEviction) verifyHeap() bool {\n\tcacheKeys := 0\n\tfor k, v := range p.keys {\n\t\tif v.size > 0 {\n\t\t\tcacheKeys += 1\n\t\t} else if v.pos != notInLru {\n\t\t\tlogger.Warnf(\"Staging block %s has size %d but index %d in lruHeap\", k, v.size, v.pos)\n\t\t\treturn false\n\t\t}\n\t}\n\tif p.lruHeap.Len() != cacheKeys {\n\t\tlogger.Warnf(\"atime heap length %d does not match keys length %d\", p.lruHeap.Len(), len(p.keys))\n\t\treturn false\n\t}\n\tfor i, item := range p.lruHeap {\n\t\tif item.pos != i {\n\t\t\tlogger.Warnf(\"atime heap item %d index %d does not match its position %d\", i, item.pos, i)\n\t\t\treturn false\n\t\t}\n\t\tif it, ok := p.keys[*item.key]; !ok {\n\t\t\tlogger.Warnf(\"heap item %d key %s not found in keys map\", i, item.key)\n\t\t\treturn false\n\t\t} else if it.cacheItem != item.cacheItem {\n\t\t\tlogger.Warnf(\"heap item %d key %s does not match cacheItem in keys map\", i, item.key)\n\t\t\treturn false\n\t\t}\n\t}\n\t// Also validate the min-heap property based on atime\n\tn := p.lruHeap.Len()\n\tfor i := 0; i < n/2; i++ {\n\t\tleft := 2*i + 1\n\t\tright := 2*i + 2\n\t\tif left < n && p.lruHeap[i].atime > p.lruHeap[left].atime {\n\t\t\tlogger.Warnf(\"heap property violated: parent atime %d > left child atime %d at index %d\", p.lruHeap[i].atime, p.lruHeap[left].atime, i)\n\t\t\treturn false\n\t\t}\n\t\tif right < n && p.lruHeap[i].atime > p.lruHeap[right].atime {\n\t\t\tlogger.Warnf(\"heap property violated: parent atime %d > right child atime %d at index %d\", p.lruHeap[i].atime, p.lruHeap[right].atime, i)\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n" }, { "path": "pkg/chunk/cached_store.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/compress\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juju/ratelimit\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nconst chunkSize = 1 << 26 // 64M\nconst pageSize = 1 << 16 // 64K\nconst SlowRequest = time.Second * time.Duration(10)\n\nvar (\n\tlogger = utils.GetLogger(\"juicefs\")\n)\n\ntype pendingItem struct {\n\tkey string\n\tfpath string // full path of local file corresponding to the key\n\tts time.Time // timestamp when this item is added\n\tuploading atomic.Bool\n}\n\n// slice for read and remove\ntype rSlice struct {\n\tid uint64\n\tlength int\n\tstore *cachedStore\n}\n\nfunc sliceForRead(id uint64, length int, store *cachedStore) *rSlice {\n\treturn &rSlice{id, length, store}\n}\n\nfunc (s *rSlice) blockSize(indx int) int {\n\tbsize := s.length - indx*s.store.conf.BlockSize\n\tif bsize > s.store.conf.BlockSize {\n\t\tbsize = s.store.conf.BlockSize\n\t}\n\treturn bsize\n}\n\nfunc (s *rSlice) key(indx int) string {\n\tif s.store.conf.HashPrefix {\n\t\treturn fmt.Sprintf(\"chunks/%02X/%v/%v_%v_%v\", s.id%256, s.id/1000/1000, s.id, indx, s.blockSize(indx))\n\t}\n\treturn fmt.Sprintf(\"chunks/%v/%v/%v_%v_%v\", s.id/1000/1000, s.id/1000, s.id, indx, s.blockSize(indx))\n}\n\nfunc (s *rSlice) index(off int) int {\n\treturn off / s.store.conf.BlockSize\n}\n\nfunc (s *rSlice) keys() []string {\n\tif s.length <= 0 {\n\t\treturn nil\n\t}\n\tlastIndx := (s.length - 1) / s.store.conf.BlockSize\n\tkeys := make([]string, lastIndx+1)\n\tfor i := 0; i <= lastIndx; i++ {\n\t\tkeys[i] = s.key(i)\n\t}\n\treturn keys\n}\n\nfunc (s *rSlice) ReadAt(ctx context.Context, page *Page, off int) (n int, err error) {\n\tp := page.Data\n\tif len(p) == 0 {\n\t\treturn 0, nil\n\t}\n\tif off >= s.length {\n\t\treturn 0, io.EOF\n\t}\n\n\tindx := s.index(off)\n\tboff := off % s.store.conf.BlockSize\n\tblockSize := s.blockSize(indx)\n\tif boff+len(p) > blockSize {\n\t\t// read beyond current page\n\t\tvar got int\n\t\tfor got < len(p) {\n\t\t\t// aligned to current page\n\t\t\tl := min(len(p)-got, s.blockSize(s.index(off))-off%s.store.conf.BlockSize)\n\t\t\tpp := page.Slice(got, l)\n\t\t\tn, err = s.ReadAt(ctx, pp, off)\n\t\t\tpp.Release()\n\t\t\tif err != nil {\n\t\t\t\treturn got + n, err\n\t\t\t}\n\t\t\tif n == 0 {\n\t\t\t\treturn got, io.EOF\n\t\t\t}\n\t\t\tgot += n\n\t\t\toff += n\n\t\t}\n\t\treturn got, nil\n\t}\n\n\tkey := s.key(indx)\n\tif s.store.conf.CacheEnabled() {\n\t\tstart := time.Now()\n\t\tr, err := s.store.bcache.load(key)\n\t\tif err == nil {\n\t\t\tn, err = r.ReadAt(p, int64(boff))\n\t\t\tif !s.store.conf.OSCache {\n\t\t\t\tdropOSCache(r)\n\t\t\t}\n\t\t\t_ = r.Close()\n\t\t\tif err == nil {\n\t\t\t\ts.store.cacheHits.Add(1)\n\t\t\t\ts.store.cacheHitBytes.Add(float64(n))\n\t\t\t\ts.store.cacheReadHist.Observe(time.Since(start).Seconds())\n\t\t\t\treturn n, nil\n\t\t\t}\n\t\t\tlogger.Warnf(\"remove partial cached block %s: %d %s\", key, n, err)\n\t\t\ts.store.bcache.remove(key, false)\n\t\t}\n\t}\n\n\ts.store.cacheMiss.Add(1)\n\ts.store.cacheMissBytes.Add(float64(len(p)))\n\n\tif s.store.seekable &&\n\t\t(!s.store.conf.CacheEnabled() || (boff > 0 && len(p) <= blockSize/4)) {\n\t\tn, err = s.store.loadRange(ctx, key, page, boff)\n\t\tif err == nil || !errors.Is(err, errTryFullRead) {\n\t\t\treturn n, err\n\t\t}\n\t}\n\n\tblock, err := s.store.group.Execute(key, func() (*Page, error) {\n\t\ttmp := page\n\t\tif boff > 0 || len(p) < blockSize {\n\t\t\ttmp = NewOffPage(blockSize)\n\t\t} else {\n\t\t\ttmp.Acquire()\n\t\t}\n\t\terr = s.store.load(ctx, key, tmp, s.store.shouldCache(blockSize), false)\n\t\treturn tmp, err\n\t})\n\tdefer block.Release()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif block != page {\n\t\tcopy(p, block.Data[boff:])\n\t}\n\treturn len(p), nil\n}\n\nfunc (s *rSlice) delete(indx int) error {\n\tkey := s.key(indx)\n\treturn s.store.delete(key)\n}\n\nfunc (s *rSlice) Remove() error {\n\tif s.length == 0 {\n\t\t// no block\n\t\treturn nil\n\t}\n\n\tlastIndx := (s.length - 1) / s.store.conf.BlockSize\n\tfor i := 0; i <= lastIndx; i++ {\n\t\t// there could be multiple clients try to remove the same chunk in the same time,\n\t\t// any of them should succeed if any blocks is removed\n\t\tkey := s.key(i)\n\t\ts.store.removePending(key)\n\t\ts.store.bcache.remove(key, true)\n\t}\n\n\tvar err error\n\tfor i := 0; i <= lastIndx; i++ {\n\t\tif e := s.delete(i); e != nil {\n\t\t\terr = e\n\t\t}\n\t}\n\treturn err\n}\n\nvar pagePool = make(chan *Page, 128)\n\nfunc allocPage(sz int) *Page {\n\tif sz != pageSize {\n\t\treturn NewOffPage(sz)\n\t}\n\tselect {\n\tcase p := <-pagePool:\n\t\treturn p\n\tdefault:\n\t\treturn NewOffPage(pageSize)\n\t}\n}\n\nfunc freePage(p *Page) {\n\tif cap(p.Data) != pageSize {\n\t\tp.Release()\n\t\treturn\n\t}\n\tselect {\n\tcase pagePool <- p:\n\tdefault:\n\t\tp.Release()\n\t}\n}\n\n// slice for write only\ntype wSlice struct {\n\trSlice\n\tpages [][]*Page\n\tuploaded int\n\terrors chan error\n\tuploadError error\n\tpendings int\n\twriteback bool\n}\n\nfunc sliceForWrite(id uint64, store *cachedStore) *wSlice {\n\treturn &wSlice{\n\t\trSlice: rSlice{id, 0, store},\n\t\tpages: make([][]*Page, chunkSize/store.conf.BlockSize),\n\t\terrors: make(chan error, chunkSize/store.conf.BlockSize),\n\t\twriteback: store.conf.Writeback,\n\t}\n}\n\nfunc (s *wSlice) SetID(id uint64) {\n\ts.id = id\n}\n\nfunc (s *wSlice) SetWriteback(enabled bool) {\n\ts.writeback = enabled\n}\n\nfunc (s *wSlice) WriteAt(p []byte, off int64) (n int, err error) {\n\tif int(off)+len(p) > chunkSize {\n\t\treturn 0, fmt.Errorf(\"write out of chunk boudary: %d > %d\", int(off)+len(p), chunkSize)\n\t}\n\tif off < int64(s.uploaded) {\n\t\treturn 0, fmt.Errorf(\"Cannot overwrite uploaded block: %d < %d\", off, s.uploaded)\n\t}\n\n\t// Fill previous blocks with zeros\n\tif s.length < int(off) {\n\t\tzeros := make([]byte, int(off)-s.length)\n\t\t_, _ = s.WriteAt(zeros, int64(s.length))\n\t}\n\n\tfor n < len(p) {\n\t\tindx := s.index(int(off) + n)\n\t\tboff := (int(off) + n) % s.store.conf.BlockSize\n\t\tvar bs = pageSize\n\t\tif indx > 0 || bs > s.store.conf.BlockSize {\n\t\t\tbs = s.store.conf.BlockSize\n\t\t}\n\t\tbi := boff / bs\n\t\tbo := boff % bs\n\t\tvar page *Page\n\t\tif bi < len(s.pages[indx]) {\n\t\t\tpage = s.pages[indx][bi]\n\t\t} else {\n\t\t\tpage = allocPage(bs)\n\t\t\tpage.Data = page.Data[:0]\n\t\t\ts.pages[indx] = append(s.pages[indx], page)\n\t\t}\n\t\tleft := len(p) - n\n\t\tif bo+left > bs {\n\t\t\tpage.Data = page.Data[:bs]\n\t\t} else if len(page.Data) < bo+left {\n\t\t\tpage.Data = page.Data[:bo+left]\n\t\t}\n\t\tn += copy(page.Data[bo:], p[n:])\n\t}\n\tif int(off)+n > s.length {\n\t\ts.length = int(off) + n\n\t}\n\treturn n, nil\n}\n\nfunc (store *cachedStore) put(key string, p *Page) error {\n\tif store.upLimit != nil {\n\t\tstore.upLimit.Wait(int64(len(p.Data)))\n\t}\n\tp.Acquire()\n\tvar (\n\t\treqID string\n\t\tsc = object.DefaultStorageClass\n\t)\n\treturn utils.WithTimeout(context.TODO(), func(ctx context.Context) error {\n\t\tdefer p.Release()\n\t\tst := time.Now()\n\t\terr := store.storage.Put(ctx, key, bytes.NewReader(p.Data), object.WithRequestID(&reqID), object.WithStorageClass(&sc))\n\t\tused := time.Since(st)\n\t\tlogRequest(\"PUT\", key, \"\", reqID, err, used)\n\t\tstore.objectDataBytes.WithLabelValues(\"PUT\", sc).Add(float64(len(p.Data)))\n\t\tstore.objectReqsHistogram.WithLabelValues(\"PUT\", sc).Observe(used.Seconds())\n\t\tif err != nil {\n\t\t\tstore.objectReqErrors.Add(1)\n\t\t}\n\t\treturn err\n\t}, store.conf.PutTimeout)\n}\n\nfunc (store *cachedStore) delete(key string) error {\n\tst := time.Now()\n\tvar reqID string\n\terr := utils.WithTimeout(context.TODO(), func(ctx context.Context) error {\n\t\treturn store.storage.Delete(ctx, key, object.WithRequestID(&reqID))\n\t}, store.conf.PutTimeout)\n\tused := time.Since(st)\n\tif err != nil && (strings.Contains(err.Error(), \"NoSuchKey\") ||\n\t\tstrings.Contains(err.Error(), \"not found\") ||\n\t\tstrings.Contains(err.Error(), \"No such file\")) {\n\t\terr = nil\n\t}\n\tlogRequest(\"DELETE\", key, \"\", reqID, err, used)\n\tstore.objectReqsHistogram.WithLabelValues(\"DELETE\", \"\").Observe(used.Seconds())\n\tif err != nil {\n\t\tstore.objectReqErrors.Add(1)\n\t}\n\treturn err\n}\n\nfunc (store *cachedStore) upload(key string, block *Page, s *wSlice) error {\n\tsync := s != nil\n\tblen := len(block.Data)\n\tbufSize := store.compressor.CompressBound(blen)\n\tvar buf *Page\n\tif bufSize > blen {\n\t\tbuf = NewOffPage(bufSize)\n\t} else {\n\t\tbuf = block\n\t\tbuf.Acquire()\n\t}\n\tdefer buf.Release()\n\tif sync && (blen < store.conf.BlockSize || store.conf.CacheLargeWrite) {\n\t\t// block will be freed after written into disk\n\t\tstore.bcache.cache(key, block, false, false)\n\t}\n\tn, err := store.compressor.Compress(buf.Data, block.Data)\n\tblock.Release()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Compress block key %s: %s\", key, err)\n\t}\n\tbuf.Data = buf.Data[:n]\n\n\ttry, max := 0, 3\n\tif sync {\n\t\tmax = store.conf.MaxRetries + 1\n\t}\n\tfor ; try < max; try++ {\n\t\ttime.Sleep(time.Second * time.Duration(try*try))\n\t\tif s != nil && s.uploadError != nil {\n\t\t\terr = fmt.Errorf(\"(cancelled) upload block %s: %s (after %d tries)\", key, err, try)\n\t\t\tbreak\n\t\t}\n\t\tif err = store.put(key, buf); err == nil {\n\t\t\tbreak\n\t\t}\n\t\tlogger.Debugf(\"Upload %s: %s (try %d)\", key, err, try+1)\n\t}\n\tif err != nil && try >= max {\n\t\terr = fmt.Errorf(\"(max tries) upload block %s: %s (after %d tries)\", key, err, try)\n\t}\n\treturn err\n}\n\nfunc (s *wSlice) upload(indx int) {\n\tblen := s.blockSize(indx)\n\tkey := s.key(indx)\n\tpages := s.pages[indx]\n\ts.pages[indx] = nil\n\ts.pendings++\n\n\tgo func() {\n\t\tvar block *Page\n\t\tvar off int\n\t\tif len(pages) == 1 {\n\t\t\tblock = pages[0]\n\t\t\toff = len(block.Data)\n\t\t} else {\n\t\t\tblock = NewOffPage(blen)\n\t\t\tfor _, b := range pages {\n\t\t\t\toff += copy(block.Data[off:], b.Data)\n\t\t\t\tfreePage(b)\n\t\t\t}\n\t\t}\n\t\tif off != blen {\n\t\t\tpanic(fmt.Sprintf(\"block length does not match: %v != %v\", off, blen))\n\t\t}\n\t\tif s.writeback && blen < s.store.conf.WritebackThresholdSize {\n\t\t\tstagingPath := \"unknown\"\n\t\t\tstageFailed := false\n\t\t\tblock.Acquire()\n\t\t\terr := utils.WithTimeout(context.TODO(), func(context.Context) (err error) { // In case it hangs for more than 5 minutes(see fileWriter.flush), fallback to uploading directly to avoid `EIO`\n\t\t\t\tdefer block.Release()\n\t\t\t\tstagingPath, err = s.store.bcache.stage(key, block.Data)\n\t\t\t\tif err == nil && stageFailed { // upload thread already marked me as failed because of timeout\n\t\t\t\t\t_ = s.store.bcache.removeStage(key)\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}, s.store.conf.PutTimeout)\n\t\t\tif err != nil {\n\t\t\t\tstageFailed = true\n\t\t\t\tif !errors.Is(err, errStageConcurrency) {\n\t\t\t\t\ts.store.stageBlockErrors.Add(1)\n\t\t\t\t\tlogger.Warnf(\"write %s to disk: %s, upload it directly\", key, err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\ts.errors <- nil\n\t\t\t\tif s.store.conf.UploadDelay == 0 && s.store.canUpload() {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase s.store.currentUpload <- struct{}{}:\n\t\t\t\t\t\tdefer func() { <-s.store.currentUpload }()\n\t\t\t\t\t\tif err = s.store.upload(key, block, nil); err == nil {\n\t\t\t\t\t\t\ts.store.bcache.uploaded(key, blen)\n\t\t\t\t\t\t\tif err := s.store.bcache.removeStage(key); err != nil {\n\t\t\t\t\t\t\t\tlogger.Warnf(\"failed to remove stage %s in upload\", stagingPath)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else { // add to delay list and wait for later scanning\n\t\t\t\t\t\t\ts.store.addDelayedStaging(key, stagingPath, time.Now(), false)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tblock.Release()\n\t\t\t\ts.store.addDelayedStaging(key, stagingPath, time.Now(), false)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\ts.store.currentUpload <- struct{}{}\n\t\tdefer func() { <-s.store.currentUpload }()\n\t\ts.errors <- s.store.upload(key, block, s)\n\t}()\n}\n\nfunc (s *wSlice) ID() uint64 {\n\treturn s.id\n}\n\nfunc (s *wSlice) Len() int {\n\treturn s.length\n}\n\nfunc (s *wSlice) FlushTo(offset int) error {\n\tif offset < s.uploaded {\n\t\tpanic(fmt.Sprintf(\"Invalid offset: %d < %d\", offset, s.uploaded))\n\t}\n\tfor i, block := range s.pages {\n\t\tstart := i * s.store.conf.BlockSize\n\t\tend := start + s.store.conf.BlockSize\n\t\tif start >= s.uploaded && end <= offset {\n\t\t\tif block != nil {\n\t\t\t\ts.upload(i)\n\t\t\t}\n\t\t\ts.uploaded = end\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (s *wSlice) Finish(length int) error {\n\tif s.length != length {\n\t\treturn fmt.Errorf(\"Length mismatch: %v != %v\", s.length, length)\n\t}\n\n\tn := (length-1)/s.store.conf.BlockSize + 1\n\tif err := s.FlushTo(n * s.store.conf.BlockSize); err != nil {\n\t\treturn err\n\t}\n\tfor i := 0; i < s.pendings; i++ {\n\t\tif err := <-s.errors; err != nil {\n\t\t\ts.uploadError = err\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (s *wSlice) Abort() {\n\tfor i := range s.pages {\n\t\tfor _, b := range s.pages[i] {\n\t\t\tfreePage(b)\n\t\t}\n\t\ts.pages[i] = nil\n\t}\n\t// delete uploaded blocks\n\ts.length = s.uploaded\n\t_ = s.Remove()\n}\n\n// Config contains options for cachedStore\ntype Config struct {\n\tCacheDir string\n\tCacheMode os.FileMode\n\tCacheSize uint64\n\tCacheItems int64\n\tCacheChecksum string\n\tCacheEviction string\n\tCacheScanInterval time.Duration\n\tCacheExpire time.Duration\n\tOSCache bool\n\tFreeSpace float32\n\tAutoCreate bool\n\tCompress string\n\tMaxUpload int\n\tMaxDownload int\n\tMaxStageWrite int\n\tMaxRetries int\n\tUploadLimit int64 // bytes per second\n\tDownloadLimit int64 // bytes per second\n\tWriteback bool\n\tWritebackThresholdSize int\n\tUploadDelay time.Duration\n\tUploadHours string\n\tHashPrefix bool\n\tBlockSize int\n\tGetTimeout time.Duration\n\tPutTimeout time.Duration\n\tCacheFullBlock bool\n\tCacheLargeWrite bool\n\tBufferSize uint64\n\tReadahead int\n\tPrefetch int\n}\n\nfunc (c *Config) SelfCheck(uuid string) {\n\tif !c.CacheEnabled() {\n\t\tif c.Writeback || c.Prefetch > 0 {\n\t\t\tlogger.Warnf(\"cache-size is 0, writeback and prefetch will be disabled\")\n\t\t\tc.Writeback = false\n\t\t\tc.Prefetch = 0\n\t\t}\n\t\tc.CacheDir = \"memory\"\n\t}\n\tif c.MaxUpload <= 0 {\n\t\tlogger.Warnf(\"max-uploads should be greater than 0, set it to 1\")\n\t\tc.MaxUpload = 1\n\t}\n\tif c.UploadLimit > 0 && int64(c.MaxUpload*c.BlockSize) > c.UploadLimit*int64(c.GetTimeout/time.Second)/2 {\n\t\tlogger.Warnf(\"max-upload %d may exceed bandwidth limit (bw: %d Mbps)\", c.MaxUpload, c.UploadDelay*8>>20)\n\t}\n\tif c.MaxDownload <= 0 {\n\t\tlogger.Warnf(\"max-downloads should be greater than 0, set it to 200\")\n\t\tc.MaxDownload = 200\n\t}\n\tif c.DownloadLimit > 0 && int64(c.MaxDownload*c.BlockSize) > c.DownloadLimit*int64(c.GetTimeout/time.Second)/2 {\n\t\tlogger.Warnf(\"max-download %d may exceed bandwidth limit (bw: %d Mbps)\", c.MaxDownload, (c.DownloadLimit*8)>>20)\n\t}\n\tif c.BufferSize <= 32<<20 {\n\t\tlogger.Warnf(\"buffer-size is too small, setting it to 32 MiB\")\n\t\tc.BufferSize = 32 << 20\n\t}\n\tif c.CacheDir != \"memory\" {\n\t\tds := utils.SplitDir(c.CacheDir)\n\t\tfor i := range ds {\n\t\t\tds[i] = filepath.Join(ds[i], uuid)\n\t\t}\n\t\tc.CacheDir = strings.Join(ds, string(os.PathListSeparator))\n\t\tif cs := []string{CsNone, CsFull, CsShrink, CsExtend}; !utils.StringContains(cs, c.CacheChecksum) {\n\t\t\tlogger.Warnf(\"verify-cache-checksum should be one of %v\", cs)\n\t\t\tc.CacheChecksum = CsExtend\n\t\t}\n\t} else if c.Writeback {\n\t\tlogger.Warnf(\"writeback is not supported in memory cache mode\")\n\t\tc.Writeback = false\n\t}\n\tif c.Writeback {\n\t\tif !c.CacheFullBlock {\n\t\t\tlogger.Warnf(\"cache-partial-only is ineffective for stage blocks with writeback enabled\")\n\t\t}\n\t\tif c.WritebackThresholdSize == 0 {\n\t\t\tc.WritebackThresholdSize = c.BlockSize + 1\n\t\t}\n\t} else {\n\t\tif c.UploadDelay > 0 || c.UploadHours != \"\" {\n\t\t\tlogger.Warnf(\"delayed upload is disabled in non-writeback mode\")\n\t\t\tc.UploadDelay = 0\n\t\t\tc.UploadHours = \"\"\n\t\t}\n\t}\n\tif _, _, err := c.parseHours(); err != nil {\n\t\tlogger.Warnf(\"invalid value (%s) for upload-hours: %s\", c.UploadHours, err)\n\t\tc.UploadHours = \"\"\n\t}\n\tif c.CacheEviction == \"\" {\n\t\tc.CacheEviction = Eviction2Random\n\t} else if c.CacheEviction != Eviction2Random && c.CacheEviction != EvictionNone && c.CacheEviction != EvictionLRU {\n\t\tlogger.Warnf(\"cache-eviction should be one of [%s, %s, %s]\", EvictionNone, Eviction2Random, EvictionLRU)\n\t\tc.CacheEviction = Eviction2Random\n\t}\n\tif c.CacheDir == \"memory\" && c.CacheEviction == EvictionLRU {\n\t\tlogger.Warnf(\"LRU eviction is not supported in memory cache mode yet, setting it to 2-random\")\n\t\tc.CacheEviction = Eviction2Random\n\t}\n\tif c.CacheExpire > 0 && c.CacheExpire < time.Second {\n\t\tlogger.Warnf(\"cache-expire it too short, setting it to 1 second\")\n\t\tc.CacheExpire = time.Second\n\t}\n}\n\nfunc (c *Config) parseHours() (start, end int, err error) {\n\tif c.UploadHours == \"\" {\n\t\treturn\n\t}\n\tsplit := \",\"\n\tif strings.Contains(c.UploadHours, \"-\") {\n\t\tsplit = \"-\"\n\t}\n\tps := strings.Split(c.UploadHours, split)\n\tif len(ps) != 2 {\n\t\terr = errors.New(\"unexpected number of fields\")\n\t\treturn\n\t}\n\tif start, err = strconv.Atoi(ps[0]); err != nil {\n\t\treturn\n\t}\n\tif end, err = strconv.Atoi(ps[1]); err != nil {\n\t\treturn\n\t}\n\tif start < 0 || start > 23 || end < 0 || end > 23 {\n\t\terr = errors.New(\"invalid hour number\")\n\t}\n\treturn\n}\n\nfunc (c *Config) CacheEnabled() bool {\n\treturn c.CacheSize > 0\n}\n\ntype cachedStore struct {\n\tstorage object.ObjectStorage\n\tbcache CacheManager\n\tfetcher *prefetcher\n\tconf Config\n\tgroup *Controller\n\tcurrentUpload chan struct{}\n\tcurrentDownload chan struct{}\n\tpendingCh chan *pendingItem\n\tpendingKeys map[string]*pendingItem\n\tpendingMutex sync.Mutex\n\tstartHour int\n\tendHour int\n\tcompressor compress.Compressor\n\tseekable bool\n\tupLimit *ratelimit.Bucket\n\tdownLimit *ratelimit.Bucket\n\n\tcacheHits prometheus.Counter\n\tcacheMiss prometheus.Counter\n\tcacheHitBytes prometheus.Counter\n\tcacheMissBytes prometheus.Counter\n\tcacheReadHist prometheus.Histogram\n\tobjectReqsHistogram *prometheus.HistogramVec\n\tobjectReqErrors prometheus.Counter\n\tobjectDataBytes *prometheus.CounterVec\n\tstageBlockDelay prometheus.Counter\n\tstageBlockErrors prometheus.Counter\n}\n\nfunc logRequest(typeStr, key, param, reqID string, err error, used time.Duration) {\n\tif used > SlowRequest {\n\t\tlogger.Warnf(\"slow request: %s %s %s(req_id: %q, err: %v, cost: %s)\", typeStr, key, param, reqID, err, used)\n\t} else {\n\t\tlogger.Debugf(\"%s %s %s(req_id: %q, err: %v, cost: %s)\", typeStr, key, param, reqID, err, used)\n\t}\n}\n\nvar errTryFullRead = errors.New(\"try full read\")\n\nfunc (store *cachedStore) loadRange(ctx context.Context, key string, page *Page, off int) (n int, err error) {\n\tp := page.Data\n\tfullPage, err := store.group.TryPiggyback(key)\n\tif fullPage != nil {\n\t\tdefer fullPage.Release()\n\t\tif err == nil { // piggybacked a full read\n\t\t\tn = copy(p, fullPage.Data[off:])\n\t\t\treturn n, nil\n\t\t}\n\t}\n\n\tstore.currentDownload <- struct{}{}\n\tdefer func() { <-store.currentDownload }()\n\tif store.downLimit != nil {\n\t\tstore.downLimit.Wait(int64(len(p)))\n\t}\n\n\tstart := time.Now()\n\tvar (\n\t\treqID string\n\t\tsc = object.DefaultStorageClass\n\t)\n\tpage.Acquire()\n\terr = utils.WithTimeout(ctx, func(cCtx context.Context) error {\n\t\tdefer page.Release()\n\t\tin, err := store.storage.Get(cCtx, key, int64(off), int64(len(p)), object.WithRequestID(&reqID), object.WithStorageClass(&sc))\n\t\tif err == nil {\n\t\t\tn, err = io.ReadFull(in, p)\n\t\t\t_ = in.Close()\n\t\t}\n\t\treturn err\n\t}, store.conf.GetTimeout)\n\n\tused := time.Since(start)\n\tlogRequest(\"GET\", key, fmt.Sprintf(\"RANGE(%d,%d) \", off, len(p)), reqID, err, used)\n\tif errors.Is(err, context.Canceled) {\n\t\treturn 0, err\n\t}\n\tstore.objectDataBytes.WithLabelValues(\"GET\", sc).Add(float64(n))\n\tstore.objectReqsHistogram.WithLabelValues(\"GET\", sc).Observe(used.Seconds())\n\tif err == nil {\n\t\tstore.fetcher.fetch(key)\n\t\treturn n, nil\n\t}\n\tstore.objectReqErrors.Add(1)\n\t// fall back to full read\n\treturn 0, errTryFullRead\n}\n\nfunc (store *cachedStore) load(ctx context.Context, key string, page *Page, cache bool, forceCache bool) (err error) {\n\tdefer func() {\n\t\te := recover()\n\t\tif e != nil {\n\t\t\terr = fmt.Errorf(\"recovered from %s\", e)\n\t\t}\n\t}()\n\tstore.currentDownload <- struct{}{}\n\tdefer func() { <-store.currentDownload }()\n\tneeded := store.compressor.CompressBound(len(page.Data))\n\tcompressed := needed > len(page.Data)\n\t// we don't know the actual size for compressed block\n\tif store.downLimit != nil && !compressed {\n\t\tstore.downLimit.Wait(int64(len(page.Data)))\n\t}\n\tvar (\n\t\tin io.ReadCloser\n\t\tn int\n\t\tp *Page\n\t\treqID string\n\t\tsc = object.DefaultStorageClass\n\t\tstart = time.Now()\n\t)\n\tif compressed {\n\t\tc := NewOffPage(needed)\n\t\tdefer c.Release()\n\t\tp = c\n\t} else {\n\t\tp = page\n\t}\n\tp.Acquire()\n\terr = utils.WithTimeout(ctx, func(cCtx context.Context) error {\n\t\tdefer p.Release()\n\t\t// it will be retried in the upper layer.\n\t\tin, err = store.storage.Get(cCtx, key, 0, -1, object.WithRequestID(&reqID), object.WithStorageClass(&sc))\n\t\tif err == nil {\n\t\t\tn, err = io.ReadFull(in, p.Data)\n\t\t\t_ = in.Close()\n\t\t}\n\t\tif compressed && err == io.ErrUnexpectedEOF {\n\t\t\terr = nil\n\t\t}\n\t\treturn err\n\t}, store.conf.GetTimeout)\n\tif errors.Is(err, context.Canceled) {\n\t\treturn err\n\t}\n\tused := time.Since(start)\n\tlogRequest(\"GET\", key, \"\", reqID, err, used)\n\tif store.downLimit != nil && compressed {\n\t\tstore.downLimit.Wait(int64(n))\n\t}\n\tstore.objectDataBytes.WithLabelValues(\"GET\", sc).Add(float64(n))\n\tstore.objectReqsHistogram.WithLabelValues(\"GET\", sc).Observe(used.Seconds())\n\tif err != nil {\n\t\tstore.objectReqErrors.Add(1)\n\t\treturn fmt.Errorf(\"get %s: %s\", key, err)\n\t}\n\tif compressed {\n\t\tn, err = store.compressor.Decompress(page.Data, p.Data[:n])\n\t}\n\tif err != nil || n < len(page.Data) {\n\t\treturn fmt.Errorf(\"read %s fully: %v (%d < %d) after %s\", key, err, n, len(page.Data), used)\n\t}\n\tif cache {\n\t\tstore.bcache.cache(key, page, forceCache, !store.conf.OSCache)\n\t}\n\treturn nil\n}\n\n// NewCachedStore create a cached store.\nfunc NewCachedStore(storage object.ObjectStorage, config Config, reg prometheus.Registerer) ChunkStore {\n\tcompressor := compress.NewCompressor(config.Compress)\n\tif compressor == nil {\n\t\tlogger.Fatalf(\"unknown compress algorithm: %s\", config.Compress)\n\t}\n\tif config.MaxRetries == 0 {\n\t\tconfig.MaxRetries = 10\n\t}\n\tif config.GetTimeout == 0 {\n\t\tconfig.GetTimeout = time.Second * 60\n\t}\n\tif config.PutTimeout == 0 {\n\t\tconfig.PutTimeout = time.Second * 60\n\t}\n\tstore := &cachedStore{\n\t\tstorage: storage,\n\t\tconf: config,\n\t\tcurrentUpload: make(chan struct{}, config.MaxUpload),\n\t\tcurrentDownload: make(chan struct{}, config.MaxDownload),\n\t\tcompressor: compressor,\n\t\tseekable: compressor.CompressBound(0) == 0,\n\t\tpendingCh: make(chan *pendingItem, 100*config.MaxUpload),\n\t\tpendingKeys: make(map[string]*pendingItem),\n\t\tgroup: NewController(),\n\t}\n\tif config.UploadLimit > 0 {\n\t\t// there are overheads coming from HTTP/TCP/IP\n\t\tstore.upLimit = ratelimit.NewBucketWithRate(float64(config.UploadLimit)*0.85, config.UploadLimit/10)\n\t}\n\tif config.DownloadLimit > 0 {\n\t\tstore.downLimit = ratelimit.NewBucketWithRate(float64(config.DownloadLimit)*0.85, config.DownloadLimit/10)\n\t}\n\tstore.initMetrics()\n\tif store.conf.Writeback {\n\t\tstore.startHour, store.endHour, _ = config.parseHours()\n\t\tif store.startHour != store.endHour {\n\t\t\tlogger.Infof(\"background upload at %d:00 ~ %d:00\", store.startHour, store.endHour)\n\t\t}\n\t}\n\tstore.bcache = newCacheManager(&config, reg, func(key, fpath string, force bool) bool {\n\t\tif fi, err := os.Stat(fpath); err == nil {\n\t\t\treturn store.addDelayedStaging(key, fpath, fi.ModTime(), force)\n\t\t} else {\n\t\t\tlogger.Warnf(\"Stat staging block %s: %s\", fpath, err)\n\t\t\treturn false\n\t\t}\n\t})\n\n\tgo func() {\n\t\tfor {\n\t\t\tif store.bcache.isEmpty() {\n\t\t\t\tlogger.Warn(\"cache store is empty, use memory cache\")\n\t\t\t\tconfig.CacheSize = 100 << 20\n\t\t\t\tconfig.CacheDir = \"memory\"\n\t\t\t\tstore.bcache = newMemStore(&config, store.bcache.getMetrics())\n\t\t\t}\n\t\t\ttime.Sleep(time.Second)\n\t\t}\n\t}()\n\n\tif !config.CacheEnabled() {\n\t\tconfig.Prefetch = 0 // disable prefetch if cache is disabled\n\t}\n\tstore.fetcher = newPrefetcher(config.Prefetch, func(key string) {\n\t\tsize := parseObjOrigSize(key)\n\t\tif size == 0 || size > store.conf.BlockSize {\n\t\t\treturn\n\t\t}\n\t\tp := NewOffPage(size)\n\t\tdefer p.Release()\n\t\tblock, err := store.group.Execute(key, func() (*Page, error) { // dedup requests with full read\n\t\t\tp.Acquire()\n\t\t\terr := store.load(context.TODO(), key, p, false, false) // delay writing cache until singleflight ends to prevent blocking waiters\n\t\t\treturn p, err\n\t\t})\n\t\tdefer block.Release()\n\t\tif err == nil && block == p {\n\t\t\tstore.bcache.cache(key, block, true, !store.conf.OSCache)\n\t\t}\n\t})\n\n\tif store.conf.Writeback {\n\t\tfor i := 0; i < store.conf.MaxUpload; i++ {\n\t\t\tgo store.uploader()\n\t\t}\n\t\tinterval := time.Minute\n\t\tif d := store.conf.UploadDelay; d > 0 {\n\t\t\tif d < time.Minute {\n\t\t\t\tinterval = d\n\t\t\t\tlogger.Warnf(\"delay uploading by %s (this value is too small, and is not recommended)\", d)\n\t\t\t} else {\n\t\t\t\tlogger.Infof(\"delay uploading by %s\", d)\n\t\t\t}\n\t\t}\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\ttime.Sleep(interval)\n\t\t\t\tstore.scanDelayedStaging()\n\t\t\t}\n\t\t}()\n\t}\n\tstore.regMetrics(reg)\n\treturn store\n}\n\nfunc (store *cachedStore) initMetrics() {\n\tstore.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_hits\",\n\t\tHelp: \"read from cached block\",\n\t})\n\tstore.cacheMiss = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_miss\",\n\t\tHelp: \"missed read from cached block\",\n\t})\n\tstore.cacheHitBytes = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_hit_bytes\",\n\t\tHelp: \"read bytes from cached block\",\n\t})\n\tstore.cacheMissBytes = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_miss_bytes\",\n\t\tHelp: \"missed bytes from cached block\",\n\t})\n\tstore.cacheReadHist = prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"blockcache_read_hist_seconds\",\n\t\tHelp: \"read cached block latency distribution\",\n\t\tBuckets: prometheus.ExponentialBuckets(0.00001, 2, 20),\n\t})\n\tstore.objectReqsHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{\n\t\tName: \"object_request_durations_histogram_seconds\",\n\t\tHelp: \"Object requests latency distributions.\",\n\t\tBuckets: prometheus.ExponentialBuckets(0.01, 1.5, 25),\n\t}, []string{\"method\", \"storage_class\"})\n\tstore.objectReqErrors = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"object_request_errors\",\n\t\tHelp: \"failed requests to object store\",\n\t})\n\tstore.objectDataBytes = prometheus.NewCounterVec(prometheus.CounterOpts{\n\t\tName: \"object_request_data_bytes\",\n\t\tHelp: \"Object requests size in bytes.\",\n\t}, []string{\"method\", \"storage_class\"})\n\tstore.stageBlockDelay = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"staging_block_delay_seconds\",\n\t\tHelp: \"Total seconds of delay for staging blocks\",\n\t})\n\tstore.stageBlockErrors = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"staging_block_errors\",\n\t\tHelp: \"Total errors when staging blocks\",\n\t})\n}\n\nfunc (store *cachedStore) regMetrics(reg prometheus.Registerer) {\n\tif reg == nil {\n\t\treturn\n\t}\n\treg.MustRegister(store.cacheHits)\n\treg.MustRegister(store.cacheHitBytes)\n\treg.MustRegister(store.cacheMiss)\n\treg.MustRegister(store.cacheMissBytes)\n\treg.MustRegister(store.cacheReadHist)\n\treg.MustRegister(store.objectReqsHistogram)\n\treg.MustRegister(store.objectReqErrors)\n\treg.MustRegister(store.objectDataBytes)\n\treg.MustRegister(store.stageBlockDelay)\n\treg.MustRegister(store.stageBlockErrors)\n\treg.MustRegister(prometheus.NewGaugeFunc(\n\t\tprometheus.GaugeOpts{\n\t\t\tName: \"blockcache_blocks\",\n\t\t\tHelp: \"number of cached blocks\",\n\t\t},\n\t\tfunc() float64 {\n\t\t\tcnt, _ := store.bcache.stats()\n\t\t\treturn float64(cnt)\n\t\t}))\n\treg.MustRegister(prometheus.NewGaugeFunc(\n\t\tprometheus.GaugeOpts{\n\t\t\tName: \"blockcache_bytes\",\n\t\t\tHelp: \"number of cached bytes\",\n\t\t},\n\t\tfunc() float64 {\n\t\t\t_, used := store.bcache.stats()\n\t\t\treturn float64(used)\n\t\t}))\n\treg.MustRegister(prometheus.NewGaugeFunc(\n\t\tprometheus.GaugeOpts{\n\t\t\tName: \"object_request_uploading\",\n\t\t\tHelp: \"number of uploading requests\",\n\t\t},\n\t\tfunc() float64 {\n\t\t\treturn float64(len(store.currentUpload))\n\t\t}))\n}\n\nfunc (store *cachedStore) shouldCache(size int) bool {\n\treturn store.conf.CacheFullBlock || size < store.conf.BlockSize\n}\n\nfunc parseObjOrigSize(key string) int {\n\tp := strings.LastIndexByte(key, '_')\n\tl, _ := strconv.Atoi(key[p+1:])\n\treturn l\n}\n\nfunc (store *cachedStore) uploadStagingFile(key string, stagingPath string) {\n\tstore.currentUpload <- struct{}{}\n\tdefer func() {\n\t\t<-store.currentUpload\n\t}()\n\n\tstore.pendingMutex.Lock()\n\titem, ok := store.pendingKeys[key]\n\tstore.pendingMutex.Unlock()\n\tif !ok {\n\t\tlogger.Debugf(\"Key %s is not needed, drop it\", key)\n\t\treturn\n\t}\n\tdefer func() {\n\t\titem.uploading.Store(false)\n\t}()\n\n\tif !store.canUpload() {\n\t\treturn\n\t}\n\n\tblen := parseObjOrigSize(key)\n\tf, err := openCacheFile(stagingPath, blen, store.conf.CacheChecksum)\n\tif err != nil {\n\t\tif store.isPendingValid(key) {\n\t\t\tlogger.Errorf(\"Open staging file %s: %s\", stagingPath, err)\n\t\t} else {\n\t\t\tlogger.Debugf(\"Key %s is not needed, drop it\", key)\n\t\t}\n\t\treturn\n\t}\n\tblock := NewOffPage(blen)\n\t_, err = f.ReadAt(block.Data, 0)\n\t_ = f.Close()\n\tif err != nil {\n\t\tblock.Release()\n\t\tlogger.Errorf(\"Read staging file %s: %s\", stagingPath, err)\n\t\treturn\n\t}\n\tif !store.isPendingValid(key) {\n\t\tblock.Release()\n\t\tlogger.Debugf(\"Key %s is not needed, drop it\", key)\n\t\treturn\n\t}\n\n\tstore.stageBlockDelay.Add(time.Since(item.ts).Seconds())\n\tif err = store.upload(key, block, nil); err == nil {\n\t\tif !store.isPendingValid(key) { // Delete leaked objects if it's already deleted by other goroutines\n\t\t\terr := store.delete(key)\n\t\t\tlogger.Infof(\"Key %s is not needed, abandoned, err: %v\", key, err)\n\t\t} else {\n\t\t\tstore.bcache.uploaded(key, blen)\n\t\t\tstore.removePending(key)\n\t\t\tif err := store.bcache.removeStage(key); err != nil {\n\t\t\t\tlogger.Warnf(\"failed to remove stage %s, in upload staging file\", stagingPath)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (store *cachedStore) addDelayedStaging(key, stagingPath string, added time.Time, force bool) bool {\n\tstore.pendingMutex.Lock()\n\titem := store.pendingKeys[key]\n\tif item == nil {\n\t\titem = &pendingItem{key, stagingPath, added, atomic.Bool{}}\n\t\tstore.pendingKeys[key] = item\n\t}\n\tstore.pendingMutex.Unlock()\n\tif force || store.canUpload() && time.Since(added) > store.conf.UploadDelay {\n\t\tif item.uploading.CompareAndSwap(false, true) {\n\t\t\tselect {\n\t\t\tcase store.pendingCh <- item:\n\t\t\t\treturn true\n\t\t\tdefault:\n\t\t\t\titem.uploading.Store(false)\n\t\t\t}\n\t\t} else {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (store *cachedStore) removePending(key string) {\n\tstore.pendingMutex.Lock()\n\tdelete(store.pendingKeys, key)\n\tstore.pendingMutex.Unlock()\n}\n\nfunc (store *cachedStore) isPendingValid(key string) bool {\n\tstore.pendingMutex.Lock()\n\tdefer store.pendingMutex.Unlock()\n\t_, ok := store.pendingKeys[key]\n\treturn ok\n}\n\nfunc (store *cachedStore) scanDelayedStaging() {\n\tif !store.canUpload() {\n\t\treturn\n\t}\n\tcutoff := time.Now().Add(-store.conf.UploadDelay)\n\tstore.pendingMutex.Lock()\n\tdefer store.pendingMutex.Unlock()\n\tfor _, item := range store.pendingKeys {\n\t\tstore.pendingMutex.Unlock()\n\t\tif item.ts.Before(cutoff) && item.uploading.CompareAndSwap(false, true) {\n\t\t\tstore.pendingCh <- item\n\t\t}\n\t\tstore.pendingMutex.Lock()\n\t}\n}\n\nfunc (store *cachedStore) uploader() {\n\tfor it := range store.pendingCh {\n\t\tstore.uploadStagingFile(it.key, it.fpath)\n\t}\n}\n\nfunc (store *cachedStore) canUpload() bool {\n\tif store.startHour == store.endHour {\n\t\treturn true\n\t}\n\th := time.Now().Hour()\n\treturn store.startHour < store.endHour && h >= store.startHour && h < store.endHour ||\n\t\tstore.startHour > store.endHour && (h >= store.startHour || h < store.endHour)\n}\n\nfunc (store *cachedStore) NewReader(id uint64, length int) Reader {\n\treturn sliceForRead(id, length, store)\n}\n\nfunc (store *cachedStore) NewWriter(id uint64) Writer {\n\treturn sliceForWrite(id, store)\n}\n\nfunc (store *cachedStore) Remove(id uint64, length int) error {\n\tr := sliceForRead(id, length, store)\n\treturn r.Remove()\n}\n\nfunc (store *cachedStore) FillCache(id uint64, length uint32) error {\n\tr := sliceForRead(id, int(length), store)\n\tkeys := r.keys()\n\tvar err error\n\tfor _, k := range keys {\n\t\tif _, existed := store.bcache.exist(k); existed { // already cached\n\t\t\tcontinue\n\t\t}\n\t\tsize := parseObjOrigSize(k)\n\t\tif size == 0 || size > store.conf.BlockSize {\n\t\t\tlogger.Warnf(\"Invalid size: %s %d\", k, size)\n\t\t\tcontinue\n\t\t}\n\t\tp := NewOffPage(size)\n\t\tif e := store.load(context.TODO(), k, p, true, true); e != nil {\n\t\t\tlogger.Warnf(\"Failed to load key: %s %s\", k, e)\n\t\t\terr = e\n\t\t}\n\t\tp.Release()\n\t}\n\treturn err\n}\n\nfunc (store *cachedStore) EvictCache(id uint64, length uint32) error {\n\tr := sliceForRead(id, int(length), store)\n\tkeys := r.keys()\n\tfor _, k := range keys {\n\t\tstore.bcache.remove(k, false)\n\t}\n\treturn nil\n}\n\nfunc (store *cachedStore) CheckCache(id uint64, length uint32, handler func(exists bool, loc string, size int)) error {\n\tr := sliceForRead(id, int(length), store)\n\tkeys := r.keys()\n\tvar loc string\n\tvar existed bool\n\tfor i, k := range keys {\n\t\tloc, existed = store.bcache.exist(k)\n\t\tif handler != nil {\n\t\t\thandler(existed, loc, r.blockSize(i))\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (store *cachedStore) UsedMemory() int64 {\n\treturn store.bcache.usedMemory()\n}\n\nfunc (store *cachedStore) UpdateLimit(upload, download int64) {\n\tif upload = upload * 1e6 / 8; upload != store.conf.UploadLimit {\n\t\tlogger.Infof(\"Upload limit changed from %d to %d\", store.conf.UploadLimit, upload)\n\t\tstore.conf.UploadLimit = upload\n\t\tif upload > 0 {\n\t\t\tstore.upLimit = ratelimit.NewBucketWithRate(float64(upload)*0.85, upload/10)\n\t\t} else {\n\t\t\tstore.upLimit = nil\n\t\t}\n\t}\n\tif download = download * 1e6 / 8; download != store.conf.DownloadLimit {\n\t\tlogger.Infof(\"Download limit changed from %d to %d\", store.conf.DownloadLimit, download)\n\t\tstore.conf.DownloadLimit = download\n\t\tif download > 0 {\n\t\t\tstore.downLimit = ratelimit.NewBucketWithRate(float64(download)*0.85, download/10)\n\t\t} else {\n\t\t\tstore.downLimit = nil\n\t\t}\n\t}\n}\n\nvar _ ChunkStore = (*cachedStore)(nil)\n" }, { "path": "pkg/chunk/cached_store_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n//nolint:errcheck\npackage chunk\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc forgetSlice(store ChunkStore, sliceId uint64, size int) error {\n\tw := store.NewWriter(sliceId)\n\tbuf := bytes.Repeat([]byte{0x41}, size)\n\tif _, err := w.WriteAt(buf, 0); err != nil {\n\t\treturn err\n\t}\n\treturn w.Finish(size)\n}\n\nfunc testStore(t *testing.T, store ChunkStore) {\n\twriter := store.NewWriter(1)\n\tdata := []byte(\"hello world\")\n\tif n, err := writer.WriteAt(data, 0); n != 11 || err != nil {\n\t\tt.Fatalf(\"write fail: %d %s\", n, err)\n\t}\n\toffset := defaultConf.BlockSize - 3\n\tif n, err := writer.WriteAt(data, int64(offset)); err != nil || n != 11 {\n\t\tt.Fatalf(\"write fail: %d %s\", n, err)\n\t}\n\tif err := writer.FlushTo(defaultConf.BlockSize + 3); err != nil {\n\t\tt.Fatalf(\"flush fail: %s\", err)\n\t}\n\tsize := offset + len(data)\n\tif err := writer.Finish(size); err != nil {\n\t\tt.Fatalf(\"finish fail: %s\", err)\n\t}\n\tdefer store.Remove(1, size)\n\n\treader := store.NewReader(1, size)\n\tp := NewPage(make([]byte, 5))\n\tif n, err := reader.ReadAt(context.Background(), p, 6); n != 5 || err != nil {\n\t\tt.Fatalf(\"read failed: %d %s\", n, err)\n\t} else if string(p.Data[:n]) != \"world\" {\n\t\tt.Fatalf(\"not expected: %s\", string(p.Data[:n]))\n\t}\n\tp = NewPage(make([]byte, 5))\n\tif n, err := reader.ReadAt(context.Background(), p, 0); n != 5 || err != nil {\n\t\tt.Fatalf(\"read failed: %d %s\", n, err)\n\t} else if string(p.Data[:n]) != \"hello\" {\n\t\tt.Fatalf(\"not expected: %s\", string(p.Data[:n]))\n\t}\n\tp = NewPage(make([]byte, 20))\n\tif n, err := reader.ReadAt(context.Background(), p, offset); n != 11 || err != nil && err != io.EOF {\n\t\tt.Fatalf(\"read failed: %d %s\", n, err)\n\t} else if string(p.Data[:n]) != \"hello world\" {\n\t\tt.Fatalf(\"not expected: %s\", string(p.Data[:n]))\n\t}\n\n\tbsize := defaultConf.BlockSize / 2\n\terrs := make(chan error, 3)\n\tfor i := 2; i < 5; i++ {\n\t\tgo func(sliceId uint64) {\n\t\t\tif err := forgetSlice(store, sliceId, bsize); err != nil {\n\t\t\t\terrs <- err\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttime.Sleep(time.Millisecond * 100) // waiting for flush\n\t\t\terrs <- store.Remove(sliceId, bsize)\n\t\t}(uint64(i))\n\t}\n\tfor i := 0; i < 3; i++ {\n\t\tif err := <-errs; err != nil {\n\t\t\tt.Fatalf(\"test concurrent write failed: %s\", err)\n\t\t}\n\t}\n}\n\nvar defaultConf = Config{\n\tBlockSize: 1 << 20,\n\tCacheDir: filepath.Join(os.TempDir(), \"diskCache\"),\n\tCacheMode: 0600,\n\tCacheSize: 10 << 20,\n\tCacheChecksum: CsNone,\n\tCacheScanInterval: time.Second * 300,\n\tMaxUpload: 1,\n\tMaxDownload: 200,\n\tMaxRetries: 10,\n\tPutTimeout: time.Second,\n\tGetTimeout: time.Second * 2,\n\tAutoCreate: true,\n\tBufferSize: 10 << 20,\n}\n\nvar ctx = context.Background()\n\nfunc TestStoreDefault(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\t_ = os.RemoveAll(defaultConf.CacheDir)\n\tstore := NewCachedStore(mem, defaultConf, nil)\n\ttestStore(t, store)\n\tif used := store.UsedMemory(); used != 0 {\n\t\tt.Fatalf(\"used memory %d != expect 0\", used)\n\t}\n\tif cnt, used := store.(*cachedStore).bcache.stats(); cnt != 0 || used != 0 {\n\t\tt.Fatalf(\"cache cnt %d used %d, expect both 0\", cnt, used)\n\t}\n}\n\nfunc TestStoreMemCache(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.CacheDir = \"memory\"\n\tstore := NewCachedStore(mem, conf, nil)\n\ttestStore(t, store)\n\tif used := store.UsedMemory(); used != 0 {\n\t\tt.Fatalf(\"used memory %d != expect 0\", used)\n\t}\n\tif cnt, used := store.(*cachedStore).bcache.stats(); cnt != 0 || used != 0 {\n\t\tt.Fatalf(\"cache cnt %d used %d, expect both 0\", cnt, used)\n\t}\n}\nfunc TestStoreCompressed(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.Compress = \"lz4\"\n\tconf.AutoCreate = false\n\tstore := NewCachedStore(mem, conf, nil)\n\ttestStore(t, store)\n}\n\nfunc TestStoreLimited(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.UploadLimit = 1e6\n\tconf.DownloadLimit = 1e6\n\tstore := NewCachedStore(mem, conf, nil)\n\ttestStore(t, store)\n}\n\nfunc TestStoreFull(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.FreeSpace = 0.9999\n\tstore := NewCachedStore(mem, conf, nil)\n\ttestStore(t, store)\n}\n\nfunc TestStoreSmallBuffer(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.BufferSize = 1 << 20\n\tstore := NewCachedStore(mem, conf, nil)\n\ttestStore(t, store)\n}\n\nfunc TestStoreAsync(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.Writeback = true\n\tp := filepath.Join(conf.CacheDir, stagingDir, \"chunks/0/0/123_0_4\")\n\tos.MkdirAll(filepath.Dir(p), 0744)\n\tf, _ := os.Create(p)\n\tf.WriteString(\"good\")\n\tf.Close()\n\tstore := NewCachedStore(mem, conf, nil)\n\ttime.Sleep(time.Millisecond * 50) // wait for scan to finish\n\tin, err := mem.Get(ctx, \"chunks/0/0/123_0_4\", 0, -1)\n\tif err != nil {\n\t\tt.Fatalf(\"staging object should be upload\")\n\t}\n\tdata, _ := io.ReadAll(in)\n\tif string(data) != \"good\" {\n\t\tt.Fatalf(\"data %s != expect good\", data)\n\t}\n\ttestStore(t, store)\n}\n\nfunc TestForceUpload(t *testing.T) {\n\tblob, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconfig := defaultConf\n\t_ = os.RemoveAll(config.CacheDir)\n\tconfig.Writeback = true\n\tconfig.WritebackThresholdSize = config.BlockSize + 1\n\tconfig.UploadDelay = time.Hour\n\tconfig.BlockSize = 4 << 20\n\tstore := NewCachedStore(blob, config, nil)\n\tcleanCache := func() {\n\t\trSlice := sliceForRead(1, 1024, store.(*cachedStore))\n\t\tkeys := rSlice.keys()\n\t\tfor _, k := range keys {\n\t\t\tstore.(*cachedStore).bcache.remove(k, true)\n\t\t}\n\t}\n\treadSlice := func(id uint64, length int) error {\n\t\tp := NewPage(make([]byte, length))\n\t\tr := store.NewReader(id, length)\n\t\t_, err := r.ReadAt(context.Background(), p, 0)\n\t\treturn err\n\t}\n\n\t// write to cache\n\tw := store.NewWriter(1)\n\tif _, err := w.WriteAt(make([]byte, 1024), 0); err != nil {\n\t\tt.Fatalf(\"write fail: %s\", err)\n\t}\n\tif err := w.Finish(1024); err != nil {\n\t\tt.Fatalf(\"write fail: %s\", err)\n\t}\n\tcleanCache()\n\tif readSlice(1, 1024) == nil {\n\t\tt.Fatalf(\"read slice 1 should fail\")\n\t}\n\n\t// write to os\n\tw = store.NewWriter(2)\n\tw.SetWriteback(false)\n\tif _, err := w.WriteAt(make([]byte, 1024), 0); err != nil {\n\t\tt.Fatalf(\"write fail: %s\", err)\n\t}\n\tif err := w.Finish(1024); err != nil {\n\t\tt.Fatalf(\"write fail: %s\", err)\n\t}\n\tcleanCache()\n\tif readSlice(2, 1024) != nil {\n\t\tt.Fatalf(\"check slice 2 should success\")\n\t}\n}\n\nfunc TestStoreDelayed(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.Writeback = true\n\tconf.UploadDelay = time.Millisecond * 200\n\tstore := NewCachedStore(mem, conf, nil)\n\ttime.Sleep(time.Second) // waiting for cache scanned\n\ttestStore(t, store)\n\tif err := forgetSlice(store, 10, 1024); err != nil {\n\t\tt.Fatalf(\"forge slice 10 1024: %s\", err)\n\t}\n\tdefer store.Remove(10, 1024)\n\ttime.Sleep(time.Second) // waiting for upload\n\tif _, err := mem.Head(ctx, \"chunks/0/0/10_0_1024\"); err != nil {\n\t\tt.Fatalf(\"head object 10_0_1024: %s\", err)\n\t}\n}\n\nfunc TestStoreMultiBuckets(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.HashPrefix = true\n\tstore := NewCachedStore(mem, conf, nil)\n\ttestStore(t, store)\n}\n\nfunc TestFillCache(t *testing.T) {\n\tmem, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconf := defaultConf\n\tconf.CacheSize = 10 << 20\n\tconf.FreeSpace = 0.01\n\t_ = os.RemoveAll(conf.CacheDir)\n\tstore := NewCachedStore(mem, conf, nil)\n\tif err := forgetSlice(store, 10, 1024); err != nil {\n\t\tt.Fatalf(\"forge slice 10 1024: %s\", err)\n\t}\n\tdefer store.Remove(10, 1024)\n\tbsize := conf.BlockSize\n\tif err := forgetSlice(store, 11, bsize); err != nil {\n\t\tt.Fatalf(\"forge slice 11 %d: %s\", bsize, err)\n\t}\n\tdefer store.Remove(11, bsize)\n\n\ttime.Sleep(time.Millisecond * 100) // waiting for flush\n\tbcache := store.(*cachedStore).bcache\n\tif cnt, used := bcache.stats(); cnt != 1 || used != 1024+4096 { // only chunk 10 cached\n\t\tt.Fatalf(\"cache cnt %d used %d, expect cnt 1 used 5120\", cnt, used)\n\t}\n\tif err := store.FillCache(10, 1024); err != nil {\n\t\tt.Fatalf(\"fill cache 10 1024: %s\", err)\n\t}\n\tif err := store.FillCache(11, uint32(bsize)); err != nil {\n\t\tt.Fatalf(\"fill cache 11 %d: %s\", bsize, err)\n\t}\n\ttime.Sleep(time.Second)\n\texpect := int64(1024 + 4096 + bsize + 4096)\n\tif cnt, used := bcache.stats(); cnt != 2 || used != expect {\n\t\tt.Fatalf(\"cache cnt %d used %d, expect cnt 2 used %d\", cnt, used, expect)\n\t}\n\n\tvar missBytes uint64\n\thandler := func(exists bool, loc string, size int) {\n\t\tif !exists {\n\t\t\tmissBytes += uint64(size)\n\t\t}\n\t}\n\t// check\n\terr := store.CheckCache(10, 1024, handler)\n\tassert.Nil(t, err)\n\tassert.Equal(t, uint64(0), missBytes)\n\n\tmissBytes = 0\n\terr = store.CheckCache(11, uint32(bsize), handler)\n\tassert.Nil(t, err)\n\tassert.Equal(t, uint64(0), missBytes)\n\n\t// evict slice 11\n\terr = store.EvictCache(11, uint32(bsize))\n\tassert.Nil(t, err)\n\n\t// stat\n\tif cnt, used := bcache.stats(); cnt != 1 || used != 1024+4096 { // only chunk 10 cached\n\t\tt.Fatalf(\"cache cnt %d used %d, expect cnt 1 used 5120\", cnt, used)\n\t}\n\n\t// check again\n\tmissBytes = 0\n\terr = store.CheckCache(11, uint32(bsize), handler)\n\tassert.Nil(t, err)\n\tassert.Equal(t, uint64(bsize), missBytes)\n}\n\nfunc BenchmarkCachedRead(b *testing.B) {\n\tblob, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconfig := defaultConf\n\tconfig.BlockSize = 4 << 20\n\tstore := NewCachedStore(blob, config, nil)\n\tw := store.NewWriter(1)\n\tif _, err := w.WriteAt(make([]byte, 1024), 0); err != nil {\n\t\tb.Fatalf(\"write fail: %s\", err)\n\t}\n\tif err := w.Finish(1024); err != nil {\n\t\tb.Fatalf(\"write fail: %s\", err)\n\t}\n\ttime.Sleep(time.Millisecond * 100)\n\tp := NewPage(make([]byte, 1024))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tr := store.NewReader(1, 1024)\n\t\tif n, err := r.ReadAt(context.Background(), p, 0); err != nil || n != 1024 {\n\t\t\tb.FailNow()\n\t\t}\n\t}\n}\n\nfunc BenchmarkUncachedRead(b *testing.B) {\n\tblob, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tconfig := defaultConf\n\tconfig.BlockSize = 4 << 20\n\tconfig.CacheSize = 0\n\tstore := NewCachedStore(blob, config, nil)\n\tw := store.NewWriter(2)\n\tif _, err := w.WriteAt(make([]byte, 1024), 0); err != nil {\n\t\tb.Fatalf(\"write fail: %s\", err)\n\t}\n\tif err := w.Finish(1024); err != nil {\n\t\tb.Fatalf(\"write fail: %s\", err)\n\t}\n\tp := NewPage(make([]byte, 1024))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tr := store.NewReader(2, 1024)\n\t\tif n, err := r.ReadAt(context.Background(), p, 0); err != nil || n != 1024 {\n\t\t\tb.FailNow()\n\t\t}\n\t}\n}\n\ntype dStore struct {\n\tobject.ObjectStorage\n\tcnt int32\n}\n\nfunc (s *dStore) Get(ctx context.Context, key string, off, limit int64, getters ...object.AttrGetter) (io.ReadCloser, error) {\n\tatomic.AddInt32(&s.cnt, 1)\n\treturn nil, errors.New(\"not found\")\n}\n\nfunc TestStoreRetry(t *testing.T) {\n\ts := &dStore{}\n\tcs := NewCachedStore(s, defaultConf, nil)\n\tp := NewPage(nil)\n\tdefer p.Release()\n\tcs.(*cachedStore).load(context.TODO(), \"non\", p, false, false) // wont retry\n\trequire.Equal(t, int32(1), s.cnt)\n}\n" }, { "path": "pkg/chunk/chunk.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"context\"\n\t\"io\"\n)\n\ntype Reader interface {\n\tReadAt(ctx context.Context, p *Page, off int) (int, error)\n}\n\ntype Writer interface {\n\tio.WriterAt\n\tID() uint64\n\tSetID(id uint64)\n\tSetWriteback(enabled bool)\n\tFlushTo(offset int) error\n\tFinish(length int) error\n\tAbort()\n}\n\ntype ChunkStore interface {\n\tNewReader(id uint64, length int) Reader\n\tNewWriter(id uint64) Writer\n\tRemove(id uint64, length int) error\n\tFillCache(id uint64, length uint32) error\n\tEvictCache(id uint64, length uint32) error\n\tCheckCache(id uint64, length uint32, handler func(exists bool, loc string, size int)) error\n\tUsedMemory() int64\n\tUpdateLimit(upload, download int64)\n}\n" }, { "path": "pkg/chunk/disk_cache.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"hash/fnv\"\n\t\"io\"\n\t\"io/fs\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/charlievieth/fastwalk\"\n\t\"github.com/davies/groupcache/consistenthash\"\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/google/uuid\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/twmb/murmur3\"\n)\n\nvar (\n\tstagingDir = \"rawstaging\"\n\tcacheDir = \"raw\"\n\tmaxIODur = time.Second * 30\n\tstagingBlocks atomic.Int64\n\terrNotCached = errors.New(\"not cached\")\n\terrStageFull = errors.New(\"space not enough on device\")\n\terrStageConcurrency = errors.New(\"concurrent staging limit reached\")\n)\n\ntype cacheKey struct {\n\tid uint64\n\tindx uint32\n\tsize uint32\n}\n\nfunc (k cacheKey) String() string { return fmt.Sprintf(\"%d_%d_%d\", k.id, k.indx, k.size) }\n\ntype pendingFile struct {\n\tkey string\n\tpage *Page\n\tdropCache bool\n}\n\ntype cacheStore struct {\n\tid string\n\ttotalPages int64\n\tsync.Mutex\n\tdir string\n\tmode os.FileMode\n\tmaxStageWrite int\n\tcapacity int64\n\tmaxItems int64\n\tfreeRatio float32\n\thashPrefix bool\n\tscanInterval time.Duration\n\tcacheExpire time.Duration\n\tpending chan pendingFile\n\tpages map[string]*Page\n\tm *cacheManagerMetrics\n\n\tused int64\n\tkeys KeyIndex\n\tscanned bool\n\tstageFull bool\n\trawFull bool\n\tchecksum string // checksum level\n\tuploader func(key, path string, force bool) bool\n\n\topTs map[time.Duration]func() error\n\topMu sync.Mutex\n\n\tstate dcState\n\tstateLock sync.Mutex\n\n\t// newBlockCooldown reduces the initial access time for newly cached staged blocks.\n\t// This helps prevent a surge of writes from evicting active read blocks.\n\tstagedBlockCooldown time.Duration\n}\n\nfunc newCacheStore(m *cacheManagerMetrics, dir string, cacheSize, maxItems int64, pendingPages int, config *Config, uploader func(key, path string, force bool) bool) *cacheStore {\n\tif config.CacheMode == 0 {\n\t\tconfig.CacheMode = 0600 // only owner can read/write cache\n\t}\n\tif config.FreeSpace == 0.0 {\n\t\tconfig.FreeSpace = 0.1 // 10%\n\t}\n\tkeyIndex, err := NewKeyIndex(config)\n\tif err != nil {\n\t\tlogger.Warnf(\"%s, fallback to %s\", err, Eviction2Random)\n\t\tconfig.CacheEviction = Eviction2Random\n\t\tkeyIndex, _ = NewKeyIndex(config)\n\t}\n\tc := &cacheStore{\n\t\tm: m,\n\t\tdir: dir,\n\t\tmode: config.CacheMode,\n\t\tcapacity: cacheSize,\n\t\tmaxItems: maxItems,\n\t\tmaxStageWrite: config.MaxStageWrite,\n\t\tfreeRatio: config.FreeSpace,\n\t\tchecksum: config.CacheChecksum,\n\t\thashPrefix: config.HashPrefix,\n\t\tscanInterval: config.CacheScanInterval,\n\t\tcacheExpire: config.CacheExpire,\n\t\tkeys: keyIndex,\n\t\tpending: make(chan pendingFile, pendingPages),\n\t\tpages: make(map[string]*Page),\n\t\tuploader: uploader,\n\t\topTs: make(map[time.Duration]func() error),\n\t\tstagedBlockCooldown: config.CacheExpire / 2,\n\t}\n\tc.stateLock = sync.Mutex{}\n\tif config.Writeback {\n\t\tc.state = newDCState(dcUnchanged, c)\n\t} else {\n\t\tc.state = newDCState(dcNormal, c)\n\t}\n\n\tc.createDir(c.dir)\n\tusage := c.curFreeRatio()\n\tif usage.br < c.freeRatio || usage.fr < c.freeRatio {\n\t\tlogger.Warnf(\"not enough space (%d%%) or inodes (%d%%) for caching in %s: free ratio should be >= %d%%\", int(usage.br*100), int(usage.fr*100), c.dir, int(c.freeRatio*100))\n\t}\n\tlogger.Infof(\"Disk cache (%s): used ratio - [space %s%%, inode %s%%]\",\n\t\tc.dir, humanize.FtoaWithDigits(float64((1-usage.br)*100), 1), humanize.FtoaWithDigits(float64((1-usage.fr)*100), 1))\n\n\tc.setLimitByFreeRatio(usage, c.freeRatio)\n\n\tc.createLockFile()\n\tgo c.checkLockFile()\n\tgo c.flush()\n\tgo c.checkFreeSpace()\n\tif c.cacheExpire > 0 {\n\t\tgo c.cleanupExpire()\n\t}\n\tgo c.refreshCacheKeys()\n\tgo c.scanStaging()\n\tgo c.checkTimeout()\n\treturn c\n}\n\nfunc (cache *cacheStore) setLimitByFreeRatio(usage DiskFreeRatio, freeRatio float32) {\n\tsizeLimit := int64(float64(1-freeRatio) * float64(usage.spaceCap))\n\tif sizeLimit < cache.capacity {\n\t\tlimit := cache.capacity\n\t\tcache.capacity = sizeLimit\n\t\tlogger.Infof(\"Adjusted cache capacity based on freeratio: from %d to %d bytes\", limit, cache.capacity)\n\t}\n\tif usage.inodeCap <= 0 {\n\t\treturn\n\t}\n\tinodeLimit := int64(float64(1-freeRatio) * float64(usage.inodeCap))\n\tif inodeLimit < cache.maxItems || cache.maxItems == 0 {\n\t\tlimit := cache.maxItems\n\t\tcache.maxItems = inodeLimit\n\n\t\tmaxItems := \"unlimited\"\n\t\tif cache.maxItems != 0 {\n\t\t\tmaxItems = strconv.FormatInt(cache.maxItems, 10)\n\t\t}\n\t\tlogger.Infof(\"Adjusted max items based on freeratio: from %d to %s items\", limit, maxItems)\n\t}\n}\n\nfunc (cache *cacheStore) lockFilePath() string {\n\treturn filepath.Join(cache.dir, \".lock\")\n}\n\nfunc (cache *cacheStore) createLockFile() {\n\tlockfile := cache.lockFilePath()\n\terr := cache.checkErr(func() error {\n\t\tf, err := os.OpenFile(lockfile, os.O_CREATE|os.O_RDWR, 0666)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"open lock file %s: %w\", lockfile, err)\n\t\t}\n\t\tdefer f.Close()\n\t\trawId, err := io.ReadAll(f)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"read lock file %s: %w\", lockfile, err)\n\t\t}\n\t\tif len(rawId) > 0 {\n\t\t\tcache.id = string(rawId)\n\t\t} else {\n\t\t\tcache.id = uuid.New().String()\n\t\t\t_, err = f.Write([]byte(cache.id))\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"write lock file %s: %w\", lockfile, err)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"create lock file %s: %s\", lockfile, err)\n\t}\n}\n\nfunc (cache *cacheStore) checkLockFile() {\n\tlockfile := cache.lockFilePath()\n\tfor cache.available() {\n\t\ttime.Sleep(time.Second * 10)\n\t\tif err := cache.statFile(lockfile); err != nil && os.IsNotExist(err) {\n\t\t\tlogger.Infof(\"lockfile %s is lost, cache device maybe broken\", lockfile)\n\t\t\tif inRootVolume(cache.dir) && cache.freeRatio < 0.2 {\n\t\t\t\tlogger.Infof(\"cache directory %s is in root volume, keep 20%% space free\", cache.dir)\n\t\t\t\tcache.freeRatio = 0.2\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (c *cacheStore) available() bool {\n\treturn c.state.state() != dcDown\n}\n\nfunc (c *cacheStore) enabled() bool {\n\treturn c.capacity > 0\n}\n\nfunc (c *cacheStore) full() bool {\n\treturn c.used > c.capacity || (c.maxItems != 0 && int64(c.keys.len()) > c.maxItems)\n}\n\nfunc (cache *cacheStore) checkErr(f func() error) error {\n\tif !cache.available() {\n\t\treturn errCacheDown\n\t}\n\tcache.state.beforeCacheOp()\n\tdefer cache.state.afterCacheOp()\n\tif err := cache.state.checkCacheOp(); err != nil {\n\t\treturn err\n\t}\n\n\tstart := utils.Clock()\n\tcache.opMu.Lock()\n\tcache.opTs[start] = f\n\tcache.opMu.Unlock()\n\terr := f()\n\tcache.opMu.Lock()\n\tdelete(cache.opTs, start)\n\tcache.opMu.Unlock()\n\n\tif err != nil {\n\t\tif errors.Is(err, syscall.EIO) || errors.Is(err, utils.ErrFuncTimeout) {\n\t\t\tlogger.Errorf(\"cache store is unavailable: %s\", err)\n\t\t\tcache.state.onIOErr()\n\t\t}\n\t} else {\n\t\tcache.state.onIOSucc()\n\t}\n\treturn err\n}\n\nfunc getFunctionName(f interface{}) string {\n\treturn runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()\n}\n\nfunc (c *cacheStore) checkTimeout() {\n\tfor c.available() {\n\t\tnow := utils.Clock()\n\t\tcutOff := now - maxIODur\n\t\tc.opMu.Lock()\n\t\tfor ts := range c.opTs {\n\t\t\tif ts < cutOff {\n\t\t\t\tlogger.Warnf(\"IO operation %s on %s is timeout after %s, \", getFunctionName(c.opTs[ts]), c.dir, now-ts)\n\t\t\t\tc.state.onIOErr()\n\t\t\t\tdelete(c.opTs, ts)\n\t\t\t}\n\t\t}\n\t\tc.opMu.Unlock()\n\t\ttime.Sleep(time.Second)\n\t}\n}\n\nfunc (c *cacheStore) statFile(path string) error {\n\treturn c.checkErr(func() error {\n\t\t_, err := os.Stat(path)\n\t\treturn err\n\t})\n}\n\nfunc (cache *cacheStore) removeFile(path string) error {\n\treturn cache.checkErr(func() error {\n\t\treturn os.Remove(path)\n\t})\n}\n\nfunc (cache *cacheStore) renameFile(oldpath, newpath string) error {\n\treturn cache.checkErr(func() error {\n\t\treturn os.Rename(oldpath, newpath)\n\t})\n}\n\nfunc (cache *cacheStore) writeFile(f *os.File, data []byte) error {\n\treturn cache.checkErr(func() error {\n\t\t_, err := f.Write(data)\n\t\treturn err\n\t})\n}\n\nfunc (cache *cacheStore) closeFile(f *os.File) error {\n\treturn cache.checkErr(func() error {\n\t\treturn f.Close()\n\t})\n}\n\nfunc (cache *cacheStore) usedMemory() int64 {\n\treturn atomic.LoadInt64(&cache.totalPages)\n}\n\nfunc (cache *cacheStore) stats() (int64, int64) {\n\tcache.Lock()\n\tdefer cache.Unlock()\n\treturn int64(len(cache.pages) + cache.keys.len()), cache.used + cache.usedMemory()\n}\n\nfunc (cache *cacheStore) checkFreeSpace() {\n\tfor cache.available() {\n\t\tusage := cache.curFreeRatio()\n\t\tcache.stageFull = usage.br < cache.freeRatio/2 || (usage.inodeCap > 0 && usage.fr < cache.freeRatio/2)\n\t\tcache.rawFull = usage.br < cache.freeRatio || (usage.inodeCap > 0 && usage.fr < cache.freeRatio)\n\t\tif cache.rawFull && cache.keys.name() != EvictionNone {\n\t\t\tlogger.Tracef(\"Cleanup cache when check free space (%s): free ratio (%d%%), space usage (%d%%), inodes usage (%d%%)\", cache.dir, int(cache.freeRatio*100), int(usage.br*100), int(usage.fr*100))\n\t\t\tcache.Lock()\n\t\t\tcache.cleanupFull()\n\t\t\tcache.Unlock()\n\t\t\tusage = cache.curFreeRatio()\n\t\t\tcache.rawFull = usage.br < cache.freeRatio || (usage.inodeCap > 0 && usage.fr < cache.freeRatio)\n\t\t}\n\t\tif cache.rawFull {\n\t\t\tcache.uploadStaging()\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n\tlogger.Infof(\"stop checkFreeSpace at %s\", cache.dir)\n}\n\nfunc (cache *cacheStore) cleanupExpire() {\n\tvar todel []cacheKey\n\tvar interval = time.Minute\n\tif cache.cacheExpire < time.Minute {\n\t\tinterval = cache.cacheExpire\n\t}\n\tfor {\n\t\tvar freed int64\n\t\tvar cnt, deleted int\n\t\tvar cutoff = uint32(time.Now().Unix()) - uint32(cache.cacheExpire/time.Second)\n\t\tcache.Lock()\n\t\tfor k, v := range cache.keys.randomIter() {\n\t\t\tcnt++\n\t\t\tif cnt > 1e3 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif v.size < 0 {\n\t\t\t\tcontinue // staging\n\t\t\t}\n\t\t\tif v.atime < cutoff {\n\t\t\t\tif cache.keys.remove(k, false) != nil {\n\t\t\t\t\tdeleted++\n\t\t\t\t\tfreed += int64(v.size + 4096)\n\t\t\t\t\tcache.used -= int64(v.size + 4096)\n\t\t\t\t\ttodel = append(todel, k)\n\t\t\t\t\tcache.m.cacheEvicts.Add(1)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(todel) > 0 {\n\t\t\tlogger.Debugf(\"cleanup expired cache (%s): %d blocks (%s), expired %d blocks (%s)\", cache.dir, cache.keys.len(), humanize.IBytes(uint64(cache.used)), len(todel), humanize.IBytes(uint64(freed)))\n\t\t}\n\t\tcache.Unlock()\n\t\tfor _, k := range todel {\n\t\t\tif !cache.available() {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t_ = cache.removeFile(cache.cachePath(cache.getPathFromKey(k)))\n\t\t}\n\t\ttodel = todel[:0]\n\t\ttime.Sleep(interval / 1000 * time.Duration((cnt+1-deleted)*1000/(cnt+1)))\n\t}\n}\n\nfunc (cache *cacheStore) refreshCacheKeys() {\n\tif cache.scanInterval < 0 {\n\t\treturn\n\t}\n\tcache.scanCached()\n\tif cache.scanInterval > 0 {\n\t\tfor {\n\t\t\ttime.Sleep(cache.scanInterval)\n\t\t\tcache.scanCached()\n\t\t}\n\t}\n}\n\nfunc (cache *cacheStore) removeStage(key string) error {\n\tvar err error\n\tif err = cache.removeFile(cache.stagePath(key)); err == nil {\n\t\tcache.m.stageBlocks.Sub(1)\n\t\tcache.m.stageBlockBytes.Sub(float64(parseObjOrigSize(key)))\n\t}\n\t// ignore ENOENT error\n\tif err != nil && os.IsNotExist(err) {\n\t\treturn nil\n\t}\n\treturn err\n}\n\nfunc (cache *cacheStore) cache(key string, p *Page, force, dropCache bool) {\n\tif !cache.enabled() {\n\t\treturn\n\t}\n\tif cache.rawFull && cache.keys.name() == EvictionNone {\n\t\tlogger.Debugf(\"Caching directory is full (%s), drop %s (%d bytes)\", cache.dir, key, len(p.Data))\n\t\tcache.m.cacheDrops.Add(1)\n\t\treturn\n\t}\n\tcache.Lock()\n\tdefer cache.Unlock()\n\tif _, ok := cache.pages[key]; ok {\n\t\treturn\n\t}\n\tk := cache.getCacheKey(key)\n\tif cache.keys.get(k) != nil {\n\t\treturn\n\t}\n\tp.Acquire()\n\tcache.pages[key] = p\n\tatomic.AddInt64(&cache.totalPages, int64(cap(p.Data)))\n\tselect {\n\tcase cache.pending <- pendingFile{key, p, dropCache}:\n\tdefault:\n\t\tif force {\n\t\t\tcache.Unlock()\n\t\t\tcache.pending <- pendingFile{key, p, dropCache}\n\t\t\tcache.Lock()\n\t\t} else {\n\t\t\t// does not have enough bandwidth to write it into disk, discard it\n\t\t\tlogger.Debugf(\"Caching queue is full (%s), drop %s (%d bytes)\", cache.dir, key, len(p.Data))\n\t\t\tcache.m.cacheDrops.Add(1)\n\t\t\tdelete(cache.pages, key)\n\t\t\tatomic.AddInt64(&cache.totalPages, -int64(cap(p.Data)))\n\t\t\tp.Release()\n\t\t}\n\t}\n}\n\ntype DiskFreeRatio struct {\n\tbr float32\n\tfr float32\n\tspaceCap uint64\n\tinodeCap uint64\n}\n\n// caller should not hold cache lock\nfunc (cache *cacheStore) curFreeRatio() DiskFreeRatio {\n\tvar total, free, files, ffree uint64\n\t_ = cache.checkErr(func() error {\n\t\ttotal, free, files, ffree = getDiskUsage(cache.dir)\n\t\treturn nil\n\t})\n\tusage := DiskFreeRatio{\n\t\tspaceCap: total,\n\t\tinodeCap: files,\n\t}\n\tif total != 0 {\n\t\tusage.br = float32(free) / float32(total)\n\t}\n\tif files != 0 {\n\t\tusage.fr = float32(ffree) / float32(files)\n\t}\n\treturn usage\n}\n\nfunc (cache *cacheStore) flushPage(path string, data []byte, dropCache bool) (err error) {\n\tif !cache.available() {\n\t\treturn errCacheDown\n\t}\n\n\tstart := time.Now()\n\tcache.m.cacheWrites.Add(1)\n\tcache.m.cacheWriteBytes.Add(float64(len(data)))\n\tdefer func() {\n\t\tcache.m.cacheWriteHist.Observe(time.Since(start).Seconds())\n\t}()\n\tcache.createDir(filepath.Dir(path))\n\ttmp := path + \".tmp\"\n\n\tvar f *os.File\n\terr = cache.checkErr(func() error {\n\t\tf, err = os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE, cache.mode)\n\t\treturn err\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"Can't create cache file %s: %s\", tmp, err)\n\t\treturn err\n\t}\n\n\tdefer func() {\n\t\tif err != nil {\n\t\t\t_ = cache.removeFile(tmp)\n\t\t}\n\t}()\n\n\tif err = cache.writeFile(f, data); err != nil {\n\t\tlogger.Warnf(\"Write to cache file %s failed: %s\", tmp, err)\n\t\t_ = f.Close()\n\t\treturn\n\t}\n\tif cache.checksum != CsNone {\n\t\tif err = cache.writeFile(f, checksum(data)); err != nil {\n\t\t\tlogger.Warnf(\"Write checksum to cache file %s failed: %s\", tmp, err)\n\t\t\t_ = f.Close()\n\t\t\treturn\n\t\t}\n\t}\n\tif dropCache {\n\t\tdropOSCache(f)\n\t}\n\tif err = cache.closeFile(f); err != nil {\n\t\tlogger.Warnf(\"Close cache file %s failed: %s\", tmp, err)\n\t\treturn\n\t}\n\tif err = cache.renameFile(tmp, path); err != nil {\n\t\tlogger.Warnf(\"Rename cache file %s -> %s failed: %s\", tmp, path, err)\n\t}\n\treturn\n}\n\nfunc (cache *cacheStore) createDir(dir string) {\n\t// who can read the cache, should be able to access the directories and add new file.\n\t_ = cache.checkErr(func() error {\n\t\treadmode := cache.mode & 0444\n\t\tmode := cache.mode | (readmode >> 2) | (readmode >> 1)\n\t\tvar st os.FileInfo\n\t\tvar err error\n\t\tdir = filepath.Clean(dir) // `CacheManager` appends \"/\" to dir, remove it so that following `filepath.Dir` returns the parent dir\n\t\tif st, err = os.Stat(dir); os.IsNotExist(err) {\n\t\t\tif filepath.Dir(dir) != dir {\n\t\t\t\tcache.createDir(filepath.Dir(dir))\n\t\t\t}\n\t\t\t_ = os.Mkdir(dir, mode)\n\t\t\t// umask may remove some permissions\n\t\t\treturn os.Chmod(dir, mode)\n\t\t} else if strings.HasPrefix(dir, cache.dir) && err == nil && st.Mode().Perm() != mode.Perm() { // check permission only\n\t\t\tchangeMode(dir, st, mode)\n\t\t}\n\t\treturn err\n\t})\n}\n\nfunc (cache *cacheStore) getCacheKey(key string) cacheKey {\n\tp := strings.LastIndexByte(key, '/')\n\tp++\n\tvar k cacheKey\n\tl := len(key)\n\tfor p < l {\n\t\tif key[p] == '_' {\n\t\t\tp++\n\t\t\tbreak\n\t\t}\n\t\tk.id *= 10\n\t\tk.id += uint64(key[p] - '0')\n\t\tp++\n\t}\n\tfor p < l {\n\t\tif key[p] == '_' {\n\t\t\tp++\n\t\t\tbreak\n\t\t}\n\t\tk.indx *= 10\n\t\tk.indx += uint32(key[p] - '0')\n\t\tp++\n\t}\n\tfor p < l {\n\t\tk.size *= 10\n\t\tk.size += uint32(key[p] - '0')\n\t\tp++\n\t}\n\treturn k\n}\n\nfunc (cache *cacheStore) getPathFromKey(k cacheKey) string {\n\tif cache.hashPrefix {\n\t\treturn fmt.Sprintf(\"chunks/%02X/%v/%v_%v_%v\", k.id%256, k.id/1000/1000, k.id, k.indx, k.size)\n\t} else {\n\t\treturn fmt.Sprintf(\"chunks/%v/%v/%v_%v_%v\", k.id/1000/1000, k.id/1000, k.id, k.indx, k.size)\n\t}\n}\n\nfunc (cache *cacheStore) remove(key string, staging bool) {\n\tcache.Lock()\n\tdelete(cache.pages, key)\n\tpath := cache.cachePath(key)\n\tk := cache.getCacheKey(key)\n\tif it := cache.keys.remove(k, staging); it != nil {\n\t\tif it.size > 0 {\n\t\t\tcache.used -= int64(it.size + 4096)\n\t\t}\n\t} else if cache.scanned || !staging {\n\t\tpath = \"\" // not existed or staging block\n\t}\n\tcache.Unlock()\n\n\tif path != \"\" {\n\t\tif err := cache.removeFile(path); err != nil && !os.IsNotExist(err) {\n\t\t\tlogger.Warnf(\"remove %s failed: %s\", path, err)\n\t\t}\n\t\tif staging {\n\t\t\tif err := cache.removeStage(key); err != nil && !os.IsNotExist(err) {\n\t\t\t\tlogger.Warnf(\"remove stage %s failed: %s\", cache.stagePath(key), err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (cache *cacheStore) load(key string) (ReadCloser, error) {\n\tcache.Lock()\n\tdefer cache.Unlock()\n\tif p, ok := cache.pages[key]; ok {\n\t\treturn NewPageReader(p), nil\n\t}\n\tk := cache.getCacheKey(key)\n\tif cache.scanned && cache.keys.get(k) == nil {\n\t\treturn nil, errNotCached\n\t}\n\tcache.Unlock()\n\n\tvar f *cacheFile\n\tvar err error\n\terr = cache.checkErr(func() error {\n\t\tf, err = openCacheFile(cache.cachePath(key), parseObjOrigSize(key), cache.checksum)\n\t\tif err != nil && !os.IsNotExist(err) {\n\t\t\tlogger.Warnf(\"Open cache file %s failed: %s\", cache.cachePath(key), err)\n\t\t}\n\t\treturn err\n\t})\n\n\tcache.Lock()\n\tif err != nil {\n\t\tif it := cache.keys.remove(k, false); it != nil {\n\t\t\tcache.used -= int64(it.size + 4096)\n\t\t}\n\t}\n\treturn f, err\n}\n\nfunc (cache *cacheStore) exist(key string) (bool, error) {\n\tcache.Lock()\n\tdefer cache.Unlock()\n\tif _, ok := cache.pages[key]; ok {\n\t\treturn true, nil\n\t}\n\tk := cache.getCacheKey(key)\n\tif cache.scanned && cache.keys.get(k) == nil {\n\t\treturn false, errNotCached\n\t}\n\tcache.Unlock()\n\tvar err error\n\terr = cache.checkErr(func() error {\n\t\t_, err = os.Stat(cache.cachePath(key))\n\t\tif err != nil && !os.IsNotExist(err) {\n\t\t\tlogger.Warnf(\"Stat %s failed: %s\", cache.cachePath(key), err)\n\t\t}\n\t\treturn err\n\t})\n\n\tcache.Lock()\n\tif err == nil {\n\t\treturn true, nil\n\t} else if it := cache.keys.remove(k, false); it != nil {\n\t\tcache.used -= int64(it.size + 4096)\n\t}\n\treturn false, err\n}\n\nfunc (cache *cacheStore) cachePath(key string) string {\n\treturn filepath.Join(cache.dir, cacheDir, key)\n}\n\nfunc (cache *cacheStore) stagePath(key string) string {\n\treturn filepath.Join(cache.dir, stagingDir, key)\n}\n\n// flush cached block into disk\nfunc (cache *cacheStore) flush() {\n\tfor {\n\t\tw := <-cache.pending\n\t\tpath := cache.cachePath(w.key)\n\t\tif cache.enabled() && cache.flushPage(path, w.page.Data, w.dropCache) == nil {\n\t\t\tcache.add(w.key, int32(len(w.page.Data)), uint32(time.Now().Unix()))\n\t\t}\n\t\tcache.Lock()\n\t\t_, ok := cache.pages[w.key]\n\t\tdelete(cache.pages, w.key)\n\t\tatomic.AddInt64(&cache.totalPages, -int64(cap(w.page.Data)))\n\t\tcache.Unlock()\n\t\tw.page.Release()\n\t\tif !ok {\n\t\t\tcache.remove(w.key, false)\n\t\t}\n\t}\n}\n\nfunc (cache *cacheStore) add(key string, size int32, atime uint32) {\n\tif size == 0 {\n\t\tlogger.Warnf(\"Cache add %s with size 0, atime %d\", key, atime) // should not happen\n\t\treturn\n\t}\n\tk := cache.getCacheKey(key)\n\tcache.Lock()\n\tdefer cache.Unlock()\n\titer := cache.keys.get(k)\n\tif iter == nil {\n\t\titer = &cacheItem{size: size, atime: atime}\n\t} else {\n\t\tif iter.size > 0 {\n\t\t\tcache.used -= int64(iter.size + 4096)\n\t\t}\n\t\titer.size = size\n\t}\n\tcache.keys.add(k, *iter) // add or update\n\tif size > 0 {\n\t\tcache.used += int64(size + 4096)\n\t}\n\tif cache.full() && cache.keys.name() != EvictionNone {\n\t\tlogger.Debugf(\"Cleanup cache when add new data (%s): %d blocks (%s)\", cache.dir, cache.keys.len(), humanize.IBytes(uint64(cache.used)))\n\t\tcache.cleanupFull()\n\t}\n}\n\nfunc (cache *cacheStore) stage(key string, data []byte) (string, error) {\n\tstagingPath := cache.stagePath(key)\n\tif cache.stageFull {\n\t\treturn stagingPath, errStageFull\n\t}\n\tif cache.maxStageWrite != 0 && stagingBlocks.Load() > int64(cache.maxStageWrite) {\n\t\treturn stagingPath, errStageConcurrency\n\t}\n\tstagingBlocks.Add(1)\n\tdefer stagingBlocks.Add(-1)\n\terr := cache.flushPage(stagingPath, data, false)\n\tif err == nil {\n\t\tcache.m.stageBlocks.Add(1)\n\t\tcache.m.stageBlockBytes.Add(float64(len(data)))\n\t\tcache.m.stageWriteBytes.Add(float64(len(data)))\n\t\tif cache.enabled() {\n\t\t\tpath := cache.cachePath(key)\n\t\t\tcache.createDir(filepath.Dir(path))\n\t\t\tif err = os.Link(stagingPath, path); err == nil {\n\t\t\t\tcache.add(key, -int32(len(data)), uint32(time.Now().Add(-cache.stagedBlockCooldown).Unix()))\n\t\t\t} else {\n\t\t\t\tlogger.Warnf(\"link %s to %s failed: %s\", stagingPath, path, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn stagingPath, err\n}\n\nfunc (cache *cacheStore) uploaded(key string, size int) {\n\tcache.add(key, int32(size), 0)\n}\n\n// locked\nfunc (cache *cacheStore) cleanupFull() {\n\tif !cache.available() {\n\t\treturn\n\t}\n\n\tgoal := cache.capacity * 95 / 100\n\tnum := int64(cache.keys.len()) * 99 / 100\n\tif cache.maxItems != 0 && num > cache.maxItems*99/100 {\n\t\tnum = cache.maxItems * 99 / 100\n\t}\n\tcache.Unlock()\n\t// make sure we have enough free space after cleanup\n\tusage := cache.curFreeRatio()\n\tcache.Lock()\n\tif usage.br < cache.freeRatio {\n\t\ttoFree := int64(float32(usage.spaceCap) * (cache.freeRatio - usage.br))\n\t\tif toFree > cache.used {\n\t\t\tgoal = 0\n\t\t} else if cache.used-toFree < goal {\n\t\t\tgoal = (cache.used - toFree) * 95 / 100\n\t\t}\n\t}\n\tif usage.fr < cache.freeRatio {\n\t\ttoFree := int(float32(usage.inodeCap) * (cache.freeRatio - usage.fr))\n\t\tif toFree > cache.keys.len() {\n\t\t\tnum = 0\n\t\t} else {\n\t\t\tnum = int64(cache.keys.len()-toFree) * 99 / 100\n\t\t}\n\t}\n\tif int64(cache.keys.len()) <= num && cache.used <= goal {\n\t\treturn // some other thread has done the cleanup\n\t}\n\n\tvar todel []cacheKey\n\tvar freed int64\n\tvar now = uint32(time.Now().Unix())\n\n\tfor k, item := range cache.keys.evictionIter() {\n\t\tfreed += int64(item.size + 4096)\n\t\tcache.used -= int64(item.size + 4096)\n\t\ttodel = append(todel, k)\n\n\t\tlogger.Debugf(\"remove %s from cache, age: %ds\", k, now-item.atime)\n\t\tcache.m.cacheEvicts.Add(1)\n\n\t\tif int64(cache.keys.len()) <= num && cache.used <= goal {\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(todel) > 0 {\n\t\tlogger.Debugf(\"cleanup cache (%s) using %s eviction: %d blocks (%s), freed %d blocks (%s)\", cache.dir, cache.keys.name(), cache.keys.len(), humanize.IBytes(uint64(cache.used)), len(todel), humanize.IBytes(uint64(freed)))\n\t}\n\tcache.Unlock()\n\tfor _, k := range todel {\n\t\tif !cache.available() {\n\t\t\tbreak\n\t\t}\n\t\t_ = cache.removeFile(cache.cachePath(cache.getPathFromKey(k)))\n\t}\n\tcache.Lock()\n}\n\nfunc (cache *cacheStore) uploadStaging() {\n\tif !cache.scanned || cache.uploader == nil {\n\t\treturn\n\t}\n\tvar toFree int64\n\tusage := cache.curFreeRatio()\n\tif usage.br < cache.freeRatio || usage.fr < cache.freeRatio {\n\t\ttoFree = int64(float64(usage.spaceCap)*float64(cache.freeRatio) - math.Min(float64(usage.br), float64(usage.fr)))\n\t}\n\tcache.Lock()\n\tdefer cache.Unlock()\n\tvar cnt int\n\tvar lastK cacheKey\n\tvar lastValue cacheItem\n\t// for each two random keys, then compare the access time, upload the older one\n\tfor k, value := range cache.keys.randomIter() {\n\t\tif value.size > 0 {\n\t\t\tcontinue // read cache\n\t\t}\n\n\t\t// pick the bigger one if they were accessed within the same minute\n\t\tif cnt == 0 || lastValue.atime/60 > value.atime/60 ||\n\t\t\tlastValue.atime/60 == value.atime/60 && lastValue.size > value.size { // both size are < 0\n\t\t\tlastK = k\n\t\t\tlastValue = value\n\t\t}\n\t\tcnt++\n\t\tif cnt > 1 {\n\t\t\tcache.Unlock()\n\t\t\tkey := cache.getPathFromKey(lastK)\n\t\t\tif !cache.uploader(key, cache.stagePath(key), true) {\n\t\t\t\tlogger.Warnf(\"Upload list is too full\")\n\t\t\t\tcache.Lock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tlogger.Debugf(\"upload %s, age: %d\", key, uint32(time.Now().Unix())-lastValue.atime)\n\t\t\tcache.Lock()\n\t\t\t// the size in keys should be updated\n\t\t\ttoFree -= int64(-lastValue.size + 4096)\n\t\t\tcnt = 0\n\t\t}\n\n\t\tif toFree < 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\tif cnt > 0 {\n\t\tcache.Unlock()\n\t\tkey := cache.getPathFromKey(lastK)\n\t\tif cache.uploader(key, cache.stagePath(key), true) {\n\t\t\tlogger.Debugf(\"upload %s, age: %d\", key, uint32(time.Now().Unix())-lastValue.atime)\n\t\t}\n\t\tcache.Lock()\n\t}\n}\n\nfunc (cache *cacheStore) scanCached() {\n\tcache.Lock()\n\tcache.used = 0\n\t// atime in memory is more accurate than on disk, inherit it for the next round\n\tlastSnap := cache.keys.reset()\n\tcache.scanned = false\n\tcache.Unlock()\n\n\tvar start = time.Now()\n\tvar oneMinAgo = start.Add(-time.Minute)\n\n\tcachePrefix := filepath.Join(cache.dir, cacheDir)\n\tlogger.Debugf(\"Scan %s to find cached blocks\", cachePrefix)\n\t_ = fastwalk.Walk(nil, cachePrefix, func(path string, d fs.DirEntry, err error) error {\n\t\t// this func should be concurrent safe\n\t\tif err != nil {\n\t\t\treturn nil\n\t\t}\n\t\tfi, _ := d.Info()\n\t\tif fi != nil {\n\t\t\tif fi.IsDir() || strings.HasSuffix(path, \".tmp\") {\n\t\t\t\tif fi.ModTime().Before(oneMinAgo) {\n\t\t\t\t\t// try to remove empty directory\n\t\t\t\t\tif cache.removeFile(path) == nil {\n\t\t\t\t\t\tlogger.Debugf(\"Remove empty directory: %s\", path)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tkey := path[len(cachePrefix)+1:]\n\t\t\t\tif runtime.GOOS == \"windows\" {\n\t\t\t\t\tkey = strings.ReplaceAll(key, \"\\\\\", \"/\")\n\t\t\t\t}\n\t\t\t\tatime := uint32(getAtime(fi).Unix())\n\t\t\t\tif lastAtime := lastSnap.peekAtime(cache.getCacheKey(key)); lastAtime > atime {\n\t\t\t\t\tatime = lastAtime\n\t\t\t\t}\n\t\t\t\tsize := parseObjOrigSize(key) // track logical size\n\t\t\t\tif size == 0 {\n\t\t\t\t\tlogger.Warnf(\"Ignore file with unknown size: %s\", path)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tif getNlink(fi) > 1 {\n\t\t\t\t\tcache.add(key, -int32(size), atime)\n\t\t\t\t} else {\n\t\t\t\t\tcache.add(key, int32(size), atime)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tcache.Lock()\n\tcache.scanned = true\n\tlogger.Debugf(\"Found %s cached blocks (%s) in %s with %s\", humanize.Comma(int64(cache.keys.len())), humanize.IBytes(uint64(cache.used)), cache.dir, time.Since(start))\n\tcache.Unlock()\n}\n\nvar pathReg, _ = regexp.Compile(`^chunks/((\\d+)|([0-9a-fA-F]{2}))/\\d+/\\d+_\\d+_\\d+$`)\n\nfunc (cache *cacheStore) scanStaging() {\n\tif cache.uploader == nil {\n\t\treturn\n\t}\n\n\tvar start = time.Now()\n\tvar oneMinAgo = start.Add(-time.Minute)\n\tvar count, usage uint64\n\tstagingPrefix := filepath.Join(cache.dir, stagingDir)\n\tlogger.Debugf(\"Scan %s to find staging blocks\", stagingPrefix)\n\t_ = fastwalk.Walk(nil, stagingPrefix, func(path string, d fs.DirEntry, err error) error {\n\t\t// this func should be concurrent safe\n\t\tif err != nil {\n\t\t\treturn nil // ignore it\n\t\t}\n\t\tif d.IsDir() || strings.HasSuffix(path, \".tmp\") {\n\t\t\tfi, err := d.Info()\n\t\t\tif err != nil {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif fi.ModTime().Before(oneMinAgo) {\n\t\t\t\t// try to remove empty directory\n\t\t\t\tif cache.removeFile(path) == nil {\n\t\t\t\t\tlogger.Debugf(\"Remove empty directory: %s\", path)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tkey := path[len(stagingPrefix)+1:]\n\t\t\tif runtime.GOOS == \"windows\" {\n\t\t\t\tkey = strings.ReplaceAll(key, \"\\\\\", \"/\")\n\t\t\t}\n\t\t\tif !pathReg.MatchString(key) {\n\t\t\t\tlogger.Warnf(\"Ignore invalid file in staging: %s\", path)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\torigSize := parseObjOrigSize(key)\n\t\t\tif origSize == 0 {\n\t\t\t\tlogger.Warnf(\"Ignore file with zero size: %s\", path)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tlogger.Debugf(\"Found staging block: %s\", path)\n\t\t\tcache.m.stageBlocks.Add(1)\n\t\t\tcache.m.stageBlockBytes.Add(float64(origSize))\n\t\t\tcache.uploader(key, path, false)\n\t\t\tatomic.AddUint64(&count, 1)\n\t\t\tatomic.AddUint64(&usage, uint64(origSize))\n\t\t}\n\t\treturn nil\n\t})\n\tif count > 0 {\n\t\tlogger.Infof(\"Found %d staging blocks (%s) in %s with %s\", count, humanize.IBytes(usage), cache.dir, time.Since(start))\n\t}\n}\n\ntype cacheManager struct {\n\tsync.Mutex\n\tconsistentMap *consistenthash.Map\n\tstoreMap map[string]*cacheStore\n\tstores []*cacheStore\n\tmetrics *cacheManagerMetrics\n}\n\nfunc legacyKeyHash(s string) uint32 {\n\thash := fnv.New32()\n\t_, _ = hash.Write([]byte(s))\n\treturn hash.Sum32()\n}\n\n// hasMeta reports whether path contains any of the magic characters\n// recognized by Match.\nfunc hasMeta(path string) bool {\n\tmagicChars := `*?[`\n\tif runtime.GOOS != \"windows\" {\n\t\tmagicChars = `*?[\\`\n\t}\n\treturn strings.ContainsAny(path, magicChars)\n}\n\nvar osPathSeparator = string([]byte{os.PathSeparator})\n\nfunc expandDir(pattern string) []string {\n\tpattern = strings.TrimRight(pattern, \"/\")\n\tif runtime.GOOS == \"windows\" {\n\t\tpattern = strings.TrimRight(pattern, osPathSeparator)\n\t}\n\tif pattern == \"\" {\n\t\treturn []string{\"/\"}\n\t}\n\tif !hasMeta(pattern) {\n\t\treturn []string{pattern}\n\t}\n\tdir, f := filepath.Split(pattern)\n\tif hasMeta(f) {\n\t\tmatched, err := filepath.Glob(pattern)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"glob %s: %s\", pattern, err)\n\t\t\treturn []string{pattern}\n\t\t}\n\t\treturn matched\n\t}\n\tvar rs []string\n\tfor _, p := range expandDir(dir) {\n\t\trs = append(rs, filepath.Join(p, f))\n\t}\n\treturn rs\n}\n\ntype CacheManager interface {\n\tcache(key string, p *Page, force, dropCache bool)\n\tremove(key string, staging bool)\n\tload(key string) (ReadCloser, error)\n\texist(key string) (string, bool)\n\tuploaded(key string, size int)\n\tstage(key string, data []byte) (string, error)\n\tremoveStage(key string) error\n\tstats() (int64, int64)\n\tusedMemory() int64\n\tisEmpty() bool\n\tgetMetrics() *cacheManagerMetrics\n}\n\nfunc newCacheManager(config *Config, reg prometheus.Registerer, uploader func(key, path string, force bool) bool) CacheManager {\n\tgetEnvs()\n\tmetrics := newCacheManagerMetrics(reg)\n\tif config.CacheDir == \"memory\" || !config.CacheEnabled() {\n\t\treturn newMemStore(config, metrics)\n\t}\n\tvar dirs []string\n\tfor _, d := range utils.SplitDir(config.CacheDir) {\n\t\tdd := expandDir(d)\n\t\tif config.AutoCreate {\n\t\t\tdirs = append(dirs, dd...)\n\t\t} else {\n\t\t\tfor _, d := range dd {\n\t\t\t\tif fi, err := os.Stat(d); err == nil && fi.IsDir() {\n\t\t\t\t\tdirs = append(dirs, d)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif len(dirs) == 0 {\n\t\tconfig.CacheSize = 100 << 20\n\t\tlogger.Warnf(\"No cache dir existed, use memory cache instead, cache size: 100 MiB\")\n\t\treturn newMemStore(config, metrics)\n\t}\n\tsort.Strings(dirs)\n\tdirCacheSize := int64(config.CacheSize) / int64(len(dirs))\n\tdirCacheItems := config.CacheItems / int64(len(dirs))\n\tm := &cacheManager{\n\t\tconsistentMap: consistenthash.New(100, murmur3.Sum32),\n\t\tstoreMap: make(map[string]*cacheStore, len(dirs)),\n\t\tstores: make([]*cacheStore, len(dirs)),\n\t\tmetrics: metrics,\n\t}\n\n\t// 20% of buffer could be used for pending pages\n\tpendingPages := int(config.BufferSize) * 2 / 10 / config.BlockSize / len(dirs)\n\tfor i, d := range dirs {\n\t\tstore := newCacheStore(metrics, strings.TrimSpace(d)+string(filepath.Separator), dirCacheSize, dirCacheItems, pendingPages, config, uploader)\n\t\tm.stores[i] = store\n\t\tm.storeMap[store.id] = store\n\t\tm.consistentMap.Add(store.id)\n\t}\n\tgo m.cleanup()\n\treturn m\n}\n\nfunc (m *cacheManager) getMetrics() *cacheManagerMetrics {\n\treturn m.metrics\n}\n\nfunc (m *cacheManager) cleanup() {\n\tfor !m.isEmpty() {\n\t\tvar ids []string\n\t\tm.Lock()\n\t\tfor id, s := range m.storeMap {\n\t\t\tif s == nil || !s.available() {\n\t\t\t\tids = append(ids, id)\n\t\t\t}\n\t\t}\n\t\tm.Unlock()\n\t\tfor _, id := range ids {\n\t\t\tm.removeStore(id)\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n}\n\nfunc (m *cacheManager) isEmpty() bool {\n\treturn m.length() == 0\n}\n\nfunc (m *cacheManager) length() int {\n\tm.Lock()\n\tdefer m.Unlock()\n\treturn len(m.storeMap)\n}\n\nfunc (m *cacheManager) removeStore(id string) {\n\tm.Lock()\n\tm.consistentMap.Remove(id)\n\tvar dir string\n\tif s := m.storeMap[id]; s != nil {\n\t\tdir = s.dir\n\t}\n\tdelete(m.storeMap, id)\n\tfor i, c := range m.stores {\n\t\tif c != nil && c.id == id {\n\t\t\tm.stores[i] = nil\n\t\t}\n\t}\n\tm.Unlock()\n\tlogger.Errorf(\"cache dir `%s`(%s) is unavailable, removed\", dir, id)\n}\n\nfunc (m *cacheManager) getStore(key string) *cacheStore {\n\tfor {\n\t\tm.Lock()\n\t\tid := m.consistentMap.Get(key)\n\t\ts := m.storeMap[id]\n\t\tm.Unlock()\n\t\tif s == nil || s.available() {\n\t\t\treturn s\n\t\t}\n\t\tm.removeStore(id)\n\t}\n}\n\nfunc (m *cacheManager) removeStage(key string) error {\n\tif s := m.getStore(key); s == nil {\n\t\treturn errCacheDown\n\t} else {\n\t\treturn s.removeStage(key)\n\t}\n}\n\n// Deprecated: use getStore instead\nfunc (m *cacheManager) getStoreLegacy(key string) *cacheStore {\n\treturn m.stores[legacyKeyHash(key)%uint32(len(m.stores))]\n}\n\nfunc (m *cacheManager) usedMemory() int64 {\n\tvar used int64\n\tfor _, s := range m.stores {\n\t\tif s != nil {\n\t\t\tused += s.usedMemory()\n\t\t}\n\t}\n\treturn used\n}\n\nfunc (m *cacheManager) stats() (int64, int64) {\n\tvar cnt, used int64\n\tfor _, s := range m.stores {\n\t\tif s != nil {\n\t\t\tc, u := s.stats()\n\t\t\tcnt += c\n\t\t\tused += u\n\t\t}\n\t}\n\treturn cnt, used\n}\n\nfunc (m *cacheManager) cache(key string, p *Page, force, dropCache bool) {\n\tstore := m.getStore(key)\n\tif store != nil {\n\t\tstore.cache(key, p, force, dropCache)\n\t}\n}\n\ntype ReadCloser interface {\n\t// io.Reader\n\tio.ReaderAt\n\tio.Closer\n}\n\nfunc (m *cacheManager) load(key string) (ReadCloser, error) {\n\tstore := m.getStore(key)\n\tif store == nil {\n\t\treturn nil, errors.New(\"no available cache dir\")\n\t}\n\tr, err := store.load(key)\n\tif err == errNotCached {\n\t\tlegacy := m.getStoreLegacy(key)\n\t\tif legacy != store && legacy != nil {\n\t\t\tr, err = legacy.load(key)\n\t\t}\n\t}\n\treturn r, err\n}\n\nfunc (m *cacheManager) exist(key string) (string, bool) {\n\tstore := m.getStore(key)\n\tif store == nil {\n\t\treturn \"\", false\n\t}\n\tloc := store.dir\n\texisted, err := m.getStore(key).exist(key)\n\tif err == errNotCached {\n\t\tlegacy := m.getStoreLegacy(key)\n\t\tif legacy != store && legacy != nil {\n\t\t\texisted, _ = legacy.exist(key)\n\t\t\tloc = legacy.dir\n\t\t}\n\t}\n\treturn loc, existed\n}\n\nfunc (m *cacheManager) remove(key string, staging bool) {\n\tstore := m.getStore(key)\n\tif store != nil {\n\t\tstore.remove(key, staging)\n\t}\n}\n\nfunc (m *cacheManager) stage(key string, data []byte) (string, error) {\n\tstore := m.getStore(key)\n\tif store != nil {\n\t\treturn store.stage(key, data)\n\t}\n\treturn \"\", errors.New(\"no available cache dir\")\n}\n\nfunc (m *cacheManager) uploaded(key string, size int) {\n\tstore := m.getStore(key)\n\tif store != nil {\n\t\tstore.uploaded(key, size)\n\t}\n}\n\n/* --- Checksum --- */\nconst (\n\tCsNone = \"none\"\n\tCsFull = \"full\"\n\tCsShrink = \"shrink\"\n\tCsExtend = \"extend\"\n\n\tcsBlock = 32 << 10\n)\n\nvar crc32c = crc32.MakeTable(crc32.Castagnoli)\n\ntype cacheFile struct {\n\t*os.File\n\tlength int // length of data\n\tcsLevel string\n}\n\n// Calculate 32-bits checksum for every 32 KiB data, so 512 Bytes for 4 MiB in total\nfunc checksum(data []byte) []byte {\n\tlength := len(data)\n\tbuf := utils.NewBuffer(uint32((length-1)/csBlock+1) * 4)\n\tfor start, end := 0, 0; start < length; start = end {\n\t\tend = start + csBlock\n\t\tif end > length {\n\t\t\tend = length\n\t\t}\n\t\tsum := crc32.Checksum(data[start:end], crc32c)\n\t\tbuf.Put32(sum)\n\t}\n\treturn buf.Bytes()\n}\n\nfunc openCacheFile(name string, length int, level string) (*cacheFile, error) {\n\tfp, err := os.Open(name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfi, err := fp.Stat()\n\tif err != nil {\n\t\t_ = fp.Close()\n\t\treturn nil, err\n\t}\n\tchecksumLength := ((length-1)/csBlock + 1) * 4\n\tswitch fi.Size() - int64(length) {\n\tcase 0:\n\t\treturn &cacheFile{fp, length, CsNone}, nil\n\tcase int64(checksumLength):\n\t\treturn &cacheFile{fp, length, level}, nil\n\tdefault:\n\t\t_ = fp.Close()\n\t\treturn nil, fmt.Errorf(\"invalid file size %d, data length %d\", fi.Size(), length)\n\t}\n}\n\nfunc (cf *cacheFile) ReadAt(b []byte, off int64) (n int, err error) {\n\tlogger.Tracef(\"CacheFile length %d level %s, readat off %d buffer size %d\", cf.length, cf.csLevel, off, len(b))\n\tdefer func() {\n\t\tlogger.Tracef(\"CacheFile readat returns n %d err %s\", n, err)\n\t}()\n\tif cf.csLevel == CsNone || cf.csLevel == CsFull && (off != 0 || len(b) != cf.length) {\n\t\treturn cf.File.ReadAt(b, off)\n\t}\n\tvar rb = b // read buffer\n\tvar roff = off // read offset\n\tif cf.csLevel == CsExtend {\n\t\troff = off / csBlock * csBlock\n\t\trend := int(off) + len(b)\n\t\tif rend%csBlock != 0 {\n\t\t\trend = (rend/csBlock + 1) * csBlock\n\t\t\tif rend > cf.length {\n\t\t\t\trend = cf.length\n\t\t\t}\n\t\t}\n\t\tif size := rend - int(roff); size != len(b) {\n\t\t\tp := NewOffPage(size)\n\t\t\trb = p.Data\n\t\t\tdefer func() {\n\t\t\t\tif err == nil {\n\t\t\t\t\tn = copy(b, rb[off-roff:])\n\t\t\t\t} else {\n\t\t\t\t\tn = 0\n\t\t\t\t}\n\t\t\t\tp.Release()\n\t\t\t}()\n\t\t}\n\t}\n\tif n, err = cf.File.ReadAt(rb, roff); err != nil {\n\t\treturn\n\t}\n\n\tioff := int(roff) / csBlock // index offset\n\tif cf.csLevel == CsShrink {\n\t\tif roff%csBlock != 0 {\n\t\t\tif o := csBlock - int(roff)%csBlock; len(rb) <= o {\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\trb = rb[o:]\n\t\t\t\tioff += 1\n\t\t\t}\n\t\t}\n\t\tif end := int(roff) + n; end != cf.length && end%csBlock != 0 {\n\t\t\tif len(rb) <= end%csBlock {\n\t\t\t\treturn\n\t\t\t}\n\t\t\trb = rb[:len(rb)-end%csBlock]\n\t\t}\n\t}\n\t// now rb contains the data to check\n\tlength := len(rb)\n\tbuf := utils.NewBuffer(uint32((length-1)/csBlock+1) * 4)\n\tif _, err = cf.File.ReadAt(buf.Bytes(), int64(cf.length+ioff*4)); err != nil {\n\t\tlogger.Warnf(\"Read checksum of data length %d checksum offset %d: %s\", length, cf.length+ioff*4, err)\n\t\treturn\n\t}\n\tfor start, end := 0, 0; start < length; start = end {\n\t\tend = start + csBlock\n\t\tif end > length {\n\t\t\tend = length\n\t\t}\n\t\tsum := crc32.Checksum(rb[start:end], crc32c)\n\t\texpect := buf.Get32()\n\t\tlogger.Debugf(\"Cache file read data start %d end %d checksum %d, expected %d\", start, end, sum, expect)\n\t\tif sum != expect {\n\t\t\terr = fmt.Errorf(\"data checksum %d != expect %d\", sum, expect)\n\t\t\tbreak\n\t\t}\n\t}\n\treturn\n}\n" }, { "path": "pkg/chunk/disk_cache_state.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nvar (\n\tnumIOErrToUnstable uint32 = 3 // from normal to unstable\n\tminIOSuccToNormal uint32 = 60 // from unstable to normal\n\tmaxIOErrPercentageToNormal float64 = 0 // from unstable to normal\n\tmaxDurToDown = 30 * time.Minute // from unstable to down\n\tmaxConcurrencyForUnstable int64 = 10\n\ttickDurForNormal = 1 * time.Minute\n\ttickDurForUnstable = 1 * time.Minute\n\n\tprobeDur = 500 * time.Millisecond\n\tprobeDir = \"probe\"\n\tprobeData = []byte{1, 2, 3}\n\tprobeBuff = make([]byte, 3)\n)\n\nvar (\n\terrCacheDown = errors.New(\"cache down\")\n\terrUnstableCoLimit = fmt.Errorf(\"exceed concurrency %d limit for unstable disk cache\", maxConcurrencyForUnstable)\n)\n\nvar diskStateNames = map[int]string{\n\tdcUnknown: \"unknown\",\n\tdcNormal: \"normal\",\n\tdcUnstable: \"unstable\",\n\tdcDown: \"down\",\n\tdcUnchanged: \"unchanged\",\n}\n\nconst (\n\tdcUnknown = iota\n\tdcNormal\n\tdcUnstable\n\tdcDown\n\tdcUnchanged\n)\n\nconst (\n\teventUnknown = iota\n\teventToNormal\n\teventToUnstable\n\teventToDown\n)\n\n// dcState disk cache state\ntype dcState interface {\n\tinit(cs *cacheStore)\n\ttick()\n\tstop()\n\tstate() int\n\tcheckCacheOp() error\n\tbeforeCacheOp()\n\tafterCacheOp()\n\tonIOErr()\n\tonIOSucc()\n}\n\ntype baseDC struct {\n\tcache *cacheStore\n\tstopCh chan struct{}\n}\n\nfunc newDCState(state int, cs *cacheStore) dcState {\n\tvar s dcState\n\tswitch state {\n\tcase dcNormal:\n\t\ts = &normalDC{}\n\tcase dcUnstable:\n\t\ts = &unstableDC{}\n\tcase dcDown:\n\t\ts = &downDC{}\n\tcase dcUnchanged:\n\t\ts = &unchangedDC{}\n\t}\n\ts.init(cs)\n\ts.tick()\n\treturn s\n}\n\nfunc (dc *baseDC) init(cs *cacheStore) {\n\tdc.cache = cs\n\tdc.stopCh = make(chan struct{})\n}\n\nfunc (dc *baseDC) stop() {\n\tclose(dc.stopCh)\n}\nfunc (dc *baseDC) onIOErr() {}\nfunc (dc *baseDC) onIOSucc() {}\nfunc (dc *baseDC) state() int { return dcUnknown }\nfunc (dc *baseDC) tick() {}\nfunc (dc *baseDC) checkCacheOp() error { return nil }\nfunc (dc *baseDC) beforeCacheOp() {}\nfunc (dc *baseDC) afterCacheOp() {}\n\ntype unchangedDC struct {\n\tbaseDC\n}\n\nfunc (dc *unchangedDC) state() int { return dcUnchanged }\n\ntype normalDC struct {\n\tbaseDC\n\tioErrCnt uint32\n}\n\nfunc (dc *normalDC) state() int { return dcNormal }\n\nfunc (dc *normalDC) init(cs *cacheStore) {\n\tdc.baseDC.init(cs)\n\t_ = os.RemoveAll(dc.cache.cachePath(probeDir))\n}\n\nfunc (dc *normalDC) tick() {\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-dc.stopCh:\n\t\t\t\treturn\n\t\t\tcase <-time.After(tickDurForNormal):\n\t\t\t\tatomic.StoreUint32(&dc.ioErrCnt, 0)\n\t\t\t}\n\t\t}\n\t}()\n}\n\nfunc (dc *normalDC) onIOErr() {\n\tcnt := atomic.AddUint32(&dc.ioErrCnt, 1)\n\tif cnt >= uint32(numIOErrToUnstable) {\n\t\tdc.cache.event(eventToUnstable)\n\t}\n}\n\ntype unstableDC struct {\n\tbaseDC\n\tstartTime time.Time\n\tioErrCnt uint32\n\tioCnt uint32\n\n\tconcurrency atomic.Int64\n}\n\nfunc (dc *unstableDC) state() int { return dcUnstable }\n\nfunc (dc *unstableDC) init(cs *cacheStore) {\n\tdc.baseDC.init(cs)\n\tdc.startTime = time.Now()\n}\n\nfunc (dc *unstableDC) onIOErr() {\n\tatomic.AddUint32(&dc.ioCnt, 1)\n\tatomic.AddUint32(&dc.ioErrCnt, 1)\n}\n\nfunc (dc *unstableDC) onIOSucc() {\n\tatomic.AddUint32(&dc.ioCnt, 1)\n}\n\nfunc probeCacheKey(id, size int) string {\n\treturn fmt.Sprintf(\"%s/%02X/%v/%v_%v_%v\", probeDir, id%256, id/1000/1000, id, 0, size)\n}\n\nfunc (dc *unstableDC) tick() {\n\tgo dc.probe()\n\tgo func() {\n\t\tticker := time.NewTicker(tickDurForUnstable)\n\t\tdefer ticker.Stop()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-dc.stopCh:\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t\terrCnt, ioCnt := atomic.LoadUint32(&dc.ioErrCnt), atomic.LoadUint32(&dc.ioCnt)\n\t\t\t\tif ioCnt >= minIOSuccToNormal && float64(errCnt)/float64(ioCnt) <= maxIOErrPercentageToNormal {\n\t\t\t\t\tdc.cache.event(eventToNormal)\n\t\t\t\t} else if time.Since(dc.startTime) >= maxDurToDown {\n\t\t\t\t\tdc.cache.event(eventToDown)\n\t\t\t\t} else {\n\t\t\t\t\tatomic.StoreUint32(&dc.ioErrCnt, 0)\n\t\t\t\t\tatomic.StoreUint32(&dc.ioCnt, 0)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n}\n\nfunc (dc *unstableDC) probe() {\n\tpage := NewPage(probeData)\n\tdefer page.Release()\n\tcnt := 0\n\n\tfor {\n\t\tselect {\n\t\tcase <-dc.stopCh:\n\t\t\treturn\n\t\tdefault:\n\t\t\tcnt++\n\t\t\tstart := time.Now()\n\t\t\tdc.doProbe(probeCacheKey(cnt, len(probeData)), page)\n\t\t\tdiff := probeDur - time.Since(start)\n\t\t\tif diff > 0 {\n\t\t\t\ttime.Sleep(diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (dc *unstableDC) doProbe(key string, page *Page) {\n\tdc.cache.cache(key, page, true, false)\n\treader, err := dc.cache.load(key)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer reader.Close()\n\t_, _ = reader.ReadAt(probeBuff, 0)\n\tdc.cache.remove(key, false)\n}\n\nfunc (dc *unstableDC) beforeCacheOp() { dc.concurrency.Add(1) }\nfunc (dc *unstableDC) afterCacheOp() { dc.concurrency.Add(-1) }\n\nfunc (dc *unstableDC) checkCacheOp() error {\n\tif dc.concurrency.Load() >= maxConcurrencyForUnstable {\n\t\treturn errUnstableCoLimit\n\t}\n\treturn nil\n}\n\ntype downDC struct {\n\tbaseDC\n}\n\nfunc (dc *downDC) state() int { return dcDown }\nfunc (dc *downDC) checkCacheOp() error { return errCacheDown }\n\nfunc (cache *cacheStore) event(eventType int) {\n\tcache.stateLock.Lock()\n\tdefer cache.stateLock.Unlock()\n\tstate := cache.state.state()\n\tswitch state {\n\tcase dcNormal:\n\t\tif eventType == eventToUnstable {\n\t\t\tcache.state.stop()\n\t\t\tcache.state = newDCState(dcUnstable, cache)\n\t\t}\n\tcase dcUnstable:\n\t\tswitch eventType {\n\t\tcase eventToNormal:\n\t\t\tcache.state.stop()\n\t\t\tcache.state = newDCState(dcNormal, cache)\n\t\tcase eventToDown:\n\t\t\tcache.state.stop()\n\t\t\tcache.state = newDCState(dcDown, cache)\n\t\t}\n\t}\n\tlogger.Infof(\"disk cache %s state change from %s to %s\", cache.dir, diskStateNames[state], diskStateNames[cache.state.state()])\n}\n\nfunc getEnvs() {\n\tif os.Getenv(\"JFS_MAX_IO_DURATION\") != \"\" {\n\t\tdur, err := time.ParseDuration(os.Getenv(\"JFS_MAX_IO_DURATION\"))\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"parse JFS_MAX_IO_DURATION error: %v\", err)\n\t\t} else {\n\t\t\tmaxIODur = dur\n\t\t}\n\t\tlogger.Infof(\"set maxIODur to %v\", maxIODur)\n\t}\n\tif os.Getenv(\"JFS_MAX_IO_ERR_PERCENTAGE\") != \"\" {\n\t\tpercentage, err := strconv.ParseFloat(os.Getenv(\"JFS_MAX_IO_ERR_PERCENTAGE\"), 64)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"parse JFS_MAX_IO_ERR_PERCENTAGE error: %v\", err)\n\t\t} else {\n\t\t\tmaxIOErrPercentageToNormal = percentage\n\t\t}\n\t\tlogger.Infof(\"set maxIOErrPercentageToNormal to %f\", maxIOErrPercentageToNormal)\n\t}\n\tif os.Getenv(\"JFS_MAX_DURATION_TO_DOWN\") != \"\" {\n\t\tdur, err := time.ParseDuration(os.Getenv(\"JFS_MAX_DURATION_TO_DOWN\"))\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"parse JFS_MAX_DURATION_TO_DOWN error: %v\", err)\n\t\t} else {\n\t\t\tmaxDurToDown = dur\n\t\t}\n\t\tlogger.Infof(\"set maxDurToDown to %v\", maxDurToDown)\n\t}\n\tif os.Getenv(\"JFS_MAX_CONCURRENCY_FOR_UNSTABLE\") != \"\" {\n\t\tco, err := strconv.ParseInt(os.Getenv(\"JFS_MAX_CONCURRENCY_FOR_UNSTABLE\"), 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"parse JFS_MAX_CONCURRENCY_FOR_UNSTABLE error: %v\", err)\n\t\t} else {\n\t\t\tmaxConcurrencyForUnstable = co\n\t\t}\n\t\tlogger.Infof(\"set maxConcurrencyForUnstable to %d\", maxConcurrencyForUnstable)\n\t}\n}\n" }, { "path": "pkg/chunk/disk_cache_state_test.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc setState(s *cacheStore, state int) {\n\ts.stateLock.Lock()\n\tdefer s.stateLock.Unlock()\n\ts.state.stop()\n\ts.state = newDCState(state, s)\n}\n\nfunc testDiskCacheState(t *testing.T, cacheNum int) {\n\toriTickDurForUnstable, oriMinIOSuccToNormal, oriMaxDurToDown := tickDurForUnstable, minIOSuccToNormal, maxDurToDown\n\tdefer func() {\n\t\ttickDurForUnstable, minIOSuccToNormal, maxDurToDown = oriTickDurForUnstable, oriMinIOSuccToNormal, oriMaxDurToDown\n\t}()\n\n\tgenDirs := func(num int) []string {\n\t\tdirs := make([]string, 0, num)\n\t\tfor i := 0; i < num; i++ {\n\t\t\tdirs = append(dirs, fmt.Sprintf(\"/tmp/diskCache%d\", i))\n\t\t}\n\t\treturn dirs\n\t}\n\n\tconf := defaultConf\n\tdirs := genDirs(cacheNum)\n\tconf.CacheDir = strings.Join(dirs, \":\")\n\tconf.AutoCreate = true\n\tdefer func() {\n\t\tfor _, dir := range dirs {\n\t\t\t_ = os.RemoveAll(dir)\n\t\t}\n\t}()\n\n\tmanager := newCacheManager(&conf, nil, nil)\n\trequire.False(t, manager.isEmpty())\n\n\tm, ok := manager.(*cacheManager)\n\trequire.True(t, ok)\n\trequire.Equal(t, cacheNum, m.length())\n\n\t// case: cache\n\tdata := []byte{1, 2, 3}\n\tpage := NewPage(data)\n\tdefer page.Release()\n\tk1 := probeCacheKey(0, len(data))\n\tm.cache(k1, page, true, false)\n\ttime.Sleep(time.Second)\n\n\t// case: normal -> unstable\n\ts1 := m.getStore(k1)\n\tfor i := 0; i <= int(numIOErrToUnstable); i++ {\n\t\ts1.state.onIOErr()\n\t}\n\trequire.Equal(t, dcUnstable, s1.state.state())\n\n\t// case: probe in unstable\n\ttime.Sleep(time.Second)\n\trequire.GreaterOrEqual(t, atomic.LoadUint32(&s1.state.(*unstableDC).ioCnt), uint32(1))\n\n\t// case: unstable concurrency limit\n\tfor i := 0; i < int(maxConcurrencyForUnstable); i++ {\n\t\ts1.state.beforeCacheOp()\n\t}\n\t_, err := m.load(k1)\n\tassert.Equal(t, errUnstableCoLimit, err)\n\tfor i := 0; i < int(maxConcurrencyForUnstable); i++ {\n\t\ts1.state.afterCacheOp()\n\t}\n\n\t// case: unstable -> normal\n\ttickDurForUnstable = time.Second\n\tminIOSuccToNormal = 1\n\tsetState(s1, dcUnstable)\n\ts1.state.(*unstableDC).doProbe(k1, page)\n\ttime.Sleep(2 * time.Second)\n\trequire.Equal(t, dcNormal, s1.state.state())\n\n\t// case: unstable -> down\n\ttickDurForUnstable = time.Second\n\tmaxDurToDown = 1\n\tminIOSuccToNormal = 5 * 60\n\tsetState(s1, dcUnstable)\n\ttime.Sleep(2 * time.Second)\n\trequire.Equal(t, dcDown, s1.state.state())\n}\n\nfunc TestDiskCacheState(t *testing.T) {\n\ttestDiskCacheState(t, 1)\n\ttestDiskCacheState(t, 10)\n}\n" }, { "path": "pkg/chunk/disk_cache_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tdto \"github.com/prometheus/client_model/go\"\n\t\"github.com/stretchr/testify/require\"\n\n\t. \"github.com/bytedance/mockey\"\n\t. \"github.com/smartystreets/goconvey/convey\"\n)\n\n// Copy from https://github.com/prometheus/client_golang/blob/v1.14.0/prometheus/testutil/testutil.go\nfunc toFloat64(c prometheus.Collector) float64 {\n\tvar (\n\t\tm prometheus.Metric\n\t\tmCount int\n\t\tmChan = make(chan prometheus.Metric)\n\t\tdone = make(chan struct{})\n\t)\n\n\tgo func() {\n\t\tfor m = range mChan {\n\t\t\tmCount++\n\t\t}\n\t\tclose(done)\n\t}()\n\n\tc.Collect(mChan)\n\tclose(mChan)\n\t<-done\n\n\tif mCount != 1 {\n\t\tpanic(fmt.Errorf(\"collected %d metrics instead of exactly 1\", mCount))\n\t}\n\n\tpb := &dto.Metric{}\n\tif err := m.Write(pb); err != nil {\n\t\tpanic(fmt.Errorf(\"error happened while collecting metrics: %w\", err))\n\t}\n\tif pb.Gauge != nil {\n\t\treturn pb.Gauge.GetValue()\n\t}\n\tif pb.Counter != nil {\n\t\treturn pb.Counter.GetValue()\n\t}\n\tif pb.Untyped != nil {\n\t\treturn pb.Untyped.GetValue()\n\t}\n\tpanic(fmt.Errorf(\"collected a non-gauge/counter/untyped metric: %s\", pb))\n}\n\nfunc testConf() Config {\n\tconf := defaultConf\n\tconf.CacheDir = filepath.Join(conf.CacheDir, fmt.Sprintf(\"%d\", time.Now().UnixNano()))\n\treturn conf\n}\n\nfunc TestNewCacheStore(t *testing.T) {\n\tconf := testConf()\n\tdefer os.RemoveAll(conf.CacheDir)\n\ts := newCacheStore(nil, conf.CacheDir, 1<<30, conf.CacheItems, 1, &conf, nil)\n\tif s == nil {\n\t\tt.Fatalf(\"Create new cache store failed\")\n\t}\n}\n\nfunc TestMetrics(t *testing.T) {\n\tconf := testConf()\n\tdefer os.RemoveAll(conf.CacheDir)\n\tm := newCacheManager(&conf, nil, nil)\n\tmetrics := m.(*cacheManager).metrics\n\ts := m.(*cacheManager).stores[0]\n\tcontent := []byte(\"helloworld\")\n\tp := NewPage(content)\n\ts.cache(\"test\", p, true, false)\n\t// Waiting for the cache to be flushed\n\ttime.Sleep(time.Millisecond * 100)\n\tif toFloat64(metrics.cacheWrites) != 1.0 {\n\t\tt.Fatalf(\"expect the cacheWrites is 1\")\n\t}\n\n\tif toFloat64(metrics.cacheWriteBytes) != float64(len(content)) {\n\t\tt.Fatalf(\"expect the cacheWriteBytes is %d\", len(content))\n\t}\n\n\tif toFloat64(metrics.stageBlocks) != 0.0 {\n\t\tt.Fatalf(\"expect the stageBlocks is %d\", len(content))\n\t}\n\n\tif toFloat64(metrics.stageBlockBytes) != 0.0 {\n\t\tt.Fatalf(\"expect the stageBlockBytes is %d\", len(content))\n\t}\n\tkey := fmt.Sprintf(\"chunks/0/5/5000_2_%d\", len(content))\n\tstagingPath, err := m.stage(key, content)\n\tif err != nil {\n\t\tt.Fatalf(\"stage failed: %s\", err)\n\t}\n\tif toFloat64(metrics.stageBlocks) != 1.0 {\n\t\tt.Fatalf(\"expect the stageBlocks is %d\", len(content))\n\t}\n\n\tif toFloat64(metrics.stageBlockBytes) != float64(len(content)) {\n\t\tt.Fatalf(\"expect the stageBlockBytes is %d\", len(content))\n\t}\n\terr = m.removeStage(key)\n\tif err != nil {\n\t\tt.Fatalf(\"faild to remove stage\")\n\t}\n\n\tif toFloat64(metrics.stageBlocks) != 0.0 {\n\t\tt.Fatalf(\"expect the stageBlocks is %d\", len(content))\n\t}\n\n\tif toFloat64(metrics.stageBlockBytes) != 0.0 {\n\t\tt.Fatalf(\"expect the stageBlockBytes is %d\", len(content))\n\t}\n\n\tif _, err := os.Stat(stagingPath); err != nil && !os.IsNotExist(err) {\n\t\tt.Fatalf(\"expect the stageingPath %s not exists\", stagingPath)\n\t}\n}\n\nfunc TestScanCached(t *testing.T) {\n\tvar err error\n\tcfg := defaultConf\n\tcfg.CacheEviction = EvictionNone\n\tcache := &cacheStore{\n\t\topTs: make(map[time.Duration]func() error),\n\t}\n\tcache.state = newDCState(dcUnchanged, cache)\n\tcache.keys, err = NewKeyIndex(&cfg)\n\trequire.NoError(t, err)\n\tcache.dir = \"/tmp/jfstest_scan\"\n\trawDir := filepath.Join(cache.dir, cacheDir)\n\tif err := os.MkdirAll(rawDir, 0755); err != nil {\n\t\tt.Fatalf(\"mkdir %s: %s\", rawDir, err)\n\t}\n\tnum := 10\n\tfor i := 0; i < num; i++ {\n\t\tif f, err := os.Create(filepath.Join(rawDir, fmt.Sprintf(\"test%d_1024\", i))); err == nil {\n\t\t\t_ = f.Close()\n\t\t}\n\t}\n\tdefer os.RemoveAll(rawDir)\n\tcache.scanCached()\n\trequire.Equal(t, num, cache.keys.len())\n}\n\nfunc TestChecksum(t *testing.T) {\n\tconf := testConf()\n\tconf.FreeSpace = 0.01\n\tconf.CacheEviction = EvictionNone\n\tdefer os.RemoveAll(conf.CacheDir)\n\tm := new(cacheManagerMetrics)\n\tm.initMetrics()\n\ts := newCacheStore(m, conf.CacheDir, 1<<30, conf.CacheItems, 1, &conf, nil)\n\tk1 := \"0_0_10\" // no checksum\n\tk2 := \"1_0_10\"\n\tk3 := \"2_1_102400\"\n\tk4 := \"3_5_102400\" // corrupt data\n\tk5 := \"4_8_1048576\"\n\n\tp := NewPage([]byte(\"helloworld\"))\n\tdefer p.Release()\n\ts.cache(k1, p, true, false)\n\n\ts.checksum = CsFull\n\ts.cache(k2, p, true, false)\n\n\tbuf := make([]byte, 102400)\n\tutils.RandRead(buf)\n\ts.cache(k3, NewPage(buf), true, false)\n\n\tfpath := s.cachePath(k4)\n\tdir := filepath.Dir(fpath)\n\tif err := os.MkdirAll(dir, 0755); err != nil {\n\t\tt.Fatalf(\"mkdir parent dir %s: %s\", dir, err)\n\t}\n\tf, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE, s.mode)\n\tif err != nil {\n\t\tt.Fatalf(\"Create cache file %s: %s\", fpath, err)\n\t}\n\tif _, err = f.Write(buf); err != nil {\n\t\t_ = f.Close()\n\t\tt.Fatalf(\"Write cache file %s: %s\", fpath, err)\n\t}\n\tcorrupt := make([]byte, 102400)\n\tcopy(corrupt, buf)\n\tfor i := 98304; i < 102400; i++ { // reset 96K ~ 100K\n\t\tcorrupt[i] = 0\n\t}\n\tif _, err = f.Write(checksum(corrupt)); err != nil {\n\t\t_ = f.Close()\n\t\tt.Fatalf(\"Write checksum to cache file %s: %s\", fpath, err)\n\t}\n\t_ = f.Close()\n\ts.add(k4, 102400, uint32(time.Now().Unix()))\n\n\tbuf = make([]byte, 1048576)\n\tutils.RandRead(buf)\n\ts.cache(k5, NewPage(buf), true, false)\n\ttime.Sleep(time.Second * 5) // wait for cache file flushed\n\n\tcheck := func(key string, off int64, size int) error {\n\t\trc, err := s.load(key)\n\t\tif err != nil {\n\t\t\tt.Logf(\"CacheStore files in %s:\", s.dir)\n\t\t\tfilepath.Walk(s.dir, func(path string, info os.FileInfo, err error) error {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"error accessing %s: %v\", path, err)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tt.Logf(\"cache file: %s\", path)\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tt.Fatalf(\"CacheStore load key %s: %s\", key, err)\n\t\t}\n\t\tdefer rc.Close()\n\t\tbuf := make([]byte, size)\n\t\t_, err = rc.ReadAt(buf, off)\n\t\treturn err\n\t}\n\tcases := []struct {\n\t\tkey string\n\t\toff int64\n\t\tsize int\n\t\texpect bool\n\t}{\n\t\t{k1, 0, 10, true},\n\t\t{k1, 3, 5, true},\n\t\t{k2, 0, 10, true},\n\t\t{k2, 3, 5, true},\n\t\t{k3, 0, 102400, true},\n\t\t{k3, 8192, 92160, true}, // 8K ~ 98K\n\t\t{k4, 0, 102400, true},\n\t\t{k4, 8192, 92160, true}, // only CsExtend can detect the error\n\t\t{k5, 0, 1048576, true},\n\t\t{k5, 131072, 131072, true},\n\t\t{k5, 102400, 512000, true},\n\t}\n\tfor _, l := range []string{CsNone, CsFull, CsShrink, CsExtend} {\n\t\ts.checksum = l\n\t\tif l != CsNone {\n\t\t\tcases[6].expect = false\n\t\t}\n\t\tif l == CsExtend {\n\t\t\tcases[7].expect = false\n\t\t}\n\t\tfor _, c := range cases {\n\t\t\tif err = check(c.key, c.off, c.size); (err == nil) != c.expect {\n\t\t\t\tt.Fatalf(\"CacheStore check level %s case %+v: %s\", l, c, err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestExpand(t *testing.T) {\n\trs := expandDir(\"/not/exists/jfsCache\")\n\tif len(rs) != 1 || rs[0] != \"/not/exists/jfsCache\" {\n\t\tt.Errorf(\"expand: %v\", rs)\n\t\tt.FailNow()\n\t}\n\n\tdir := t.TempDir()\n\t_ = os.Mkdir(filepath.Join(dir, \"aaa1\"), 0755)\n\t_ = os.Mkdir(filepath.Join(dir, \"aaa2\"), 0755)\n\t_ = os.Mkdir(filepath.Join(dir, \"aaa3\"), 0755)\n\t_ = os.Mkdir(filepath.Join(dir, \"aaa3\", \"jfscache\"), 0755)\n\t_ = os.Mkdir(filepath.Join(dir, \"aaa3\", \"jfscache\", \"jfs\"), 0755)\n\n\trs = expandDir(filepath.Join(dir, \"aaa*\", \"jfscache\", \"jfs\"))\n\tif len(rs) != 3 || rs[0] != filepath.Join(dir, \"aaa1\", \"jfscache\", \"jfs\") {\n\t\tt.Errorf(\"expand: %v\", rs)\n\t\tt.FailNow()\n\t}\n}\n\nfunc BenchmarkLoadCached(b *testing.B) {\n\tconf := testConf()\n\tdefer os.RemoveAll(conf.CacheDir)\n\ts := newCacheStore(nil, conf.CacheDir, 1<<30, conf.CacheItems, 1, &conf, nil)\n\tp := NewPage(make([]byte, 1024))\n\tkey := \"/chunks/1_1024\"\n\ts.cache(key, p, false, false)\n\ttime.Sleep(time.Millisecond * 100)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif f, e := s.load(key); e == nil {\n\t\t\t_ = f.Close()\n\t\t} else {\n\t\t\tb.FailNow()\n\t\t}\n\t}\n}\n\nfunc BenchmarkLoadUncached(b *testing.B) {\n\tconf := testConf()\n\tdefer os.RemoveAll(conf.CacheDir)\n\ts := newCacheStore(nil, conf.CacheDir, 1<<30, conf.CacheItems, 1, &conf, nil)\n\tkey := \"chunks/222_1024\"\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif f, e := s.load(key); e == nil {\n\t\t\t_ = f.Close()\n\t\t}\n\t}\n}\n\nfunc TestCheckPath(t *testing.T) {\n\tcases := []struct {\n\t\tpath string\n\t\texpected bool\n\t}{\n\t\t// unix path style\n\t\t{path: \"chunks/111/222/3333_3333_3333\", expected: true},\n\t\t{path: \"chunks/111/222/3333_3333_0\", expected: true},\n\t\t{path: \"chunks/0/0/0_0_0\", expected: true},\n\t\t{path: \"chunks/01/10/0_01_0\", expected: true},\n\t\t{path: \"achunks/111/222/3333_3333_3333\", expected: false},\n\t\t{path: \"chunksa/111/222/3333_3333_3333\", expected: false},\n\t\t{path: \"chunksa\", expected: false},\n\t\t{path: \"chunks/111\", expected: false},\n\t\t{path: \"chunks/111/2222\", expected: false},\n\t\t{path: \"chunks/111/2222/3\", expected: false},\n\t\t{path: \"chunks/111/2222/3333_3333\", expected: false},\n\t\t{path: \"chunks/111/2222/3333_3333_3333_4444\", expected: false},\n\t\t{path: \"chunks/111/2222/3333_3333_3333/4444\", expected: false},\n\t\t{path: \"chunks/111_/2222/3333_3333_3333\", expected: false},\n\t\t{path: \"chunks/111/22_22/3333_3333_3333\", expected: false},\n\t\t{path: \"chunks/111/22_22/3333_3333_3333\", expected: false},\n\t\t{path: \"chunks/dd/222/3333_3333_0\", expected: true}, // hash prefix\n\t\t{path: \"chunks/FF/222/3333_3333_0\", expected: true}, // hash prefix\n\t\t{path: \"chunks/5D/222/3333_3333_0\", expected: true}, // hash prefix\n\t\t{path: \"chunks/D1/222/3333_3333_0\", expected: true}, // hash prefix\n\t\t{path: \"chunks/5DD/222/3333_3333_0\", expected: false},\n\t\t{path: \"chunks/111D/222/3333_3333_0\", expected: false},\n\t}\n\tfor _, c := range cases {\n\t\tif res := pathReg.MatchString(c.path); res != c.expected {\n\t\t\tt.Fatalf(\"check path %s expected %v but got %v\", c.path, c.expected, res)\n\t\t}\n\t}\n}\n\nfunc shutdownStore(s *cacheStore) {\n\ts.stateLock.Lock()\n\tdefer s.stateLock.Unlock()\n\ts.state.stop()\n\ts.state = newDCState(dcDown, s)\n}\n\nfunc TestCacheManager(t *testing.T) {\n\tconf := defaultConf\n\tconf.CacheDir = \"/tmp/diskCache0:/tmp/diskCache1:/tmp/diskCache2\"\n\tconf.AutoCreate = true\n\tdefer os.RemoveAll(\"/tmp/diskCache0\")\n\tdefer os.RemoveAll(\"/tmp/diskCache1\")\n\tdefer os.RemoveAll(\"/tmp/diskCache2\")\n\tmanager := newCacheManager(&conf, nil, nil)\n\trequire.True(t, !manager.isEmpty())\n\n\tm, ok := manager.(*cacheManager)\n\trequire.True(t, ok)\n\trequire.Equal(t, 3, m.length())\n\n\t// case: key rehash after store removal\n\tk1 := \"k1\"\n\tp1 := NewPage([]byte{1, 2, 3})\n\tdefer p1.Release()\n\tm.cache(k1, p1, true, false)\n\n\ts1 := m.getStore(k1)\n\trequire.NotNil(t, s1)\n\n\tPatchConvey(\"test getDiskUsage\", t, func() {\n\t\tMock(getDiskUsage).To(func(path string) (uint64, uint64, uint64, uint64) {\n\t\t\ttime.Sleep(time.Second * 10)\n\t\t\treturn 1, 1, 1, 1\n\t\t}).Build()\n\t\tvar wg sync.WaitGroup\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\ts1.Lock()\n\t\t\twg.Done()\n\t\t\ts1.cleanupFull()\n\t\t\ts1.Unlock()\n\t\t}()\n\n\t\twg.Wait()\n\t\tstart := time.Now()\n\t\ts1.load(k1)\n\t\tSo(time.Since(start), ShouldBeLessThan, time.Second*3)\n\t})\n\n\tm.Lock()\n\tshutdownStore(s1)\n\tm.Unlock()\n\ttime.Sleep(3 * time.Second)\n\n\trc, _ := m.load(k1)\n\trequire.Nil(t, rc)\n\t_, exist := m.exist(k1)\n\trequire.False(t, exist)\n\n\ts2 := m.getStore(k1)\n\trequire.NotNil(t, s2)\n\n\t// case: remove all store\n\tm.Lock()\n\tfor _, s := range m.storeMap {\n\t\tshutdownStore(s)\n\t}\n\tm.Unlock()\n\ttime.Sleep(3 * time.Second)\n\trequire.True(t, m.isEmpty())\n}\n\nfunc TestAtimeNotLost(t *testing.T) {\n\tfor _, eviction := range []string{EvictionNone, Eviction2Random, EvictionLRU} {\n\t\tcfg := defaultConf\n\t\tcfg.CacheEviction = eviction\n\t\tcfg.FreeSpace = 0.03\n\t\tm := newCacheManager(&cfg, nil, nil)\n\t\tkey := \"0_0_10\"\n\n\t\tp := NewPage([]byte(\"helloworld\"))\n\t\tdefer p.Release()\n\t\tm.cache(key, p, true, false)\n\t\ttime.Sleep(3 * time.Second)\n\n\t\t_, exist := m.exist(key) // touch atime\n\t\tif !exist {\n\t\t\tt.Fatalf(\"CacheStore key %s not exist\", key)\n\t\t}\n\t\ts := m.(*cacheManager).stores[0]\n\t\tatimeMem := s.keys.peekAtime(s.getCacheKey(key))\n\t\tif atimeMem == 0 {\n\t\t\tt.Fatalf(\"CacheStore key %s atime lost\", key)\n\t\t}\n\t\ts.scanCached() // should use atime from memory\n\t\tatimeAfterScan := s.keys.peekAtime(s.getCacheKey(key))\n\t\tif atimeAfterScan != atimeMem {\n\t\t\tt.Fatalf(\"CacheStore key %s atime lost after scan, before: %d, after: %d\", key, atimeMem, atimeAfterScan)\n\t\t}\n\t}\n}\nfunc TestSetlimitByFreeRatio(t *testing.T) {\n\tconf := testConf()\n\tdefer os.RemoveAll(conf.CacheDir)\n\tcache := newCacheStore(nil, conf.CacheDir, 1<<30, 1000, 1, &conf, nil)\n\n\tusage := DiskFreeRatio{\n\t\tspaceCap: 1 << 30,\n\t\tinodeCap: 1000,\n\t}\n\tfreeRatio := float32(0.2)\n\tcache.setLimitByFreeRatio(usage, 0.2)\n\n\texpectedSizeLimit := int64((1 - freeRatio + 0.05) * float32(usage.spaceCap))\n\tif cache.capacity > expectedSizeLimit {\n\t\tt.Fatalf(\"Expected capacity <= %d, but got %d\", expectedSizeLimit, cache.capacity)\n\t}\n\texpectedInodeLimit := int64((1 - freeRatio + 0.05) * float32(usage.inodeCap))\n\tif cache.maxItems > expectedInodeLimit && cache.maxItems != 0 {\n\t\tt.Fatalf(\"Expected maxItems <= %d, but got %d\", expectedInodeLimit, cache.maxItems)\n\t}\n}\n\nfunc TestSetLimitByFreeRatioUnknownInodesKeepExplicitMaxItems(t *testing.T) {\n\tconf := testConf()\n\tdefer os.RemoveAll(conf.CacheDir)\n\tcache := newCacheStore(nil, conf.CacheDir, 1<<30, 1000, 1, &conf, nil)\n\n\tusage := DiskFreeRatio{\n\t\tspaceCap: 1 << 30,\n\t\tinodeCap: 0,\n\t}\n\tcache.setLimitByFreeRatio(usage, 0.2)\n\trequire.Equal(t, int64(1000), cache.maxItems)\n}\n\nfunc TestUnknownInodeStatsShouldNotMarkCacheAsRawFull(t *testing.T) {\n\tPatchConvey(\"unknown inode stats should not trigger rawFull\", t, func() {\n\t\tMock(getDiskUsage).To(func(path string) (uint64, uint64, uint64, uint64) {\n\t\t\treturn 1 << 30, 1 << 30, 0, 0\n\t\t}).Build()\n\n\t\tconf := defaultConf\n\t\tconf.CacheDir = t.TempDir()\n\t\tm := new(cacheManagerMetrics)\n\t\tm.initMetrics()\n\t\ts := newCacheStore(m, conf.CacheDir, 1<<30, conf.CacheItems, 1, &conf, nil)\n\t\tdefer shutdownStore(s)\n\n\t\trequire.Never(t, func() bool {\n\t\t\ts.Lock()\n\t\t\tdefer s.Unlock()\n\t\t\treturn s.rawFull\n\t\t}, 1500*time.Millisecond, 100*time.Millisecond)\n\t})\n}\n\nfunc Test2RandomEviction(t *testing.T) {\n\tConvey(\"Test2RandomEviction-CacheFull\", t, func() {\n\t\tdir := t.TempDir()\n\t\tdefer os.RemoveAll(dir)\n\t\tconf := defaultConf\n\t\tconf.FreeSpace = 0.00001\n\t\tconf.CacheScanInterval = -1 // Disable periodic scan\n\t\tconf.CacheSize = 1 << 30\n\t\tconf.CacheItems = 10 // Max 10 items to easily trigger eviction\n\n\t\tm := new(cacheManagerMetrics)\n\t\tm.initMetrics()\n\t\ts := newCacheStore(m, filepath.Join(dir, \"diskCache\"), int64(conf.CacheSize), conf.CacheItems, 1, &conf, nil)\n\t\trequire.NotNil(t, s)\n\t\tif _, ok := s.keys.(*randomEviction); !ok {\n\t\t\tt.Fatalf(\"Expected randomEviction, but got %T\", s.keys)\n\t\t}\n\n\t\t// Add items with distinct atimes\n\t\tfor i := 1; i <= 20; i++ {\n\t\t\tkey := fmt.Sprintf(\"%d_%d_1024\", i, i)\n\t\t\ts.add(key, 1024, uint32(time.Now().Add(time.Duration(i)*time.Second).Unix())) // New items have larger atime\n\t\t\trequire.LessOrEqual(t, int64(s.keys.len()), conf.CacheItems, \"Cache should not exceed max items limit during addition\")\n\t\t\trequire.Greater(t, s.keys.len(), 0, \"Cache should always have items after addition\")\n\t\t}\n\t})\n}\n\nfunc TestLruEviction(t *testing.T) {\n\tConvey(\"TestLruEviction-CacheFull\", t, func() {\n\t\tdir := t.TempDir()\n\t\tdefer os.RemoveAll(dir)\n\t\tconf := defaultConf\n\t\tconf.CacheEviction = EvictionLRU\n\t\tconf.FreeSpace = 0.00001\n\t\tconf.CacheScanInterval = -1 // Disable periodic scan\n\t\tconf.CacheSize = 1 << 30\n\t\tconf.CacheItems = 10 // Max 10 items to easily trigger eviction\n\n\t\tm := new(cacheManagerMetrics)\n\t\tm.initMetrics()\n\t\ts := newCacheStore(m, filepath.Join(dir, \"diskCache\"), int64(conf.CacheSize), conf.CacheItems, 1, &conf, nil)\n\t\trequire.NotNil(t, s)\n\t\tle := s.keys.(*lruEviction)\n\n\t\t// Add items with distinct atimes\n\t\tfor i := 1; i <= 20; i++ {\n\t\t\tkey := fmt.Sprintf(\"%d_%d_1024\", i, i)\n\t\t\ts.add(key, 1024, uint32(time.Now().Add(time.Duration(i)*time.Second).Unix())) // New items have larger atime\n\t\t\trequire.True(t, le.verifyHeap())\n\t\t\trequire.LessOrEqual(t, int64(s.keys.len()), conf.CacheItems, \"Cache should not exceed max items limit during addition\")\n\t\t\trequire.Greater(t, s.keys.len(), 0, \"Cache should always have items after addition\")\n\t\t}\n\n\t\tcutIndex := 20 - conf.CacheItems\n\t\texpectedKeys := make(map[string]bool)\n\t\t// After eviction, the cache should only contain the newest items.\n\t\tfor i := cutIndex + 1; i <= 20; i++ {\n\t\t\tkey := fmt.Sprintf(\"%d_%d_1024\", i, i)\n\t\t\texpectedKeys[key] = true\n\t\t}\n\n\t\trequire.Equal(t, le.lruHeap.Len(), len(le.keys), \"Heap length should match keys length after insertion\")\n\t\trequire.Equal(t, len(expectedKeys), len(le.keys), \"Number of items in cache after eviction mismatch\")\n\t\trequire.Equal(t, len(expectedKeys), le.lruHeap.Len(), \"Number of items in heap after eviction mismatch\")\n\n\t\t// Verify the heap also contains the expected keys\n\t\ttempHeap := make(atimeHeap, le.lruHeap.Len())\n\t\tcopy(tempHeap, le.lruHeap)\n\t\tfor tempHeap.Len() > 0 {\n\t\t\titem := tempHeap.Pop().(heapItem)\n\t\t\trequire.Contains(t, expectedKeys, item.key.String(), \"Unexpected key found in heap: %s\", item.key.String())\n\t\t}\n\n\t\t// Verify all evicted keys are no longer in the cache\n\t\tfor i := int64(1); i <= cutIndex; i++ {\n\t\t\tkey := fmt.Sprintf(\"%d_%d_1024\", i, i)\n\t\t\t_, ok := le.keys[s.getCacheKey(key)]\n\t\t\trequire.False(t, ok, \"Evicted key %s still found in cache\", key)\n\t\t}\n\t})\n\n\tConvey(\"TestLruEviction-WriteBack\", t, func() {\n\t\tdir := t.TempDir()\n\t\tdefer os.RemoveAll(dir)\n\t\tconf := defaultConf\n\t\tconf.CacheEviction = EvictionLRU\n\t\tconf.Writeback = true\n\t\tconf.FreeSpace = 0.00001\n\t\tconf.CacheScanInterval = -1 // Disable periodic scan\n\t\tconf.CacheSize = 1 << 30\n\t\tconf.CacheItems = 10 // Max 10 items to easily trigger eviction\n\n\t\t// TODO: delete me\n\t\tm := new(cacheManagerMetrics)\n\t\tm.initMetrics()\n\t\ts := newCacheStore(m, filepath.Join(dir, \"diskCache\"), int64(conf.CacheSize), conf.CacheItems, 1, &conf, nil)\n\t\trequire.NotNil(t, s)\n\t\tle := s.keys.(*lruEviction)\n\n\t\t// Add items with distinct atimes\n\t\tblockPlaceHolder := []byte(\"test data\")\n\t\tfor i := 1; i <= 20; i++ {\n\t\t\tkey := fmt.Sprintf(\"%d_%d_9\", i, i)\n\t\t\t_, err := s.stage(key, blockPlaceHolder)\n\t\t\trequire.True(t, le.verifyHeap())\n\t\t\trequire.NoError(t, err, \"Failed to stage data for key %s\", key)\n\t\t}\n\t\trequire.Equal(t, 20, len(le.keys), \"Cache should contain 20 staged items even if full\")\n\t\trequire.Equal(t, 0, len(le.lruHeap), \"Staged items should not be in the LRU heap\")\n\n\t\ts.Lock()\n\t\ts.cleanupFull()\n\t\ts.Unlock()\n\t\tfor i := 1; i <= 20; i++ {\n\t\t\tkey := fmt.Sprintf(\"%d_%d_9\", i, i)\n\t\t\ts.uploaded(key, len(blockPlaceHolder))\n\t\t}\n\t\trequire.Equal(t, len(le.keys), le.lruHeap.Len(), \"Heap length should match keys length after staged items are uploaded\")\n\n\t\ts.maxItems = 1\n\t\ts.Lock()\n\t\ts.cleanupFull()\n\t\ts.Unlock()\n\t\trequire.Equal(t, 0, len(le.keys), \"Cache should be empty by cleanupFull after setting maxItems to 1\")\n\t\trequire.Equal(t, 0, len(le.lruHeap), \"LRU heap should be empty by cleanupFull after setting maxItems to 1\")\n\t})\n}\n\nfunc TestCooldownAtimeOnWriteFixedOnLoad(t *testing.T) {\n\tdir := t.TempDir()\n\tconf := defaultConf\n\tconf.CacheExpire = time.Hour\n\tconf.CacheEviction = EvictionNone\n\tm := new(cacheManagerMetrics)\n\tm.initMetrics()\n\tcache := newCacheStore(m, dir, 1<<30, 1000, 1, &conf, nil)\n\tkey := \"0_0_4\"\n\n\tPatchConvey(\"mock time.Now to avoid drift\", t, func() {\n\t\tfixedTime := time.Date(2025, 1, 28, 12, 0, 0, 0, time.UTC)\n\t\tMock(time.Now).Return(fixedTime).Build()\n\t\tpath, err := cache.stage(key, []byte(\"test\"))\n\t\trequire.NoError(t, err)\n\t\trequire.NotEmpty(t, path)\n\t\texpectedCooldownAtime := uint32(fixedTime.Add(-conf.CacheExpire / 2).Unix())\n\t\trequire.Equal(t, expectedCooldownAtime, cache.keys.peekAtime(cache.getCacheKey(key)))\n\t\trc, err := cache.load(key)\n\t\trequire.NoError(t, err)\n\t\trequire.NotNil(t, rc)\n\t\tdefer rc.Close()\n\t\trequire.Equal(t, uint32(fixedTime.Unix()), cache.keys.peekAtime(cache.getCacheKey(key)))\n\t})\n}\n" }, { "path": "pkg/chunk/mem_cache.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"errors\"\n\t\"runtime\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n)\n\ntype memItem struct {\n\tatime time.Time\n\tpage *Page\n}\n\ntype memcache struct {\n\tsync.Mutex\n\tcapacity int64\n\tmaxItems int64\n\tused int64\n\tpages map[string]memItem\n\teviction string\n\tcacheExpire time.Duration\n\n\tmetrics *cacheManagerMetrics\n}\n\nfunc newMemStore(config *Config, metrics *cacheManagerMetrics) *memcache {\n\tc := &memcache{\n\t\tcapacity: int64(config.CacheSize),\n\t\tmaxItems: config.CacheItems,\n\t\tpages: make(map[string]memItem),\n\t\teviction: config.CacheEviction,\n\t\tcacheExpire: config.CacheExpire,\n\t\tmetrics: metrics,\n\t}\n\truntime.SetFinalizer(c, func(c *memcache) {\n\t\tfor _, p := range c.pages {\n\t\t\tp.page.Release()\n\t\t}\n\t\tc.pages = nil\n\t})\n\tif c.cacheExpire > 0 {\n\t\tgo c.cleanupExpire()\n\t}\n\treturn c\n}\n\nfunc (c *memcache) removeStage(key string) error {\n\treturn nil\n}\n\nfunc (c *memcache) usedMemory() int64 {\n\tc.Lock()\n\tdefer c.Unlock()\n\treturn c.used\n}\n\nfunc (c *memcache) stats() (int64, int64) {\n\tc.Lock()\n\tdefer c.Unlock()\n\treturn int64(len(c.pages)), c.used\n}\n\nfunc (c *memcache) cache(key string, p *Page, force, dropCache bool) {\n\tif !c.enabled() {\n\t\treturn\n\t}\n\tc.Lock()\n\tdefer c.Unlock()\n\tif c.full() && c.eviction == EvictionNone {\n\t\tlogger.Debugf(\"Caching is full, drop %s (%d bytes)\", key, len(p.Data))\n\t\tc.metrics.cacheDrops.Add(1)\n\t\treturn\n\t}\n\tif _, ok := c.pages[key]; ok {\n\t\treturn\n\t}\n\tsize := int64(cap(p.Data))\n\tc.metrics.cacheWrites.Add(1)\n\tc.metrics.cacheWriteBytes.Add(float64(size))\n\tp.Acquire()\n\tc.pages[key] = memItem{time.Now(), p}\n\tc.used += size\n\tif c.full() && c.eviction != EvictionNone {\n\t\tc.cleanup()\n\t}\n}\n\nfunc (c *memcache) delete(key string, p *Page) {\n\tsize := int64(cap(p.Data))\n\tc.used -= size\n\tp.Release()\n\tdelete(c.pages, key)\n}\n\nfunc (c *memcache) remove(key string, staging bool) {\n\tc.Lock()\n\tdefer c.Unlock()\n\tif item, ok := c.pages[key]; ok {\n\t\tc.delete(key, item.page)\n\t\tlogger.Debugf(\"remove %s from cache\", key)\n\t}\n}\n\nfunc (c *memcache) load(key string) (ReadCloser, error) {\n\tc.Lock()\n\tdefer c.Unlock()\n\tif item, ok := c.pages[key]; ok {\n\t\tc.pages[key] = memItem{time.Now(), item.page}\n\t\treturn NewPageReader(item.page), nil\n\t}\n\treturn nil, errNotCached\n}\n\nfunc (c *memcache) exist(key string) (string, bool) {\n\tif !c.enabled() {\n\t\treturn \"\", false\n\t}\n\tc.Lock()\n\tdefer c.Unlock()\n\tif item, ok := c.pages[key]; ok {\n\t\tc.pages[key] = memItem{time.Now(), item.page}\n\t\treturn \"memory\", true\n\t}\n\treturn \"\", false\n}\n\n// locked\nfunc (c *memcache) cleanup() {\n\tvar cnt int\n\tvar lastKey string\n\tvar lastValue memItem\n\tvar now = time.Now()\n\t// for each two random keys, then compare the access time, evict the older one\n\tfor k, v := range c.pages {\n\t\tif cnt == 0 || lastValue.atime.After(v.atime) {\n\t\t\tlastKey = k\n\t\t\tlastValue = v\n\t\t}\n\t\tcnt++\n\t\tif cnt > 1 {\n\t\t\tlogger.Debugf(\"remove %s from cache, age: %d\", lastKey, now.Sub(lastValue.atime))\n\t\t\tc.metrics.cacheEvicts.Add(1)\n\t\t\tc.delete(lastKey, lastValue.page)\n\t\t\tcnt = 0\n\t\t\tif !c.full() {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (c *memcache) enabled() bool {\n\treturn c.capacity > 0\n}\n\nfunc (c *memcache) full() bool {\n\treturn c.used > c.capacity || (c.maxItems != 0 && int64(len(c.pages)) > c.maxItems)\n}\n\nfunc (c *memcache) cleanupExpire() {\n\tvar interval = time.Minute\n\tif c.cacheExpire < time.Minute {\n\t\tinterval = c.cacheExpire\n\t}\n\tfor {\n\t\tvar freed int64\n\t\tvar cnt, deleted int\n\t\tc.Lock()\n\t\tcutoff := time.Now().Add(-c.cacheExpire)\n\t\tfor k, v := range c.pages {\n\t\t\tcnt++\n\t\t\tif cnt > 1e3 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif v.atime.Before(cutoff) {\n\t\t\t\tdeleted++\n\t\t\t\tfreed += int64(cap(v.page.Data))\n\t\t\t\tc.metrics.cacheEvicts.Add(1)\n\t\t\t\tc.delete(k, v.page)\n\t\t\t}\n\t\t}\n\t\tc.Unlock()\n\t\tif deleted > 0 {\n\t\t\tlogger.Debugf(\"Expired cache blocks: %d blocks (%s), remaining: %d blocks (%s)\", deleted, humanize.IBytes(uint64(freed)), len(c.pages), humanize.IBytes(uint64(c.used)))\n\t\t}\n\t\ttime.Sleep(interval / 1000 * time.Duration((cnt+1-deleted)*1000/(cnt+1)))\n\t}\n}\n\nfunc (c *memcache) stage(key string, data []byte) (string, error) {\n\treturn \"\", errors.New(\"not supported\")\n}\nfunc (c *memcache) uploaded(key string, size int) {}\nfunc (c *memcache) isEmpty() bool { return false }\nfunc (c *memcache) getMetrics() *cacheManagerMetrics { return c.metrics }\n" }, { "path": "pkg/chunk/metrics.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\n// CacheManager Metrics\ntype cacheManagerMetrics struct {\n\tcacheDrops prometheus.Counter\n\tcacheWrites prometheus.Counter\n\tcacheEvicts prometheus.Counter\n\tcacheWriteBytes prometheus.Counter\n\tcacheWriteHist prometheus.Histogram\n\tstageBlocks prometheus.Gauge\n\tstageBlockBytes prometheus.Gauge\n\tstageWriteBytes prometheus.Counter\n}\n\nfunc newCacheManagerMetrics(reg prometheus.Registerer) *cacheManagerMetrics {\n\tmetrics := &cacheManagerMetrics{}\n\tmetrics.initMetrics()\n\tmetrics.registerMetrics(reg)\n\treturn metrics\n}\n\nfunc (c *cacheManagerMetrics) registerMetrics(reg prometheus.Registerer) {\n\tif reg != nil {\n\t\treg.MustRegister(c.cacheDrops)\n\t\treg.MustRegister(c.cacheWrites)\n\t\treg.MustRegister(c.cacheEvicts)\n\t\treg.MustRegister(c.cacheWriteHist)\n\t\treg.MustRegister(c.cacheWriteBytes)\n\t\treg.MustRegister(c.stageBlocks)\n\t\treg.MustRegister(c.stageBlockBytes)\n\t\treg.MustRegister(c.stageWriteBytes)\n\t\treg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\t\tName: \"staging_writing_blocks\",\n\t\t\tHelp: \"Number of writing blocks in staging.\",\n\t\t}, func() float64 {\n\t\t\treturn float64(stagingBlocks.Load())\n\t\t}))\n\t}\n}\n\nfunc (c *cacheManagerMetrics) initMetrics() {\n\tc.cacheDrops = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_drops\",\n\t\tHelp: \"dropped block\",\n\t})\n\tc.cacheWrites = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_writes\",\n\t\tHelp: \"written cached block\",\n\t})\n\tc.cacheEvicts = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_evicts\",\n\t\tHelp: \"evicted cache blocks\",\n\t})\n\tc.cacheWriteBytes = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"blockcache_write_bytes\",\n\t\tHelp: \"write bytes of cached block\",\n\t})\n\tc.cacheWriteHist = prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"blockcache_write_hist_seconds\",\n\t\tHelp: \"write cached block latency distribution\",\n\t\tBuckets: prometheus.ExponentialBuckets(0.00001, 2, 20),\n\t})\n\tc.stageBlocks = prometheus.NewGauge(prometheus.GaugeOpts{\n\t\tName: \"staging_blocks\",\n\t\tHelp: \"Number of blocks in the staging path.\",\n\t})\n\tc.stageBlockBytes = prometheus.NewGauge(prometheus.GaugeOpts{\n\t\tName: \"staging_block_bytes\",\n\t\tHelp: \"Total bytes of blocks in the staging path.\",\n\t})\n\tc.stageWriteBytes = prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"staging_write_bytes\",\n\t\tHelp: \"write bytes of blocks in the staging path.\",\n\t})\n}\n" }, { "path": "pkg/chunk/page.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"os\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"sync/atomic\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nvar pageStack = os.Getenv(\"JFS_PAGE_STACK\") != \"\"\n\n// Page is a page with refcount\ntype Page struct {\n\trefs int32\n\toffheap bool\n\tdep *Page\n\tData []byte\n\tstack []byte\n}\n\n// NewPage create a new page.\nfunc NewPage(data []byte) *Page {\n\treturn &Page{refs: 1, Data: data}\n}\n\nfunc NewOffPage(size int) *Page {\n\tif size <= 0 {\n\t\tpanic(\"size of page should > 0\")\n\t}\n\tp := utils.Alloc(size)\n\tpage := &Page{refs: 1, offheap: true, Data: p}\n\tif pageStack {\n\t\tpage.stack = debug.Stack()\n\t}\n\truntime.SetFinalizer(page, func(p *Page) {\n\t\trefcnt := atomic.LoadInt32(&p.refs)\n\t\tif refcnt != 0 {\n\t\t\tlogger.Errorf(\"refcount of page %p (%d bytes) is not zero: %d, created by: %s\", p, cap(p.Data), refcnt, string(p.stack))\n\t\t\tif refcnt > 0 {\n\t\t\t\tp.Release()\n\t\t\t}\n\t\t}\n\t})\n\treturn page\n}\n\nfunc (p *Page) Slice(off, len int) *Page {\n\tp.Acquire()\n\tnp := NewPage(p.Data[off : off+len])\n\tnp.dep = p\n\treturn np\n}\n\n// Acquire increase the refcount\nfunc (p *Page) Acquire() {\n\tif pageStack {\n\t\tp.stack = append(p.stack, debug.Stack()...)\n\t}\n\tatomic.AddInt32(&p.refs, 1)\n}\n\n// Release decrease the refcount\nfunc (p *Page) Release() {\n\tif pageStack {\n\t\tp.stack = append(p.stack, debug.Stack()...)\n\t}\n\tif atomic.AddInt32(&p.refs, -1) == 0 {\n\t\tif p.offheap {\n\t\t\tutils.Free(p.Data)\n\t\t}\n\t\tif p.dep != nil {\n\t\t\tp.dep.Release()\n\t\t\tp.dep = nil\n\t\t}\n\t\tp.Data = nil\n\t}\n}\n\ntype pageReader struct {\n\tp *Page\n\toff int\n}\n\nfunc NewPageReader(p *Page) *pageReader {\n\tp.Acquire()\n\treturn &pageReader{p, 0}\n}\n\nfunc (r *pageReader) Read(buf []byte) (int, error) {\n\tn, err := r.ReadAt(buf, int64(r.off))\n\tr.off += n\n\treturn n, err\n}\n\nfunc (r *pageReader) ReadAt(buf []byte, off int64) (int, error) {\n\tif len(buf) == 0 {\n\t\treturn 0, nil\n\t}\n\tif r.p == nil {\n\t\treturn 0, errors.New(\"page is already released\")\n\t}\n\tif int(off) == len(r.p.Data) {\n\t\treturn 0, io.EOF\n\t}\n\tn := copy(buf, r.p.Data[off:])\n\tif n < len(buf) {\n\t\treturn n, io.EOF\n\t}\n\treturn n, nil\n}\n\nfunc (r *pageReader) Close() error {\n\tif r.p != nil {\n\t\tr.p.Release()\n\t\tr.p = nil\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/chunk/page_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"io\"\n\t\"testing\"\n)\n\nfunc TestPage(t *testing.T) {\n\tp1 := NewOffPage(1)\n\tif len(p1.Data) != 1 {\n\t\tt.Fail()\n\t}\n\tif cap(p1.Data) != 1 {\n\t\tt.Fail()\n\t}\n\tp1.Acquire()\n\tp1.Release()\n\tif p1.Data == nil {\n\t\tt.Fail()\n\t}\n\n\tp2 := p1.Slice(0, 1)\n\tp1.Release()\n\tif p1.Data == nil {\n\t\tt.Fail()\n\t}\n\n\tp2.Release()\n\tif p2.Data != nil {\n\t\tt.Fail()\n\t}\n\tif p1.Data != nil {\n\t\tt.Fail()\n\t}\n}\n\nfunc TestPageReader(t *testing.T) {\n\tdata := []byte(\"hello\")\n\tp := NewPage(data)\n\tr := NewPageReader(p)\n\n\tif n, err := r.Read(nil); n != 0 || err != nil {\n\t\tt.Fatalf(\"read should return 0\")\n\t}\n\tbuf := make([]byte, 3)\n\tif n, err := r.Read(buf); n != 3 || err != nil {\n\t\tt.Fatalf(\"read should return 3 but got %d\", n)\n\t}\n\tif n, err := r.Read(buf); n != 2 || (err != nil && err != io.EOF) {\n\t\tt.Fatalf(\"read should return 2 but got %d\", n)\n\t}\n\tif n, err := r.Read(buf); n != 0 || err != io.EOF {\n\t\tt.Fatalf(\"read should return 0\")\n\t}\n\tif n, err := r.ReadAt(buf, 4); n != 1 || (err != nil && err != io.EOF) {\n\t\tt.Fatalf(\"read should return 1\")\n\t}\n\tif n, err := r.ReadAt(buf, 5); n != 0 || err != io.EOF {\n\t\tt.Fatalf(\"read should return 0\")\n\t}\n\t_ = r.Close()\n\tif n, err := r.ReadAt(buf, 5); n != 0 || err == nil {\n\t\tt.Fatalf(\"read should fail after close\")\n\t}\n}\n" }, { "path": "pkg/chunk/prefetch.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"sync\"\n)\n\ntype prefetcher struct {\n\tsync.Mutex\n\tpending chan string\n\tbusy map[string]bool\n\top func(key string)\n}\n\nfunc newPrefetcher(parallel int, fetch func(string)) *prefetcher {\n\tp := &prefetcher{\n\t\tpending: make(chan string, 10),\n\t\tbusy: make(map[string]bool),\n\t\top: fetch,\n\t}\n\tfor i := 0; i < parallel; i++ {\n\t\tgo p.do()\n\t}\n\treturn p\n}\n\nfunc (p *prefetcher) do() {\n\tfor key := range p.pending {\n\t\tp.op(key)\n\n\t\tp.Lock()\n\t\tdelete(p.busy, key)\n\t\tp.Unlock()\n\t}\n}\n\nfunc (p *prefetcher) fetch(key string) {\n\tp.Lock()\n\tdefer p.Unlock()\n\tif _, ok := p.busy[key]; ok {\n\t\treturn\n\t}\n\tselect {\n\tcase p.pending <- key:\n\t\tp.busy[key] = true\n\tdefault:\n\t}\n}\n" }, { "path": "pkg/chunk/prefetch_test.go", "content": "package chunk\n\nimport (\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestPrefetcher(t *testing.T) {\n\tt.Run(\"should fetch given keys\", func(t *testing.T) {\n\t\tkeys := []string{\"source/1\", \"source/2\", \"source/3\", \"source/4\"}\n\t\tchRes := make(chan string, len(keys))\n\t\tdefer close(chRes)\n\t\tf := newPrefetcher(2, func(k string) {\n\t\t\tchRes <- k + \"Done\"\n\t\t})\n\t\tfor _, k := range keys {\n\t\t\tf.fetch(k)\n\t\t}\n\t\tres := make(map[string]bool, len(keys))\n\t\tfor range keys {\n\t\t\tres[<-chRes] = true\n\t\t}\n\t\tif len(res) != len(keys) {\n\t\t\tt.Errorf(\"Incorrect number of keys fetched, expect: %d, got: %d\", len(keys), len(res))\n\t\t}\n\t\tfor _, k := range keys {\n\t\t\tif !res[k+\"Done\"] {\n\t\t\t\tt.Errorf(\"Key not fetched: %s\", k)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"should ignore duplicate keys\", func(t *testing.T) {\n\t\tvar counter int32\n\t\tf := newPrefetcher(4, func(k string) {\n\t\t\t// Introduce a little latency to mimic a slower fetch operation\n\t\t\t// so that our few duplicate keys can reach the prefetcher in the time period\n\t\t\ttime.Sleep(time.Millisecond)\n\t\t\tatomic.AddInt32(&counter, 1)\n\t\t})\n\t\tfor i := 0; i < 5; i++ {\n\t\t\tf.fetch(\"a\")\n\t\t}\n\t\tif atomic.LoadInt32(&counter) > 1 {\n\t\t\tt.Errorf(\"Duplicate keys fetched\")\n\t\t}\n\t})\n\n\tt.Run(\"should drop keys when pending queue is full\", func(t *testing.T) {\n\t\tconst maxPending = 10\n\t\tvar counter int32\n\n\t\tf := newPrefetcher(1, func(k string) {\n\t\t\tatomic.AddInt32(&counter, 1)\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t})\n\n\t\tfor i := 0; i < maxPending+1; i++ {\n\t\t\tf.fetch(string(rune('a' + i)))\n\t\t}\n\n\t\ttime.Sleep(50 * time.Millisecond)\n\n\t\tfinalCount := atomic.LoadInt32(&counter)\n\t\tif finalCount > maxPending {\n\t\t\tt.Errorf(\"Processed count %d exceeds queue capacity %d\", finalCount, maxPending)\n\t\t}\n\t})\n}\n" }, { "path": "pkg/chunk/singleflight.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport \"sync\"\n\ntype request struct {\n\twg sync.WaitGroup\n\tval *Page\n\tdups int\n\terr error\n}\n\ntype Controller struct {\n\tsync.Mutex\n\trs map[string]*request\n}\n\nfunc NewController() *Controller {\n\treturn &Controller{\n\t\trs: make(map[string]*request),\n\t}\n}\n\nfunc (con *Controller) Execute(key string, fn func() (*Page, error)) (*Page, error) {\n\tcon.Lock()\n\tif c, ok := con.rs[key]; ok {\n\t\tc.dups++\n\t\tcon.Unlock()\n\t\tc.wg.Wait()\n\t\treturn c.val, c.err\n\t}\n\tc := new(request)\n\tc.wg.Add(1)\n\tcon.rs[key] = c\n\tcon.Unlock()\n\n\tc.val, c.err = fn()\n\n\tcon.Lock()\n\tfor i := 0; i < c.dups; i++ {\n\t\t// Acquire for the pending Execute\n\t\tc.val.Acquire()\n\t}\n\tdelete(con.rs, key)\n\tcon.Unlock()\n\n\tc.wg.Done()\n\n\treturn c.val, c.err\n}\n\nfunc (con *Controller) TryPiggyback(key string) (*Page, error) {\n\tcon.Lock()\n\tif c, ok := con.rs[key]; ok {\n\t\tc.dups++\n\t\tcon.Unlock()\n\t\tc.wg.Wait()\n\t\treturn c.val, c.err\n\t}\n\tcon.Unlock()\n\treturn nil, nil\n}\n" }, { "path": "pkg/chunk/singleflight_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestSingleFlight(t *testing.T) {\n\tg := NewController()\n\tgp := &sync.WaitGroup{}\n\tvar cache sync.Map\n\tvar n int32\n\tvar piggyback atomic.Int64\n\titers := 100000\n\terrCh := make(chan error, iters)\n\n\tfor i := 0; i < iters; i++ {\n\t\tgp.Add(2)\n\t\tgo func(k int) {\n\t\t\tp, _ := g.Execute(strconv.Itoa(k/100), func() (*Page, error) {\n\t\t\t\ttime.Sleep(time.Microsecond * 500000) // In most cases 500ms is enough to run 100 goroutines\n\t\t\t\tatomic.AddInt32(&n, 1)\n\t\t\t\tpage := NewOffPage(100)\n\t\t\t\tcopy(page.Data, make([]byte, 100)) // zeroed\n\t\t\t\tcopy(page.Data, strconv.Itoa(k/100))\n\t\t\t\treturn page, nil\n\t\t\t})\n\t\t\tp.Release()\n\t\t\tcache.LoadOrStore(strconv.Itoa(k/100), p)\n\t\t\tgp.Done()\n\t\t}(i)\n\t\tgo func(k int) {\n\t\t\tdefer gp.Done()\n\t\t\tpage, _ := g.TryPiggyback(strconv.Itoa(k / 100))\n\t\t\tif page != nil {\n\t\t\t\texpected := make([]byte, 100)\n\t\t\t\tcopy(expected, strconv.Itoa(k/100))\n\t\t\t\tif bytes.Compare(page.Data, expected) != 0 {\n\t\t\t\t\terrCh <- fmt.Errorf(\"got %x, want %x, key: %d\", page.Data, expected, k/100)\n\t\t\t\t}\n\t\t\t\tpage.Release()\n\t\t\t\tpiggyback.Add(1)\n\t\t\t}\n\t\t}(i)\n\t}\n\tgp.Wait()\n\tclose(errCh)\n\n\tfor err := range errCh {\n\t\tt.Fatalf(\"Test failed: %v\", err)\n\t}\n\n\tnv := int(atomic.LoadInt32(&n))\n\tif nv != iters/100 {\n\t\tt.Fatalf(\"singleflight doesn't take effect: %v\", nv)\n\t}\n\tif piggyback.Load() == 0 {\n\t\tt.Fatal(\"never piggybacked?\")\n\t}\n\n\t// verify the ref\n\tcache.Range(func(key any, value any) bool {\n\t\tif value.(*Page).refs != 0 {\n\t\t\tt.Fatalf(\"refs of page is not 0, got: %d, key: %s\", value.(*Page).refs, key)\n\t\t}\n\t\treturn true\n\t})\n}\n" }, { "path": "pkg/chunk/utils_darwin.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n)\n\nfunc getAtime(fi os.FileInfo) time.Time {\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn time.Unix(sst.Atimespec.Unix())\n\t} else {\n\t\treturn fi.ModTime()\n\t}\n}\n\nfunc dropOSCache(r ReadCloser) {}\n" }, { "path": "pkg/chunk/utils_linux.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\nfunc getAtime(fi os.FileInfo) time.Time {\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn time.Unix(sst.Atim.Unix())\n\t}\n\treturn fi.ModTime()\n}\n\nfunc dropOSCache(r ReadCloser) {\n\tif cf, ok := r.(*cacheFile); ok {\n\t\t_ = unix.Fadvise(int(cf.Fd()), 0, 0, unix.FADV_DONTNEED)\n\t} else if f, ok := r.(*os.File); ok {\n\t\t_ = unix.Fadvise(int(f.Fd()), 0, 0, unix.FADV_DONTNEED)\n\t}\n}\n" }, { "path": "pkg/chunk/utils_unix.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"os\"\n\t\"syscall\"\n)\n\nfunc getNlink(fi os.FileInfo) int {\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn int(sst.Nlink)\n\t}\n\treturn 1\n}\n\nfunc getDiskUsage(path string) (uint64, uint64, uint64, uint64) {\n\tvar stat syscall.Statfs_t\n\tif err := syscall.Statfs(path, &stat); err == nil {\n\t\treturn stat.Blocks * uint64(stat.Bsize), stat.Bavail * uint64(stat.Bsize), stat.Files, stat.Ffree\n\t} else {\n\t\tlogger.Warnf(\"statfs %s: %s\", path, err)\n\t\treturn 1, 1, 1, 1\n\t}\n}\n\nfunc changeMode(dir string, st os.FileInfo, mode os.FileMode) {\n\tsst := st.Sys().(*syscall.Stat_t)\n\tif os.Getuid() == int(sst.Uid) {\n\t\t_ = os.Chmod(dir, mode)\n\t}\n}\n\nfunc inRootVolume(dir string) bool {\n\tdstat, err := os.Stat(dir)\n\tif err != nil {\n\t\tlogger.Warnf(\"stat `%s`: %s\", dir, err.Error())\n\t\treturn false\n\t}\n\trstat, err := os.Stat(\"/\")\n\tif err != nil {\n\t\tlogger.Warnf(\"stat `/`: %s\", err.Error())\n\t\treturn false\n\t}\n\treturn dstat.Sys().(*syscall.Stat_t).Dev == rstat.Sys().(*syscall.Stat_t).Dev\n}\n" }, { "path": "pkg/chunk/utils_unix_test.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"os\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc TestInRootVolume(t *testing.T) {\n\tif runtime.GOOS == \"windows\" {\n\t\tt.SkipNow()\n\t}\n\tif !inRootVolume(\"/\") {\n\t\tt.Fatal(\"`/` is in root volume\")\n\t}\n\tif inRootVolume(\".\") {\n\t\terr := os.MkdirAll(\"./__test__\", 0755)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tdefer os.RemoveAll(\"./__test__\")\n\t\tif !inRootVolume(\"./__test__\") {\n\t\t\tt.Fatal(\"`./__test__` is in root volume\")\n\t\t}\n\t}\n\tif !inRootVolume(\"/tmp\") {\n\t\terr := os.MkdirAll(\"/tmp/__jfs_test__\", 0755)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tdefer os.RemoveAll(\"/tmp/__jfs_test__\")\n\t\tif inRootVolume(\"/tmp/__jfs_test__\") {\n\t\t\tt.Fatal(\"`/tmp/__jfs_test__` is not in root volume\")\n\t\t}\n\t}\n}\n" }, { "path": "pkg/chunk/utils_windows.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage chunk\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\tsys \"golang.org/x/sys/windows\"\n)\n\nfunc getAtime(fi os.FileInfo) time.Time {\n\tstat, ok := fi.Sys().(*syscall.Win32FileAttributeData)\n\tif ok {\n\t\treturn time.Unix(0, stat.LastAccessTime.Nanoseconds())\n\t} else {\n\t\treturn time.Unix(0, 0)\n\t}\n}\n\nfunc dropOSCache(r ReadCloser) {}\n\nfunc getNlink(fi os.FileInfo) int {\n\treturn 1\n}\n\nfunc getDiskUsage(path string) (uint64, uint64, uint64, uint64) {\n\tvar freeBytes, total, totalFree uint64\n\terr := sys.GetDiskFreeSpaceEx(sys.StringToUTF16Ptr(path), &freeBytes, &total, &totalFree)\n\tif err != nil {\n\t\tlogger.Errorf(\"GetDiskFreeSpaceEx %s: %s\", path, err.Error())\n\t\treturn 1, 1, 1, 1\n\t}\n\treturn total, freeBytes, 1, 1\n}\n\nfunc changeMode(dir string, st os.FileInfo, mode os.FileMode) {}\n\nfunc inRootVolume(dir string) bool { return false }\n" }, { "path": "pkg/compress/compress.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage compress\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/DataDog/zstd\"\n\t\"github.com/hungys/go-lz4\"\n)\n\n// ZSTD_LEVEL compression level used by Zstd\nconst ZSTD_LEVEL = 1 // fastest\n\n// Compressor interface to be implemented by a compression algo\ntype Compressor interface {\n\tName() string\n\tCompressBound(int) int\n\tCompress(dst, src []byte) (int, error)\n\tDecompress(dst, src []byte) (int, error)\n}\n\n// NewCompressor returns a struct implementing Compressor interface\nfunc NewCompressor(algr string) Compressor {\n\talgr = strings.ToLower(algr)\n\tif algr == \"zstd\" {\n\t\treturn ZStandard{ZSTD_LEVEL}\n\t} else if algr == \"lz4\" {\n\t\treturn LZ4{}\n\t} else if algr == \"none\" || algr == \"\" {\n\t\treturn noOp{}\n\t}\n\treturn nil\n}\n\ntype noOp struct{}\n\nfunc (n noOp) Name() string { return \"Noop\" }\nfunc (n noOp) CompressBound(l int) int { return l }\nfunc (n noOp) Compress(dst, src []byte) (int, error) {\n\tif len(dst) < len(src) {\n\t\treturn 0, fmt.Errorf(\"buffer too short: %d < %d\", len(dst), len(src))\n\t}\n\tcopy(dst, src)\n\treturn len(src), nil\n}\nfunc (n noOp) Decompress(dst, src []byte) (int, error) {\n\tif len(dst) < len(src) {\n\t\treturn 0, fmt.Errorf(\"buffer too short: %d < %d\", len(dst), len(src))\n\t}\n\tcopy(dst, src)\n\treturn len(src), nil\n}\n\n// ZStandard implements Compressor interface using zstd library\ntype ZStandard struct {\n\tlevel int\n}\n\n// Name returns name of the algorithm Zstd\nfunc (n ZStandard) Name() string { return \"Zstd\" }\n\n// CompressBound max size of compressed data\nfunc (n ZStandard) CompressBound(l int) int { return zstd.CompressBound(l) }\n\n// Compress using Zstd\nfunc (n ZStandard) Compress(dst, src []byte) (int, error) {\n\td, err := zstd.CompressLevel(dst, src, n.level)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif len(d) > 0 && len(dst) > 0 && &d[0] != &dst[0] {\n\t\treturn 0, fmt.Errorf(\"buffer too short: %d < %d\", cap(dst), cap(d))\n\t}\n\treturn len(d), err\n}\n\n// Decompress using Zstd\nfunc (n ZStandard) Decompress(dst, src []byte) (int, error) {\n\td, err := zstd.Decompress(dst, src)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif len(d) > 0 && len(dst) > 0 && &d[0] != &dst[0] {\n\t\treturn 0, fmt.Errorf(\"buffer too short: %d < %d\", len(dst), len(d))\n\t}\n\treturn len(d), err\n}\n\n// LZ4 implements Compressor using LZ4 library\ntype LZ4 struct{}\n\n// Name returns name of the algorithm LZ4\nfunc (l LZ4) Name() string { return \"LZ4\" }\n\n// CompressBound max size of compressed data\nfunc (l LZ4) CompressBound(size int) int { return lz4.CompressBound(size) }\n\n// Compress using LZ4 algorithm\nfunc (l LZ4) Compress(dst, src []byte) (int, error) {\n\treturn lz4.CompressDefault(src, dst)\n}\n\n// Decompress using LZ4 algorithm\nfunc (l LZ4) Decompress(dst, src []byte) (int, error) {\n\tif len(src) == 0 {\n\t\treturn 0, fmt.Errorf(\"decompress an empty input\")\n\t}\n\treturn lz4.DecompressSafe(src, dst)\n}\n" }, { "path": "pkg/compress/compress_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage compress\n\nimport (\n\t\"io\"\n\t\"os\"\n\t\"testing\"\n)\n\nfunc testCompress(t *testing.T, c Compressor) {\n\tsrc := []byte(c.Name())\n\ttestIt := func(src []byte) {\n\t\tif len(src) > 1 {\n\t\t\t_, err := c.Compress(make([]byte, 1), src)\n\t\t\tif err == nil {\n\t\t\t\tt.Fatal(\"expect short buffer error, but got nil \")\n\t\t\t}\n\t\t}\n\t\tdst := make([]byte, c.CompressBound(len(src)))\n\t\tn, err := c.Compress(dst, src)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"compress: %s\", err)\n\t\t}\n\t\tif len(src) > 1 {\n\t\t\t_, err = c.Decompress(make([]byte, 1), dst[:n])\n\t\t\tif err == nil {\n\t\t\t\tt.Fatalf(\"expect short buffer error, but got nil\")\n\t\t\t}\n\t\t}\n\t\tsrc2 := make([]byte, len(src))\n\t\tn, err = c.Decompress(src2, dst[:n])\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"decompress: %s\", err)\n\t\t}\n\t\tif string(src2[:n]) != string(src) {\n\t\t\tt.Fatalf(\"expect %s but got %s\", string(src), string(src2))\n\t\t}\n\t}\n\n\ttestIt(src)\n\ttestIt(nil)\n\n\tif c.CompressBound(0) > 0 {\n\t\tn, err := c.Decompress(make([]byte, 100), src[:0])\n\t\tif err == nil || n > 0 {\n\t\t\tt.Fatalf(\"decompress should fail, but got %d\", n)\n\t\t}\n\t}\n}\n\nfunc TestUncompressed(t *testing.T) {\n\ttestCompress(t, NewCompressor(\"none\"))\n}\n\nfunc TestZstd(t *testing.T) {\n\ttestCompress(t, NewCompressor(\"zstd\"))\n}\n\nfunc TestLZ4(t *testing.T) {\n\ttestCompress(t, NewCompressor(\"lz4\"))\n}\n\nfunc benchmarkDecompress(b *testing.B, comp Compressor) {\n\tf, _ := os.Open(os.Getenv(\"PAYLOAD\"))\n\tvar c = make([]byte, 5<<20)\n\tvar d = make([]byte, 4<<20)\n\tn, err := io.ReadFull(f, d)\n\tf.Close()\n\tif err != nil {\n\t\tb.Skip()\n\t\treturn\n\t}\n\td = d[:n]\n\tn, err = comp.Compress(c[:4<<20], d)\n\tif err != nil {\n\t\tb.Errorf(\"compress: %s\", err)\n\t\tb.FailNow()\n\t}\n\tc = c[:n]\n\t// println(\"compres\", comp.Name(), len(c), len(d))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tn, err := comp.Decompress(d, c)\n\t\tif err != nil {\n\t\t\tb.Errorf(\"decompress %d %s\", n, err)\n\t\t\tb.FailNow()\n\t\t}\n\t\tb.SetBytes(int64(len(d)))\n\t}\n}\n\nfunc BenchmarkDecompressZstd(b *testing.B) {\n\tbenchmarkDecompress(b, NewCompressor(\"zstd\"))\n}\n\nfunc BenchmarkDecompressLZ4(b *testing.B) {\n\tbenchmarkDecompress(b, LZ4{})\n}\n\nfunc BenchmarkDecompressNone(b *testing.B) {\n\tbenchmarkDecompress(b, NewCompressor(\"none\"))\n}\n\nfunc benchmarkCompress(b *testing.B, comp Compressor) {\n\tf, _ := os.Open(os.Getenv(\"PAYLOAD\"))\n\tvar d = make([]byte, 4<<20)\n\tn, err := io.ReadFull(f, d)\n\tf.Close()\n\tif err != nil {\n\t\tb.Skip()\n\t\treturn\n\t}\n\td = d[:n]\n\tvar c = make([]byte, 5<<20)\n\t// println(\"compres\", comp.Name(), len(c), len(d))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tn, err := comp.Compress(c, d)\n\t\tif err != nil {\n\t\t\tb.Errorf(\"compress %d %s\", n, err)\n\t\t\tb.FailNow()\n\t\t}\n\t\tb.SetBytes(int64(len(d)))\n\t}\n}\n\nfunc BenchmarkCompressZstd(b *testing.B) {\n\tbenchmarkCompress(b, NewCompressor(\"Zstd\"))\n}\n\nfunc BenchmarkCompressCLZ4(b *testing.B) {\n\tbenchmarkCompress(b, LZ4{})\n}\nfunc BenchmarkCompressNone(b *testing.B) {\n\tbenchmarkCompress(b, NewCompressor(\"none\"))\n}\n" }, { "path": "pkg/fs/fs.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fs\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path\"\n\t\"runtime/trace\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/acl\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nvar logger = utils.GetLogger(\"juicefs\")\n\ntype Ino = meta.Ino\ntype Attr = meta.Attr\ntype LogContext = vfs.LogContext\n\nfunc IsExist(err error) bool {\n\treturn err == syscall.EEXIST || err == syscall.EACCES || err == syscall.EPERM\n}\n\nfunc IsNotExist(err error) bool {\n\treturn err == syscall.ENOENT\n}\n\nfunc IsNotEmpty(err error) bool {\n\treturn err == syscall.ENOTEMPTY\n}\n\nfunc errstr(e error) string {\n\tif e == nil {\n\t\treturn \"OK\"\n\t}\n\tif eno, ok := e.(syscall.Errno); ok && eno == 0 {\n\t\treturn \"OK\"\n\t}\n\treturn e.Error()\n}\n\ntype FileStat struct {\n\tname string\n\tinode Ino\n\tattr *Attr\n}\n\nfunc (fs *FileStat) Inode() Ino { return fs.inode }\nfunc (fs *FileStat) Name() string { return fs.name }\nfunc (fs *FileStat) Size() int64 { return int64(fs.attr.Length) }\nfunc (fs *FileStat) Mode() os.FileMode {\n\tattr := fs.attr\n\tmode := os.FileMode(attr.Mode & 0777)\n\tif attr.Mode&04000 != 0 {\n\t\tmode |= os.ModeSetuid\n\t}\n\tif attr.Mode&02000 != 0 {\n\t\tmode |= os.ModeSetgid\n\t}\n\tif attr.Mode&01000 != 0 {\n\t\tmode |= os.ModeSticky\n\t}\n\tif attr.AccessACL+attr.DefaultACL > 0 {\n\t\tmode |= 1 << 18\n\t}\n\tswitch attr.Typ {\n\tcase meta.TypeDirectory:\n\t\tmode |= os.ModeDir\n\tcase meta.TypeSymlink:\n\t\tmode |= os.ModeSymlink\n\tcase meta.TypeFile:\n\tdefault:\n\t}\n\treturn mode\n}\nfunc (fs *FileStat) ModTime() time.Time {\n\treturn time.Unix(fs.attr.Mtime, int64(fs.attr.Mtimensec))\n}\nfunc (fs *FileStat) IsDir() bool { return fs.attr.Typ == meta.TypeDirectory }\nfunc (fs *FileStat) IsSymlink() bool { return fs.attr.Typ == meta.TypeSymlink }\nfunc (fs *FileStat) Sys() interface{} { return fs.attr }\nfunc (fs *FileStat) Uid() int { return int(fs.attr.Uid) }\nfunc (fs *FileStat) Gid() int { return int(fs.attr.Gid) }\n\nfunc (fs *FileStat) Atime() int64 { return fs.attr.Atime*1000 + int64(fs.attr.Atimensec/1e6) }\nfunc (fs *FileStat) Mtime() int64 { return fs.attr.Mtime*1000 + int64(fs.attr.Mtimensec/1e6) }\n\nfunc (fs *FileStat) Attr() *Attr { return fs.attr }\n\nfunc AttrToFileInfo(inode Ino, attr *Attr) *FileStat {\n\treturn &FileStat{inode: inode, attr: attr}\n}\n\ntype entryCache struct {\n\tinode Ino\n\ttyp uint8\n\texpire time.Time\n}\n\ntype attrCache struct {\n\tattr Attr\n\texpire time.Time\n}\n\ntype FileSystem struct {\n\tconf *vfs.Config\n\treader vfs.DataReader\n\twriter vfs.DataWriter\n\tm meta.Meta\n\tstore chunk.ChunkStore\n\tcacheFiller *vfs.CacheFiller\n\n\tSuperuser string\n\tSupergroup string\n\n\tcacheM sync.Mutex\n\tentries map[Ino]map[string]*entryCache\n\tattrs map[Ino]*attrCache\n\tcheckAccessFile time.Duration\n\trotateAccessLog int64\n\tlogBuffer chan string\n\n\treadSizeHistogram prometheus.Histogram\n\twrittenSizeHistogram prometheus.Histogram\n\topsDurationsHistogram prometheus.Histogram\n\n\tregistry *prometheus.Registry\n\n\t// Pre-parsed subdir prefixes for fast path checking\n\tsubdirPrefixes []string\n}\n\ntype File struct {\n\tpath string\n\tinode Ino\n\tinfo *FileStat\n\tfs *FileSystem\n\n\tsync.Mutex\n\tflags uint32\n\toffset int64\n\trdata vfs.FileReader\n\twdata vfs.FileWriter\n\tdircache []os.FileInfo\n\tentries []*meta.Entry\n\tdata []byte\n}\n\nfunc NewFileSystem(conf *vfs.Config, m meta.Meta, d chunk.ChunkStore, registry *prometheus.Registry) (*FileSystem, error) {\n\treader := vfs.NewDataReader(conf, m, d)\n\tfs := &FileSystem{\n\t\tm: m,\n\t\tstore: d,\n\t\tconf: conf,\n\t\tcacheFiller: vfs.NewCacheFiller(conf, m, d),\n\t\treader: reader,\n\t\twriter: vfs.NewDataWriter(conf, m, d, reader),\n\t\tentries: make(map[meta.Ino]map[string]*entryCache),\n\t\tattrs: make(map[meta.Ino]*attrCache),\n\t\tcheckAccessFile: time.Minute,\n\t\trotateAccessLog: 300 << 20, // 300 MiB\n\n\t\treadSizeHistogram: prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\t\tName: \"sdk_read_size_bytes\",\n\t\t\tHelp: \"size of read distributions.\",\n\t\t\tBuckets: prometheus.LinearBuckets(4096, 4096, 32),\n\t\t}),\n\t\twrittenSizeHistogram: prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\t\tName: \"sdk_written_size_bytes\",\n\t\t\tHelp: \"size of write distributions.\",\n\t\t\tBuckets: prometheus.LinearBuckets(4096, 4096, 32),\n\t\t}),\n\t\topsDurationsHistogram: prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\t\tName: \"sdk_ops_durations_histogram_seconds\",\n\t\t\tHelp: \"Operations latency distributions.\",\n\t\t\tBuckets: prometheus.ExponentialBuckets(0.00001, 1.8, 29),\n\t\t}),\n\t\tregistry: registry,\n\t}\n\n\t// Pre-parse subdir prefixes for fast path checking\n\tif conf.Subdir != \"\" {\n\t\tsubdirs := strings.Split(conf.Subdir, \",\")\n\t\tfs.subdirPrefixes = make([]string, 0, len(subdirs))\n\t\tfor _, prefix := range subdirs {\n\t\t\tprefix = strings.TrimSpace(prefix)\n\t\t\tif prefix != \"\" {\n\t\t\t\tfs.subdirPrefixes = append(fs.subdirPrefixes, prefix)\n\t\t\t}\n\t\t}\n\t}\n\n\tgo fs.cleanupCache()\n\tif conf.AccessLog != \"\" {\n\t\tf, err := os.OpenFile(conf.AccessLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Open access log %s: %s\", conf.AccessLog, err)\n\t\t} else {\n\t\t\t_ = os.Chmod(conf.AccessLog, 0666)\n\t\t\tfs.logBuffer = make(chan string, 1024)\n\t\t\tgo fs.flushLog(f, fs.logBuffer, conf.AccessLog)\n\t\t}\n\t}\n\treturn fs, nil\n}\n\nfunc (fs *FileSystem) InitMetrics(reg prometheus.Registerer) {\n\tif reg != nil {\n\t\treg.MustRegister(fs.readSizeHistogram)\n\t\treg.MustRegister(fs.writtenSizeHistogram)\n\t\treg.MustRegister(fs.opsDurationsHistogram)\n\t\tvfs.InitMemoryBufferMetrics(fs.writer, fs.reader, reg)\n\t}\n}\n\nfunc (fs *FileSystem) cleanupCache() {\n\tfor {\n\t\tfs.cacheM.Lock()\n\t\tnow := time.Now()\n\t\tvar cnt int\n\t\tfor inode, it := range fs.attrs {\n\t\t\tif now.After(it.expire) {\n\t\t\t\tdelete(fs.attrs, inode)\n\t\t\t}\n\t\t\tcnt++\n\t\t\tif cnt > 1000 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tcnt = 0\n\tOUTER:\n\t\tfor inode, es := range fs.entries {\n\t\t\tfor n, e := range es {\n\t\t\t\tif now.After(e.expire) {\n\t\t\t\t\tdelete(es, n)\n\t\t\t\t\tif len(es) == 0 {\n\t\t\t\t\t\tdelete(fs.entries, inode)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tcnt++\n\t\t\t\tif cnt > 1000 {\n\t\t\t\t\tbreak OUTER\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfs.cacheM.Unlock()\n\t\ttime.Sleep(time.Second)\n\t}\n}\n\nfunc (fs *FileSystem) InvalidateEntry(parent Ino, name string) {\n\tfs.cacheM.Lock()\n\tdefer fs.cacheM.Unlock()\n\tes, ok := fs.entries[parent]\n\tif ok {\n\t\tdelete(es, name)\n\t\tif len(es) == 0 {\n\t\t\tdelete(fs.entries, parent)\n\t\t}\n\t}\n}\n\nfunc (fs *FileSystem) InvalidateAttr(ino Ino) {\n\tfs.cacheM.Lock()\n\tdefer fs.cacheM.Unlock()\n\tdelete(fs.attrs, ino)\n}\n\nfunc (fs *FileSystem) log(ctx LogContext, format string, args ...interface{}) {\n\tused := ctx.Duration()\n\tfs.opsDurationsHistogram.Observe(used.Seconds())\n\tif fs.logBuffer == nil {\n\t\treturn\n\t}\n\tnow := utils.Now()\n\tcmd := fmt.Sprintf(format, args...)\n\tts := now.Format(\"2006.01.02 15:04:05.000000\")\n\tcmd += fmt.Sprintf(\" <%.6f>\", used.Seconds())\n\tline := fmt.Sprintf(\"%s [uid:%d,gid:%d,pid:%d] %s\\n\", ts, ctx.Uid(), ctx.Gid(), ctx.Pid(), cmd)\n\tselect {\n\tcase fs.logBuffer <- line:\n\tdefault:\n\t\tlogger.Debugf(\"log dropped: %s\", line[:len(line)-1])\n\t}\n}\n\nfunc (fs *FileSystem) flushLog(f *os.File, logBuffer chan string, path string) {\n\tbuf := make([]byte, 0, 128<<10)\n\tvar lastcheck = time.Now()\n\tfor {\n\t\tline := <-logBuffer\n\t\tbuf = append(buf[:0], []byte(line)...)\n\tLOOP:\n\t\tfor len(buf) < (128 << 10) {\n\t\t\tselect {\n\t\t\tcase line = <-logBuffer:\n\t\t\t\tbuf = append(buf, []byte(line)...)\n\t\t\tdefault:\n\t\t\t\tbreak LOOP\n\t\t\t}\n\t\t}\n\t\t_, err := f.Write(buf)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"write access log: %s\", err)\n\t\t\tbreak\n\t\t}\n\t\tif lastcheck.Add(fs.checkAccessFile).After(time.Now()) {\n\t\t\tcontinue\n\t\t}\n\t\tlastcheck = time.Now()\n\t\tvar fi os.FileInfo\n\t\tfi, err = f.Stat()\n\t\tif err == nil && fi.Size() > fs.rotateAccessLog {\n\t\t\t_ = f.Close()\n\t\t\tfi, err = os.Stat(path)\n\t\t\tif err == nil && fi.Size() > fs.rotateAccessLog {\n\t\t\t\ttmp := fmt.Sprintf(\"%s.%p\", path, fs)\n\t\t\t\tif os.Rename(path, tmp) == nil {\n\t\t\t\t\tfor i := 6; i > 0; i-- {\n\t\t\t\t\t\t_ = os.Rename(path+\".\"+strconv.Itoa(i), path+\".\"+strconv.Itoa(i+1))\n\t\t\t\t\t}\n\t\t\t\t\t_ = os.Rename(tmp, path+\".1\")\n\t\t\t\t} else {\n\t\t\t\t\tfi, err = os.Stat(path)\n\t\t\t\t\tif err == nil && fi.Size() > fs.rotateAccessLog*7 {\n\t\t\t\t\t\tlogger.Infof(\"can't rename %s, truncate it\", path)\n\t\t\t\t\t\t_ = os.Truncate(path, 0)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tf, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"open %s: %s\", path, err)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t_ = os.Chmod(path, 0666)\n\t\t}\n\t}\n}\n\nfunc (fs *FileSystem) Meta() meta.Meta {\n\treturn fs.m\n}\n\nfunc (fs *FileSystem) StatFS(ctx meta.Context) (totalspace uint64, availspace uint64) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.StatFS\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"StatFS (): (%d,%d)\", totalspace, availspace) }()\n\tvar iused, iavail uint64\n\t_ = fs.m.StatFS(ctx, meta.RootInode, &totalspace, &availspace, &iused, &iavail)\n\treturn\n}\n\n// open file without following symlink\nfunc (fs *FileSystem) Lopen(ctx meta.Context, path string, flags uint32) (f *File, err syscall.Errno) {\n\treturn fs.open(ctx, path, flags, false)\n}\n\nfunc (fs *FileSystem) Open(ctx meta.Context, path string, flags uint32) (*File, syscall.Errno) {\n\treturn fs.open(ctx, path, flags, true)\n}\n\nfunc (fs *FileSystem) open(ctx meta.Context, path string, flags uint32, followLink bool) (f *File, err syscall.Errno) {\n\t_, task := trace.NewTask(context.TODO(), \"Open\")\n\tdefer task.End()\n\tl := vfs.NewLogContext(ctx)\n\tif flags != 0 {\n\t\tdefer func() { fs.log(l, \"Open (%s,%d): %s\", path, flags, errstr(err)) }()\n\t} else {\n\t\tdefer func() { fs.log(l, \"Lookup (%s): %s\", path, errstr(err)) }()\n\t}\n\tvar fi *FileStat\n\tfi, err = fs.resolve(ctx, path, followLink)\n\tif err != 0 {\n\t\treturn\n\t}\n\n\tif flags != 0 && !fi.IsDir() {\n\t\tvar oflags uint32 = syscall.O_RDONLY\n\t\tif flags == vfs.MODE_MASK_W {\n\t\t\toflags = syscall.O_WRONLY\n\t\t} else if flags&vfs.MODE_MASK_W != 0 {\n\t\t\toflags = syscall.O_RDWR\n\t\t}\n\t\terr = fs.m.Open(ctx, fi.inode, oflags, fi.attr)\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t}\n\n\tf = &File{}\n\tf.path = path\n\tf.inode = fi.inode\n\tf.info = fi\n\tf.fs = fs\n\tf.flags = flags\n\tswitch fi.inode {\n\tcase vfs.ConfigInode:\n\t\tfs.conf.Format = fs.Meta().GetFormat()\n\t\tfs.conf.Format.RemoveSecret()\n\t\tf.data, _ = json.MarshalIndent(fs.conf, \"\", \" \")\n\t\tf.info.attr.Length = uint64(len(f.data))\n\tcase vfs.StatsInode:\n\t\tf.data = vfs.CollectMetrics(fs.registry)\n\t\tf.info.attr.Length = uint64(len(f.data))\n\t}\n\treturn\n}\n\nfunc (fs *FileSystem) Access(ctx meta.Context, path string, flags int) (err syscall.Errno) {\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Access (%s): %s\", path, errstr(err)) }()\n\tvar fi *FileStat\n\tfi, err = fs.resolve(ctx, path, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\n\tif ctx.Uid() != 0 && flags != 0 {\n\t\terr = fs.m.Access(ctx, fi.inode, uint8(flags), fi.attr)\n\t}\n\treturn\n}\n\nfunc (fs *FileSystem) Stat(ctx meta.Context, path string) (fi *FileStat, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Stat\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Stat (%s): %s\", path, errstr(err)) }()\n\treturn fs.resolve(ctx, path, true)\n}\n\nfunc (fs *FileSystem) Lstat(ctx meta.Context, path string) (fi *FileStat, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Lstat\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Lstat (%s): %s\", path, errstr(err)) }()\n\treturn fs.resolve(ctx, path, false)\n}\n\n// parentDir returns parent of /foo/bar/ as /foo\nfunc parentDir(p string) string {\n\treturn path.Dir(strings.TrimRight(p, \"/\"))\n}\n\nfunc (fs *FileSystem) Mkdir(ctx meta.Context, p string, mode uint16, umask uint16) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Mkdir\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Mkdir (%s, %o): %s\", p, mode, errstr(err)) }()\n\tif p == \"/\" {\n\t\treturn syscall.EEXIST\n\t}\n\tfi, err := fs.resolve(ctx, parentDir(p), true)\n\tif err != 0 {\n\t\treturn err\n\t}\n\tvar inode Ino\n\terr = fs.m.Mkdir(ctx, fi.inode, path.Base(p), mode, umask, 0, &inode, nil)\n\tif err == syscall.ENOENT && fi.inode != 1 {\n\t\t// parent be moved into trash, try again\n\t\tif fs.conf.DirEntryTimeout > 0 {\n\t\t\tparent := parentDir(p)\n\t\t\tif fi, err := fs.resolve(ctx, parentDir(parent), true); err == 0 {\n\t\t\t\tfs.InvalidateEntry(fi.inode, path.Base(parent))\n\t\t\t}\n\t\t}\n\t\tif fi2, e := fs.resolve(ctx, parentDir(p), true); e != 0 {\n\t\t\treturn e\n\t\t} else if fi2.inode != fi.inode {\n\t\t\terr = fs.m.Mkdir(ctx, fi2.inode, path.Base(p), mode, umask, 0, &inode, nil)\n\t\t}\n\t}\n\tfs.InvalidateEntry(fi.inode, path.Base(p))\n\treturn\n}\n\nfunc (fs *FileSystem) MkdirAll(ctx meta.Context, p string, mode uint16, umask uint16) (err syscall.Errno) {\n\treturn fs.MkdirAll0(ctx, p, mode, umask, true)\n}\n\nfunc (fs *FileSystem) MkdirAll0(ctx meta.Context, p string, mode uint16, umask uint16, existOK bool) (err syscall.Errno) {\n\terr = fs.Mkdir(ctx, p, mode, umask)\n\tif err == syscall.ENOENT {\n\t\terr = fs.MkdirAll(ctx, parentDir(p), mode, umask)\n\t\tif err == 0 {\n\t\t\terr = fs.Mkdir(ctx, p, mode, umask)\n\t\t}\n\t}\n\tif existOK && err == syscall.EEXIST {\n\t\terr = 0\n\t}\n\treturn err\n}\n\nfunc (fs *FileSystem) Unlink(ctx meta.Context, p string) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Unlink\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Unlink (%s): %s\", p, errstr(err)) }()\n\treturn fs.Delete0(ctx, p, true)\n}\n\nfunc (fs *FileSystem) Delete(ctx meta.Context, p string) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Delete\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Delete (%s): %s\", p, errstr(err)) }()\n\treturn fs.Delete0(ctx, p, false)\n}\n\nfunc (fs *FileSystem) BatchDeleteEntries(ctx meta.Context, parent string, ps []string) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.BatchDeleteEntries\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"BatchDeleteEntries : %s\", errstr(err)) }()\n\tparentInfo, errno := fs.Stat(ctx, parent)\n\tif errno != 0 {\n\t\treturn errno\n\t}\n\tvar entries []*meta.Entry\n\tfor _, p := range ps {\n\t\tfi, e := fs.Stat(ctx, p)\n\t\tif errors.Is(e, syscall.ENOENT) {\n\t\t\tcontinue\n\t\t}\n\t\tif e != 0 {\n\t\t\treturn e\n\t\t}\n\t\tentries = append(entries, &meta.Entry{Inode: fi.Inode(), Name: []byte(fi.Name()), Attr: fi.Attr()})\n\t}\n\tif len(entries) == 0 {\n\t\treturn 0\n\t}\n\teno := fs.m.BatchUnlink(ctx, parentInfo.inode, entries, nil, false)\n\tfor _, p := range ps {\n\t\tfs.InvalidateEntry(parentInfo.inode, path.Base(p))\n\t}\n\treturn eno\n}\n\nfunc (fs *FileSystem) Delete0(ctx meta.Context, p string, callByUnlink bool) (err syscall.Errno) {\n\tparent, err := fs.resolve(ctx, parentDir(p), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\tfi, err := fs.resolve(ctx, p, false)\n\tif err != 0 {\n\t\treturn\n\t}\n\tif fi.IsDir() {\n\t\tif callByUnlink {\n\t\t\terr = syscall.EISDIR\n\t\t\treturn\n\t\t}\n\t\terr = fs.m.Rmdir(ctx, parent.inode, path.Base(p))\n\t} else {\n\t\terr = fs.m.Unlink(ctx, parent.inode, path.Base(p))\n\t}\n\tfs.InvalidateEntry(parent.inode, path.Base(p))\n\treturn\n}\n\nfunc (fs *FileSystem) Rmdir(ctx meta.Context, p string) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Rmdir\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Rmdir (%s): %s\", p, errstr(err)) }()\n\tparent, err := fs.resolve(ctx, parentDir(p), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.Rmdir(ctx, parent.inode, path.Base(p))\n\tfs.InvalidateEntry(parent.inode, path.Base(p))\n\treturn\n}\n\nfunc (fs *FileSystem) Rmr(ctx meta.Context, p string, skipTrash bool, numthreads int) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Rmr\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Rmr (%s): %s\", p, errstr(err)) }()\n\tparent, err := fs.resolve(ctx, parentDir(p), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.Remove(ctx, parent.inode, path.Base(p), skipTrash, numthreads, nil)\n\tfs.InvalidateEntry(parent.inode, path.Base(p))\n\treturn\n}\n\nfunc trimDotsForRename(paths []string) (res []string) {\n\tfor i, p := range paths {\n\t\tif p == \".\" {\n\t\t\tpaths[i] = \"\"\n\t\t} else if p == \"..\" {\n\t\t\tif i > 0 {\n\t\t\t\tpaths[i] = \"\"\n\t\t\t\tpaths[i-1] = \"\"\n\t\t\t}\n\t\t}\n\t}\n\tfor _, p := range paths {\n\t\tif p != \"\" {\n\t\t\tres = append(res, p)\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc (fs *FileSystem) Rename(ctx meta.Context, oldpath string, newpath string, flags uint32) (err syscall.Errno) {\n\toss := trimDotsForRename(strings.Split(oldpath, \"/\"))\n\tnss := trimDotsForRename(strings.Split(newpath, \"/\"))\n\tvar err0 syscall.Errno\n\n\t// check if oldpath is ancestor of newpath\n\tfor i := 0; i < len(oss); {\n\t\tif i >= len(nss) || oss[i] != nss[i] {\n\t\t\tbreak\n\t\t} else { // oss[i] == nss[i]\n\t\t\ti++\n\t\t\tif i == len(oss) && i == len(nss) {\n\t\t\t\tbreak\n\t\t\t} else if i == len(oss) {\n\t\t\t\terr0 = syscall.EINVAL\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tdefer trace.StartRegion(context.TODO(), \"fs.Rename\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Rename (%s,%s,%d): %s\", oldpath, newpath, flags, errstr(err)) }()\n\toldfi, err := fs.resolve(ctx, parentDir(oldpath), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\tnewfi, err := fs.resolve(ctx, parentDir(newpath), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\tif err0 != 0 {\n\t\treturn err0\n\t}\n\terr = fs.m.Rename(ctx, oldfi.inode, path.Base(oldpath), newfi.inode, path.Base(newpath), flags, nil, nil)\n\tfs.InvalidateEntry(oldfi.inode, path.Base(oldpath))\n\tfs.InvalidateEntry(newfi.inode, path.Base(newpath))\n\treturn\n}\n\nfunc (fs *FileSystem) Link(ctx meta.Context, src string, dst string) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Link\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Link (%s,%s): %s\", src, dst, errstr(err)) }()\n\n\tfi, err := fs.resolve(ctx, src, false)\n\tif err != 0 {\n\t\treturn\n\t}\n\tpi, err := fs.resolve(ctx, parentDir(dst), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.Link(ctx, fi.inode, pi.inode, path.Base(dst), nil)\n\tfs.InvalidateEntry(pi.inode, path.Base(dst))\n\treturn\n}\n\nfunc (fs *FileSystem) Symlink(ctx meta.Context, target string, link string) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Symlink\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Symlink (%s,%s): %s\", target, link, errstr(err)) }()\n\tif strings.HasSuffix(link, \"/\") {\n\t\treturn syscall.EINVAL\n\t}\n\tfi, err := fs.resolve(ctx, parentDir(link), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.Symlink(ctx, fi.inode, path.Base(link), target, nil, nil)\n\tfs.InvalidateEntry(fi.inode, path.Base(link))\n\treturn\n}\n\nfunc (fs *FileSystem) Readlink(ctx meta.Context, link string) (path []byte, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Readlink\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Readlink (%s): %s (%d)\", link, errstr(err), len(path)) }()\n\tfi, err := fs.resolve(ctx, link, false)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.ReadLink(ctx, fi.inode, &path)\n\treturn\n}\n\nfunc (fs *FileSystem) Truncate(ctx meta.Context, path string, length uint64) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Truncate\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Truncate (%s,%d): %s\", path, length, errstr(err)) }()\n\tfi, err := fs.resolve(ctx, path, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\tif fi.IsDir() {\n\t\treturn syscall.EISDIR\n\t}\n\terr = fs.m.Truncate(ctx, fi.inode, 0, length, nil, false)\n\treturn\n}\n\nfunc (fs *FileSystem) CopyFileRange(ctx meta.Context, src string, soff uint64, dst string, doff uint64, size uint64) (written uint64, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.CopyFileRange\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() {\n\t\tfs.log(l, \"CopyFileRange (%s,%d,%s,%d,%d): (%d,%s)\", dst, doff, src, soff, size, written, errstr(err))\n\t}()\n\tvar dfi, sfi *FileStat\n\tdfi, err = fs.resolve(ctx, dst, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\tsfi, err = fs.resolve(ctx, src, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.CopyFileRange(ctx, sfi.inode, soff, dfi.inode, doff, size, 0, &written, nil)\n\treturn\n}\n\nfunc (fs *FileSystem) SetXattr(ctx meta.Context, p string, name string, value []byte, flags uint32) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.SetXattr\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"SetXAttr (%s,%s,%d,%d): %s\", p, name, len(value), flags, errstr(err)) }()\n\tfi, err := fs.resolve(ctx, p, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.SetXattr(ctx, fi.inode, name, value, flags)\n\treturn\n}\n\nfunc (fs *FileSystem) GetXattr(ctx meta.Context, p string, name string) (result []byte, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.GetXattr\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"GetXattr (%s,%s): (%d,%s)\", p, name, len(result), errstr(err)) }()\n\tfi, err := fs.resolve(ctx, p, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.GetXattr(ctx, fi.inode, name, &result)\n\treturn\n}\n\nfunc (fs *FileSystem) ListXattr(ctx meta.Context, p string) (names []byte, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.ListXattr\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"ListXattr (%s): (%d,%s)\", p, len(names), errstr(err)) }()\n\tfi, err := fs.resolve(ctx, p, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.ListXattr(ctx, fi.inode, &names)\n\treturn\n}\n\nfunc (fs *FileSystem) RemoveXattr(ctx meta.Context, p string, name string) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.RemoveXattr\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"RemoveXattr (%s,%s): %s\", p, name, errstr(err)) }()\n\tfi, err := fs.resolve(ctx, p, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.RemoveXattr(ctx, fi.inode, name)\n\treturn\n}\n\nfunc (fs *FileSystem) GetFacl(ctx meta.Context, p string, acltype uint8, rule *acl.Rule) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.GetFacl\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"GetFacl (%s,%d): %s\", p, acltype, errstr(err)) }()\n\tfi, err := fs.resolve(ctx, p, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.GetFacl(ctx, fi.inode, acltype, rule)\n\treturn\n}\n\nfunc (fs *FileSystem) SetFacl(ctx meta.Context, p string, acltype uint8, rule *acl.Rule) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.SetFacl\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() {\n\t\tfs.log(l, \"SetFacl (%s,%d,%v): %s\", p, acltype, rule, errstr(err))\n\t}()\n\tfi, err := fs.resolve(ctx, p, true)\n\tif err != 0 {\n\t\treturn\n\t}\n\tif acltype == acl.TypeDefault && fi.Mode().IsRegular() {\n\t\tif rule.IsEmpty() {\n\t\t\treturn\n\t\t} else {\n\t\t\treturn syscall.ENOTSUP\n\t\t}\n\t}\n\tif rule.IsEmpty() {\n\t\toldRule := acl.EmptyRule()\n\t\tif err = fs.m.GetFacl(ctx, fi.inode, acltype, oldRule); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\trule.Owner = oldRule.Owner\n\t\trule.Other = oldRule.Other\n\t\trule.Group = oldRule.Group & oldRule.Mask\n\t}\n\terr = fs.m.SetFacl(ctx, fi.inode, acltype, rule)\n\treturn\n}\n\nfunc (fs *FileSystem) lookup(ctx meta.Context, parent Ino, name string, inode *Ino, attr *Attr) (err syscall.Errno) {\n\tnow := time.Now()\n\tif fs.conf.DirEntryTimeout > 0 || fs.conf.EntryTimeout > 0 {\n\t\tfs.cacheM.Lock()\n\t\tes, ok := fs.entries[parent]\n\t\tif ok {\n\t\t\te, ok := es[name]\n\t\t\tif ok {\n\t\t\t\tif now.Before(e.expire) {\n\t\t\t\t\tac := fs.attrs[e.inode]\n\t\t\t\t\tfs.cacheM.Unlock()\n\t\t\t\t\t*inode = e.inode\n\t\t\t\t\tif ac == nil || now.After(ac.expire) {\n\t\t\t\t\t\terr = fs.m.GetAttr(ctx, e.inode, attr)\n\t\t\t\t\t\tif err == 0 && fs.conf.AttrTimeout > 0 {\n\t\t\t\t\t\t\tfs.cacheM.Lock()\n\t\t\t\t\t\t\tfs.attrs[e.inode] = &attrCache{*attr, now.Add(fs.conf.AttrTimeout)}\n\t\t\t\t\t\t\tfs.cacheM.Unlock()\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\t*attr = ac.attr\n\t\t\t\t\t}\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tdelete(es, name)\n\t\t\t\tif len(es) == 0 {\n\t\t\t\t\tdelete(fs.entries, parent)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfs.cacheM.Unlock()\n\t}\n\n\terr = fs.m.Lookup(ctx, parent, name, inode, attr, false)\n\tif err == 0 && (fs.conf.DirEntryTimeout > 0 && attr.Typ == meta.TypeDirectory || fs.conf.EntryTimeout > 0 && attr.Typ != meta.TypeDirectory) {\n\t\tfs.cacheM.Lock()\n\t\tif fs.conf.AttrTimeout > 0 {\n\t\t\tfs.attrs[*inode] = &attrCache{*attr, now.Add(fs.conf.AttrTimeout)}\n\t\t}\n\t\tes, ok := fs.entries[parent]\n\t\tif !ok {\n\t\t\tes = make(map[string]*entryCache)\n\t\t\tfs.entries[parent] = es\n\t\t}\n\t\tvar expire time.Time\n\t\tif attr.Typ == meta.TypeDirectory {\n\t\t\texpire = now.Add(fs.conf.DirEntryTimeout)\n\t\t} else {\n\t\t\texpire = now.Add(fs.conf.EntryTimeout)\n\t\t}\n\t\tes[name] = &entryCache{*inode, attr.Typ, expire}\n\t\tfs.cacheM.Unlock()\n\t}\n\t// TODO: support for `negative_dentry_cache`?\n\treturn err\n}\n\nfunc (fs *FileSystem) resolve(ctx meta.Context, p string, followLastSymlink bool) (fi *FileStat, err syscall.Errno) {\n\treturn fs.doResolve(ctx, p, followLastSymlink, make(map[Ino]struct{}))\n}\n\nfunc (fs *FileSystem) doResolve(ctx meta.Context, p string, followLastSymlink bool, visited map[Ino]struct{}) (fi *FileStat, err syscall.Errno) {\n\tp = path.Clean(p)\n\n\t// Check if path is allowed by any of the configured subdirs\n\tif len(fs.subdirPrefixes) > 0 {\n\t\tallowed := false\n\t\tplen := len(p)\n\t\tfor _, prefix := range fs.subdirPrefixes {\n\t\t\tprefixLen := len(prefix)\n\t\t\t// Fast path: check length first to avoid string comparison if possible\n\t\t\tif prefixLen > plen {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Check if path starts with prefix and is either the prefix itself or has '/' after prefix\n\t\t\t// This prevents matching \"/test\" with \"/testfile\" (should match \"/test\" or \"/test/...\")\n\t\t\tif strings.HasPrefix(p, prefix) && (prefixLen == plen || p[prefixLen] == '/') {\n\t\t\t\tallowed = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !allowed {\n\t\t\treturn nil, syscall.EACCES\n\t\t}\n\t}\n\tvar inode Ino\n\tvar attr = &Attr{}\n\n\tif fs.conf.FastResolve {\n\t\terr = fs.m.Resolve(ctx, 1, p, &inode, attr)\n\t\tif err == 0 {\n\t\t\tfi = AttrToFileInfo(inode, attr)\n\t\t\tp = strings.TrimRight(p, \"/\")\n\t\t\tss := strings.Split(p, \"/\")\n\t\t\tfi.name = ss[len(ss)-1]\n\t\t\tif fi.IsSymlink() && followLastSymlink {\n\t\t\t\t// fast resolve can't follow symlink\n\t\t\t\terr = syscall.ENOTSUP\n\t\t\t}\n\t\t}\n\t\tif err != syscall.ENOTSUP {\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Fallback to the default implementation that calls `fs.m.Lookup` for each directory along the path.\n\t// It might be slower for deep directories, but it works for every meta that implements `Lookup`.\n\tparent := Ino(1)\n\tss := strings.Split(p, \"/\")\n\tfor i, name := range ss {\n\t\tif len(name) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tif parent == meta.RootInode && i == len(ss)-1 && vfs.IsSpecialName(name) {\n\t\t\tinode, attr := vfs.GetInternalNodeByName(name)\n\t\t\tfi = AttrToFileInfo(inode, attr)\n\t\t\tparent = inode\n\t\t\tbreak\n\t\t}\n\t\tif parent > 1 {\n\t\t\tif (name == \".\" || name == \"..\") && attr.Typ != meta.TypeDirectory {\n\t\t\t\treturn nil, syscall.ENOTDIR\n\t\t\t}\n\t\t\tif err := fs.m.Access(ctx, parent, meta.MODE_MASK_X, attr); err != 0 {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\n\t\tvar inode Ino\n\t\tvar resolved bool\n\n\t\terr = fs.lookup(ctx, parent, name, &inode, attr)\n\t\tif i == len(ss)-1 {\n\t\t\tresolved = true\n\t\t}\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t\tfi = AttrToFileInfo(inode, attr)\n\t\tif (!resolved || followLastSymlink) && fi.IsSymlink() {\n\t\t\tif _, ok := visited[inode]; ok {\n\t\t\t\tlogger.Errorf(\"find a loop symlink: %d\", inode)\n\t\t\t\treturn nil, syscall.ELOOP\n\t\t\t} else {\n\t\t\t\tvisited[inode] = struct{}{}\n\t\t\t}\n\t\t\tvar buf []byte\n\t\t\terr = fs.m.ReadLink(ctx, inode, &buf)\n\t\t\tif err != 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttarget := string(buf)\n\t\t\tif strings.Contains(target, \"://\") {\n\t\t\t\treturn &FileStat{name: target}, syscall.ENOTSUP\n\t\t\t}\n\t\t\tif strings.HasPrefix(target, \"/\") {\n\t\t\t\tmp := fs.conf.Mountpoint\n\t\t\t\tif !strings.HasSuffix(mp, \"/\") {\n\t\t\t\t\tmp += \"/\"\n\t\t\t\t}\n\t\t\t\tif strings.HasPrefix(target, mp) {\n\t\t\t\t\ttarget = target[len(mp):]\n\t\t\t\t} else {\n\t\t\t\t\tfi.name = \"file:\" + target\n\t\t\t\t\tlogger.Errorf(\"external link: %s -> %s\", p, target)\n\t\t\t\t\treturn fi, utils.ErrExtlink\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\ttarget = path.Join(strings.Join(ss[:i], \"/\"), target)\n\t\t\t}\n\t\t\tfi, err = fs.doResolve(ctx, target, followLastSymlink, visited)\n\t\t\tif err != 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tinode = fi.Inode()\n\t\t\tattr = fi.attr\n\t\t}\n\t\tfi.name = name\n\t\tparent = inode\n\t}\n\tif parent == meta.RootInode {\n\t\terr = fs.m.GetAttr(ctx, parent, attr)\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t\tfi = AttrToFileInfo(1, attr)\n\t}\n\treturn fi, 0\n}\n\nfunc (fs *FileSystem) Create(ctx meta.Context, p string, mode uint16, umask uint16) (f *File, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Create\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { fs.log(l, \"Create (%s,%o): %s\", p, mode, errstr(err)) }()\n\tif strings.HasSuffix(p, \"/\") {\n\t\treturn nil, syscall.EINVAL\n\t}\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tvar fi *FileStat\n\tfi, err = fs.resolve(ctx, parentDir(p), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = fs.m.Create(ctx, fi.inode, path.Base(p), mode&07777, umask, syscall.O_EXCL, &inode, attr)\n\tif err == syscall.ENOENT && fi.inode != 1 {\n\t\t// dir be moved into trash, try again\n\t\tif fs.conf.DirEntryTimeout > 0 {\n\t\t\tparent := parentDir(p)\n\t\t\tif fi, err := fs.resolve(ctx, parentDir(parent), true); err == 0 {\n\t\t\t\tfs.InvalidateEntry(fi.inode, path.Base(parent))\n\t\t\t}\n\t\t}\n\t\tif fi2, e := fs.resolve(ctx, parentDir(p), true); e != 0 {\n\t\t\treturn nil, e\n\t\t} else if fi2.inode != fi.inode {\n\t\t\terr = fs.m.Create(ctx, fi2.inode, path.Base(p), mode&07777, umask, syscall.O_EXCL, &inode, attr)\n\t\t}\n\t}\n\tif err == 0 {\n\t\tfi = AttrToFileInfo(inode, attr)\n\t\tfi.name = path.Base(p)\n\t\tf = &File{}\n\t\tf.flags = vfs.MODE_MASK_W\n\t\tf.path = p\n\t\tf.inode = fi.inode\n\t\tf.info = fi\n\t\tf.fs = fs\n\t}\n\tfs.InvalidateEntry(fi.inode, path.Base(p))\n\treturn\n}\n\nfunc (fs *FileSystem) Flush() error {\n\tbuffer := fs.logBuffer\n\tif buffer != nil {\n\t\tbuffer <- \"\" // flush\n\t}\n\tfs.Meta().FlushSession()\n\treturn nil\n}\n\nfunc (fs *FileSystem) Close() error {\n\t_ = fs.Flush()\n\tbuffer := fs.logBuffer\n\tif buffer != nil {\n\t\tfs.logBuffer = nil\n\t\tclose(buffer)\n\t}\n\treturn nil\n}\n\nfunc (fs *FileSystem) Clone(ctx meta.Context, src, dst string, preserve bool) (err syscall.Errno) {\n\tsrcParent, err := fs.resolve(ctx, parentDir(src), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\tvar srcIno Ino\n\terr = fs.lookup(ctx, srcParent.Inode(), path.Base(src), &srcIno, &Attr{})\n\tif err != 0 {\n\t\treturn\n\t}\n\tdstParent, err := fs.resolve(ctx, parentDir(dst), true)\n\tif err != 0 {\n\t\treturn\n\t}\n\n\tvar count, total uint64\n\tumask := uint16(utils.GetUmask())\n\n\tvar cmode uint8\n\tif preserve {\n\t\tcmode |= meta.CLONE_MODE_PRESERVE_ATTR\n\t}\n\n\tif err = fs.m.Clone(meta.NewContext(ctx.Pid(), ctx.Uid(), ctx.Gids()), srcParent.Inode(), srcIno, dstParent.Inode(), path.Base(dst), cmode, umask, meta.CLONE_DEFAULT_CONCURRENCY, &count, &total); err != 0 {\n\t\tlogger.Errorf(\"clone failed srcIno:%d,dstParentIno:%d,dstName:%s,cmode:%d,umask:%d,eno:%v\", srcIno, dstParent.Inode(), path.Base(dst), cmode, umask, err)\n\t}\n\treturn\n}\n\nfunc (fs *FileSystem) Warmup(ctx meta.Context, paths []string, numthreads int, background bool, isEvict bool, isCheck bool, resp *vfs.CacheResponse) {\n\taction := vfs.WarmupCache\n\tif isEvict {\n\t\taction = vfs.EvictCache\n\t}\n\tif isCheck {\n\t\taction = vfs.CheckCache\n\t}\n\n\tif background {\n\t\tgo fs.cacheFiller.Cache(meta.NewContext(ctx.Pid(), ctx.Uid(), ctx.Gids()), action, paths, int(numthreads), resp)\n\t} else {\n\t\tfs.cacheFiller.Cache(meta.NewContext(ctx.Pid(), ctx.Uid(), ctx.Gids()), action, paths, int(numthreads), resp)\n\t}\n}\n\nfunc (fs *FileSystem) HandleQuota(ctx meta.Context, path string, cmd uint8, capacity, inodes uint64, strict, repair, create bool) (qs map[string]*meta.Quota, err syscall.Errno) {\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() {\n\t\tfs.log(l, \"QuotaCtl (%s,%d,%d,%d,%t,%t,%t): %s\", path, cmd, capacity, inodes, create, repair, strict, errstr(err))\n\t}()\n\tif cmd == meta.QuotaSet && capacity == 0 && inodes == 0 {\n\t\treturn nil, syscall.EINVAL\n\t}\n\tqs = make(map[string]*meta.Quota)\n\tif cmd == meta.QuotaSet {\n\t\tq := &meta.Quota{MaxSpace: -1, MaxInodes: -1} // negative means no change\n\t\tif capacity > 0 {\n\t\t\tq.MaxSpace = int64(capacity)\n\t\t}\n\t\tif inodes > 0 {\n\t\t\tq.MaxInodes = int64(inodes)\n\t\t}\n\t\tqs[path] = q\n\t}\n\n\tif _err := fs.m.HandleQuota(meta.Background(), cmd, path, 0, 0, qs, strict, repair, create); _err != nil {\n\t\tif strings.HasPrefix(_err.Error(), \"no quota for inode\") {\n\t\t\treturn qs, 0\n\t\t}\n\t\terr = syscall.EINVAL\n\t}\n\treturn\n}\n\n// File\n\nfunc (f *File) FS() *FileSystem {\n\treturn f.fs\n}\n\nfunc (f *File) Inode() Ino {\n\treturn f.inode\n}\n\nfunc (f *File) Name() string {\n\treturn f.path\n}\n\nfunc (f *File) Stat() (fi os.FileInfo, err error) {\n\treturn f.info, nil\n}\n\nfunc (f *File) Chmod(ctx meta.Context, mode uint16) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Chmod\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Chmod (%s,%o): %s\", f.path, mode, errstr(err)) }()\n\tvar attr = Attr{Mode: mode}\n\terr = f.fs.m.SetAttr(ctx, f.inode, meta.SetAttrMode, 0, &attr)\n\tf.fs.InvalidateAttr(f.inode)\n\treturn\n}\n\nfunc (f *File) Chown(ctx meta.Context, uid uint32, gid uint32) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Chown\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Chown (%s,%d,%d): %s\", f.path, uid, gid, errstr(err)) }()\n\tvar flag uint16\n\tif uid != uint32(f.info.Uid()) {\n\t\tflag |= meta.SetAttrUID\n\t}\n\tif gid != uint32(f.info.Gid()) {\n\t\tflag |= meta.SetAttrGID\n\t}\n\tvar attr = Attr{Uid: uid, Gid: gid}\n\terr = f.fs.m.SetAttr(ctx, f.inode, flag, 0, &attr)\n\tf.fs.InvalidateAttr(f.inode)\n\treturn\n}\n\nfunc (f *File) Utime(ctx meta.Context, atime, mtime int64) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Utime\").End()\n\tvar flag uint16\n\tif atime >= 0 {\n\t\tflag |= meta.SetAttrAtime\n\t}\n\tif mtime >= 0 {\n\t\tflag |= meta.SetAttrMtime\n\t}\n\tif flag == 0 {\n\t\treturn 0\n\t}\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Utime (%s,%d,%d): %s\", f.path, atime, mtime, errstr(err)) }()\n\tvar attr Attr\n\tattr.Atime = atime / 1000\n\tattr.Atimensec = uint32(atime%1000) * 1e6\n\tattr.Mtime = mtime / 1000\n\tattr.Mtimensec = uint32(mtime%1000) * 1e6\n\terr = f.fs.m.SetAttr(ctx, f.inode, flag, 0, &attr)\n\tf.fs.InvalidateAttr(f.inode)\n\treturn\n}\n\nfunc (f *File) Utime2(ctx meta.Context, atimeSec, atimeNSec, mtimeSec, mtimeNsec int64) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Utime2\").End()\n\tvar flag uint16\n\tif atimeSec >= 0 || atimeNSec >= 0 {\n\t\tflag |= meta.SetAttrAtime\n\t}\n\tif mtimeSec >= 0 || mtimeNsec >= 0 {\n\t\tflag |= meta.SetAttrMtime\n\t}\n\tif flag == 0 {\n\t\treturn 0\n\t}\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() {\n\t\tf.fs.log(l, \"Utime2 (%s,%d,%d,%d,%d): %s\", f.path, atimeSec, atimeNSec, mtimeSec, mtimeNsec, errstr(err))\n\t}()\n\tvar attr Attr\n\tattr.Atime = atimeSec\n\tattr.Atimensec = uint32(atimeNSec)\n\tattr.Mtime = mtimeSec\n\tattr.Mtimensec = uint32(mtimeNsec)\n\terr = f.fs.m.SetAttr(ctx, f.inode, flag, 0, &attr)\n\tf.fs.InvalidateAttr(f.inode)\n\treturn\n}\n\nfunc (f *File) Seek(ctx meta.Context, offset int64, whence int) (int64, error) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Seek\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Seek (%s,%d,%d): %d\", f.path, offset, whence, f.offset) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tswitch whence {\n\tcase io.SeekStart:\n\t\tf.offset = offset\n\tcase io.SeekCurrent:\n\t\tf.offset += offset\n\tcase io.SeekEnd:\n\t\tf.offset = f.info.Size() + offset\n\t}\n\treturn f.offset, nil\n}\n\nfunc (f *File) Read(ctx meta.Context, b []byte) (n int, err error) {\n\t_, task := trace.NewTask(context.TODO(), \"Read\")\n\tdefer task.End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Read (%s,%d): (%d,%s)\", f.path, len(b), n, errstr(err)) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tn, err = f.pread(ctx, b, f.offset)\n\tf.offset += int64(n)\n\treturn\n}\n\nfunc (f *File) Pread(ctx meta.Context, b []byte, offset int64) (n int, err error) {\n\t_, task := trace.NewTask(context.TODO(), \"Pread\")\n\tdefer task.End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Pread (%s,%d,%d): (%d,%s)\", f.path, len(b), offset, n, errstr(err)) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tn, err = f.pread(ctx, b, offset)\n\treturn\n}\n\nfunc (f *File) pread(ctx meta.Context, b []byte, offset int64) (n int, err error) {\n\tif offset >= f.info.Size() {\n\t\treturn 0, io.EOF\n\t}\n\tif int64(len(b))+offset > f.info.Size() {\n\t\tb = b[:f.info.Size()-offset]\n\t}\n\tif f.data != nil {\n\t\tn := copy(b, f.data[offset:])\n\t\treturn n, nil\n\t}\n\tif f.wdata != nil {\n\t\teno := f.wdata.Flush(ctx)\n\t\tif eno != 0 {\n\t\t\terr = eno\n\t\t\treturn\n\t\t}\n\t}\n\tif f.rdata == nil {\n\t\tf.rdata = f.fs.reader.Open(f.inode, uint64(f.info.Size()))\n\t}\n\n\tgot, eno := f.rdata.Read(ctx, uint64(offset), b)\n\tfor eno == syscall.EAGAIN {\n\t\tgot, eno = f.rdata.Read(ctx, uint64(offset), b)\n\t}\n\tif eno != 0 {\n\t\terr = eno\n\t\treturn\n\t}\n\tif got == 0 {\n\t\treturn 0, io.EOF\n\t}\n\tf.fs.readSizeHistogram.Observe(float64(got))\n\treturn got, nil\n}\n\nfunc (f *File) Write(ctx meta.Context, b []byte) (n int, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Write\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Write (%s,%d): (%d,%s)\", f.path, len(b), n, errstr(err)) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tn, err = f.pwrite(ctx, b, f.offset)\n\tf.offset += int64(n)\n\treturn\n}\n\nfunc (f *File) Pwrite(ctx meta.Context, b []byte, offset int64) (n int, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Pwrite\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Pwrite (%s,%d,%d): (%d,%s)\", f.path, len(b), offset, n, errstr(err)) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tn, err = f.pwrite(ctx, b, offset)\n\treturn\n}\n\nfunc (f *File) pwrite(ctx meta.Context, b []byte, offset int64) (n int, err syscall.Errno) {\n\tif f.wdata == nil {\n\t\tf.wdata = f.fs.writer.Open(f.inode, uint64(f.info.Size()))\n\t}\n\terr = f.wdata.Write(ctx, uint64(offset), b)\n\tif err != 0 {\n\t\t_ = f.wdata.Close(meta.Background())\n\t\tf.wdata = nil\n\t\treturn\n\t}\n\tif offset+int64(len(b)) > int64(f.info.attr.Length) {\n\t\tf.info.attr.Length = uint64(offset + int64(len(b)))\n\t}\n\tf.fs.writtenSizeHistogram.Observe(float64(len(b)))\n\treturn len(b), 0\n}\n\nfunc (f *File) Truncate(ctx meta.Context, length uint64) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Truncate\").End()\n\tf.Lock()\n\tdefer f.Unlock()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Truncate (%s,%d): %s\", f.path, length, errstr(err)) }()\n\tif f.wdata != nil {\n\t\terr = f.wdata.Flush(ctx)\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t}\n\terr = f.fs.m.Truncate(ctx, f.inode, 0, length, nil, false)\n\tif err == 0 {\n\t\t_ = f.fs.m.InvalidateChunkCache(ctx, f.inode, uint32(((length - 1) >> meta.ChunkBits)))\n\t\tf.fs.writer.Truncate(f.inode, length)\n\t\tf.fs.reader.Truncate(f.inode, length)\n\t\tf.info.attr.Length = length\n\t\tf.fs.InvalidateAttr(f.inode)\n\t}\n\treturn\n}\n\nfunc (f *File) Flush(ctx meta.Context) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Flush\").End()\n\tf.Lock()\n\tdefer f.Unlock()\n\tif f.wdata == nil {\n\t\treturn\n\t}\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Flush (%s): %s\", f.path, errstr(err)) }()\n\terr = f.wdata.Flush(ctx)\n\tf.fs.InvalidateAttr(f.inode)\n\treturn\n}\n\nfunc (f *File) Fsync(ctx meta.Context) (err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Fsync\").End()\n\tf.Lock()\n\tdefer f.Unlock()\n\tif f.wdata == nil {\n\t\treturn 0\n\t}\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Fsync (%s): %s\", f.path, errstr(err)) }()\n\terr = f.wdata.Flush(ctx)\n\tf.fs.InvalidateAttr(f.inode)\n\treturn\n}\n\nfunc (f *File) Close(ctx meta.Context) (err syscall.Errno) {\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Close (%s): %s\", f.path, errstr(err)) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tif f.flags != 0 && !f.info.IsDir() {\n\t\tf.offset = 0\n\t\tif f.rdata != nil {\n\t\t\trdata := f.rdata\n\t\t\tf.rdata = nil\n\t\t\ttime.AfterFunc(time.Second, func() {\n\t\t\t\trdata.Close(meta.Background())\n\t\t\t})\n\t\t}\n\t\tif f.wdata != nil {\n\t\t\terr = f.wdata.Close(meta.Background())\n\t\t\tf.fs.InvalidateAttr(f.inode)\n\t\t\tf.wdata = nil\n\t\t}\n\t\t_ = f.fs.m.Close(ctx, f.inode)\n\t}\n\treturn\n}\n\nfunc (f *File) Readdir(ctx meta.Context, count int) (fi []os.FileInfo, err syscall.Errno) {\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"Readdir (%s,%d): (%s,%d)\", f.path, count, errstr(err), len(fi)) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tfi = f.dircache\n\tif fi == nil {\n\t\tvar inodes []*meta.Entry\n\t\terr = f.fs.m.Readdir(ctx, f.inode, 1, &inodes)\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t\tif f.fs.conf.Meta.SortDir {\n\t\t\tsort.Slice(inodes[2:], func(i, j int) bool {\n\t\t\t\treturn string(inodes[i].Name) < string(inodes[j].Name)\n\t\t\t})\n\t\t}\n\t\t// skip . and ..\n\t\tfor _, n := range inodes[2:] {\n\t\t\ti := AttrToFileInfo(n.Inode, n.Attr)\n\t\t\ti.name = string(n.Name)\n\t\t\tfi = append(fi, i)\n\t\t}\n\t\tf.dircache = fi\n\t}\n\n\tif len(fi) < int(f.offset) {\n\t\treturn nil, 0\n\t}\n\tfi = fi[f.offset:]\n\tif count > 0 && len(fi) > count {\n\t\tfi = fi[:count]\n\t}\n\tf.offset += int64(len(fi))\n\treturn\n}\n\nfunc (f *File) ReaddirPlus(ctx meta.Context, offset int) (entries []*meta.Entry, err syscall.Errno) {\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() { f.fs.log(l, \"ReaddirPlus (%s,%d): (%s,%d)\", f.path, offset, errstr(err), len(entries)) }()\n\tf.Lock()\n\tdefer f.Unlock()\n\tif f.entries == nil {\n\t\tvar es []*meta.Entry\n\t\terr = f.fs.m.Readdir(ctx, f.inode, 1, &es)\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t\t// filter out . and ..\n\t\tf.entries = make([]*meta.Entry, 0, len(es))\n\t\tfor _, e := range es {\n\t\t\tif !bytes.Equal(e.Name, []byte{'.'}) && !bytes.Equal(e.Name, []byte(\"..\")) {\n\t\t\t\tf.entries = append(f.entries, e)\n\t\t\t}\n\t\t}\n\t\tif f.fs.conf.Meta.SortDir {\n\t\t\tsort.Slice(f.entries, func(i, j int) bool {\n\t\t\t\treturn string(f.entries[i].Name) < string(f.entries[j].Name)\n\t\t\t})\n\t\t}\n\t}\n\tif offset >= len(f.entries) {\n\t\toffset = len(f.entries)\n\t}\n\tentries = f.entries[offset:]\n\treturn\n}\n\nfunc (f *File) Summary(ctx meta.Context, recursive, strict bool) (s *meta.Summary, err syscall.Errno) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.Summary\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() {\n\t\tf.fs.log(l, \"Summary (%s): %s (%d,%d,%d,%d)\", f.path, errstr(err), s.Length, s.Size, s.Files, s.Dirs)\n\t}()\n\ts = &meta.Summary{}\n\terr = f.fs.m.GetSummary(ctx, f.inode, s, recursive, strict)\n\treturn\n}\n\nfunc (f *File) GetTreeSummary(ctx meta.Context, depth, entries uint8, strict bool) (s *meta.TreeSummary, err syscall.Errno) {\n\ts = &meta.TreeSummary{\n\t\tInode: f.inode,\n\t\tPath: \"\",\n\t\tType: meta.TypeDirectory,\n\t}\n\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() {\n\t\tf.fs.log(l, \"GetTreeSummary (%s,%d,%d,%t): %s (%d,%d,%d)\", f.path, depth, entries, strict, errstr(err), s.Size, s.Files, s.Dirs)\n\t}()\n\terr = f.fs.m.GetTreeSummary(ctx, s, depth, entries, strict, nil)\n\ts.Path = path.Base(f.path)\n\treturn\n}\n\nfunc (f *File) GetQuota(ctx meta.Context) (quota *meta.Quota, err error) {\n\tdefer trace.StartRegion(context.TODO(), \"fs.getQuota\").End()\n\tl := vfs.NewLogContext(ctx)\n\tdefer func() {\n\t\tf.fs.log(l, \"getQuota (%s): %s\", f.path, errstr(err))\n\t}()\n\terr = nil\n\tqs := make(map[string]*meta.Quota)\n\t// get filesystem quota if root\n\tif f.inode == meta.RootInode {\n\t\tformat := f.fs.m.GetFormat()\n\t\tquota = &meta.Quota{\n\t\t\tMaxSpace: int64(format.Capacity),\n\t\t\tMaxInodes: int64(format.Inodes),\n\t\t}\n\t\treturn quota, err\n\t}\n\t// get directory quota\n\terr = f.fs.m.HandleQuota(ctx, meta.QuotaGet, f.path, 0, 0, qs, false, false, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tquota = qs[f.path]\n\treturn quota, err\n}\n" }, { "path": "pkg/fs/fs_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fs\n\nimport (\n\t\"io\"\n\t\"os\"\n\t\"sort\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n)\n\n// mutate_test_job_number: 5\nfunc TestFileStat(t *testing.T) {\n\tattr := meta.Attr{\n\t\tTyp: meta.TypeDirectory,\n\t\tMode: 07740,\n\t\tAtime: 1,\n\t\tMtime: 2,\n\t}\n\tst := AttrToFileInfo(2, &attr)\n\tif st.Inode() != 2 {\n\t\tt.Fatalf(\"inode should be 2\")\n\t}\n\tif !st.IsDir() {\n\t\tt.Fatalf(\"should be a dir\")\n\t}\n\tmode := st.Mode()\n\tif mode&os.ModeSticky == 0 {\n\t\tt.Fatalf(\"sticky bit should be set\")\n\t}\n\tif mode&os.ModeSetuid == 0 {\n\t\tt.Fatalf(\"suid should be set\")\n\t}\n\tif mode&os.ModeSetgid == 0 {\n\t\tt.Fatalf(\"sgid should be set\")\n\t}\n\tif st.ModTime().Unix() != 2 {\n\t\tt.Fatalf(\"unixtimestamp : %d\", st.ModTime().Unix())\n\t}\n\tif st.Sys() != &attr {\n\t\tt.Fatalf(\"sys should be meta attr\")\n\t}\n\tattr.Typ = meta.TypeSymlink\n\tif !st.IsSymlink() {\n\t\tt.Fatalf(\"should be a symlink\")\n\t}\n}\n\n// nolint:errcheck\nfunc TestFileSystem(t *testing.T) {\n\tfs := createTestFS(t)\n\tctx := meta.NewContext(1, 1, []uint32{2})\n\tif total, avail := fs.StatFS(ctx); total != 1<<30 || avail != (1<<30) {\n\t\tt.Fatalf(\"statfs: %d %d\", total, avail)\n\t}\n\tif e := fs.Access(ctx, \"/\", 7); e != 0 {\n\t\tt.Fatalf(\"access /: %s\", e)\n\t}\n\tf, err := fs.Create(ctx, \"/hello\", 0666, 022)\n\tif err != 0 {\n\t\tt.Fatalf(\"create /hello: %s\", err)\n\t}\n\tif f.Name() != \"/hello\" {\n\t\tt.Fatalf(\"name: %s\", f.Name())\n\t}\n\t_ = f.Close(ctx)\n\tf, err = fs.Open(ctx, \"/hello\", meta.MODE_MASK_R|meta.MODE_MASK_W)\n\tif err != 0 {\n\t\tt.Fatalf(\"open %s\", err)\n\t}\n\tif fi, err := f.Stat(); err != nil || fi.Mode() != 0644 {\n\t\tt.Fatalf(\"stat: %s %+v\", err, fi)\n\t}\n\tif n, err := f.Write(ctx, []byte(\"world\")); err != 0 || n != 5 {\n\t\tt.Fatalf(\"write 5 bytes: %d %s\", n, err)\n\t}\n\tif err := f.Fsync(ctx); err != 0 {\n\t\tt.Fatalf(\"fsync: %s\", err)\n\t}\n\tvar buf = make([]byte, 10)\n\tif n, err := f.Pread(ctx, buf, 2); err != nil || n != 3 || string(buf[:n]) != \"rld\" {\n\t\tt.Fatalf(\"pread(2): %d %s %s\", n, err, string(buf[:n]))\n\t}\n\tif n, err := f.Seek(ctx, -3, io.SeekEnd); err != nil || n != 2 {\n\t\tt.Fatalf(\"seek 3 bytes before end: %d %s\", n, err)\n\t}\n\tif n, err := f.Write(ctx, []byte(\"t\")); err != 0 || n != 1 {\n\t\tt.Fatalf(\"write 1 bytes: %d %s\", n, err)\n\t}\n\tif n, err := f.Seek(ctx, -2, io.SeekCurrent); err != nil || n != 1 {\n\t\tt.Fatalf(\"seek 2 bytes before current: %d %s\", n, err)\n\t}\n\tif n, err := f.Read(ctx, buf); err != nil || n != 4 || string(buf[:n]) != \"otld\" {\n\t\tt.Fatalf(\"read(): %d %s %s\", n, err, string(buf[:n]))\n\t}\n\tif n, err := f.Read(ctx, buf); err != io.EOF || n != 0 {\n\t\tt.Fatalf(\"read(): %d %s %s\", n, err, string(buf[:n]))\n\t}\n\tif n, err := f.Pwrite(ctx, []byte(\"t\"), 1); err != 0 || n != 1 {\n\t\tt.Fatalf(\"write 1 bytes: %d %s\", n, err)\n\t}\n\tif e := f.Flush(ctx); e != 0 {\n\t\tt.Fatalf(\"flush /hello: %s\", e)\n\t}\n\n\tif e := f.Chmod(ctx, 0640); e != 0 {\n\t\tt.Fatalf(\"chown: %s\", e)\n\t}\n\tif e := f.Chown(ctx, 1, 2); e != 0 {\n\t\tt.Fatalf(\"chown: %s\", e)\n\t}\n\tif e := f.Utime(ctx, 1, 2); e != 0 {\n\t\tt.Fatalf(\"utime: %s\", e)\n\t}\n\tif s, e := f.Summary(ctx, true, true); e != 0 || s.Dirs != 0 || s.Files != 1 || s.Length != 5 || s.Size != 4<<10 {\n\t\tt.Fatalf(\"summary: %s %+v\", e, s)\n\t}\n\tif e := f.Close(ctx); e != 0 {\n\t\tt.Fatalf(\"close /hello: %s\", e)\n\t}\n\tif fi, err := fs.Stat(ctx, \"/hello\"); err != 0 {\n\t\tt.Fatalf(\"stat /hello: %s\", err)\n\t} else if fi.Mode() != 0640 || fi.Uid() != 1 || fi.Gid() != 2 || fi.Atime() != 1 || fi.Mtime() != 2 {\n\t\tt.Fatalf(\"stat /hello: %+v\", fi)\n\t}\n\tif e := fs.Truncate(ctx, \"/hello\", 2); e != 0 {\n\t\tt.Fatalf(\"truncate : %s\", e)\n\t}\n\tif n, e := fs.CopyFileRange(ctx, \"/hello\", 0, \"/hello\", 5, 5); e != 0 || n != 2 {\n\t\tt.Fatalf(\"copyfilerange: %s %d\", e, n)\n\t}\n\n\tif e := fs.SetXattr(ctx, \"/hello\", \"k\", []byte(\"value\"), 0); e != 0 {\n\t\tt.Fatalf(\"setxattr /hello: %s\", e)\n\t}\n\tif v, e := fs.GetXattr(ctx, \"/hello\", \"k\"); e != 0 || string(v) != \"value\" {\n\t\tt.Fatalf(\"getxattr /hello: %s %s\", e, string(v))\n\t}\n\tif names, e := fs.ListXattr(ctx, \"/hello\"); e != 0 || string(names) != \"k\\x00\" {\n\t\tt.Fatalf(\"listxattr /hello: %s %+v\", e, names)\n\t}\n\tif e := fs.RemoveXattr(ctx, \"/hello\", \"k\"); e != 0 {\n\t\tt.Fatalf(\"removexattr /hello: %s\", e)\n\t}\n\n\tif e := fs.Symlink(ctx, \"hello\", \"/sym\"); e != 0 {\n\t\tt.Fatalf(\"symlink: %s\", e)\n\t}\n\tif target, e := fs.Readlink(ctx, \"/sym\"); e != 0 || string(target) != \"hello\" {\n\t\tt.Fatalf(\"readlink: %s\", string(target))\n\t}\n\tif fi, err := fs.Stat(ctx, \"/sym\"); err != 0 || fi.name != \"sym\" || fi.IsSymlink() {\n\t\tt.Fatalf(\"stat symlink: %s %+v\", err, fi)\n\t}\n\tif fi, err := fs.Lstat(ctx, \"/sym\"); err != 0 || fi.name != \"sym\" || !fi.IsSymlink() {\n\t\tt.Fatalf(\"lstat symlink: %s %+v\", err, fi)\n\t}\n\tif err := fs.Delete(ctx, \"/sym\"); err != 0 {\n\t\tt.Fatalf(\"delete /sym: %s\", err)\n\t}\n\n\tif _, e := fs.Open(meta.NewContext(2, 2, []uint32{3}), \"/hello\", meta.MODE_MASK_W); e == 0 || e != syscall.EACCES {\n\t\tt.Fatalf(\"open without permission: %s\", e)\n\t}\n\n\tif err := fs.Mkdir(ctx, \"/d\", 0777, 022); err != 0 {\n\t\tt.Fatalf(\"mkdir /d: %s\", err)\n\t}\n\td, e := fs.Open(ctx, \"/\", 0)\n\tif e != 0 {\n\t\tt.Fatalf(\"open /: %s\", e)\n\t}\n\tdefer d.Close(ctx)\n\tif fis, e := d.Readdir(ctx, 0); e != 0 || len(fis) != 2 {\n\t\tt.Fatalf(\"readdir /: %s, %d entries\", e, len(fis))\n\t} else {\n\t\tsort.Slice(fis, func(i, j int) bool { return fis[i].Name() < fis[j].Name() })\n\t\tif fis[0].Name() != \"d\" || fis[1].Name() != \"hello\" {\n\t\t\tt.Fatalf(\"readdir names: %+v\", fis)\n\t\t}\n\t}\n\tif es, e := d.ReaddirPlus(ctx, 0); e != 0 || len(es) != 2 {\n\t\tt.Fatalf(\"readdirplus: %s, %d entries\", e, len(es))\n\t} else {\n\t\tsort.Slice(es, func(i, j int) bool { return es[i].Inode < es[j].Inode })\n\t\tif string(es[0].Name) != \"hello\" || string(es[1].Name) != \"d\" {\n\t\t\tt.Fatalf(\"readdirplus names: %+v\", es)\n\t\t}\n\t}\n\tif e := fs.Rename(ctx, \"/hello\", \"/d/f\", 0); e != 0 {\n\t\tt.Fatalf(\"rename: %s\", e)\n\t}\n\tif e := fs.Symlink(ctx, \"d\", \"/sd\"); e != 0 {\n\t\tt.Fatalf(\"symlink: %s\", e)\n\t}\n\tif fi, e := fs.Stat(ctx, \"/sd/f\"); e != 0 || fi.name != \"f\" {\n\t\tt.Fatalf(\"follow symlink: %s %+v\", e, fi)\n\t}\n\n\tif s, e := d.Summary(ctx, true, true); e != 0 || s.Dirs != 2 || s.Files != 2 || s.Length != 7 || s.Size != 16<<10 {\n\t\tt.Fatalf(\"summary: %s %+v\", e, s)\n\t}\n\tif q, e := d.GetQuota(ctx); e != nil || q.MaxInodes != 0 || q.MaxSpace != (1<<30) {\n\t\tt.Fatalf(\"quota: %s %+v\", e, q)\n\t}\n\tif e := fs.Delete(ctx, \"/d\"); e == 0 || !IsNotEmpty(e) {\n\t\tt.Fatalf(\"rmdir: %s\", e)\n\t}\n\tif err := fs.Delete(ctx, \"/d/f\"); err != 0 {\n\t\tt.Fatalf(\"delete /d/f: %s\", err)\n\t}\n\tif err := fs.Delete(ctx, \"/d/f\"); err == 0 || !IsNotExist(err) {\n\t\tt.Fatalf(\"delete /d/f: %s\", err)\n\t}\n\tif e := fs.Rmr(ctx, \"/d\", false, meta.RmrDefaultThreads); e != 0 {\n\t\tt.Fatalf(\"delete /d -r: %s\", e)\n\t}\n\n\ttime.Sleep(time.Second * 2)\n\tif e := fs.Flush(); e != nil {\n\t\tt.Fatalf(\"flush : %s\", e)\n\t}\n\tif e := fs.Close(); e != nil {\n\t\tt.Fatalf(\"close: %s\", e)\n\t}\n\tif e := fs.Close(); e != nil {\n\t\tt.Fatalf(\"close: %s\", e)\n\t}\n\n\t// path with trailing /\n\tif err := fs.Mkdir(ctx, \"/ddd/\", 0777, 000); err != 0 {\n\t\tt.Fatalf(\"mkdir /ddd/: %s\", err)\n\t}\n\tif _, err := fs.Create(ctx, \"/ddd/ddd\", 0777, 000); err != 0 {\n\t\tt.Fatalf(\"create /ddd/ddd: %s\", err)\n\t}\n\tif _, err := fs.Create(ctx, \"/ddd/fff/\", 0777, 000); err != syscall.EINVAL {\n\t\tt.Fatalf(\"create /ddd/fff/: %s\", err)\n\t}\n\tif err := fs.Delete(ctx, \"/ddd/\"); err != syscall.ENOTEMPTY {\n\t\tt.Fatalf(\"delete /ddd/: %s\", err)\n\t}\n\tif err := fs.Rename(ctx, \"/ddd/\", \"/ttt/\", 0); err != 0 {\n\t\tt.Fatalf(\"delete /ddd/: %s\", err)\n\t}\n\tif err := fs.Rmr(ctx, \"/ttt/\", false, meta.RmrDefaultThreads); err != 0 {\n\t\tt.Fatalf(\"rmr /ttt/: %s\", err)\n\t}\n\tif _, err := fs.Stat(ctx, \"/ttt/\"); err != syscall.ENOENT {\n\t\tt.Fatalf(\"stat /ttt/: %s\", err)\n\t}\n}\n\nfunc createTestFS(t *testing.T) *FileSystem {\n\tm := meta.NewClient(\"memkv://\", nil)\n\tformat := &meta.Format{\n\t\tName: \"test\",\n\t\tBlockSize: 4096,\n\t\tCapacity: 1 << 30,\n\t\tDirStats: true,\n\t}\n\t_ = m.Init(format, true)\n\tvar conf = vfs.Config{\n\t\tMeta: meta.DefaultConf(),\n\t\tChunk: &chunk.Config{\n\t\t\tBlockSize: format.BlockSize << 10,\n\t\t\tMaxUpload: 1,\n\t\t\tMaxDownload: 200,\n\t\t\tBufferSize: 100 << 20,\n\t\t},\n\t\tDirEntryTimeout: time.Millisecond * 100,\n\t\tEntryTimeout: time.Millisecond * 100,\n\t\tAttrTimeout: time.Millisecond * 100,\n\t\tAccessLog: \"/tmp/juicefs.access.log\",\n\t}\n\tobjStore, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tstore := chunk.NewCachedStore(objStore, *conf.Chunk, nil)\n\tjfs, err := NewFileSystem(&conf, m, store, nil)\n\tjfs.checkAccessFile = time.Millisecond\n\tjfs.rotateAccessLog = 500\n\tif err != nil {\n\t\tt.Fatalf(\"initialize failed: %s\", err)\n\t}\n\treturn jfs\n}\n" }, { "path": "pkg/fs/http.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fs\n\nimport (\n\t\"compress/gzip\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"encoding/xml\"\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"golang.org/x/net/webdav\"\n)\n\ntype gzipResponseWriter struct {\n\tio.Writer\n\thttp.ResponseWriter\n}\n\nfunc (w gzipResponseWriter) Write(b []byte) (int, error) {\n\treturn w.Writer.Write(b)\n}\n\ntype gzipHandler struct {\n\thandler http.Handler\n}\n\nfunc (g *gzipHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tif !strings.Contains(r.Header.Get(\"Accept-Encoding\"), \"gzip\") {\n\t\tg.handler.ServeHTTP(w, r)\n\t\treturn\n\t}\n\tw.Header().Set(\"Content-Encoding\", \"gzip\")\n\tgz := gzip.NewWriter(w)\n\tdefer gz.Close()\n\tgzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}\n\tg.handler.ServeHTTP(gzr, r)\n}\n\nfunc makeGzipHandler(h http.Handler) http.Handler {\n\treturn &gzipHandler{h}\n}\n\nvar errmap = map[syscall.Errno]error{\n\t0: nil,\n\tsyscall.EPERM: os.ErrPermission,\n\tsyscall.ENOENT: os.ErrNotExist,\n\tsyscall.EEXIST: os.ErrExist,\n}\n\nfunc econv(err error) error {\n\tif err == nil {\n\t\treturn nil\n\t}\n\teno, ok := err.(syscall.Errno)\n\tif !ok {\n\t\treturn err\n\t}\n\tif e, ok := errmap[eno]; ok {\n\t\treturn e\n\t}\n\treturn err\n}\n\ntype webdavFS struct {\n\tctx meta.Context\n\tfs *FileSystem\n\tumask uint16\n\tconfig WebdavConfig\n}\n\nfunc (hfs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) error {\n\treturn econv(hfs.fs.Mkdir(hfs.ctx, name, uint16(perm), hfs.umask))\n}\n\nfunc (hfs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {\n\tvar mode int\n\tif flag&(os.O_RDONLY|os.O_RDWR) != 0 {\n\t\tmode |= vfs.MODE_MASK_R\n\t}\n\tif flag&(os.O_APPEND|os.O_RDWR|os.O_WRONLY) != 0 {\n\t\tmode |= vfs.MODE_MASK_W\n\t}\n\tif flag&(os.O_EXCL) != 0 {\n\t\tmode |= vfs.MODE_MASK_X\n\t}\n\tname = strings.TrimRight(name, \"/\")\n\tf, err := hfs.fs.Open(hfs.ctx, name, uint32(mode))\n\tif err != 0 {\n\t\tif err == syscall.ENOENT && flag&os.O_CREATE != 0 {\n\t\t\tf, err = hfs.fs.Create(hfs.ctx, name, uint16(perm), hfs.umask)\n\t\t}\n\t} else if flag&os.O_TRUNC != 0 {\n\t\tif errno := hfs.fs.Truncate(hfs.ctx, name, 0); errno != 0 {\n\t\t\treturn nil, errno\n\t\t}\n\t} else if flag&os.O_APPEND != 0 {\n\t\tif _, err := f.Seek(hfs.ctx, 0, 2); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn &davFile{f, hfs.ctx, hfs.fs, hfs.config}, econv(err)\n}\n\nfunc (hfs *webdavFS) RemoveAll(ctx context.Context, name string) error {\n\treturn econv(hfs.fs.Rmr(hfs.ctx, name, false, hfs.config.MaxDeletes))\n}\n\nfunc (hfs *webdavFS) Rename(ctx context.Context, oldName, newName string) error {\n\treturn econv(hfs.fs.Rename(hfs.ctx, oldName, newName, 0))\n}\n\nfunc (hfs *webdavFS) Stat(ctx context.Context, name string) (os.FileInfo, error) {\n\tfi, err := hfs.fs.Stat(hfs.ctx, removeNewLine(name))\n\treturn fi, econv(err)\n}\n\ntype davFile struct {\n\t*File\n\tmctx meta.Context\n\tfs *FileSystem\n\tconfig WebdavConfig\n}\n\nconst webdavDeadProps = \"webdav-dead-props\"\n\ntype localProperty struct {\n\tN xml.Name `json:\"name\"`\n\tP webdav.Property `json:\"property\"`\n}\n\nfunc (f *davFile) DeadProps() (map[xml.Name]webdav.Property, error) {\n\tif !f.config.EnableProppatch {\n\t\treturn nil, nil\n\t}\n\tresult, err := f.fs.GetXattr(f.mctx, f.path, webdavDeadProps)\n\tif err != 0 {\n\t\tif errors.Is(err, meta.ENOATTR) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, econv(err)\n\t}\n\n\tvar lProperty []localProperty\n\tif err := json.Unmarshal(result, &lProperty); err != nil {\n\t\treturn nil, econv(err)\n\t}\n\tvar property = make(map[xml.Name]webdav.Property)\n\tfor _, p := range lProperty {\n\t\tproperty[p.N] = p.P\n\t}\n\treturn property, nil\n}\n\nfunc (f *davFile) Patch(patches []webdav.Proppatch) ([]webdav.Propstat, error) {\n\tif !f.config.EnableProppatch {\n\t\treturn nil, nil\n\t}\n\tpstat := webdav.Propstat{Status: http.StatusOK}\n\tdeadProps, err := f.DeadProps()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, patch := range patches {\n\t\tfor _, p := range patch.Props {\n\t\t\tpstat.Props = append(pstat.Props, webdav.Property{XMLName: p.XMLName})\n\t\t\tif patch.Remove && deadProps != nil {\n\t\t\t\tdelete(deadProps, p.XMLName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif deadProps == nil {\n\t\t\t\tdeadProps = map[xml.Name]webdav.Property{}\n\t\t\t}\n\t\t\tdeadProps[p.XMLName] = p\n\t\t}\n\t}\n\n\tif deadProps != nil {\n\t\tvar property []localProperty\n\t\tfor name, p := range deadProps {\n\t\t\tproperty = append(property, localProperty{N: name, P: p})\n\t\t}\n\n\t\tjsonData, err := json.Marshal(&property)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\terrno := f.fs.SetXattr(f.mctx, f.path, webdavDeadProps, jsonData, 0)\n\t\tif errno != 0 {\n\t\t\treturn nil, econv(errno)\n\t\t}\n\t}\n\treturn []webdav.Propstat{pstat}, nil\n}\n\nfunc (f *davFile) Seek(offset int64, whence int) (int64, error) {\n\tn, err := f.File.Seek(meta.Background(), offset, whence)\n\treturn n, econv(err)\n}\n\nfunc (f *davFile) Read(b []byte) (n int, err error) {\n\tn, err = f.File.Read(meta.Background(), b)\n\treturn n, econv(err)\n}\n\nfunc (f *davFile) Write(buf []byte) (n int, err error) {\n\tn, err = f.File.Write(meta.Background(), buf)\n\treturn n, econv(err)\n}\n\nfunc (f *davFile) Readdir(count int) (fi []os.FileInfo, err error) {\n\tfi, err = f.File.Readdir(meta.Background(), count)\n\t// skip the first two (. and ..)\n\tfor len(fi) > 0 && (fi[0].Name() == \".\" || fi[0].Name() == \"..\") {\n\t\tfi = fi[1:]\n\t}\n\treturn fi, econv(err)\n}\n\nfunc (f *davFile) Close() error {\n\treturn econv(f.File.Close(meta.Background()))\n}\n\ntype WebdavConfig struct {\n\tAddr string\n\tDisallowList bool\n\tEnableProppatch bool\n\tEnableGzip bool\n\tUsername string\n\tPassword string\n\tCertFile string\n\tKeyFile string\n\tMaxDeletes int\n}\n\ntype indexHandler struct {\n\t*webdav.Handler\n\tWebdavConfig\n}\n\nfunc (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\n\t// http://www.webdav.org/specs/rfc4918.html#n-guidance-for-clients-desiring-to-authenticate\n\tif h.Username != \"\" && h.Password != \"\" {\n\t\tuserName, pwd, ok := r.BasicAuth()\n\t\tif !ok {\n\t\t\tw.Header().Set(\"WWW-Authenticate\", `Basic realm=\"Restricted\"`)\n\t\t\tw.WriteHeader(http.StatusUnauthorized)\n\t\t\treturn\n\t\t}\n\t\tif userName != h.Username || pwd != h.Password {\n\t\t\thttp.Error(w, \"WebDAV: need authorized!\", http.StatusUnauthorized)\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Excerpt from RFC4918, section 9.4:\n\t//\n\t// \t\tGET, when applied to a collection, may return the contents of an\n\t//\t\t\"index.html\" resource, a human-readable view of the contents of\n\t//\t\tthe collection, or something else altogether.\n\t//\n\t// Get, when applied to collection, will return the same as PROPFIND method.\n\tif r.Method == \"GET\" && strings.HasPrefix(r.URL.Path, h.Handler.Prefix) {\n\t\tinfo, err := h.Handler.FileSystem.Stat(context.TODO(), strings.TrimPrefix(r.URL.Path, h.Handler.Prefix))\n\t\tif err == nil && info.IsDir() {\n\t\t\tif h.DisallowList {\n\t\t\t\thttp.Error(w, \"Forbidden\", http.StatusForbidden)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tr.Method = \"PROPFIND\"\n\t\t\tif r.Header.Get(\"Depth\") == \"\" {\n\t\t\t\tr.Header.Add(\"Depth\", \"1\")\n\t\t\t}\n\t\t}\n\t}\n\n\t// The next line would normally be:\n\t//\thttp.Handle(\"/\", h)\n\t// but we wrap that HTTP handler h to cater for a special case.\n\t//\n\t// The propfind_invalid2 litmus test case expects an empty namespace prefix\n\t// declaration to be an error. The FAQ in the webdav litmus test says:\n\t//\n\t// \"What does the \"propfind_invalid2\" test check for?...\n\t//\n\t// If a request was sent with an XML body which included an empty namespace\n\t// prefix declaration (xmlns:ns1=\"\"), then the server must reject that with\n\t// a \"400 Bad Request\" response, as it is invalid according to the XML\n\t// Namespace specification.\"\n\t//\n\t// On the other hand, the Go standard library's encoding/xml package\n\t// accepts an empty xmlns namespace, as per the discussion at\n\t// https://github.com/golang/go/issues/8068\n\t//\n\t// Empty namespaces seem disallowed in the second (2006) edition of the XML\n\t// standard, but allowed in a later edition. The grammar differs between\n\t// http://www.w3.org/TR/2006/REC-xml-names-20060816/#ns-decl and\n\t// http://www.w3.org/TR/REC-xml-names/#dt-prefix\n\t//\n\t// Thus, we assume that the propfind_invalid2 test is obsolete, and\n\t// hard-code the 400 Bad Request response that the test expects.\n\tif r.Header.Get(\"X-Litmus\") == \"props: 3 (propfind_invalid2)\" {\n\t\thttp.Error(w, \"400 Bad Request\", http.StatusBadRequest)\n\t\treturn\n\t}\n\n\tif !h.EnableProppatch && r.Method == \"PROPPATCH\" {\n\t\thttp.Error(w, \"The PROPPATCH method is not currently enabled,please add the --enable-proppatch parameter and run it again\", http.StatusNotImplemented)\n\t\treturn\n\t}\n\n\th.Handler.ServeHTTP(w, r)\n}\n\nfunc StartHTTPServer(fs *FileSystem, config WebdavConfig) {\n\tctx := meta.NewContext(uint32(os.Getpid()), uint32(utils.GetCurrentUID()), []uint32{uint32(utils.GetCurrentGID())})\n\thfs := &webdavFS{ctx, fs, uint16(utils.GetUmask()), config}\n\tsrv := &webdav.Handler{\n\t\tFileSystem: hfs,\n\t\tLockSystem: webdav.NewMemLS(),\n\t\tLogger: func(r *http.Request, err error) {\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"WEBDAV [%s]: %s, ERROR: %s\", r.Method, r.URL, err)\n\t\t\t} else {\n\t\t\t\tlogger.Debugf(\"WEBDAV [%s]: %s\", r.Method, r.URL)\n\t\t\t}\n\t\t},\n\t}\n\tvar h http.Handler = &indexHandler{Handler: srv, WebdavConfig: config}\n\tif config.EnableGzip {\n\t\th = makeGzipHandler(h)\n\t}\n\thttp.Handle(\"/\", h)\n\tlogger.Infof(\"WebDAV listening on %s\", config.Addr)\n\tvar err error\n\tif config.CertFile != \"\" && config.KeyFile != \"\" {\n\t\terr = http.ListenAndServeTLS(config.Addr, config.CertFile, config.KeyFile, nil)\n\t} else {\n\t\terr = http.ListenAndServe(config.Addr, nil)\n\t}\n\tif err != nil {\n\t\tlogger.Fatalf(\"Error with WebDAV server: %v\", err)\n\t}\n}\n\nfunc removeNewLine(input string) string {\n\treturn strings.Replace(strings.Replace(input, \"\\n\", \"\", -1), \"\\r\", \"\", -1)\n}\n" }, { "path": "pkg/fs/http_test.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fs\n\nimport (\n\t\"context\"\n\t\"io\"\n\t\"io/fs\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nfunc TestWebdav(t *testing.T) {\n\tjfs := createTestFS(t)\n\twebdavFS := &webdavFS{meta.NewContext(uint32(os.Getpid()), uint32(os.Getuid()), []uint32{uint32(os.Getgid())}), jfs, uint16(utils.GetUmask()), WebdavConfig{EnableProppatch: true}}\n\tctx := context.Background()\n\t_, err := webdavFS.Stat(ctx, \"/\")\n\tif err != nil {\n\t\tt.Fatalf(\"webdavFS stat failed: %s\", err)\n\t}\n\taFile, err := webdavFS.OpenFile(ctx, \"/a\", os.O_CREATE, 0644)\n\tif err != nil {\n\t\tt.Fatalf(\"webdavFS create failed: %s\", err)\n\t}\n\t_, err = webdavFS.OpenFile(ctx, \"/b/\", os.O_CREATE, 0644)\n\tif err != nil {\n\t\tt.Fatalf(\"webdavFS create failed: %s\", err)\n\t}\n\taInfo, err := aFile.Stat()\n\tif err != nil || aInfo.Name() != \"a\" || aInfo.Mode().Perm() != fs.FileMode(0644) {\n\t\tt.Fatalf(\"webdavFS stat failed: %s\", err)\n\t}\n\tif n, err := aFile.Write([]byte(\"world\")); err != nil || n != 5 {\n\t\tt.Fatalf(\"webdavFS write 5 bytes: %d %s\", n, err)\n\t}\n\tif n, err := aFile.Seek(-3, io.SeekEnd); err != nil || n != 2 {\n\t\tt.Fatalf(\"webdavFS seek 3 bytes before end: %d %s\", n, err)\n\t}\n\tbuf := make([]byte, 100)\n\tif n, err := aFile.Read(buf); err != nil || n != 3 || string(buf[:n]) != \"rld\" {\n\t\tt.Fatalf(\"webdavFS read(): %d %s %s\", n, err, string(buf[:n]))\n\t}\n\n\tif err = webdavFS.Mkdir(ctx, \"/d1\", 0755); err != nil {\n\t\tt.Fatalf(\"webdavFS mkdir failed: %s\", err)\n\t}\n\tif d1Info, err := webdavFS.Stat(ctx, \"/d1\"); err != nil || d1Info.Name() != \"d1\" || d1Info.Mode().Perm() != fs.FileMode(0755) {\n\t\tt.Fatalf(\"webdavFS stat failed: %s\", err)\n\t}\n\tif webdavFS.Rename(ctx, \"/d1\", \"/d2\") != nil {\n\t\tt.Fatalf(\"webdavFS rename failed: %s\", err)\n\t}\n\tif stat, err := webdavFS.Stat(ctx, \"/d2\"); err != nil || !stat.IsDir() {\n\t\tt.Fatalf(\"webdavFS rename failed: %s\", err)\n\t}\n\tfor _, name := range []string{\"/d2/a\", \"/d2/b\", \"/d2/c\", \"/d2/d\"} {\n\t\tif _, err := webdavFS.OpenFile(ctx, name, os.O_CREATE, 0644); err != nil {\n\t\t\tt.Fatalf(\"webdavFS create failed: %s\", err)\n\t\t}\n\t}\n\tif webdavFS.RemoveAll(ctx, \"/d2\") != nil {\n\t\tt.Fatalf(\"webdavFS removeAll failed: %s\", err)\n\t}\n\tif _, err = webdavFS.Stat(ctx, \"/d2\"); err != os.ErrNotExist {\n\t\tt.Fatalf(\"webdavFS removeAll failed: %s\", err)\n\t}\n\tif err = aFile.Close(); err != nil {\n\t\tt.Fatalf(\"webdavFS close file failed: %s\", err)\n\t}\n}\n" }, { "path": "pkg/fs/metrics.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fs\n" }, { "path": "pkg/fuse/context.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fuse\n\nimport (\n\t\"context\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\n\t\"github.com/hanwen/go-fuse/v2/fuse\"\n)\n\n// Ino is an alias to meta.Ino\ntype Ino = meta.Ino\n\n// Attr is an alias to meta.Attr\ntype Attr = meta.Attr\n\n// Context is an alias to vfs.LogContext\ntype Context = vfs.LogContext\n\ntype fuseContext struct {\n\tcontext.Context\n\tstart time.Time\n\theader *fuse.InHeader\n\tcanceled bool\n\tcancel <-chan struct{}\n\n\tcheckPermission bool\n}\n\nvar gidcache = newGidCache(time.Minute * 5)\n\nvar contextPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn &fuseContext{}\n\t},\n}\n\nfunc (fs *fileSystem) newContext(cancel <-chan struct{}, header *fuse.InHeader) *fuseContext {\n\tctx := contextPool.Get().(*fuseContext)\n\tctx.Context = context.Background()\n\tctx.start = time.Now()\n\tctx.canceled = false\n\tctx.cancel = cancel\n\tctx.header = header\n\tctx.checkPermission = fs.conf.NonDefaultPermission && header.Uid != 0\n\tif header.Uid == 0 && fs.conf.RootSquash != nil {\n\t\tctx.checkPermission = true\n\t\tctx.header.Uid = fs.conf.RootSquash.Uid\n\t\tctx.header.Gid = fs.conf.RootSquash.Gid\n\t}\n\tif fs.conf.AllSquash != nil {\n\t\tctx.checkPermission = true\n\t\tctx.header.Uid = fs.conf.AllSquash.Uid\n\t\tctx.header.Gid = fs.conf.AllSquash.Gid\n\t}\n\treturn ctx\n}\n\nfunc releaseContext(ctx *fuseContext) {\n\tcontextPool.Put(ctx)\n}\n\nfunc (c *fuseContext) Uid() uint32 {\n\treturn c.header.Uid\n}\n\nfunc (c *fuseContext) Gid() uint32 {\n\treturn c.header.Gid\n}\n\nfunc (c *fuseContext) Gids() []uint32 {\n\tif c.checkPermission {\n\t\treturn gidcache.get(c.Pid(), c.Gid())\n\t}\n\treturn []uint32{c.header.Gid}\n}\n\nfunc (c *fuseContext) Pid() uint32 {\n\treturn c.header.Pid\n}\n\nfunc (c *fuseContext) Duration() time.Duration {\n\treturn time.Since(c.start)\n}\n\nfunc (c *fuseContext) Cancel() {\n\tc.canceled = true\n}\n\nfunc (c *fuseContext) CheckPermission() bool {\n\treturn c.checkPermission\n}\n\nfunc (c *fuseContext) Canceled() bool {\n\tif c.Duration() < time.Second {\n\t\treturn false\n\t}\n\tif c.canceled {\n\t\treturn true\n\t}\n\tselect {\n\tcase <-c.cancel:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n\nfunc (c *fuseContext) WithValue(k, v interface{}) meta.Context {\n\twc := *c // gids is a const, so it's safe to shallow copy\n\twc.Context = context.WithValue(c.Context, k, v)\n\treturn &wc\n}\n\nfunc (c *fuseContext) Err() error {\n\treturn syscall.EINTR\n}\n\n// func (c *fuseContext) Done() <-chan struct{} {\n// \treturn c.cancel\n// }\n" }, { "path": "pkg/fuse/device_darwin.go", "content": "// Copyright 2020 Chaos Mesh Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fuse\n\nfunc ensureFuseDev() {}\n\nfunc grantAccess() error {\n\treturn nil\n}\n" }, { "path": "pkg/fuse/device_linux.go", "content": "// Copyright 2020 Chaos Mesh Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fuse\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\n// ensureFuseDev ensures /dev/fuse exists. If not, it will create one\nfunc ensureFuseDev() {\n\tif _, err := os.Open(\"/dev/fuse\"); os.IsNotExist(err) {\n\t\t// 10, 229 according to https://www.kernel.org/doc/Documentation/admin-guide/devices.txt\n\t\tfuse := unix.Mkdev(10, 229)\n\t\tif err := syscall.Mknod(\"/dev/fuse\", 0o666|syscall.S_IFCHR, int(fuse)); err != nil {\n\t\t\tlogger.Errorf(\"mknod /dev/fuse: %v\", err)\n\t\t}\n\t}\n}\n\n// grantAccess appends 'c 10:229 rwm' to devices.allow\nfunc grantAccess() error {\n\tpid := os.Getpid()\n\tcgroupPath := fmt.Sprintf(\"/proc/%d/cgroup\", pid)\n\tcgroupFile, err := os.Open(cgroupPath)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"open %s\", cgroupPath)\n\t}\n\tdefer cgroupFile.Close()\n\n\tcgroupScanner := bufio.NewScanner(cgroupFile)\n\tvar deviceCgroup string\n\tfor cgroupScanner.Scan() {\n\t\tif err := cgroupScanner.Err(); err != nil {\n\t\t\treturn errors.Wrap(err, \"read cgroup file\")\n\t\t}\n\t\tvar (\n\t\t\ttext = cgroupScanner.Text()\n\t\t\tparts = strings.SplitN(text, \":\", 3)\n\t\t)\n\t\tif len(parts) < 3 {\n\t\t\treturn errors.Errorf(\"invalid cgroup entry: %q\", text)\n\t\t}\n\n\t\tif parts[1] == \"devices\" {\n\t\t\tdeviceCgroup = parts[2]\n\t\t}\n\t}\n\n\tif len(deviceCgroup) == 0 {\n\t\treturn errors.Errorf(\"fail to find device cgroup\")\n\t}\n\n\tdeviceListPath := path.Join(\"/sys/fs/cgroup/devices\" + deviceCgroup, \"/devices.list\")\n\tdeviceAllowPath := path.Join(\"/sys/fs/cgroup/devices\" + deviceCgroup, \"/devices.allow\")\n\n\t// check if fuse is already allowed\n\tdeviceListFile, err := os.OpenFile(deviceListPath, os.O_RDONLY, 0)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"open %s\", deviceListPath)\n\t}\n\tdefer deviceListFile.Close()\n\tdeviceListScanner := bufio.NewScanner(deviceListFile)\n\tfor deviceListScanner.Scan() {\n\t\tif err := deviceListScanner.Err(); err != nil {\n\t\t\treturn errors.Wrap(err, \"read device list file\")\n\t\t}\n\t\tvar (\n\t\t\ttext = deviceListScanner.Text()\n\t\t\tparts = strings.SplitN(text, \" \", 3)\n\t\t)\n\t\tif len(parts) < 3 {\n\t\t\treturn errors.Errorf(\"invalid device list entry: %q\", text)\n\t\t}\n\n\t\tif (parts[0] == \"c\" || parts[0] == \"a\") && (parts[1] == \"10:229\" || parts[1] == \"*:*\") && parts[2] == \"rwm\" {\n\t\t\tlogger.Debug(\"/dev/fuse is already granted\")\n\t\t\t// fuse is already allowed\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tf, err := os.OpenFile(deviceAllowPath, os.O_WRONLY, 0)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"open %s\", deviceAllowPath)\n\t}\n\tdefer f.Close()\n\t// 10, 229 according to https://www.kernel.org/doc/Documentation/admin-guide/devices.txt\n\tcontent := \"c 10:229 rwm\"\n\t_, err = f.WriteString(content)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"write %s to %s\", content, deviceAllowPath)\n\t}\n\tlogger.Debug(\"/dev/fuse is granted\")\n\treturn nil\n}\n" }, { "path": "pkg/fuse/fuse.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fuse\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"math\"\n\t\"os\"\n\t\"os/exec\"\n\t\"runtime\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/hanwen/go-fuse/v2/fuse\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n)\n\nvar logger = utils.GetLogger(\"juicefs\")\n\ntype fileSystem struct {\n\tfuse.RawFileSystem\n\tconf *vfs.Config\n\tv *vfs.VFS\n}\n\nfunc newFileSystem(conf *vfs.Config, v *vfs.VFS) *fileSystem {\n\treturn &fileSystem{\n\t\tRawFileSystem: fuse.NewDefaultRawFileSystem(),\n\t\tconf: conf,\n\t\tv: v,\n\t}\n}\n\ntype setTimeout func(time.Duration)\n\nfunc (fs *fileSystem) replyAttr(ctx *fuseContext, entry *meta.Entry, attr *fuse.Attr, set setTimeout) {\n\tif vfs.IsSpecialNode(entry.Inode) {\n\t\tset(time.Hour)\n\t} else if entry.Attr.Typ == meta.TypeFile && fs.v.ModifiedSince(entry.Inode, ctx.start) {\n\t\tlogger.Debugf(\"refresh attr for %d\", entry.Inode)\n\t\tvar attr meta.Attr\n\t\tst := fs.v.Meta.GetAttr(ctx, entry.Inode, &attr)\n\t\tif st == 0 {\n\t\t\t*entry.Attr = attr\n\t\t\tset(fs.conf.AttrTimeout)\n\t\t}\n\t} else {\n\t\tset(fs.conf.AttrTimeout)\n\t}\n\tfs.v.UpdateLength(entry.Inode, entry.Attr)\n\tattrToStat(entry.Inode, entry.Attr, attr)\n}\n\nfunc (fs *fileSystem) replyEntry(ctx *fuseContext, out *fuse.EntryOut, e *meta.Entry) fuse.Status {\n\tout.NodeId = uint64(e.Inode)\n\tout.Generation = 1\n\tif e.Attr.Typ == meta.TypeDirectory {\n\t\tout.SetEntryTimeout(fs.conf.DirEntryTimeout)\n\t} else {\n\t\tout.SetEntryTimeout(fs.conf.EntryTimeout)\n\t}\n\tfs.replyAttr(ctx, e, &out.Attr, out.SetAttrTimeout)\n\treturn 0\n}\n\nfunc (fs *fileSystem) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name string, out *fuse.EntryOut) (status fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\tentry, err := fs.v.Lookup(ctx, Ino(header.NodeId), name)\n\tif err != 0 {\n\t\tif fs.conf.NegEntryTimeout != 0 && err == syscall.ENOENT {\n\t\t\tout.NodeId = 0 // zero nodeid is same as ENOENT, but with valid timeout\n\t\t\tout.SetEntryTimeout(fs.conf.NegEntryTimeout)\n\t\t\treturn 0\n\t\t}\n\t\treturn fuse.Status(err)\n\t}\n\treturn fs.replyEntry(ctx, out, entry)\n}\n\nfunc (fs *fileSystem) GetAttr(cancel <-chan struct{}, in *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tvar opened uint8\n\tif in.Fh() != 0 {\n\t\topened = 1\n\t}\n\tentry, err := fs.v.GetAttr(ctx, Ino(in.NodeId), opened)\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\tfs.replyAttr(ctx, entry, &out.Attr, out.SetTimeout)\n\treturn 0\n}\n\nfunc (fs *fileSystem) SetAttr(cancel <-chan struct{}, in *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentry, err := fs.v.SetAttr(ctx, Ino(in.NodeId), int(in.Valid), in.Fh, in.Mode, in.Uid, in.Gid, int64(in.Atime), int64(in.Mtime), in.Atimensec, in.Mtimensec, in.Size)\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\tfs.replyAttr(ctx, entry, &out.Attr, out.SetTimeout)\n\treturn 0\n}\n\nfunc (fs *fileSystem) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out *fuse.EntryOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentry, err := fs.v.Mknod(ctx, Ino(in.NodeId), name, uint16(in.Mode), getUmask(in.Umask, fs.v.Conf.UMask, false), in.Rdev)\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\treturn fs.replyEntry(ctx, out, entry)\n}\n\nfunc (fs *fileSystem) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out *fuse.EntryOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentry, err := fs.v.Mkdir(ctx, Ino(in.NodeId), name, uint16(in.Mode), getUmask(in.Umask, fs.v.Conf.UMask, true))\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\treturn fs.replyEntry(ctx, out, entry)\n}\n\nfunc (fs *fileSystem) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Unlink(ctx, Ino(header.NodeId), name)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Rmdir(ctx, Ino(header.NodeId), name)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string, newName string) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Rename(ctx, Ino(in.NodeId), oldName, Ino(in.Newdir), newName, in.Flags)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *fuse.EntryOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentry, err := fs.v.Link(ctx, Ino(in.Oldnodeid), Ino(in.NodeId), name)\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\treturn fs.replyEntry(ctx, out, entry)\n}\n\nfunc (fs *fileSystem) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target string, name string, out *fuse.EntryOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\tentry, err := fs.v.Symlink(ctx, target, Ino(header.NodeId), name)\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\treturn fs.replyEntry(ctx, out, entry)\n}\n\nfunc (fs *fileSystem) Readlink(cancel <-chan struct{}, header *fuse.InHeader) (out []byte, code fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\tpath, err := fs.v.Readlink(ctx, Ino(header.NodeId))\n\treturn path, fuse.Status(err)\n}\n\nfunc (fs *fileSystem) GetXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string, dest []byte) (sz uint32, code fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\tvalue, err := fs.v.GetXattr(ctx, Ino(header.NodeId), attr, uint32(len(dest)))\n\tif err != 0 {\n\t\treturn 0, fuse.Status(err)\n\t}\n\tcopy(dest, value)\n\treturn uint32(len(value)), 0\n}\n\nfunc (fs *fileSystem) ListXAttr(cancel <-chan struct{}, header *fuse.InHeader, dest []byte) (uint32, fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\tdata, err := fs.v.ListXattr(ctx, Ino(header.NodeId), len(dest))\n\tif err != 0 {\n\t\treturn 0, fuse.Status(err)\n\t}\n\tcopy(dest, data)\n\treturn uint32(len(data)), 0\n}\n\nfunc (fs *fileSystem) SetXAttr(cancel <-chan struct{}, in *fuse.SetXAttrIn, attr string, data []byte) fuse.Status {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.SetXattr(ctx, Ino(in.NodeId), attr, data, in.Flags)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string) (code fuse.Status) {\n\tctx := fs.newContext(cancel, header)\n\tdefer releaseContext(ctx)\n\terr := fs.v.RemoveXattr(ctx, Ino(header.NodeId), attr)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, out *fuse.CreateOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentry, fh, err := fs.v.Create(ctx, Ino(in.NodeId), name, uint16(in.Mode), getCreateUmask(in.Umask, fs.v.Conf.UMask), in.Flags)\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\tout.Fh = fh\n\treturn fs.replyEntry(ctx, &out.EntryOut, entry)\n}\n\nfunc (fs *fileSystem) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentry, fh, err := fs.v.Open(ctx, Ino(in.NodeId), in.Flags)\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\tout.Fh = fh\n\tif vfs.IsSpecialNode(Ino(in.NodeId)) {\n\t\tout.OpenFlags |= fuse.FOPEN_DIRECT_IO\n\t} else if entry.Attr.KeepCache {\n\t\tout.OpenFlags |= fuse.FOPEN_KEEP_CACHE\n\t} else {\n\t\tif runtime.GOOS == \"darwin\" {\n\t\t\tgo fsserv.InodeNotify(uint64(in.NodeId), -1, 0)\n\t\t} else {\n\t\t\tfsserv.InodeNotify(uint64(in.NodeId), -1, 0)\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (fs *fileSystem) Read(cancel <-chan struct{}, in *fuse.ReadIn, buf []byte) (fuse.ReadResult, fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tn, err := fs.v.Read(ctx, Ino(in.NodeId), buf, in.Offset, in.Fh)\n\tif err != 0 {\n\t\treturn nil, fuse.Status(err)\n\t}\n\treturn fuse.ReadResultData(buf[:n]), 0\n}\n\nfunc (fs *fileSystem) Release(cancel <-chan struct{}, in *fuse.ReleaseIn) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tfs.v.Release(ctx, Ino(in.NodeId), in.Fh)\n}\n\nfunc (fs *fileSystem) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (written uint32, code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Write(ctx, Ino(in.NodeId), data, in.Offset, in.Fh)\n\tif err != 0 {\n\t\treturn 0, fuse.Status(err)\n\t}\n\treturn uint32(len(data)), 0\n}\n\nfunc (fs *fileSystem) Flush(cancel <-chan struct{}, in *fuse.FlushIn) fuse.Status {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Flush(ctx, Ino(in.NodeId), in.Fh, in.LockOwner)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) Fsync(cancel <-chan struct{}, in *fuse.FsyncIn) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Fsync(ctx, Ino(in.NodeId), int(in.FsyncFlags), in.Fh)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) Fallocate(cancel <-chan struct{}, in *fuse.FallocateIn) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Fallocate(ctx, Ino(in.NodeId), uint8(in.Mode), int64(in.Offset), int64(in.Length), in.Fh)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn) (written uint32, code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tvar len = in.Len\n\tif len > math.MaxUint32 {\n\t\t// written may overflow\n\t\tlen = math.MaxUint32 + 1 - meta.ChunkSize\n\t}\n\tcopied, err := fs.v.CopyFileRange(ctx, Ino(in.NodeId), in.FhIn, in.OffIn, Ino(in.NodeIdOut), in.FhOut, in.OffOut, len, uint32(in.Flags))\n\tif err != 0 {\n\t\treturn 0, fuse.Status(err)\n\t}\n\treturn uint32(copied), 0\n}\n\nfunc (fs *fileSystem) GetLk(cancel <-chan struct{}, in *fuse.LkIn, out *fuse.LkOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tl := in.Lk\n\terr := fs.v.Getlk(ctx, Ino(in.NodeId), in.Fh, in.Owner, &l.Start, &l.End, &l.Typ, &l.Pid)\n\tif err == 0 {\n\t\tout.Lk = l\n\t}\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) SetLk(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) {\n\treturn fs.setLk(cancel, in, false)\n}\n\nfunc (fs *fileSystem) SetLkw(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) {\n\treturn fs.setLk(cancel, in, true)\n}\n\nfunc (fs *fileSystem) setLk(cancel <-chan struct{}, in *fuse.LkIn, block bool) (code fuse.Status) {\n\tif in.LkFlags&fuse.FUSE_LK_FLOCK != 0 {\n\t\treturn fs.Flock(cancel, in, block)\n\t}\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tl := in.Lk\n\terr := fs.v.Setlk(ctx, Ino(in.NodeId), in.Fh, in.Owner, l.Start, l.End, l.Typ, l.Pid, block)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) Flock(cancel <-chan struct{}, in *fuse.LkIn, block bool) (code fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Flock(ctx, Ino(in.NodeId), in.Fh, in.Owner, in.Lk.Typ, block)\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) OpenDir(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tino := Ino(in.NodeId)\n\tfh, err := fs.v.Opendir(ctx, ino, in.Flags)\n\tout.Fh = fh\n\tif fs.conf.ReaddirCache && !vfs.IsSpecialNode(ino) {\n\t\tout.OpenFlags |= fuse.FOPEN_CACHE_DIR | fuse.FOPEN_KEEP_CACHE // both flags are required\n\t}\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) ReadDir(cancel <-chan struct{}, in *fuse.ReadIn, out *fuse.DirEntryList) fuse.Status {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentries, _, err := fs.v.Readdir(ctx, Ino(in.NodeId), in.Size, int(in.Offset), in.Fh, false)\n\tvar de fuse.DirEntry\n\tfor i, e := range entries {\n\t\tde.Ino = uint64(e.Inode)\n\t\tde.Name = string(e.Name)\n\t\tde.Mode = e.Attr.SMode()\n\t\tif !out.AddDirEntry(de) {\n\t\t\tfs.v.UpdateReaddirOffset(ctx, Ino(in.NodeId), in.Fh, int(in.Offset)+i)\n\t\t\tbreak\n\t\t}\n\t}\n\treturn fuse.Status(err)\n}\n\nfunc (fs *fileSystem) ReadDirPlus(cancel <-chan struct{}, in *fuse.ReadIn, out *fuse.DirEntryList) fuse.Status {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tentries, readAt, err := fs.v.Readdir(ctx, Ino(in.NodeId), in.Size, int(in.Offset), in.Fh, true)\n\tctx.start = readAt\n\tvar de fuse.DirEntry\n\tfor i, e := range entries {\n\t\tde.Ino = uint64(e.Inode)\n\t\tde.Name = string(e.Name)\n\t\tde.Mode = e.Attr.SMode()\n\t\teo := out.AddDirLookupEntry(de)\n\t\tif eo == nil {\n\t\t\tfs.v.UpdateReaddirOffset(ctx, Ino(in.NodeId), in.Fh, int(in.Offset)+i)\n\t\t\tbreak\n\t\t}\n\t\tif e.Attr.Full {\n\t\t\tfs.replyEntry(ctx, eo, e)\n\t\t} else {\n\t\t\teo.Ino = uint64(e.Inode)\n\t\t\teo.Generation = 1\n\t\t}\n\t}\n\treturn fuse.Status(err)\n}\n\nvar cancelReleaseDir = make(chan struct{})\n\nfunc (fs *fileSystem) ReleaseDir(in *fuse.ReleaseIn) {\n\tctx := fs.newContext(cancelReleaseDir, &in.InHeader)\n\tdefer releaseContext(ctx)\n\tfs.v.Releasedir(ctx, Ino(in.NodeId), in.Fh)\n}\n\nfunc (fs *fileSystem) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.StatfsOut) (code fuse.Status) {\n\tctx := fs.newContext(cancel, in)\n\tdefer releaseContext(ctx)\n\tst, err := fs.v.StatFS(ctx, Ino(in.NodeId))\n\tif err != 0 {\n\t\treturn fuse.Status(err)\n\t}\n\tout.NameLen = 255\n\tout.Frsize = 4096\n\tout.Bsize = 4096\n\tout.Blocks = st.Total / uint64(out.Bsize)\n\tif out.Blocks < 1 {\n\t\tout.Blocks = 1\n\t}\n\tout.Bavail = st.Avail / uint64(out.Bsize)\n\tout.Bfree = out.Bavail\n\tout.Files = st.Files\n\tout.Ffree = st.Favail\n\treturn 0\n}\n\nfunc (fs *fileSystem) Ioctl(cancel <-chan struct{}, in *fuse.IoctlIn, out *fuse.IoctlOut, bufIn, bufOut []byte) (status fuse.Status) {\n\tctx := fs.newContext(cancel, &in.InHeader)\n\tdefer releaseContext(ctx)\n\terr := fs.v.Ioctl(ctx, Ino(in.NodeId), in.Cmd, in.Arg, bufIn, bufOut)\n\treturn fuse.Status(err)\n}\n\n// Serve starts a server to serve requests from FUSE.\nfunc Serve(v *vfs.VFS, options string, xattrs, ioctl bool) error {\n\tif err := syscall.Setpriority(syscall.PRIO_PROCESS, os.Getpid(), -19); err != nil {\n\t\tlogger.Warnf(\"setpriority: %s\", err)\n\t}\n\terr := grantAccess()\n\tif err != nil {\n\t\tlogger.Debugf(\"grant access to /dev/fuse: %s\", err)\n\t}\n\tensureFuseDev()\n\n\tconf := v.Conf\n\timp := newFileSystem(conf, v)\n\n\tvar opt fuse.MountOptions\n\topt.FsName = \"JuiceFS:\" + conf.Format.Name\n\topt.Name = \"juicefs\"\n\topt.SingleThreaded = false\n\topt.MaxBackground = 50\n\topt.EnableLocks = true\n\topt.EnableSymlinkCaching = conf.FuseOpts.EnableSymlinkCaching\n\topt.EnableAcl = conf.Format.EnableACL\n\topt.DontUmask = conf.Format.EnableACL\n\topt.DisableXAttrs = !xattrs\n\topt.EnableIoctl = ioctl\n\topt.MaxWrite = conf.FuseOpts.MaxWrite\n\topt.MaxReadAhead = 1 << 20\n\topt.DirectMount = true\n\topt.AllowOther = os.Getuid() == 0\n\topt.Timeout = conf.FuseOpts.Timeout\n\topt.EnableReadDirPlusAuto = conf.FuseOpts.EnableReadDirPlusAuto\n\n\tif opt.EnableAcl && conf.NonDefaultPermission {\n\t\tlogger.Warnf(\"it is recommended to turn on 'default-permissions' when enable acl\")\n\t}\n\n\tif opt.EnableAcl && opt.DisableXAttrs {\n\t\tlogger.Infof(\"The format \\\"enable-acl\\\" flag will enable the xattrs feature.\")\n\t\topt.DisableXAttrs = false\n\t}\n\topt.IgnoreSecurityLabels = false\n\n\tfor _, n := range strings.Split(options, \",\") {\n\t\tif n == \"allow_other\" || n == \"allow_root\" {\n\t\t\topt.AllowOther = true\n\t\t} else if n == \"nonempty\" || n == \"ro\" {\n\t\t} else if n == \"debug\" {\n\t\t\topt.Debug = true\n\t\t} else if n == \"writeback_cache\" {\n\t\t\topt.EnableWriteback = true\n\t\t} else if n == \"async_dio\" {\n\t\t\topt.OtherCaps |= fuse.CAP_ASYNC_DIO\n\t\t} else if strings.TrimSpace(n) != \"\" {\n\t\t\topt.Options = append(opt.Options, strings.TrimSpace(n))\n\t\t}\n\t}\n\tif !conf.NonDefaultPermission {\n\t\topt.Options = append(opt.Options, \"default_permissions\")\n\t}\n\tif runtime.GOOS == \"darwin\" {\n\t\topt.Options = append(opt.Options, \"fssubtype=juicefs\")\n\t\topt.Options = append(opt.Options, \"volname=\"+conf.Format.Name)\n\t\topt.Options = append(opt.Options, \"daemon_timeout=60\", \"iosize=65536\", \"novncache\")\n\t}\n\tfssrv, err := fuse.NewServer(imp, conf.Meta.MountPoint, &opt)\n\tif err != nil {\n\t\tif execErr, ok := err.(*exec.Error); ok {\n\t\t\tif pathErr, ok := execErr.Unwrap().(*os.PathError); ok &&\n\t\t\t\tstrings.Contains(pathErr.Path, \"fusermount\") &&\n\t\t\t\tpathErr.Unwrap() == syscall.ENOENT {\n\t\t\t\treturn fmt.Errorf(\"fuse is not installed. Please install it first\")\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"fuse: %s\", err)\n\t}\n\tdefer func() {\n\t\tif runtime.GOOS == \"darwin\" {\n\t\t\t_ = fssrv.Unmount()\n\t\t}\n\t}()\n\n\tif runtime.GOOS == \"linux\" {\n\t\tv.InvalidateEntry = func(parent Ino, name string) syscall.Errno {\n\t\t\treturn syscall.Errno(fssrv.EntryNotify(uint64(parent), name))\n\t\t}\n\t}\n\n\tfsserv = fssrv\n\tfssrv.Serve()\n\treturn nil\n}\n\nfunc GenFuseOpt(conf *vfs.Config, options string, mt int, noxattr, noacl bool, maxWrite int) fuse.MountOptions {\n\tvar opt fuse.MountOptions\n\topt.FsName = \"JuiceFS:\" + conf.Format.Name\n\topt.Name = \"juicefs\"\n\topt.SingleThreaded = mt == 0\n\topt.MaxBackground = 200\n\topt.EnableLocks = true\n\topt.EnableSymlinkCaching = true\n\topt.DisableXAttrs = noxattr\n\topt.EnableAcl = !noacl\n\topt.IgnoreSecurityLabels = false\n\topt.MaxWrite = maxWrite\n\topt.MaxReadAhead = 1 << 20\n\topt.DirectMount = true\n\topt.DontUmask = true\n\topt.Timeout = time.Minute * 15\n\topt.EnableReadDirPlusAuto = true\n\tfor _, n := range strings.Split(options, \",\") {\n\t\t// TODO allow_root\n\t\tif n == \"allow_other\" {\n\t\t\topt.AllowOther = true\n\t\t} else if strings.HasPrefix(n, \"fsname=\") {\n\t\t\topt.FsName = n[len(\"fsname=\"):]\n\t\t} else if n == \"writeback_cache\" {\n\t\t\topt.EnableWriteback = true\n\t\t} else if n == \"debug\" {\n\t\t\topt.Debug = true\n\t\t\tlog.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)\n\t\t} else if strings.TrimSpace(n) != \"\" {\n\t\t\topt.Options = append(opt.Options, strings.TrimSpace(n))\n\t\t}\n\t}\n\topt.Options = append(opt.Options, \"default_permissions\")\n\tif runtime.GOOS == \"darwin\" {\n\t\topt.Options = append(opt.Options, \"fssubtype=juicefs\", \"volname=\"+conf.Format.Name)\n\t\topt.Options = append(opt.Options, \"daemon_timeout=60\", \"iosize=65536\", \"novncache\")\n\t}\n\treturn opt\n}\n\nvar fsserv *fuse.Server\n\nfunc Shutdown() bool {\n\tif fsserv != nil {\n\t\treturn fsserv.Shutdown()\n\t}\n\treturn false\n}\n" }, { "path": "pkg/fuse/fuse_darwin.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fuse\n\nimport (\n\t\"github.com/hanwen/go-fuse/v2/fuse\"\n)\n\nfunc getCreateUmask(mask uint32, defMask uint16) uint16 {\n\tif defMask != 0xFFFF {\n\t\treturn defMask\n\t}\n\treturn 0\n}\n\nfunc getUmask(mask uint32, defMask uint16, isDir bool) uint16 {\n\tif defMask != 0xFFFF {\n\t\treturn defMask\n\t}\n\tif isDir {\n\t\treturn uint16(mask)\n\t}\n\treturn 0\n}\n\nfunc setBlksize(out *fuse.Attr, size uint32) {\n}\n" }, { "path": "pkg/fuse/fuse_linux.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fuse\n\nimport (\n\t\"github.com/hanwen/go-fuse/v2/fuse\"\n)\n\nfunc getCreateUmask(mask uint32, defMask uint16) uint16 {\n\tif defMask != 0xFFFF {\n\t\treturn defMask\n\t}\n\treturn uint16(mask)\n}\n\nfunc getUmask(mask uint32, defMask uint16, isDir bool) uint16 {\n\tif defMask != 0xFFFF {\n\t\treturn defMask\n\t}\n\treturn uint16(mask)\n}\n\nfunc setBlksize(out *fuse.Attr, size uint32) {\n\tout.Blksize = size\n}\n" }, { "path": "pkg/fuse/fuse_test.go", "content": "//go:build linux\n// +build linux\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n//nolint:errcheck\npackage fuse\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"log\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/gofrs/flock\"\n\t\"github.com/google/uuid\"\n\t\"github.com/hanwen/go-fuse/v2/posixtest\"\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/pkg/xattr\"\n)\n\nfunc format(url string) {\n\tm := meta.NewClient(url, nil)\n\tformat := &meta.Format{\n\t\tName: \"test\",\n\t\tUUID: uuid.New().String(),\n\t\tStorage: \"file\",\n\t\tBucket: os.TempDir() + \"/\",\n\t\tBlockSize: 4096,\n\t\tDirStats: true,\n\t}\n\terr := m.Init(format, true)\n\tif err != nil {\n\t\tlog.Fatalf(\"format: %s\", err)\n\t}\n}\n\nfunc mount(url, mp string) {\n\tif err := os.MkdirAll(mp, 0777); err != nil {\n\t\tlog.Fatalf(\"create %s: %s\", mp, err)\n\t}\n\n\tmetaConf := meta.DefaultConf()\n\tmetaConf.MountPoint = mp\n\tm := meta.NewClient(url, metaConf)\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\tlog.Fatalf(\"load setting: %s\", err)\n\t}\n\n\tchunkConf := chunk.Config{\n\t\tBlockSize: format.BlockSize * 1024,\n\t\tCompress: format.Compression,\n\t\tMaxUpload: 20,\n\t\tMaxDownload: 200,\n\t\tBufferSize: 300 << 20,\n\t\tCacheSize: 1024,\n\t\tCacheDir: \"memory\",\n\t}\n\n\tblob, err := object.CreateStorage(strings.ToLower(format.Storage), format.Bucket, format.AccessKey, format.SecretKey, format.SessionToken)\n\tif err != nil {\n\t\tlog.Fatalf(\"object storage: %s\", err)\n\t}\n\tblob = object.WithPrefix(blob, format.Name+\"/\")\n\tstore := chunk.NewCachedStore(blob, chunkConf, nil)\n\n\tm.OnMsg(meta.CompactChunk, meta.MsgCallback(func(args ...interface{}) error {\n\t\tslices := args[0].([]meta.Slice)\n\t\tsliceId := args[1].(uint64)\n\t\treturn vfs.Compact(chunkConf, store, slices, sliceId)\n\t}))\n\n\tconf := &vfs.Config{\n\t\tMeta: metaConf,\n\t\tFormat: *format,\n\t\tChunk: &chunkConf,\n\t\tFuseOpts: &vfs.FuseOptions{},\n\t}\n\n\terr = m.NewSession(true)\n\tif err != nil {\n\t\tlog.Fatalf(\"new session: %s\", err)\n\t}\n\n\tconf.AttrTimeout = time.Second\n\tconf.EntryTimeout = time.Second\n\tconf.DirEntryTimeout = time.Second\n\tconf.HideInternal = true\n\tv := vfs.NewVFS(conf, m, store, nil, nil)\n\terr = Serve(v, \"\", true, true)\n\tif err != nil {\n\t\tlog.Fatalf(\"fuse server err: %s\\n\", err)\n\t}\n\t_ = m.CloseSession()\n}\n\nfunc umount(mp string, force bool) {\n\tvar cmd *exec.Cmd\n\tif _, err := exec.LookPath(\"fusermount\"); err == nil {\n\t\tif force {\n\t\t\tcmd = exec.Command(\"fusermount\", \"-uz\", mp)\n\t\t} else {\n\t\t\tcmd = exec.Command(\"fusermount\", \"-u\", mp)\n\t\t}\n\t} else {\n\t\tif force {\n\t\t\tcmd = exec.Command(\"umount\", \"-l\", mp)\n\t\t} else {\n\t\t\tcmd = exec.Command(\"umount\", mp)\n\t\t}\n\t}\n\n\tout, err := cmd.CombinedOutput()\n\tif err != nil {\n\t\tlog.Print(string(out))\n\t}\n}\n\nfunc waitMountpoint(mp string) chan error {\n\tch := make(chan error, 1)\n\tfor i := 0; i < 20; i++ {\n\t\ttime.Sleep(time.Millisecond * 500)\n\t\tst, err := os.Stat(mp)\n\t\tif err == nil {\n\t\t\tif sys, ok := st.Sys().(*syscall.Stat_t); ok && sys.Ino == 1 {\n\t\t\t\tch <- nil\n\t\t\t\treturn ch\n\t\t\t}\n\t\t}\n\t}\n\tch <- errors.New(\"not ready in 10 seconds\")\n\treturn ch\n}\n\nfunc setUp(metaUrl, mp string) error {\n\tformat(metaUrl)\n\tgo mount(metaUrl, mp)\n\treturn <-waitMountpoint(mp)\n}\n\nfunc cleanup(mp string) {\n\tparent, err := os.Open(mp)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer parent.Close()\n\tnames, err := parent.Readdirnames(-1)\n\tif err != nil {\n\t\treturn\n\t}\n\tfor _, n := range names {\n\t\tos.RemoveAll(filepath.Join(mp, n))\n\t}\n}\n\nfunc StatFS(t *testing.T, mp string) {\n\tvar st syscall.Statfs_t\n\tif err := syscall.Statfs(mp, &st); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif st.Bsize != 4096 {\n\t\tt.Fatalf(\"bsize should be 4096 but got %d \", st.Bsize)\n\t}\n\tif st.Blocks-st.Bavail != 0 {\n\t\tt.Fatalf(\"used blocks should be 0 but got %d\", st.Blocks-st.Bavail)\n\t}\n\tif st.Files-st.Ffree != 0 {\n\t\tt.Fatalf(\"used files should be 0 but got %d\", st.Files)\n\t}\n}\n\nfunc Xattrs(t *testing.T, mp string) {\n\tpath := filepath.Join(mp, \"myfile\")\n\tos.WriteFile(path, []byte(\"\"), 0644)\n\n\tconst prefix = \"user.\"\n\tvar value = []byte(\"test-attr-value\")\n\tif err := xattr.Set(path, prefix+\"test\", value); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif _, err := xattr.List(path); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif data, err := xattr.Get(path, prefix+\"test\"); err != nil {\n\t\tt.Fatal(err)\n\t} else if !bytes.Equal(data, value) {\n\t\tt.Fatalf(\"expect %v bot got %v\", value, data)\n\t}\n\tif err := xattr.Remove(path, prefix+\"test\"); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// One can also specify the flags parameter to be passed to the OS.\n\tif err := xattr.SetWithFlags(path, prefix+\"test\", []byte(\"test-attr-value2\"), xattr.XATTR_CREATE); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc Flock(t *testing.T, mp string) {\n\tpath := filepath.Join(mp, \"go-lock.lock\")\n\tos.WriteFile(path, []byte(\"\"), 0644)\n\n\tfileLock := flock.New(path)\n\tlocked, err := fileLock.TryLock()\n\tif err != nil {\n\t\tt.Fatalf(\"try lock: %s\", err)\n\t}\n\tif locked {\n\t\tfileLock.Unlock()\n\t} else {\n\t\tt.Fatal(\"no lock\")\n\t}\n}\n\nfunc PosixLock(t *testing.T, mp string) {\n\tpath := filepath.Join(mp, \"go-lock.lock\")\n\tf, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer f.Close()\n\tf.WriteString(\"hello\")\n\tif err := f.Sync(); err != nil {\n\t\tt.Fatalf(\"fsync: %s\", err)\n\t}\n\tvar fl syscall.Flock_t\n\tfl.Pid = int32(os.Getpid())\n\tfl.Type = syscall.F_WRLCK\n\tfl.Whence = io.SeekStart\n\terr = syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &fl)\n\tfor err == syscall.EAGAIN {\n\t\terr = syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &fl)\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"lock: %s\", err)\n\t}\n\tif err = syscall.FcntlFlock(f.Fd(), syscall.F_GETLK, &fl); err != nil {\n\t\tt.Fatalf(\"getlk: %s\", err)\n\t}\n\tif int(fl.Pid) != os.Getpid() {\n\t\tt.Fatalf(\"pid: %d != %d\", fl.Pid, os.Getpid())\n\t}\n\tfl.Type = syscall.F_UNLCK\n\tif err = syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &fl); err != nil {\n\t\tt.Fatalf(\"unlock: %s\", err)\n\t}\n}\n\nfunc TestFUSE(t *testing.T) {\n\tf, err := os.CreateTemp(\"\", \"meta\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer os.Remove(f.Name())\n\tmetaUrl := \"sqlite3://\" + f.Name()\n\tmp, err := os.MkdirTemp(\"\", \"mp\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = setUp(metaUrl, mp)\n\tif err != nil {\n\t\tt.Fatalf(\"setup: %s\", err)\n\t}\n\tdefer umount(mp, true)\n\n\tt.Run(\"StatFS\", func(t *testing.T) {\n\t\tStatFS(t, mp)\n\t})\n\tdelete(posixtest.All, \"FdLeak\")\n\tdelete(posixtest.All, \"FcntlFlockLocksFile\") // FIXME: check gofuse in posixtest/posixtest_test.go\n\tposixtest.All[\"Xattrs\"] = Xattrs\n\tposixtest.All[\"Flock\"] = Flock\n\tposixtest.All[\"POSIXLock\"] = PosixLock\n\tfor c, f := range posixtest.All {\n\t\tcleanup(mp)\n\t\tt.Run(c, func(t *testing.T) {\n\t\t\tf(t, mp)\n\t\t})\n\t}\n}\n" }, { "path": "pkg/fuse/gidcache.go", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fuse\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype cItem struct {\n\tgids []uint32\n\texpire time.Time\n}\n\ntype gidCache struct {\n\tsync.Mutex\n\tgroups map[uint32]*cItem\n\tcacheto time.Duration\n}\n\nfunc newGidCache(cacheto time.Duration) *gidCache {\n\tg := &gidCache{\n\t\tgroups: make(map[uint32]*cItem),\n\t\tcacheto: cacheto,\n\t}\n\tgo g.cleanup()\n\treturn g\n}\n\nfunc (g *gidCache) cleanup() {\n\tfor {\n\t\tg.Lock()\n\t\tnow := time.Now()\n\t\tfor k, gs := range g.groups {\n\t\t\tif gs.expire.Before(now) {\n\t\t\t\tdelete(g.groups, k)\n\t\t\t}\n\t\t}\n\t\tg.Unlock()\n\t\ttime.Sleep(time.Second * 10)\n\t}\n}\n\nfunc findProcessGroups(pid, gid uint32) []uint32 {\n\tif runtime.GOOS == \"darwin\" {\n\t\treturn []uint32{gid}\n\t}\n\tpath := fmt.Sprintf(\"/proc/%d/status\", pid)\n\tf, err := os.Open(path)\n\tif err != nil {\n\t\treturn []uint32{gid}\n\t}\n\tdefer f.Close()\n\tbuf, err := io.ReadAll(f)\n\tif err != nil {\n\t\treturn []uint32{gid}\n\t}\n\n\tp := bytes.Index(buf, []byte(\"Groups:\"))\n\tif p < 0 {\n\t\treturn []uint32{gid}\n\t}\n\tbuf = buf[p+7:]\n\tlast := bytes.IndexByte(buf, '\\n')\n\tif last >= 0 {\n\t\tbuf = buf[:last]\n\t}\n\tparts := bytes.Split(buf, []byte(\" \"))\n\tgids := []uint32{gid}\n\tfor _, p := range parts {\n\t\tg, err := strconv.Atoi(string(bytes.TrimSpace(p)))\n\t\tif err == nil && uint32(g) != gid {\n\t\t\tgids = append(gids, uint32(g))\n\t\t}\n\t}\n\treturn gids\n}\n\nfunc (g *gidCache) get(pid, gid uint32) []uint32 {\n\tif g.cacheto == 0 || pid == 0 || gid == 0 {\n\t\treturn []uint32{gid}\n\t}\n\tnow := time.Now()\n\tg.Lock()\n\tdefer g.Unlock()\n\tit := g.groups[pid]\n\tif it != nil && it.expire.Before(now) {\n\t\tit = nil\n\t}\n\tif it == nil {\n\t\tit = &cItem{findProcessGroups(pid, gid), now.Add(g.cacheto)}\n\t\tg.groups[pid] = it\n\t}\n\treturn it.gids\n}\n" }, { "path": "pkg/fuse/utils.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage fuse\n\nimport (\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\n\t\"github.com/hanwen/go-fuse/v2/fuse\"\n)\n\nfunc attrToStat(inode Ino, attr *Attr, out *fuse.Attr) {\n\tout.Ino = uint64(inode)\n\tout.Uid = attr.Uid\n\tout.Gid = attr.Gid\n\tout.Mode = attr.SMode()\n\tout.Nlink = attr.Nlink\n\tout.Atime = uint64(attr.Atime)\n\tout.Atimensec = attr.Atimensec\n\tout.Mtime = uint64(attr.Mtime)\n\tout.Mtimensec = attr.Mtimensec\n\tout.Ctime = uint64(attr.Ctime)\n\tout.Ctimensec = attr.Ctimensec\n\n\tvar size, blocks uint64\n\tswitch attr.Typ {\n\tcase meta.TypeDirectory:\n\t\tfallthrough\n\tcase meta.TypeSymlink:\n\t\tfallthrough\n\tcase meta.TypeFile:\n\t\tsize = attr.Length\n\t\tblocks = (size + 511) / 512\n\tcase meta.TypeBlockDev:\n\t\tfallthrough\n\tcase meta.TypeCharDev:\n\t\tout.Rdev = attr.Rdev\n\t}\n\tout.Size = size\n\tout.Blocks = blocks\n\tsetBlksize(out, 0x10000)\n}\n" }, { "path": "pkg/gateway/gateway.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage gateway\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\tjsoniter \"github.com/json-iterator/go\"\n\t\"github.com/minio/minio-go/v7/pkg/tags\"\n\t\"github.com/minio/minio/pkg/bucket/policy\"\n\t\"github.com/minio/minio/pkg/madmin\"\n\t\"golang.org/x/sync/errgroup\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/minio/minio-go/v7/pkg/s3utils\"\n\tminio \"github.com/minio/minio/cmd\"\n\txhttp \"github.com/minio/minio/cmd/http\"\n\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n)\n\nconst (\n\tsep = \"/\"\n\tmetaBucket = \".sys\"\n\tsubDirPrefix = 3 // 16^3=4096 slots\n)\n\nvar mctx meta.Context\nvar logger = utils.GetLogger(\"juicefs\")\n\ntype Config struct {\n\tMultiBucket bool\n\tKeepEtag bool\n\tUmask uint16\n\tObjTag bool\n\tObjMeta bool\n\tHeadDir bool\n\tHideDir bool\n\tReadOnly bool\n}\n\nfunc NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) (minio.ObjectLayer, error) {\n\tmctx = meta.NewContext(uint32(os.Getpid()), uint32(utils.GetCurrentUID()), []uint32{uint32(utils.GetCurrentGID())})\n\tjfsObj := &jfsObjects{fs: jfs, conf: conf, listPool: minio.NewTreeWalkPool(time.Second * 10), gConf: gConf, nsMutex: minio.NewNSLock(false)}\n\tgo jfsObj.cleanup()\n\treturn jfsObj, nil\n}\n\ntype jfsObjects struct {\n\tconf *vfs.Config\n\tfs *fs.FileSystem\n\tlistPool *minio.TreeWalkPool\n\tnsMutex *minio.NsLockMap\n\tgConf *Config\n}\n\nfunc (n *jfsObjects) PutObjectMetadata(ctx context.Context, s string, s2 string, options minio.ObjectOptions) (minio.ObjectInfo, error) {\n\treturn minio.ObjectInfo{}, minio.NotImplemented{}\n}\n\nfunc (n *jfsObjects) NSScanner(ctx context.Context, bf *minio.BloomFilter, updates chan<- madmin.DataUsageInfo) error {\n\treturn nil\n}\n\nfunc (n *jfsObjects) IsCompressionSupported() bool {\n\treturn false\n}\n\nfunc (n *jfsObjects) IsEncryptionSupported() bool {\n\treturn false\n}\n\n// IsReady returns whether the layer is ready to take requests.\nfunc (n *jfsObjects) IsReady(_ context.Context) bool {\n\treturn true\n}\n\nfunc (n *jfsObjects) Shutdown(ctx context.Context) error {\n\treturn n.fs.Close()\n}\n\nfunc (n *jfsObjects) StorageInfo(ctx context.Context) (info minio.StorageInfo, errors []error) {\n\tsinfo := minio.StorageInfo{}\n\tsinfo.Backend.Type = madmin.FS\n\treturn sinfo, nil\n}\n\nfunc jfsToObjectErr(ctx context.Context, err error, params ...string) error {\n\tif err == nil {\n\t\treturn nil\n\t}\n\tbucket := \"\"\n\tobject := \"\"\n\tuploadID := \"\"\n\tswitch len(params) {\n\tcase 3:\n\t\tuploadID = params[2]\n\t\tfallthrough\n\tcase 2:\n\t\tobject = params[1]\n\t\tfallthrough\n\tcase 1:\n\t\tbucket = params[0]\n\t}\n\n\tif eno, ok := err.(syscall.Errno); !ok {\n\t\tlogger.Errorf(\"error: %s bucket: %s, object: %s, uploadID: %s\", err, bucket, object, uploadID)\n\t\treturn err\n\t} else if eno == 0 {\n\t\treturn nil\n\t}\n\n\tswitch {\n\tcase fs.IsNotExist(err):\n\t\tif uploadID != \"\" {\n\t\t\treturn minio.InvalidUploadID{\n\t\t\t\tUploadID: uploadID,\n\t\t\t}\n\t\t}\n\t\tif object != \"\" {\n\t\t\treturn minio.ObjectNotFound{Bucket: bucket, Object: object}\n\t\t}\n\t\treturn minio.BucketNotFound{Bucket: bucket}\n\tcase fs.IsExist(err):\n\t\tif object != \"\" {\n\t\t\treturn minio.PrefixAccessDenied{Bucket: bucket, Object: object}\n\t\t}\n\t\treturn minio.BucketAlreadyOwnedByYou{Bucket: bucket}\n\tcase fs.IsNotEmpty(err):\n\t\tif object != \"\" {\n\t\t\treturn minio.PrefixAccessDenied{Bucket: bucket, Object: object}\n\t\t}\n\t\treturn minio.BucketNotEmpty{Bucket: bucket}\n\tdefault:\n\t\tlogger.Errorf(\"other error: %s bucket: %s, object: %s, uploadID: %s\", err, bucket, object, uploadID)\n\t\treturn err\n\t}\n}\n\n// isValidBucketName verifies whether a bucket name is valid.\nfunc (n *jfsObjects) isValidBucketName(bucket string) error {\n\tif strings.HasPrefix(bucket, minio.MinioMetaBucket) {\n\t\treturn nil\n\t}\n\tif s3utils.CheckValidBucketNameStrict(bucket) != nil {\n\t\treturn minio.BucketNameInvalid{Bucket: bucket}\n\t}\n\tif !n.gConf.MultiBucket && bucket != n.conf.Format.Name {\n\t\treturn minio.BucketNotFound{Bucket: bucket}\n\t}\n\treturn nil\n}\n\nfunc (n *jfsObjects) path(p ...string) string {\n\tif !n.gConf.MultiBucket && len(p) > 0 && p[0] == n.conf.Format.Name {\n\t\tp = p[1:]\n\t}\n\treturn sep + minio.PathJoin(p...)\n}\n\nfunc (n *jfsObjects) tpath(p ...string) string {\n\treturn sep + metaBucket + n.path(p...)\n}\n\nfunc (n *jfsObjects) upath(bucket, uploadID string) string {\n\treturn n.tpath(bucket, \"uploads\", uploadID[:subDirPrefix], uploadID)\n}\n\nfunc (n *jfsObjects) ppath(bucket, uploadID, part string) string {\n\treturn n.tpath(bucket, \"uploads\", uploadID[:subDirPrefix], uploadID, part)\n}\n\nfunc (n *jfsObjects) ppathFlat(bucket, uploadID, part string) string { // compatible with tmp files uploaded by old versions(<1.2)\n\treturn n.tpath(bucket, \"uploads\", uploadID, part)\n}\n\nfunc (n *jfsObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {\n\tif err := n.isValidBucketName(bucket); err != nil {\n\t\treturn err\n\t}\n\tif !n.gConf.MultiBucket {\n\t\treturn minio.BucketNotEmpty{Bucket: bucket}\n\t}\n\tif eno := n.fs.Delete(mctx, n.path(minio.MinioMetaBucket, minio.BucketMetaPrefix, bucket, minio.BucketMetadataFile)); eno != 0 {\n\t\tlogger.Errorf(\"delete bucket metadata: %s\", eno)\n\t}\n\t_ = n.fs.Delete(mctx, n.path(minio.MinioMetaBucket, minio.BucketMetaPrefix, bucket))\n\teno := n.fs.Delete(mctx, n.path(bucket))\n\treturn jfsToObjectErr(ctx, eno, bucket)\n}\n\nfunc (n *jfsObjects) MakeBucketWithLocation(ctx context.Context, bucket string, options minio.BucketOptions) error {\n\tif bucket != minio.MinioMetaBucket {\n\t\tif err := n.isValidBucketName(bucket); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !n.gConf.MultiBucket {\n\t\t\treturn nil\n\t\t}\n\t}\n\teno := n.fs.Mkdir(mctx, n.path(bucket), 0777, n.gConf.Umask)\n\tif eno == 0 {\n\t\tmetadata := minio.NewBucketMetadata(bucket)\n\t\tif err := metadata.Save(ctx, n); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn jfsToObjectErr(ctx, eno, bucket)\n}\n\nfunc (n *jfsObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {\n\tif err := n.isValidBucketName(bucket); err != nil {\n\t\treturn bi, err\n\t}\n\tfi, eno := n.fs.Stat(mctx, n.path(bucket))\n\tif eno == 0 {\n\t\tbi = minio.BucketInfo{\n\t\t\tName: bucket,\n\t\t\tCreated: time.Unix(fi.Atime()/1000, 0),\n\t\t}\n\t}\n\treturn bi, jfsToObjectErr(ctx, eno, bucket)\n}\n\n// Ignores all reserved bucket names or invalid bucket names.\nfunc isReservedOrInvalidBucket(bucketEntry string, strict bool) bool {\n\tif err := s3utils.CheckValidBucketName(bucketEntry); err != nil {\n\t\treturn true\n\t}\n\treturn bucketEntry == metaBucket\n}\n\nfunc (n *jfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {\n\tif !n.gConf.MultiBucket {\n\t\tfi, eno := n.fs.Stat(mctx, \"/\")\n\t\tif eno != 0 {\n\t\t\treturn nil, jfsToObjectErr(ctx, eno)\n\t\t}\n\t\tbuckets = []minio.BucketInfo{{\n\t\t\tName: n.conf.Format.Name,\n\t\t\tCreated: time.Unix(fi.Atime()/1000, 0),\n\t\t}}\n\t\treturn buckets, nil\n\t}\n\tf, eno := n.fs.Open(mctx, sep, 0)\n\tif eno != 0 {\n\t\treturn nil, jfsToObjectErr(ctx, eno)\n\t}\n\tdefer f.Close(mctx)\n\tentries, eno := f.Readdir(mctx, 10000)\n\tif eno != 0 {\n\t\treturn nil, jfsToObjectErr(ctx, eno)\n\t}\n\n\tfor _, entry := range entries {\n\t\t// Ignore all reserved bucket names and invalid bucket names.\n\t\tif isReservedOrInvalidBucket(entry.Name(), false) || n.isValidBucketName(entry.Name()) != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif entry.IsDir() {\n\t\t\tbuckets = append(buckets, minio.BucketInfo{\n\t\t\t\tName: entry.Name(),\n\t\t\t\tCreated: time.Unix(entry.(*fs.FileStat).Atime()/1000, 0),\n\t\t\t})\n\t\t}\n\t}\n\n\t// Sort bucket infos by bucket name.\n\tsort.Slice(buckets, func(i, j int) bool {\n\t\treturn buckets[i].Name < buckets[j].Name\n\t})\n\treturn buckets, nil\n}\n\nfunc (n *jfsObjects) isLeafDir(bucket, leafPath string) bool {\n\treturn false\n}\n\nfunc (n *jfsObjects) isLeaf(bucket, leafPath string) bool {\n\treturn !strings.HasSuffix(leafPath, \"/\")\n}\n\nfunc (n *jfsObjects) listDirFactory() minio.ListDirFunc {\n\treturn func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []*minio.Entry, delayIsLeaf bool) {\n\t\tf, eno := n.fs.Open(mctx, n.path(bucket, prefixDir), 0)\n\t\tif eno != 0 {\n\t\t\treturn fs.IsNotExist(eno), nil, false\n\t\t}\n\t\tdefer f.Close(mctx)\n\t\tif !n.gConf.HideDir {\n\t\t\tif fi, _ := f.Stat(); fi.(*fs.FileStat).Atime() == 0 && prefixEntry == \"\" {\n\t\t\t\tentries = append(entries, &minio.Entry{Name: \"\"})\n\t\t\t}\n\t\t}\n\n\t\tfis, eno := f.Readdir(mctx, 0)\n\t\tif eno != 0 {\n\t\t\treturn\n\t\t}\n\t\troot := n.path(bucket, prefixDir) == \"/\"\n\t\tfor _, fi := range fis {\n\t\t\tif root && (fi.Name() == metaBucket || fi.Name() == minio.MinioMetaBucket) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif stat, ok := fi.(*fs.FileStat); ok && stat.IsSymlink() {\n\t\t\t\tvar err syscall.Errno\n\t\t\t\tp := n.path(bucket, prefixDir, fi.Name())\n\t\t\t\tif fi, err = n.fs.Stat(mctx, p); err != 0 {\n\t\t\t\t\tlogger.Errorf(\"stat %s: %s\", p, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tentry := &minio.Entry{Name: fi.Name(), Info: fi}\n\n\t\t\tif fi.IsDir() {\n\t\t\t\tentry.Name += sep\n\t\t\t}\n\t\t\tentries = append(entries, entry)\n\t\t}\n\t\tif len(entries) == 0 {\n\t\t\treturn true, nil, false\n\t\t}\n\t\tentries, delayIsLeaf = minio.FilterListEntries(bucket, prefixDir, entries, prefixEntry, n.isLeaf)\n\t\treturn false, entries, delayIsLeaf\n\t}\n}\n\nfunc (n *jfsObjects) checkBucket(ctx context.Context, bucket string) error {\n\tif err := n.isValidBucketName(bucket); err != nil {\n\t\treturn err\n\t}\n\tbucketPath := n.path(bucket)\n\tif bucketPath != \"/\" { // no need to stat \"/\" in every request\n\t\tif _, eno := n.fs.Stat(mctx, bucketPath); eno != 0 {\n\t\t\treturn jfsToObjectErr(ctx, eno, bucket)\n\t\t}\n\t}\n\treturn nil\n}\n\n// ListObjects lists all blobs in JFS bucket filtered by prefix.\nfunc (n *jfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {\n\tif err := n.checkBucket(ctx, bucket); err != nil {\n\t\treturn loi, err\n\t}\n\tgetObjectInfo := func(ctx context.Context, bucket, object string, fi_ any) (obj minio.ObjectInfo, err error) {\n\t\tvar eno syscall.Errno\n\t\tvar info *minio.ObjectInfo\n\t\tif fi_ == nil {\n\t\t\tvar fi *fs.FileStat\n\t\t\tfi, eno = n.fs.Stat(mctx, n.path(bucket, object))\n\t\t\tif eno == 0 {\n\t\t\t\tsize := fi.Size()\n\t\t\t\tif fi.IsDir() {\n\t\t\t\t\tsize = 0\n\t\t\t\t}\n\t\t\t\tinfo = &minio.ObjectInfo{\n\t\t\t\t\tBucket: bucket,\n\t\t\t\t\tModTime: fi.ModTime(),\n\t\t\t\t\tSize: size,\n\t\t\t\t\tIsDir: fi.IsDir(),\n\t\t\t\t\tAccTime: fi.ModTime(),\n\t\t\t\t\tIsLatest: true,\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// replace links to external file systems with empty files\n\t\t\tif errors.Is(eno, syscall.ENOTSUP) {\n\t\t\t\tnow := time.Now()\n\t\t\t\tinfo = &minio.ObjectInfo{\n\t\t\t\t\tBucket: bucket,\n\t\t\t\t\tModTime: now,\n\t\t\t\t\tSize: 0,\n\t\t\t\t\tIsDir: false,\n\t\t\t\t\tAccTime: now,\n\t\t\t\t\tIsLatest: true,\n\t\t\t\t}\n\t\t\t\teno = 0\n\t\t\t}\n\t\t} else {\n\t\t\tfi := fi_.(*fs.FileStat)\n\t\t\tinfo = &minio.ObjectInfo{\n\t\t\t\tBucket: bucket,\n\t\t\t\tName: fi.Name(),\n\t\t\t\tModTime: fi.ModTime(),\n\t\t\t\tSize: fi.Size(),\n\t\t\t\tIsDir: fi.IsDir(),\n\t\t\t\tAccTime: fi.ModTime(),\n\t\t\t\tIsLatest: true,\n\t\t\t}\n\t\t\tif fi.IsDir() {\n\t\t\t\tinfo.Size = 0\n\t\t\t}\n\t\t}\n\n\t\tif info == nil {\n\t\t\treturn obj, jfsToObjectErr(ctx, eno, bucket, object)\n\t\t}\n\t\tinfo.Name = object\n\t\tif n.gConf.KeepEtag && !strings.HasSuffix(object, sep) {\n\t\t\tetag, _ := n.fs.GetXattr(mctx, n.path(bucket, object), s3Etag)\n\t\t\tinfo.ETag = string(etag)\n\t\t}\n\t\treturn *info, jfsToObjectErr(ctx, eno, bucket, object)\n\t}\n\n\tif maxKeys == 0 {\n\t\tmaxKeys = -1 // list as many objects as possible\n\t}\n\treturn minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), n.isLeaf, n.isLeafDir, getObjectInfo, getObjectInfo)\n}\n\n// ListObjectsV2 lists all blobs in JFS bucket filtered by prefix\nfunc (n *jfsObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,\n\tfetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {\n\tif err := n.isValidBucketName(bucket); err != nil {\n\t\treturn minio.ListObjectsV2Info{}, err\n\t}\n\t// fetchOwner is not supported and unused.\n\tmarker := continuationToken\n\tif marker == \"\" {\n\t\tmarker = startAfter\n\t}\n\tresultV1, err := n.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)\n\tif err == nil {\n\t\tloi = minio.ListObjectsV2Info{\n\t\t\tObjects: resultV1.Objects,\n\t\t\tPrefixes: resultV1.Prefixes,\n\t\t\tContinuationToken: continuationToken,\n\t\t\tNextContinuationToken: resultV1.NextMarker,\n\t\t\tIsTruncated: resultV1.IsTruncated,\n\t\t}\n\t}\n\treturn loi, err\n}\n\nfunc (n *jfsObjects) setFileAtime(p string, atime int64) {\n\tif f, eno := n.fs.Open(mctx, p, 0); eno == 0 {\n\t\tdefer f.Close(mctx)\n\t\tif eno := f.Utime(mctx, atime, -1); eno != 0 {\n\t\t\tlogger.Warnf(\"set atime of %s: %s\", p, eno)\n\t\t}\n\t} else if eno != syscall.ENOENT {\n\t\tlogger.Warnf(\"open %s: %s\", p, eno)\n\t}\n}\n\nfunc (n *jfsObjects) DeleteObject(ctx context.Context, bucket, object string, options minio.ObjectOptions) (info minio.ObjectInfo, err error) {\n\tif err = n.checkBucket(ctx, bucket); err != nil {\n\t\treturn\n\t}\n\terr = n.delObj(bucket, object)\n\tinfo.Bucket = bucket\n\tinfo.Name = object\n\treturn info, jfsToObjectErr(ctx, err, bucket, object)\n}\n\nfunc (n *jfsObjects) delObj(bucket string, object string) error {\n\tp := path.Clean(n.path(bucket, object))\n\troot := n.path(bucket)\n\tif strings.HasSuffix(object, sep) {\n\t\t// reset atime\n\t\tn.setFileAtime(p, time.Now().Unix())\n\t}\n\tvar err error\n\tfor p != root {\n\t\tif eno := n.fs.Delete(mctx, p); eno != 0 {\n\t\t\tif fs.IsNotEmpty(eno) || fs.IsNotExist(eno) {\n\t\t\t\terr = nil\n\t\t\t} else {\n\t\t\t\terr = eno\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tp = path.Dir(p)\n\t\tif fi, _ := n.fs.Stat(mctx, p); fi == nil || fi.Atime() == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (n *jfsObjects) DeleteObjects(ctx context.Context, bucket string, objects []minio.ObjectToDelete, options minio.ObjectOptions) (objs []minio.DeletedObject, errs []error) {\n\tobjs = make([]minio.DeletedObject, len(objects))\n\terrs = make([]error, len(objects))\n\tif err := n.checkBucket(ctx, bucket); err != nil {\n\t\tfor idx := range objects {\n\t\t\terrs[idx] = minio.BucketNotFound{Bucket: bucket}\n\t\t}\n\t\treturn\n\t}\n\tdelMap := make(map[string][]int)\n\tfor idx, o := range objects {\n\t\tp := path.Dir(path.Clean(n.path(bucket, o.ObjectName)))\n\t\tdelMap[p] = append(delMap[p], idx)\n\t}\n\tvar g errgroup.Group\n\tg.SetLimit(runtime.NumCPU())\n\tfor ppath := range delMap {\n\t\tppath := ppath\n\t\tidxs := delMap[ppath]\n\t\tps := make([]string, len(idxs))\n\t\tfor i, idx := range idxs {\n\t\t\tps[i] = n.path(bucket, objects[idx].ObjectName)\n\t\t}\n\t\tg.Go(func() error {\n\t\t\t// will ignore dir\n\t\t\terr := n.fs.BatchDeleteEntries(mctx, ppath, ps)\n\t\t\tif err != 0 {\n\t\t\t\tfor _, idx := range idxs {\n\t\t\t\t\terrs[idx] = jfsToObjectErr(ctx, err, bucket, objects[idx].ObjectName)\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif e := n.delObj(bucket, ppath); e != nil {\n\t\t\t\tfor _, idx := range idxs {\n\t\t\t\t\terrs[idx] = e\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t}\n\t_ = g.Wait()\n\treturn\n}\n\ntype fReader struct {\n\t*fs.File\n}\n\nfunc (f *fReader) Read(b []byte) (int, error) {\n\treturn f.File.Read(mctx, b)\n}\n\nfunc (n *jfsObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {\n\tobjInfo, err := n.GetObjectInfo(ctx, bucket, object, opts)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar startOffset, length int64\n\tstartOffset, length, err = rs.GetOffsetLength(objInfo.Size)\n\tif err != nil {\n\t\treturn\n\t}\n\tf, eno := n.fs.Open(mctx, n.path(bucket, object), vfs.MODE_MASK_R)\n\tif eno != 0 {\n\t\treturn nil, jfsToObjectErr(ctx, eno, bucket, object)\n\t}\n\t_, _ = f.Seek(mctx, startOffset, 0)\n\tr := &io.LimitedReader{R: &fReader{f}, N: length}\n\tcloser := func() { _ = f.Close(mctx) }\n\treturn minio.NewGetObjectReaderFromReader(r, objInfo, opts, closer)\n}\n\nfunc (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (info minio.ObjectInfo, err error) {\n\tif err = n.checkBucket(ctx, srcBucket); err != nil {\n\t\treturn\n\t}\n\tif err = n.checkBucket(ctx, dstBucket); err != nil {\n\t\treturn\n\t}\n\tdst := n.path(dstBucket, dstObject)\n\tsrc := n.path(srcBucket, srcObject)\n\n\tif minio.IsStringEqual(src, dst) {\n\t\t// if we copy the same object for set metadata\n\t\terr = n.setObjMeta(dst, srcInfo.UserDefined)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"set object metadata error, path: %s error %s\", dst, err)\n\t\t}\n\t\treturn n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{})\n\t}\n\tuuid := minio.MustGetUUID()\n\ttmp := n.tpath(dstBucket, \"tmp\", uuid[:subDirPrefix], uuid)\n\tf, eno := n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)\n\tif eno == syscall.ENOENT {\n\t\t_ = n.mkdirAll(ctx, path.Dir(tmp))\n\t\tf, eno = n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)\n\t}\n\tif eno != 0 {\n\t\tlogger.Errorf(\"create %s: %s\", tmp, eno)\n\t\treturn\n\t}\n\tdefer func() {\n\t\t_ = f.Close(mctx)\n\t\tif err != nil {\n\t\t\t_ = n.fs.Delete(mctx, tmp)\n\t\t}\n\t}()\n\n\t_, eno = n.fs.CopyFileRange(mctx, src, 0, tmp, 0, 1<<63)\n\tif eno != 0 {\n\t\terr = jfsToObjectErr(ctx, eno, srcBucket, srcObject)\n\t\tlogger.Errorf(\"copy %s to %s: %s\", src, tmp, err)\n\t\treturn\n\t}\n\n\tvar etag []byte\n\tif n.gConf.KeepEtag {\n\t\tetag, _ = n.fs.GetXattr(mctx, src, s3Etag)\n\t\tif len(etag) != 0 {\n\t\t\teno = n.fs.SetXattr(mctx, tmp, s3Etag, etag, 0)\n\t\t\tif eno != 0 {\n\t\t\t\tlogger.Warnf(\"set xattr error, path: %s,xattr: %s,value: %s,flags: %d\", tmp, s3Etag, etag, 0)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar tagStr string\n\tif n.gConf.ObjTag && srcInfo.UserDefined != nil {\n\t\tif tagStr = srcInfo.UserDefined[xhttp.AmzObjectTagging]; tagStr != \"\" {\n\t\t\tif eno := n.fs.SetXattr(mctx, tmp, s3Tags, []byte(tagStr), 0); eno != 0 {\n\t\t\t\tlogger.Errorf(\"set object tags error, path: %s, value: %s error %s\", tmp, tagStr, eno)\n\t\t\t}\n\t\t}\n\t}\n\terr = n.setObjMeta(tmp, srcInfo.UserDefined)\n\tif err != nil {\n\t\tlogger.Errorf(\"set object metadata error, path: %s error %s\", dst, err)\n\t}\n\n\teno = n.fs.Rename(mctx, tmp, dst, 0)\n\tif eno == syscall.ENOENT {\n\t\tif err = n.mkdirAll(ctx, path.Dir(dst)); err != nil {\n\t\t\tlogger.Errorf(\"mkdirAll %s: %s\", path.Dir(dst), err)\n\t\t\terr = jfsToObjectErr(ctx, err, dstBucket, dstObject)\n\t\t\treturn\n\t\t}\n\t\teno = n.fs.Rename(mctx, tmp, dst, 0)\n\t}\n\tif eno != 0 {\n\t\terr = jfsToObjectErr(ctx, eno, srcBucket, srcObject)\n\t\tlogger.Errorf(\"rename %s to %s: %s\", tmp, dst, err)\n\t\treturn\n\t}\n\tfi, eno := n.fs.Stat(mctx, dst)\n\tif eno != 0 {\n\t\terr = jfsToObjectErr(ctx, eno, dstBucket, dstObject)\n\t\treturn\n\t}\n\n\treturn minio.ObjectInfo{\n\t\tBucket: dstBucket,\n\t\tName: dstObject,\n\t\tETag: string(etag),\n\t\tModTime: fi.ModTime(),\n\t\tSize: fi.Size(),\n\t\tIsDir: fi.IsDir(),\n\t\tAccTime: fi.ModTime(),\n\t\tUserTags: tagStr,\n\t\tUserDefined: minio.CleanMetadata(srcInfo.UserDefined),\n\t\tIsLatest: true,\n\t}, nil\n}\n\nvar buffPool = sync.Pool{\n\tNew: func() interface{} {\n\t\tbuf := make([]byte, 1<<17)\n\t\treturn &buf\n\t},\n}\n\nfunc (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {\n\tif err = n.checkBucket(ctx, bucket); err != nil {\n\t\treturn\n\t}\n\tfi, eno := n.fs.Stat(mctx, n.path(bucket, object))\n\tif eno != 0 {\n\t\terr = jfsToObjectErr(ctx, eno, bucket, object)\n\t\treturn\n\t}\n\t// put /dir1/key1; head /dir1 return 404; head /dir1/ return 404; head /dir1/key1 return 200\n\t// put /dir1/key1/; head /dir1/key1 return 404; head /dir1/key1/ return 200\n\tvar isObject bool\n\tif strings.HasSuffix(object, sep) && fi.IsDir() && fi.Atime() == 0 {\n\t\tisObject = true\n\t} else if !strings.HasSuffix(object, sep) && !fi.IsDir() {\n\t\tisObject = true\n\t}\n\tif !n.gConf.HeadDir && !isObject {\n\t\terr = jfsToObjectErr(ctx, syscall.ENOENT, bucket, object)\n\t\treturn\n\t}\n\tvar etag []byte\n\tif n.gConf.KeepEtag && !fi.IsDir() {\n\t\tetag, _ = n.fs.GetXattr(mctx, n.path(bucket, object), s3Etag)\n\t}\n\tsize := fi.Size()\n\tif fi.IsDir() {\n\t\tsize = 0\n\t}\n\t// key1=value1&key2=value2\n\tvar tagStr []byte\n\tif n.gConf.ObjTag {\n\t\tvar errno syscall.Errno\n\t\tif tagStr, errno = n.fs.GetXattr(mctx, n.path(bucket, object), s3Tags); errno != 0 && errno != meta.ENOATTR {\n\t\t\treturn minio.ObjectInfo{}, errno\n\t\t}\n\t}\n\tobjMeta, err := n.getObjMeta(n.path(bucket, object))\n\tif err != nil {\n\t\treturn minio.ObjectInfo{}, err\n\t}\n\tif opts.UserDefined == nil {\n\t\topts.UserDefined = make(map[string]string)\n\t}\n\tfor k, v := range objMeta {\n\t\topts.UserDefined[k] = v\n\t}\n\tcontentType := utils.GuessMimeType(object)\n\tif c, exist := objMeta[\"content-type\"]; exist && len(c) > 0 {\n\t\tcontentType = c\n\t}\n\treturn minio.ObjectInfo{\n\t\tBucket: bucket,\n\t\tName: object,\n\t\tModTime: fi.ModTime(),\n\t\tSize: size,\n\t\tIsDir: fi.IsDir(),\n\t\tAccTime: fi.ModTime(),\n\t\tETag: string(etag),\n\t\tContentType: contentType,\n\t\tUserTags: string(tagStr),\n\t\tUserDefined: minio.CleanMetadata(opts.UserDefined),\n\t\tIsLatest: true,\n\t}, nil\n}\n\nfunc (n *jfsObjects) mkdirAll(ctx context.Context, p string) error {\n\tif fi, eno := n.fs.Stat(mctx, p); eno == 0 {\n\t\tif !fi.IsDir() {\n\t\t\treturn fmt.Errorf(\"%s is not directory\", p)\n\t\t}\n\t\treturn nil\n\t}\n\teno := n.fs.Mkdir(mctx, p, 0777, n.gConf.Umask)\n\tif eno != 0 && fs.IsNotExist(eno) {\n\t\tif err := n.mkdirAll(ctx, path.Dir(p)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\teno = n.fs.Mkdir(mctx, p, 0777, n.gConf.Umask)\n\t}\n\tif eno != 0 && fs.IsExist(eno) {\n\t\teno = 0\n\t}\n\tif eno == 0 {\n\t\treturn nil\n\t}\n\treturn eno\n}\n\nfunc (n *jfsObjects) putObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions, applyObjTaggingFunc func(tmpName string)) (err error) {\n\tuuid := minio.MustGetUUID()\n\ttmpname := n.tpath(bucket, \"tmp\", uuid[:subDirPrefix], uuid)\n\tf, eno := n.fs.Create(mctx, tmpname, 0666, n.gConf.Umask)\n\tif eno == syscall.ENOENT {\n\t\t_ = n.mkdirAll(ctx, path.Dir(tmpname))\n\t\tf, eno = n.fs.Create(mctx, tmpname, 0666, n.gConf.Umask)\n\t}\n\tif eno != 0 {\n\t\tlogger.Errorf(\"create %s: %s\", tmpname, eno)\n\t\terr = eno\n\t\treturn\n\t}\n\tdefer func() {\n\t\tif err != nil {\n\t\t\t_ = n.fs.Delete(mctx, tmpname)\n\t\t}\n\t}()\n\tvar buf = buffPool.Get().(*[]byte)\n\tdefer buffPool.Put(buf)\n\tfor {\n\t\tvar n int\n\t\tn, err = io.ReadFull(r, *buf)\n\t\tif n == 0 {\n\t\t\tif err == io.EOF {\n\t\t\t\terr = nil\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\t_, eno := f.Write(mctx, (*buf)[:n])\n\t\tif eno != 0 {\n\t\t\terr = eno\n\t\t\tbreak\n\t\t}\n\t}\n\tif err == nil {\n\t\teno = f.Close(mctx)\n\t\tif eno != 0 {\n\t\t\terr = eno\n\t\t}\n\t} else {\n\t\t_ = f.Close(mctx)\n\t}\n\tif err != nil {\n\t\treturn\n\t}\n\n\tapplyObjTaggingFunc(tmpname)\n\n\teno = n.fs.Rename(mctx, tmpname, object, 0)\n\tif eno == syscall.ENOENT {\n\t\tif err = n.mkdirAll(ctx, path.Dir(object)); err != nil {\n\t\t\tlogger.Errorf(\"mkdirAll %s: %s\", path.Dir(object), err)\n\t\t\terr = jfsToObjectErr(ctx, err, bucket, object)\n\t\t\treturn\n\t\t}\n\t\teno = n.fs.Rename(mctx, tmpname, object, 0)\n\t}\n\tif eno != 0 {\n\t\terr = jfsToObjectErr(ctx, eno, bucket, object)\n\t}\n\treturn\n}\n\nfunc (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {\n\tif err = n.checkBucket(ctx, bucket); err != nil {\n\t\treturn\n\t}\n\tvar tagStr string\n\tvar etag string\n\tp := n.path(bucket, object)\n\tif strings.HasSuffix(object, sep) {\n\t\tif err = n.mkdirAll(ctx, p); err != nil {\n\t\t\terr = jfsToObjectErr(ctx, err, bucket, object)\n\t\t\treturn\n\t\t}\n\t\tif r.Size() > 0 {\n\t\t\terr = minio.ObjectExistsAsDirectory{\n\t\t\t\tBucket: bucket,\n\t\t\t\tObject: object,\n\t\t\t\tErr: syscall.EEXIST,\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\t// if the put object is a directory, set its atime to 0\n\t\tn.setFileAtime(p, 0)\n\t} else {\n\t\tif err = n.putObject(ctx, bucket, p, r, opts, func(tmpName string) {\n\t\t\tetag = r.MD5CurrentHexString()\n\t\t\tif n.gConf.KeepEtag && !strings.HasSuffix(object, sep) {\n\t\t\t\tif eno := n.fs.SetXattr(mctx, tmpName, s3Etag, []byte(etag), 0); eno != 0 {\n\t\t\t\t\tlogger.Errorf(\"set xattr error, path: %s,xattr: %s,value: %s,flags: %d\", tmpName, s3Etag, etag, 0)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// tags: key1=value1&key2=value2&key3=value3\n\t\t\tif n.gConf.ObjTag && opts.UserDefined != nil {\n\t\t\t\tif tagStr = opts.UserDefined[xhttp.AmzObjectTagging]; tagStr != \"\" {\n\t\t\t\t\tif eno := n.fs.SetXattr(mctx, tmpName, s3Tags, []byte(tagStr), 0); eno != 0 {\n\t\t\t\t\t\tlogger.Errorf(\"set object tags error, path: %s, value: %s error: %s\", tmpName, tagStr, eno)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\terr = n.setObjMeta(tmpName, opts.UserDefined)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"set object metadata error, path: %s error %s\", p, err)\n\t\t\t}\n\t\t}); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\tfi, eno := n.fs.Stat(mctx, p)\n\tif eno != 0 {\n\t\treturn objInfo, jfsToObjectErr(ctx, eno, bucket, object)\n\t}\n\n\treturn minio.ObjectInfo{\n\t\tBucket: bucket,\n\t\tName: object,\n\t\tETag: etag,\n\t\tModTime: fi.ModTime(),\n\t\tSize: fi.Size(),\n\t\tIsDir: fi.IsDir(),\n\t\tAccTime: fi.ModTime(),\n\t\tUserTags: tagStr,\n\t\tUserDefined: minio.CleanMetadata(opts.UserDefined),\n\t\tIsLatest: true,\n\t}, nil\n}\n\nfunc (n *jfsObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (uploadID string, err error) {\n\tif err = n.checkBucket(ctx, bucket); err != nil {\n\t\treturn\n\t}\n\tuploadID = minio.MustGetUUID()\n\tp := n.upath(bucket, uploadID)\n\terr = n.mkdirAll(ctx, p)\n\tif err == nil {\n\t\teno := n.fs.SetXattr(mctx, p, uploadKeyName, []byte(object), 0)\n\t\tif eno != 0 {\n\t\t\tlogger.Warnf(\"set object %s on upload %s: %s\", object, uploadID, eno)\n\t\t}\n\t\tif n.gConf.ObjTag && opts.UserDefined != nil {\n\t\t\tif tagStr := opts.UserDefined[xhttp.AmzObjectTagging]; tagStr != \"\" {\n\t\t\t\tif eno := n.fs.SetXattr(mctx, p, s3Tags, []byte(tagStr), 0); eno != 0 {\n\t\t\t\t\tlogger.Errorf(\"set object tags error, path: %s, value: %s errors: %s\", p, tagStr, eno)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\terr = n.setObjMeta(p, opts.UserDefined)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"set object metadata error, path: %s error %s\", p, err)\n\t\t}\n\t}\n\treturn\n}\n\nconst uploadKeyName = \"s3-object\"\nconst s3Etag = \"s3-etag\"\n\n// less than 64k ref: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions\nconst s3Tags = \"s3-tags\"\n\n// S3 object metadata\nconst s3Meta = \"s3-meta\"\nconst amzMeta = \"x-amz-meta-\"\n\nvar s3UserControlledSystemMeta = []string{\n\t\"cache-control\",\n\t\"content-disposition\",\n\t\"content-type\",\n}\n\nfunc (n *jfsObjects) getObjMeta(p string) (objMeta map[string]string, err error) {\n\tif n.gConf.ObjMeta {\n\t\tvar errno syscall.Errno\n\t\tvar metadataStr []byte\n\t\tif metadataStr, errno = n.fs.GetXattr(mctx, p, s3Meta); errno != 0 && errno != meta.ENOATTR {\n\t\t\treturn objMeta, errno\n\t\t}\n\t\tif len(metadataStr) > 0 {\n\t\t\terr = json.Unmarshal(metadataStr, &objMeta)\n\t\t\treturn objMeta, err\n\t\t}\n\t} else {\n\t\tobjMeta = make(map[string]string)\n\t}\n\treturn objMeta, nil\n}\n\nfunc (n *jfsObjects) setObjMeta(p string, metadata map[string]string) error {\n\tif n.gConf.ObjMeta && metadata != nil {\n\t\tmeta := make(map[string]string)\n\t\tfor k, v := range metadata {\n\t\t\tk = strings.ToLower(k)\n\t\t\tif strings.HasPrefix(k, amzMeta) {\n\t\t\t\tmeta[k] = v\n\t\t\t} else {\n\t\t\t\tfor _, systemMetaKey := range s3UserControlledSystemMeta {\n\t\t\t\t\tif k == systemMetaKey {\n\t\t\t\t\t\tmeta[k] = v\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(meta) > 0 {\n\t\t\ts3MetadataValue, err := json.Marshal(meta)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif eno := n.fs.SetXattr(mctx, p, s3Meta, s3MetadataValue, 0); eno != 0 {\n\t\t\t\tlogger.Errorf(\"set object metadata error, path: %s,value: %s error: %s\", p, string(s3Meta), eno)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {\n\tif err = n.checkBucket(ctx, bucket); err != nil {\n\t\treturn\n\t}\n\tf, eno := n.fs.Open(mctx, n.tpath(bucket, \"uploads\"), 0)\n\tif eno != 0 {\n\t\treturn // no found\n\t}\n\tdefer f.Close(mctx)\n\tparents, eno := f.ReaddirPlus(mctx, 0)\n\tif eno != 0 {\n\t\terr = jfsToObjectErr(ctx, eno, bucket)\n\t\treturn\n\t}\n\tlmi.Prefix = prefix\n\tlmi.KeyMarker = keyMarker\n\tlmi.UploadIDMarker = uploadIDMarker\n\tlmi.MaxUploads = maxUploads\n\tlmi.Delimiter = delimiter\n\tcommPrefixSet := make(map[string]struct{})\n\tfor _, p := range parents {\n\t\tf, eno := n.fs.Open(mctx, n.tpath(bucket, \"uploads\", string(p.Name)), 0)\n\t\tif eno != 0 {\n\t\t\treturn\n\t\t}\n\t\tdefer f.Close(mctx)\n\t\tentries, eno := f.ReaddirPlus(mctx, 0)\n\t\tif eno != 0 {\n\t\t\terr = jfsToObjectErr(ctx, eno, bucket)\n\t\t\treturn\n\t\t}\n\n\t\tfor _, e := range entries {\n\t\t\tif len(e.Name) != 36 {\n\t\t\t\tcontinue // not an uuid\n\t\t\t}\n\t\t\tuploadID := string(e.Name)\n\t\t\t// todo: parallel\n\t\t\tobject_, eno := n.fs.GetXattr(mctx, n.upath(bucket, uploadID), uploadKeyName)\n\t\t\tif eno != 0 {\n\t\t\t\tlogger.Warnf(\"get object xattr error %s: %s, ignore this item\", n.upath(bucket, uploadID), eno)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tobject := string(object_)\n\t\t\tif strings.HasPrefix(object, prefix) {\n\t\t\t\tif keyMarker != \"\" && object+uploadID > keyMarker+uploadIDMarker || keyMarker == \"\" {\n\t\t\t\t\tlmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{\n\t\t\t\t\t\tObject: object,\n\t\t\t\t\t\tUploadID: uploadID,\n\t\t\t\t\t\tInitiated: time.Unix(e.Attr.Atime, int64(e.Attr.Atimensec)),\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tsort.Slice(lmi.Uploads, func(i, j int) bool {\n\t\tif lmi.Uploads[i].Object == lmi.Uploads[j].Object {\n\t\t\treturn lmi.Uploads[i].UploadID < lmi.Uploads[j].UploadID\n\t\t} else {\n\t\t\treturn lmi.Uploads[i].Object < lmi.Uploads[j].Object\n\t\t}\n\t})\n\n\tif delimiter != \"\" {\n\t\tvar tmp []minio.MultipartInfo\n\t\tfor _, info := range lmi.Uploads {\n\t\t\tif maxUploads == 0 {\n\t\t\t\tlmi.IsTruncated = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tindex := strings.Index(strings.TrimPrefix(info.Object, prefix), delimiter)\n\t\t\tif index == -1 {\n\t\t\t\ttmp = append(tmp, info)\n\t\t\t\tmaxUploads--\n\t\t\t} else {\n\t\t\t\tcommPrefix := info.Object[:index+1]\n\t\t\t\tif _, ok := commPrefixSet[commPrefix]; ok {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tcommPrefixSet[commPrefix] = struct{}{}\n\t\t\t\tmaxUploads--\n\t\t\t}\n\t\t}\n\t\tlmi.Uploads = tmp\n\t\tfor prefix := range commPrefixSet {\n\t\t\tlmi.CommonPrefixes = append(lmi.CommonPrefixes, prefix)\n\t\t}\n\t\tsort.Strings(lmi.CommonPrefixes)\n\t} else if len(lmi.Uploads) > maxUploads {\n\t\tlmi.IsTruncated = true\n\t\tlmi.Uploads = lmi.Uploads[:maxUploads]\n\t}\n\n\tif len(lmi.Uploads) != 0 {\n\t\tlmi.NextKeyMarker = lmi.Uploads[len(lmi.Uploads)-1].Object\n\t\tlmi.NextUploadIDMarker = lmi.Uploads[len(lmi.Uploads)-1].UploadID\n\t}\n\treturn lmi, jfsToObjectErr(ctx, err, bucket)\n}\n\nfunc (n *jfsObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string) (err error) {\n\tif err = n.checkBucket(ctx, bucket); err != nil {\n\t\treturn\n\t}\n\t_, eno := n.fs.Stat(mctx, n.upath(bucket, uploadID))\n\treturn jfsToObjectErr(ctx, eno, bucket, object, uploadID)\n}\n\nfunc (n *jfsObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (result minio.ListPartsInfo, err error) {\n\tif err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {\n\t\treturn result, err\n\t}\n\tf, e := n.fs.Open(mctx, n.upath(bucket, uploadID), 0)\n\tif e != 0 {\n\t\terr = jfsToObjectErr(ctx, e, bucket, object, uploadID)\n\t\treturn\n\t}\n\tdefer func() { _ = f.Close(mctx) }()\n\tentries, e := f.ReaddirPlus(mctx, 0)\n\tif e != 0 {\n\t\terr = jfsToObjectErr(ctx, e, bucket, object, uploadID)\n\t\treturn\n\t}\n\tresult.Bucket = bucket\n\tresult.Object = object\n\tresult.UploadID = uploadID\n\tresult.PartNumberMarker = partNumberMarker\n\tresult.MaxParts = maxParts\n\tfor _, entry := range entries {\n\t\tnum, er := strconv.Atoi(string(entry.Name))\n\t\tif er == nil && num > partNumberMarker {\n\t\t\tetag, _ := n.fs.GetXattr(mctx, n.ppath(bucket, uploadID, string(entry.Name)), s3Etag)\n\t\t\tresult.Parts = append(result.Parts, minio.PartInfo{\n\t\t\t\tPartNumber: num,\n\t\t\t\tSize: int64(entry.Attr.Length),\n\t\t\t\tLastModified: time.Unix(entry.Attr.Mtime, 0),\n\t\t\t\tETag: string(etag),\n\t\t\t})\n\t\t}\n\t}\n\tsort.Slice(result.Parts, func(i, j int) bool {\n\t\treturn result.Parts[i].PartNumber < result.Parts[j].PartNumber\n\t})\n\tif len(result.Parts) > maxParts {\n\t\tresult.IsTruncated = true\n\t\tresult.Parts = result.Parts[:maxParts]\n\t\tresult.NextPartNumberMarker = result.Parts[maxParts-1].PartNumber\n\t}\n\treturn\n}\n\nfunc (n *jfsObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int,\n\tstartOffset int64, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (result minio.PartInfo, err error) {\n\tif err = n.isValidBucketName(srcBucket); err != nil {\n\t\treturn\n\t}\n\tif err = n.checkUploadIDExists(ctx, dstBucket, dstObject, uploadID); err != nil {\n\t\treturn\n\t}\n\t// TODO: use CopyFileRange\n\treturn n.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)\n}\n\nfunc (n *jfsObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {\n\tif err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {\n\t\treturn\n\t}\n\tp := n.ppath(bucket, uploadID, strconv.Itoa(partID))\n\tvar etag string\n\tif err = n.putObject(ctx, bucket, p, r, opts, func(tmpName string) {\n\t\tetag = r.MD5CurrentHexString()\n\t\tif n.fs.SetXattr(mctx, tmpName, s3Etag, []byte(etag), 0) != 0 {\n\t\t\tlogger.Warnf(\"set xattr error, path: %s,xattr: %s,value: %s,flags: %d\", tmpName, s3Etag, etag, 0)\n\t\t}\n\t}); err != nil {\n\t\terr = jfsToObjectErr(ctx, err, bucket, object)\n\t\treturn\n\t}\n\tinfo.PartNumber = partID\n\tinfo.ETag = etag\n\tinfo.LastModified = minio.UTCNow()\n\tinfo.Size = r.Reader.Size()\n\treturn\n}\n\nfunc (n *jfsObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts minio.ObjectOptions) (result minio.MultipartInfo, err error) {\n\tif err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {\n\t\treturn\n\t}\n\tresult.Bucket = bucket\n\tresult.Object = object\n\tresult.UploadID = uploadID\n\treturn\n}\n\nfunc (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []minio.CompletePart, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {\n\tif err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {\n\t\treturn\n\t}\n\tg, ectx := errgroup.WithContext(ctx)\n\tg.SetLimit(10)\n\tfor i := 0; i < len(parts); i++ {\n\t\ti := i\n\t\tg.Go(func() error {\n\t\t\tselect {\n\t\t\tcase <-ectx.Done():\n\t\t\t\treturn ectx.Err()\n\t\t\tdefault:\n\t\t\t}\n\t\t\tppath := n.ppath(bucket, uploadID, strconv.Itoa(parts[i].PartNumber))\n\t\t\tetag, _ := n.fs.GetXattr(mctx, ppath, s3Etag)\n\t\t\tif string(etag) != \"\" && string(etag) != parts[i].ETag {\n\t\t\t\tlogger.Warnf(\"path: %s,expect etag: %s,but got: %s\", ppath, etag, parts[i].ETag)\n\t\t\t\treturn minio.ErrInvalidEtag\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tif err = g.Wait(); err != nil {\n\t\treturn objInfo, err\n\t}\n\ttmp := n.ppath(bucket, uploadID, \"complete\")\n\t_ = n.fs.Delete(mctx, tmp)\n\tf, eno := n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)\n\tif eno != 0 {\n\t\terr = jfsToObjectErr(ctx, eno, bucket, object, uploadID)\n\t\tlogger.Errorf(\"create complete: %s\", err)\n\t\treturn\n\t}\n\tdefer func() {\n\t\t_ = f.Close(mctx)\n\t}()\n\tvar total uint64\n\tfor _, part := range parts {\n\t\tp := n.ppath(bucket, uploadID, strconv.Itoa(part.PartNumber))\n\t\tcopied, eno := n.fs.CopyFileRange(mctx, p, 0, tmp, total, 5<<30)\n\t\tif eno == syscall.ENOENT { // try lookup from old path\n\t\t\tp = n.ppathFlat(bucket, uploadID, strconv.Itoa(part.PartNumber))\n\t\t\tcopied, eno = n.fs.CopyFileRange(mctx, p, 0, tmp, total, 5<<30)\n\t\t}\n\t\tif eno != 0 {\n\t\t\terr = jfsToObjectErr(ctx, eno, bucket, object, uploadID)\n\t\t\tlogger.Errorf(\"merge parts: %s\", err)\n\t\t\treturn\n\t\t}\n\t\ttotal += copied\n\t}\n\n\t// Calculate s3 compatible md5sum for complete multipart.\n\ts3MD5 := minio.ComputeCompleteMultipartMD5(parts)\n\tif n.gConf.KeepEtag {\n\t\teno = n.fs.SetXattr(mctx, tmp, s3Etag, []byte(s3MD5), 0)\n\t\tif eno != 0 {\n\t\t\tlogger.Warnf(\"set xattr error, path: %s,xattr: %s,value: %s,flags: %d\", tmp, s3Etag, s3MD5, 0)\n\t\t}\n\t}\n\n\tvar tagStr []byte\n\tif n.gConf.ObjTag {\n\t\tvar eno syscall.Errno\n\t\tif tagStr, eno = n.fs.GetXattr(mctx, n.upath(bucket, uploadID), s3Tags); eno != 0 {\n\t\t\tif eno != meta.ENOATTR {\n\t\t\t\tlogger.Errorf(\"get object tags error, path: %s, error: %s\", n.upath(bucket, uploadID), eno)\n\t\t\t}\n\t\t} else if len(tagStr) > 0 {\n\t\t\tif eno = n.fs.SetXattr(mctx, tmp, s3Tags, tagStr, 0); eno != 0 {\n\t\t\t\tlogger.Errorf(\"set object tags error, path: %s, tags: %s, error: %s\", tmp, string(tagStr), eno)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar objMeta map[string]string\n\tif n.gConf.ObjMeta {\n\t\tif objMeta, err = n.getObjMeta(n.upath(bucket, uploadID)); err != nil {\n\t\t\tlogger.Errorf(\"get object meta error, path: %s, error: %s\", n.upath(bucket, uploadID), err)\n\t\t} else if err = n.setObjMeta(tmp, objMeta); err != nil {\n\t\t\tlogger.Errorf(\"set object meta error, path: %s, error: %s\", tmp, err)\n\t\t}\n\t}\n\n\tname := n.path(bucket, object)\n\teno = n.fs.Rename(mctx, tmp, name, 0)\n\tif eno == syscall.ENOENT {\n\t\tif err = n.mkdirAll(ctx, path.Dir(name)); err != nil {\n\t\t\tlogger.Errorf(\"mkdirAll %s: %s\", path.Dir(name), err)\n\t\t\t_ = n.fs.Delete(mctx, tmp)\n\t\t\terr = jfsToObjectErr(ctx, err, bucket, object, uploadID)\n\t\t\treturn\n\t\t}\n\t\teno = n.fs.Rename(mctx, tmp, name, 0)\n\t}\n\tif eno != 0 {\n\t\t_ = n.fs.Delete(mctx, tmp)\n\t\terr = jfsToObjectErr(ctx, eno, bucket, object, uploadID)\n\t\tlogger.Errorf(\"Rename %s -> %s: %s\", tmp, name, err)\n\t\treturn\n\t}\n\n\tfi, eno := n.fs.Stat(mctx, name)\n\tif eno != 0 {\n\t\t_ = n.fs.Delete(mctx, name)\n\t\terr = jfsToObjectErr(ctx, eno, bucket, object, uploadID)\n\t\treturn\n\t}\n\n\t// remove parts\n\t_ = n.fs.Rmr(mctx, n.upath(bucket, uploadID), true, meta.RmrDefaultThreads)\n\treturn minio.ObjectInfo{\n\t\tBucket: bucket,\n\t\tName: object,\n\t\tETag: s3MD5,\n\t\tModTime: fi.ModTime(),\n\t\tSize: fi.Size(),\n\t\tIsDir: fi.IsDir(),\n\t\tAccTime: fi.ModTime(),\n\t\tUserTags: string(tagStr),\n\t\tUserDefined: minio.CleanMetadata(opts.UserDefined),\n\t\tIsLatest: true,\n\t}, nil\n}\n\nfunc (n *jfsObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, option minio.ObjectOptions) (err error) {\n\tif err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {\n\t\treturn\n\t}\n\teno := n.fs.Rmr(mctx, n.upath(bucket, uploadID), true, meta.RmrDefaultThreads)\n\treturn jfsToObjectErr(ctx, eno, bucket, object, uploadID)\n}\n\nfunc (n *jfsObjects) cleanup() {\n\tfor range time.Tick(24 * time.Hour) {\n\t\t// default bucket tmp dirs\n\t\ttmpDirs := []string{\".sys/tmp/\", \".sys/uploads/\"}\n\t\tif n.gConf.MultiBucket {\n\t\t\tbuckets, err := n.ListBuckets(context.Background())\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"list buckets error: %v\", err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, bucket := range buckets {\n\t\t\t\ttmpDirs = append(tmpDirs, fmt.Sprintf(\".sys/%s/tmp\", bucket.Name))\n\t\t\t\ttmpDirs = append(tmpDirs, fmt.Sprintf(\".sys/%s/uploads\", bucket.Name))\n\t\t\t}\n\t\t}\n\t\tfor _, dir := range tmpDirs {\n\t\t\tn.cleanupDir(dir)\n\t\t}\n\t}\n}\n\nfunc (n *jfsObjects) cleanupDir(dir string) bool {\n\tf, errno := n.fs.Open(mctx, dir, 0)\n\tif errno != 0 {\n\t\treturn false\n\t}\n\tdefer f.Close(mctx)\n\tentries, _ := f.ReaddirPlus(mctx, 0)\n\tnow := time.Now()\n\tdeleted := 0\n\tfor _, entry := range entries {\n\t\tdirPath := n.path(dir, string(entry.Name))\n\t\tif entry.Attr.Typ == meta.TypeDirectory && len(entry.Name) == subDirPrefix {\n\t\t\tif !n.cleanupDir(strings.TrimPrefix(dirPath, \"/\")) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else if _, err := uuid.Parse(string(entry.Name)); err != nil {\n\t\t\tlogger.Warnf(\"unexpected file path: %s\", dirPath)\n\t\t\tcontinue\n\t\t}\n\t\tif now.Sub(time.Unix(entry.Attr.Mtime, 0)) > 7*24*time.Hour {\n\t\t\tif errno = n.fs.Rmr(mctx, dirPath, true, meta.RmrDefaultThreads); errno != 0 {\n\t\t\t\tlogger.Errorf(\"failed to delete expired temporary files path: %s, err: %s\", dirPath, errno)\n\t\t\t} else {\n\t\t\t\tdeleted += 1\n\t\t\t\tlogger.Infof(\"delete expired temporary files path: %s, mtime: %s\", dirPath, time.Unix(entry.Attr.Mtime, 0).Format(time.RFC3339))\n\t\t\t}\n\t\t}\n\t}\n\treturn deleted == len(entries)\n}\n\ntype jfsFLock struct {\n\tinode meta.Ino\n\towner uint64\n\tmeta meta.Meta\n\tlocalLock sync.RWMutex\n\treadonly bool\n}\n\nfunc (j *jfsFLock) GetLock(ctx context.Context, timeout *minio.DynamicTimeout) (newCtx context.Context, timedOutErr error) {\n\treturn j.getFlockWithTimeOut(ctx, meta.F_WRLCK, timeout)\n}\n\nfunc (j *jfsFLock) getFlockWithTimeOut(ctx context.Context, ltype uint32, timeout *minio.DynamicTimeout) (context.Context, error) {\n\tif j.readonly || j.inode == 0 {\n\t\treturn ctx, nil\n\t}\n\tstart := time.Now()\n\tdeadline := start.Add(timeout.Timeout())\n\tlockStr := \"write\"\n\n\tvar getLockFunc func() bool\n\tvar unlockFunc func()\n\tvar getLock bool\n\tif ltype == meta.F_RDLCK {\n\t\tgetLockFunc = j.localLock.TryRLock\n\t\tunlockFunc = j.localLock.RUnlock\n\t\tlockStr = \"read\"\n\t} else {\n\t\tgetLockFunc = j.localLock.TryLock\n\t\tunlockFunc = j.localLock.Unlock\n\t}\n\n\tfor {\n\t\tgetLock = getLockFunc()\n\t\tif getLock {\n\t\t\tbreak\n\t\t}\n\t\tif time.Now().After(deadline) {\n\t\t\ttimeout.LogFailure()\n\t\t\tlogger.Errorf(\"get %s lock timed out ino:%d\", lockStr, j.inode)\n\t\t\treturn ctx, minio.OperationTimedOut{}\n\t\t}\n\t\ttime.Sleep(5 * time.Millisecond)\n\t}\n\n\tfor {\n\t\tif errno := j.meta.Flock(mctx, j.inode, j.owner, ltype, false); errno != 0 {\n\t\t\tif !errors.Is(errno, syscall.EAGAIN) {\n\t\t\t\tlogger.Errorf(\"failed to get %s lock for inode %d by owner %d, error : %s\", lockStr, j.inode, j.owner, errno)\n\t\t\t}\n\t\t} else {\n\t\t\ttimeout.LogSuccess(time.Since(start))\n\t\t\treturn ctx, nil\n\t\t}\n\n\t\tif time.Now().After(deadline) {\n\t\t\tunlockFunc()\n\t\t\ttimeout.LogFailure()\n\t\t\tlogger.Errorf(\"get %s lock timed out ino:%d\", lockStr, j.inode)\n\t\t\treturn ctx, minio.OperationTimedOut{}\n\t\t}\n\t\ttime.Sleep(5 * time.Millisecond)\n\t}\n}\n\nfunc (j *jfsFLock) Unlock() {\n\tif j.inode == 0 || j.readonly {\n\t\treturn\n\t}\n\tif errno := j.meta.Flock(mctx, j.inode, j.owner, meta.F_UNLCK, true); errno != 0 {\n\t\tlogger.Errorf(\"failed to release lock for inode %d by owner %d, error : %s\", j.inode, j.owner, errno)\n\t}\n\tj.localLock.Unlock()\n}\n\nfunc (j *jfsFLock) GetRLock(ctx context.Context, timeout *minio.DynamicTimeout) (newCtx context.Context, timedOutErr error) {\n\treturn j.getFlockWithTimeOut(ctx, meta.F_RDLCK, timeout)\n}\n\nfunc (j *jfsFLock) RUnlock() {\n\tif j.inode == 0 || j.readonly {\n\t\treturn\n\t}\n\tif errno := j.meta.Flock(mctx, j.inode, j.owner, meta.F_UNLCK, true); errno != 0 {\n\t\tlogger.Errorf(\"failed to release lock for inode %d by owner %d, error : %s\", j.inode, j.owner, errno)\n\t}\n\tj.localLock.RUnlock()\n}\n\nfunc (n *jfsObjects) NewNSLock(bucket string, objects ...string) minio.RWLocker {\n\tif n.gConf.ReadOnly {\n\t\treturn &jfsFLock{readonly: true}\n\t}\n\tif len(objects) != 1 {\n\t\tpanic(fmt.Errorf(\"jfsObjects.NewNSLock: the length of the objects parameter must be 1, current %s\", objects))\n\t}\n\n\tlockfile := path.Join(minio.MinioMetaBucket, minio.MinioMetaLockFile)\n\tvar file *fs.File\n\tvar errno syscall.Errno\n\tfile, errno = n.fs.Open(mctx, lockfile, vfs.MODE_MASK_W)\n\tif errno != 0 && !errors.Is(errno, syscall.ENOENT) {\n\t\tlogger.Errorf(\"failed to open the file to be locked: %s error %s\", lockfile, errno)\n\t\treturn &jfsFLock{}\n\t}\n\tif errors.Is(errno, syscall.ENOENT) {\n\t\tif file, errno = n.fs.Create(mctx, lockfile, 0666, n.gConf.Umask); errno != 0 {\n\t\t\tif errors.Is(errno, syscall.EEXIST) {\n\t\t\t\tif file, errno = n.fs.Open(mctx, lockfile, vfs.MODE_MASK_W); errno != 0 {\n\t\t\t\t\tlogger.Errorf(\"failed to open the file to be locked: %s error %s\", lockfile, errno)\n\t\t\t\t\treturn &jfsFLock{}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tlogger.Errorf(\"failed to create gateway lock file err %s\", errno)\n\t\t\t\treturn &jfsFLock{}\n\t\t\t}\n\t\t}\n\t}\n\tdefer file.Close(mctx)\n\treturn &jfsFLock{owner: n.conf.Meta.Sid, inode: file.Inode(), meta: n.fs.Meta()}\n}\n\nfunc (n *jfsObjects) BackendInfo() madmin.BackendInfo {\n\treturn madmin.BackendInfo{Type: madmin.FS}\n}\n\nfunc (n *jfsObjects) LocalStorageInfo(ctx context.Context) (minio.StorageInfo, []error) {\n\treturn n.StorageInfo(ctx)\n}\n\nfunc (n *jfsObjects) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (loi minio.ListObjectVersionsInfo, err error) {\n\tobjs, err := n.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)\n\tif err == nil {\n\t\tloi.Objects = objs.Objects\n\t\tloi.Prefixes = objs.Prefixes\n\t}\n\treturn loi, err\n}\n\nfunc (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string, info any) (oi minio.ObjectInfo, e error) {\n\tif info != nil {\n\t\tfi := info.(*fs.FileStat)\n\t\toi = minio.ObjectInfo{\n\t\t\tBucket: bucket,\n\t\t\tName: object,\n\t\t\tModTime: fi.ModTime(),\n\t\t\tSize: fi.Size(),\n\t\t\tIsDir: fi.IsDir(),\n\t\t\tAccTime: fi.ModTime(),\n\t\t\tIsLatest: true,\n\t\t}\n\t\tif fi.IsDir() {\n\t\t\toi.Size = 0\n\t\t}\n\t\treturn\n\t}\n\treturn n.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})\n}\n\nfunc (n *jfsObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- minio.ObjectInfo, opts minio.ObjectOptions) error {\n\treturn minio.FsWalk(ctx, n, bucket, prefix, n.listDirFactory(), n.isLeaf, n.isLeafDir, results, n.getObjectInfoNoFSLock, n.getObjectInfoNoFSLock)\n}\n\nfunc (n *jfsObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {\n\tmeta, err := minio.LoadBucketMetadata(ctx, n, bucket)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tjson := jsoniter.ConfigCompatibleWithStandardLibrary\n\tconfigData, err := json.Marshal(policy)\n\tif err != nil {\n\t\treturn err\n\t}\n\tmeta.PolicyConfigJSON = configData\n\n\treturn meta.Save(ctx, n)\n}\n\nfunc (n *jfsObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {\n\tmeta, err := minio.LoadBucketMetadata(ctx, n, bucket)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif meta.PolicyConfig == nil {\n\t\treturn nil, minio.BucketPolicyNotFound{Bucket: bucket}\n\t}\n\treturn meta.PolicyConfig, nil\n}\n\nfunc (n *jfsObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error {\n\tmeta, err := minio.LoadBucketMetadata(ctx, n, bucket)\n\tif err != nil {\n\t\treturn err\n\t}\n\tmeta.PolicyConfigJSON = nil\n\treturn meta.Save(ctx, n)\n}\n\nfunc (n *jfsObjects) SetDriveCounts() []int {\n\treturn nil\n}\n\nfunc (n *jfsObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {\n\treturn madmin.HealResultItem{}, minio.NotImplemented{}\n}\n\nfunc (n *jfsObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {\n\treturn madmin.HealResultItem{}, minio.NotImplemented{}\n}\n\nfunc (n *jfsObjects) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (res madmin.HealResultItem, err error) {\n\treturn res, minio.NotImplemented{}\n}\n\nfunc (n *jfsObjects) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn minio.HealObjectFn) error {\n\treturn minio.NotImplemented{}\n}\n\nfunc (n *jfsObjects) GetMetrics(ctx context.Context) (*minio.BackendMetrics, error) {\n\treturn &minio.BackendMetrics{}, minio.NotImplemented{}\n}\n\nfunc (n *jfsObjects) Health(ctx context.Context, opts minio.HealthOptions) minio.HealthResult {\n\tif _, errno := n.fs.Stat(mctx, minio.MinioMetaBucket); errno != 0 {\n\t\treturn minio.HealthResult{}\n\t}\n\treturn minio.HealthResult{\n\t\tHealthy: true,\n\t}\n}\n\nfunc (n *jfsObjects) ReadHealth(ctx context.Context) bool {\n\t_, errno := n.fs.Stat(mctx, minio.MinioMetaBucket)\n\treturn errno == 0\n}\n\nfunc (n *jfsObjects) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {\n\tif !n.gConf.ObjTag {\n\t\treturn minio.ObjectInfo{}, minio.NotImplemented{}\n\t}\n\tif eno := n.fs.SetXattr(mctx, n.path(bucket, object), s3Tags, []byte(tags), 0); eno != 0 {\n\t\treturn minio.ObjectInfo{}, eno\n\t}\n\treturn n.GetObjectInfo(ctx, bucket, object, opts)\n}\n\nfunc (n *jfsObjects) GetObjectTags(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (*tags.Tags, error) {\n\tif !n.gConf.ObjTag {\n\t\treturn nil, minio.NotImplemented{}\n\t}\n\toi, err := n.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn tags.ParseObjectTags(oi.UserTags)\n}\n\nfunc (n *jfsObjects) DeleteObjectTags(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {\n\tif !n.gConf.ObjTag {\n\t\treturn minio.ObjectInfo{}, minio.NotImplemented{}\n\t}\n\tif errno := n.fs.RemoveXattr(mctx, n.path(bucket, object), s3Tags); errno != 0 && errno != meta.ENOATTR {\n\t\treturn minio.ObjectInfo{}, errno\n\t}\n\treturn n.GetObjectInfo(ctx, bucket, object, opts)\n}\n\nfunc (n *jfsObjects) IsNotificationSupported() bool {\n\treturn true\n}\n\nfunc (n *jfsObjects) IsListenSupported() bool {\n\treturn true\n}\n\nfunc (n *jfsObjects) IsTaggingSupported() bool {\n\treturn true\n}\n" }, { "path": "pkg/gateway/gateway_test.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage gateway\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\tminio \"github.com/minio/minio/cmd\"\n)\n\nfunc TestGatewayLock(t *testing.T) {\n\tm := meta.NewClient(\"memkv://\", nil)\n\tformat := &meta.Format{\n\t\tName: \"test\",\n\t\tBlockSize: 4096,\n\t\tCapacity: 1 << 30,\n\t\tDirStats: true,\n\t}\n\t_ = m.Init(format, true)\n\tvar conf = vfs.Config{\n\t\tMeta: meta.DefaultConf(),\n\t\tChunk: &chunk.Config{\n\t\t\tBlockSize: format.BlockSize << 10,\n\t\t\tMaxUpload: 1,\n\t\t\tMaxDownload: 200,\n\t\t\tBufferSize: 100 << 20,\n\t\t},\n\t\tDirEntryTimeout: time.Millisecond * 100,\n\t\tEntryTimeout: time.Millisecond * 100,\n\t\tAttrTimeout: time.Millisecond * 100,\n\t}\n\tobjStore, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tstore := chunk.NewCachedStore(objStore, *conf.Chunk, nil)\n\tjfs, err := fs.NewFileSystem(&conf, m, store, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"initialize failed: %s\", err)\n\t}\n\tjfsObj := &jfsObjects{fs: jfs, conf: &conf, listPool: minio.NewTreeWalkPool(time.Minute * 30), gConf: &Config{Umask: 022}, nsMutex: minio.NewNSLock(false)}\n\tmctx = meta.NewContext(uint32(os.Getpid()), uint32(os.Getuid()), []uint32{uint32(os.Getgid())})\n\tif err := jfs.Mkdir(mctx, minio.MinioMetaBucket, 0777, 022); err != 0 {\n\t\tt.Fatalf(\"mkdir failed: %s\", err)\n\t}\n\n\trwLocker := jfsObj.NewNSLock(minio.MinioMetaBucket, minio.MinioMetaLockFile)\n\n\tif _, err := rwLocker.GetLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); err != nil {\n\t\tt.Fatalf(\"get lock failed: %s\", err)\n\t}\n\tif _, err := rwLocker.GetLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); !errors.As(err, &minio.OperationTimedOut{}) {\n\t\tt.Fatalf(\"GetLock should return timeout error: %s\", err)\n\t}\n\trwLocker.Unlock()\n\n\tif _, err := rwLocker.GetRLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); err != nil {\n\t\tt.Fatalf(\"get lock failed: %s\", err)\n\t}\n\tif _, err := rwLocker.GetRLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); err != nil {\n\t\tt.Fatalf(\"GetRLock should return nil: %s\", err)\n\t}\n\trwLocker.RUnlock()\n\trwLocker.RUnlock()\n\n\tif _, err := rwLocker.GetLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); err != nil {\n\t\tt.Fatalf(\"get lock failed: %s\", err)\n\t}\n\tif _, err := rwLocker.GetRLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); !errors.As(err, &minio.OperationTimedOut{}) {\n\t\tt.Fatalf(\"GetRLock should return timeout error: %s\", err)\n\t}\n\trwLocker.Unlock()\n\n\tif _, err := rwLocker.GetRLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); err != nil {\n\t\tt.Fatalf(\"GetRLock failed: %s\", err)\n\t}\n\tif _, err := rwLocker.GetLock(context.Background(), minio.NewDynamicTimeout(2*time.Second, 1*time.Second)); !errors.As(err, &minio.OperationTimedOut{}) {\n\t\tt.Fatalf(\"GetRLock should return timeout error: %s\", err)\n\t}\n\trwLocker.RUnlock()\n\n}\n" }, { "path": "pkg/meta/backup.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"unsafe\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta/pb\"\n\t\"google.golang.org/protobuf/encoding/protojson\"\n\t\"google.golang.org/protobuf/proto\"\n\t\"google.golang.org/protobuf/reflect/protoreflect\"\n\t\"google.golang.org/protobuf/reflect/protoregistry\"\n)\n\nconst (\n\tBakMagic = 0x747083\n\tBakVersion = 1\n\tBakEOS = BakMagic // end of segments\n)\n\nconst (\n\tsegTypeUnknown = iota\n\tsegTypeFormat\n\tsegTypeCounter\n\tsegTypeNode\n\tsegTypeEdge\n\tsegTypeChunk\n\tsegTypeSliceRef\n\tsegTypeSymlink\n\tsegTypeSustained\n\tsegTypeDelFile\n\tsegTypeXattr\n\tsegTypeAcl\n\tsegTypeStat\n\tsegTypeQuota\n\tsegTypeParent // for redis/tkv only\n\tsegTypeMax\n)\n\nvar SegType2Name = map[int]string{\n\tsegTypeFormat: \"format\",\n\tsegTypeCounter: \"counter\",\n\tsegTypeNode: \"node\",\n\tsegTypeEdge: \"edge\",\n\tsegTypeChunk: \"chunk\",\n\tsegTypeSliceRef: \"sliceRef\",\n\tsegTypeSymlink: \"symlink\",\n\tsegTypeSustained: \"sustained\",\n\tsegTypeDelFile: \"delFile\",\n\tsegTypeXattr: \"xattr\",\n\tsegTypeAcl: \"acl\",\n\tsegTypeStat: \"stat\",\n\tsegTypeQuota: \"quota\",\n\tsegTypeParent: \"parent\",\n}\n\nvar errBakEOF = fmt.Errorf(\"reach backup EOF\")\n\nfunc getMessageFromType(typ int) (proto.Message, error) {\n\tvar name protoreflect.FullName\n\tif typ == segTypeFormat {\n\t\tname = proto.MessageName(&pb.Format{})\n\t} else if typ < segTypeMax {\n\t\tname = proto.MessageName(&pb.Batch{})\n\t}\n\tif name == \"\" {\n\t\treturn nil, fmt.Errorf(\"unknown message type %d\", typ)\n\t}\n\treturn createMessageByName(name)\n}\n\nfunc createMessageByName(name protoreflect.FullName) (proto.Message, error) {\n\ttyp, err := protoregistry.GlobalTypes.FindMessageByName(name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to find message %s's type: %v\", name, err)\n\t}\n\treturn typ.New().Interface(), nil\n}\n\n// BakFormat: BakSegment... + BakEOS + BakFooter\ntype BakFormat struct {\n\tPos uint64\n\tFooter *BakFooter\n}\n\nfunc newBakFormat() *BakFormat {\n\treturn &BakFormat{\n\t\tFooter: &BakFooter{\n\t\t\tMsg: &pb.Footer{\n\t\t\t\tMagic: BakMagic,\n\t\t\t\tVersion: BakVersion,\n\t\t\t\tInfos: make(map[string]*pb.Footer_SegInfo),\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc (f *BakFormat) writeSegment(w io.Writer, seg *BakSegment) error {\n\tif seg == nil {\n\t\treturn nil\n\t}\n\n\tn, err := seg.Marshal(w)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to marshal segment %s: %v\", seg, err)\n\t}\n\n\tname := seg.Name()\n\tinfo, ok := f.Footer.Msg.Infos[name]\n\tif !ok {\n\t\tinfo = &pb.Footer_SegInfo{Offset: []uint64{}, Num: 0}\n\t\tf.Footer.Msg.Infos[name] = info\n\t}\n\n\tinfo.Offset = append(info.Offset, f.Pos)\n\tinfo.Num += seg.num()\n\tf.Pos += uint64(n)\n\treturn nil\n}\n\nfunc (f *BakFormat) ReadSegment(r io.Reader) (*BakSegment, error) {\n\tseg := &BakSegment{}\n\tif err := seg.Unmarshal(r); err != nil {\n\t\treturn nil, err\n\t}\n\treturn seg, nil\n}\n\nfunc (f *BakFormat) writeFooter(w io.Writer) error {\n\tif err := f.writeEOS(w); err != nil {\n\t\treturn err\n\t}\n\treturn f.Footer.Marshal(w)\n}\n\nfunc (f *BakFormat) writeEOS(w io.Writer) error {\n\tif n, err := w.Write(binary.BigEndian.AppendUint32(nil, BakEOS)); err != nil && n != 4 {\n\t\treturn fmt.Errorf(\"failed to write EOS: err %w, write len %d, expect len 4\", err, n)\n\t}\n\treturn nil\n}\n\nfunc (f *BakFormat) ReadFooter(r io.ReadSeeker) (*BakFooter, error) { // nolint:unused\n\tfooter := &BakFooter{}\n\tif err := footer.Unmarshal(r); err != nil {\n\t\treturn nil, err\n\t}\n\tif footer.Msg.Magic != BakMagic {\n\t\treturn nil, fmt.Errorf(\"invalid magic number %d, expect %d\", footer.Msg.Magic, BakMagic)\n\t}\n\tf.Footer = footer\n\treturn footer, nil\n}\n\ntype BakFooter struct {\n\tMsg *pb.Footer\n\tLen uint64\n}\n\nfunc (h *BakFooter) Marshal(w io.Writer) error {\n\tdata, err := proto.Marshal(h.Msg)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to marshal footer: %w\", err)\n\t}\n\n\tif n, err := w.Write(data); err != nil && n != len(data) {\n\t\treturn fmt.Errorf(\"failed to write footer data: err %w, write len %d, expect len %d\", err, n, len(data))\n\t}\n\n\th.Len = uint64(len(data))\n\tif n, err := w.Write(binary.BigEndian.AppendUint64(nil, h.Len)); err != nil && n != 8 {\n\t\treturn fmt.Errorf(\"failed to write footer length: err %w, write len %d, expect len 8\", err, n)\n\t}\n\treturn nil\n}\n\nfunc (h *BakFooter) Unmarshal(r io.ReadSeeker) error {\n\tlenSize := int64(unsafe.Sizeof(h.Len))\n\t_, _ = r.Seek(-lenSize, io.SeekEnd)\n\n\tdata := make([]byte, lenSize)\n\tif n, err := r.Read(data); err != nil && n != int(lenSize) {\n\t\treturn fmt.Errorf(\"failed to read footer length: err %w, read len %d, expect len %d\", err, n, lenSize)\n\t}\n\n\th.Len = binary.BigEndian.Uint64(data)\n\t_, _ = r.Seek(-int64(h.Len)-lenSize, io.SeekEnd)\n\tdata = make([]byte, h.Len)\n\tif n, err := r.Read(data); err != nil && n != int(h.Len) {\n\t\treturn fmt.Errorf(\"failed to read footer: err %w, read len %d, expect len %d\", err, n, h.Len)\n\t}\n\n\th.Msg = &pb.Footer{}\n\tif err := proto.Unmarshal(data, h.Msg); err != nil {\n\t\treturn fmt.Errorf(\"failed to unmarshal footer: %w\", err)\n\t}\n\treturn nil\n}\n\ntype BakSegment struct {\n\ttyp uint32\n\tlen uint64\n\tval proto.Message\n}\n\nfunc (s *BakSegment) Name() string {\n\tif name, ok := SegType2Name[int(s.typ)]; ok {\n\t\treturn name\n\t}\n\treturn fmt.Sprintf(\"type-%d\", s.typ)\n}\n\nfunc (s *BakSegment) String() string {\n\tswitch s.val.(type) {\n\tcase *pb.Format:\n\t\treturn string(s.val.(*pb.Format).Data)\n\tcase *pb.Batch:\n\t\treturn protojson.Format(s.val)\n\t}\n\treturn \"unknown segment\"\n}\n\nfunc newBakSegment(val proto.Message) *BakSegment {\n\ts := &BakSegment{val: val}\n\tswitch v := s.val.(type) {\n\tcase *pb.Format:\n\t\ts.typ = uint32(segTypeFormat)\n\tcase *pb.Batch:\n\t\tif v.Counters != nil {\n\t\t\ts.typ = uint32(segTypeCounter)\n\t\t} else if v.Sustained != nil {\n\t\t\ts.typ = uint32(segTypeSustained)\n\t\t} else if v.Delfiles != nil {\n\t\t\ts.typ = uint32(segTypeDelFile)\n\t\t} else if v.Acls != nil {\n\t\t\ts.typ = uint32(segTypeAcl)\n\t\t} else if v.Xattrs != nil {\n\t\t\ts.typ = uint32(segTypeXattr)\n\t\t} else if v.Quotas != nil {\n\t\t\ts.typ = uint32(segTypeQuota)\n\t\t} else if v.Dirstats != nil {\n\t\t\ts.typ = uint32(segTypeStat)\n\t\t} else if v.Nodes != nil {\n\t\t\ts.typ = uint32(segTypeNode)\n\t\t} else if v.Chunks != nil {\n\t\t\ts.typ = uint32(segTypeChunk)\n\t\t} else if v.SliceRefs != nil {\n\t\t\ts.typ = uint32(segTypeSliceRef)\n\t\t} else if v.Edges != nil {\n\t\t\ts.typ = uint32(segTypeEdge)\n\t\t} else if v.Symlinks != nil {\n\t\t\ts.typ = uint32(segTypeSymlink)\n\t\t} else if v.Parents != nil {\n\t\t\ts.typ = uint32(segTypeParent)\n\t\t} else {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn s\n}\n\nfunc (s *BakSegment) num() uint64 {\n\tswitch s.typ {\n\tcase segTypeFormat:\n\t\treturn 1\n\tdefault:\n\t\tb := s.val.(*pb.Batch)\n\t\tswitch s.typ {\n\t\tcase segTypeCounter:\n\t\t\treturn uint64(len(b.Counters))\n\t\tcase segTypeNode:\n\t\t\treturn uint64(len(b.Nodes))\n\t\tcase segTypeEdge:\n\t\t\treturn uint64(len(b.Edges))\n\t\tcase segTypeChunk:\n\t\t\treturn uint64(len(b.Chunks))\n\t\tcase segTypeSliceRef:\n\t\t\treturn uint64(len(b.SliceRefs))\n\t\tcase segTypeSymlink:\n\t\t\treturn uint64(len(b.Symlinks))\n\t\tcase segTypeSustained:\n\t\t\treturn uint64(len(b.Sustained))\n\t\tcase segTypeDelFile:\n\t\t\treturn uint64(len(b.Delfiles))\n\t\tcase segTypeXattr:\n\t\t\treturn uint64(len(b.Xattrs))\n\t\tcase segTypeAcl:\n\t\t\treturn uint64(len(b.Acls))\n\t\tcase segTypeStat:\n\t\t\treturn uint64(len(b.Dirstats))\n\t\tcase segTypeQuota:\n\t\t\treturn uint64(len(b.Quotas))\n\t\tcase segTypeParent:\n\t\t\treturn uint64(len(b.Parents))\n\t\t}\n\t\treturn 0\n\t}\n}\n\nfunc (s *BakSegment) Marshal(w io.Writer) (int, error) {\n\tif s == nil || s.val == nil {\n\t\treturn 0, fmt.Errorf(\"segment %s is nil\", s)\n\t}\n\n\tif err := binary.Write(w, binary.BigEndian, s.typ); err != nil {\n\t\treturn 0, fmt.Errorf(\"failed to write segment type %s : %w\", s, err)\n\t}\n\tdata, err := proto.Marshal(s.val)\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"failed to marshal segment message %s : %w\", s, err)\n\t}\n\ts.len = uint64(len(data))\n\tif err := binary.Write(w, binary.BigEndian, s.len); err != nil {\n\t\treturn 0, fmt.Errorf(\"failed to write segment length %s: %w\", s, err)\n\t}\n\n\tif n, err := w.Write(data); err != nil || n != len(data) {\n\t\treturn 0, fmt.Errorf(\"failed to write segment data %s: err %w, write len %d, expect len %d\", s, err, n, len(data))\n\t}\n\n\treturn binary.Size(s.typ) + binary.Size(s.len) + len(data), nil\n}\n\nfunc (s *BakSegment) Unmarshal(r io.Reader) error {\n\tif err := binary.Read(r, binary.BigEndian, &s.typ); err != nil {\n\t\treturn fmt.Errorf(\"failed to read segment type: %v\", err)\n\t}\n\n\tif s.typ == BakEOS {\n\t\treturn errBakEOF\n\t}\n\n\tif err := binary.Read(r, binary.BigEndian, &s.len); err != nil {\n\t\treturn fmt.Errorf(\"failed to read segment %s length: %v\", s, err)\n\t}\n\tdata := make([]byte, s.len)\n\tn, err := r.Read(data)\n\tif err != nil && n != int(s.len) {\n\t\treturn fmt.Errorf(\"failed to read segment value: err %v, read len %d, expect len %d\", err, n, s.len)\n\t}\n\n\tmsg, err := getMessageFromType(int(s.typ))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create message by type %d: %w\", s.typ, err)\n\t}\n\tif err = proto.Unmarshal(data, msg); err != nil {\n\t\treturn fmt.Errorf(\"failed to unmarshal segment msg %d: %w\", s.typ, err)\n\t}\n\ts.val = msg\n\treturn nil\n}\n\ntype DumpOption struct {\n\tKeepSecret bool\n\tThreads int\n\tProgress func(name string, cnt int)\n}\n\nfunc (opt *DumpOption) check() *DumpOption {\n\tif opt == nil {\n\t\topt = &DumpOption{}\n\t}\n\tif opt.Threads < 1 {\n\t\topt.Threads = 10\n\t}\n\treturn opt\n}\n\nfunc (m *baseMeta) dumpFormat(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tf := m.GetFormat()\n\tif !opt.KeepSecret {\n\t\tf.RemoveSecret()\n\t}\n\tdata, err := json.MarshalIndent(f, \"\", \"\")\n\tif err != nil {\n\t\tlogger.Errorf(\"failed to marshal format %s: %v\", f.Name, err)\n\t\treturn nil\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Format{Data: data}})\n}\n\ntype dumpedResult struct {\n\tmsg proto.Message\n\trelease func(m proto.Message)\n}\n\nfunc dumpResult(ctx context.Context, ch chan<- *dumpedResult, res *dumpedResult) error {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tcase ch <- res:\n\t\treturn nil\n\t}\n}\n\ntype LoadOption struct {\n\tThreads int\n\tProgress func(name string, cnt int)\n}\n\nfunc (opt *LoadOption) check() {\n\tif opt.Threads < 1 {\n\t\topt.Threads = 10\n\t}\n}\n\n// transaction\n\ntype txSessionKey struct{}\ntype txMaxRetryKey struct{}\n" }, { "path": "pkg/meta/base.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"path\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"golang.org/x/sync/errgroup\"\n\t\"google.golang.org/protobuf/proto\"\n)\n\nconst (\n\tinodeBatch = 1 << 10\n\tsliceIdBatch = 4 << 10\n\tnlocks = 1024\n\tmaxSymCacheNum = int32(10000)\n\tunknownUsage = -1\n)\n\nvar (\n\tDirBatchNum = map[string]int{\n\t\t\"redis\": 4096,\n\t\t\"kv\": 4096,\n\t\t\"db\": 40960,\n\t}\n\tmaxCompactSlices = 1000\n\tmaxSlices = 2500\n\tinodeNeedPrefetch = uint64(utils.JitterIt(inodeBatch * 0.1)) // Add jitter to reduce probability of txn conflicts\n)\n\nfunc checkInodeName(name string) syscall.Errno {\n\tif len(name) == 0 || strings.ContainsAny(name, \"/\\x00\") {\n\t\treturn syscall.EINVAL\n\t}\n\treturn 0\n}\n\ntype engine interface {\n\t// Get the value of counter name.\n\tgetCounter(name string) (int64, error)\n\t// Increase counter name by value. Do not use this if value is 0, use getCounter instead.\n\tincrCounter(name string, value int64) (int64, error)\n\t// Set counter name to value if old <= value - diff.\n\tsetIfSmall(name string, value, diff int64) (bool, error)\n\tupdateStats(space int64, inodes int64)\n\tdoFlushStats()\n\n\tdoLoad() ([]byte, error)\n\n\tdoNewSession(sinfo []byte, update bool) error\n\tdoRefreshSession() error\n\tdoFindStaleSessions(limit int) ([]uint64, error) // limit < 0 means all\n\tdoCleanStaleSession(sid uint64) error\n\tdoInit(format *Format, force bool) error\n\n\tscanAllChunks(ctx Context, ch chan<- cchunk, bar *utils.Bar) error\n\tdoDeleteSustainedInode(sid uint64, inode Ino) error\n\tdoFindDeletedFiles(ts int64, limit int) (map[Ino]uint64, error) // limit < 0 means all\n\tdoDeleteFileData(inode Ino, length uint64)\n\tdoCleanupSlices(ctx Context, count *uint64) error\n\tdoCleanupDelayedSlices(ctx Context, edge int64) (int, error)\n\tdoDeleteSlice(id uint64, size uint32) error\n\n\tdoCloneEntry(ctx Context, srcIno Ino, parent Ino, name string, ino Ino, attr *Attr, cmode uint8, cumask uint16, top bool) syscall.Errno\n\tdoBatchClone(ctx Context, srcParent Ino, dstParent Ino, entries []*Entry, cmode uint8, cumask uint16, result *batchCloneResult) syscall.Errno\n\tdoAttachDirNode(ctx Context, parent Ino, dstIno Ino, name string) syscall.Errno\n\tdoFindDetachedNodes(t time.Time) []Ino\n\tdoCleanupDetachedNode(ctx Context, detachedNode Ino) syscall.Errno\n\n\tdoGetQuota(ctx Context, qtype uint32, key uint64) (*Quota, error)\n\t// set quota, return true if there is no quota exists before\n\tdoSetQuota(ctx Context, qtype uint32, key uint64, quota *Quota) (created bool, err error)\n\tdoDelQuota(ctx Context, qtype uint32, key uint64) error\n\tdoLoadQuotas(ctx Context) (map[uint64]*Quota, map[uint64]*Quota, map[uint64]*Quota, error)\n\tdoFlushQuotas(ctx Context, quotas []*iQuota) error\n\n\tdoGetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno\n\tdoSetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr, oldAttr *Attr) syscall.Errno\n\tdoLookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr) syscall.Errno\n\tdoMknod(ctx Context, parent Ino, name string, _type uint8, mode, cumask uint16, path string, inode *Ino, attr *Attr) syscall.Errno\n\tdoLink(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno\n\tdoUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno\n\tdoRmdir(ctx Context, parent Ino, name string, inode *Ino, attr *Attr, skipCheckTrash ...bool) syscall.Errno\n\tdoBatchUnlink(ctx Context, parent Ino, entries []*Entry, delta *dirStat, skipCheckTrash ...bool) syscall.Errno\n\tdoReadlink(ctx Context, inode Ino, noatime bool) (int64, []byte, error)\n\tdoReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry, limit int) syscall.Errno\n\tdoRename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode, tinode *Ino, attr, tattr *Attr) syscall.Errno\n\tdoSetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno\n\tdoRemoveXattr(ctx Context, inode Ino, name string) syscall.Errno\n\tdoRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno\n\tdoTouchAtime(ctx Context, inode Ino, attr *Attr, ts time.Time) (bool, error)\n\tdoRead(ctx Context, inode Ino, indx uint32) ([]*slice, syscall.Errno)\n\tdoList(ctx Context, inode Ino) ([]*slice, syscall.Errno)\n\tdoWrite(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time, numSlices *int, delta *dirStat, attr *Attr) syscall.Errno\n\tdoTruncate(ctx Context, inode Ino, flags uint8, length uint64, delta *dirStat, attr *Attr, skipPermCheck bool) syscall.Errno\n\tdoFallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, delta *dirStat, attr *Attr) syscall.Errno\n\tdoCompactChunk(inode Ino, indx uint32, origin []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno\n\n\tdoGetParents(ctx Context, inode Ino) map[Ino]int\n\tdoUpdateDirStat(ctx Context, batch map[Ino]dirStat) error\n\t// @trySync: try sync dir stat if broken or not existed\n\tdoGetDirStat(ctx Context, ino Ino, trySync bool) (*dirStat, syscall.Errno)\n\tdoSyncDirStat(ctx Context, ino Ino) (*dirStat, syscall.Errno)\n\tdoSyncVolumeStat(ctx Context) error\n\n\tscanTrashSlices(Context, trashSliceScan) error\n\tscanPendingSlices(Context, pendingSliceScan) error\n\tscanPendingFiles(Context, pendingFileScan) error\n\n\tGetSession(sid uint64, detail bool) (*Session, error)\n\n\tdoSetFacl(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) syscall.Errno\n\tdoGetFacl(ctx Context, ino Ino, aclType uint8, aclId uint32, rule *aclAPI.Rule) syscall.Errno\n\tcacheACLs(ctx Context) error\n\n\t// kerberos delegation token\n\tdoStoreToken(ctx Context, token []byte) (id uint32, st syscall.Errno)\n\tdoUpdateToken(ctx Context, id uint32, token []byte) syscall.Errno\n\tdoLoadToken(ctx Context, id uint32) (token []byte, st syscall.Errno)\n\tdoDeleteTokens(ctx Context, ids []uint32) syscall.Errno\n\tdoListTokens(ctx Context) (tokens map[uint32][]byte, st syscall.Errno)\n\n\tnewDirHandler(inode Ino, plus bool, entries []*Entry) DirHandler\n\n\tdump(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error\n\tload(ctx Context, typ int, opt *LoadOption, val proto.Message) error\n\tprepareLoad(ctx Context, opt *LoadOption) error\n}\n\ntype trashSliceScan func(ss []Slice, ts int64) (clean bool, err error)\ntype pendingSliceScan func(id uint64, size uint32) (clean bool, err error)\ntype trashFileScan func(inode Ino, size uint64, ts time.Time) (clean bool, err error)\ntype pendingFileScan func(ino Ino, size uint64, ts int64) (clean bool, err error)\n\n// fsStat aligned for atomic operations\n// nolint:structcheck\ntype fsStat struct {\n\tnewSpace int64\n\tnewInodes int64\n\tusedSpace int64\n\tusedInodes int64\n}\n\n// chunk for compaction\ntype cchunk struct {\n\tinode Ino\n\tindx uint32\n\tslices int\n}\n\ntype symlinkCache struct {\n\t*sync.Map\n\tsize atomic.Int32\n\tcap int32\n}\n\n// ugQuotaDelta represents quota changes for a specific user and group.\ntype ugQuotaDelta struct {\n\tUid uint32\n\tGid uint32\n\tSpace int64\n\tInodes int64\n}\n\ntype ugQuotaDeltas map[uint64]*ugQuotaDelta\n\nfunc (ds ugQuotaDeltas) add(delta *ugQuotaDelta) {\n\tkey := ugKey(delta.Uid, delta.Gid)\n\tif existing, ok := ds[key]; ok {\n\t\texisting.Space += delta.Space\n\t\texisting.Inodes += delta.Inodes\n\t} else {\n\t\tds[key] = delta\n\t}\n}\n\ntype batchCloneResult struct {\n\tlength int64\n\tspace int64\n\tinodes int64\n\tdeltas ugQuotaDeltas\n}\n\nfunc ugKey(uid, gid uint32) uint64 {\n\treturn (uint64(uid) << 32) | uint64(gid)\n}\n\nfunc newSymlinkCache(cap int32) *symlinkCache {\n\treturn &symlinkCache{\n\t\tMap: &sync.Map{},\n\t\tcap: cap,\n\t}\n}\n\nfunc (symCache *symlinkCache) Store(inode Ino, path []byte) {\n\tif _, loaded := symCache.Swap(inode, path); !loaded {\n\t\tsymCache.size.Add(1)\n\t}\n}\n\nfunc (symCache *symlinkCache) clean(ctx Context, wg *sync.WaitGroup) {\n\tdefer wg.Done()\n\tticker := time.NewTicker(time.Minute)\n\tdefer ticker.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-ticker.C:\n\t\t\tsymCache.doClean()\n\t\t}\n\t}\n}\n\nfunc (symCache *symlinkCache) doClean() {\n\tif symCache.size.Load() < int32(float64(symCache.cap)*0.75) {\n\t\treturn\n\t}\n\n\ttodo := symCache.size.Load() / 5\n\tcnt := int32(0)\n\tsymCache.Range(func(key, value interface{}) bool {\n\t\tsymCache.Delete(key)\n\t\tsymCache.size.Add(-1)\n\t\tcnt++\n\t\treturn cnt < todo\n\t})\n}\n\ntype baseMeta struct {\n\tsync.Mutex\n\taddr string\n\tconf *Config\n\tfmt *Format\n\n\troot Ino\n\ttxlocks [nlocks]sync.Mutex // Pessimistic locks to reduce conflict\n\tsubTrash internalNode\n\tsid uint64\n\tof *openfiles\n\tremovedFiles map[Ino]bool\n\tcompacting map[uint64]bool\n\tmaxDeleting chan struct{}\n\tdslices chan Slice // slices to delete\n\tsymlinks *symlinkCache\n\tmsgCallbacks *msgCallbacks\n\treloadCb []func(*Format)\n\tumounting bool\n\tsesMu sync.Mutex\n\taclCache aclAPI.Cache\n\n\tsessCtx Context\n\tsessWG sync.WaitGroup\n\n\tdSliceMu sync.Mutex\n\tdSliceWG sync.WaitGroup\n\n\tdirStatsLock sync.Mutex\n\tdirStats map[Ino]dirStat\n\n\tfsStatsLock sync.Mutex\n\t*fsStat\n\n\tparentMu sync.Mutex // protect dirParents\n\tquotaMu sync.RWMutex // protect dirQuotas\n\tdirParents map[Ino]Ino // directory inode -> parent inode\n\tdirQuotas map[uint64]*Quota // directory inode -> quota\n\tuserQuotas map[uint64]*Quota // uid -> quota\n\tgroupQuotas map[uint64]*Quota // gid -> quota\n\n\tquotaMetricMu sync.Mutex\n\tdirQuotaMetricKeys map[uint64]bool\n\tuserQuotaMetricKeys map[uint64]bool\n\tgroupQuotaMetricKeys map[uint64]bool\n\n\tfreeMu sync.Mutex\n\tfreeInodes freeID\n\tfreeSlices freeID\n\tprefetchMu sync.Mutex\n\tprefetchedInodes freeID\n\n\tusedSpaceG prometheus.Gauge\n\tusedInodesG prometheus.Gauge\n\ttotalSpaceG prometheus.Gauge\n\ttotalInodesG prometheus.Gauge\n\ttxDist prometheus.Histogram\n\ttxRestart *prometheus.CounterVec\n\topDist prometheus.Histogram\n\topCount *prometheus.CounterVec\n\topDuration *prometheus.CounterVec\n\n\t// Subdir info metric\n\tsubdirInfoG *prometheus.GaugeVec\n\n\t// Quota metrics\n\tdirQuotaMaxSpaceG *prometheus.GaugeVec\n\tdirQuotaMaxInodesG *prometheus.GaugeVec\n\tdirQuotaUsedSpaceG *prometheus.GaugeVec\n\tdirQuotaUsedInodesG *prometheus.GaugeVec\n\n\tuserQuotaMaxSpaceG *prometheus.GaugeVec\n\tuserQuotaMaxInodesG *prometheus.GaugeVec\n\tuserQuotaUsedSpaceG *prometheus.GaugeVec\n\tuserQuotaUsedInodesG *prometheus.GaugeVec\n\n\tgroupQuotaMaxSpaceG *prometheus.GaugeVec\n\tgroupQuotaMaxInodesG *prometheus.GaugeVec\n\tgroupQuotaUsedSpaceG *prometheus.GaugeVec\n\tgroupQuotaUsedInodesG *prometheus.GaugeVec\n\n\tbgjobDels *prometheus.CounterVec\n\tbgjobDuration *prometheus.HistogramVec\n\n\ten engine\n}\n\nfunc newBaseMeta(addr string, conf *Config) *baseMeta {\n\treturn &baseMeta{\n\t\taddr: utils.RemovePassword(addr),\n\t\tconf: conf,\n\t\tsid: conf.Sid,\n\t\troot: RootInode,\n\t\tof: newOpenFiles(conf.OpenCache, conf.OpenCacheLimit),\n\t\tremovedFiles: make(map[Ino]bool),\n\t\tcompacting: make(map[uint64]bool),\n\t\tmaxDeleting: make(chan struct{}, 100),\n\t\tsymlinks: newSymlinkCache(maxSymCacheNum),\n\t\tfsStat: &fsStat{\n\t\t\tusedSpace: unknownUsage,\n\t\t\tusedInodes: unknownUsage,\n\t\t},\n\t\tdirStats: make(map[Ino]dirStat),\n\t\tdirParents: make(map[Ino]Ino),\n\t\tdirQuotas: make(map[uint64]*Quota),\n\t\tuserQuotas: make(map[uint64]*Quota),\n\t\tgroupQuotas: make(map[uint64]*Quota),\n\t\tmsgCallbacks: &msgCallbacks{\n\t\t\tcallbacks: make(map[uint32]MsgCallback),\n\t\t},\n\t\taclCache: aclAPI.NewCache(),\n\n\t\tusedSpaceG: prometheus.NewGauge(prometheus.GaugeOpts{\n\t\t\tName: \"used_space\",\n\t\t\tHelp: \"Total used space in bytes.\",\n\t\t}),\n\t\tusedInodesG: prometheus.NewGauge(prometheus.GaugeOpts{\n\t\t\tName: \"used_inodes\",\n\t\t\tHelp: \"Total used number of inodes.\",\n\t\t}),\n\t\ttotalSpaceG: prometheus.NewGauge(prometheus.GaugeOpts{\n\t\t\tName: \"total_space\",\n\t\t\tHelp: \"Total space in bytes.\",\n\t\t}),\n\t\ttotalInodesG: prometheus.NewGauge(prometheus.GaugeOpts{\n\t\t\tName: \"total_inodes\",\n\t\t\tHelp: \"Total number of inodes.\",\n\t\t}),\n\t\ttxDist: prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\t\tName: \"transaction_durations_histogram_seconds\",\n\t\t\tHelp: \"Transactions latency distributions.\",\n\t\t\tBuckets: prometheus.ExponentialBuckets(0.0001, 1.5, 30),\n\t\t}),\n\t\ttxRestart: prometheus.NewCounterVec(prometheus.CounterOpts{\n\t\t\tName: \"transaction_restart\",\n\t\t\tHelp: \"The number of times a transaction is restarted.\",\n\t\t}, []string{\"method\"}),\n\t\topDist: prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\t\tName: \"meta_ops_durations_histogram_seconds\",\n\t\t\tHelp: \"Operation latency distributions.\",\n\t\t\tBuckets: prometheus.ExponentialBuckets(0.0001, 1.5, 30),\n\t\t}),\n\t\topCount: prometheus.NewCounterVec(prometheus.CounterOpts{\n\t\t\tName: \"meta_ops_total\",\n\t\t\tHelp: \"Meta operation count\",\n\t\t}, []string{\"method\"}),\n\t\topDuration: prometheus.NewCounterVec(prometheus.CounterOpts{\n\t\t\tName: \"meta_ops_duration_seconds\",\n\t\t\tHelp: \"Meta operation duration in seconds.\",\n\t\t}, []string{\"method\"}),\n\n\t\t// Subdir info metric\n\t\tsubdirInfoG: prometheus.NewGaugeVec(prometheus.GaugeOpts{\n\t\t\tName: \"subdir_info\",\n\t\t\tHelp: \"Subdir configuration for JuiceFS mount (empty string means root mount)\",\n\t\t}, []string{\"subdir\"}),\n\n\t\t// quota metrics\n\t\tdirQuotaMaxSpaceG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"dir_quota_max_space_bytes\",\n\t\t\t\tHelp: \"Directory quota maximum space in bytes.\",\n\t\t\t},\n\t\t\t[]string{\"inode\"},\n\t\t),\n\t\tdirQuotaMaxInodesG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"dir_quota_max_inodes\",\n\t\t\t\tHelp: \"Directory quota maximum number of inodes.\",\n\t\t\t},\n\t\t\t[]string{\"inode\"},\n\t\t),\n\t\tdirQuotaUsedSpaceG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"dir_quota_used_space_bytes\",\n\t\t\t\tHelp: \"Directory quota used space in bytes.\",\n\t\t\t},\n\t\t\t[]string{\"inode\"},\n\t\t),\n\t\tdirQuotaUsedInodesG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"dir_quota_used_inodes\",\n\t\t\t\tHelp: \"Directory quota used number of inodes.\",\n\t\t\t},\n\t\t\t[]string{\"inode\"},\n\t\t),\n\t\tuserQuotaMaxSpaceG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"user_quota_max_space_bytes\",\n\t\t\t\tHelp: \"User quota maximum space in bytes.\",\n\t\t\t},\n\t\t\t[]string{\"uid\"},\n\t\t),\n\t\tuserQuotaMaxInodesG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"user_quota_max_inodes\",\n\t\t\t\tHelp: \"User quota maximum number of inodes.\",\n\t\t\t},\n\t\t\t[]string{\"uid\"},\n\t\t),\n\t\tuserQuotaUsedSpaceG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"user_quota_used_space_bytes\",\n\t\t\t\tHelp: \"User quota used space in bytes.\",\n\t\t\t},\n\t\t\t[]string{\"uid\"},\n\t\t),\n\t\tuserQuotaUsedInodesG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"user_quota_used_inodes\",\n\t\t\t\tHelp: \"User quota used number of inodes.\",\n\t\t\t},\n\t\t\t[]string{\"uid\"},\n\t\t),\n\t\tgroupQuotaMaxSpaceG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"group_quota_max_space_bytes\",\n\t\t\t\tHelp: \"Group quota maximum space in bytes.\",\n\t\t\t},\n\t\t\t[]string{\"gid\"},\n\t\t),\n\t\tgroupQuotaMaxInodesG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"group_quota_max_inodes\",\n\t\t\t\tHelp: \"Group quota maximum number of inodes.\",\n\t\t\t},\n\t\t\t[]string{\"gid\"},\n\t\t),\n\t\tgroupQuotaUsedSpaceG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"group_quota_used_space_bytes\",\n\t\t\t\tHelp: \"Group quota used space in bytes.\",\n\t\t\t},\n\t\t\t[]string{\"gid\"},\n\t\t),\n\t\tgroupQuotaUsedInodesG: prometheus.NewGaugeVec(\n\t\t\tprometheus.GaugeOpts{\n\t\t\t\tName: \"group_quota_used_inodes\",\n\t\t\t\tHelp: \"Group quota used number of inodes.\",\n\t\t\t},\n\t\t\t[]string{\"gid\"},\n\t\t),\n\n\t\tbgjobDuration: prometheus.NewHistogramVec(\n\t\t\tprometheus.HistogramOpts{\n\t\t\t\tName: \"juicefs_bgjob_duration_seconds\",\n\t\t\t\tHelp: \"Background job duration in seconds.\",\n\t\t\t\tBuckets: prometheus.ExponentialBuckets(1, 2, 13),\n\t\t\t},\n\t\t\t[]string{\"job\", \"status\"},\n\t\t),\n\t\tbgjobDels: prometheus.NewCounterVec(\n\t\t\tprometheus.CounterOpts{\n\t\t\t\tName: \"juicefs_bgjob_deletions_total\",\n\t\t\t\tHelp: \"Number of deletions (files or slices) by background jobs.\",\n\t\t\t},\n\t\t\t[]string{\"job\"},\n\t\t),\n\n\t\tdirQuotaMetricKeys: make(map[uint64]bool),\n\t\tuserQuotaMetricKeys: make(map[uint64]bool),\n\t\tgroupQuotaMetricKeys: make(map[uint64]bool),\n\t}\n}\n\n// InitSharedMetrics initialize the metrics that are same for all clients.\nfunc (m *baseMeta) InitSharedMetrics(reg prometheus.Registerer) {\n\tif reg == nil {\n\t\treturn\n\t}\n\n\treg.MustRegister(m.usedSpaceG)\n\treg.MustRegister(m.usedInodesG)\n\treg.MustRegister(m.totalSpaceG)\n\treg.MustRegister(m.totalInodesG)\n\treg.MustRegister(m.dirQuotaMaxSpaceG)\n\treg.MustRegister(m.dirQuotaMaxInodesG)\n\treg.MustRegister(m.dirQuotaUsedSpaceG)\n\treg.MustRegister(m.dirQuotaUsedInodesG)\n\treg.MustRegister(m.userQuotaMaxSpaceG)\n\treg.MustRegister(m.userQuotaMaxInodesG)\n\treg.MustRegister(m.userQuotaUsedSpaceG)\n\treg.MustRegister(m.userQuotaUsedInodesG)\n\treg.MustRegister(m.groupQuotaMaxSpaceG)\n\treg.MustRegister(m.groupQuotaMaxInodesG)\n\treg.MustRegister(m.groupQuotaUsedSpaceG)\n\treg.MustRegister(m.groupQuotaUsedInodesG)\n\treg.MustRegister(m.bgjobDuration)\n\treg.MustRegister(m.bgjobDels)\n\treg.MustRegister(m.subdirInfoG)\n\n\t// Initialize subdir info metric\n\tsubdir := m.conf.Subdir\n\tif subdir == \"/\" {\n\t\tsubdir = \"\"\n\t}\n\tm.subdirInfoG.WithLabelValues(subdir).Set(1)\n\n\tgo func() {\n\t\tfor {\n\t\t\tif m.sessCtx != nil && m.sessCtx.Canceled() {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tvar totalSpace, availSpace, iused, iavail uint64\n\t\t\terr := m.StatFS(Background(), m.root, &totalSpace, &availSpace, &iused, &iavail)\n\t\t\tif err == 0 {\n\t\t\t\tm.usedSpaceG.Set(float64(totalSpace - availSpace))\n\t\t\t\tm.usedInodesG.Set(float64(iused))\n\t\t\t\tm.totalSpaceG.Set(float64(totalSpace))\n\t\t\t\tm.totalInodesG.Set(float64(iused + iavail))\n\t\t\t}\n\t\t\tm.updateQuotaMetrics()\n\t\t\tutils.SleepWithJitter(time.Second * 10)\n\t\t}\n\t}()\n\n\tgo func() {\n\t\tfor {\n\t\t\tif m.sessCtx != nil && m.sessCtx.Canceled() {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tm.cleanupQuotaMetrics()\n\t\t\tutils.SleepWithJitter(time.Hour)\n\t\t}\n\t}()\n}\n\nfunc (m *baseMeta) InitMetrics(reg prometheus.Registerer) {\n\tif reg == nil {\n\t\treturn\n\t}\n\treg.MustRegister(m.txDist)\n\treg.MustRegister(m.txRestart)\n\treg.MustRegister(m.opDist)\n\treg.MustRegister(m.opCount)\n\treg.MustRegister(m.opDuration)\n}\n\nfunc (m *baseMeta) timeit(method string, start time.Time) {\n\tused := time.Since(start).Seconds()\n\tm.opDist.Observe(used)\n\tm.opCount.WithLabelValues(method).Inc()\n\tm.opDuration.WithLabelValues(method).Add(used)\n}\n\nfunc (m *baseMeta) getBase() *baseMeta {\n\treturn m\n}\n\nfunc (m *baseMeta) checkRoot(inode Ino) Ino {\n\tswitch inode {\n\tcase 0:\n\t\treturn RootInode // force using Root inode\n\tcase RootInode:\n\t\treturn m.root\n\tdefault:\n\t\treturn inode\n\t}\n}\n\nfunc (r *baseMeta) txLock(idx uint) {\n\tr.txlocks[idx%nlocks].Lock()\n}\n\nfunc (r *baseMeta) txUnlock(idx uint) {\n\tr.txlocks[idx%nlocks].Unlock()\n}\n\nfunc (r *baseMeta) txBatchLock(inodes ...Ino) func() {\n\tswitch len(inodes) {\n\tcase 0:\n\t\treturn func() {}\n\tcase 1: // most cases\n\t\tr.txLock(uint(inodes[0]))\n\t\treturn func() { r.txUnlock(uint(inodes[0])) }\n\tdefault: // for rename and more\n\t\tinodeSlots := make([]int, len(inodes))\n\t\tfor i, ino := range inodes {\n\t\t\tinodeSlots[i] = int(ino % nlocks)\n\t\t}\n\t\tsort.Ints(inodeSlots)\n\t\tuniqInodeSlots := inodeSlots[:0]\n\t\tfor i := 0; i < len(inodeSlots); i++ { // Go does not support recursive locks\n\t\t\tif i == 0 || inodeSlots[i] != inodeSlots[i-1] {\n\t\t\t\tuniqInodeSlots = append(uniqInodeSlots, inodeSlots[i])\n\t\t\t}\n\t\t}\n\t\tfor _, idx := range uniqInodeSlots {\n\t\t\tr.txlocks[idx].Lock()\n\t\t}\n\t\treturn func() {\n\t\t\tfor _, idx := range uniqInodeSlots {\n\t\t\t\tr.txlocks[idx].Unlock()\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (r *baseMeta) OnMsg(mtype uint32, cb MsgCallback) {\n\tr.msgCallbacks.Lock()\n\tdefer r.msgCallbacks.Unlock()\n\tr.msgCallbacks.callbacks[mtype] = cb\n}\n\nfunc (r *baseMeta) newMsg(mid uint32, args ...interface{}) error {\n\tr.msgCallbacks.Lock()\n\tcb, ok := r.msgCallbacks.callbacks[mid]\n\tr.msgCallbacks.Unlock()\n\tif ok {\n\t\treturn cb(args...)\n\t}\n\treturn fmt.Errorf(\"message %d is not supported\", mid)\n}\n\nfunc (m *baseMeta) Load(checkVersion bool) (*Format, error) {\n\tbody, err := m.en.doLoad()\n\tif err == nil && len(body) == 0 {\n\t\terr = fmt.Errorf(\"database is not formatted, please run `juicefs format ...` first\")\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar format = new(Format)\n\tif err = json.Unmarshal(body, format); err != nil {\n\t\treturn nil, fmt.Errorf(\"json: %s\", err)\n\t}\n\tif checkVersion {\n\t\tif err = format.CheckVersion(); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"check version: %s\", err)\n\t\t}\n\t}\n\tm.Lock()\n\tm.fmt = format\n\tm.Unlock()\n\treturn format, nil\n}\n\nfunc (m *baseMeta) newSessionInfo() []byte {\n\thost, err := os.Hostname()\n\tif err != nil {\n\t\tlogger.Warnf(\"Failed to get hostname: %s\", err)\n\t}\n\tips, err := utils.FindLocalIPs(m.conf.NetworkInterfaces...)\n\tif err != nil {\n\t\tlogger.Warnf(\"Failed to get local IP: %s\", err)\n\t}\n\taddrs := make([]string, 0, len(ips))\n\tfor _, i := range ips {\n\t\tif ip := i.String(); ip[0] == '?' {\n\t\t\tlogger.Warnf(\"Invalid IP address: %s\", ip)\n\t\t} else {\n\t\t\taddrs = append(addrs, ip)\n\t\t}\n\t}\n\tbuf, err := json.Marshal(&SessionInfo{\n\t\tVersion: version.Version(),\n\t\tHostName: host,\n\t\tIPAddrs: addrs,\n\t\tMountPoint: m.conf.MountPoint,\n\t\tMountTime: time.Now(),\n\t\tProcessID: os.Getpid(),\n\t})\n\tif err != nil {\n\t\tpanic(err) // marshal SessionInfo should never fail\n\t}\n\treturn buf\n}\n\nfunc (m *baseMeta) NewSession(record bool) error {\n\tm.sessCtx = Background()\n\tctx := m.sessCtx\n\tgo m.refresh(ctx)\n\n\tif err := m.en.cacheACLs(ctx); err != nil {\n\t\treturn err\n\t}\n\n\tif m.conf.ReadOnly {\n\t\tlogger.Infof(\"Create read-only session OK with version: %s\", version.Version())\n\t\treturn nil\n\t}\n\n\tif record {\n\t\t// use the original sid if it's not 0\n\t\taction := \"Update\"\n\t\tif m.sid == 0 {\n\t\t\tv, err := m.en.incrCounter(\"nextSession\", 1)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"get session ID: %s\", err)\n\t\t\t}\n\t\t\tm.sid = uint64(v)\n\t\t\tm.conf.Sid = m.sid\n\t\t\taction = \"Create\"\n\t\t}\n\t\tif err := m.en.doNewSession(m.newSessionInfo(), action == \"Update\"); err != nil {\n\t\t\treturn fmt.Errorf(\"create session: %s\", err)\n\t\t}\n\t\tlogger.Infof(\"%s session %d OK with version: %s\", action, m.sid, version.Version())\n\t}\n\n\tm.loadQuotas()\n\n\tm.sessWG.Add(3)\n\tgo m.flushStats(ctx)\n\tgo m.flushDirStat(ctx)\n\tgo m.flushQuotas(ctx)\n\tm.startDeleteSliceTasks() // start MaxDeletes tasks\n\n\tif !m.conf.NoBGJob {\n\t\tm.sessWG.Add(4)\n\t\tgo m.cleanupDeletedFiles(ctx)\n\t\tgo m.cleanupSlices(ctx)\n\t\tgo m.cleanupTrash(ctx)\n\t\tgo m.symlinks.clean(ctx, &m.sessWG)\n\t}\n\treturn nil\n}\n\nconst (\n\tbgJobSucc = \"success\"\n\tbgJobFail = \"failed\"\n\tbgJobCanceled = \"canceled\"\n)\n\nfunc (m *baseMeta) startDeleteSliceTasks() {\n\tm.Lock()\n\tdefer m.Unlock()\n\tif m.conf.MaxDeletes <= 0 || m.dslices != nil {\n\t\treturn\n\t}\n\tm.sessWG.Add(m.conf.MaxDeletes)\n\tm.dSliceWG.Add(m.conf.MaxDeletes)\n\tm.dslices = make(chan Slice, m.conf.MaxDeletes*10240)\n\tfor i := 0; i < m.conf.MaxDeletes; i++ {\n\t\tgo func(dslices chan Slice) {\n\t\t\tdefer m.sessWG.Done()\n\t\t\tdefer m.dSliceWG.Done()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-m.sessCtx.Done():\n\t\t\t\t\treturn\n\t\t\t\tcase s, ok := <-dslices:\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tm.deleteSlice_(s.Id, s.Size)\n\t\t\t\t}\n\t\t\t}\n\t\t}(m.dslices)\n\t}\n}\n\nfunc (m *baseMeta) stopDeleteSliceTasks() {\n\tm.dSliceMu.Lock()\n\tif m.conf.MaxDeletes <= 0 || m.dslices == nil {\n\t\tm.dSliceMu.Unlock()\n\t\treturn\n\t}\n\tclose(m.dslices)\n\tm.dslices = nil\n\tm.dSliceMu.Unlock()\n\tm.dSliceWG.Wait()\n}\n\nfunc (m *baseMeta) expireTime() int64 {\n\tif m.conf.Heartbeat > 0 {\n\t\treturn time.Now().Add(m.conf.Heartbeat * 5).Unix()\n\t} else {\n\t\treturn time.Now().Add(time.Hour * 24 * 365).Unix()\n\t}\n}\n\nfunc (m *baseMeta) OnReload(fn func(f *Format)) {\n\tm.msgCallbacks.Lock()\n\tdefer m.msgCallbacks.Unlock()\n\tm.reloadCb = append(m.reloadCb, fn)\n}\n\nconst UmountCode = 11\n\nfunc (m *baseMeta) refresh(ctx Context) {\n\tfor {\n\t\tif ctx.Canceled() {\n\t\t\treturn\n\t\t}\n\t\tif m.conf.Heartbeat > 0 {\n\t\t\tutils.SleepWithJitter(m.conf.Heartbeat)\n\t\t} else { // use default value\n\t\t\tutils.SleepWithJitter(time.Second * 12)\n\t\t}\n\t\tm.sesMu.Lock()\n\t\tif m.umounting {\n\t\t\tm.sesMu.Unlock()\n\t\t\treturn\n\t\t}\n\t\tif !m.conf.ReadOnly && m.conf.Heartbeat > 0 && m.sid > 0 {\n\t\t\tif err := m.en.doRefreshSession(); err != nil {\n\t\t\t\tlogger.Errorf(\"Refresh session %d: %s\", m.sid, err)\n\t\t\t}\n\t\t}\n\t\tm.sesMu.Unlock()\n\n\t\told := m.getFormat()\n\t\tif format, err := m.Load(false); err != nil {\n\t\t\tif strings.HasPrefix(err.Error(), \"database is not formatted\") {\n\t\t\t\tlogger.Errorf(\"reload setting: %s\", err)\n\t\t\t\tos.Exit(UmountCode)\n\t\t\t}\n\t\t\tlogger.Warnf(\"reload setting: %s\", err)\n\t\t} else if format.MetaVersion > MaxVersion {\n\t\t\tlogger.Errorf(\"incompatible metadata version %d > max version %d\", format.MetaVersion, MaxVersion)\n\t\t\tos.Exit(UmountCode)\n\t\t} else if format.UUID != old.UUID {\n\t\t\tlogger.Errorf(\"UUID changed from %s to %s\", old.UUID, format.UUID)\n\t\t\tos.Exit(UmountCode)\n\t\t} else if !reflect.DeepEqual(format, old) {\n\t\t\tm.msgCallbacks.Lock()\n\t\t\tcbs := m.reloadCb\n\t\t\tm.msgCallbacks.Unlock()\n\t\t\tfor _, cb := range cbs {\n\t\t\t\tcb(format)\n\t\t\t}\n\t\t}\n\n\t\tif v, err := m.en.getCounter(usedSpace); err == nil {\n\t\t\tatomic.StoreInt64(&m.usedSpace, v)\n\t\t} else {\n\t\t\tlogger.Warnf(\"Get counter %s: %s\", usedSpace, err)\n\t\t}\n\t\tif v, err := m.en.getCounter(totalInodes); err == nil {\n\t\t\tatomic.StoreInt64(&m.usedInodes, v)\n\t\t} else {\n\t\t\tlogger.Warnf(\"Get counter %s: %s\", totalInodes, err)\n\t\t}\n\t\tm.loadQuotas()\n\n\t\tif m.conf.ReadOnly || m.conf.NoBGJob || m.conf.Heartbeat == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tif ok, err := m.en.setIfSmall(\"lastCleanupSessions\", time.Now().Unix(), int64((m.conf.Heartbeat * 9 / 10).Seconds())); err != nil {\n\t\t\tlogger.Warnf(\"checking counter lastCleanupSessions: %s\", err)\n\t\t} else if ok {\n\t\t\tgo m.CleanStaleSessions(ctx)\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) CleanStaleSessions(ctx Context) {\n\tsids, err := m.en.doFindStaleSessions(1000)\n\tif err != nil {\n\t\tlogger.Warnf(\"scan stale sessions: %s\", err)\n\t\treturn\n\t}\n\tfor _, sid := range sids {\n\t\tif ctx.Canceled() {\n\t\t\treturn\n\t\t}\n\t\ts, err := m.en.GetSession(sid, false)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"Get session info %d: %v\", sid, err)\n\t\t\ts = &Session{Sid: sid}\n\t\t}\n\t\tlogger.Infof(\"clean up stale session %d %+v: %v\", sid, s.SessionInfo, m.en.doCleanStaleSession(sid))\n\t}\n}\n\nfunc (m *baseMeta) CloseSession() error {\n\tm.FlushSession()\n\tm.sesMu.Lock()\n\tm.umounting = true\n\tm.sesMu.Unlock()\n\tvar err error\n\tif m.sid > 0 {\n\t\terr = m.en.doCleanStaleSession(m.sid)\n\t}\n\tm.sessCtx.Cancel()\n\tm.sessWG.Wait()\n\tm.stopDeleteSliceTasks()\n\tlogger.Infof(\"close session %d: %v\", m.sid, err)\n\treturn err\n}\n\nfunc (m *baseMeta) FlushSession() {\n\tif m.conf.ReadOnly {\n\t\treturn\n\t}\n\tm.doFlushStats()\n\tm.doFlushDirStat()\n\tm.doFlushQuotas()\n\tlogger.Infof(\"flush session %d:\", m.sid)\n}\n\nfunc (m *baseMeta) Init(format *Format, force bool) error {\n\treturn m.en.doInit(format, force)\n}\n\nfunc (m *baseMeta) cleanupDeletedFiles(ctx Context) {\n\tdefer m.sessWG.Done()\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-time.After(utils.JitterIt(time.Hour)):\n\t\t}\n\t\tif ok, err := m.en.setIfSmall(\"lastCleanupFiles\", time.Now().Unix(), int64(time.Hour.Seconds())*9/10); err != nil {\n\t\t\tlogger.Warnf(\"checking counter lastCleanupFiles: %s\", err)\n\t\t} else if ok {\n\t\t\tjob := \"cleanupDeletedFiles\"\n\t\t\tjobStart := time.Now()\n\t\t\tfiles, err := m.en.doFindDeletedFiles(time.Now().Add(-time.Hour).Unix(), 6e5)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"scan deleted files: %s\", err)\n\t\t\t\tm.bgjobDuration.WithLabelValues(job, bgJobFail).Observe(time.Since(jobStart).Seconds())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvar processed int64\n\t\t\tstatus := bgJobSucc\n\t\t\tfor inode, length := range files {\n\t\t\t\tlogger.Debugf(\"cleanup chunks of inode %d with %d bytes\", inode, length)\n\t\t\t\tm.en.doDeleteFileData(inode, length)\n\t\t\t\tprocessed++\n\t\t\t\tif time.Since(jobStart) > 50*time.Minute { // Yield my time slice to avoid conflicts with other clients\n\t\t\t\t\tstatus = bgJobCanceled\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tm.bgjobDuration.WithLabelValues(job, status).Observe(time.Since(jobStart).Seconds())\n\t\t\tm.bgjobDels.WithLabelValues(job).Add(float64(processed))\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) cleanupSlices(ctx Context) {\n\tdefer m.sessWG.Done()\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-time.After(utils.JitterIt(time.Hour)):\n\t\t}\n\t\tif ok, err := m.en.setIfSmall(\"nextCleanupSlices\", time.Now().Unix(), int64(time.Hour.Seconds())*9/10); err != nil {\n\t\t\tlogger.Warnf(\"checking counter nextCleanupSlices: %s\", err)\n\t\t} else if ok {\n\t\t\tfunc() {\n\t\t\t\tcCtx := WrapWithTimeout(ctx, time.Minute*50)\n\t\t\t\tdefer cCtx.Cancel()\n\t\t\t\tjobStart := time.Now()\n\t\t\t\tstatus := bgJobSucc\n\t\t\t\tvar cnt uint64\n\t\t\t\tif err := m.en.doCleanupSlices(cCtx, &cnt); err != nil {\n\t\t\t\t\tif errors.Is(err, context.DeadlineExceeded) {\n\t\t\t\t\t\tstatus = bgJobCanceled\n\t\t\t\t\t} else {\n\t\t\t\t\t\tstatus = bgJobFail\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tm.bgjobDuration.WithLabelValues(\"cleanupSlices\", status).Observe(time.Since(jobStart).Seconds())\n\t\t\t\tm.bgjobDels.WithLabelValues(\"cleanupSlices\").Add(float64(cnt))\n\t\t\t}()\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) StatFS(ctx Context, ino Ino, totalspace, availspace, iused, iavail *uint64) syscall.Errno {\n\tdefer m.timeit(\"StatFS\", time.Now())\n\tif st := m.statRootFs(ctx, totalspace, availspace, iused, iavail); st != 0 {\n\t\treturn st\n\t}\n\tino = m.checkRoot(ino)\n\tvar usage, quota *Quota\n\tfor ino >= RootInode {\n\t\tino, quota = m.getQuotaParent(ctx, ino)\n\t\tif quota == nil {\n\t\t\tbreak\n\t\t}\n\t\tq := quota.snap()\n\t\tq.sanitize()\n\t\tif usage == nil {\n\t\t\tusage = &q\n\t\t}\n\t\tif q.MaxSpace > 0 {\n\t\t\tls := uint64(q.MaxSpace - q.UsedSpace)\n\t\t\tif ls < *availspace {\n\t\t\t\t*availspace = ls\n\t\t\t}\n\t\t}\n\t\tif q.MaxInodes > 0 {\n\t\t\tli := uint64(q.MaxInodes - q.UsedInodes)\n\t\t\tif li < *iavail {\n\t\t\t\t*iavail = li\n\t\t\t}\n\t\t}\n\t\tif ino == RootInode {\n\t\t\tbreak\n\t\t}\n\t\tif parent, st := m.getDirParent(ctx, ino); st != 0 {\n\t\t\tlogger.Warnf(\"Get directory parent of inode %d: %s\", ino, st)\n\t\t\tbreak\n\t\t} else {\n\t\t\tino = parent\n\t\t}\n\t}\n\tif usage != nil {\n\t\t*totalspace = uint64(usage.UsedSpace) + *availspace\n\t\t*iused = uint64(usage.UsedInodes)\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) statRootFs(ctx Context, totalspace, availspace, iused, iavail *uint64) syscall.Errno {\n\tused, inodes := atomic.LoadInt64(&m.usedSpace), atomic.LoadInt64(&m.usedInodes)\n\tvar err error\n\tif !m.conf.FastStatfs || used == unknownUsage || inodes == unknownUsage {\n\t\tvar remoteUsed int64 // using an additional variable here to ensure the assignment inside `utils.WithTimeout` does not change the `used` variable again after a timeout.\n\t\terr = utils.WithTimeout(ctx, func(context.Context) error {\n\t\t\tremoteUsed, err = m.en.getCounter(usedSpace)\n\t\t\treturn err\n\t\t}, time.Millisecond*150)\n\t\tif err == nil {\n\t\t\tused = remoteUsed\n\t\t}\n\t\tvar remoteInodes int64\n\t\terr = utils.WithTimeout(ctx, func(context.Context) error {\n\t\t\tremoteInodes, err = m.en.getCounter(totalInodes)\n\t\t\treturn err\n\t\t}, time.Millisecond*150)\n\t\tif err == nil {\n\t\t\tinodes = remoteInodes\n\t\t}\n\t}\n\n\tused += atomic.LoadInt64(&m.newSpace)\n\tinodes += atomic.LoadInt64(&m.newInodes)\n\tif used < 0 {\n\t\tused = 0\n\t}\n\tformat := m.getFormat()\n\tif format.Capacity > 0 {\n\t\t*totalspace = format.Capacity\n\t\tif *totalspace < uint64(used) {\n\t\t\t*totalspace = uint64(used)\n\t\t}\n\t} else {\n\t\t*totalspace = 1 << 50\n\t\tconst maxVal = math.MaxUint64 >> 1\n\t\tfor *totalspace*8 < uint64(used)*10 {\n\t\t\tif *totalspace >= maxVal {\n\t\t\t\t*totalspace = math.MaxUint64\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t*totalspace <<= 1\n\t\t}\n\t}\n\t*availspace = *totalspace - uint64(used)\n\tif inodes < 0 {\n\t\tinodes = 0\n\t}\n\t*iused = uint64(inodes)\n\tif format.Inodes > 0 {\n\t\tif *iused > format.Inodes {\n\t\t\t*iavail = 0\n\t\t} else {\n\t\t\t*iavail = format.Inodes - *iused\n\t\t}\n\t} else {\n\t\t*iavail = 10 << 20\n\t\tconst maxVal = math.MaxUint64 >> 1\n\t\tfor *iused > *iavail*4 {\n\t\t\tif *iavail >= maxVal {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t*iavail <<= 1\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) resolveCase(ctx Context, parent Ino, name string) *Entry {\n\tvar entries []*Entry\n\t_ = m.en.doReaddir(ctx, parent, 0, &entries, -1)\n\tfor _, e := range entries {\n\t\tn := string(e.Name)\n\t\tif strings.EqualFold(name, n) {\n\t\t\treturn e\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *baseMeta) Lookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr, checkPerm bool) syscall.Errno {\n\tif inode == nil || attr == nil {\n\t\treturn syscall.EINVAL // bad request\n\t}\n\tdefer m.timeit(\"Lookup\", time.Now())\n\tparent = m.checkRoot(parent)\n\tif checkPerm {\n\t\tif st := m.Access(ctx, parent, MODE_MASK_X, nil); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\tif name == \"..\" {\n\t\tif parent == m.root {\n\t\t\tname = \".\"\n\t\t} else {\n\t\t\tif st := m.GetAttr(ctx, parent, attr); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t\tif attr.Typ != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t\t*inode = attr.Parent\n\t\t\treturn m.GetAttr(ctx, *inode, attr)\n\t\t}\n\t}\n\tif name == \".\" {\n\t\tif st := m.GetAttr(ctx, parent, attr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\t*inode = parent\n\t\treturn 0\n\t}\n\tif parent == RootInode && name == TrashName {\n\t\tif st := m.GetAttr(ctx, TrashInode, attr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\t*inode = TrashInode\n\t\treturn 0\n\t}\n\tst := m.en.doLookup(ctx, parent, name, inode, attr)\n\tif st == syscall.ENOENT && m.conf.CaseInsensi {\n\t\tif e := m.resolveCase(ctx, parent, name); e != nil {\n\t\t\t*inode = e.Inode\n\t\t\tif st = m.GetAttr(ctx, *inode, attr); st == syscall.ENOENT {\n\t\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", e.Inode, parent, e.Name)\n\t\t\t\t*attr = *e.Attr\n\t\t\t\tst = 0\n\t\t\t}\n\t\t}\n\t}\n\tif st == 0 && attr.Typ == TypeDirectory && !parent.IsTrash() {\n\t\tm.parentMu.Lock()\n\t\tm.dirParents[*inode] = parent\n\t\tm.parentMu.Unlock()\n\t}\n\treturn st\n}\n\nfunc (attr *Attr) reset() {\n\tattr.Flags = 0\n\tattr.Mode = 0\n\tattr.Typ = 0\n\tattr.Uid = 0\n\tattr.Gid = 0\n\tattr.Atime = 0\n\tattr.Atimensec = 0\n\tattr.Mtime = 0\n\tattr.Mtimensec = 0\n\tattr.Ctime = 0\n\tattr.Ctimensec = 0\n\tattr.Nlink = 0\n\tattr.Length = 0\n\tattr.Rdev = 0\n\tattr.Parent = 0\n\tattr.AccessACL = aclAPI.None\n\tattr.DefaultACL = aclAPI.None\n\tattr.Full = false\n}\n\nfunc (m *baseMeta) parseAttr(buf []byte, attr *Attr) {\n\tattr.Unmarshal(buf)\n}\n\nfunc (m *baseMeta) marshal(attr *Attr) []byte {\n\treturn attr.Marshal()\n}\n\nfunc (m *baseMeta) encodeDelayedSlice(id uint64, size uint32) []byte {\n\tw := utils.NewBuffer(8 + 4)\n\tw.Put64(id)\n\tw.Put32(size)\n\treturn w.Bytes()\n}\n\nfunc (m *baseMeta) decodeDelayedSlices(buf []byte, ss *[]Slice) {\n\tif len(buf) == 0 || len(buf)%12 != 0 {\n\t\treturn\n\t}\n\tfor rb := utils.FromBuffer(buf); rb.HasMore(); {\n\t\t*ss = append(*ss, Slice{Id: rb.Get64(), Size: rb.Get32()})\n\t}\n}\n\nfunc clearSUGID(ctx Context, cur *Attr, set *Attr) {\n\tswitch runtime.GOOS {\n\tcase \"darwin\":\n\t\tif ctx.Uid() != 0 {\n\t\t\t// clear SUID and SGID\n\t\t\tcur.Mode &= 01777\n\t\t\tset.Mode &= 01777\n\t\t}\n\tcase \"linux\":\n\t\t// same as ext\n\t\tif cur.Typ != TypeDirectory {\n\t\t\tif ctx.Uid() != 0 || (cur.Mode>>3)&1 != 0 {\n\t\t\t\t// clear SUID and SGID\n\t\t\t\tcur.Mode &= 01777\n\t\t\t\tset.Mode &= 01777\n\t\t\t} else {\n\t\t\t\t// keep SGID if the file is non-group-executable\n\t\t\t\tcur.Mode &= 03777\n\t\t\t\tset.Mode &= 03777\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (r *baseMeta) Resolve(ctx Context, parent Ino, path string, inode *Ino, attr *Attr) syscall.Errno {\n\treturn syscall.ENOTSUP\n}\n\nfunc (m *baseMeta) Access(ctx Context, inode Ino, mmask uint8, attr *Attr) syscall.Errno {\n\tif ctx.Uid() == 0 {\n\t\treturn 0\n\t}\n\tif !ctx.CheckPermission() {\n\t\treturn 0\n\t}\n\n\tif attr == nil || !attr.Full {\n\t\tif attr == nil {\n\t\t\tattr = &Attr{}\n\t\t}\n\t\terr := m.GetAttr(ctx, inode, attr)\n\t\tif err != 0 {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// ref: https://github.com/torvalds/linux/blob/e5eb28f6d1afebed4bb7d740a797d0390bd3a357/fs/namei.c#L352-L357\n\t// dont check acl if mask is 0\n\tif attr.AccessACL != aclAPI.None && (attr.Mode&00070) != 0 {\n\t\trule := &aclAPI.Rule{}\n\t\tif st := m.en.doGetFacl(ctx, inode, aclAPI.TypeAccess, attr.AccessACL, rule); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif rule.CanAccess(ctx.Uid(), ctx.Gids(), attr.Uid, attr.Gid, mmask) {\n\t\t\treturn 0\n\t\t}\n\t\treturn syscall.EACCES\n\t}\n\n\tmode := accessMode(attr, ctx.Uid(), ctx.Gids())\n\tif mode&mmask != mmask {\n\t\tlogger.Debugf(\"Access inode %d %o, mode %o, request mode %o\", inode, attr.Mode, mode, mmask)\n\t\treturn syscall.EACCES\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) GetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno {\n\tinode = m.checkRoot(inode)\n\tif m.conf.OpenCache > 0 && m.of.Check(inode, attr) {\n\t\treturn 0\n\t}\n\tdefer m.timeit(\"GetAttr\", time.Now())\n\tvar err syscall.Errno\n\tif inode == RootInode || inode == TrashInode {\n\t\t// doGetAttr could overwrite the `attr` after timeout\n\t\tvar a Attr\n\t\te := utils.WithTimeout(ctx, func(context.Context) error {\n\t\t\terr = m.en.doGetAttr(ctx, inode, &a)\n\t\t\treturn nil\n\t\t}, time.Millisecond*300)\n\t\tif e == nil && err == 0 {\n\t\t\t*attr = a\n\t\t} else {\n\t\t\terr = 0\n\t\t\tattr.Typ = TypeDirectory\n\t\t\tattr.Mode = 0777\n\t\t\tattr.Nlink = 2\n\t\t\tattr.Length = 4 << 10\n\t\t\tif inode == TrashInode {\n\t\t\t\tattr.Mode = 0555\n\t\t\t}\n\t\t\tattr.Parent = RootInode\n\t\t\tattr.Full = true\n\t\t}\n\t} else {\n\t\terr = m.en.doGetAttr(ctx, inode, attr)\n\t}\n\tif err == 0 {\n\t\tm.of.Update(inode, attr)\n\t\tif attr.Typ == TypeDirectory && inode != RootInode && !attr.Parent.IsTrash() {\n\t\t\tm.parentMu.Lock()\n\t\t\tm.dirParents[inode] = attr.Parent\n\t\t\tm.parentMu.Unlock()\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *baseMeta) SetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr) syscall.Errno {\n\tdefer m.timeit(\"SetAttr\", time.Now())\n\tinode = m.checkRoot(inode)\n\tvar oldAttr Attr\n\n\terr := m.en.doSetAttr(ctx, inode, set, sugidclearmode, attr, &oldAttr)\n\tif err == 0 {\n\t\tm.of.InvalidateChunk(inode, invalidateAttrOnly)\n\t\tm.of.Update(inode, attr)\n\n\t\tuidChanged := oldAttr.Uid != attr.Uid\n\t\tgidChanged := oldAttr.Gid != attr.Gid\n\t\tif uidChanged || gidChanged {\n\t\t\tvar space, inodes int64\n\t\t\tif attr.Typ == TypeFile {\n\t\t\t\tspace = align4K(attr.Length)\n\t\t\t\tinodes = 1\n\t\t\t} else if attr.Typ == TypeDirectory {\n\t\t\t\tspace = align4K(0)\n\t\t\t\tinodes = 1\n\t\t\t}\n\n\t\t\tif uidChanged {\n\t\t\t\tm.updateUserGroupStat(ctx, oldAttr.Uid, 0, -space, -inodes)\n\t\t\t\tm.updateUserGroupStat(ctx, attr.Uid, 0, space, inodes)\n\t\t\t}\n\t\t\tif gidChanged {\n\t\t\t\tm.updateUserGroupStat(ctx, 0, oldAttr.Gid, -space, -inodes)\n\t\t\t\tm.updateUserGroupStat(ctx, 0, attr.Gid, space, inodes)\n\t\t\t}\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *baseMeta) nextInode() (Ino, error) {\n\tm.freeMu.Lock()\n\tdefer m.freeMu.Unlock()\n\tif m.freeInodes.next >= m.freeInodes.maxid {\n\n\t\tm.prefetchMu.Lock() // Wait until prefetchInodes() is done\n\t\tif m.prefetchedInodes.maxid > m.freeInodes.maxid {\n\t\t\tm.freeInodes = m.prefetchedInodes\n\t\t\tm.prefetchedInodes = freeID{}\n\t\t}\n\t\tm.prefetchMu.Unlock()\n\n\t\tif m.freeInodes.next >= m.freeInodes.maxid { // Prefetch missed, try again\n\t\t\tnextInodes, err := m.allocateInodes()\n\t\t\tif err != nil {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t\tm.freeInodes = nextInodes\n\t\t}\n\t}\n\tn := m.freeInodes.next\n\tm.freeInodes.next++\n\tfor n <= 1 {\n\t\tn = m.freeInodes.next\n\t\tm.freeInodes.next++\n\t}\n\tif m.freeInodes.maxid-m.freeInodes.next == inodeNeedPrefetch {\n\t\tgo m.prefetchInodes()\n\t}\n\treturn Ino(n), nil\n}\n\nfunc (m *baseMeta) prefetchInodes() {\n\tm.prefetchMu.Lock()\n\tdefer m.prefetchMu.Unlock()\n\tif m.prefetchedInodes.maxid > m.freeInodes.maxid {\n\t\treturn // Someone else has done the job\n\t}\n\tnextInodes, err := m.allocateInodes()\n\tif err == nil {\n\t\tm.prefetchedInodes = nextInodes\n\t} else {\n\t\tlogger.Warnf(\"Failed to prefetch inodes: %s, current limit: %d\", err, m.freeInodes.maxid)\n\t}\n}\n\nfunc (m *baseMeta) allocateInodes() (freeID, error) {\n\tv, err := m.en.incrCounter(\"nextInode\", inodeBatch)\n\tif err != nil {\n\t\treturn freeID{}, err\n\t}\n\treturn freeID{next: uint64(v) - inodeBatch, maxid: uint64(v)}, nil\n}\n\nfunc (m *baseMeta) Mknod(ctx Context, parent Ino, name string, _type uint8, mode, cumask uint16, rdev uint32, path string, inode *Ino, attr *Attr) syscall.Errno {\n\tif _type < TypeFile || _type > TypeSocket {\n\t\treturn syscall.EINVAL\n\t}\n\tif parent.IsTrash() {\n\t\treturn syscall.EPERM\n\t}\n\tif parent == RootInode && name == TrashName {\n\t\treturn syscall.EPERM\n\t}\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tif name == \".\" || name == \"..\" {\n\t\treturn syscall.EEXIST\n\t}\n\tif errno := checkInodeName(name); errno != 0 {\n\t\treturn errno\n\t}\n\n\tdefer m.timeit(\"Mknod\", time.Now())\n\tparent = m.checkRoot(parent)\n\tvar space, inodes int64 = align4K(0), 1\n\tif err := m.checkQuota(ctx, space, inodes, ctx.Uid(), ctx.Gid(), parent); err != 0 {\n\t\treturn err\n\t}\n\n\tino, err := m.nextInode()\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tif inode == nil {\n\t\tinode = &ino\n\t}\n\t*inode = ino\n\tif attr == nil {\n\t\tattr = &Attr{}\n\t}\n\tattr.Typ = _type\n\tattr.Uid = ctx.Uid()\n\tattr.Gid = ctx.Gid()\n\tif _type == TypeDirectory {\n\t\tattr.Nlink = 2\n\t\tattr.Length = 4 << 10\n\t} else {\n\t\tattr.Nlink = 1\n\t\tif _type == TypeSymlink {\n\t\t\tattr.Length = uint64(len(path))\n\t\t} else {\n\t\t\tattr.Length = 0\n\t\t\tattr.Rdev = rdev\n\t\t}\n\t}\n\tattr.Parent = parent\n\tattr.Full = true\n\tst := m.en.doMknod(ctx, parent, name, _type, mode, cumask, path, inode, attr)\n\tif st == 0 {\n\t\tm.en.updateStats(space, inodes)\n\t\tm.updateDirStat(ctx, parent, 0, space, inodes)\n\t\tm.updateDirQuota(ctx, parent, space, inodes)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, space, inodes)\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) Create(ctx Context, parent Ino, name string, mode uint16, cumask uint16, flags uint32, inode *Ino, attr *Attr) syscall.Errno {\n\tif attr == nil {\n\t\tattr = &Attr{}\n\t}\n\teno := m.Mknod(ctx, parent, name, TypeFile, mode, cumask, 0, \"\", inode, attr)\n\tif eno == syscall.EEXIST && (flags&syscall.O_EXCL) == 0 && attr.Typ == TypeFile {\n\t\teno = 0\n\t}\n\tif eno == 0 && inode != nil {\n\t\tm.of.Open(*inode, attr)\n\t}\n\treturn eno\n}\n\nfunc (m *baseMeta) Mkdir(ctx Context, parent Ino, name string, mode uint16, cumask uint16, copysgid uint8, inode *Ino, attr *Attr) syscall.Errno {\n\tst := m.Mknod(ctx, parent, name, TypeDirectory, mode, cumask, 0, \"\", inode, attr)\n\tif st == 0 {\n\t\tm.parentMu.Lock()\n\t\tm.dirParents[*inode] = parent\n\t\tm.parentMu.Unlock()\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) Symlink(ctx Context, parent Ino, name string, path string, inode *Ino, attr *Attr) syscall.Errno {\n\tif len(path) == 0 || len(path) > MaxSymlink {\n\t\treturn syscall.EINVAL\n\t}\n\tfor _, c := range path {\n\t\tif c == 0 {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t}\n\t// mode of symlink is ignored in POSIX\n\treturn m.Mknod(ctx, parent, name, TypeSymlink, 0777, 0, 0, path, inode, attr)\n}\n\nfunc (m *baseMeta) Link(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno {\n\tif parent.IsTrash() {\n\t\treturn syscall.EPERM\n\t}\n\tif parent == RootInode && name == TrashName {\n\t\treturn syscall.EPERM\n\t}\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tif errno := checkInodeName(name); errno != 0 {\n\t\treturn errno\n\t}\n\tif name == \".\" || name == \"..\" {\n\t\treturn syscall.EEXIST\n\t}\n\n\tdefer m.timeit(\"Link\", time.Now())\n\tif attr == nil {\n\t\tattr = &Attr{}\n\t}\n\tparent = m.checkRoot(parent)\n\tif st := m.GetAttr(ctx, inode, attr); st != 0 {\n\t\treturn st\n\t}\n\tif attr.Typ == TypeDirectory {\n\t\treturn syscall.EPERM\n\t}\n\n\tif m.checkUserQuota(ctx, uint64(attr.Uid), 0, 1) {\n\t\treturn syscall.EDQUOT\n\t}\n\tif m.checkGroupQuota(ctx, uint64(attr.Gid), 0, 1) {\n\t\treturn syscall.EDQUOT\n\t}\n\tif m.checkDirQuota(ctx, parent, align4K(attr.Length), 1) {\n\t\treturn syscall.EDQUOT\n\t}\n\n\tdefer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }()\n\terr := m.en.doLink(ctx, inode, parent, name, attr)\n\tif err == 0 {\n\t\tm.updateDirStat(ctx, parent, int64(attr.Length), align4K(attr.Length), 1)\n\t\tm.updateDirQuota(ctx, parent, align4K(attr.Length), 1)\n\t}\n\treturn err\n}\n\nfunc (m *baseMeta) ReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno {\n\tnoatime := m.conf.AtimeMode == NoAtime || m.conf.ReadOnly\n\tif target, ok := m.symlinks.Load(inode); ok {\n\t\tif noatime {\n\t\t\t*path = target.([]byte)\n\t\t\treturn 0\n\t\t} else {\n\t\t\tbuf := target.([]byte)\n\t\t\t// ctime and mtime are ignored since symlink can't be modified\n\t\t\tatime := int64(binary.BigEndian.Uint64(buf[:8]))\n\t\t\tattr := &Attr{Atime: atime / int64(time.Second), Atimensec: uint32(atime % int64(time.Second))}\n\t\t\tif !m.atimeNeedsUpdate(attr, time.Now()) {\n\t\t\t\t*path = buf[8:]\n\t\t\t\treturn 0\n\t\t\t}\n\t\t}\n\t}\n\tdefer m.timeit(\"ReadLink\", time.Now())\n\tatime, target, err := m.en.doReadlink(ctx, inode, noatime)\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tif len(target) == 0 {\n\t\tvar attr Attr\n\t\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif attr.Typ != TypeSymlink {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\treturn syscall.EIO\n\t}\n\t*path = target\n\tif noatime {\n\t\tm.symlinks.Store(inode, target)\n\t} else {\n\t\tbuf := make([]byte, 8+len(target))\n\t\tbinary.BigEndian.PutUint64(buf[:8], uint64(atime))\n\t\tcopy(buf[8:], target)\n\t\tm.symlinks.Store(inode, buf)\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) Unlink(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno {\n\tif parent == RootInode && name == TrashName || parent.IsTrash() && ctx.Uid() != 0 {\n\t\treturn syscall.EPERM\n\t}\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\n\tdefer m.timeit(\"Unlink\", time.Now())\n\tparent = m.checkRoot(parent)\n\tvar attr Attr\n\terr := m.en.doUnlink(ctx, parent, name, &attr, skipCheckTrash...)\n\tif err == 0 {\n\t\tvar diffLength uint64\n\t\tif attr.Typ == TypeFile {\n\t\t\tdiffLength = attr.Length\n\t\t}\n\t\tm.updateDirStat(ctx, parent, -int64(diffLength), -align4K(diffLength), -1)\n\t\tif !parent.IsTrash() {\n\t\t\tm.updateDirQuota(ctx, parent, -align4K(diffLength), -1)\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *baseMeta) Rmdir(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno {\n\tif name == \".\" {\n\t\treturn syscall.EINVAL\n\t}\n\tif name == \"..\" {\n\t\treturn syscall.ENOTEMPTY\n\t}\n\tif parent == RootInode && name == TrashName || parent == TrashInode || parent.IsTrash() && ctx.Uid() != 0 {\n\t\treturn syscall.EPERM\n\t}\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\n\tdefer m.timeit(\"Rmdir\", time.Now())\n\tparent = m.checkRoot(parent)\n\tvar inode Ino\n\tvar oldAttr Attr\n\tst := m.en.doRmdir(ctx, parent, name, &inode, &oldAttr, skipCheckTrash...)\n\tif st == 0 {\n\t\tif !parent.IsTrash() {\n\t\t\tm.parentMu.Lock()\n\t\t\tdelete(m.dirParents, inode)\n\t\t\tm.parentMu.Unlock()\n\t\t}\n\t\tm.updateDirStat(ctx, parent, 0, -align4K(0), -1)\n\t\tm.updateDirQuota(ctx, parent, -align4K(0), -1)\n\t}\n\treturn st\n}\n\n// BatchUnlink delete multiple files in the same directory (case-sensitive filenames)\nfunc (m *baseMeta) BatchUnlink(ctx Context, parent Ino, entries []*Entry, count *uint64, skipCheckTrash bool) syscall.Errno {\n\tif len(entries) == 0 {\n\t\treturn 0\n\t}\n\tvar delta dirStat\n\tst := m.en.doBatchUnlink(ctx, parent, entries, &delta, skipCheckTrash)\n\tif st == 0 {\n\t\tm.updateDirStat(ctx, parent, delta.length, delta.space, delta.inodes)\n\t\tif !parent.IsTrash() {\n\t\t\tm.updateDirQuota(ctx, parent, delta.space, delta.inodes)\n\t\t}\n\t\tif count != nil && len(entries) > 0 {\n\t\t\tatomic.AddUint64(count, uint64(len(entries)))\n\t\t}\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) BatchClone(ctx Context, srcParent Ino, dstParent Ino, entries []*Entry, cmode uint8, cumask uint16, count *uint64) syscall.Errno {\n\tif len(entries) == 0 {\n\t\treturn 0\n\t}\n\tvar r batchCloneResult\n\tst := m.en.doBatchClone(ctx, srcParent, dstParent, entries, cmode, cumask, &r)\n\tif st == 0 {\n\t\tm.en.updateStats(r.space, r.inodes)\n\t\tm.updateDirQuota(ctx, dstParent, r.space, r.inodes)\n\t\t// TODO\n\t\tfor _, q := range r.deltas {\n\t\t\tm.updateUserGroupStat(ctx, q.Uid, q.Gid, q.Space, q.Inodes)\n\t\t}\n\t\tif count != nil {\n\t\t\tatomic.AddUint64(count, uint64(r.inodes))\n\t\t}\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) Rename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode *Ino, attr *Attr) syscall.Errno {\n\tif parentSrc == RootInode && nameSrc == TrashName || parentDst == RootInode && nameDst == TrashName {\n\t\treturn syscall.EPERM\n\t}\n\tif parentDst.IsTrash() || parentSrc.IsTrash() && ctx.Uid() != 0 {\n\t\treturn syscall.EPERM\n\t}\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tif errno := checkInodeName(nameDst); errno != 0 {\n\t\treturn errno\n\t}\n\n\tswitch flags {\n\tcase 0, RenameNoReplace, RenameExchange, RenameNoReplace | RenameRestore:\n\tcase RenameWhiteout, RenameNoReplace | RenameWhiteout:\n\t\treturn syscall.ENOTSUP\n\tdefault:\n\t\treturn syscall.EINVAL\n\t}\n\n\tdefer m.timeit(\"Rename\", time.Now())\n\tif inode == nil {\n\t\tinode = new(Ino)\n\t}\n\tif attr == nil {\n\t\tattr = &Attr{}\n\t}\n\tparentSrc = m.checkRoot(parentSrc)\n\tparentDst = m.checkRoot(parentDst)\n\tvar quotaSrc, quotaDst Ino\n\tif !parentSrc.IsTrash() {\n\t\tquotaSrc, _ = m.getQuotaParent(ctx, parentSrc)\n\t}\n\tif parentSrc == parentDst {\n\t\tquotaDst = quotaSrc\n\t} else {\n\t\tquotaDst, _ = m.getQuotaParent(ctx, parentDst)\n\t}\n\tvar space, inodes int64\n\tif quotaSrc != quotaDst {\n\t\tif st := m.Lookup(ctx, parentSrc, nameSrc, inode, attr, false); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif attr.Typ == TypeDirectory {\n\t\t\tm.quotaMu.RLock()\n\t\t\tq := m.dirQuotas[uint64(*inode)]\n\t\t\tm.quotaMu.RUnlock()\n\t\t\tif q != nil {\n\t\t\t\tspace, inodes = q.UsedSpace+align4K(0), q.UsedInodes+1\n\t\t\t} else {\n\t\t\t\tvar sum Summary\n\t\t\t\tlogger.Debugf(\"Start to get summary of inode %d\", *inode)\n\t\t\t\tif st := m.GetSummary(ctx, *inode, &sum, true, false); st != 0 {\n\t\t\t\t\tlogger.Warnf(\"Get summary of inode %d: %s\", *inode, st)\n\t\t\t\t\treturn st\n\t\t\t\t}\n\t\t\t\tspace, inodes = int64(sum.Size), int64(sum.Dirs+sum.Files)\n\t\t\t}\n\t\t} else {\n\t\t\tspace, inodes = align4K(attr.Length), 1\n\t\t}\n\t\t// TODO: dst exists and is replaced or exchanged\n\t\tif quotaDst > 0 && m.checkDirQuota(ctx, parentDst, space, inodes) {\n\t\t\treturn syscall.EDQUOT\n\t\t}\n\t}\n\ttinode := new(Ino)\n\ttattr := new(Attr)\n\tst := m.en.doRename(ctx, parentSrc, nameSrc, parentDst, nameDst, flags, inode, tinode, attr, tattr)\n\tif st == 0 {\n\t\tvar diffLength uint64\n\t\tif attr.Typ == TypeDirectory {\n\t\t\tm.parentMu.Lock()\n\t\t\tm.dirParents[*inode] = parentDst\n\t\t\tm.parentMu.Unlock()\n\t\t} else if attr.Typ == TypeFile {\n\t\t\tdiffLength = attr.Length\n\t\t}\n\t\tif parentSrc != parentDst {\n\t\t\tm.updateDirStat(ctx, parentSrc, -int64(diffLength), -align4K(diffLength), -1)\n\t\t\tm.updateDirStat(ctx, parentDst, int64(diffLength), align4K(diffLength), 1)\n\t\t\tif quotaSrc != quotaDst {\n\t\t\t\tif quotaSrc > 0 {\n\t\t\t\t\tm.updateDirQuota(ctx, parentSrc, -space, -inodes)\n\t\t\t\t}\n\t\t\t\tif quotaDst > 0 {\n\t\t\t\t\tm.updateDirQuota(ctx, parentDst, space, inodes)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif *tinode > 0 && flags != RenameExchange {\n\t\t\tdiffLength = 0\n\t\t\tif tattr.Typ == TypeDirectory {\n\t\t\t\tm.parentMu.Lock()\n\t\t\t\tdelete(m.dirParents, *tinode)\n\t\t\t\tm.parentMu.Unlock()\n\t\t\t} else if attr.Typ == TypeFile {\n\t\t\t\tdiffLength = tattr.Length\n\t\t\t}\n\t\t\tm.updateDirStat(ctx, parentDst, -int64(diffLength), -align4K(diffLength), -1)\n\t\t\tif quotaDst > 0 {\n\t\t\t\tm.updateDirQuota(ctx, parentDst, -align4K(diffLength), -1)\n\t\t\t}\n\t\t}\n\t}\n\treturn st\n}\n\n// caller makes sure inode is not special inode.\nfunc (m *baseMeta) touchAtime(ctx Context, inode Ino, attr *Attr) {\n\tif m.conf.AtimeMode == NoAtime || m.conf.ReadOnly {\n\t\treturn\n\t}\n\n\tif attr == nil {\n\t\tattr = new(Attr)\n\t\tif of := m.of.find(inode); of != nil {\n\t\t\t*attr = of.attr\n\t\t}\n\t}\n\tnow := time.Now()\n\tif attr.Full && !m.atimeNeedsUpdate(attr, now) {\n\t\treturn\n\t}\n\n\tupdated, err := m.en.doTouchAtime(ctx, inode, attr, now)\n\tif updated {\n\t\tm.of.Update(inode, attr)\n\t} else if err != nil {\n\t\tlogger.Warnf(\"Update atime of inode %d: %s\", inode, err)\n\t}\n}\n\nfunc (m *baseMeta) Open(ctx Context, inode Ino, flags uint32, attr *Attr) (st syscall.Errno) {\n\tif m.conf.ReadOnly && flags&(syscall.O_WRONLY|syscall.O_RDWR|syscall.O_TRUNC|syscall.O_APPEND) != 0 {\n\t\treturn syscall.EROFS\n\t}\n\tdefer func() {\n\t\tif st == 0 {\n\t\t\tm.touchAtime(ctx, inode, attr)\n\t\t}\n\t}()\n\tif m.conf.OpenCache > 0 && m.of.OpenCheck(inode, attr) {\n\t\treturn 0\n\t}\n\t// attr may be valid, see fs.Open()\n\tif attr != nil && !attr.Full {\n\t\tif st = m.GetAttr(ctx, inode, attr); st != 0 {\n\t\t\treturn\n\t\t}\n\t}\n\tvar mmask uint8 = 0\n\tswitch flags & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) {\n\tcase syscall.O_RDONLY:\n\t\tmmask = MODE_MASK_R\n\t\t// 0x20 means O_FMODE_EXEC\n\t\tif (flags & 0x20) != 0 {\n\t\t\tmmask = MODE_MASK_X\n\t\t}\n\tcase syscall.O_WRONLY:\n\t\tmmask = MODE_MASK_W\n\tcase syscall.O_RDWR:\n\t\tmmask = MODE_MASK_R | MODE_MASK_W\n\t}\n\tif st = m.Access(ctx, inode, mmask, attr); st != 0 {\n\t\treturn\n\t}\n\n\tif attr.Flags&FlagImmutable != 0 || attr.Parent > TrashInode {\n\t\tif flags&(syscall.O_WRONLY|syscall.O_RDWR) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t}\n\tif attr.Flags&FlagAppend != 0 {\n\t\tif (flags&(syscall.O_WRONLY|syscall.O_RDWR)) != 0 && (flags&syscall.O_APPEND) == 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif flags&syscall.O_TRUNC != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t}\n\tm.of.Open(inode, attr)\n\treturn 0\n}\n\nfunc (m *baseMeta) InvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno {\n\tm.of.InvalidateChunk(inode, indx)\n\treturn 0\n}\n\nfunc (m *baseMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) (st syscall.Errno) {\n\tdefer func() {\n\t\tif st == 0 {\n\t\t\tm.touchAtime(ctx, inode, nil)\n\t\t}\n\t}()\n\n\tf := m.of.find(inode)\n\tif f != nil {\n\t\tf.RLock()\n\t\tdefer f.RUnlock()\n\t}\n\tif ss, ok := m.of.ReadChunk(inode, indx); ok {\n\t\t*slices = ss\n\t\treturn 0\n\t}\n\n\t*slices = nil\n\tdefer m.timeit(\"Read\", time.Now())\n\tss, st := m.en.doRead(ctx, inode, indx)\n\tif st != 0 {\n\t\treturn st\n\t}\n\tif ss == nil {\n\t\treturn syscall.EIO\n\t}\n\tif len(ss) == 0 {\n\t\tvar attr Attr\n\t\tif st = m.en.doGetAttr(ctx, inode, &attr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif attr.Typ != TypeFile {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\treturn 0\n\t}\n\n\t*slices = buildSlice(ss)\n\tm.of.CacheChunk(inode, indx, *slices)\n\tif !m.conf.ReadOnly && (len(ss) >= 5 || len(*slices) >= 5) {\n\t\tgo m.compactChunk(inode, indx, false, false)\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) NewSlice(ctx Context, id *uint64) syscall.Errno {\n\tm.freeMu.Lock()\n\tdefer m.freeMu.Unlock()\n\tif m.freeSlices.next >= m.freeSlices.maxid {\n\t\tv, err := m.en.incrCounter(\"nextChunk\", sliceIdBatch)\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\t\tm.freeSlices.next = uint64(v) - sliceIdBatch\n\t\tm.freeSlices.maxid = uint64(v)\n\t}\n\t*id = m.freeSlices.next\n\tm.freeSlices.next++\n\treturn 0\n}\n\nfunc (m *baseMeta) Close(ctx Context, inode Ino) syscall.Errno {\n\tif m.of.Close(inode) {\n\t\tm.Lock()\n\t\t_, removed := m.removedFiles[inode]\n\t\tif removed {\n\t\t\tdelete(m.removedFiles, inode)\n\t\t}\n\t\tm.Unlock()\n\t\tif removed {\n\t\t\t_ = m.en.doDeleteSustainedInode(m.sid, inode)\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno {\n\tdefer m.timeit(\"Write\", time.Now())\n\tf := m.of.find(inode)\n\tif f != nil {\n\t\tf.Lock()\n\t\tdefer f.Unlock()\n\t}\n\tdefer func() { m.of.InvalidateChunk(inode, indx) }()\n\tvar numSlices int\n\tvar delta dirStat\n\tvar attr Attr\n\tst := m.en.doWrite(ctx, inode, indx, off, slice, mtime, &numSlices, &delta, &attr)\n\tif st == 0 {\n\t\tm.updateParentStat(ctx, inode, attr.Parent, delta.length, delta.space)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, delta.space, 0)\n\t\tif numSlices%100 == 99 || numSlices > 350 {\n\t\t\tif numSlices < maxSlices {\n\t\t\t\tgo m.compactChunk(inode, indx, false, false)\n\t\t\t} else {\n\t\t\t\tm.compactChunk(inode, indx, true, false)\n\t\t\t}\n\t\t}\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) Truncate(ctx Context, inode Ino, flags uint8, length uint64, attr *Attr, skipPermCheck bool) syscall.Errno {\n\tdefer m.timeit(\"Truncate\", time.Now())\n\tf := m.of.find(inode)\n\tif f != nil {\n\t\tf.Lock()\n\t\tdefer f.Unlock()\n\t}\n\tdefer func() { m.of.InvalidateChunk(inode, invalidateAllChunks) }()\n\tif attr == nil {\n\t\tattr = &Attr{}\n\t}\n\tvar delta dirStat\n\tst := m.en.doTruncate(ctx, inode, flags, length, &delta, attr, skipPermCheck)\n\tif st == 0 {\n\t\tm.updateParentStat(ctx, inode, attr.Parent, delta.length, delta.space)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, delta.space, 0)\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, flength *uint64) syscall.Errno {\n\tif mode&fallocCollapesRange != 0 && mode != fallocCollapesRange {\n\t\treturn syscall.EINVAL\n\t}\n\tif mode&fallocInsertRange != 0 && mode != fallocInsertRange {\n\t\treturn syscall.EINVAL\n\t}\n\tif mode == fallocInsertRange || mode == fallocCollapesRange {\n\t\treturn syscall.ENOTSUP\n\t}\n\tif mode&fallocPunchHole != 0 && mode&fallocKeepSize == 0 {\n\t\treturn syscall.EINVAL\n\t}\n\tif size == 0 {\n\t\treturn syscall.EINVAL\n\t}\n\tdefer m.timeit(\"Fallocate\", time.Now())\n\tf := m.of.find(inode)\n\tif f != nil {\n\t\tf.Lock()\n\t\tdefer f.Unlock()\n\t}\n\tdefer func() { m.of.InvalidateChunk(inode, invalidateAllChunks) }()\n\tvar delta dirStat\n\tvar attr Attr\n\tst := m.en.doFallocate(ctx, inode, mode, off, size, &delta, &attr)\n\tif st == 0 {\n\t\tif flength != nil {\n\t\t\t*flength = attr.Length\n\t\t}\n\t\tm.updateParentStat(ctx, inode, attr.Parent, delta.length, delta.space)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, delta.space, 0)\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) Readdir(ctx Context, inode Ino, plus uint8, entries *[]*Entry) (rerr syscall.Errno) {\n\tvar attr Attr\n\tdefer func() {\n\t\tif rerr == 0 {\n\t\t\tm.touchAtime(ctx, inode, &attr)\n\t\t}\n\t}()\n\tinode = m.checkRoot(inode)\n\tif err := m.GetAttr(ctx, inode, &attr); err != 0 {\n\t\treturn err\n\t}\n\tdefer m.timeit(\"Readdir\", time.Now())\n\tvar mmask uint8 = MODE_MASK_R\n\tif plus != 0 {\n\t\tmmask |= MODE_MASK_X\n\t}\n\tif st := m.Access(ctx, inode, mmask, &attr); st != 0 {\n\t\treturn st\n\t}\n\tif inode == m.root {\n\t\tattr.Parent = m.root\n\t}\n\t*entries = []*Entry{\n\t\t{\n\t\t\tInode: inode,\n\t\t\tName: []byte(\".\"),\n\t\t\tAttr: &Attr{Typ: TypeDirectory},\n\t\t},\n\t}\n\t*entries = append(*entries, &Entry{\n\t\tInode: attr.Parent,\n\t\tName: []byte(\"..\"),\n\t\tAttr: &Attr{Typ: TypeDirectory},\n\t})\n\tst := m.en.doReaddir(ctx, inode, plus, entries, -1)\n\tif st == syscall.ENOENT && inode == TrashInode {\n\t\tst = 0\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) SetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tif name == \"\" {\n\t\treturn syscall.EINVAL\n\t}\n\tswitch flags {\n\tcase 0, XattrCreate, XattrReplace:\n\tdefault:\n\t\treturn syscall.EINVAL\n\t}\n\n\tdefer m.timeit(\"SetXattr\", time.Now())\n\treturn m.en.doSetXattr(ctx, m.checkRoot(inode), name, value, flags)\n}\n\nfunc (m *baseMeta) RemoveXattr(ctx Context, inode Ino, name string) syscall.Errno {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tif name == \"\" {\n\t\treturn syscall.EINVAL\n\t}\n\n\tdefer m.timeit(\"RemoveXattr\", time.Now())\n\treturn m.en.doRemoveXattr(ctx, m.checkRoot(inode), name)\n}\n\nfunc (m *baseMeta) GetParents(ctx Context, inode Ino) map[Ino]int {\n\tif inode == RootInode || inode == TrashInode {\n\t\treturn map[Ino]int{1: 1}\n\t}\n\tvar attr Attr\n\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\tlogger.Warnf(\"GetAttr inode %d: %s\", inode, st)\n\t\treturn nil\n\t}\n\tif attr.Parent > 0 {\n\t\treturn map[Ino]int{attr.Parent: 1}\n\t} else {\n\t\treturn m.en.doGetParents(ctx, inode)\n\t}\n}\n\nfunc (m *baseMeta) GetPaths(ctx Context, inode Ino) []string {\n\tif inode == RootInode {\n\t\treturn []string{\"/\"}\n\t}\n\n\tif inode == TrashInode {\n\t\treturn []string{\"/.trash\"}\n\t}\n\n\toutside := \"path not shown because it's outside of the mounted root\"\n\tgetDirPath := func(ino Ino) (string, error) {\n\t\tvar names []string\n\t\tvar attr Attr\n\t\tfor ino != RootInode && ino != m.root {\n\t\t\tif st := m.en.doGetAttr(ctx, ino, &attr); st != 0 {\n\t\t\t\treturn \"\", fmt.Errorf(\"getattr inode %d: %s\", ino, st)\n\t\t\t}\n\t\t\tif attr.Typ != TypeDirectory {\n\t\t\t\treturn \"\", fmt.Errorf(\"inode %d is not a directory\", ino)\n\t\t\t}\n\t\t\tvar entries []*Entry\n\t\t\tif st := m.en.doReaddir(ctx, attr.Parent, 0, &entries, -1); st != 0 {\n\t\t\t\treturn \"\", fmt.Errorf(\"readdir inode %d: %s\", ino, st)\n\t\t\t}\n\t\t\tvar name string\n\t\t\tfor _, e := range entries {\n\t\t\t\tif e.Inode == ino {\n\t\t\t\t\tname = string(e.Name)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif attr.Parent == RootInode && ino == TrashInode {\n\t\t\t\tname = TrashName\n\t\t\t}\n\t\t\tif name == \"\" {\n\t\t\t\treturn \"\", fmt.Errorf(\"entry %d/%d not found\", attr.Parent, ino)\n\t\t\t}\n\t\t\tnames = append(names, name)\n\t\t\tino = attr.Parent\n\t\t}\n\t\tif m.root != RootInode && ino == RootInode {\n\t\t\treturn outside, nil\n\t\t}\n\t\tnames = append(names, \"/\") // add root\n\n\t\tfor i, j := 0, len(names)-1; i < j; i, j = i+1, j-1 { // reverse\n\t\t\tnames[i], names[j] = names[j], names[i]\n\t\t}\n\t\treturn path.Join(names...), nil\n\t}\n\n\tvar paths []string\n\t// inode != RootInode, parent is the real parent inode\n\tfor parent, count := range m.GetParents(ctx, inode) {\n\t\tif count <= 0 {\n\t\t\tcontinue\n\t\t}\n\t\tdir, err := getDirPath(parent)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"Get directory path of %d: %s\", parent, err)\n\t\t\tcontinue\n\t\t} else if dir == outside {\n\t\t\tpaths = append(paths, outside)\n\t\t\tcontinue\n\t\t}\n\t\tvar entries []*Entry\n\t\tif st := m.en.doReaddir(ctx, parent, 0, &entries, -1); st != 0 {\n\t\t\tlogger.Warnf(\"Readdir inode %d: %s\", parent, st)\n\t\t\tcontinue\n\t\t}\n\t\tvar c int\n\t\tfor _, e := range entries {\n\t\t\tif e.Inode == inode {\n\t\t\t\tc++\n\t\t\t\tpaths = append(paths, path.Join(dir, string(e.Name)))\n\t\t\t}\n\t\t}\n\t\tif c != count {\n\t\t\tlogger.Warnf(\"Expect to find %d entries under parent %d, but got %d\", count, parent, c)\n\t\t}\n\t}\n\treturn paths\n}\n\nfunc (m *baseMeta) countDirNlink(ctx Context, inode Ino) (uint32, syscall.Errno) {\n\tvar entries []*Entry\n\tif st := m.en.doReaddir(ctx, inode, 0, &entries, -1); st != 0 {\n\t\treturn 0, st\n\t}\n\tvar dirCounter uint32 = 2\n\tfor _, e := range entries {\n\t\tif e.Attr.Typ == TypeDirectory {\n\t\t\tdirCounter++\n\t\t}\n\t}\n\treturn dirCounter, 0\n}\n\ntype metaWalkFunc func(ctx Context, inode Ino, p string, attr *Attr)\n\nfunc (m *baseMeta) walk(ctx Context, inode Ino, p string, attr *Attr, walkFn metaWalkFunc) syscall.Errno {\n\twalkFn(ctx, inode, p, attr)\n\tif attr.Full && attr.Typ != TypeDirectory {\n\t\treturn 0\n\t}\n\tvar entries []*Entry\n\tst := m.en.doReaddir(ctx, inode, 1, &entries, -1)\n\tif st != 0 && st != syscall.ENOENT {\n\t\tlogger.Errorf(\"list %s: %s\", p, st)\n\t\treturn st\n\t}\n\tfor _, entry := range entries {\n\t\tif ctx.Canceled() {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t\tif !entry.Attr.Full {\n\t\t\tentry.Attr.Parent = inode\n\t\t}\n\t\tif st := m.walk(ctx, entry.Inode, path.Join(p, string(entry.Name)), entry.Attr, walkFn); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) Check(ctx Context, fpath string, opt *CheckOpt) error {\n\tvar attr Attr\n\tvar inode = RootInode\n\tvar parent = RootInode\n\tattr.Typ = TypeDirectory\n\tif fpath == \"/\" {\n\t\tif st := m.GetAttr(ctx, inode, &attr); st != 0 && st != syscall.ENOENT {\n\t\t\tlogger.Errorf(\"GetAttr inode %d: %s\", inode, st)\n\t\t\treturn st\n\t\t}\n\t} else {\n\t\tps := strings.FieldsFunc(fpath, func(r rune) bool {\n\t\t\treturn r == '/'\n\t\t})\n\t\tfor i, name := range ps {\n\t\t\tparent = inode\n\t\t\tif st := m.Lookup(ctx, parent, name, &inode, &attr, false); st != 0 {\n\t\t\t\tlogger.Errorf(\"Lookup parent %d name %s: %s\", parent, name, st)\n\t\t\t\treturn st\n\t\t\t}\n\t\t\tif !attr.Full && i < len(ps)-1 {\n\t\t\t\t// missing attribute\n\t\t\t\tp := \"/\" + path.Join(ps[:i+1]...)\n\t\t\t\tif attr.Typ != TypeDirectory { // TODO: determine file size?\n\t\t\t\t\tlogger.Warnf(\"Attribute of %s (inode %d type %d) is missing and cannot be auto-repaired, please repair it manually or remove it\", p, inode, attr.Typ)\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Warnf(\"Attribute of %s (inode %d) is missing, please re-run with '--path %s --repair' to fix it\", p, inode, p)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif !attr.Full {\n\t\tattr.Parent = parent\n\t}\n\n\tprogress := utils.NewProgress(false)\n\tdefer progress.Done()\n\tnodeBar := progress.AddCountBar(\"Checked nodes\", 0)\n\n\tvar hasError bool\n\ttype node struct {\n\t\tinode Ino\n\t\tpath string\n\t\tattr *Attr\n\t}\n\tnodes := make(chan *node, 1000)\n\tgo func() {\n\t\tdefer close(nodes)\n\t\tvar count int64\n\t\tif opt.Recursive {\n\t\t\tif st := m.walk(ctx, inode, fpath, &attr, func(ctx Context, inode Ino, path string, attr *Attr) {\n\t\t\t\tnodes <- &node{inode, path, attr}\n\t\t\t\tatomic.AddInt64(&count, 1)\n\t\t\t}); st != 0 {\n\t\t\t\thasError = true\n\t\t\t\tlogger.Errorf(\"Walk %s: %s\", fpath, st)\n\t\t\t}\n\t\t} else {\n\t\t\tnodes <- &node{inode, fpath, &attr}\n\t\t\tcount = 1\n\t\t}\n\t\tnodeBar.SetTotal(count)\n\t}()\n\n\tformat, err := m.Load(false)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"load meta format\")\n\t}\n\tif opt.SyncDirStat && !format.DirStats {\n\t\tlogger.Warn(\"dir stats is disabled, flag '--sync-dir-stat' will be ignored\")\n\t}\n\tvar lock sync.Mutex\n\tlistSlices := func(inode Ino, path string) {\n\t\tlock.Lock()\n\t\tif _, ok := opt.Slices[inode]; ok {\n\t\t\tlock.Unlock()\n\t\t\treturn\n\t\t}\n\t\topt.Slices[inode] = []Slice{}\n\t\tlock.Unlock()\n\t\trawSlices, st := m.en.doList(ctx, inode)\n\t\tif st != 0 {\n\t\t\tlogger.Errorf(\"dolist %s: %s\", path, st)\n\t\t\treturn\n\t\t}\n\t\tss := make([]Slice, 0, len(rawSlices))\n\t\tfor _, rs := range rawSlices {\n\t\t\tif rs.id > 0 {\n\t\t\t\tss = append(ss, Slice{Id: rs.id, Size: rs.size})\n\t\t\t}\n\t\t}\n\t\tlock.Lock()\n\t\topt.Slices[inode] = ss\n\t\tif opt.ShowProgress != nil {\n\t\t\topt.ShowProgress(len(opt.Slices[inode]))\n\t\t}\n\t\tlock.Unlock()\n\t}\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 20; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor e := range nodes {\n\t\t\t\tinode := e.inode\n\t\t\t\tpath := e.path\n\t\t\t\tattr := e.attr\n\t\t\t\tif attr.Typ != TypeDirectory {\n\t\t\t\t\tif attr.Typ == TypeFile {\n\t\t\t\t\t\tlistSlices(inode, path)\n\t\t\t\t\t\tnodeBar.Increment()\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tvar attrBroken, statBroken bool\n\t\t\t\tif attr.Full {\n\t\t\t\t\tnlink, st := m.countDirNlink(ctx, inode)\n\t\t\t\t\tif st == syscall.ENOENT {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif st != 0 {\n\t\t\t\t\t\thasError = true\n\t\t\t\t\t\tlogger.Errorf(\"Count nlink for inode %d: %s\", inode, st)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif attr.Nlink != nlink {\n\t\t\t\t\t\tlogger.Warnf(\"nlink of %s should be %d, but got %d\", path, nlink, attr.Nlink)\n\t\t\t\t\t\tattrBroken = true\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Warnf(\"attribute of %s is missing\", path)\n\t\t\t\t\tattrBroken = true\n\t\t\t\t}\n\n\t\t\t\tif attrBroken {\n\t\t\t\t\tif opt.Repair {\n\t\t\t\t\t\tif !attr.Full {\n\t\t\t\t\t\t\tnow := time.Now().Unix()\n\t\t\t\t\t\t\tattr.Mode = opt.RepairDirMode\n\t\t\t\t\t\t\tattr.Uid = ctx.Uid()\n\t\t\t\t\t\t\tattr.Gid = ctx.Gid()\n\t\t\t\t\t\t\tattr.Atime = now\n\t\t\t\t\t\t\tattr.Mtime = now\n\t\t\t\t\t\t\tattr.Ctime = now\n\t\t\t\t\t\t\tattr.Length = 4 << 10\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif st1 := m.en.doRepair(ctx, inode, attr); st1 == 0 || st1 == syscall.ENOENT {\n\t\t\t\t\t\t\tlogger.Debugf(\"Path %s (inode %d) is successfully repaired\", path, inode)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\thasError = true\n\t\t\t\t\t\t\tlogger.Errorf(\"Repair path %s inode %d: %s\", path, inode, st1)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlogger.Warnf(\"Path %s (inode %d) can be repaired, please re-run with '--path %s --repair' to fix it\", path, inode, path)\n\t\t\t\t\t\thasError = true\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif format.DirStats {\n\t\t\t\t\tstat, st := m.en.doGetDirStat(ctx, inode, false)\n\t\t\t\t\tif st == syscall.ENOENT {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif st != 0 {\n\t\t\t\t\t\thasError = true\n\t\t\t\t\t\tlogger.Errorf(\"get dir stat for inode %d: %v\", inode, st)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif stat == nil || stat.space < 0 || stat.inodes < 0 {\n\t\t\t\t\t\tlogger.Warnf(\"usage stat of %s is missing or broken\", path)\n\t\t\t\t\t\tstatBroken = true\n\t\t\t\t\t}\n\n\t\t\t\t\tif !opt.Repair && opt.SyncDirStat {\n\t\t\t\t\t\ts, st := m.calcDirStat(ctx, inode)\n\t\t\t\t\t\tif st != 0 {\n\t\t\t\t\t\t\thasError = true\n\t\t\t\t\t\t\tlogger.Errorf(\"calc dir stat for inode %d: %v\", inode, st)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif stat.space != s.space || stat.inodes != s.inodes {\n\t\t\t\t\t\t\tlogger.Warnf(\"usage stat of %s should be %v, but got %v\", path, s, stat)\n\t\t\t\t\t\t\tstatBroken = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tif opt.Repair {\n\t\t\t\t\t\tif statBroken || opt.SyncDirStat {\n\t\t\t\t\t\t\tif _, st := m.en.doSyncDirStat(ctx, inode); st == 0 || st == syscall.ENOENT {\n\t\t\t\t\t\t\t\tlogger.Debugf(\"Stat of path %s (inode %d) is successfully synced\", path, inode)\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\thasError = true\n\t\t\t\t\t\t\t\tlogger.Errorf(\"Sync stat of path %s inode %d: %s\", path, inode, st)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if statBroken {\n\t\t\t\t\t\tlogger.Warnf(\"Stat of path %s (inode %d) should be synced, please re-run with '--path %s --repair --sync-dir-stat' to fix it\", path, inode, path)\n\t\t\t\t\t\thasError = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tnodeBar.Increment()\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tif fpath == \"/\" && opt.Repair && opt.Recursive && opt.SyncDirStat {\n\t\tif err := m.syncVolumeStat(ctx); err != nil {\n\t\t\tlogger.Errorf(\"Sync used space: %s\", err)\n\t\t\thasError = true\n\t\t}\n\t}\n\tif hasError {\n\t\treturn errors.New(\"some errors occurred, please check the log of fsck\")\n\t}\n\n\tif progress.Quiet {\n\t\tlogger.Infof(\"Checked %d nodes\", nodeBar.Current())\n\t}\n\n\treturn nil\n}\n\nfunc (m *baseMeta) Chroot(ctx Context, subdir string) syscall.Errno {\n\tfor subdir != \"\" {\n\t\tps := strings.SplitN(subdir, \"/\", 2)\n\t\tif ps[0] != \"\" {\n\t\t\tvar attr Attr\n\t\t\tvar inode Ino\n\t\t\tr := m.Lookup(ctx, m.root, ps[0], &inode, &attr, true)\n\t\t\tif r == syscall.ENOENT {\n\t\t\t\tr = m.Mkdir(ctx, m.root, ps[0], 0777, 0, 0, &inode, &attr)\n\t\t\t}\n\t\t\tif r != 0 {\n\t\t\t\treturn r\n\t\t\t}\n\t\t\tif attr.Typ != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t\tm.chroot(inode)\n\t\t}\n\t\tif len(ps) == 1 {\n\t\t\tbreak\n\t\t}\n\t\tsubdir = ps[1]\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) chroot(inode Ino) {\n\tm.root = inode\n}\n\nfunc (m *baseMeta) resolve(ctx Context, dpath string, inode *Ino, create bool) syscall.Errno {\n\tvar attr Attr\n\t*inode = RootInode\n\tumask := utils.GetUmask()\n\tfor dpath != \"\" {\n\t\tps := strings.SplitN(dpath, \"/\", 2)\n\t\tif ps[0] != \"\" {\n\t\t\tr := m.en.doLookup(ctx, *inode, ps[0], inode, &attr)\n\t\t\tif errors.Is(r, syscall.ENOENT) && create {\n\t\t\t\tr = m.Mkdir(ctx, *inode, ps[0], 0777, uint16(umask), 0, inode, &attr)\n\t\t\t}\n\t\t\tif r != 0 {\n\t\t\t\treturn r\n\t\t\t}\n\t\t\tif attr.Typ != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t}\n\t\tif len(ps) == 1 {\n\t\t\tbreak\n\t\t}\n\t\tdpath = ps[1]\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) getFormat() *Format {\n\tm.Lock()\n\tdefer m.Unlock()\n\treturn m.fmt\n}\n\nfunc (m *baseMeta) GetFormat() Format {\n\treturn *m.getFormat()\n}\n\nfunc (m *baseMeta) CompactAll(ctx Context, threads int, bar *utils.Bar) syscall.Errno {\n\tvar wg sync.WaitGroup\n\tch := make(chan cchunk, 1000000)\n\tfor i := 0; i < threads; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tfor c := range ch {\n\t\t\t\tlogger.Debugf(\"Compacting chunk %d:%d (%d slices)\", c.inode, c.indx, c.slices)\n\t\t\t\tm.compactChunk(c.inode, c.indx, false, true)\n\t\t\t\tbar.Increment()\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\terr := m.en.scanAllChunks(ctx, ch, bar)\n\tclose(ch)\n\twg.Wait()\n\tif err != nil {\n\t\tlogger.Warnf(\"Scan chunks: %s\", err)\n\t\treturn errno(err)\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) {\n\t// avoid too many or duplicated compaction\n\tk := uint64(inode) + (uint64(indx) << 40)\n\tm.Lock()\n\tif m.sessCtx != nil && m.sessCtx.Canceled() {\n\t\tm.Unlock()\n\t\treturn\n\t}\n\tif once || force {\n\t\tfor m.compacting[k] {\n\t\t\tm.Unlock()\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t\tm.Lock()\n\t\t}\n\t} else if len(m.compacting) > 10 || m.compacting[k] {\n\t\tm.Unlock()\n\t\treturn\n\t}\n\tm.compacting[k] = true\n\tm.Unlock()\n\tdefer func() {\n\t\tm.Lock()\n\t\tdelete(m.compacting, k)\n\t\tm.Unlock()\n\t}()\n\n\tss, st := m.en.doRead(Background(), inode, indx)\n\tif st != 0 {\n\t\treturn\n\t}\n\tif ss == nil {\n\t\tlogger.Errorf(\"Corrupt value for inode %d chunk indx %d\", inode, indx)\n\t\treturn\n\t}\n\tif once && len(ss) < maxSlices {\n\t\treturn\n\t}\n\tif len(ss) > maxCompactSlices {\n\t\tss = ss[:maxCompactSlices]\n\t}\n\tskipped := skipSome(ss)\n\tcompacted := ss[skipped:]\n\tpos, size, slices := compactChunk(compacted)\n\tif len(compacted) < 2 || size == 0 {\n\t\treturn\n\t}\n\tfor _, s := range ss[:skipped] {\n\t\tif pos+size > s.pos && s.pos+s.len > pos {\n\t\t\tvar sstring string\n\t\t\tfor _, s := range ss {\n\t\t\t\tsstring += fmt.Sprintf(\"\\n%+v\", *s)\n\t\t\t}\n\t\t\tpanic(fmt.Sprintf(\"invalid compaction skipped %d, pos %d, size %d; slices: %s\", skipped, pos, size, sstring))\n\t\t}\n\t}\n\n\tvar id uint64\n\tif st = m.NewSlice(Background(), &id); st != 0 {\n\t\treturn\n\t}\n\tlogger.Debugf(\"compact %d:%d: skipped %d slices (%d bytes) %d slices (%d bytes)\", inode, indx, skipped, pos, len(compacted), size)\n\terr := m.newMsg(CompactChunk, slices, id)\n\tif err != nil {\n\t\tif !strings.Contains(err.Error(), \"not exist\") && !strings.Contains(err.Error(), \"not found\") {\n\t\t\tlogger.Warnf(\"compact %d %d with %d slices: %s\", inode, indx, len(compacted), err)\n\t\t}\n\t\treturn\n\t}\n\n\tvar dsbuf []byte\n\ttrash := m.toTrash(0)\n\tif trash {\n\t\tdsbuf = make([]byte, 0, len(compacted)*12)\n\t\tfor _, s := range compacted {\n\t\t\tif s.id > 0 {\n\t\t\t\tdsbuf = append(dsbuf, m.encodeDelayedSlice(s.id, s.size)...)\n\t\t\t}\n\t\t}\n\t}\n\torigin := make([]byte, 0, len(ss)*sliceBytes)\n\tfor _, s := range ss {\n\t\torigin = append(origin, marshalSlice(s.pos, s.id, s.size, s.off, s.len)...)\n\t}\n\tst = m.en.doCompactChunk(inode, indx, origin, compacted, skipped, pos, id, size, dsbuf)\n\tif st == syscall.EINVAL {\n\t\tlogger.Infof(\"compaction for %d:%d is wasted, delete slice %d (%d bytes)\", inode, indx, id, size)\n\t\tm.deleteSlice(id, size)\n\t} else if st == 0 {\n\t\tm.of.InvalidateChunk(inode, indx)\n\t} else {\n\t\tlogger.Warnf(\"compact %d %d: %s\", inode, indx, err)\n\t}\n\n\tif force {\n\t\tm.Lock()\n\t\tdelete(m.compacting, k)\n\t\tm.Unlock()\n\t\tm.compactChunk(inode, indx, once, force)\n\t}\n}\n\nfunc (m *baseMeta) Compact(ctx Context, inode Ino, concurrency int, preFunc, postFunc func()) syscall.Errno {\n\tvar attr Attr\n\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\tlogger.Errorf(\"get attr error [inode %v]: %v\", inode, st)\n\t\treturn st\n\t}\n\n\tvar wg sync.WaitGroup\n\t// compact\n\tchunkChan := make(chan cchunk, 10000)\n\tfor i := 0; i < concurrency; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor c := range chunkChan {\n\t\t\t\tm.compactChunk(c.inode, c.indx, false, true)\n\t\t\t\tpostFunc()\n\t\t\t\tif ctx.Canceled() {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// scan\n\tst := m.walk(ctx, inode, \"\", &attr, func(ctx Context, fIno Ino, path string, fAttr *Attr) {\n\t\tif fAttr.Typ != TypeFile {\n\t\t\treturn\n\t\t}\n\t\t// calc chunk index in local\n\t\tchunkCnt := uint32((fAttr.Length + ChunkSize - 1) / ChunkSize)\n\t\tfor i := uint32(0); i < chunkCnt; i++ {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase chunkChan <- cchunk{inode: fIno, indx: i}:\n\t\t\t\tpreFunc()\n\t\t\t}\n\t\t}\n\t})\n\n\t// finish\n\tclose(chunkChan)\n\twg.Wait()\n\n\tif st != 0 {\n\t\tlogger.Errorf(\"walk error [inode %v]: %v\", inode, st)\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) fileDeleted(opened, force bool, inode Ino, length uint64) {\n\tif opened {\n\t\tm.Lock()\n\t\tm.removedFiles[inode] = true\n\t\tm.Unlock()\n\t} else {\n\t\tm.tryDeleteFileData(inode, length, force)\n\t}\n}\n\nfunc (m *baseMeta) tryDeleteFileData(inode Ino, length uint64, force bool) {\n\tif force {\n\t\tm.maxDeleting <- struct{}{}\n\t} else {\n\t\tselect {\n\t\tcase m.maxDeleting <- struct{}{}:\n\t\tdefault:\n\t\t\treturn // will be cleanup later\n\t\t}\n\t}\n\tgo func() {\n\t\tm.en.doDeleteFileData(inode, length)\n\t\t<-m.maxDeleting\n\t}()\n}\n\nfunc (m *baseMeta) deleteSlice_(id uint64, size uint32) {\n\tif err := m.newMsg(DeleteSlice, id, size); err != nil {\n\t\tlogger.Warnf(\"Delete data blocks of slice %d (%d bytes): %s\", id, size, err)\n\t\treturn\n\t}\n\tif err := m.en.doDeleteSlice(id, size); err != nil {\n\t\tlogger.Errorf(\"Delete meta entry of slice %d (%d bytes): %s\", id, size, err)\n\t}\n}\n\nfunc (m *baseMeta) deleteSlice(id uint64, size uint32) {\n\tif id == 0 || m.conf.MaxDeletes == 0 {\n\t\treturn\n\t}\n\tm.dSliceMu.Lock()\n\tif m.dslices == nil {\n\t\tm.dSliceMu.Unlock()\n\t\tm.deleteSlice_(id, size)\n\t\treturn\n\t}\n\tselect {\n\tcase <-m.sessCtx.Done():\n\tcase m.dslices <- Slice{Id: id, Size: size}:\n\t}\n\tm.dSliceMu.Unlock()\n}\n\nfunc (m *baseMeta) toTrash(parent Ino) bool {\n\tif parent.IsTrash() {\n\t\treturn false\n\t}\n\treturn m.getFormat().TrashDays > 0\n}\n\nfunc (m *baseMeta) checkTrash(parent Ino, trash *Ino) syscall.Errno {\n\tif !m.toTrash(parent) {\n\t\treturn 0\n\t}\n\tname := time.Now().UTC().Format(\"2006-01-02-15\")\n\tm.Lock()\n\tdefer m.Unlock()\n\tif name == m.subTrash.name {\n\t\t*trash = m.subTrash.inode\n\t\treturn 0\n\t}\n\tm.Unlock()\n\n\tst := m.en.doLookup(Background(), TrashInode, name, trash, nil)\n\tif st == syscall.ENOENT {\n\t\tattr := Attr{Typ: TypeDirectory, Nlink: 2, Length: 4 << 10, Parent: TrashInode, Full: true}\n\t\tst = m.en.doMknod(Background(), TrashInode, name, TypeDirectory, 0555, 0, \"\", trash, &attr)\n\t\tm.en.updateStats(align4K(0), 1)\n\t}\n\n\tm.Lock()\n\tif st != 0 && st != syscall.EEXIST {\n\t\tlogger.Warnf(\"create subTrash %s: %s\", name, st)\n\t} else if *trash <= TrashInode {\n\t\tlogger.Warnf(\"invalid trash inode: %d\", *trash)\n\t\tst = syscall.EBADF\n\t} else {\n\t\tm.subTrash.inode = *trash\n\t\tm.subTrash.name = name\n\t\tst = 0\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) trashEntry(parent, inode Ino, name string) string {\n\ts := fmt.Sprintf(\"%d-%d-%s\", parent, inode, name)\n\tif len(s) > MaxName {\n\t\ts = s[:MaxName]\n\t\tlogger.Warnf(\"File name is too long as a trash entry, truncating it: %s -> %s\", name, s)\n\t}\n\treturn s\n}\n\nfunc (m *baseMeta) cleanupTrash(ctx Context) {\n\tdefer m.sessWG.Done()\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-time.After(utils.JitterIt(time.Hour)):\n\t\t}\n\t\tif st := m.en.doGetAttr(ctx, TrashInode, nil); st != 0 {\n\t\t\tif st != syscall.ENOENT {\n\t\t\t\tlogger.Warnf(\"getattr inode %d: %s\", TrashInode, st)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif ok, err := m.en.setIfSmall(\"lastCleanupTrash\", time.Now().Unix(), int64(time.Hour.Seconds())*9/10); err != nil {\n\t\t\tlogger.Warnf(\"checking counter lastCleanupTrash: %s\", err)\n\t\t} else if ok {\n\t\t\tfunc() {\n\t\t\t\tcCtx := WrapWithTimeout(ctx, 50*time.Minute)\n\t\t\t\tdefer cCtx.Cancel()\n\t\t\t\tjobStart := time.Now()\n\t\t\t\tdays := m.getFormat().TrashDays\n\t\t\t\tvar wg sync.WaitGroup\n\t\t\t\twg.Add(2)\n\t\t\t\tdefer wg.Wait()\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tstats := &CleanupTrashStats{}\n\t\t\t\t\tstatus := bgJobSucc\n\t\t\t\t\tif st := m.doCleanupTrash(cCtx, days, false, stats); st != 0 {\n\t\t\t\t\t\tif st == syscall.ETIMEDOUT {\n\t\t\t\t\t\t\tstatus = bgJobCanceled\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tstatus = bgJobFail\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tm.bgjobDuration.WithLabelValues(\"cleanTrashFile\", status).Observe(time.Since(jobStart).Seconds())\n\t\t\t\t\tm.bgjobDels.WithLabelValues(\"cleanTrashFile\").Add(float64(atomic.LoadInt64(&stats.DeletedFiles)))\n\t\t\t\t}()\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tstatus := bgJobSucc\n\t\t\t\t\tvar cnt uint64\n\t\t\t\t\tif err := m.cleanupDelayedSlices(cCtx, days, &cnt); err != nil {\n\t\t\t\t\t\tif errors.Is(err, context.DeadlineExceeded) {\n\t\t\t\t\t\t\tstatus = bgJobCanceled\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tstatus = bgJobFail\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tm.bgjobDuration.WithLabelValues(\"cleanDelayedSlice\", status).Observe(time.Since(jobStart).Seconds())\n\t\t\t\t\tm.bgjobDels.WithLabelValues(\"cleanDelayedSlice\").Add(float64(cnt))\n\t\t\t\t}()\n\t\t\t}()\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) CleanupDetachedNodesBefore(ctx Context, edge time.Time, increProgress func()) {\n\tfor _, inode := range m.en.doFindDetachedNodes(edge) {\n\t\tif eno := m.en.doCleanupDetachedNode(Background(), inode); eno != 0 {\n\t\t\tlogger.Errorf(\"cleanupDetachedNode: remove detached tree (%d) error: %s\", inode, eno)\n\t\t} else {\n\t\t\tif increProgress != nil {\n\t\t\t\tincreProgress()\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) CleanupTrashBefore(ctx Context, edge time.Time, increProgress func(int), stats *CleanupTrashStats) syscall.Errno {\n\tlogger.Debugf(\"cleanup trash: started\")\n\tnow := time.Now()\n\tvar st syscall.Errno\n\tvar entries []*Entry\n\tif st = m.en.doReaddir(ctx, TrashInode, 0, &entries, -1); st != 0 {\n\t\tlogger.Warnf(\"readdir trash %d: %s\", TrashInode, st)\n\t\treturn st\n\t}\n\tsort.Slice(entries, func(i, j int) bool { return entries[i].Inode < entries[j].Inode })\n\tvar count uint64\n\tdone := make(chan struct{})\n\tdefer func() {\n\t\tclose(done)\n\t\tif count > 0 {\n\t\t\tlogger.Infof(\"cleanup trash: deleted %d files in %v\", count, time.Since(now))\n\t\t\tif stats != nil {\n\t\t\t\tatomic.StoreInt64(&stats.DeletedFiles, int64(count))\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Debugf(\"cleanup trash: nothing to delete\")\n\t\t}\n\t}()\n\n\tif increProgress != nil {\n\t\tgo func() {\n\t\t\tvar last uint64\n\t\t\tticker := time.NewTicker(time.Second)\n\t\t\tdefer ticker.Stop()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-done:\n\t\t\t\t\treturn\n\t\t\t\tcase <-ticker.C:\n\t\t\t\t\tcurr := atomic.LoadUint64(&count)\n\t\t\t\t\tif curr != last {\n\t\t\t\t\t\tincreProgress(int(curr - last))\n\t\t\t\t\t\tlast = curr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tconcurrent := make(chan int, 1) // no effect for flatterned trash dirs\n\tfor len(entries) > 0 {\n\t\tif ctx.Canceled() {\n\t\t\treturn errno(ctx.Err())\n\t\t}\n\t\te := entries[0]\n\t\tts, err := time.Parse(\"2006-01-02-15\", string(e.Name))\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"bad entry as a subTrash: %s\", e.Name)\n\t\t\tentries = entries[1:]\n\t\t\tcontinue\n\t\t}\n\t\tif !ts.Before(edge) {\n\t\t\tbreak\n\t\t}\n\t\tif st = m.emptyDir(ctx, e.Inode, true, &count, concurrent); st != 0 {\n\t\t\tif st != syscall.ETIMEDOUT && st != syscall.EINTR {\n\t\t\t\tlogger.Warnf(\"empty subTrash %d/%s: %s\", e.Inode, e.Name, st)\n\t\t\t}\n\t\t} else {\n\t\t\tentries = entries[1:]\n\t\t\tif st = m.en.doRmdir(ctx, TrashInode, string(e.Name), nil, nil); st != 0 {\n\t\t\t\tlogger.Warnf(\"rmdir subTrash %s: %s\", e.Name, st)\n\t\t\t}\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) scanTrashEntry(ctx Context, scan func(inode Ino, size uint64)) error {\n\tvar st syscall.Errno\n\tvar entries []*Entry\n\tif st = m.en.doReaddir(ctx, TrashInode, 1, &entries, -1); st != 0 {\n\t\treturn errors.Wrap(st, \"read trash\")\n\t}\n\n\tvar subEntries []*Entry\n\tfor _, entry := range entries {\n\t\tscan(entry.Inode, entry.Attr.Length)\n\t\tsubEntries = subEntries[:0]\n\t\tif st = m.en.doReaddir(ctx, entry.Inode, 1, &subEntries, -1); st != 0 {\n\t\t\tlogger.Warnf(\"readdir subEntry %d: %s\", entry.Inode, st)\n\t\t\tcontinue\n\t\t}\n\t\tfor _, se := range subEntries {\n\t\t\tscan(se.Inode, se.Attr.Length)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *baseMeta) scanTrashFiles(ctx Context, scan trashFileScan) error {\n\tvar st syscall.Errno\n\tvar entries []*Entry\n\tif st = m.en.doReaddir(ctx, TrashInode, 1, &entries, -1); st != 0 {\n\t\treturn errors.Wrap(st, \"read trash\")\n\t}\n\n\tvar subEntries []*Entry\n\tfor _, entry := range entries {\n\t\tts, err := time.Parse(\"2006-01-02-15\", string(entry.Name))\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"bad entry as a subTrash: %s\", entry.Name)\n\t\t\tcontinue\n\t\t}\n\t\tsubEntries = subEntries[:0]\n\t\tif st = m.en.doReaddir(ctx, entry.Inode, 1, &subEntries, -1); st != 0 {\n\t\t\tlogger.Warnf(\"readdir subEntry %d: %s\", entry.Inode, st)\n\t\t\tcontinue\n\t\t}\n\t\tfor _, se := range subEntries {\n\t\t\tif se.Attr.Typ == TypeFile {\n\t\t\t\tclean, err := scan(se.Inode, se.Attr.Length, ts)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn errors.Wrap(err, \"scan trash files\")\n\t\t\t\t}\n\t\t\t\tif clean {\n\t\t\t\t\t// TODO: m.en.doUnlink(ctx, entry.Attr.Parent, string(entry.Name))\n\t\t\t\t\t// avoid lint warning\n\t\t\t\t\t_ = clean\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *baseMeta) doCleanupTrash(ctx Context, days int, force bool, stats *CleanupTrashStats) syscall.Errno {\n\tedge := time.Now().Add(-time.Duration(24*days+2) * time.Hour)\n\tif force {\n\t\tedge = time.Now()\n\t}\n\treturn m.CleanupTrashBefore(ctx, edge, nil, stats)\n}\n\nfunc (m *baseMeta) cleanupDelayedSlices(ctx Context, days int, count *uint64) error {\n\tnow := time.Now()\n\tedge := now.Unix() - int64(days)*24*3600\n\tlogger.Debugf(\"Cleanup delayed slices: started with edge %d\", edge)\n\tvar err error\n\tvar cnt int\n\tif cnt, err = m.en.doCleanupDelayedSlices(ctx, edge); err != nil && !errors.Is(err, context.DeadlineExceeded) {\n\t\tlogger.Warnf(\"Cleanup delayed slices: deleted %d slices in %v, but got error: %s\", count, time.Since(now), err)\n\t\treturn err\n\t} else if cnt > 0 {\n\t\tlogger.Infof(\"Cleanup delayed slices: deleted %d slices in %v\", cnt, time.Since(now))\n\t\tif count != nil {\n\t\t\tatomic.AddUint64(count, uint64(cnt))\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *baseMeta) ScanDeletedObject(ctx Context, tss trashSliceScan, pss pendingSliceScan, tfs trashFileScan, pfs pendingFileScan) error {\n\teg := errgroup.Group{}\n\tif tss != nil {\n\t\teg.Go(func() error {\n\t\t\treturn m.en.scanTrashSlices(ctx, tss)\n\t\t})\n\t}\n\tif pss != nil {\n\t\teg.Go(func() error {\n\t\t\treturn m.en.scanPendingSlices(ctx, pss)\n\t\t})\n\t}\n\tif tfs != nil {\n\t\teg.Go(func() error {\n\t\t\treturn m.scanTrashFiles(ctx, tfs)\n\t\t})\n\t}\n\tif pfs != nil {\n\t\teg.Go(func() error {\n\t\t\tconcurrency := m.conf.MaxDeletes\n\t\t\tcleanChan := make(chan struct {\n\t\t\t\tino Ino\n\t\t\t\tsize uint64\n\t\t\t}, concurrency)\n\t\t\tvar wg sync.WaitGroup\n\n\t\t\tfor i := 0; i < concurrency; i++ {\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tfor p := range cleanChan {\n\t\t\t\t\t\tm.en.doDeleteFileData(p.ino, p.size)\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\n\t\t\tcpfs := func(ino Ino, size uint64, ts int64) (bool, error) {\n\t\t\t\tclean, err := pfs(ino, size, ts)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn false, err\n\t\t\t\t}\n\t\t\t\tif clean {\n\t\t\t\t\tcleanChan <- struct {\n\t\t\t\t\t\tino Ino\n\t\t\t\t\t\tsize uint64\n\t\t\t\t\t}{ino, size}\n\t\t\t\t}\n\t\t\t\treturn clean, nil\n\t\t\t}\n\n\t\t\terr := m.en.scanPendingFiles(ctx, cpfs)\n\t\t\tclose(cleanChan)\n\t\t\twg.Wait()\n\t\t\treturn err\n\t\t})\n\t}\n\treturn eg.Wait()\n}\n\nfunc (m *baseMeta) Clone(ctx Context, srcParentIno, srcIno, parent Ino, name string, cmode uint8, cumask uint16, concurrency uint8, count, total *uint64) syscall.Errno {\n\n\tif srcIno.IsTrash() || srcParentIno.IsTrash() || parent.IsTrash() || (parent == RootInode && name == TrashName) {\n\t\treturn syscall.EPERM\n\t}\n\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tif name == \"\" {\n\t\treturn syscall.ENOENT\n\t}\n\n\tdefer m.timeit(\"Clone\", time.Now())\n\tparent = m.checkRoot(parent)\n\n\tvar attr Attr\n\tvar eno syscall.Errno\n\tif eno = m.en.doGetAttr(ctx, srcIno, &attr); eno != 0 {\n\t\treturn eno\n\t}\n\tif eno = m.Access(ctx, srcIno, MODE_MASK_R, &attr); eno != 0 {\n\t\treturn eno\n\t}\n\tif eno = m.Access(ctx, parent, MODE_MASK_X|MODE_MASK_W, nil); eno != 0 {\n\t\treturn eno\n\t}\n\tvar dstIno Ino\n\tvar _a Attr\n\tif eno = m.en.doLookup(ctx, parent, name, &dstIno, &_a); eno == 0 {\n\t\treturn syscall.EEXIST\n\t} else if eno != syscall.ENOENT {\n\t\treturn eno\n\t}\n\tvar sum Summary\n\teno = m.GetSummary(ctx, srcIno, &sum, true, false)\n\tif eno != 0 {\n\t\treturn eno\n\t}\n\tif err := m.checkQuota(ctx, int64(sum.Size), int64(sum.Dirs)+int64(sum.Files), ctx.Uid(), ctx.Gid(), parent); err != 0 {\n\t\treturn err\n\t}\n\t*total = sum.Dirs + sum.Files\n\tif concurrency < 1 {\n\t\tconcurrency = 1\n\t}\n\tconcurrent := make(chan struct{}, concurrency)\n\tif attr.Typ == TypeDirectory {\n\t\teno = m.cloneEntry(ctx, srcIno, parent, name, &dstIno, cmode, cumask, count, true, concurrent)\n\t\tif eno == 0 {\n\t\t\teno = m.en.doAttachDirNode(ctx, parent, dstIno, name)\n\t\t}\n\t\tif eno != 0 && dstIno != 0 {\n\t\t\tif eno := m.en.doCleanupDetachedNode(ctx, dstIno); eno != 0 {\n\t\t\t\tlogger.Errorf(\"remove detached tree (%d): %s\", dstIno, eno)\n\t\t\t}\n\t\t}\n\t} else {\n\t\teno = m.cloneEntry(ctx, srcIno, parent, name, nil, cmode, cumask, count, true, concurrent)\n\t}\n\tif eno == 0 {\n\t\tm.updateDirStat(ctx, parent, int64(attr.Length), align4K(attr.Length), 1)\n\t\tm.updateDirQuota(ctx, parent, int64(sum.Size), int64(sum.Dirs)+int64(sum.Files))\n\t}\n\treturn eno\n}\n\nfunc (m *baseMeta) cloneEntry(ctx Context, srcIno Ino, parent Ino, name string, dstIno *Ino, cmode uint8, cumask uint16, count *uint64, top bool, concurrent chan struct{}) syscall.Errno {\n\tino, err := m.nextInode()\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tif dstIno != nil {\n\t\t*dstIno = ino\n\t}\n\tvar attr Attr\n\teno := m.en.doCloneEntry(ctx, srcIno, parent, name, ino, &attr, cmode, cumask, top)\n\tif eno != 0 {\n\t\treturn eno\n\t}\n\tm.en.updateStats(align4K(attr.Length), 1)\n\tatomic.AddUint64(count, 1)\n\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, align4K(attr.Length), 1)\n\tif attr.Typ != TypeDirectory {\n\t\treturn 0\n\t}\n\tif eno = m.Access(ctx, srcIno, MODE_MASK_R|MODE_MASK_X, &attr); eno != 0 {\n\t\treturn eno\n\t}\n\t// Use DirHandler for batch processing to avoid loading all entries at once\n\thandler, eno := m.NewDirHandler(ctx, srcIno, true, nil)\n\tif eno == syscall.ENOENT {\n\t\teno = 0 // empty dir\n\t}\n\tif eno != 0 {\n\t\treturn eno\n\t}\n\tdefer handler.Close()\n\n\tcloneCtx := WrapWithCancel(ctx, ctx.Pid(), ctx.Uid(), ctx.Gids())\n\tdefer cloneCtx.Cancel()\n\n\tvar g errgroup.Group\n\tvar skipped uint32\n\n\tcloneChild := func(e *Entry) syscall.Errno {\n\t\tchildEno := m.cloneEntry(cloneCtx, e.Inode, ino, string(e.Name), nil, cmode, cumask, count, false, concurrent)\n\t\tif childEno == syscall.ENOENT {\n\t\t\tlogger.Warnf(\"ignore deleted %s in dir %d\", string(e.Name), srcIno)\n\t\t\tif e.Attr.Typ == TypeDirectory {\n\t\t\t\tatomic.AddUint32(&skipped, 1)\n\t\t\t}\n\t\t\treturn 0\n\t\t}\n\t\tif childEno != 0 {\n\t\t\tcloneCtx.Cancel()\n\t\t}\n\t\treturn childEno\n\t}\n\n\toffset := 0\n\tfor {\n\t\tbatchEntries, batchEno := handler.List(cloneCtx, offset)\n\t\tif batchEno != 0 {\n\t\t\teno = batchEno\n\t\t\tbreak\n\t\t}\n\t\tif len(batchEntries) == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tvar nonDirEntries []*Entry\n\t\tfor _, e := range batchEntries {\n\t\t\tif string(e.Name) == \".\" || string(e.Name) == \"..\" {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif e.Attr.Typ == TypeDirectory {\n\t\t\t\tselect {\n\t\t\t\tcase concurrent <- struct{}{}:\n\t\t\t\t\tentry := e\n\t\t\t\t\tg.Go(func() error {\n\t\t\t\t\t\tdefer func() { <-concurrent }()\n\t\t\t\t\t\tif childEno := cloneChild(entry); childEno != 0 {\n\t\t\t\t\t\t\treturn childEno\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\tdefault:\n\t\t\t\t\t// Synchronous fallback when concurrency limit reached\n\t\t\t\t\tif childEno := cloneChild(e); childEno != 0 && eno == 0 {\n\t\t\t\t\t\teno = childEno\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tnonDirEntries = append(nonDirEntries, e)\n\t\t\t}\n\n\t\t\tif cloneCtx.Canceled() {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif eno != 0 || cloneCtx.Canceled() {\n\t\t\tbreak\n\t\t}\n\n\t\t// Batch clone files immediately (don't wait for subdirs to finish)\n\t\tif len(nonDirEntries) > 0 {\n\t\t\tbatchEno := m.BatchClone(cloneCtx, srcIno, ino, nonDirEntries, cmode, cumask, count)\n\t\t\tif batchEno == syscall.ENOTSUP {\n\t\t\t\t// Fallback: clone each file concurrently\n\t\t\t\tfor _, e := range nonDirEntries {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase concurrent <- struct{}{}:\n\t\t\t\t\t\tentry := e\n\t\t\t\t\t\tg.Go(func() error {\n\t\t\t\t\t\t\tdefer func() { <-concurrent }()\n\t\t\t\t\t\t\tif childEno := cloneChild(entry); childEno != 0 {\n\t\t\t\t\t\t\t\treturn childEno\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t})\n\t\t\t\t\tdefault:\n\t\t\t\t\t\t// Synchronous fallback when concurrency limit reached\n\t\t\t\t\t\tif childEno := cloneChild(e); childEno != 0 && eno == 0 {\n\t\t\t\t\t\t\teno = childEno\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tif cloneCtx.Canceled() {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif eno == syscall.ENOTSUP {\n\t\t\t\t\teno = 0\n\t\t\t\t}\n\t\t\t} else if batchEno != 0 {\n\t\t\t\teno = batchEno\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\toffset += len(batchEntries)\n\t\tif cloneCtx.Canceled() {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Wait for all goroutines; preserve the first non-cancel error when possible.\n\tif err := g.Wait(); eno == 0 && err != nil {\n\t\teno = errno(err)\n\t}\n\tif eno == 0 && cloneCtx.Canceled() {\n\t\teno = syscall.EINTR\n\t}\n\n\tif eno == 0 && skipped > 0 {\n\t\tattr.Nlink -= skipped\n\t\tif eno := m.en.doRepair(ctx, ino, &attr); eno != 0 {\n\t\t\tlogger.Warnf(\"fix nlink of %d: %s\", ino, eno)\n\t\t}\n\t}\n\treturn eno\n}\n\nfunc (m *baseMeta) mergeAttr(ctx Context, inode Ino, set uint16, cur, attr *Attr, now time.Time, rule *aclAPI.Rule) (*Attr, syscall.Errno) {\n\tdirtyAttr := *cur\n\tif (set&(SetAttrUID|SetAttrGID)) != 0 && (set&SetAttrMode) != 0 {\n\t\tattr.Mode |= (cur.Mode & 06000)\n\t}\n\tvar changed bool\n\tif (cur.Mode&06000) != 0 && (set&(SetAttrUID|SetAttrGID)) != 0 {\n\t\tclearSUGID(ctx, &dirtyAttr, attr)\n\t\tchanged = true\n\t}\n\tif set&SetAttrGID != 0 {\n\t\tif ctx.Uid() != 0 && ctx.Uid() != cur.Uid {\n\t\t\treturn nil, syscall.EPERM\n\t\t}\n\t\tif cur.Gid != attr.Gid {\n\t\t\tif ctx.CheckPermission() && ctx.Uid() != 0 && !containsGid(ctx, attr.Gid) {\n\t\t\t\treturn nil, syscall.EPERM\n\t\t\t}\n\t\t\tdirtyAttr.Gid = attr.Gid\n\t\t\tchanged = true\n\t\t}\n\t}\n\tif set&SetAttrUID != 0 && cur.Uid != attr.Uid {\n\t\tif ctx.CheckPermission() && ctx.Uid() != 0 {\n\t\t\treturn nil, syscall.EPERM\n\t\t}\n\t\tdirtyAttr.Uid = attr.Uid\n\t\tchanged = true\n\t}\n\tif set&SetAttrMode != 0 {\n\t\tif ctx.Uid() != 0 && (attr.Mode&02000) != 0 {\n\t\t\tif ctx.Gid() != cur.Gid {\n\t\t\t\tattr.Mode &= 05777\n\t\t\t}\n\t\t}\n\n\t\tif rule != nil {\n\t\t\trule.SetMode(attr.Mode)\n\t\t\tdirtyAttr.Mode = attr.Mode&07000 | rule.GetMode()\n\t\t\tchanged = true\n\t\t} else if attr.Mode != cur.Mode {\n\t\t\tif ctx.Uid() != 0 && ctx.Uid() != cur.Uid &&\n\t\t\t\t(cur.Mode&01777 != attr.Mode&01777 || attr.Mode&02000 > cur.Mode&02000 || attr.Mode&04000 > cur.Mode&04000) {\n\t\t\t\treturn nil, syscall.EPERM\n\t\t\t}\n\t\t\tdirtyAttr.Mode = attr.Mode\n\t\t\tchanged = true\n\t\t}\n\t}\n\tif set&SetAttrAtimeNow != 0 || (set&SetAttrAtime) != 0 && attr.Atime < 0 {\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W, cur); ctx.Uid() != cur.Uid && st != 0 {\n\t\t\treturn nil, syscall.EACCES\n\t\t}\n\t\tdirtyAttr.Atime = now.Unix()\n\t\tdirtyAttr.Atimensec = uint32(now.Nanosecond())\n\t\tchanged = true\n\t} else if set&SetAttrAtime != 0 && (cur.Atime != attr.Atime || cur.Atimensec != attr.Atimensec) {\n\t\tif cur.Uid == 0 && ctx.Uid() != 0 {\n\t\t\treturn nil, syscall.EPERM\n\t\t}\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W, cur); ctx.Uid() != cur.Uid && st != 0 {\n\t\t\treturn nil, syscall.EACCES\n\t\t}\n\t\tdirtyAttr.Atime = attr.Atime\n\t\tdirtyAttr.Atimensec = attr.Atimensec\n\t\tchanged = true\n\t}\n\tif set&SetAttrMtimeNow != 0 || (set&SetAttrMtime) != 0 && attr.Mtime < 0 {\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W, cur); ctx.Uid() != cur.Uid && st != 0 {\n\t\t\treturn nil, syscall.EACCES\n\t\t}\n\t\tdirtyAttr.Mtime = now.Unix()\n\t\tdirtyAttr.Mtimensec = uint32(now.Nanosecond())\n\t\tchanged = true\n\t} else if set&SetAttrMtime != 0 && (cur.Mtime != attr.Mtime || cur.Mtimensec != attr.Mtimensec) {\n\t\tif cur.Uid == 0 && ctx.Uid() != 0 {\n\t\t\treturn nil, syscall.EPERM\n\t\t}\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W, cur); ctx.Uid() != cur.Uid && st != 0 {\n\t\t\treturn nil, syscall.EACCES\n\t\t}\n\t\tdirtyAttr.Mtime = attr.Mtime\n\t\tdirtyAttr.Mtimensec = attr.Mtimensec\n\t\tchanged = true\n\t}\n\tif set&SetAttrFlag != 0 {\n\t\tdirtyAttr.Flags = attr.Flags\n\t\tchanged = true\n\t}\n\tif !changed {\n\t\t*attr = *cur\n\t\treturn nil, 0\n\t}\n\treturn &dirtyAttr, 0\n}\n\nfunc (m *baseMeta) CheckSetAttr(ctx Context, inode Ino, set uint16, attr Attr) syscall.Errno {\n\tvar cur Attr\n\tinode = m.checkRoot(inode)\n\tif st := m.en.doGetAttr(ctx, inode, &cur); st != 0 {\n\t\treturn st\n\t}\n\t_, st := m.mergeAttr(ctx, inode, set, &cur, &attr, time.Now(), nil)\n\treturn st\n}\n\nvar errACLNotInCache = errors.New(\"acl not in cache\")\n\nfunc (m *baseMeta) getFaclFromCache(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) error {\n\tino = m.checkRoot(ino)\n\tcAttr := &Attr{}\n\tif m.conf.OpenCache > 0 && m.of.Check(ino, cAttr) {\n\t\taclId := getAttrACLId(cAttr, aclType)\n\t\tif aclId == aclAPI.None {\n\t\t\treturn ENOATTR\n\t\t}\n\n\t\tif cRule := m.aclCache.Get(aclId); cRule != nil {\n\t\t\t*rule = *cRule\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn errACLNotInCache\n}\n\nfunc setAttrACLId(attr *Attr, aclType uint8, id uint32) {\n\tswitch aclType {\n\tcase aclAPI.TypeAccess:\n\t\tattr.AccessACL = id\n\tcase aclAPI.TypeDefault:\n\t\tattr.DefaultACL = id\n\t}\n}\n\nfunc getAttrACLId(attr *Attr, aclType uint8) uint32 {\n\tswitch aclType {\n\tcase aclAPI.TypeAccess:\n\t\treturn attr.AccessACL\n\tcase aclAPI.TypeDefault:\n\t\treturn attr.DefaultACL\n\t}\n\treturn aclAPI.None\n}\n\nfunc setXAttrACL(xattrs *[]byte, accessACL, defaultACL uint32) {\n\tif accessACL != aclAPI.None {\n\t\t*xattrs = append(*xattrs, []byte(\"system.posix_acl_access\")...)\n\t\t*xattrs = append(*xattrs, 0)\n\t}\n\tif defaultACL != aclAPI.None {\n\t\t*xattrs = append(*xattrs, []byte(\"system.posix_acl_default\")...)\n\t\t*xattrs = append(*xattrs, 0)\n\t}\n}\n\nfunc (m *baseMeta) saveACL(rule *aclAPI.Rule, aclMaxId *uint32) uint32 {\n\tif rule == nil {\n\t\treturn aclAPI.None\n\t}\n\tid := m.aclCache.GetId(rule)\n\tif id == aclAPI.None {\n\t\t(*aclMaxId)++\n\t\tid = *aclMaxId\n\t\tm.aclCache.Put(id, rule)\n\t}\n\treturn id\n}\n\nfunc (m *baseMeta) SetFacl(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) syscall.Errno {\n\tif aclType != aclAPI.TypeAccess && aclType != aclAPI.TypeDefault {\n\t\treturn syscall.EINVAL\n\t}\n\n\tif !ino.IsNormal() {\n\t\treturn syscall.EPERM\n\t}\n\n\tnow := time.Now()\n\tdefer func() {\n\t\tm.timeit(\"SetFacl\", now)\n\t\tm.of.InvalidateChunk(ino, invalidateAttrOnly)\n\t}()\n\n\treturn m.en.doSetFacl(ctx, ino, aclType, rule)\n}\n\nfunc (m *baseMeta) GetFacl(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) syscall.Errno {\n\tvar err error\n\tif err = m.getFaclFromCache(ctx, ino, aclType, rule); err == nil {\n\t\treturn 0\n\t}\n\n\tif !errors.Is(err, errACLNotInCache) {\n\t\treturn errno(err)\n\t}\n\n\tnow := time.Now()\n\tdefer m.timeit(\"GetFacl\", now)\n\n\treturn m.en.doGetFacl(ctx, ino, aclType, aclAPI.None, rule)\n}\n\nfunc (m *baseMeta) StoreToken(ctx Context, token []byte) (id uint32, st syscall.Errno) {\n\tdefer m.timeit(\"StoreToken\", time.Now())\n\treturn m.en.doStoreToken(ctx, token)\n}\n\nfunc (m *baseMeta) UpdateToken(ctx Context, id uint32, token []byte) syscall.Errno {\n\tdefer m.timeit(\"UpdateToken\", time.Now())\n\treturn m.en.doUpdateToken(ctx, id, token)\n}\n\nfunc (m *baseMeta) LoadToken(ctx Context, id uint32) (token []byte, st syscall.Errno) {\n\tdefer m.timeit(\"LoadToken\", time.Now())\n\treturn m.en.doLoadToken(ctx, id)\n}\n\nfunc (m *baseMeta) DeleteTokens(ctx Context, ids []uint32) syscall.Errno {\n\tdefer m.timeit(\"DeleteTokens\", time.Now())\n\treturn m.en.doDeleteTokens(ctx, ids)\n}\n\nfunc (m *baseMeta) ListTokens(ctx Context) (tokens map[uint32][]byte, st syscall.Errno) {\n\tdefer m.timeit(\"ListTokens\", time.Now())\n\treturn m.en.doListTokens(ctx)\n}\n\nfunc inGroup(ctx Context, gid uint32) bool {\n\tfor _, egid := range ctx.Gids() {\n\t\tif egid == gid {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\ntype DirHandler interface {\n\tList(ctx Context, offset int) ([]*Entry, syscall.Errno)\n\tInsert(inode Ino, name string, attr *Attr)\n\tDelete(name string)\n\tRead(offset int)\n\tClose()\n}\n\nfunc (m *baseMeta) NewDirHandler(ctx Context, inode Ino, plus bool, initEntries []*Entry) (DirHandler, syscall.Errno) {\n\tvar attr Attr\n\tvar st syscall.Errno\n\tdefer func() {\n\t\tif st == 0 {\n\t\t\tm.touchAtime(ctx, inode, &attr)\n\t\t}\n\t}()\n\n\tinode = m.checkRoot(inode)\n\tif st = m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\treturn nil, st\n\t}\n\tdefer m.timeit(\"NewDirHandler\", time.Now())\n\tvar mmask uint8 = MODE_MASK_R\n\tif plus {\n\t\tmmask |= MODE_MASK_X\n\t}\n\n\tif st = m.Access(ctx, inode, mmask, &attr); st != 0 {\n\t\treturn nil, st\n\t}\n\tif inode == m.root {\n\t\tattr.Parent = m.root\n\t}\n\n\tinitEntries = append(initEntries, &Entry{\n\t\tInode: inode,\n\t\tName: []byte(\".\"),\n\t\tAttr: &attr,\n\t})\n\n\tparent := &Entry{\n\t\tInode: attr.Parent,\n\t\tName: []byte(\"..\"),\n\t\tAttr: &Attr{Typ: TypeDirectory},\n\t}\n\tif plus {\n\t\tif attr.Parent == inode {\n\t\t\tparent.Attr = &attr\n\t\t} else {\n\t\t\tif st := m.GetAttr(ctx, attr.Parent, parent.Attr); st != 0 {\n\t\t\t\treturn nil, st\n\t\t\t}\n\t\t}\n\t}\n\tinitEntries = append(initEntries, parent)\n\n\treturn m.en.newDirHandler(inode, plus, initEntries), 0\n}\n\ntype dirBatch struct {\n\tisEnd bool\n\toffset int\n\tcursor interface{}\n\tentries []*Entry\n\tindexes map[string]int\n}\n\nfunc (b *dirBatch) contain(offset int) bool {\n\tif b == nil {\n\t\treturn false\n\t}\n\treturn b.offset <= offset && offset < b.offset+len(b.entries) || (len(b.entries) == 0 && b.offset == offset)\n}\n\nfunc (b *dirBatch) predecessor(offset int) bool {\n\treturn b.offset+len(b.entries) == offset\n}\n\ntype dirFetcher func(ctx Context, inode Ino, cursor interface{}, offset, limit int, plus bool) (interface{}, []*Entry, error)\n\ntype dirHandler struct {\n\tsync.Mutex\n\tinode Ino\n\tplus bool\n\tinitEntries []*Entry\n\tbatch *dirBatch\n\tfetcher dirFetcher\n\treadOff int\n\tbatchNum int\n}\n\nfunc (h *dirHandler) fetch(ctx Context, offset int) (*dirBatch, error) {\n\tvar cursor interface{}\n\tif h.batch != nil && h.batch.predecessor(offset) {\n\t\tif h.batch.isEnd {\n\t\t\treturn h.batch, nil\n\t\t}\n\t\tcursor = h.batch.cursor\n\t}\n\tnextCursor, entries, err := h.fetcher(ctx, h.inode, cursor, offset, h.batchNum, h.plus)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif entries == nil {\n\t\tentries = []*Entry{}\n\t\tnextCursor = cursor\n\t}\n\tindexes := make(map[string]int, len(entries))\n\tfor i, e := range entries {\n\t\tindexes[string(e.Name)] = i\n\t}\n\treturn &dirBatch{isEnd: len(entries) < h.batchNum, offset: offset, cursor: nextCursor, entries: entries, indexes: indexes}, nil\n}\n\nfunc (h *dirHandler) List(ctx Context, offset int) ([]*Entry, syscall.Errno) {\n\tvar prefix []*Entry\n\tif offset < len(h.initEntries) {\n\t\tprefix = h.initEntries[offset:]\n\t\toffset = 0\n\t} else {\n\t\toffset -= len(h.initEntries)\n\t}\n\n\tvar err error\n\th.Lock()\n\tdefer h.Unlock()\n\tif !h.batch.contain(offset) {\n\t\th.batch, err = h.fetch(ctx, offset)\n\t}\n\n\tif err != nil {\n\t\treturn nil, errno(err)\n\t}\n\n\th.readOff = h.batch.offset + len(h.batch.entries)\n\tif len(prefix) > 0 {\n\t\treturn append(prefix, h.batch.entries...), 0\n\t}\n\treturn h.batch.entries[offset-h.batch.offset:], 0\n}\n\nfunc (h *dirHandler) Delete(name string) {\n\th.Lock()\n\tdefer h.Unlock()\n\tif h.batch == nil || len(h.batch.entries) == 0 {\n\t\treturn\n\t}\n\n\tif idx, ok := h.batch.indexes[name]; ok && idx+h.batch.offset >= h.readOff {\n\t\tdelete(h.batch.indexes, name)\n\t\tn := len(h.batch.entries)\n\t\tif idx < n-1 {\n\t\t\t// TODO: sorted\n\t\t\th.batch.entries[idx] = h.batch.entries[n-1]\n\t\t\th.batch.indexes[string(h.batch.entries[idx].Name)] = idx\n\t\t}\n\t\th.batch.entries = h.batch.entries[:n-1]\n\t}\n}\n\nfunc (h *dirHandler) Insert(inode Ino, name string, attr *Attr) {\n\th.Lock()\n\tdefer h.Unlock()\n\tif h.batch == nil {\n\t\treturn\n\t}\n\tif h.batch.isEnd || bytes.Compare([]byte(name), h.batch.cursor.([]byte)) < 0 {\n\t\t// TODO: sorted\n\t\th.batch.entries = append(h.batch.entries, &Entry{Inode: inode, Name: []byte(name), Attr: attr})\n\t\th.batch.indexes[name] = len(h.batch.entries) - 1\n\t}\n}\n\nfunc (h *dirHandler) Read(offset int) {\n\th.readOff = offset - len(h.initEntries) // TODO: what if fuse only reads one entry?\n}\n\nfunc (h *dirHandler) Close() {\n\th.Lock()\n\th.batch = nil\n\th.readOff = 0\n\th.Unlock()\n}\n\nfunc (m *baseMeta) DumpMetaV2(ctx Context, w io.Writer, opt *DumpOption) error {\n\topt = opt.check()\n\n\tbak := newBakFormat()\n\tch := make(chan *dumpedResult, 100)\n\twg := &sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\terr := m.en.dump(ctx, opt, ch)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"dump meta err: %v\", err)\n\t\t\tctx.Cancel()\n\t\t} else {\n\t\t\tclose(ch)\n\t\t}\n\t}()\n\n\tvar res *dumpedResult\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\twg.Wait()\n\t\t\treturn ctx.Err()\n\t\tcase res = <-ch:\n\t\t}\n\t\tif res == nil {\n\t\t\tbreak\n\t\t}\n\t\tseg := newBakSegment(res.msg)\n\t\tif err := bak.writeSegment(w, seg); err != nil {\n\t\t\tlogger.Errorf(\"write %d err: %v\", seg.typ, err)\n\t\t\tctx.Cancel()\n\t\t\twg.Wait()\n\t\t\treturn err\n\t\t}\n\t\tif opt.Progress != nil {\n\t\t\topt.Progress(seg.Name(), int(seg.num()))\n\t\t}\n\t\tif res.release != nil {\n\t\t\tres.release(res.msg)\n\t\t}\n\t}\n\n\twg.Wait()\n\treturn bak.writeFooter(w)\n}\n\nfunc (m *baseMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error {\n\tif opt == nil {\n\t\topt = &LoadOption{}\n\t}\n\tif err := m.en.prepareLoad(ctx, opt); err != nil {\n\t\treturn err\n\t}\n\n\ttype task struct {\n\t\ttyp int\n\t\tmsg proto.Message\n\t}\n\n\tvar wg sync.WaitGroup\n\ttaskCh := make(chan *task, 100)\n\n\tworkerFunc := func(ctx Context, taskCh <-chan *task) {\n\t\tdefer wg.Done()\n\t\tvar task *task\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase task = <-taskCh:\n\t\t\t}\n\t\t\tif task == nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\terr := m.en.load(ctx, task.typ, opt, task.msg)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"failed to insert %d: %s\", task.typ, err)\n\t\t\t\tctx.Cancel()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tfor i := 0; i < opt.Threads; i++ {\n\t\twg.Add(1)\n\t\tgo workerFunc(ctx, taskCh)\n\t}\n\n\tbak := &BakFormat{}\n\tfor {\n\t\tseg, err := bak.ReadSegment(r)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, errBakEOF) {\n\t\t\t\tclose(taskCh)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tctx.Cancel()\n\t\t\twg.Wait()\n\t\t\treturn err\n\t\t}\n\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\twg.Wait()\n\t\t\treturn ctx.Err()\n\t\tcase taskCh <- &task{int(seg.typ), seg.val}:\n\t\t\tif opt.Progress != nil {\n\t\t\t\topt.Progress(seg.Name(), int(seg.num()))\n\t\t\t}\n\t\t}\n\t}\n\twg.Wait()\n\treturn nil\n}\n" }, { "path": "pkg/meta/base_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n//\n//mutate:disable\n//nolint:errcheck\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n\t\"xorm.io/xorm\"\n)\n\nfunc testConfig() *Config {\n\tconf := DefaultConf()\n\tconf.DirStatFlushPeriod = 100 * time.Millisecond\n\treturn conf\n}\n\nfunc testFormat() *Format {\n\treturn &Format{Name: \"test\", DirStats: true}\n}\n\nfunc TestRedisClient(t *testing.T) {\n\tm, err := newRedisMeta(\"redis\", \"127.0.0.1:6379/10\", testConfig())\n\tif err != nil || m.Name() != \"redis\" {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestKeyDB(t *testing.T) { // skip mutate\n\tif os.Getenv(\"SKIP_NON_CORE\") == \"true\" {\n\t\tt.Skipf(\"skip non-core test\")\n\t}\n\t// 127.0.0.1:6378 enable flash, 127.0.0.1:6377 disable flash\n\tfor _, addr := range []string{\"127.0.0.1:6378/10\", \"127.0.0.1:6377/10\"} {\n\t\tm, err := newRedisMeta(\"redis\", addr, testConfig())\n\t\tif err != nil || m.Name() != \"redis\" {\n\t\t\tt.Fatalf(\"create meta: %s\", err)\n\t\t}\n\t\tif r, ok := m.(*redisMeta); ok {\n\t\t\trawInfo, err := r.rdb.Info(Background()).Result()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"parse info: %s\", err)\n\t\t\t}\n\t\t\tvar storageProvider, maxMemoryPolicy string\n\t\t\tfor _, l := range strings.Split(strings.TrimSpace(rawInfo), \"\\n\") {\n\t\t\t\tl = strings.TrimSpace(l)\n\t\t\t\tif l == \"\" || strings.HasPrefix(l, \"#\") {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tkvPair := strings.SplitN(l, \":\", 2)\n\t\t\t\tif len(kvPair) < 2 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tkey, val := kvPair[0], kvPair[1]\n\t\t\t\tswitch key {\n\t\t\t\tcase \"maxmemory_policy\":\n\t\t\t\t\tmaxMemoryPolicy = val\n\t\t\t\tcase \"storage_provider\":\n\t\t\t\t\tstorageProvider = val\n\t\t\t\t}\n\t\t\t}\n\t\t\tif storageProvider == \"none\" && maxMemoryPolicy != \"noeviction\" {\n\t\t\t\tt.Fatalf(\"maxmemory_policy should be noeviction\")\n\t\t\t}\n\t\t\tif storageProvider == \"flash\" && maxMemoryPolicy == \"noeviction\" {\n\t\t\t\tt.Fatalf(\"maxmemory_policy should not be noeviction\")\n\t\t\t}\n\t\t} else {\n\t\t\tt.Fatalf(\"should be redisMeta\")\n\t\t}\n\t}\n}\n\nfunc TestRedisCluster(t *testing.T) { // skip mutate\n\tif os.Getenv(\"SKIP_NON_CORE\") == \"true\" {\n\t\tt.Skipf(\"skip non-core test\")\n\t}\n\tm, err := newRedisMeta(\"redis\", \"127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003/2\", testConfig())\n\tif err != nil {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc testMeta(t *testing.T, m Meta) {\n\tif err := m.Reset(); err != nil {\n\t\tt.Fatalf(\"reset meta: %s\", err)\n\t}\n\n\ttestMetaClient(t, m)\n\ttestTruncateAndDelete(t, m)\n\ttestTrash(t, m)\n\ttestParents(t, m)\n\ttestRemove(t, m)\n\ttestResolve(t, m)\n\ttestStickyBit(t, m)\n\ttestLocks(t, m)\n\ttestListLocks(t, m)\n\ttestConcurrentWrite(t, m)\n\ttestCompaction(t, m, false)\n\ttime.Sleep(time.Second)\n\ttestCompaction(t, m, true)\n\ttestCopyFileRange(t, m)\n\ttestCloseSession(t, m)\n\ttestConcurrentDir(t, m)\n\ttestAttrFlags(t, m)\n\ttestQuota(t, m)\n\ttestUserGroupQuota(t, m)\n\ttestAtime(t, m)\n\ttestAccess(t, m)\n\tbase := m.getBase()\n\tbase.conf.OpenCache = time.Second\n\tbase.of.expire = time.Second\n\ttestOpenCache(t, m)\n\tbase.conf.CaseInsensi = true\n\ttestCaseIncensi(t, m)\n\ttestCaseIncensiRename(t, m)\n\ttestCaseIncensiHardlinkRename(t, m)\n\ttestCheckAndRepair(t, m)\n\ttestDirStat(t, m)\n\ttestClone(t, m)\n\ttestBatchClone(t, m)\n\ttestACL(t, m)\n\ttestKerberosToken(t, m)\n\tbase.conf.ReadOnly = true\n\ttestReadOnly(t, m)\n}\n\nfunc testAccess(t *testing.T, m Meta) {\n\tif err := m.Init(testFormat(), false); err != nil {\n\t\tt.Fatalf(\"init error: %s\", err)\n\t}\n\n\tdefer m.getBase().aclCache.Clear()\n\n\tvar testNode Ino = 2\n\tctx := NewContext(1, 1, []uint32{2})\n\tattr := &Attr{\n\t\tMode: 0541,\n\t\tUid: 0,\n\t\tGid: 0,\n\t\tAccessACL: 1,\n\t\tDefaultACL: 0,\n\t\tFull: true,\n\t}\n\n\tr1 := &aclAPI.Rule{\n\t\tOwner: 5,\n\t\tGroup: 4,\n\t\tMask: 2,\n\t\tOther: 1,\n\t\tNamedUsers: aclAPI.Entries{\n\t\t\t{\n\t\t\t\tId: 1,\n\t\t\t\tPerm: 6,\n\t\t\t},\n\t\t},\n\t\tNamedGroups: aclAPI.Entries{\n\t\t\t{\n\t\t\t\tId: 2,\n\t\t\t\tPerm: 6,\n\t\t\t},\n\t\t},\n\t}\n\tm.getBase().aclCache.Put(1, r1)\n\n\t// case: match owner, skip named entries\n\tst := m.Access(ctx, testNode, MODE_MASK_R|MODE_MASK_W, attr)\n\tassert.Equal(t, syscall.EACCES, st)\n\n\t// case: match named grouped entry, but group perm & mask failed\n\tctx = NewContext(1, 2, []uint32{2})\n\tst = m.Access(ctx, testNode, MODE_MASK_R|MODE_MASK_W, attr)\n\tassert.Equal(t, syscall.EACCES, st)\n\n\t// case: same as above, make mask to pass test\n\tr2 := &aclAPI.Rule{}\n\t*r2 = *r1\n\tr2.Mask = 7\n\tm.getBase().aclCache.Put(2, r2)\n\tattr.AccessACL = 2\n\n\tctx = NewContext(1, 2, []uint32{2})\n\tst = m.Access(ctx, testNode, MODE_MASK_R|MODE_MASK_W, attr)\n\tassert.Equal(t, syscall.Errno(0), st)\n}\n\nfunc testACL(t *testing.T, m Meta) {\n\tformat := testFormat()\n\tformat.EnableACL = true\n\n\tif err := m.Init(format, false); err != nil {\n\t\tt.Fatalf(\"test acl failed: %s\", err)\n\t}\n\n\tdefer m.getBase().aclCache.Clear()\n\n\tctx := Background()\n\ttestDir := \"test_dir\"\n\tvar testDirIno Ino\n\tattr1 := &Attr{}\n\n\tif st := m.Mkdir(ctx, RootInode, testDir, 0644, 0, 0, &testDirIno, attr1); st != 0 {\n\t\tt.Fatalf(\"create %s: %s\", testDir, st)\n\t}\n\tdefer m.Rmdir(ctx, RootInode, testDir)\n\n\trule := &aclAPI.Rule{\n\t\tOwner: 7,\n\t\tGroup: 7,\n\t\tMask: 7,\n\t\tOther: 7,\n\t\tNamedUsers: []aclAPI.Entry{\n\t\t\t{\n\t\t\t\tId: 1001,\n\t\t\t\tPerm: 4,\n\t\t\t},\n\t\t},\n\t\tNamedGroups: nil,\n\t}\n\n\t// case: setfacl\n\tif st := m.SetFacl(ctx, testDirIno, aclAPI.TypeAccess, rule); st != 0 {\n\t\tt.Fatalf(\"setfacl error: %s\", st)\n\t}\n\n\t// case: getfacl\n\trule2 := &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, testDirIno, aclAPI.TypeAccess, rule2); st != 0 {\n\t\tt.Fatalf(\"getfacl error: %s\", st)\n\t}\n\tassert.True(t, rule.IsEqual(rule2))\n\n\t// case: setfacl will sync mode (group class is mask)\n\tattr2 := &Attr{}\n\tif st := m.GetAttr(ctx, testDirIno, attr2); st != 0 {\n\t\tt.Fatalf(\"getattr error: %s\", st)\n\t}\n\tassert.Equal(t, uint16(0777), attr2.Mode)\n\n\t// case: setattr will sync acl\n\tset := uint16(0) | SetAttrMode\n\tattr2 = &Attr{\n\t\tMode: 0555,\n\t}\n\tif st := m.SetAttr(ctx, testDirIno, set, 0, attr2); st != 0 {\n\t\tt.Fatalf(\"setattr error: %s\", st)\n\t}\n\n\trule3 := &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, testDirIno, aclAPI.TypeAccess, rule3); st != 0 {\n\t\tt.Fatalf(\"getfacl error: %s\", st)\n\t}\n\trule2.Owner = 5\n\trule2.Mask = 5\n\trule2.Other = 5\n\tassert.True(t, rule3.IsEqual(rule2))\n\n\t// case: remove acl\n\trule3.Mask = 0xFFFF\n\trule3.NamedUsers = nil\n\trule3.NamedGroups = nil\n\tif st := m.SetFacl(ctx, testDirIno, aclAPI.TypeAccess, rule3); st != 0 {\n\t\tt.Fatalf(\"setattr error: %s\", st)\n\t}\n\n\tst := m.GetFacl(ctx, testDirIno, aclAPI.TypeAccess, nil)\n\tassert.Equal(t, ENOATTR, st)\n\n\tattr2 = &Attr{}\n\tif st := m.GetAttr(ctx, testDirIno, attr2); st != 0 {\n\t\tt.Fatalf(\"getattr error: %s\", st)\n\t}\n\tassert.Equal(t, uint16(0575), attr2.Mode)\n\n\t// case: set normal default acl\n\tif st := m.SetFacl(ctx, testDirIno, aclAPI.TypeDefault, rule); st != 0 {\n\t\tt.Fatalf(\"setfacl error: %s\", st)\n\t}\n\n\t// case: get normal default acl\n\trule2 = &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, testDirIno, aclAPI.TypeDefault, rule2); st != 0 {\n\t\tt.Fatalf(\"getfacl error: %s\", st)\n\t}\n\tassert.True(t, rule2.IsEqual(rule))\n\n\t// case: mk subdir with normal default acl\n\tsubDir := \"sub_dir\"\n\tvar subDirIno Ino\n\tattr2 = &Attr{}\n\n\tmode := uint16(0222)\n\t// cumask will be ignored\n\tif st := m.Mkdir(ctx, testDirIno, subDir, mode, 0022, 0, &subDirIno, attr2); st != 0 {\n\t\tt.Fatalf(\"create %s: %s\", subDir, st)\n\t}\n\tdefer m.Rmdir(ctx, testDirIno, subDir)\n\n\t// subdir inherit default acl\n\trule3 = &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, subDirIno, aclAPI.TypeDefault, rule3); st != 0 {\n\t\tt.Fatalf(\"getfacl error: %s\", st)\n\t}\n\tassert.True(t, rule3.IsEqual(rule2))\n\n\t// subdir access acl\n\trule3 = &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, subDirIno, aclAPI.TypeAccess, rule3); st != 0 {\n\t\tt.Fatalf(\"getfacl error: %s\", st)\n\t}\n\trule2.Owner &= (mode >> 6) & 7\n\trule2.Mask &= (mode >> 3) & 7\n\trule2.Other &= mode & 7\n\tassert.True(t, rule3.IsEqual(rule2))\n\n\t// case: set minimal default acl\n\trule = &aclAPI.Rule{\n\t\tOwner: 5,\n\t\tGroup: 5,\n\t\tMask: 0xFFFF,\n\t\tOther: 5,\n\t\tNamedUsers: nil,\n\t\tNamedGroups: nil,\n\t}\n\tif st := m.SetFacl(ctx, testDirIno, aclAPI.TypeDefault, rule); st != 0 {\n\t\tt.Fatalf(\"setfacl error: %s\", st)\n\t}\n\n\t// case: get minimal default acl\n\trule2 = &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, testDirIno, aclAPI.TypeDefault, rule2); st != 0 {\n\t\tt.Fatalf(\"getfacl error: %s\", st)\n\t}\n\tassert.True(t, rule2.IsEqual(rule))\n\n\t// case: mk subdir with minimal default acl\n\tsubDir2 := \"sub_dir2\"\n\tvar subDirIno2 Ino\n\tattr2 = &Attr{}\n\n\tmode = uint16(0222)\n\tif st := m.Mkdir(ctx, testDirIno, subDir2, mode, 0022, 0, &subDirIno2, attr2); st != 0 {\n\t\tt.Fatalf(\"create %s: %s\", subDir, st)\n\t}\n\tdefer m.Rmdir(ctx, testDirIno, subDir2)\n\tassert.Equal(t, uint16(0), attr2.Mode)\n\n\t// subdir inherit default acl\n\trule3 = &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, subDirIno2, aclAPI.TypeDefault, rule3); st != 0 {\n\t\tt.Fatalf(\"getfacl error: %s\", st)\n\t}\n\tassert.True(t, rule3.IsEqual(rule2))\n\n\t// subdir have no access acl\n\trule3 = &aclAPI.Rule{}\n\tst = m.GetFacl(ctx, subDirIno2, aclAPI.TypeAccess, rule3)\n\tassert.Equal(t, ENOATTR, st)\n\n\t// test cache all\n\tsz := m.getBase().aclCache.Size()\n\terr := m.getBase().en.cacheACLs(ctx)\n\tassert.Nil(t, err)\n\tassert.Equal(t, sz, m.getBase().aclCache.Size())\n}\n\nfunc testKerberosToken(t *testing.T, m Meta) {\n\ttype token struct {\n\t\tUser string\n\t\tRenewer string\n\t\tPassword string\n\t\tIssued int64\n\t\tExpire int64\n\t}\n\n\tformat := testFormat()\n\tif err := m.Init(format, false); err != nil {\n\t\tt.Fatalf(\"test acl failed: %s\", err)\n\t}\n\tctx := Background()\n\n\tissueToken := func() (uint32, *token) {\n\t\tnow := time.Now()\n\t\ttk := &token{\n\t\t\tUser: \"tom\",\n\t\t\tRenewer: \"yarn\",\n\t\t\tPassword: \"password123\",\n\t\t\tIssued: now.Unix(),\n\t\t\tExpire: now.Add(2 * time.Second).Unix(),\n\t\t}\n\t\ttb, err := json.Marshal(tk)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"marshal token failed: %s\", err)\n\t\t}\n\t\tid, eno := m.StoreToken(ctx, tb)\n\t\tif eno != 0 {\n\t\t\tt.Fatalf(\"store token failed: %s\", eno)\n\t\t}\n\t\treturn id, tk\n\t}\n\n\tbuildToken := func(data []byte) *token {\n\t\ttk := &token{}\n\t\tif err := json.Unmarshal(data, tk); err != nil {\n\t\t\tt.Fatalf(\"unmarshal token: %s\", err)\n\t\t}\n\t\treturn tk\n\t}\n\n\tid1, tk1 := issueToken()\n\tretb, eno := m.LoadToken(ctx, id1)\n\tif eno != 0 {\n\t\tt.Fatalf(\"load token failed: %s\", eno)\n\t}\n\tvar rettk token\n\tif err := json.Unmarshal(retb, &rettk); err != nil {\n\t\tt.Fatalf(\"unmarshal token: %s\", err)\n\t}\n\tif !reflect.DeepEqual(tk1, &rettk) {\n\t\tt.Fatalf(\"token mismatch: %+v != %+v\", tk1, &rettk)\n\t}\n\ttk1.Expire = time.Now().Add(2 * time.Second).Unix()\n\ttb, err := json.Marshal(tk1)\n\tif err != nil {\n\t\tt.Fatalf(\"marshal token failed: %s\", err)\n\t}\n\teno = m.UpdateToken(ctx, id1, tb)\n\tif eno != 0 {\n\t\tt.Fatalf(\"update token failed: %s\", eno)\n\t}\n\n\tid2, tk2 := issueToken()\n\ttokens, eno := m.ListTokens(ctx)\n\tif eno != 0 {\n\t\tt.Fatalf(\"list tokens failed: %s\", eno)\n\t}\n\tif !reflect.DeepEqual(tk2, buildToken(tokens[id2])) {\n\t\tt.Fatalf(\"token2 mismatch: %+v != %+v\", tk2, buildToken(tokens[id2]))\n\t}\n\tif !reflect.DeepEqual(tk1, buildToken(tokens[id1])) {\n\t\tt.Fatalf(\"token1 mismatch: %+v != %+v\", tk1, buildToken(tokens[id1]))\n\t}\n\n\teno = m.DeleteTokens(ctx, []uint32{id1, id2})\n\tif eno != 0 {\n\t\tt.Fatalf(\"delete tokens failed: %s\", eno)\n\t}\n\ttokens, eno = m.ListTokens(ctx)\n\tif eno != 0 {\n\t\tt.Fatalf(\"list tokens failed: %s\", eno)\n\t}\n\tif tokens[id1] != nil || tokens[id2] != nil {\n\t\tt.Fatalf(\"tokens not deleted\")\n\t}\n}\n\nfunc testMetaClient(t *testing.T, m Meta) {\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error { return nil })\n\tctx := Background()\n\tvar attr = &Attr{}\n\tif st := m.GetAttr(ctx, 1, attr); st != 0 || attr.Mode != 0777 { // getattr of root always succeed\n\t\tt.Fatalf(\"getattr root: %s\", st)\n\t}\n\n\tif err := m.Init(testFormat(), true); err != nil {\n\t\tt.Fatalf(\"initialize failed: %s\", err)\n\t}\n\tif err := m.Init(&Format{Name: \"test2\"}, false); err == nil { // not allowed\n\t\tt.Fatalf(\"change name without --force is not allowed\")\n\t}\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\tt.Fatalf(\"load failed after initialization: %s\", err)\n\t}\n\tif format.Name != \"test\" {\n\t\tt.Fatalf(\"load got volume name %s, expected %s\", format.Name, \"test\")\n\t}\n\tif err = m.NewSession(true); err != nil {\n\t\tt.Fatalf(\"new session: %s\", err)\n\t}\n\tdefer m.CloseSession()\n\tses, err := m.ListSessions()\n\tif err != nil || len(ses) != 1 {\n\t\tt.Fatalf(\"list sessions %+v: %s\", ses, err)\n\t}\n\tbase := m.getBase()\n\tif base.sid != ses[0].Sid {\n\t\tt.Fatalf(\"my sid %d != registered sid %d\", base.sid, ses[0].Sid)\n\t}\n\tgo m.CleanStaleSessions(Background())\n\n\tvar parent, inode, dummyInode Ino\n\tif st := m.Mkdir(ctx, 1, \"d\", 0640, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tdefer m.Rmdir(ctx, 1, \"d\")\n\tif st := m.Unlink(ctx, 1, \"d\"); st != syscall.EPERM {\n\t\tt.Fatalf(\"unlink d: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, parent, \".\"); st != syscall.EINVAL {\n\t\tt.Fatalf(\"unlink d.: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, parent, \"..\"); st != syscall.ENOTEMPTY {\n\t\tt.Fatalf(\"unlink d..: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, 1, \"d\", &parent, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup d: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, 1, \"d\", &parent, nil, true); st != syscall.EINVAL {\n\t\tt.Fatalf(\"lookup d: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, 1, \"..\", &inode, attr, true); st != 0 || inode != 1 {\n\t\tt.Fatalf(\"lookup ..: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, parent, \".\", &inode, attr, true); st != 0 || inode != parent {\n\t\tt.Fatalf(\"lookup .: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, parent, \"..\", &inode, attr, true); st != 0 || inode != 1 {\n\t\tt.Fatalf(\"lookup ..: %s\", st)\n\t}\n\tif attr.Nlink != 3 {\n\t\tt.Fatalf(\"nlink expect 3, but got %d\", attr.Nlink)\n\t}\n\tif st := m.Access(ctx, parent, 4, attr); st != 0 {\n\t\tt.Fatalf(\"access d: %s\", st)\n\t}\n\tif st := m.Create(ctx, parent, \"f\", 0650, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\t_ = m.Close(ctx, inode)\n\tvar tino Ino\n\tif st := m.Lookup(ctx, inode, \".\", &tino, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup /d/f/.: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, inode, \"..\", &tino, attr, true); st != syscall.ENOTDIR {\n\t\tt.Fatalf(\"lookup /d/f/..: %s\", st)\n\t}\n\tdefer m.Unlink(ctx, parent, \"f\")\n\tif st := m.Rmdir(ctx, parent, \"f\"); st != syscall.ENOTDIR {\n\t\tt.Fatalf(\"rmdir f: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, 1, \"d\"); st != syscall.ENOTEMPTY {\n\t\tt.Fatalf(\"rmdir d: %s\", st)\n\t}\n\tif st := m.Mknod(ctx, inode, \"df\", TypeFile, 0650, 022, 0, \"\", &dummyInode, nil); st != syscall.ENOTDIR {\n\t\tt.Fatalf(\"create fd: %s\", st)\n\t}\n\tif st := m.Mknod(ctx, parent, \"f\", TypeFile, 0650, 022, 0, \"\", &inode, attr); st != syscall.EEXIST {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, parent, \"f\", &inode, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup f: %s\", st)\n\t}\n\tif st := m.Resolve(ctx, 1, \"d/f\", &inode, attr); st != 0 && st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"resolve d/f: %s\", st)\n\t}\n\tif st := m.Resolve(ctx, parent, \"/f\", &inode, attr); st != 0 && st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"resolve f: %s\", st)\n\t}\n\tvar ctx2 = NewContext(0, 1, []uint32{1})\n\tif st := m.Resolve(ctx2, parent, \"/f\", &inode, attr); st != syscall.EACCES && st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"resolve f: %s\", st)\n\t}\n\tif st := m.Resolve(ctx, parent, \"/f/c\", &inode, attr); st != syscall.ENOTDIR && st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"resolve f: %s\", st)\n\t}\n\tif st := m.Resolve(ctx, parent, \"/f2\", &inode, attr); st != syscall.ENOENT && st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"resolve f2: %s\", st)\n\t}\n\t// check owner permission\n\tvar p1, c1 Ino\n\tif st := m.Mkdir(ctx2, 1, \"d1\", 02777, 0, 0, &p1, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d1: %s\", st)\n\t}\n\tattr.Gid = 1\n\tm.SetAttr(ctx, p1, SetAttrGID, 0, attr)\n\tif attr.Mode&02000 == 0 {\n\t\tt.Fatalf(\"SGID is lost\")\n\t}\n\tvar ctx3 = NewContext(2, 2, []uint32{2})\n\tif st := m.Mkdir(ctx3, p1, \"d2\", 0777, 022, 0, &c1, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d2: %s\", st)\n\t}\n\tif attr.Gid != ctx2.Gid() {\n\t\tt.Fatalf(\"inherit gid: %d != %d\", attr.Gid, ctx2.Gid())\n\t}\n\tif runtime.GOOS == \"linux\" {\n\t\tif attr.Mode&02000 == 0 {\n\t\t\tt.Fatalf(\"not inherit sgid\")\n\t\t}\n\t\tif st := m.Mknod(ctx2, p1, \"f1\", TypeFile, 02777, 022, 0, \"\", &dummyInode, attr); st != 0 {\n\t\t\tt.Fatalf(\"create f1: %s\", st)\n\t\t} else if attr.Mode&02010 != 02010 {\n\t\t\tt.Fatalf(\"sgid should not be cleared\")\n\t\t}\n\t\tif st := m.Mknod(ctx3, p1, \"f2\", TypeFile, 02777, 022, 0, \"\", &dummyInode, attr); st != 0 {\n\t\t\tt.Fatalf(\"create f2: %s\", st)\n\t\t} else if attr.Mode&02010 != 00010 {\n\t\t\tt.Fatalf(\"sgid should be cleared\")\n\t\t}\n\n\t}\n\tif st := m.Resolve(ctx2, 1, \"/d1/d2\", nil, nil); st != 0 && st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"resolve /d1/d2: %s\", st)\n\t}\n\tif st := m.Remove(ctx, 1, \"d1\", false, RmrDefaultThreads, nil); st != 0 {\n\t\tt.Fatalf(\"Remove d1: %s\", st)\n\t}\n\tattr.Atime = 2\n\tattr.Mtime = 2\n\tattr.Uid = 1\n\tattr.Gid = 1\n\tattr.Mode = 0640\n\tif st := m.SetAttr(ctx, inode, SetAttrAtime|SetAttrMtime|SetAttrUID|SetAttrGID|SetAttrMode, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, inode, 0, 0, attr); st != 0 { // changes nothing\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.GetAttr(ctx, inode, attr); st != 0 {\n\t\tt.Fatalf(\"getattr f: %s\", st)\n\t}\n\tif attr.Atime != 2 || attr.Mtime != 2 || attr.Uid != 1 || attr.Gid != 1 || attr.Mode != 0640 {\n\t\tt.Fatalf(\"atime:%d mtime:%d uid:%d gid:%d mode:%o\", attr.Atime, attr.Mtime, attr.Uid, attr.Gid, attr.Mode)\n\t}\n\tif st := m.SetAttr(ctx, inode, SetAttrAtimeNow|SetAttrMtimeNow, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tfakeCtx := NewContext(100, 2, []uint32{2, 1})\n\tif st := m.Access(fakeCtx, parent, 2, nil); st != syscall.EACCES {\n\t\tt.Fatalf(\"access d: %s\", st)\n\t}\n\tif st := m.Access(fakeCtx, inode, 4, nil); st != 0 {\n\t\tt.Fatalf(\"access f: %s\", st)\n\t}\n\tvar entries []*Entry\n\tif st := m.Readdir(ctx, parent, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t} else if len(entries) != 3 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t} else if string(entries[0].Name) != \".\" || string(entries[1].Name) != \"..\" || string(entries[2].Name) != \"f\" {\n\t\tt.Fatalf(\"entries: %+v\", entries)\n\t}\n\tif st := m.Rename(ctx, parent, \"f\", 1, \"f2\", RenameWhiteout, &inode, attr); st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"rename d/f -> f2: %s\", st)\n\t}\n\tif st := m.Rename(ctx, parent, \"f\", 1, \"f2\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename d/f -> f2: %s\", st)\n\t}\n\tdefer func() {\n\t\t_ = m.Unlink(ctx, 1, \"f2\")\n\t}()\n\tif st := m.Rename(ctx, 1, \"f2\", 1, \"f2\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename f2 -> f2: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f2\", 1, \"f\", RenameExchange, &inode, attr); st != syscall.ENOENT {\n\t\tt.Fatalf(\"rename f2 -> f: %s\", st)\n\t}\n\tif st := m.Create(ctx, 1, \"f\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\t_ = m.Close(ctx, inode)\n\tdefer m.Unlink(ctx, 1, \"f\")\n\tif st := m.Rename(ctx, 1, \"f2\", 1, \"f\", RenameNoReplace, &inode, attr); st != syscall.EEXIST {\n\t\tt.Fatalf(\"rename f2 -> f: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f2\", 1, \"f\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename f2 -> f: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f\", 1, \"d\", RenameExchange, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename f <-> d: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f\", 1, \"d\", RenameExchange, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename f <-> d: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"d\", 1, \"f\", 0, &inode, attr); st != syscall.ENOTDIR {\n\t\tt.Fatalf(\"rename d -> f: %s\", st)\n\t}\n\tif st := m.GetAttr(ctx, 1, attr); st != 0 {\n\t\tt.Fatalf(\"getattr f: %s\", st)\n\t}\n\tif attr.Nlink != 3 {\n\t\tt.Fatalf(\"nlink expect 3, but got %d\", attr.Nlink)\n\t}\n\t// Test rename with parent change\n\tvar parent2 Ino\n\tif st := m.Mkdir(ctx, 1, \"d4\", 0777, 0, 0, &parent2, attr); st != 0 {\n\t\tt.Fatalf(\"create dir d4: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, parent2, \"d5\", 0777, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create dir d4/d5: %s\", st)\n\t}\n\tif st := m.Rename(ctx, parent2, \"d5\", 1, \"d5\", RenameNoReplace, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename d4/d5 <-> d5: %s\", st)\n\t} else if attr.Parent != 1 {\n\t\tt.Fatalf(\"after rename d4/d5 <-> d5 parent %d expect 1\", attr.Parent)\n\t}\n\tif st := m.Mknod(ctx, parent2, \"f6\", TypeFile, 0650, 022, 0, \"\", &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create dir d4/f6: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"d5\", parent2, \"f6\", RenameExchange, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename d5 <-> d4/d6: %s\", st)\n\t} else if attr.Parent != parent2 {\n\t\tt.Fatalf(\"after exchange d5 <-> d4/f6 parent %d expect %d\", attr.Parent, parent2)\n\t} else if attr.Typ != TypeDirectory {\n\t\tt.Fatalf(\"after exchange d5 <-> d4/f6 type %d expect %d\", attr.Typ, TypeDirectory)\n\t}\n\tif st := m.Lookup(ctx, 1, \"d5\", &inode, attr, true); st != 0 || attr.Parent != 1 {\n\t\tt.Fatalf(\"lookup d5 after exchange: %s; parent %d expect 1\", st, attr.Parent)\n\t} else if attr.Typ != TypeFile {\n\t\tt.Fatalf(\"after exchange d5 <-> d4/f6 type %d expect %d\", attr.Typ, TypeFile)\n\t}\n\tif st := m.Rmdir(ctx, parent2, \"f6\"); st != 0 {\n\t\tt.Fatalf(\"rmdir d4/f6 : %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, 1, \"d4\"); st != 0 {\n\t\tt.Fatalf(\"rmdir d4 first : %s\", st)\n\t}\n\tif st := m.Unlink(ctx, 1, \"d5\"); st != 0 {\n\t\tt.Fatalf(\"rmdir d6 : %s\", st)\n\t}\n\tif st := m.Lookup(ctx, 1, \"f\", &inode, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup f: %s\", st)\n\t}\n\tif st := m.Link(ctx, inode, 1, \"f3\", attr); st != 0 {\n\t\tt.Fatalf(\"link f3 -> f: %s\", st)\n\t}\n\tdefer m.Unlink(ctx, 1, \"f3\")\n\tif st := m.Link(ctx, inode, 1, \"F3\", attr); st != 0 { // CaseInsensi = false\n\t\tt.Fatalf(\"link F3 -> f: %s\", st)\n\t}\n\tif st := m.Link(ctx, parent, 1, \"d2\", attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"link d2 -> d: %s\", st)\n\t}\n\tif st := m.Symlink(ctx, 1, \"s\", \"/f\", &inode, attr); st != 0 {\n\t\tt.Fatalf(\"symlink s -> /f: %s\", st)\n\t}\n\tif attr.Mode&0777 != 0777 {\n\t\tt.Fatalf(\"mode of symlink should be 0777\")\n\t}\n\tdefer m.Unlink(ctx, 1, \"s\")\n\tvar target1, target2 []byte\n\tif st := m.ReadLink(ctx, inode, &target1); st != 0 {\n\t\tt.Fatalf(\"readlink s: %s\", st)\n\t}\n\tif st := m.ReadLink(ctx, inode, &target2); st != 0 { // cached\n\t\tt.Fatalf(\"readlink s: %s\", st)\n\t}\n\tif !bytes.Equal(target1, target2) || !bytes.Equal(target1, []byte(\"/f\")) {\n\t\tt.Fatalf(\"readlink got %s %s, expected %s\", target1, target2, \"/f\")\n\t}\n\tif st := m.ReadLink(ctx, parent, &target1); st != syscall.EINVAL {\n\t\tt.Fatalf(\"readlink d: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, 1, \"f\", &inode, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup f: %s\", st)\n\t}\n\n\t// data\n\tvar sliceId uint64\n\t// try to open a file that does not exist\n\tif st := m.Open(ctx, 99999, syscall.O_RDWR, &Attr{}); st != syscall.ENOENT {\n\t\tt.Fatalf(\"open not exist inode got %d, expected %d\", st, syscall.ENOENT)\n\t}\n\tif st := m.Open(ctx, inode, syscall.O_RDWR, attr); st != 0 {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\t_ = m.Close(ctx, inode)\n\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\tt.Fatalf(\"write chunk: %s\", st)\n\t}\n\tvar s = Slice{Id: sliceId, Size: 100, Len: 100}\n\tif st := m.Write(ctx, inode, 0, 100, s, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write end: %s\", st)\n\t}\n\tvar slices []Slice\n\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\tt.Fatalf(\"read chunk: %s\", st)\n\t}\n\tif len(slices) != 2 || slices[0].Id != 0 || slices[0].Size != 100 || slices[1].Id != sliceId || slices[1].Size != 100 {\n\t\tt.Fatalf(\"slices: %v\", slices)\n\t}\n\tif st := m.Fallocate(ctx, inode, fallocPunchHole|fallocKeepSize, 100, 50, nil); st != 0 {\n\t\tt.Fatalf(\"fallocate: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, inode, fallocPunchHole|fallocCollapesRange, 100, 50, nil); st != syscall.EINVAL {\n\t\tt.Fatalf(\"fallocate: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, inode, fallocPunchHole|fallocInsertRange, 100, 50, nil); st != syscall.EINVAL {\n\t\tt.Fatalf(\"fallocate: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, inode, fallocCollapesRange, 100, 50, nil); st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"fallocate: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, inode, fallocPunchHole, 100, 50, nil); st != syscall.EINVAL {\n\t\tt.Fatalf(\"fallocate: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, inode, fallocPunchHole|fallocKeepSize, 0, 0, nil); st != syscall.EINVAL {\n\t\tt.Fatalf(\"fallocate: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, parent, fallocPunchHole|fallocKeepSize, 100, 50, nil); st != syscall.EPERM {\n\t\tt.Fatalf(\"fallocate dir: %s\", st)\n\t}\n\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\tt.Fatalf(\"read chunk: %s\", st)\n\t}\n\tif len(slices) != 3 || slices[1].Id != 0 || slices[1].Len != 50 || slices[2].Id != sliceId || slices[2].Len != 50 {\n\t\tt.Fatalf(\"slices: %v\", slices)\n\t}\n\n\t// xattr\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v\"), XattrCreateOrReplace); st != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v2\"), XattrCreateOrReplace); st != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tvar value []byte\n\tif st := m.GetXattr(ctx, inode, \"a\", &value); st != 0 || string(value) != \"v2\" {\n\t\tt.Fatalf(\"getxattr: %s %v\", st, value)\n\t}\n\tif st := m.ListXattr(ctx, inode, &value); st != 0 || string(value) != \"a\\000\" {\n\t\tt.Fatalf(\"listxattr: %s %v\", st, value)\n\t}\n\tif st := m.Unlink(ctx, 1, \"F3\"); st != 0 {\n\t\tt.Fatalf(\"unlink F3: %s\", st)\n\t}\n\tif st := m.GetXattr(ctx, inode, \"a\", &value); st != 0 || string(value) != \"v2\" {\n\t\tt.Fatalf(\"getxattr: %s %v\", st, value)\n\t}\n\tif st := m.RemoveXattr(ctx, inode, \"a\"); st != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v\"), XattrReplace); st != ENOATTR {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v3\"), XattrCreate); st != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v3\"), XattrCreate); st != syscall.EEXIST {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v3\"), XattrReplace); st != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v4\"), XattrReplace); st != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, inode, \"a\", []byte(\"v5\"), 5); st != syscall.EINVAL {\n\t\tt.Fatalf(\"setxattr: %s\", st)\n\t}\n\n\tvar totalspace, availspace, iused, iavail uint64\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<50 || iavail != 10<<20 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\tformat.Capacity = 1 << 20\n\tformat.Inodes = 100\n\tif err = m.Init(format, false); err != nil {\n\t\tt.Fatalf(\"set quota failed: %s\", err)\n\t}\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<20 || iavail != 97 {\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\t_ = m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail)\n\t\tif totalspace != 1<<20 || iavail != 97 {\n\t\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t\t}\n\t}\n\t// test StatFS with subdir and quota\n\tvar subIno Ino\n\tif st := m.Mkdir(ctx, 1, \"subdir\", 0755, 0, 0, &subIno, nil); st != 0 {\n\t\tt.Fatalf(\"mkdir subdir: %s\", st)\n\t}\n\tif st := m.Chroot(ctx, \"subdir\"); st != 0 {\n\t\tt.Fatalf(\"chroot: %s\", st)\n\t}\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<20 || iavail != 96 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"/subdir\", 0, 0, map[string]*Quota{\n\t\t\"/subdir\": {\n\t\t\tMaxSpace: 0,\n\t\t\tMaxInodes: 0,\n\t\t},\n\t}, false, false, false); err != nil {\n\t\tt.Fatalf(\"set quota: %s\", err)\n\t}\n\tbase.loadQuotas()\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<20-4*uint64(align4K(0)) || iavail != 96 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"/subdir\", 0, 0, map[string]*Quota{\n\t\t\"/subdir\": {\n\t\t\tMaxSpace: 1 << 10,\n\t\t\tMaxInodes: 0,\n\t\t},\n\t}, false, false, false); err != nil {\n\t\tt.Fatalf(\"set quota: %s\", err)\n\t}\n\tbase.loadQuotas()\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<10 || iavail != 96 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"/subdir\", 0, 0, map[string]*Quota{\n\t\t\"/subdir\": {\n\t\t\tMaxSpace: 0,\n\t\t\tMaxInodes: 10,\n\t\t},\n\t}, false, false, false); err != nil {\n\t\tt.Fatalf(\"set quota: %s\", err)\n\t}\n\tbase.loadQuotas()\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<20-4*uint64(align4K(0)) || iavail != 10 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"/subdir\", 0, 0, map[string]*Quota{\n\t\t\"/subdir\": {\n\t\t\tMaxSpace: 1 << 10,\n\t\t\tMaxInodes: 10,\n\t\t},\n\t}, false, false, false); err != nil {\n\t\tt.Fatalf(\"set quota: %s\", err)\n\t}\n\tbase.loadQuotas()\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<10 || iavail != 10 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\n\tm.chroot(RootInode)\n\tif st := m.StatFS(ctx, RootInode, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<20 || iavail != 96 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\t// statfs subdir directly\n\tif st := m.StatFS(ctx, subIno, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<10 || iavail != 10 {\n\t\tt.Fatalf(\"total space %d, iavail %d\", totalspace, iavail)\n\t}\n\n\tbase.loadQuotas()\n\tbase.quotaMu.RLock()\n\tq := base.dirQuotas[uint64(subIno)]\n\tbase.quotaMu.RUnlock()\n\tq.update(4<<10, 15) // used > max\n\tbase.doFlushQuotas()\n\tif st := m.StatFS(ctx, subIno, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 4<<10 || availspace != 0 || iused != 15 || iavail != 0 {\n\t\tt.Fatalf(\"total space %d, availspace %d, iused %d, iavail %d\", totalspace, availspace, iused, iavail)\n\t}\n\tq.update(-8<<10, -20) // used < 0\n\tbase.doFlushQuotas()\n\tif st := m.StatFS(ctx, subIno, &totalspace, &availspace, &iused, &iavail); st != 0 {\n\t\tt.Fatalf(\"statfs: %s\", st)\n\t}\n\tif totalspace != 1<<10 || availspace != 1<<10 || iused != 0 || iavail != 10 {\n\t\tt.Fatalf(\"total space %d, availspace %d, iused %d, iavail %d\", totalspace, availspace, iused, iavail)\n\t}\n\n\tif st := m.Rmdir(ctx, 1, \"subdir\"); st != 0 {\n\t\tt.Fatalf(\"rmdir subdir: %s\", st)\n\t}\n\n\tvar summary Summary\n\tif st := m.GetSummary(ctx, parent, &summary, false, true); st != 0 {\n\t\tt.Fatalf(\"summary: %s\", st)\n\t}\n\texpected := Summary{Length: 0, Size: 4096, Files: 0, Dirs: 1}\n\tif summary != expected {\n\t\tt.Fatalf(\"summary %+v not equal to expected: %+v\", summary, expected)\n\t}\n\tsummary = Summary{}\n\tif st := m.GetSummary(ctx, 1, &summary, true, true); st != 0 {\n\t\tt.Fatalf(\"summary: %s\", st)\n\t}\n\texpected = Summary{Length: 400, Size: 20480, Files: 3, Dirs: 2}\n\tif summary != expected {\n\t\tt.Fatalf(\"summary %+v not equal to expected: %+v\", summary, expected)\n\t}\n\tif st := m.GetSummary(ctx, inode, &summary, true, true); st != 0 {\n\t\tt.Fatalf(\"summary: %s\", st)\n\t}\n\texpected = Summary{Length: 600, Size: 24576, Files: 4, Dirs: 2}\n\tif summary != expected {\n\t\tt.Fatalf(\"summary %+v not equal to expected: %+v\", summary, expected)\n\t}\n\tif st := m.Unlink(ctx, 1, \"f\"); st != 0 {\n\t\tt.Fatalf(\"unlink f: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, 1, \"f3\"); st != 0 {\n\t\tt.Fatalf(\"unlink f3: %s\", st)\n\t}\n\ttime.Sleep(time.Millisecond * 100) // wait for delete\n\tif st := m.Read(ctx, inode, 0, &slices); st != syscall.ENOENT {\n\t\tt.Fatalf(\"read chunk: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, 1, \"d\"); st != 0 {\n\t\tt.Fatalf(\"rmdir d: %s\", st)\n\t}\n}\n\nfunc testStickyBit(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar sticky, normal, inode Ino\n\tvar attr = &Attr{}\n\tm.Mkdir(ctx, 1, \"tmp\", 01777, 0, 0, &sticky, attr)\n\tm.Mkdir(ctx, 1, \"tmp2\", 0777, 0, 0, &normal, attr)\n\tctxA := NewContext(1, 1, []uint32{1})\n\t// file\n\tm.Create(ctxA, sticky, \"f\", 0777, 0, 0, &inode, attr)\n\tm.Create(ctxA, normal, \"f\", 0777, 0, 0, &inode, attr)\n\tctxB := NewContext(1, 2, []uint32{2})\n\tif e := m.Unlink(ctxB, sticky, \"f\"); e != syscall.EACCES {\n\t\tt.Fatalf(\"unlink f: %s\", e)\n\t}\n\tif e := m.Rename(ctxB, sticky, \"f\", sticky, \"f2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename f: %s\", e)\n\t}\n\tif e := m.Rename(ctxB, sticky, \"f\", normal, \"f2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename f: %s\", e)\n\t}\n\tm.Create(ctxB, sticky, \"f2\", 0777, 0, 0, &inode, attr)\n\tif e := m.Rename(ctxB, sticky, \"f2\", sticky, \"f\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"overwrite f: %s\", e)\n\t}\n\tif e := m.Rename(ctxA, sticky, \"f\", sticky, \"f2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename f: %s\", e)\n\t}\n\tif e := m.Rename(ctxA, normal, \"f\", sticky, \"f2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename f: %s\", e)\n\t}\n\tif e := m.Rename(ctxA, sticky, \"f\", sticky, \"f3\", 0, &inode, attr); e != 0 {\n\t\tt.Fatalf(\"rename f: %s\", e)\n\t}\n\tif e := m.Unlink(ctxA, sticky, \"f3\"); e != 0 {\n\t\tt.Fatalf(\"unlink f3: %s\", e)\n\t}\n\t// dir\n\tm.Mkdir(ctxA, sticky, \"d\", 0777, 0, 0, &inode, attr)\n\tm.Mkdir(ctxA, normal, \"d\", 0777, 0, 0, &inode, attr)\n\tif e := m.Rmdir(ctxB, sticky, \"d\"); e != syscall.EACCES {\n\t\tt.Fatalf(\"rmdir d: %s\", e)\n\t}\n\tif e := m.Rename(ctxB, sticky, \"d\", sticky, \"d2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename d: %s\", e)\n\t}\n\tif e := m.Rename(ctxB, sticky, \"d\", normal, \"d2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename d: %s\", e)\n\t}\n\tm.Mkdir(ctxB, sticky, \"d2\", 0777, 0, 0, &inode, attr)\n\tif e := m.Rename(ctxB, sticky, \"d2\", sticky, \"d\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"overwrite d: %s\", e)\n\t}\n\tif e := m.Rename(ctxA, sticky, \"d\", sticky, \"d2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename d: %s\", e)\n\t}\n\tif e := m.Rename(ctxA, normal, \"d\", sticky, \"d2\", 0, &inode, attr); e != syscall.EACCES {\n\t\tt.Fatalf(\"rename d: %s\", e)\n\t}\n\tif e := m.Rename(ctxA, sticky, \"d\", sticky, \"d3\", 0, &inode, attr); e != 0 {\n\t\tt.Fatalf(\"rename d: %s\", e)\n\t}\n\tif e := m.Rmdir(ctxA, sticky, \"d3\"); e != 0 {\n\t\tt.Fatalf(\"rmdir d3: %s\", e)\n\t}\n}\n\nfunc testListLocks(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tdefer m.Unlink(ctx, 1, \"f\")\n\tif st := m.Create(ctx, 1, \"f\", 0644, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 0 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\n\t// flock\n\to1 := uint64(0xF000000000000001)\n\tif st := m.Flock(ctx, inode, o1, syscall.F_WRLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 0 || len(flocks) != 1 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_UNLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock unlock: %s\", st)\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 0 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\tfor i := 2; i < 10; i++ {\n\t\tif st := m.Flock(ctx, inode, uint64(i), syscall.F_RDLCK, false); st != 0 {\n\t\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t\t}\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 0 || len(flocks) != 8 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\tfor i := 2; i < 10; i++ {\n\t\tif st := m.Flock(ctx, inode, uint64(i), syscall.F_UNLCK, false); st != 0 {\n\t\t\tt.Fatalf(\"flock unlock: %s\", st)\n\t\t}\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 0 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\n\t// plock\n\tif st := m.Setlk(ctx, inode, o1, false, syscall.F_WRLCK, 0, 0xFFFF, 1); st != 0 {\n\t\tt.Fatalf(\"plock rlock: %s\", st)\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 1 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\tif st := m.Setlk(ctx, inode, o1, false, syscall.F_UNLCK, 0, 0xFFFF, 1); st != 0 {\n\t\tt.Fatalf(\"plock unlock: %s\", st)\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 0 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\tfor i := 2; i < 10; i++ {\n\t\tif st := m.Setlk(ctx, inode, uint64(i), false, syscall.F_RDLCK, 0, 0xFFFF, 1); st != 0 {\n\t\t\tt.Fatalf(\"plock rlock: %s\", st)\n\t\t}\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 8 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n\tfor i := 2; i < 10; i++ {\n\t\tif st := m.Setlk(ctx, inode, uint64(i), false, syscall.F_UNLCK, 0, 0xFFFF, 1); st != 0 {\n\t\t\tt.Fatalf(\"plock unlock: %s\", st)\n\t\t}\n\t}\n\tif plocks, flocks, err := m.ListLocks(ctx, inode); err != nil || len(plocks) != 0 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n}\n\nfunc testLocks(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tdefer m.Unlink(ctx, 1, \"f\")\n\tif st := m.Create(ctx, 1, \"f\", 0644, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\t// flock\n\to1 := uint64(0xF000000000000001)\n\tif st := m.Flock(ctx, inode, o1, syscall.F_WRLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_WRLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_RDLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock rlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, 2, syscall.F_RDLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock rlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, 2, syscall.F_UNLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock unlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_WRLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_UNLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock unlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_RDLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock rlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, 2, syscall.F_RDLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock rlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_WRLCK, false); st != syscall.EAGAIN {\n\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, 2, syscall.F_UNLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock unlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_WRLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock wlock again: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, 2, syscall.F_WRLCK, false); st != syscall.EAGAIN {\n\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, 2, syscall.F_RDLCK, false); st != syscall.EAGAIN {\n\t\tt.Fatalf(\"flock rlock: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, o1, syscall.F_UNLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock unlock: %s\", st)\n\t}\n\tif r, ok := m.(*redisMeta); ok {\n\t\tms, err := r.rdb.SMembers(context.Background(), r.lockedKey(r.sid)).Result()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Smember %s: %s\", r.lockedKey(r.sid), err)\n\t\t}\n\t\tif len(ms) != 0 {\n\t\t\tt.Fatalf(\"locked inodes leaked: %d\", len(ms))\n\t\t}\n\t}\n\n\t// POSIX locks\n\tif st := m.Setlk(ctx, inode, o1, false, syscall.F_UNLCK, 0, 0xFFFF, 1); st != 0 {\n\t\tt.Fatalf(\"plock unlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, o1, false, syscall.F_RDLCK, 0, 0xFFFF, 1); st != 0 {\n\t\tt.Fatalf(\"plock rlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, o1, false, syscall.F_RDLCK, 0, 0xFFFF, 1); st != 0 {\n\t\tt.Fatalf(\"plock rlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, 2, false, syscall.F_RDLCK, 0, 0x2FFFF, 1); st != 0 {\n\t\tt.Fatalf(\"plock rlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, 2, false, syscall.F_WRLCK, 0, 0xFFFF, 1); st != syscall.EAGAIN {\n\t\tt.Fatalf(\"plock wlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, 2, false, syscall.F_WRLCK, 0x10000, 0x20000, 1); st != 0 {\n\t\tt.Fatalf(\"plock wlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, o1, false, syscall.F_UNLCK, 0, 0x20000, 1); st != 0 {\n\t\tt.Fatalf(\"plock unlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, 2, false, syscall.F_WRLCK, 0, 0xFFFF, 10); st != 0 {\n\t\tt.Fatalf(\"plock wlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, 2, false, syscall.F_WRLCK, 0x2000, 0xFFFF, 20); st != 0 {\n\t\tt.Fatalf(\"plock wlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, o1, false, syscall.F_WRLCK, 0, 0xFFFF, 1); st != syscall.EAGAIN {\n\t\tt.Fatalf(\"plock rlock: %s\", st)\n\t}\n\tvar ltype, pid uint32 = syscall.F_WRLCK, 1\n\tvar start, end uint64 = 0x2000, 0xFFFF\n\tif st := m.Getlk(ctx, inode, o1, <ype, &start, &end, &pid); st != 0 || ltype != syscall.F_WRLCK || pid != 20 || start != 0x2000 || end != 0xFFFF {\n\t\tt.Fatalf(\"plock get rlock: %s, %d %d %x %x\", st, ltype, pid, start, end)\n\t}\n\tif st := m.Setlk(ctx, inode, 2, false, syscall.F_UNLCK, 0, 0x2FFFF, 1); st != 0 {\n\t\tt.Fatalf(\"plock unlock: %s\", st)\n\t}\n\tltype = syscall.F_WRLCK\n\tstart, end = 0, 0xFFFFFF\n\tif st := m.Getlk(ctx, inode, o1, <ype, &start, &end, &pid); st != 0 || ltype != syscall.F_UNLCK || pid != 0 || start != 0 || end != 0 {\n\t\tt.Fatalf(\"plock get rlock: %s, %d %d %x %x\", st, ltype, pid, start, end)\n\t}\n\n\t// concurrent locks\n\tvar g sync.WaitGroup\n\tvar count int\n\tvar err syscall.Errno\n\tfor i := 0; i < 100; i++ {\n\t\tg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdefer g.Done()\n\t\t\tif st := m.Setlk(ctx, inode, uint64(i), true, syscall.F_WRLCK, 0, 0xFFFF, uint32(i)); st != 0 {\n\t\t\t\terr = st\n\t\t\t}\n\t\t\tcount++\n\t\t\ttime.Sleep(time.Millisecond)\n\t\t\tcount--\n\t\t\tif count > 0 {\n\t\t\t\tpanic(fmt.Errorf(\"count should be zero but got %d\", count))\n\t\t\t}\n\t\t\tif st := m.Setlk(ctx, inode, uint64(i), false, syscall.F_UNLCK, 0, 0xFFFF, uint32(i)); st != 0 {\n\t\t\t\tpanic(fmt.Errorf(\"plock unlock: %s\", st))\n\t\t\t}\n\t\t}(i)\n\t}\n\tg.Wait()\n\tif err != 0 {\n\t\tt.Fatalf(\"lock fail: %s\", err)\n\t}\n\n\tif r, ok := m.(*redisMeta); ok {\n\t\tms, err := r.rdb.SMembers(context.Background(), r.lockedKey(r.sid)).Result()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Smember %s: %s\", r.lockedKey(r.sid), err)\n\t\t}\n\t\tif len(ms) != 0 {\n\t\t\tt.Fatalf(\"locked inode leaked: %d\", len(ms))\n\t\t}\n\t}\n}\n\nfunc testResolve(t *testing.T, m Meta) {\n\tvar inode, parent Ino\n\tvar attr, pattr Attr\n\tif st := m.Mkdir(NewContext(1, 65534, []uint32{65534}), 1, \"d\", 0770, 0, 0, &parent, &pattr); st != 0 {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tif pattr.Gid != 65534 {\n\t\tpattr.Gid = 65534\n\t\tif st := m.SetAttr(NewContext(1, 65534, []uint32{65534}), parent, SetAttrGID, 0, &pattr); st != 0 {\n\t\t\tt.Fatalf(\"setattr gid: %s\", st)\n\t\t}\n\t}\n\n\tif pattr.Uid != 65534 || pattr.Gid != 65534 {\n\t\tt.Fatalf(\"attr %+v\", pattr)\n\t}\n\tif st := m.Create(NewContext(1, 65534, []uint32{65534}), parent, \"f\", 0644, 0, 0, &inode, &attr); st != 0 {\n\t\tt.Fatalf(\"create /d/f: %s\", st)\n\t}\n\n\tdefer func() {\n\t\tif st := m.Remove(NewContext(0, 65534, []uint32{65534}), parent, \"f\", false, RmrDefaultThreads, nil); st != 0 {\n\t\t\tt.Fatalf(\"remove /d/f by owner: %s\", st)\n\t\t}\n\t\tif st := m.Rmdir(NewContext(0, 65534, []uint32{65534}), 1, \"d\"); st != 0 {\n\t\t\tt.Fatalf(\"rmdir /d by owner: %s\", st)\n\t\t}\n\t}()\n\n\tif st := m.Resolve(NewContext(0, 65534, []uint32{65534}), 1, \"/d/f\", &inode, &attr); st != 0 {\n\t\tif st == syscall.ENOTSUP {\n\t\t\treturn\n\t\t}\n\t\tt.Fatalf(\"resolve /d/f by owner: %s\", st)\n\t}\n\tif st := m.Resolve(NewContext(0, 65533, []uint32{65534}), 1, \"/d/f\", &inode, &attr); st != 0 {\n\t\tt.Fatalf(\"resolve /d/f by group: %s\", st)\n\t}\n\tif st := m.Resolve(NewContext(0, 65533, []uint32{65533, 65534}), 1, \"/d/f\", &inode, &attr); st != 0 {\n\t\tt.Fatalf(\"resolve /d/f by multi-group: %s\", st)\n\t}\n\tif st := m.Resolve(NewContext(0, 65533, []uint32{65533}), 1, \"/d/f\", &inode, &attr); st != syscall.EACCES {\n\t\tt.Fatalf(\"resolve /d/f by non-group: %s\", st)\n\t}\n}\n\nfunc testRemove(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode, parent Ino\n\tvar attr = &Attr{}\n\tif st := m.Create(ctx, 1, \"f\", 0644, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif st := m.Remove(ctx, 1, \"f\", false, RmrDefaultThreads, nil); st != 0 {\n\t\tt.Fatalf(\"rmr f: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, 1, \"d\", 0755, 0, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, parent, \"d2\", 0755, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create d/d2: %s\", st)\n\t}\n\tif st := m.Create(ctx, parent, \"f\", 0644, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create d/f: %s\", st)\n\t}\n\tif ps := m.GetPaths(ctx, parent); len(ps) == 0 || ps[0] != \"/d\" {\n\t\tt.Fatalf(\"get path /d: %v\", ps)\n\t}\n\tif ps := m.GetPaths(ctx, inode); len(ps) == 0 || ps[0] != \"/d/f\" {\n\t\tt.Fatalf(\"get path /d/f: %v\", ps)\n\t}\n\tfor i := 0; i < 4096; i++ {\n\t\tif st := m.Create(ctx, 1, \"f\"+strconv.Itoa(i), 0644, 0, 0, &inode, attr); st != 0 {\n\t\t\tt.Fatalf(\"create f%s: %s\", strconv.Itoa(i), st)\n\t\t}\n\t}\n\tvar entries []*Entry\n\tif st := m.Readdir(ctx, 1, 1, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t} else if len(entries) != 4099 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\tif st := m.Remove(ctx, 1, \"d\", false, RmrDefaultThreads, nil); st != 0 {\n\t\tt.Fatalf(\"rmr d: %s\", st)\n\t}\n}\n\nfunc testCaseIncensi(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\t_ = m.Create(ctx, 1, \"foo\", 0755, 0, 0, &inode, attr)\n\tif st := m.Create(ctx, 1, \"Foo\", 0755, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create Foo should be ok\")\n\t}\n\tif st := m.Create(ctx, 1, \"Foo\", 0755, 0, syscall.O_EXCL, &inode, attr); st != syscall.EEXIST {\n\t\tt.Fatalf(\"create should fail with EEXIST\")\n\t}\n\tif st := m.Lookup(ctx, 1, \"Foo\", &inode, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup Foo should be OK\")\n\t}\n\tif st := m.Rename(ctx, 1, \"Foo\", 1, \"bar\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename Foo to bar should be OK, but got %s\", st)\n\t}\n\tif st := m.Create(ctx, 1, \"Foo\", 0755, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create Foo should be OK\")\n\t}\n\tif st := m.Resolve(ctx, 1, \"/Foo\", &inode, attr); st != syscall.ENOTSUP {\n\t\tt.Fatalf(\"resolve with case insensitive should be ENOTSUP\")\n\t}\n\tif st := m.Lookup(ctx, 1, \"Bar\", &inode, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup Bar should be OK\")\n\t}\n\tif st := m.Link(ctx, inode, 1, \"foo\", attr); st != syscall.EEXIST {\n\t\tt.Fatalf(\"link should fail with EEXIST\")\n\t}\n\tif st := m.Unlink(ctx, 1, \"Bar\"); st != 0 {\n\t\tt.Fatalf(\"unlink Bar should be OK\")\n\t}\n\tif st := m.Unlink(ctx, 1, \"foo\"); st != 0 {\n\t\tt.Fatalf(\"unlink foo should be OK\")\n\t}\n\tif st := m.Mkdir(ctx, 1, \"Foo\", 0755, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir Foo should be OK, but got %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, 1, \"foo\"); st != 0 {\n\t\tt.Fatalf(\"rmdir foo should be OK\")\n\t}\n}\n\nfunc testCaseIncensiRename(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\n\t_ = m.Create(ctx, 1, \"aaa\", 0755, 0, 0, &inode, attr)\n\tif st := m.Create(ctx, 1, \"AAA\", 0755, 0, syscall.O_EXCL, &inode, attr); st == 0 {\n\t\tt.Fatalf(\"create AAA should NOT be ok\")\n\t}\n\n\t_ = m.Create(ctx, 1, \"bbb\", 0755, 0, 0, &inode, attr)\n\n\t/* NOW we have:\n\t/aaa\n\t/bbb\n\t*/\n\n\tif st := m.Rename(ctx, 1, \"aaa\", 1, \"AAA\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename aaa to AAA should be OK, bug got : %s\", st)\n\t}\n\n\tif st := m.Rename(ctx, 1, \"aaa\", 1, \"AAA\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename aaa to AAA again should be OK, bug got : %s\", st)\n\t}\n\n\tif st := m.Rename(ctx, 1, \"aaa\", 1, \"BBB\", RenameNoReplace, &inode, attr); st == 0 {\n\t\tt.Fatal(\"rename aaa to BBB (RenameNoReplace) should NOT be OK\")\n\t}\n\n\tif st := m.Rename(ctx, 1, \"aaa\", 1, \"BBB\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename aaa to BBB should be OK, but got: %s\", st)\n\t}\n\n\t/* NOW we have:\n\t/BBB\n\t*/\n\n\tif st := m.Create(ctx, 1, \"aaa\", 0755, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create aaa should be ok, but got %s\", st)\n\t}\n\n\t/*NOW we have:\n\t/BBB\n\t/aaa\n\t*/\n\n\tif st := m.Rename(ctx, 1, \"aaa\", 1, \"Aaa\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename aaa to Aaa should be OK, but got %s\", st)\n\t}\n\n\tvar dirInode Ino\n\n\tif st := m.Mkdir(ctx, 1, \"case_insensi_dir\", 0755, 0, 0, &dirInode, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir case_insensi_dir should be OK, but got %s\", st)\n\t}\n\n\tif st := m.Create(ctx, dirInode, \"AAA\", 0755, 0, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create case_insensi_dir/AAA should be ok, but got %s\", st)\n\t}\n\n\t/*NOW we have:\n\t/BBB\n\t/Aaa\n\t/case_insensi_dir/AAA\n\t*/\n\n\tif st := m.Rename(ctx, 1, \"aaa\", dirInode, \"aaa\", RenameNoReplace, &inode, attr); st == 0 {\n\t\tt.Fatalf(\"rename aaa to case_insensi_dir/aaa (RenameNoReplace) should NOT be OK\")\n\t}\n\n\tif st := m.Rename(ctx, 1, \"aaa\", dirInode, \"aaa\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename Aaa to case_insensi_dir/aaa should be OK, but got %s\", st)\n\t}\n}\n\nfunc testCaseIncensiHardlinkRename(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\n\t_ = m.Create(ctx, 1, \"ccc\", 0755, 0, 0, &inode, attr)\n\tif st := m.Link(ctx, inode, 1, \"CCC\", attr); st == 0 {\n\t\tt.Fatalf(\"create hardlink CCC should NOT be ok\")\n\t}\n\n\tif st := m.Link(ctx, inode, 1, \"ddd\", attr); st != 0 {\n\t\tt.Fatalf(\"create hardlink ddd should be ok, but got %s\", st)\n\t}\n\n\t/* NOW we have:\n\t/ccc\n\t/ddd\n\t*/\n\n\tif st := m.Rename(ctx, 1, \"ccc\", 1, \"CCC\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename ccc to CCC should be OK, bug got : %s\", st)\n\t}\n\n\tif st := m.Rename(ctx, 1, \"ccc\", 1, \"DDD\", RenameNoReplace, &inode, attr); st != syscall.EEXIST {\n\t\tt.Fatal(\"rename ccc to DDD (RenameNoReplace) should fail with EEXIST\")\n\t}\n\n\tif st := m.Rename(ctx, 1, \"ccc\", 1, \"DDD\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename ccc to DDD shouldshould fail silently\")\n\t}\n\n\tif st := m.Lookup(ctx, 1, \"ccc\", &inode, attr, false); st != 0 {\n\t\tt.Fatalf(\"Lookup ccc should be OK, but got %s\", st)\n\t}\n\n\tif st := m.Lookup(ctx, 1, \"ddd\", &inode, attr, false); st != 0 {\n\t\tt.Fatalf(\"Lookup ddd should be OK, but got %s\", st)\n\t}\n\n\t/* NOW we have:\n\t/ccc\n\t/ddd\n\t*/\n\n\tvar dirInode Ino\n\n\tif st := m.Mkdir(ctx, 1, \"case_insensi_hark_dir\", 0755, 0, 0, &dirInode, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir case_insensi_hark_dir should be OK, but got %s\", st)\n\t}\n\n\tif st := m.Link(ctx, inode, dirInode, \"DDD\", attr); st != 0 {\n\t\tt.Fatalf(\"create case_insensi_dir/DDD should be ok, but got %s\", st)\n\t}\n\n\t/*NOW we have:\n\t/ccc\n\t/ddd\n\t/case_insensi_dir/DDD\n\t*/\n\n\tif st := m.Rename(ctx, 1, \"ccc\", dirInode, \"ddd\", RenameNoReplace, &inode, attr); st != syscall.EEXIST {\n\t\tt.Fatalf(\"rename ccc to case_insensi_dir/ddd (RenameNoReplace) should fail with EEXIST\")\n\t}\n\n\tif st := m.Rename(ctx, 1, \"ccc\", dirInode, \"ddd\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename ccc to case_insensi_dir/ddd should fail silently\")\n\t}\n\n\tif st := m.Lookup(ctx, 1, \"ccc\", &inode, attr, false); st != 0 {\n\t\tt.Fatalf(\"resolve ccc should be OK, but got %s\", st)\n\t}\n\n\tif st := m.Rename(ctx, 1, \"ddd\", dirInode, \"ddd\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename ddd to case_insensi_dir/ddd should fail silently\")\n\t}\n\n\tif st := m.Lookup(ctx, 1, \"ddd\", &inode, attr, false); st != 0 {\n\t\tt.Fatalf(\"lookup ddd should be OK, but got %s\", st)\n\t}\n}\n\ntype compactor interface {\n\tcompactChunk(inode Ino, indx uint32, once, force bool)\n}\n\nfunc testCompaction(t *testing.T, m Meta, trash bool) {\n\tif trash {\n\t\tformat := testFormat()\n\t\tformat.TrashDays = 1\n\t\t_ = m.Init(format, false)\n\t\tdefer func() {\n\t\t\tif err := m.Init(testFormat(), false); err != nil {\n\t\t\t\tt.Fatalf(\"init: %v\", err)\n\t\t\t}\n\t\t}()\n\t} else {\n\t\t_ = m.Init(testFormat(), false)\n\t}\n\n\tif err := m.NewSession(false); err != nil {\n\t\tt.Fatalf(\"new session: %v\", err)\n\t}\n\tdefer m.CloseSession()\n\tvar l sync.Mutex\n\tdeleted := make(map[uint64]int)\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error {\n\t\tl.Lock()\n\t\tsliceId := args[0].(uint64)\n\t\tdeleted[sliceId] = 1\n\t\tl.Unlock()\n\t\treturn nil\n\t})\n\tm.OnMsg(CompactChunk, func(args ...interface{}) error {\n\t\treturn nil\n\t})\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\t_ = m.Unlink(ctx, 1, \"f\")\n\tif st := m.Create(ctx, 1, \"f\", 0650, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create file %s\", st)\n\t}\n\tdefer func() {\n\t\t_ = m.Unlink(ctx, 1, \"f\")\n\t}()\n\n\t// random write\n\tvar sliceId uint64\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 1, uint32(0), Slice{Id: sliceId, Size: 64 << 20, Len: 64 << 20}, time.Now())\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 1, uint32(30<<20), Slice{Id: sliceId, Size: 8, Len: 8}, time.Now())\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 1, uint32(40<<20), Slice{Id: sliceId, Size: 8, Len: 8}, time.Now())\n\tvar cs1 []Slice\n\t_ = m.Read(ctx, inode, 1, &cs1)\n\tif len(cs1) != 5 {\n\t\tt.Fatalf(\"expect 5 slices, but got %+v\", cs1)\n\t}\n\tif c, ok := m.(compactor); ok {\n\t\tc.compactChunk(inode, 1, false, true)\n\t}\n\tvar cs []Slice\n\t_ = m.Read(ctx, inode, 1, &cs)\n\tif len(cs) != 1 {\n\t\tt.Fatalf(\"expect 1 slice, but got %+v\", cs)\n\t}\n\n\t// append\n\tvar size uint32 = 100000\n\tfor i := 0; i < 200; i++ {\n\t\tvar sliceId uint64\n\t\tm.NewSlice(ctx, &sliceId)\n\t\tif st := m.Write(ctx, inode, 0, uint32(i)*size, Slice{Id: sliceId, Size: size, Len: size}, time.Now()); st != 0 {\n\t\t\tt.Fatalf(\"write %d: %s\", i, st)\n\t\t}\n\t\ttime.Sleep(time.Millisecond)\n\t}\n\tif c, ok := m.(compactor); ok {\n\t\tc.compactChunk(inode, 0, false, true)\n\t}\n\tvar slices []Slice\n\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\tt.Fatalf(\"read 0: %s\", st)\n\t}\n\tif len(slices) >= 10 {\n\t\tt.Fatalf(\"inode %d should be compacted, but have %d slices\", inode, len(slices))\n\t}\n\tvar total uint32\n\tfor _, s := range slices {\n\t\ttotal += s.Len\n\t}\n\tif total != size*200 {\n\t\tt.Fatalf(\"size of slice should be %d, but got %d\", size*200, total)\n\t}\n\n\t// TODO: check result if that's predictable\n\tp, bar := utils.MockProgress()\n\tif st := m.CompactAll(ctx, 8, bar); st != 0 {\n\t\tt.Fatalf(\"compactall: %s\", st)\n\t}\n\tp.Done()\n\tsliceMap := make(map[Ino][]Slice)\n\tif st := m.ListSlices(ctx, sliceMap, false, false, nil); st != 0 {\n\t\tt.Fatalf(\"list all slices: %s\", st)\n\t}\n\n\tif trash {\n\t\tl.Lock()\n\t\tdeletes := len(deleted)\n\t\tl.Unlock()\n\t\tif deletes > 10 {\n\t\t\tt.Fatalf(\"deleted slices %d is greater than 10\", deletes)\n\t\t}\n\t\tif len(sliceMap[1]) < 200 {\n\t\t\tt.Fatalf(\"list delayed slices %d is less than 200\", len(sliceMap[1]))\n\t\t}\n\t\tm.(engine).doCleanupDelayedSlices(ctx, time.Now().Unix()+1)\n\t}\n\tm.getBase().stopDeleteSliceTasks()\n\tl.Lock()\n\tdeletes := len(deleted)\n\tl.Unlock()\n\tif deletes < 200 {\n\t\tt.Fatalf(\"deleted slices %d is less than 200\", deletes)\n\t}\n\tm.getBase().startDeleteSliceTasks()\n\n\t// truncate to 0\n\tif st := m.Truncate(ctx, inode, 0, 0, attr, false); st != 0 {\n\t\tt.Fatalf(\"truncate file: %s\", st)\n\t}\n\tif c, ok := m.(compactor); ok {\n\t\tc.compactChunk(inode, 0, false, true)\n\t}\n\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\tt.Fatalf(\"read 0: %s\", st)\n\t}\n\tif len(slices) != 1 || slices[0].Len != 1 {\n\t\tt.Fatalf(\"inode %d should be compacted, but have %d slices, size %d\", inode, len(slices), slices[0].Len)\n\t}\n\n\tif st := m.Truncate(ctx, inode, 0, 64<<10, attr, false); st != 0 {\n\t\tt.Fatalf(\"truncate file: %s\", st)\n\t}\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 0, uint32(1<<20), Slice{Id: sliceId, Size: 2 << 20, Len: 2 << 20}, time.Now())\n\tif c, ok := m.(compactor); ok {\n\t\tc.compactChunk(inode, 0, false, true)\n\t}\n\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\tt.Fatalf(\"read 0: %s\", st)\n\t}\n\tif len(slices) != 2 || slices[0].Id != 0 || slices[1].Len != 2<<20 {\n\t\tt.Fatalf(\"inode %d should be compacted, but have %d slices, id %d size %d\",\n\t\t\tinode, len(slices), slices[0].Id, slices[1].Len)\n\t}\n\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 0, uint32(512<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 64 << 10}, time.Now())\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: sliceId, Size: 1 << 20, Len: 64 << 10}, time.Now())\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 0, uint32(128<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 128 << 10}, time.Now())\n\t_ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: 0, Size: 1 << 20, Len: 1 << 20}, time.Now())\n\tif c, ok := m.(compactor); ok {\n\t\tc.compactChunk(inode, 0, false, true)\n\t}\n\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\tt.Fatalf(\"read 0: %s\", st)\n\t}\n\tif len(slices) != 1 || slices[0].Len != 3<<20 {\n\t\tt.Fatalf(\"inode %d should be compacted, but have %d slices, size %d\", inode, len(slices), slices[0].Len)\n\t}\n\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 2, 0, Slice{Id: sliceId, Size: 2338508, Len: 2338508}, time.Now())\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 2, 8829056, Slice{Id: sliceId, Size: 1074933, Len: 1074933}, time.Now())\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 2, 7663608, Slice{Id: sliceId, Size: 41480, Len: 4148}, time.Now())\n\t_ = m.Fallocate(ctx, inode, fallocZeroRange, 2*ChunkSize+4515328, 3152428, nil)\n\t_ = m.Fallocate(ctx, inode, fallocZeroRange, 2*ChunkSize+4515328, 2607724, nil)\n\tif c, ok := m.(compactor); ok {\n\t\tc.compactChunk(inode, 2, false, true)\n\t}\n\tif st := m.Read(ctx, inode, 2, &slices); st != 0 {\n\t\tt.Fatalf(\"read 1: %s\", st)\n\t}\n\t// compact twice: 4515328+2607724-2338508 = 4784544; 8829056+1074933-2338508-4784544=2780937\n\tif len(slices) != 3 || slices[0].Len != 2338508 || slices[1].Len != 4784544 || slices[2].Len != 2780937 {\n\t\tt.Fatalf(\"inode %d should be compacted, but have %d slices, size %d,%d,%d\",\n\t\t\tinode, len(slices), slices[0].Len, slices[1].Len, slices[2].Len)\n\t}\n\n\tm.NewSlice(ctx, &sliceId)\n\t_ = m.Write(ctx, inode, 3, 0, Slice{Id: sliceId, Size: 2338508, Len: 2338508}, time.Now())\n\t_ = m.CopyFileRange(ctx, inode, 3*ChunkSize, inode, 4*ChunkSize, 2338508, 0, nil, nil)\n\t_ = m.Fallocate(ctx, inode, fallocZeroRange, 4*ChunkSize, ChunkSize, nil)\n\t_ = m.CopyFileRange(ctx, inode, 3*ChunkSize, inode, 4*ChunkSize, 2338508, 0, nil, nil)\n\tif c, ok := m.(compactor); ok {\n\t\tc.compactChunk(inode, 4, false, true)\n\t}\n\tif st := m.Read(ctx, inode, 4, &slices); st != 0 {\n\t\tt.Fatalf(\"read inode %d chunk 4: %s\", inode, st)\n\t}\n\tif len(slices) != 1 || slices[0].Len != 2338508 {\n\t\tt.Fatalf(\"inode %d should be compacted, but have %d slices, size %d\", inode, len(slices), slices[0].Len)\n\t}\n}\n\nfunc testConcurrentWrite(t *testing.T, m Meta) {\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error {\n\t\treturn nil\n\t})\n\tm.OnMsg(CompactChunk, func(args ...interface{}) error {\n\t\treturn nil\n\t})\n\n\tif err := m.NewSession(false); err != nil {\n\t\tt.Fatalf(\"new session: %v\", err)\n\t}\n\tdefer m.CloseSession()\n\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\t_ = m.Unlink(ctx, 1, \"f\")\n\tif st := m.Create(ctx, 1, \"f\", 0650, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create file %s\", st)\n\t}\n\tdefer m.Unlink(ctx, 1, \"f\")\n\n\tvar errno syscall.Errno\n\tvar g sync.WaitGroup\n\tfor i := 0; i <= 10; i++ {\n\t\tg.Add(1)\n\t\tgo func(indx uint32) {\n\t\t\tdefer g.Done()\n\t\t\tfor j := 0; j < 100; j++ {\n\t\t\t\tvar sliceId uint64\n\t\t\t\tm.NewSlice(ctx, &sliceId)\n\t\t\t\tvar slice = Slice{Id: sliceId, Size: 100, Len: 100}\n\t\t\t\tst := m.Write(ctx, inode, indx, 0, slice, time.Now())\n\t\t\t\tif st != 0 {\n\t\t\t\t\terrno = st\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}(uint32(i))\n\t}\n\tg.Wait()\n\tif errno != 0 {\n\t\tt.Fatal()\n\t}\n\n\tvar g2 sync.WaitGroup\n\tfor i := 0; i <= 10; i++ {\n\t\tg2.Add(1)\n\t\tgo func() {\n\t\t\tdefer g2.Done()\n\t\t\tfor j := 0; j < 1000; j++ {\n\t\t\t\tvar sliceId uint64\n\t\t\t\tm.NewSlice(ctx, &sliceId)\n\t\t\t\tvar slice = Slice{Id: sliceId, Size: 100, Len: 100}\n\t\t\t\tst := m.Write(ctx, inode, 0, uint32(200*j), slice, time.Now())\n\t\t\t\tif st != 0 {\n\t\t\t\t\terrno = st\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\tg2.Wait()\n\tif errno != 0 {\n\t\tt.Fatal()\n\t}\n}\n\nfunc testTruncateAndDelete(t *testing.T, m Meta) {\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error {\n\t\treturn nil\n\t})\n\t// remove quota\n\tformat, _ := m.Load(false)\n\tformat.Capacity = 0\n\t_ = m.Init(format, false)\n\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tm.Unlink(ctx, 1, \"f\")\n\tif st := m.Truncate(ctx, 1, 0, 4<<10, attr, false); st != syscall.EPERM {\n\t\tt.Fatalf(\"truncate dir %s\", st)\n\t}\n\tif st := m.Create(ctx, 1, \"f\", 0650, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create file %s\", st)\n\t}\n\tdefer m.Unlink(ctx, 1, \"f\")\n\tvar sliceId uint64\n\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\tt.Fatalf(\"new chunk: %s\", st)\n\t}\n\tif st := m.Write(ctx, inode, 0, 100, Slice{sliceId, 100, 0, 100}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file %s\", st)\n\t}\n\tif st := m.Truncate(ctx, inode, 0, 200<<20, attr, false); st != 0 {\n\t\tt.Fatalf(\"truncate file %s\", st)\n\t}\n\tif st := m.Truncate(ctx, inode, 0, (10<<40)+10, attr, false); st != 0 {\n\t\tt.Fatalf(\"truncate file %s\", st)\n\t}\n\tif st := m.Truncate(ctx, inode, 0, (300<<20)+10, attr, false); st != 0 {\n\t\tt.Fatalf(\"truncate file %s\", st)\n\t}\n\tvar total int64\n\tslices := make(map[Ino][]Slice)\n\tm.ListSlices(ctx, slices, false, false, func() { total++ })\n\tvar totalSlices int\n\tfor _, ss := range slices {\n\t\ttotalSlices += len(ss)\n\t}\n\tif totalSlices != 1 {\n\t\tt.Fatalf(\"number of slices: %d != 1, %+v\", totalSlices, slices)\n\t}\n\t_ = m.Close(ctx, inode)\n\tif st := m.Unlink(ctx, 1, \"f\"); st != 0 {\n\t\tt.Fatalf(\"unlink file %s\", st)\n\t}\n\n\ttime.Sleep(time.Millisecond * 100)\n\tslices = make(map[Ino][]Slice)\n\tm.ListSlices(ctx, slices, false, false, nil)\n\ttotalSlices = 0\n\tfor _, ss := range slices {\n\t\ttotalSlices += len(ss)\n\t}\n\t// the last chunk could be found and deleted\n\tif totalSlices > 1 {\n\t\tt.Fatalf(\"number of slices: %d > 1, %+v\", totalSlices, slices)\n\t}\n}\n\nfunc testCopyFileRange(t *testing.T, m Meta) {\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error {\n\t\treturn nil\n\t})\n\n\tctx := Background()\n\tvar iin, iout Ino\n\tvar attr = &Attr{}\n\t_ = m.Unlink(ctx, 1, \"fin\")\n\t_ = m.Unlink(ctx, 1, \"fout\")\n\tif st := m.Create(ctx, 1, \"fin\", 0650, 022, 0, &iin, attr); st != 0 {\n\t\tt.Fatalf(\"create file %s\", st)\n\t}\n\tdefer m.Unlink(ctx, 1, \"fin\")\n\tif st := m.Create(ctx, 1, \"fout\", 0650, 022, 0, &iout, attr); st != 0 {\n\t\tt.Fatalf(\"create file %s\", st)\n\t}\n\tdefer m.Unlink(ctx, 1, \"fout\")\n\n\tvar sliceIds [4]uint64\n\tfor i := 0; i < len(sliceIds); i++ {\n\t\tif st := m.NewSlice(Background(), &sliceIds[i]); st != 0 {\n\t\t\tt.Fatalf(\"new chunk: %s\", st)\n\t\t}\n\t}\n\n\tif st := m.Write(ctx, iin, 0, 100, Slice{sliceIds[0], 200, 0, 100}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file %s\", st)\n\t}\n\tif st := m.Write(ctx, iin, 1, 100<<10, Slice{sliceIds[1], 40 << 20, 0, 40 << 20}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file %s\", st)\n\t}\n\tif st := m.Write(ctx, iin, 3, 0, Slice{sliceIds[2], 63 << 20, 10 << 20, 30 << 20}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file %s\", st)\n\t}\n\tif st := m.Write(ctx, iout, 2, 10<<20, Slice{sliceIds[3], 50 << 20, 10 << 20, 30 << 20}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file %s\", st)\n\t}\n\tvar copied uint64\n\tif st := m.CopyFileRange(ctx, iin, 150, iout, 30<<20, 200<<20, 0, &copied, nil); st != 0 {\n\t\tt.Fatalf(\"copy file range: %s\", st)\n\t}\n\tvar expected uint64 = 200 << 20\n\tif copied != expected {\n\t\tt.Fatalf(\"expect copy %d bytes, but got %d\", expected, copied)\n\t}\n\tvar expectedSlices = [][]Slice{\n\t\t{{0, 30 << 20, 0, 30 << 20}, {sliceIds[0], 200, 50, 50}, {0, 0, 200, ChunkSize - 30<<20 - 50}},\n\t\t{{0, 0, 150 + (ChunkSize - 30<<20), 30<<20 - 150}, {0, 0, 0, 100 << 10}, {sliceIds[1], 40 << 20, 0, (34 << 20) + 150 - (100 << 10)}},\n\t\t{{sliceIds[1], 40 << 20, (34 << 20) + 150 - (100 << 10), 6<<20 - 150 + 100<<10}, {0, 0, 40<<20 + 100<<10, ChunkSize - 40<<20 - 100<<10}, {0, 0, 0, 150 + (ChunkSize - 30<<20)}},\n\t\t{{0, 0, 150 + (ChunkSize - 30<<20), 30<<20 - 150}, {sliceIds[2], 63 << 20, 10 << 20, (8 << 20) + 150}},\n\t}\n\tfor i := uint32(0); i < 4; i++ {\n\t\tvar slices []Slice\n\t\tif st := m.Read(ctx, iout, i, &slices); st != 0 {\n\t\t\tt.Fatalf(\"read chunk %d: %s\", i, st)\n\t\t}\n\t\tif len(slices) != len(expectedSlices[i]) {\n\t\t\tt.Fatalf(\"expect chunk %d: %+v, but got %+v\", i, expectedSlices[i], slices)\n\t\t}\n\t\tfor j, s := range slices {\n\t\t\tif s != expectedSlices[i][j] {\n\t\t\t\tt.Fatalf(\"expect slice %d,%d: %+v, but got %+v\", i, j, expectedSlices[i][j], s)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc testCloseSession(t *testing.T, m Meta) {\n\t// reset session\n\tm.getBase().sid = 0\n\tif err := m.NewSession(true); err != nil {\n\t\tt.Fatalf(\"new session: %s\", err)\n\t}\n\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tif st := m.Create(ctx, 1, \"f\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif st := m.Flock(ctx, inode, 1, syscall.F_WRLCK, false); st != 0 {\n\t\tt.Fatalf(\"flock wlock: %s\", st)\n\t}\n\tif st := m.Setlk(ctx, inode, 1, false, syscall.F_WRLCK, 0x10000, 0x20000, 1); st != 0 {\n\t\tt.Fatalf(\"plock wlock: %s\", st)\n\t}\n\tif st := m.Open(ctx, inode, syscall.O_RDWR, attr); st != 0 {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, 1, \"f\"); st != 0 {\n\t\tt.Fatalf(\"unlink f: %s\", st)\n\t}\n\ttime.Sleep(10 * time.Millisecond)\n\tsid := m.getBase().sid\n\ts, err := m.GetSession(sid, true)\n\tif err != nil {\n\t\tt.Fatalf(\"get session: %s\", err)\n\t} else {\n\t\tif len(s.Flocks) != 1 || len(s.Plocks) != 1 || len(s.Sustained) != 1 {\n\t\t\tt.Fatalf(\"incorrect session: flock %d plock %d sustained %d\", len(s.Flocks), len(s.Plocks), len(s.Sustained))\n\t\t}\n\t}\n\tif err = m.CloseSession(); err != nil {\n\t\tt.Fatalf(\"close session: %s\", err)\n\t}\n\tif _, err = m.GetSession(sid, true); err == nil {\n\t\tt.Fatalf(\"get a deleted session: %s\", err)\n\t}\n\tswitch m := m.(type) {\n\tcase *redisMeta:\n\t\ts, err = m.getSession(strconv.FormatUint(sid, 10), true)\n\tcase *dbMeta:\n\t\ts, err = m.getSession(&session2{Sid: sid, Info: []byte(\"{}\")}, true)\n\tcase *kvMeta:\n\t\ts, err = m.getSession(sid, true)\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"get session: %s\", err)\n\t}\n\tif s.SessionInfo.Version != \"\" || s.SessionInfo.HostName != \"\" || s.SessionInfo.IPAddrs != nil ||\n\t\ts.SessionInfo.MountPoint != \"\" || s.SessionInfo.ProcessID != 0 {\n\t\tt.Fatalf(\"incorrect session info %+v\", s.SessionInfo)\n\t}\n\tif len(s.Flocks) != 0 || len(s.Plocks) != 0 || len(s.Sustained) != 0 {\n\t\tt.Fatalf(\"incorrect session: flock %d plock %d sustained %d\", len(s.Flocks), len(s.Plocks), len(s.Sustained))\n\t}\n}\n\nfunc testTrash(t *testing.T, m Meta) {\n\tformat := testFormat()\n\tformat.TrashDays = 1\n\tif err := m.Init(format, false); err != nil {\n\t\tt.Fatalf(\"init: %v\", err)\n\t}\n\tdefer func() {\n\t\tif err := m.Init(testFormat(), false); err != nil {\n\t\t\tt.Fatalf(\"init: %v\", err)\n\t\t}\n\t}()\n\tctx := Background()\n\tvar inode, parent Ino\n\tvar attr = &Attr{}\n\tif st := m.Create(ctx, 1, \"f1\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f1: %s\", st)\n\t}\n\tif st := m.Create(ctx, 1, \"f2\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f2: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, 1, \"d\", 0755, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tif st := m.Create(ctx, parent, \"f\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create d/f: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f1\", 1, \"d\", 0, &inode, attr); st != syscall.EISDIR {\n\t\tt.Fatalf(\"rename f1 -> d: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, parent, \"f\"); st != 0 {\n\t\tt.Fatalf(\"unlink d/f: %s\", st)\n\t}\n\tif st := m.GetAttr(ctx, inode, attr); st != 0 || attr.Parent != TrashInode+1 {\n\t\tt.Fatalf(\"getattr f(%d): %s, attr %+v\", inode, st, attr)\n\t}\n\tif st := m.Truncate(ctx, inode, 0, 1<<30, attr, false); st != syscall.EPERM {\n\t\tt.Fatalf(\"should not truncate a file in trash\")\n\t}\n\tif st := m.Open(ctx, inode, uint32(syscall.O_RDWR), attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"should not fallocate a file in trash\")\n\t}\n\tif st := m.SetAttr(ctx, inode, SetAttrMode, 1, &Attr{Mode: 0}); st != syscall.EPERM {\n\t\tt.Fatalf(\"should not change mode of a file in trash\")\n\t}\n\tvar parent2 Ino\n\tif st := m.Mkdir(ctx, 1, \"d2\", 0755, 022, 0, &parent2, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d2: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, 1, \"d2\"); st != 0 {\n\t\tt.Fatalf(\"rmdir d2: %s\", st)\n\t}\n\tif st := m.GetAttr(ctx, parent2, attr); st != 0 || attr.Parent != TrashInode+1 {\n\t\tt.Fatalf(\"getattr d2(%d): %s, attr %+v\", parent2, st, attr)\n\t}\n\tvar tino Ino\n\tif st := m.Mkdir(ctx, parent2, \"d3\", 0777, 022, 0, &tino, attr); st != syscall.ENOENT {\n\t\tt.Fatalf(\"mkdir inside trash should fail\")\n\t}\n\tif st := m.Create(ctx, parent2, \"d3\", 0755, 022, 0, &tino, attr); st != syscall.ENOENT {\n\t\tt.Fatalf(\"create inside trash should fail\")\n\t}\n\tif st := m.Link(ctx, inode, parent2, \"ttlink\", attr); st != syscall.ENOENT {\n\t\tt.Fatalf(\"link inside trash should fail\")\n\t}\n\tif st := m.Rename(ctx, 1, \"d\", parent2, \"ttlink\", 0, &tino, attr); st != syscall.ENOENT {\n\t\tt.Fatalf(\"link inside trash should fail\")\n\t}\n\tif st := m.Rmdir(ctx, 1, \"d\"); st != 0 {\n\t\tt.Fatalf(\"rmdir d: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f1\", 1, \"d\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename f1 -> d: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f2\", TrashInode, \"td\", 0, &inode, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"rename f2 -> td: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f2\", TrashInode+1, \"td\", 0, &inode, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"rename f2 -> td: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f2\", 1, \"d\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename f2 -> d: %s\", st)\n\t}\n\tif st := m.Link(ctx, inode, 1, \"l\", attr); st != 0 || attr.Nlink != 2 {\n\t\tt.Fatalf(\"link d -> l1: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, 1, \"l\"); st != 0 {\n\t\tt.Fatalf(\"unlink l: %s\", st)\n\t}\n\t// hardlink goes to the trash\n\tif st := m.GetAttr(ctx, inode, attr); st != 0 || attr.Nlink != 2 {\n\t\tt.Fatalf(\"getattr d(%d): %s, attr %+v\", inode, st, attr)\n\t}\n\tif st := m.Link(ctx, inode, 1, \"l\", attr); st != 0 || attr.Nlink != 3 {\n\t\tt.Fatalf(\"link d -> l1: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, 1, \"l\"); st != 0 {\n\t\tt.Fatalf(\"unlink l: %s\", st)\n\t}\n\t// hardlink is deleted directly\n\tif st := m.GetAttr(ctx, inode, attr); st != 0 || attr.Nlink != 2 {\n\t\tt.Fatalf(\"getattr d(%d): %s, attr %+v\", inode, st, attr)\n\t}\n\tif st := m.Unlink(ctx, 1, \"d\"); st != 0 {\n\t\tt.Fatalf(\"unlink d: %s\", st)\n\t}\n\tlname := strings.Repeat(\"f\", MaxName)\n\tif st := m.Create(ctx, 1, lname, 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create %s: %s\", lname, st)\n\t}\n\tif st := m.Unlink(ctx, 1, lname); st != 0 {\n\t\tt.Fatalf(\"unlink %s: %s\", lname, st)\n\t}\n\ttname := fmt.Sprintf(\"1-%d-%s\", inode, lname)[:MaxName]\n\tif st := m.Lookup(ctx, TrashInode+1, tname, &inode, attr, true); st != 0 || attr.Parent != TrashInode+1 {\n\t\tt.Fatalf(\"lookup subTrash/%s: %s, attr %+v\", tname, st, attr)\n\t}\n\tvar entries []*Entry\n\tif st := m.Readdir(ctx, 1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t}\n\tif len(entries) != 2 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\tentries = entries[:0]\n\tif st := m.Readdir(ctx, TrashInode+1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t}\n\tif len(entries) != 9 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\t// test Remove with skipTrash true/false\n\tif st := m.Mkdir(ctx, 1, \"d10\", 0755, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d10: %s\", st)\n\t}\n\tif st := m.Create(ctx, parent, \"f10\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create d10/f10: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, parent, \"d10\", 0755, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d10/d10: %s\", st)\n\t}\n\tif st := m.Remove(ctx, 1, \"d10\", false, RmrDefaultThreads, nil); st != 0 {\n\t\tt.Fatalf(\"rmr d10: %s\", st)\n\t}\n\tentries = entries[:0]\n\tif st := m.Readdir(ctx, TrashInode+1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t}\n\tif len(entries) != 12 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\tif st := m.Mkdir(ctx, 1, \"d10\", 0755, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d10: %s\", st)\n\t}\n\tif st := m.Create(ctx, parent, \"f10\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create d10/f10: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, parent, \"d10\", 0755, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d10/d10: %s\", st)\n\t}\n\tif st := m.Remove(ctx, 1, \"d10\", true, RmrDefaultThreads, nil); st != 0 {\n\t\tt.Fatalf(\"rmr d10: %s\", st)\n\t}\n\tentries = entries[:0]\n\tif st := m.Readdir(ctx, TrashInode+1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t}\n\tif len(entries) != 12 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\n\t// Selectively skip trash based on FS_SECRM_FL\n\tif st := m.Mkdir(ctx, 1, \"secrmd\", 0755, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir secrmd: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, parent, SetAttrFlag, 0, &Attr{Flags: FlagSkipTrash}); st != 0 {\n\t\tt.Fatalf(\"setattr secrmd secrm: %s\", st)\n\t}\n\tif st := m.Create(ctx, parent, \"f1\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create secrmd/f1: %s\", st)\n\t}\n\tif st := m.GetAttr(ctx, inode, attr); st != 0 || (attr.Flags&FlagSkipTrash) == 0 {\n\t\tt.Fatalf(\"getattr secrmd/f1(%d): %s, attr %+v\", inode, st, attr)\n\t}\n\tif st := m.Unlink(ctx, parent, \"f1\"); st != 0 {\n\t\tt.Fatalf(\"unlink secrmd/f1: %s\", st)\n\t}\n\tif st := m.Readdir(ctx, TrashInode+1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t}\n\tif len(entries) != 12 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\tentries = entries[:0]\n\tif st := m.Mkdir(ctx, parent, \"d1\", 0755, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir secrmd/d1: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, parent, \"d1\"); st != 0 {\n\t\tt.Fatalf(\"rmdir secrmd/d1: %s\", st)\n\t}\n\tif st := m.Readdir(ctx, TrashInode+1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t}\n\tif len(entries) != 12 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\tentries = entries[:0]\n\tif st := m.Create(ctx, parent, \"f2\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create secrmd/f2: %s\", st)\n\t}\n\tif st := m.Create(ctx, parent, \"f3\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create secrmd/f3: %s\", st)\n\t}\n\tif st := m.Rename(ctx, parent, \"f2\", parent, \"f3\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename secrmd/f2 -> f3: %s\", st)\n\t}\n\tif st := m.Readdir(ctx, TrashInode+1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t}\n\tif len(entries) != 12 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\tentries = entries[:0]\n\tif st := m.Unlink(ctx, parent, \"f3\"); st != 0 {\n\t\tt.Fatalf(\"unlink secrmd/f3: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, 1, \"secrmd\"); st != 0 {\n\t\tt.Fatalf(\"rmdir secrmd: %s\", st)\n\t}\n\n\tctx2 := NewContext(1000, 1, []uint32{1})\n\tif st := m.Unlink(ctx2, TrashInode+1, \"d\"); st != syscall.EPERM {\n\t\tt.Fatalf(\"unlink d: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx2, TrashInode+1, \"d\"); st != syscall.EPERM {\n\t\tt.Fatalf(\"rmdir d: %s\", st)\n\t}\n\tif st := m.Rename(ctx2, TrashInode+1, \"d\", 1, \"f\", 0, &inode, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"rename d -> f: %s\", st)\n\t}\n\tm.getBase().doCleanupTrash(Background(), format.TrashDays, true, nil)\n\tif st := m.GetAttr(ctx2, TrashInode+1, attr); st != syscall.ENOENT {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n}\n\nfunc testParents(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode, parent Ino\n\tvar attr = &Attr{}\n\tif st := m.Create(ctx, 1, \"f\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif attr.Parent != 1 {\n\t\tt.Fatalf(\"expect parent 1, but got %d\", attr.Parent)\n\t}\n\tcheckParents := func(inode Ino, expect map[Ino]int) {\n\t\tif ps := m.GetParents(ctx, inode); ps == nil {\n\t\t\tt.Fatalf(\"get parents of inode %d returns nil\", inode)\n\t\t} else if !reflect.DeepEqual(ps, expect) {\n\t\t\tt.Fatalf(\"expect parents %v, but got %v\", expect, ps)\n\t\t}\n\t}\n\tcheckParents(inode, map[Ino]int{1: 1})\n\n\tif st := m.Link(ctx, inode, 1, \"l1\", attr); st != 0 {\n\t\tt.Fatalf(\"link l1 -> f: %s\", st)\n\t}\n\tif attr.Parent != 0 {\n\t\tt.Fatalf(\"expect parent 0, but got %d\", attr.Parent)\n\t}\n\tcheckParents(inode, map[Ino]int{1: 2})\n\n\tif st := m.Mkdir(ctx, 1, \"d\", 0755, 022, 0, &parent, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tif st := m.Link(ctx, inode, parent, \"l2\", attr); st != 0 {\n\t\tt.Fatalf(\"link l2 -> f: %s\", st)\n\t}\n\tif st := m.Link(ctx, inode, parent, \"l3\", attr); st != 0 {\n\t\tt.Fatalf(\"link l3 -> f: %s\", st)\n\t}\n\tcheckParents(inode, map[Ino]int{1: 2, parent: 2})\n\n\tif st := m.Unlink(ctx, 1, \"f\"); st != 0 {\n\t\tt.Fatalf(\"unlink f: %s\", st)\n\t}\n\tif st := m.Create(ctx, 1, \"f2\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f2: %s\", st)\n\t}\n\tif st := m.Rename(ctx, 1, \"f2\", 1, \"l1\", 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"rename f2 -> l1: %s\", st)\n\t}\n\tif st := m.Lookup(ctx, parent, \"l2\", &inode, attr, true); st != 0 {\n\t\tt.Fatalf(\"lookup d/l2: %s\", st)\n\t}\n\tif attr.Parent != 0 {\n\t\tt.Fatalf(\"expect parent 0, but got %d\", attr.Parent)\n\t}\n\tif st := m.Unlink(ctx, parent, \"l2\"); st != 0 {\n\t\tt.Fatalf(\"unlink d/l2: %s\", st)\n\t}\n\tcheckParents(inode, map[Ino]int{parent: 1})\n\n\t// clean up\n\tif st := m.Unlink(ctx, 1, \"l1\"); st != 0 {\n\t\tt.Fatalf(\"unlink l1: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, parent, \"l3\"); st != 0 {\n\t\tt.Fatalf(\"unlink d/l3: %s\", st)\n\t}\n\tif st := m.Rmdir(ctx, 1, \"d\"); st != 0 {\n\t\tt.Fatalf(\"rmdir d: %s\", st)\n\t}\n}\n\nfunc testOpenCache(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tif st := m.Create(ctx, 1, \"f\", 0644, 022, 0, &inode, attr); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tdefer m.Unlink(ctx, 1, \"f\")\n\tif st := m.Open(ctx, inode, syscall.O_RDWR, attr); st != 0 {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\tdefer m.Close(ctx, inode)\n\n\tvar attr2 = &Attr{}\n\tif st := m.GetAttr(ctx, inode, attr2); st != 0 {\n\t\tt.Fatalf(\"getattr f: %s\", st)\n\t}\n\tif *attr != *attr2 {\n\t\tt.Fatalf(\"attrs not the same: attr %+v; attr2 %+v\", *attr, *attr2)\n\t}\n\tattr2.Uid = 1\n\tif st := m.SetAttr(ctx, inode, SetAttrUID, 0, attr2); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.GetAttr(ctx, inode, attr); st != 0 {\n\t\tt.Fatalf(\"getattr f: %s\", st)\n\t}\n\tif attr.Uid != 1 {\n\t\tt.Fatalf(\"attr uid should be 1: %+v\", *attr)\n\t}\n}\n\nfunc testReadOnly(t *testing.T, m Meta) {\n\tctx := Background()\n\tif err := m.NewSession(true); err != nil {\n\t\tt.Fatalf(\"new session: %s\", err)\n\t}\n\tdefer m.CloseSession()\n\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tif st := m.GetAttr(ctx, 1, attr); st != 0 {\n\t\tt.Fatalf(\"getattr 1: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, 1, \"d\", 0640, 022, 0, &inode, attr); st != syscall.EROFS {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tif st := m.Create(ctx, 1, \"f\", 0644, 022, 0, &inode, attr); st != syscall.EROFS {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif st := m.Open(ctx, inode, syscall.O_RDWR, attr); st != syscall.EROFS {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\n\tif plocks, flocks, err := m.ListLocks(ctx, 1); err != nil || len(plocks) != 0 || len(flocks) != 0 {\n\t\tt.Fatalf(\"list locks: %v %v %v\", plocks, flocks, err)\n\t}\n}\n\nfunc testConcurrentDir(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar g sync.WaitGroup\n\tvar err error\n\tformat, err := m.Load(false)\n\tformat.Capacity = 0\n\tformat.Inodes = 0\n\tif err = m.Init(format, false); err != nil {\n\t\tt.Fatalf(\"set quota failed: %s\", err)\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdefer g.Done()\n\t\t\tvar d1, d2 Ino\n\t\t\tvar attr = new(Attr)\n\t\t\tif st := m.Mkdir(ctx, 1, \"d1\", 0640, 022, 0, &d1, attr); st != 0 && st != syscall.EEXIST {\n\t\t\t\tpanic(fmt.Errorf(\"mkdir d1: %s\", st))\n\t\t\t} else if st == syscall.EEXIST {\n\t\t\t\tst = m.Lookup(ctx, 1, \"d1\", &d1, attr, true)\n\t\t\t\tif st != 0 {\n\t\t\t\t\tpanic(fmt.Errorf(\"lookup d1: %s\", st))\n\t\t\t\t}\n\t\t\t}\n\t\t\tif st := m.Mkdir(ctx, 1, \"d2\", 0640, 022, 0, &d2, attr); st != 0 && st != syscall.EEXIST {\n\t\t\t\tpanic(fmt.Errorf(\"mkdir d2: %s\", st))\n\t\t\t} else if st == syscall.EEXIST {\n\t\t\t\tst = m.Lookup(ctx, 1, \"d2\", &d2, attr, true)\n\t\t\t\tif st != 0 {\n\t\t\t\t\tpanic(fmt.Errorf(\"lookup d2: %s\", st))\n\t\t\t\t}\n\t\t\t}\n\t\t\tname := fmt.Sprintf(\"file%d\", i)\n\t\t\tvar f Ino\n\t\t\tif st := m.Create(ctx, d1, name, 0664, 0, 0, &f, attr); st != 0 {\n\t\t\t\tpanic(fmt.Errorf(\"create d1/%s: %s\", name, st))\n\t\t\t}\n\t\t\tif st := m.Rename(ctx, d1, name, d2, name, 0, &f, attr); st != 0 {\n\t\t\t\tpanic(fmt.Errorf(\"rename d1/%s -> d2/%s: %s\", name, name, st))\n\t\t\t}\n\t\t}(i)\n\t}\n\tg.Wait()\n\tif err != nil {\n\t\tt.Fatalf(\"concurrent dir: %s\", err)\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdefer g.Done()\n\t\t\tvar d2 Ino\n\t\t\tvar attr = new(Attr)\n\t\t\tst := m.Lookup(ctx, 1, \"d2\", &d2, attr, true)\n\t\t\tif st != 0 {\n\t\t\t\tpanic(fmt.Errorf(\"lookup d2: %s\", st))\n\t\t\t}\n\t\t\tname := fmt.Sprintf(\"file%d\", i)\n\t\t\tif st := m.Unlink(ctx, d2, name); st != 0 {\n\t\t\t\tpanic(fmt.Errorf(\"unlink d2/%s: %s\", name, st))\n\t\t\t}\n\t\t\tif st := m.Rmdir(ctx, 1, \"d1\"); st != 0 && st != syscall.ENOTEMPTY && st != syscall.ENOENT {\n\t\t\t\tpanic(fmt.Errorf(\"rmdir d1: %s\", st))\n\t\t\t}\n\t\t\tif st := m.Rmdir(ctx, 1, \"d2\"); st != 0 && st != syscall.ENOTEMPTY && st != syscall.ENOENT {\n\t\t\t\tpanic(fmt.Errorf(\"rmdir d2: %s\", st))\n\t\t\t}\n\t\t}(i)\n\t}\n\tg.Wait()\n}\n\nfunc testAttrFlags(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar attr = &Attr{}\n\tvar inode Ino\n\tif st := m.Create(ctx, 1, \"f\", 0644, 022, 0, &inode, nil); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tattr.Flags = FlagAppend\n\tif st := m.SetAttr(ctx, inode, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.Open(ctx, inode, syscall.O_WRONLY, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\tif st := m.Open(ctx, inode, syscall.O_WRONLY|syscall.O_APPEND, attr); st != 0 {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\tattr.Flags = FlagAppend | FlagImmutable\n\tif st := m.SetAttr(ctx, inode, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.Open(ctx, inode, syscall.O_WRONLY, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\tif st := m.Open(ctx, inode, syscall.O_WRONLY|syscall.O_APPEND, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"open f: %s\", st)\n\t}\n\n\tvar d Ino\n\tif st := m.Mkdir(ctx, 1, \"d\", 0640, 022, 0, &d, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tattr.Flags = FlagAppend\n\tif st := m.SetAttr(ctx, d, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr d: %s\", st)\n\t}\n\tif st := m.Create(ctx, d, \"f\", 0644, 022, 0, &inode, nil); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, d, \"f\"); st != syscall.EPERM {\n\t\tt.Fatalf(\"unlink f: %s\", st)\n\t}\n\tattr.Flags = FlagAppend | FlagImmutable\n\tif st := m.SetAttr(ctx, d, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr d: %s\", st)\n\t}\n\tif st := m.Create(ctx, d, \"f2\", 0644, 022, 0, &inode, nil); st != syscall.EPERM {\n\t\tt.Fatalf(\"create f2: %s\", st)\n\t}\n\n\tvar Immutable Ino\n\tif st := m.Mkdir(ctx, 1, \"ImmutFile\", 0640, 022, 0, &Immutable, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir d: %s\", st)\n\t}\n\tattr.Flags = FlagImmutable\n\tif st := m.SetAttr(ctx, Immutable, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr d: %s\", st)\n\t}\n\tif st := m.Create(ctx, Immutable, \"f2\", 0644, 022, 0, &inode, nil); st != syscall.EPERM {\n\t\tt.Fatalf(\"create f2: %s\", st)\n\t}\n\n\tvar src1, dst1, mfile Ino\n\tattr.Flags = 0\n\tif st := m.Mkdir(ctx, 1, \"src1\", 0640, 022, 0, &src1, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir src1: %s\", st)\n\t}\n\tif st := m.Create(ctx, src1, \"mfile\", 0644, 022, 0, &mfile, nil); st != 0 {\n\t\tt.Fatalf(\"create mfile: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, 1, \"dst1\", 0640, 022, 0, &dst1, attr); st != 0 {\n\t\tt.Fatalf(\"mkdir dst1: %s\", st)\n\t}\n\n\tattr.Flags = FlagAppend\n\tif st := m.SetAttr(ctx, src1, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr d: %s\", st)\n\t}\n\tif st := m.Rename(ctx, src1, \"mfile\", dst1, \"mfile\", 0, &mfile, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"rename d: %s\", st)\n\t}\n\n\tattr.Flags = FlagImmutable\n\tif st := m.SetAttr(ctx, src1, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr d: %s\", st)\n\t}\n\tif st := m.Rename(ctx, src1, \"mfile\", dst1, \"mfile\", 0, &mfile, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"rename d: %s\", st)\n\t}\n\n\tif st := m.SetAttr(ctx, dst1, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr d: %s\", st)\n\t}\n\tif st := m.Rename(ctx, src1, \"mfile\", dst1, \"mfile\", 0, &mfile, attr); st != syscall.EPERM {\n\t\tt.Fatalf(\"rename d: %s\", st)\n\t}\n\n\tvar delFile Ino\n\tif st := m.Create(ctx, 1, \"delfile\", 0644, 022, 0, &delFile, nil); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tattr.Flags = FlagImmutable | FlagAppend\n\tif st := m.SetAttr(ctx, delFile, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr d: %s\", st)\n\t}\n\tif st := m.Unlink(ctx, 1, \"delfile\"); st != syscall.EPERM {\n\t\tt.Fatalf(\"unlink f: %s\", st)\n\t}\n\n\tvar fallocFile Ino\n\tif st := m.Create(ctx, 1, \"fallocfile\", 0644, 022, 0, &fallocFile, nil); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tattr.Flags = FlagAppend\n\tif st := m.SetAttr(ctx, fallocFile, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, fallocFile, fallocKeepSize, 0, 1024, nil); st != 0 {\n\t\tt.Fatalf(\"fallocate f: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, fallocFile, fallocKeepSize|fallocZeroRange, 0, 1024, nil); st != syscall.EPERM {\n\t\tt.Fatalf(\"fallocate f: %s\", st)\n\t}\n\tattr.Flags = FlagImmutable\n\tif st := m.SetAttr(ctx, fallocFile, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, fallocFile, fallocKeepSize, 0, 1024, nil); st != syscall.EPERM {\n\t\tt.Fatalf(\"fallocate f: %s\", st)\n\t}\n\n\tvar copysrcFile, copydstFile Ino\n\tif st := m.Create(ctx, 1, \"copysrcfile\", 0644, 022, 0, ©srcFile, nil); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif st := m.Create(ctx, 1, \"copydstfile\", 0644, 022, 0, ©dstFile, nil); st != 0 {\n\t\tt.Fatalf(\"create f: %s\", st)\n\t}\n\tif st := m.Fallocate(ctx, copysrcFile, 0, 0, 1024, nil); st != 0 {\n\t\tt.Fatalf(\"fallocate f: %s\", st)\n\t}\n\tattr.Flags = FlagAppend\n\tif st := m.SetAttr(ctx, copydstFile, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.CopyFileRange(ctx, copysrcFile, 0, copydstFile, 0, 1024, 0, nil, nil); st != syscall.EPERM {\n\t\tt.Fatalf(\"copy_file_range f: %s\", st)\n\t}\n\tattr.Flags = FlagImmutable\n\tif st := m.SetAttr(ctx, copydstFile, SetAttrFlag, 0, attr); st != 0 {\n\t\tt.Fatalf(\"setattr f: %s\", st)\n\t}\n\tif st := m.CopyFileRange(ctx, copysrcFile, 0, copydstFile, 0, 1024, 0, nil, nil); st != syscall.EPERM {\n\t\tt.Fatalf(\"copy_file_range f: %s\", st)\n\t}\n}\n\nfunc setAttr(t *testing.T, m Meta, inode Ino, attr *Attr) {\n\tvar err error\n\tswitch m := m.(type) {\n\tcase *redisMeta:\n\t\terr = m.txn(Background(), func(tx *redis.Tx) error {\n\t\t\treturn tx.Set(Background(), m.inodeKey(inode), m.marshal(attr), 0).Err()\n\t\t}, m.inodeKey(inode))\n\tcase *dbMeta:\n\t\terr = m.txn(func(s *xorm.Session) error {\n\t\t\t_, err = s.ID(inode).AllCols().Update(&node{\n\t\t\t\tInode: inode,\n\t\t\t\tType: attr.Typ,\n\t\t\t\tFlags: attr.Flags,\n\t\t\t\tMode: attr.Mode,\n\t\t\t\tUid: attr.Uid,\n\t\t\t\tGid: attr.Gid,\n\t\t\t\tAtime: attr.Atime*1e6 + int64(attr.Atimensec)/1e3,\n\t\t\t\tMtime: attr.Mtime*1e6 + int64(attr.Mtimensec)/1e3,\n\t\t\t\tCtime: attr.Ctime*1e6 + int64(attr.Ctimensec)/1e3,\n\t\t\t\tAtimensec: int16(attr.Atimensec % 1e3),\n\t\t\t\tMtimensec: int16(attr.Mtimensec % 1e3),\n\t\t\t\tCtimensec: int16(attr.Ctimensec % 1e3),\n\n\t\t\t\tNlink: attr.Nlink,\n\t\t\t\tLength: attr.Length,\n\t\t\t\tRdev: attr.Rdev,\n\t\t\t\tParent: attr.Parent,\n\t\t\t})\n\t\t\treturn err\n\t\t})\n\tcase *kvMeta:\n\t\terr = m.txn(Background(), func(tx *kvTxn) error {\n\t\t\ttx.set(m.inodeKey(inode), m.marshal(attr))\n\t\t\treturn nil\n\t\t})\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"setAttr: %v\", err)\n\t}\n}\n\nfunc testCheckAndRepair(t *testing.T, m Meta) {\n\tvar checkInode, d1Inode, d2Inode, d3Inode, d4Inode Ino\n\tdirAttr := &Attr{Mode: 0644, Full: true, Typ: TypeDirectory, Nlink: 3}\n\tif st := m.Mkdir(Background(), RootInode, \"check\", 0640, 022, 0, &checkInode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", st)\n\t}\n\tif st := m.Mkdir(Background(), checkInode, \"d1\", 0640, 022, 0, &d1Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", st)\n\t}\n\tif st := m.Mkdir(Background(), d1Inode, \"d2\", 0640, 022, 0, &d2Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", st)\n\t}\n\tif st := m.Mkdir(Background(), d2Inode, \"d3\", 0640, 022, 0, &d3Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", st)\n\t}\n\tif st := m.Mkdir(Background(), d3Inode, \"d4\", 0640, 022, 0, &d4Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", st)\n\t}\n\n\tif st := m.GetAttr(Background(), checkInode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tdirAttr.Nlink = 0\n\tsetAttr(t, m, checkInode, dirAttr)\n\n\tif st := m.GetAttr(Background(), d1Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tdirAttr.Nlink = 0\n\tsetAttr(t, m, d1Inode, dirAttr)\n\n\tif st := m.GetAttr(Background(), d2Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tdirAttr.Nlink = 0\n\tsetAttr(t, m, d2Inode, dirAttr)\n\n\tif st := m.GetAttr(Background(), d3Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tdirAttr.Nlink = 0\n\tsetAttr(t, m, d3Inode, dirAttr)\n\n\tif st := m.GetAttr(Background(), d4Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tdirAttr.Full = false\n\tdirAttr.Nlink = 0\n\tsetAttr(t, m, d4Inode, dirAttr)\n\n\tshowProgress := func(n int) {}\n\tslices := make(map[Ino][]Slice)\n\tif err := m.Check(Background(), \"/check\", &CheckOpt{\n\t\tShowProgress: showProgress,\n\t\tSlices: slices,\n\t}); err == nil {\n\t\tt.Fatal(\"check should fail\")\n\t}\n\tif st := m.GetAttr(Background(), checkInode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tif dirAttr.Nlink != 0 {\n\t\tt.Fatalf(\"checkInode nlink should is 0 now: %d\", dirAttr.Nlink)\n\t}\n\n\tif err := m.Check(Background(), \"/check\", &CheckOpt{\n\t\tRepair: true,\n\t\tShowProgress: showProgress,\n\t\tSlices: slices,\n\t}); err != nil {\n\t\tt.Fatalf(\"check: %s\", err)\n\t}\n\tif st := m.GetAttr(Background(), checkInode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tif dirAttr.Nlink != 3 || dirAttr.Parent != RootInode {\n\t\tt.Fatalf(\"checkInode nlink should is 3 now: %d\", dirAttr.Nlink)\n\t}\n\n\tif err := m.Check(Background(), \"/check/d1/d2\", &CheckOpt{\n\t\tRepair: true,\n\t\tShowProgress: showProgress,\n\t\tSlices: slices,\n\t}); err != nil {\n\t\tt.Fatalf(\"check: %s\", err)\n\t}\n\tif st := m.GetAttr(Background(), d2Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tif dirAttr.Nlink != 3 || dirAttr.Parent != d1Inode {\n\t\tt.Fatalf(\"d2Inode nlink should is 3 now: %d\", dirAttr.Nlink)\n\t}\n\tif st := m.GetAttr(Background(), d1Inode, dirAttr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tif dirAttr.Nlink != 0 || dirAttr.Parent != checkInode {\n\t\tt.Fatalf(\"d1Inode nlink should is 0 now: %d\", dirAttr.Nlink)\n\t}\n\n\tif m.Name() != \"etcd\" {\n\t\tif err := m.Check(Background(), \"/\", &CheckOpt{\n\t\t\tRepair: true,\n\t\t\tRecursive: true,\n\t\t\tShowProgress: showProgress,\n\t\t\tSlices: slices,\n\t\t}); err != nil {\n\t\t\tt.Fatalf(\"check: %s\", err)\n\t\t}\n\t\tfor _, ino := range []Ino{checkInode, d1Inode, d2Inode, d3Inode} {\n\t\t\tif st := m.GetAttr(Background(), ino, dirAttr); st != 0 {\n\t\t\t\tt.Fatalf(\"getattr: %s\", st)\n\t\t\t}\n\t\t\tif !dirAttr.Full || dirAttr.Nlink != 3 {\n\t\t\t\tt.Fatalf(\"nlink should is 3 now: %d\", dirAttr.Nlink)\n\t\t\t}\n\t\t}\n\t\tif st := m.GetAttr(Background(), d4Inode, dirAttr); st != 0 {\n\t\t\tt.Fatalf(\"getattr: %s\", st)\n\t\t}\n\t\tif !dirAttr.Full || dirAttr.Nlink != 2 || dirAttr.Parent != d3Inode {\n\t\t\tt.Fatalf(\"d4Inode attr: %+v\", *dirAttr)\n\t\t}\n\t}\n}\n\nfunc testDirStat(t *testing.T, m Meta) {\n\ttestDir := \"testDirStat\"\n\tvar testInode Ino\n\t// test empty dir\n\tif st := m.Mkdir(Background(), RootInode, testDir, 0640, 022, 0, &testInode, nil); st != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", st)\n\t}\n\tif err := m.NewSession(true); err != nil {\n\t\tt.Fatalf(\"new session: %s\", err)\n\t}\n\tdefer m.CloseSession()\n\tstat, st := m.GetDirStat(Background(), testInode)\n\tcheckResult := func(length, space, inodes int64) {\n\t\tif st != 0 {\n\t\t\tt.Fatalf(\"get dir usage: %s\", st)\n\t\t}\n\t\texpect := dirStat{length, space, inodes}\n\t\tif *stat != expect {\n\t\t\tt.Fatalf(\"test dir usage: expect %+v, but got %+v\", expect, stat)\n\t\t}\n\t}\n\tcheckResult(0, 0, 0)\n\n\t// test dir with file\n\tvar fileInode Ino\n\tif st := m.Create(Background(), testInode, \"file\", 0640, 022, 0, &fileInode, nil); st != 0 {\n\t\tt.Fatalf(\"create: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(0, align4K(0), 1)\n\n\t// test dir with file and fallocate\n\tif st := m.Fallocate(Background(), fileInode, 0, 0, 4097, nil); st != 0 {\n\t\tt.Fatalf(\"fallocate: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(4097, align4K(4097), 1)\n\n\t// test dir with file and truncate\n\tif st := m.Truncate(Background(), fileInode, 0, 0, nil, false); st != 0 {\n\t\tt.Fatalf(\"truncate: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(0, align4K(0), 1)\n\n\t// test dir with file and write\n\tif st := m.Write(Background(), fileInode, 0, 0, Slice{Id: 1, Size: 1 << 20, Off: 0, Len: 4097}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(4097, align4K(4097), 1)\n\n\t// test dir with file and link\n\tif st := m.Link(Background(), fileInode, testInode, \"file2\", nil); st != 0 {\n\t\tt.Fatalf(\"link: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(2*4097, 2*align4K(4097), 2)\n\n\t// test dir with subdir\n\tvar subInode Ino\n\tif st := m.Mkdir(Background(), testInode, \"sub\", 0640, 022, 0, &subInode, nil); st != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(2*4097, align4K(0)+2*align4K(4097), 3)\n\n\t// test rename\n\tif st := m.Rename(Background(), testInode, \"file2\", subInode, \"file\", 0, nil, nil); st != 0 {\n\t\tt.Fatalf(\"rename: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(4097, align4K(0)+align4K(4097), 2)\n\tstat, st = m.GetDirStat(Background(), subInode)\n\tcheckResult(4097, align4K(4097), 1)\n\n\t// test unlink\n\tif st := m.Unlink(Background(), testInode, \"file\"); st != 0 {\n\t\tt.Fatalf(\"unlink: %s\", st)\n\t}\n\tif st := m.Unlink(Background(), subInode, \"file\"); st != 0 {\n\t\tt.Fatalf(\"unlink: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(0, align4K(0), 1)\n\tstat, st = m.GetDirStat(Background(), subInode)\n\tcheckResult(0, 0, 0)\n\n\t// test rmdir\n\tif st := m.Rmdir(Background(), testInode, \"sub\"); st != 0 {\n\t\tt.Fatalf(\"rmdir: %s\", st)\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tstat, st = m.GetDirStat(Background(), testInode)\n\tcheckResult(0, 0, 0)\n}\n\nfunc testBatchClone(t *testing.T, m Meta) {\n\tctx := Background()\n\n\t// create source directory with mixed entry types\n\tvar srcDir Ino\n\tif st := m.Mkdir(ctx, RootInode, \"batchSrc\", 0755, 022, 0, &srcDir, nil); st != 0 {\n\t\tt.Fatalf(\"mkdir batchSrc: %s\", st)\n\t}\n\n\t// file with data\n\tvar file1 Ino\n\tif st := m.Mknod(ctx, srcDir, \"file1\", TypeFile, 0644, 022, 0, \"\", &file1, nil); st != 0 {\n\t\tt.Fatalf(\"mknod file1: %s\", st)\n\t}\n\tvar sliceId1 uint64\n\tif st := m.NewSlice(ctx, &sliceId1); st != 0 {\n\t\tt.Fatalf(\"new slice: %s\", st)\n\t}\n\tif st := m.Write(ctx, file1, 0, 0, Slice{sliceId1, 1024, 0, 1024}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file1: %s\", st)\n\t}\n\tif st := m.SetXattr(ctx, file1, \"user.tag\", []byte(\"hello\"), XattrCreateOrReplace); st != 0 {\n\t\tt.Fatalf(\"setxattr file1: %s\", st)\n\t}\n\n\t// empty file\n\tvar file2 Ino\n\tif st := m.Mknod(ctx, srcDir, \"file2\", TypeFile, 0644, 022, 0, \"\", &file2, nil); st != 0 {\n\t\tt.Fatalf(\"mknod file2: %s\", st)\n\t}\n\n\t// symlink\n\tvar sym1 Ino\n\tif st := m.Symlink(ctx, srcDir, \"sym1\", \"/tmp/target\", &sym1, nil); st != 0 {\n\t\tt.Fatalf(\"symlink sym1: %s\", st)\n\t}\n\n\t// create destination directory\n\tvar dstDir Ino\n\tif st := m.Mkdir(ctx, RootInode, \"batchDst\", 0755, 022, 0, &dstDir, nil); st != 0 {\n\t\tt.Fatalf(\"mkdir batchDst: %s\", st)\n\t}\n\n\t// read source entries\n\tvar srcEntries []*Entry\n\tif st := m.Readdir(ctx, srcDir, 1, &srcEntries); st != 0 {\n\t\tt.Fatalf(\"readdir batchSrc: %s\", st)\n\t}\n\t// filter out . and ..\n\tvar nonDirEntries []*Entry\n\tfor _, e := range srcEntries {\n\t\tname := string(e.Name)\n\t\tif name == \".\" || name == \"..\" {\n\t\t\tcontinue\n\t\t}\n\t\tnonDirEntries = append(nonDirEntries, e)\n\t}\n\n\t// --- test 1: successful batch clone ---\n\tvar count uint64\n\tst := m.getBase().BatchClone(ctx, srcDir, dstDir, nonDirEntries, 0, 022, &count)\n\tif st == syscall.ENOTSUP {\n\t\tm.Remove(ctx, RootInode, \"batchSrc\", false, RmrDefaultThreads, nil)\n\t\tm.Remove(ctx, RootInode, \"batchDst\", false, RmrDefaultThreads, nil)\n\t\treturn\n\t}\n\n\tif st != 0 {\n\t\tt.Fatalf(\"BatchClone: %s\", st)\n\t}\n\tif count != uint64(len(nonDirEntries)) {\n\t\tt.Fatalf(\"BatchClone count: got %d, want %d\", count, len(nonDirEntries))\n\t}\n\n\t// verify cloned entries exist\n\tvar dstEntries []*Entry\n\tif st := m.Readdir(ctx, dstDir, 1, &dstEntries); st != 0 {\n\t\tt.Fatalf(\"readdir batchDst: %s\", st)\n\t}\n\tdstMap := make(map[string]*Entry)\n\tfor _, e := range dstEntries {\n\t\tname := string(e.Name)\n\t\tif name != \".\" && name != \"..\" {\n\t\t\tdstMap[name] = e\n\t\t}\n\t}\n\tif len(dstMap) != len(nonDirEntries) {\n\t\tt.Fatalf(\"cloned entry count: got %d, want %d\", len(dstMap), len(nonDirEntries))\n\t}\n\n\t// verify file1 clone: data, xattr\n\tif e, ok := dstMap[\"file1\"]; !ok {\n\t\tt.Fatalf(\"file1 not cloned\")\n\t} else {\n\t\tif e.Attr.Typ != TypeFile {\n\t\t\tt.Fatalf(\"file1 type: got %d, want %d\", e.Attr.Typ, TypeFile)\n\t\t}\n\t\tvar slices []Slice\n\t\tif st := m.Read(ctx, e.Inode, 0, &slices); st != 0 {\n\t\t\tt.Fatalf(\"read cloned file1: %s\", st)\n\t\t}\n\t\tif len(slices) == 0 {\n\t\t\tt.Fatal(\"cloned file1 has no slices\")\n\t\t}\n\t\tvar val []byte\n\t\tif st := m.GetXattr(ctx, e.Inode, \"user.tag\", &val); st != 0 {\n\t\t\tt.Fatalf(\"getxattr cloned file1: %s\", st)\n\t\t}\n\t\tif string(val) != \"hello\" {\n\t\t\tt.Fatalf(\"xattr value: got %q, want %q\", val, \"hello\")\n\t\t}\n\t}\n\n\t// verify sym1 clone: target\n\tif e, ok := dstMap[\"sym1\"]; !ok {\n\t\tt.Fatalf(\"sym1 not cloned\")\n\t} else {\n\t\tvar target []byte\n\t\tif st := m.ReadLink(ctx, e.Inode, &target); st != 0 {\n\t\t\tt.Fatalf(\"readlink cloned sym1: %s\", st)\n\t\t}\n\t\tif string(target) != \"/tmp/target\" {\n\t\t\tt.Fatalf(\"symlink target: got %q, want %q\", target, \"/tmp/target\")\n\t\t}\n\t}\n\n\t// verify file2 clone: empty file\n\tif e, ok := dstMap[\"file2\"]; !ok {\n\t\tt.Fatalf(\"file2 not cloned\")\n\t} else {\n\t\tif e.Attr.Typ != TypeFile {\n\t\t\tt.Fatalf(\"file2 type: got %d, want %d\", e.Attr.Typ, TypeFile)\n\t\t}\n\t\tif e.Attr.Length != 0 {\n\t\t\tt.Fatalf(\"file2 length: got %d, want 0\", e.Attr.Length)\n\t\t}\n\t}\n\n\t// --- test 2: duplicate entry names (EEXIST) ---\n\tcount = 0\n\tst = m.getBase().BatchClone(ctx, srcDir, dstDir, nonDirEntries, 0, 022, &count)\n\tif st != syscall.EEXIST {\n\t\tt.Fatalf(\"BatchClone duplicate: got %s, want EEXIST\", st)\n\t}\n\n\t// --- test 3: dst parent doesn't exist ---\n\tcount = 0\n\tst = m.getBase().BatchClone(ctx, srcDir, 999999, nonDirEntries, 0, 022, &count)\n\tif st != syscall.ENOENT {\n\t\tt.Fatalf(\"BatchClone non-existent dst: got %s, want ENOENT\", st)\n\t}\n\n\t// --- test 4: dst parent is a file, not directory ---\n\tcount = 0\n\tst = m.getBase().BatchClone(ctx, srcDir, file1, nonDirEntries, 0, 022, &count)\n\tif st != syscall.ENOTDIR {\n\t\tt.Fatalf(\"BatchClone file as dst: got %s, want ENOTDIR\", st)\n\t}\n\n\t// --- test 5: dst parent is immutable ---\n\tvar immDir Ino\n\tif st := m.Mkdir(ctx, RootInode, \"batchImm\", 0755, 022, 0, &immDir, nil); st != 0 {\n\t\tt.Fatalf(\"mkdir batchImm: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, immDir, SetAttrFlag, 0, &Attr{Flags: FlagImmutable}); st != 0 {\n\t\tt.Fatalf(\"setattr immutable: %s\", st)\n\t}\n\tcount = 0\n\tst = m.getBase().BatchClone(ctx, srcDir, immDir, nonDirEntries, 0, 022, &count)\n\tif st != syscall.EPERM {\n\t\tt.Fatalf(\"BatchClone immutable dst: got %s, want EPERM\", st)\n\t}\n\t// clean up immutable flag\n\tif st := m.SetAttr(ctx, immDir, SetAttrFlag, 0, &Attr{Flags: 0}); st != 0 {\n\t\tt.Fatalf(\"clear immutable: %s\", st)\n\t}\n\tm.Remove(ctx, RootInode, \"batchImm\", false, RmrDefaultThreads, nil)\n\n\t// --- test 6: empty entries ---\n\tcount = 0\n\tst = m.getBase().BatchClone(ctx, srcDir, dstDir, nil, 0, 022, &count)\n\tif st != 0 {\n\t\tt.Fatalf(\"BatchClone empty: %s\", st)\n\t}\n\tif count != 0 {\n\t\tt.Fatalf(\"BatchClone empty count: got %d, want 0\", count)\n\t}\n\n\t// --- test 7: preserve attr mode ---\n\tvar dstDir2 Ino\n\tif st := m.Mkdir(ctx, RootInode, \"batchDst2\", 0755, 022, 0, &dstDir2, nil); st != 0 {\n\t\tt.Fatalf(\"mkdir batchDst2: %s\", st)\n\t}\n\tcount = 0\n\tst = m.getBase().BatchClone(ctx, srcDir, dstDir2, nonDirEntries, CLONE_MODE_PRESERVE_ATTR, 022, &count)\n\tif st != 0 {\n\t\tt.Fatalf(\"BatchClone preserve: %s\", st)\n\t}\n\t// verify preserved attrs match source\n\tvar dstEntries2 []*Entry\n\tif st := m.Readdir(ctx, dstDir2, 1, &dstEntries2); st != 0 {\n\t\tt.Fatalf(\"readdir batchDst2: %s\", st)\n\t}\n\tsrcMap := make(map[string]*Entry)\n\tfor _, e := range nonDirEntries {\n\t\tsrcMap[string(e.Name)] = e\n\t}\n\tfor _, de := range dstEntries2 {\n\t\tname := string(de.Name)\n\t\tif name == \".\" || name == \"..\" {\n\t\t\tcontinue\n\t\t}\n\t\tse, ok := srcMap[name]\n\t\tif !ok {\n\t\t\tt.Fatalf(\"unexpected entry %q in batchDst2\", name)\n\t\t}\n\t\tif de.Attr.Mode != se.Attr.Mode {\n\t\t\tt.Fatalf(\"preserve mode mismatch for %s: got %o, want %o\", name, de.Attr.Mode, se.Attr.Mode)\n\t\t}\n\t}\n\n\t// cleanup\n\tm.Remove(ctx, RootInode, \"batchSrc\", false, RmrDefaultThreads, nil)\n\tm.Remove(ctx, RootInode, \"batchDst\", false, RmrDefaultThreads, nil)\n\tm.Remove(ctx, RootInode, \"batchDst2\", false, RmrDefaultThreads, nil)\n}\n\nfunc testClone(t *testing.T, m Meta) {\n\t// $ tree cloneDir\n\t// .\n\t// ├── dir\n\t// └── dir1\n\t// ├── dir2\n\t// │ ├── dir3\n\t// │ │ └── file3\n\t// │ ├── file2\n\t// │ └── file2Hardlink\n\t// ├── file1\n\t// └── file1Symlink -> file1\n\tvar cloneDir Ino\n\tif eno := m.Mkdir(Background(), RootInode, \"cloneDir\", 0777, 022, 0, &cloneDir, nil); eno != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", eno)\n\t}\n\tvar dir1 Ino\n\tif eno := m.Mkdir(Background(), cloneDir, \"dir1\", 0777, 022, 0, &dir1, nil); eno != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", eno)\n\t}\n\tvar dir Ino\n\tif eno := m.Mkdir(Background(), cloneDir, \"dir\", 0777, 022, 0, &dir, nil); eno != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", eno)\n\t}\n\tvar dir2 Ino\n\tif eno := m.Mkdir(Background(), dir1, \"dir2\", 0777, 022, 0, &dir2, nil); eno != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", eno)\n\t}\n\tvar dir3 Ino\n\tif eno := m.Mkdir(Background(), dir2, \"dir3\", 0777, 022, 0, &dir3, nil); eno != 0 {\n\t\tt.Fatalf(\"mkdir: %s\", eno)\n\t}\n\tvar file1 Ino\n\tif eno := m.Mknod(Background(), dir1, \"file1\", TypeFile, 0777, 022, 0, \"\", &file1, nil); eno != 0 {\n\t\tt.Fatalf(\"mknod: %s\", eno)\n\t}\n\tvar sliceId uint64\n\tif st := m.NewSlice(Background(), &sliceId); st != 0 {\n\t\tt.Fatalf(\"new chunk: %s\", st)\n\t}\n\tif st := m.Write(Background(), file1, 0, 0, Slice{sliceId, 67108864, 0, 67108864}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file %s\", st)\n\t}\n\n\tvar file2 Ino\n\tif eno := m.Mknod(Background(), dir2, \"file2\", TypeFile, 0777, 022, 0, \"\", &file2, nil); eno != 0 {\n\t\tt.Fatalf(\"mknod: %s\", eno)\n\t}\n\tvar sliceId2 uint64\n\tif st := m.NewSlice(Background(), &sliceId2); st != 0 {\n\t\tt.Fatalf(\"new chunk: %s\", st)\n\t}\n\tif st := m.Write(Background(), file2, 0, 0, Slice{sliceId2, 67108863, 0, 67108863}, time.Now()); st != 0 {\n\t\tt.Fatalf(\"write file %s\", st)\n\t}\n\tvar file3 Ino\n\tif eno := m.Mknod(Background(), dir3, \"file3\", TypeFile, 0777, 022, 0, \"\", &file3, nil); eno != 0 {\n\t\tt.Fatalf(\"mknod: %s\", eno)\n\t}\n\tif eno := m.Fallocate(Background(), file3, 0, 0, 67108864, nil); eno != 0 {\n\t\tt.Fatalf(\"fallocate: %s\", eno)\n\t}\n\n\tif eno := m.SetXattr(Background(), file1, \"name\", []byte(\"juicefs\"), XattrCreateOrReplace); eno != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", eno)\n\t}\n\tif eno := m.SetXattr(Background(), file1, \"name2\", []byte(\"juicefs2\"), XattrCreateOrReplace); eno != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", eno)\n\t}\n\n\tif eno := m.SetXattr(Background(), dir1, \"name\", []byte(\"juicefs\"), XattrCreateOrReplace); eno != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", eno)\n\t}\n\tif eno := m.SetXattr(Background(), dir1, \"name2\", []byte(\"juicefs2\"), XattrCreateOrReplace); eno != 0 {\n\t\tt.Fatalf(\"setxattr: %s\", eno)\n\t}\n\n\tvar file1Symlink Ino\n\tif eno := m.Symlink(Background(), dir1, \"file1Symlink\", \"file1\", &file1Symlink, nil); eno != 0 {\n\t\tt.Fatalf(\"symlink: %s\", eno)\n\t}\n\tif eno := m.Link(Background(), file2, dir2, \"file2Hardlink\", nil); eno != 0 {\n\t\tt.Fatalf(\"hardlink: %s\", eno)\n\t}\n\n\tvar attr Attr\n\tattr.Mtime = 1\n\tm.SetAttr(Background(), cloneDir, SetAttrMtime, 0, &attr)\n\tvar totalspace, availspace, iused, iavail, space, iused2 uint64\n\tm.StatFS(Background(), RootInode, &totalspace, &availspace, &iused, &iavail)\n\tspace = totalspace - availspace\n\tiused2 = iused\n\n\tcloneDstName := \"cloneDir1\"\n\tvar count, total uint64\n\tvar cmode uint8\n\tcmode |= CLONE_MODE_PRESERVE_ATTR\n\tif eno := m.Clone(Background(), cloneDir, dir1, cloneDir, cloneDstName, cmode, 022, 4, &count, &total); eno != 0 {\n\t\tt.Fatalf(\"clone: %s\", eno)\n\t}\n\tvar entries1 []*Entry\n\tif eno := m.Readdir(Background(), cloneDir, 1, &entries1); eno != 0 {\n\t\tt.Fatalf(\"readdir: %s\", eno)\n\t}\n\n\tif len(entries1) != 5 {\n\t\tt.Fatalf(\"clone dst dir not found or name not correct\")\n\t}\n\tvar idx int\n\tfor i, ent := range entries1 {\n\t\tif string(ent.Name) == cloneDstName {\n\t\t\tidx = i\n\t\t\tbreak\n\t\t}\n\t}\n\tif idx == 0 {\n\t\tt.Fatalf(\"clone dst dir not found or name not correct\")\n\t}\n\tcloneDstIno := entries1[idx].Inode\n\tcloneDstAttr := entries1[idx].Attr\n\tif cloneDstAttr.Mode != 0755 {\n\t\tt.Fatalf(\"mode should be 0755 %o\", cloneDstAttr.Mode)\n\t}\n\t// check dst parent dir nlink\n\tvar rootAttr Attr\n\tif eno := m.GetAttr(Background(), cloneDir, &rootAttr); eno != 0 {\n\t\tt.Fatalf(\"get rootAttr: %s\", eno)\n\t}\n\tif rootAttr.Nlink != 5 {\n\t\tt.Fatalf(\"rootDir nlink not correct,nlink: %d\", rootAttr.Nlink)\n\t}\n\tif rootAttr.Mtime == 1 {\n\t\tt.Fatalf(\"mtime of rootDir is not updated\")\n\t}\n\tm.StatFS(Background(), cloneDir, &totalspace, &availspace, &iused, &iavail)\n\tif totalspace-availspace-space != 268451840 {\n\t\ttime.Sleep(time.Second * 2)\n\t\tm.StatFS(Background(), cloneDir, &totalspace, &availspace, &iused, &iavail)\n\t\tif totalspace-availspace-space != 268451840 {\n\t\t\tt.Logf(\"warning: added space: %d\", totalspace-availspace-space)\n\t\t}\n\t}\n\tif iused-iused2 != 8 {\n\t\tt.Fatalf(\"added inodes: %d\", iused-iused2)\n\t}\n\tif eno := m.Clone(Background(), RootInode, dir1, cloneDir, \"no_preserve\", 0, 022, 4, &count, &total); eno != 0 {\n\t\tt.Fatalf(\"clone: %s\", eno)\n\t}\n\tvar d2 Ino\n\tvar noPreserveAttr = new(Attr)\n\tm.Lookup(Background(), cloneDir, \"no_preserve\", &d2, noPreserveAttr, true)\n\tvar cloneSrcAttr = new(Attr)\n\tm.GetAttr(Background(), dir1, cloneSrcAttr)\n\tif noPreserveAttr.Mtimensec == cloneSrcAttr.Mtimensec {\n\t\tt.Fatalf(\"clone: should not preserve mtime\")\n\t}\n\tif eno := m.Remove(Background(), cloneDir, \"no_preserve\", false, RmrDefaultThreads, nil); eno != 0 {\n\t\tt.Fatalf(\"Rmdir: %s\", eno)\n\t}\n\t// check attr\n\tvar removedItem []interface{}\n\tcheckEntryTree(t, m, dir1, cloneDstIno, func(srcEntry, dstEntry *Entry, dstIno Ino) {\n\t\tcheckEntry(t, m, srcEntry, dstEntry, dstIno)\n\n\t\tswitch m := m.(type) {\n\t\tcase *redisMeta:\n\t\t\tremovedItem = append(removedItem, m.inodeKey(dstEntry.Inode), m.entryKey(dstEntry.Inode), m.xattrKey(dstEntry.Inode), m.symKey(dstEntry.Inode))\n\t\tcase *dbMeta:\n\t\t\tremovedItem = append(removedItem, &node{Inode: dstEntry.Inode}, &edge{Inode: dstEntry.Inode, Parent: dstEntry.Attr.Parent}, &xattr{Inode: dstEntry.Inode}, &symlink{Inode: dstEntry.Inode})\n\t\tcase *kvMeta:\n\t\t\tremovedItem = append(removedItem, m.inodeKey(dstEntry.Inode), m.entryKey(dstEntry.Attr.Parent, string(dstEntry.Name)), m.symKey(dstEntry.Inode))\n\t\t}\n\t})\n\t// check slice ref after clone\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error {\n\t\tt.Fatalf(\"should not delete slice\")\n\t\treturn nil\n\t})\n\tif eno := m.Remove(Background(), cloneDir, \"dir1\", false, RmrDefaultThreads, nil); eno != 0 {\n\t\tt.Fatalf(\"Rmdir: %s\", eno)\n\t}\n\n\tvar sli1del, sli2del bool\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error {\n\t\tif args[0].(uint64) == sliceId {\n\t\t\tsli1del = true\n\t\t}\n\t\tif args[0].(uint64) == sliceId2 {\n\t\t\tsli2del = true\n\t\t}\n\t\treturn nil\n\t})\n\t// check remove tree\n\tvar dNode1, dNode2, dNode3, dNode4 Ino = 101, 102, 103, 104\n\tswitch m := m.(type) {\n\tcase *redisMeta:\n\t\t// del edge first\n\t\tif err := m.rdb.HDel(Background(), m.entryKey(cloneDstAttr.Parent), cloneDstName).Err(); err != nil {\n\t\t\tt.Fatalf(\"del edge error: %v\", err)\n\t\t}\n\t\t// check remove tree\n\t\tif eno := m.doCleanupDetachedNode(Background(), cloneDstIno); eno != 0 {\n\t\t\tt.Fatalf(\"remove tree error rootInode: %v\", cloneDstIno)\n\t\t}\n\t\tremovedKeysStr := make([]string, len(removedItem))\n\t\tfor i, key := range removedItem {\n\t\t\tremovedKeysStr[i] = key.(string)\n\t\t}\n\t\tremovedKeysStr = append(removedKeysStr, m.detachedNodes())\n\t\tif exists := m.rdb.Exists(Background(), removedKeysStr...).Val(); exists != 0 {\n\t\t\tt.Fatalf(\"has keys not removed: %v\", removedItem)\n\t\t}\n\t\t// check detached node\n\t\tm.rdb.ZAdd(Background(), m.detachedNodes(), redis.Z{Member: dNode1.String(), Score: float64(time.Now().Add(-1 * time.Minute).Unix())}).Err()\n\t\tm.rdb.ZAdd(Background(), m.detachedNodes(), redis.Z{Member: dNode2.String(), Score: float64(time.Now().Add(-5 * time.Minute).Unix())}).Err()\n\t\tm.rdb.ZAdd(Background(), m.detachedNodes(), redis.Z{Member: dNode3.String(), Score: float64(time.Now().Add(-48 * time.Hour).Unix())}).Err()\n\t\tm.rdb.ZAdd(Background(), m.detachedNodes(), redis.Z{Member: dNode4.String(), Score: float64(time.Now().Add(-48 * time.Hour).Unix())}).Err()\n\tcase *dbMeta:\n\t\tif n, err := m.db.Delete(&edge{Parent: cloneDstAttr.Parent, Name: []byte(cloneDstName)}); err != nil || n != 1 {\n\t\t\tt.Fatalf(\"del edge error: %v\", err)\n\t\t}\n\t\t// check remove tree\n\t\tif eno := m.doCleanupDetachedNode(Background(), cloneDstIno); eno != 0 {\n\t\t\tt.Fatalf(\"remove tree error rootInode: %v\", cloneDstIno)\n\t\t}\n\t\tremovedItem = append(removedItem, &detachedNode{Inode: cloneDstIno})\n\t\ttime.Sleep(1 * time.Second)\n\t\tif exists, err := m.db.Exist(removedItem...); err != nil || exists {\n\t\t\tt.Fatalf(\"has keys not removed: %v\", removedItem)\n\t\t}\n\t\tm.txn(func(s *xorm.Session) error {\n\t\t\treturn mustInsert(s,\n\t\t\t\t&detachedNode{Inode: dNode1, Added: time.Now().Add(-1 * time.Minute).Unix()},\n\t\t\t\t&detachedNode{Inode: dNode2, Added: time.Now().Add(-5 * time.Minute).Unix()},\n\t\t\t\t&detachedNode{Inode: dNode3, Added: time.Now().Add(-48 * time.Hour).Unix()},\n\t\t\t\t&detachedNode{Inode: dNode4, Added: time.Now().Add(-48 * time.Hour).Unix()},\n\t\t\t)\n\t\t})\n\tcase *kvMeta:\n\t\t// del edge first\n\t\tif err := m.deleteKeys(m.entryKey(cloneDstAttr.Parent, cloneDstName)); err != nil {\n\t\t\tt.Fatalf(\"del edge error: %v\", err)\n\t\t}\n\t\t// check remove tree\n\t\tif eno := m.doCleanupDetachedNode(Background(), cloneDstIno); eno != 0 {\n\t\t\tt.Fatalf(\"remove tree error rootInode: %v\", cloneDstIno)\n\t\t}\n\t\tremovedItem = append(removedItem, m.detachedKey(cloneDstIno))\n\t\tm.txn(Background(), func(tx *kvTxn) error {\n\t\t\tfor _, key := range removedItem {\n\t\t\t\tif buf := tx.get(key.([]byte)); buf != nil {\n\t\t\t\t\tt.Fatalf(\"has keys not removed: %v\", removedItem)\n\t\t\t\t}\n\t\t\t}\n\t\t\ttx.set(m.detachedKey(dNode1), m.packInt64(time.Now().Add(-1*time.Minute).Unix()))\n\t\t\ttx.set(m.detachedKey(dNode2), m.packInt64(time.Now().Add(-5*time.Minute).Unix()))\n\t\t\ttx.set(m.detachedKey(dNode3), m.packInt64(time.Now().Add(-48*time.Hour).Unix()))\n\t\t\ttx.set(m.detachedKey(dNode4), m.packInt64(time.Now().Add(-48*time.Hour).Unix()))\n\t\t\treturn nil\n\t\t})\n\n\t}\n\ttime.Sleep(1 * time.Second)\n\tif !sli1del || !sli2del {\n\t\tt.Fatalf(\"slice should be deleted\")\n\t}\n\tnodes := m.(engine).doFindDetachedNodes(time.Now())\n\tif len(nodes) != 4 {\n\t\tt.Fatalf(\"find detached nodes error: %v\", nodes)\n\t}\n\tnodes = m.(engine).doFindDetachedNodes(time.Now().Add(-24 * time.Hour))\n\tif len(nodes) != 2 {\n\t\tt.Fatalf(\"find detached nodes error: %v\", nodes)\n\t}\n\tif eno := m.Clone(Background(), RootInode, TrashInode, cloneDir, \"xxx\", 0, 022, 4, &count, &total); !errors.Is(eno, syscall.EPERM) {\n\t\tt.Fatalf(\"cloning trash files are not supported\")\n\t}\n\tif eno := m.Clone(Background(), TrashInode+1, 1000, cloneDir, \"xxx\", 0, 022, 4, &count, &total); !errors.Is(eno, syscall.EPERM) {\n\t\tt.Fatalf(\"cloning files in the trash is not supported\")\n\t}\n}\n\nfunc checkEntryTree(t *testing.T, m Meta, srcIno, dstIno Ino, walkFunc func(srcEntry, dstEntry *Entry, dstIno Ino)) {\n\tvar entries1 []*Entry\n\tif eno := m.Readdir(Background(), srcIno, 1, &entries1); eno != 0 {\n\t\tt.Fatalf(\"Readdir: %s\", eno)\n\t}\n\n\tvar entries2 []*Entry\n\tif eno := m.Readdir(Background(), dstIno, 1, &entries2); eno != 0 {\n\t\tt.Fatalf(\"Readdir: %s\", eno)\n\t}\n\tsort.Slice(entries1, func(i, j int) bool { return string(entries1[i].Name) < string(entries1[j].Name) })\n\tsort.Slice(entries2, func(i, j int) bool { return string(entries2[i].Name) < string(entries2[j].Name) })\n\tif len(entries1) != len(entries2) {\n\t\tt.Fatalf(\"number of children: %d != %d\", len(entries1), len(entries2))\n\t}\n\tfor idx, entry := range entries1 {\n\t\tif string(entry.Name) == \".\" || string(entry.Name) == \"..\" {\n\t\t\tcontinue\n\t\t}\n\t\tif entry.Attr.Typ == TypeDirectory {\n\t\t\tcheckEntryTree(t, m, entry.Inode, entries2[idx].Inode, walkFunc)\n\t\t}\n\t\twalkFunc(entry, entries2[idx], dstIno)\n\t}\n}\n\nfunc checkEntry(t *testing.T, m Meta, srcEntry, dstEntry *Entry, dstParentIno Ino) {\n\tif !bytes.Equal(srcEntry.Name, dstEntry.Name) {\n\t\tt.Fatalf(\"unmatched name: %s, %s\", srcEntry.Name, dstEntry.Name)\n\t}\n\tsrcAttr := srcEntry.Attr\n\tdstAttr := dstEntry.Attr\n\tif dstAttr.Parent != dstParentIno {\n\t\tt.Fatalf(\"unmatched parent: %d, %d\", dstAttr.Parent, dstParentIno)\n\t}\n\tif srcAttr.Typ == TypeFile && dstAttr.Nlink != 1 || srcAttr.Typ != TypeFile && srcAttr.Nlink != dstAttr.Nlink {\n\t\tt.Fatalf(\"nlink not correct: srcType:%d,srcNlink:%d,dstType:%d,dstNlink:%d\", srcAttr.Typ, srcAttr.Nlink, dstAttr.Typ, dstAttr.Nlink)\n\t}\n\n\tsrcAttr.Nlink = 0\n\tdstAttr.Nlink = 0\n\tsrcAttr.Parent = 0\n\tdstAttr.Parent = 0\n\tsrcAttr.Atime = 0\n\tsrcAttr.Atimensec = 0\n\tdstAttr.Atime = 0\n\tdstAttr.Atimensec = 0\n\tif *srcAttr != *dstAttr {\n\t\tt.Fatalf(\"unmatched attr: %#v, %#v\", *srcAttr, *dstAttr)\n\t}\n\n\t// check xattr\n\tvar value1 []byte\n\tif eno := m.ListXattr(Background(), srcEntry.Inode, &value1); eno != 0 {\n\t\tt.Fatalf(\"list xattr: %s\", eno)\n\t}\n\tkeys := bytes.Split(value1, []byte{0})\n\tfor _, key := range keys {\n\t\tif key == nil || len(key) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tvar v1, v2 []byte\n\t\tif eno := m.GetXattr(Background(), srcEntry.Inode, string(key), &v1); eno != 0 {\n\t\t\tt.Fatalf(\"get xattr: %s\", eno)\n\t\t}\n\t\tif eno := m.GetXattr(Background(), dstEntry.Inode, string(key), &v2); eno != 0 {\n\t\t\tt.Fatalf(\"get xattr: %s\", eno)\n\t\t}\n\t\tif !bytes.Equal(v1, v2) {\n\t\t\tt.Fatalf(\"xattr not equal\")\n\t\t}\n\t}\n}\n\nfunc testQuota(t *testing.T, m Meta) {\n\tif err := m.NewSession(true); err != nil {\n\t\tt.Fatalf(\"New session: %s\", err)\n\t}\n\tdefer m.CloseSession()\n\tctx := Background()\n\tvar inode, parent Ino\n\tvar attr Attr\n\tif st := m.Mkdir(ctx, RootInode, \"quota\", 0755, 0, 0, &parent, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir quota: %s\", st)\n\t}\n\tp := \"/quota\"\n\tif err := m.HandleQuota(ctx, QuotaSet, p, 0, 0, map[string]*Quota{p: {MaxSpace: 2 << 30, MaxInodes: 6}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set %s: %s\", p, err)\n\t}\n\tm.getBase().loadQuotas()\n\tif st := m.Mkdir(ctx, parent, \"d1\", 0755, 0, 0, &inode, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir quota/d1: %s\", st)\n\t}\n\tp = \"/quota/d1\"\n\tif err := m.HandleQuota(ctx, QuotaSet, p, 0, 0, map[string]*Quota{p: {MaxSpace: 1 << 30, MaxInodes: 5}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota %s: %s\", p, err)\n\t}\n\tm.getBase().loadQuotas()\n\tif st := m.Create(ctx, inode, \"f1\", 0644, 0, 0, nil, &attr); st != 0 {\n\t\tt.Fatalf(\"Create quota/d1/f1: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, parent, \"d2\", 0755, 0, 0, &parent, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir quota/d2: %s\", st)\n\t}\n\tif st := m.Mkdir(ctx, parent, \"d22\", 0755, 0, 0, &inode, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir quota/d2/d22: %s\", st)\n\t}\n\tp = \"/quota/d2/d22\"\n\tif err := m.HandleQuota(ctx, QuotaSet, p, 0, 0, map[string]*Quota{p: {MaxSpace: 1 << 30, MaxInodes: 5}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota %s: %s\", p, err)\n\t}\n\tm.getBase().loadQuotas()\n\t// parent -> d2, inode -> d22\n\tif st := m.Create(ctx, parent, \"f2\", 0644, 0, 0, nil, &attr); st != 0 {\n\t\tt.Fatalf(\"Create quota/d2/f2: %s\", st)\n\t}\n\tif st := m.Create(ctx, inode, \"f22\", 0644, 0, 0, nil, &attr); st != 0 {\n\t\tt.Fatalf(\"Create quota/d22/f22: %s\", st)\n\t}\n\ttime.Sleep(time.Second * 5)\n\n\tqs := make(map[string]*Quota)\n\tp = \"/quota\"\n\tif err := m.HandleQuota(ctx, QuotaGet, p, 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get %s: %s\", p, err)\n\t} else if q := qs[p]; q.MaxSpace != 2<<30 || q.MaxInodes != 6 || q.UsedSpace != 6*4<<10 || q.UsedInodes != 6 {\n\t\tt.Fatalf(\"HandleQuota get %s: %+v\", p, q)\n\t}\n\tdelete(qs, p)\n\tp = \"/quota/d1\"\n\tif err := m.HandleQuota(ctx, QuotaGet, p, 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get %s: %s\", p, err)\n\t} else if q := qs[p]; q.MaxSpace != 1<<30 || q.MaxInodes != 5 || q.UsedSpace != 4<<10 || q.UsedInodes != 1 {\n\t\tt.Fatalf(\"HandleQuota get %s: %+v\", p, q)\n\t}\n\tdelete(qs, p)\n\tp = \"/quota/d2/d22\"\n\tif err := m.HandleQuota(ctx, QuotaGet, p, 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get %s: %s\", p, err)\n\t} else if q := qs[p]; q.MaxSpace != 1<<30 || q.MaxInodes != 5 || q.UsedSpace != 4<<10 || q.UsedInodes != 1 {\n\t\tt.Fatalf(\"HandleQuota get %s: %+v\", p, q)\n\t}\n\tdelete(qs, p)\n\n\tif err := m.HandleQuota(ctx, QuotaList, \"\", 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota list: %s\", err)\n\t} else {\n\t\tif len(qs) != 3 {\n\t\t\tt.Fatalf(\"HandleQuota list bad result: %d\", len(qs))\n\t\t}\n\t}\n\n\tgetUsedInodes := func(path string) int64 {\n\t\tm.getBase().doFlushQuotas()\n\t\tqs := make(map[string]*Quota)\n\t\tif err := m.HandleQuota(ctx, QuotaGet, path, 0, 0, qs, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota list: %s\", err)\n\t\t}\n\t\treturn qs[path].UsedInodes\n\t}\n\n\t// unlink opened file\n\tvar nInode Ino\n\tif st := m.Lookup(ctx, parent, \"f2\", &nInode, &attr, false); st != 0 {\n\t\tt.Fatalf(\"Lookup quota/d2/f2: %s\", st)\n\t}\n\n\tif st := m.Open(ctx, nInode, 0, &attr); st != 0 {\n\t\tt.Fatalf(\"Open quota/d2/f2: %s\", st)\n\t}\n\n\tif st := m.Unlink(ctx, parent, \"f2\"); st != 0 {\n\t\tt.Fatalf(\"Unlink quota/d2/f2 err: %s\", st)\n\t}\n\n\tif st := m.Close(ctx, nInode); st != 0 {\n\t\tt.Fatalf(\"Close quota/d2/f2: %s\", st)\n\t}\n\n\tif used := getUsedInodes(\"/quota\"); used != 5 {\n\t\tt.Fatalf(\"used inodes of /quota should be 5, but got %d\", used)\n\t}\n\n\t// rename opened file\n\tif st := m.Lookup(ctx, inode, \"f22\", &nInode, &attr, false); st != 0 {\n\t\tt.Fatalf(\"Lookup quota/d2/d22/f22: %s\", st)\n\t}\n\n\tif st := m.Open(ctx, nInode, 0, &attr); st != 0 {\n\t\tt.Fatalf(\"Open quota/d2/d22/f22: %s\", st)\n\t}\n\n\tif st := m.Rename(ctx, inode, \"f22\", inode, \"f23\", 0, &nInode, nil); st != 0 {\n\t\tt.Fatalf(\"Rename quota/d2/d22/f22 to quota/d2/d22/f23 err: %s\", st)\n\t}\n\n\tif st := m.Close(ctx, nInode); st != 0 {\n\t\tt.Fatalf(\"Close quota/d2/d22/f23: %s\", st)\n\t}\n\n\tif used := getUsedInodes(\"/quota\"); used != 5 {\n\t\tt.Fatalf(\"used inodes of /quota should be 5, but got %d\", used)\n\t}\n\n\tif st := m.Create(ctx, parent, \"f3\", 0644, 0, 0, &nInode, &attr); st != 0 {\n\t\tt.Fatalf(\"Create quota/d2/f3: %s\", st)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"/quota/d1\", 0, 0, nil, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota del /quota/d1: %s\", err)\n\t}\n\tif err := m.HandleQuota(ctx, QuotaDel, \"/quota/d2\", 0, 0, nil, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota del /quota/d2: %s\", err)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaList, \"\", 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota list: %s\", err)\n\t} else {\n\t\tif len(qs) != 2 {\n\t\t\tt.Fatalf(\"HandleQuota list bad result: %d\", len(qs))\n\t\t}\n\t}\n\tm.getBase().loadQuotas()\n\tif st := m.Create(ctx, parent, \"f4\", 0644, 0, 0, nil, &attr); st != syscall.EDQUOT {\n\t\tt.Fatalf(\"Create quota/d22/f4: %s\", st)\n\t}\n}\n\nfunc testAtime(t *testing.T, m Meta) {\n\tctx := Background()\n\tvar inode, parent Ino\n\tvar attr Attr\n\tif st := m.Mkdir(ctx, RootInode, \"atime\", 0755, 0, 0, &parent, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir atime: %s\", st)\n\t}\n\n\t// open, read, read atime < mtime, read recent, readdir, readlink, link\n\ttestFn := func(name string) (ret [7]bool) {\n\t\tfname := \"f-\" + name\n\t\tif st := m.Create(ctx, parent, fname, 0644, 0, 0, &inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create atime/%s: %s\", fname, st)\n\t\t}\n\t\t// atime < ctime\n\t\tattr.Atime, attr.Atimensec = 1234, 5678\n\t\tif st := m.SetAttr(ctx, inode, SetAttrAtime, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Setattr atime/%s: %s\", fname, st)\n\t\t}\n\t\tif st := m.Open(ctx, inode, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Open atime/%s: %s\", fname, st)\n\t\t}\n\t\tdefer m.Close(ctx, inode)\n\t\tret[0] = attr.Atime != 1234\n\n\t\tattr.Atime, attr.Atimensec = 1234, 5678\n\t\tif st := m.SetAttr(ctx, inode, SetAttrAtime, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Setattr atime/%s: %s\", fname, st)\n\t\t}\n\t\tvar slices []Slice\n\t\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\t\tt.Fatalf(\"Read atime/%s: %s\", fname, st)\n\t\t}\n\t\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Getattr after read atime/%s: %s\", fname, st)\n\t\t}\n\t\tret[1] = attr.Atime != 1234\n\n\t\t// atime < mtime\n\t\tnow := time.Now()\n\t\tattr.Atime = now.Unix() - 2\n\t\tattr.Mtime = now.Unix()\n\t\tif st := m.SetAttr(ctx, inode, SetAttrAtime|SetAttrMtime, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Setattr atime/%s: %s\", fname, st)\n\t\t}\n\t\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\t\tt.Fatalf(\"Read atime/%s: %s\", fname, st)\n\t\t}\n\t\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Getattr after read atime/%s: %s\", fname, st)\n\t\t}\n\t\tret[2] = attr.Atime >= now.Unix()\n\n\t\t// atime = ctime = mtime, atime = now\n\t\tif st := m.SetAttr(ctx, inode, SetAttrAtimeNow|SetAttrMtimeNow, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Setattr atime/%s: %s\", fname, st)\n\t\t}\n\t\ttime.Sleep(time.Second * 2)\n\t\tnow = time.Now()\n\t\tif st := m.Read(ctx, inode, 0, &slices); st != 0 {\n\t\t\tt.Fatalf(\"Read atime/%s: %s\", fname, st)\n\t\t}\n\t\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Getattr after read atime/%s: %s\", fname, st)\n\t\t}\n\t\tret[3] = attr.Atime >= now.Unix()\n\n\t\t// readdir\n\t\tfname = \"d-\" + name\n\t\tif st := m.Mkdir(ctx, parent, fname, 0755, 0, 0, &inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Mkdir atime/%s: %s\", fname, st)\n\t\t}\n\t\tattr.Atime, attr.Atimensec = 1234, 5678\n\t\tif st := m.SetAttr(ctx, inode, SetAttrAtime, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Setattr atime/%s: %s\", fname, st)\n\t\t}\n\t\tvar entries []*Entry\n\t\tif st := m.Readdir(ctx, inode, 0, &entries); st != 0 {\n\t\t\tt.Fatalf(\"Readdir atime/%s: %s\", fname, st)\n\t\t}\n\t\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Getattr after readdir atime/%s: %s\", fname, st)\n\t\t}\n\t\tret[4] = attr.Atime != 1234\n\n\t\t// readlink\n\t\tfname = \"s-\" + name\n\t\tif st := m.Symlink(ctx, parent, fname, \"f-\"+name, &inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Symlink atime/%s: %s\", fname, st)\n\t\t}\n\t\tattr.Atime, attr.Atimensec = 1234, 5678\n\t\tif st := m.SetAttr(ctx, inode, SetAttrAtime, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Setattr atime/%s: %s\", fname, st)\n\t\t}\n\t\tvar target []byte\n\t\tif st := m.ReadLink(ctx, inode, &target); st != 0 {\n\t\t\tt.Fatalf(\"Readlink atime/%s: %s\", fname, st)\n\t\t}\n\t\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Getattr after readlink atime/%s: %s\", fname, st)\n\t\t}\n\t\tret[5] = attr.Atime != 1234 && attr.Atimensec != 5678\n\n\t\t// test link ctime\n\t\tattr.Atime, attr.Atimensec = 1234, 5678\n\t\tif st := m.SetAttr(ctx, inode, SetAttrAtime, 0, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Setattr atime/%s: %s\", fname, st)\n\t\t}\n\t\tfname = \"l-\" + name\n\t\tif st := m.Link(ctx, inode, parent, fname, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Link %s: %s\", fname, st)\n\t\t}\n\t\tret[6] = attr.Ctime != 1234 && attr.Ctimensec != 5678\n\t\treturn\n\t}\n\n\tfor name, exp := range map[string][7]bool{\n\t\tRelAtime: {true, true, true, false, true, true, true},\n\t\tStrictAtime: {true, true, true, true, true, true, true},\n\t\tNoAtime: {false, false, false, false, false, false, true},\n\t} {\n\t\tm.getBase().conf.AtimeMode = name\n\t\tif ret := testFn(name); ret != exp {\n\t\t\tt.Fatalf(\"Test %s: expected %v, got %v\", name, exp, ret)\n\t\t}\n\t}\n}\n\n// TestQuotaEdgeCases\nfunc TestQuotaEdgeCases(t *testing.T) {\n\tm := &baseMeta{}\n\n\tm.userQuotas = make(map[uint64]*Quota)\n\tm.groupQuotas = make(map[uint64]*Quota)\n\tm.quotaMu = sync.RWMutex{}\n\n\tm.fmt = &Format{\n\t\tUserGroupQuota: true,\n\t}\n\n\tfileOwnerUid := uint32(1001)\n\tfileOwnerGid := uint32(2001)\n\toperatorUid := uint32(1002)\n\toperatorGid := uint32(2002)\n\n\tt.Log(\"Testing inodes-only quota limit...\")\n\tm.userQuotas[uint64(fileOwnerUid)] = &Quota{MaxSpace: 0, MaxInodes: 3}\n\tm.groupQuotas[uint64(fileOwnerGid)] = &Quota{MaxSpace: 0, MaxInodes: 5}\n\n\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\n\tif err := m.checkQuota(operatorCtx, 10*1024*1024, 0, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass for large space usage (no space limit), got: %s\", err)\n\t}\n\n\tif err := m.checkQuota(operatorCtx, 0, 4, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding inodes limit, got: %s\", err)\n\t}\n\n\tt.Log(\"Testing space-only quota limit...\")\n\tm.userQuotas[uint64(fileOwnerUid)] = &Quota{MaxSpace: 1024 * 1024, MaxInodes: 0}\n\tm.groupQuotas[uint64(fileOwnerGid)] = &Quota{MaxSpace: 2 * 1024 * 1024, MaxInodes: 0}\n\n\tif err := m.checkQuota(operatorCtx, 0, 100, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass for large inodes usage (no inodes limit), got: %s\", err)\n\t}\n\n\tif err := m.checkQuota(operatorCtx, 2*1024*1024, 0, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding space limit, got: %s\", err)\n\t}\n\n\tt.Log(\"Testing mixed quota limits...\")\n\tm.userQuotas[uint64(fileOwnerUid)] = &Quota{MaxSpace: 0, MaxInodes: 2}\n\tm.groupQuotas[uint64(fileOwnerGid)] = &Quota{MaxSpace: 1024 * 1024, MaxInodes: 0}\n\n\tif err := m.checkQuota(operatorCtx, 512*1024, 3, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding user inodes limit, got: %s\", err)\n\t}\n\n\tif err := m.checkQuota(operatorCtx, 2*1024*1024, 1, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding group space limit, got: %s\", err)\n\t}\n\n\tif err := m.checkQuota(operatorCtx, 512*1024, 1, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass when within both limits, got: %s\", err)\n\t}\n}\n\n// TestCheckQuotaFileOwner\nfunc TestCheckQuotaFileOwner(t *testing.T) {\n\tm := &baseMeta{}\n\n\tm.userQuotas = make(map[uint64]*Quota)\n\tm.groupQuotas = make(map[uint64]*Quota)\n\tm.quotaMu = sync.RWMutex{}\n\n\tm.fmt = &Format{\n\t\tUserGroupQuota: true,\n\t}\n\n\tfileOwnerUid := uint32(1001)\n\tfileOwnerGid := uint32(2001)\n\toperatorUid := uint32(1002)\n\toperatorGid := uint32(2002)\n\n\tm.userQuotas[uint64(fileOwnerUid)] = &Quota{MaxSpace: 1 << 20, MaxInodes: 5}\n\tm.groupQuotas[uint64(fileOwnerGid)] = &Quota{MaxSpace: 2 << 20, MaxInodes: 10}\n\n\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\n\tif err := m.checkQuota(operatorCtx, 1024, 1, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass for file owner's quota, got: %s\", err)\n\t}\n\n\tif err := m.checkQuota(operatorCtx, 2<<20, 1, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding file owner's user quota, got: %s\", err)\n\t}\n\n\tif err := m.checkQuota(operatorCtx, 1024, 15, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding file owner's group quota, got: %s\", err)\n\t}\n\n\tm.userQuotas[uint64(fileOwnerUid)] = &Quota{MaxSpace: 0, MaxInodes: 0}\n\tif err := m.checkQuota(operatorCtx, 1, 1, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass when quota is zero (unlimited), got: %s\", err)\n\t}\n\n\tdelete(m.userQuotas, uint64(fileOwnerUid))\n\tdelete(m.groupQuotas, uint64(fileOwnerGid))\n\tif err := m.checkQuota(operatorCtx, 1024, 1, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass when no quota limits, got: %s\", err)\n\t}\n}\n\nfunc TestSymlinkCache(t *testing.T) {\n\tcache := newSymlinkCache(10000)\n\n\tjob := make(chan Ino)\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor ino := range job {\n\t\t\t\tcache.Store(ino, []byte(fmt.Sprintf(\"file%d\", ino)))\n\t\t\t}\n\t\t}()\n\t}\n\n\tfor i := 0; i < 10000; i++ {\n\t\tjob <- Ino(i)\n\t}\n\tclose(job)\n\twg.Wait()\n\n\tcache.doClean()\n\trequire.Equal(t, int32(8000), cache.size.Load())\n}\n\nfunc TestTxBatchLock(t *testing.T) {\n\tvar base baseMeta\n\t// 0 inode\n\tfunc() {\n\t\tdefer base.txBatchLock()()\n\t}()\n\t// 1 inodes\n\tfunc() {\n\t\tdefer base.txBatchLock(2)()\n\t}()\n\t// 2 inodes\n\tfunc() {\n\t\tdefer base.txBatchLock(1, 2)()\n\t}()\n\t// no reentrant\n\tfunc() {\n\t\tdefer base.txBatchLock(1, 1, nlocks+1)()\n\t}()\n\t// no deadlock - sequential\n\tfunc() {\n\t\tbatch1 := []Ino{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}\n\t\tbatch2 := []Ino{1 + nlocks*9, 2 + nlocks*8, 3 + nlocks*7, 4 + nlocks*6, 5 + nlocks*5, 6 + nlocks*4, 7 + nlocks*3, 8 + nlocks*2, 9 + nlocks, 10}\n\t\tvar wg sync.WaitGroup\n\t\tfor i := 0; i < 100; i++ {\n\t\t\twg.Add(2)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tdefer base.txBatchLock(batch1...)()\n\t\t\t}()\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tdefer base.txBatchLock(batch2...)()\n\t\t\t}()\n\t\t}\n\t\twg.Wait()\n\t}()\n\t// no deadlock - fuzz testing\n\tfunc() {\n\t\tvar batch1, batch2 []Ino\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tbatch1 = append(batch1, Ino(rand.Uint64()+1))\n\t\t\tbatch2 = append(batch2, Ino(rand.Uint64()+1))\n\t\t}\n\t\tvar wg sync.WaitGroup\n\t\tfor i := 0; i < 100; i++ {\n\t\t\twg.Add(2)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tdefer base.txBatchLock(batch1...)()\n\t\t\t}()\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tdefer base.txBatchLock(batch2...)()\n\t\t\t}()\n\t\t}\n\t\twg.Wait()\n\t}()\n}\n\n// testCheckQuotaFileOwnerSimple\nfunc testCheckQuotaFileOwnerSimple(t *testing.T, m Meta) {\n\tctx := Background()\n\tparent := RootInode\n\n\tfileOwnerUid := uint32(1001)\n\tfileOwnerGid := uint32(1001)\n\toperatorUid := uint32(1002)\n\toperatorGid := uint32(1002)\n\n\tformat := m.getBase().getFormat()\n\tformat.UserGroupQuota = true\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 4096, MaxInodes: 5}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set user quota: %s\", err)\n\t}\n\tm.getBase().loadQuotas()\n\n\tvar fileInode Ino\n\tvar attr Attr\n\tif st := m.Create(ctx, parent, \"testfile\", 0644, 0, 0, &fileInode, &attr); st != 0 {\n\t\tt.Fatalf(\"Create testfile: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, fileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID: %s\", st)\n\t}\n\n\tvar sliceId uint64\n\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\tt.Fatalf(\"NewSlice: %s\", st)\n\t}\n\tslice := Slice{Id: sliceId, Size: 4096, Len: 4096}\n\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\n\tif st := m.Write(operatorCtx, fileInode, 0, 0, slice, time.Now()); st != 0 {\n\t\tt.Fatalf(\"First write should succeed: %s\", st)\n\t}\n\n\tvar sliceId2 uint64\n\tif st := m.NewSlice(ctx, &sliceId2); st != 0 {\n\t\tt.Fatalf(\"NewSlice for second write: %s\", st)\n\t}\n\tslice2 := Slice{Id: sliceId2, Size: 4096, Len: 4096}\n\tif st := m.Write(operatorCtx, fileInode, 1, 0, slice2, time.Now()); st != syscall.EDQUOT {\n\t\tt.Fatalf(\"Second write should fail with EDQUOT, got: %s\", st)\n\t}\n\n\tm.CloseSession()\n}\n\n// testQuotaEdgeCases\nfunc testQuotaEdgeCases(t *testing.T, m Meta) {\n\tctx := Background()\n\n\tfileOwnerUid := uint32(1001)\n\tfileOwnerGid := uint32(1001)\n\toperatorUid := uint32(1002)\n\toperatorGid := uint32(2002)\n\n\tformat := m.getBase().getFormat()\n\tformat.UserGroupQuota = true\n\n\tt.Log(\"Testing inodes-only quota limit...\")\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 0, MaxInodes: 2}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set inodes-only quota: %s\", err)\n\t}\n\tm.getBase().loadQuotas()\n\n\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\n\tif err := m.getBase().checkQuota(operatorCtx, 10*1024*1024, 0, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass for large space usage (no space limit), got: %s\", err)\n\t}\n\n\tif err := m.getBase().checkQuota(operatorCtx, 0, 3, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding inodes limit, got: %s\", err)\n\t}\n\n\tt.Log(\"Testing space-only quota limit...\")\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1024 * 1024, MaxInodes: 0}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set space-only quota: %s\", err)\n\t}\n\tm.getBase().loadQuotas()\n\n\tif err := m.getBase().checkQuota(operatorCtx, 0, 100, fileOwnerUid, fileOwnerGid); err != 0 {\n\t\tt.Fatalf(\"checkQuota should pass for large inodes usage (no inodes limit), got: %s\", err)\n\t}\n\n\tif err := m.getBase().checkQuota(operatorCtx, 2*1024*1024, 0, fileOwnerUid, fileOwnerGid); err != syscall.EDQUOT {\n\t\tt.Fatalf(\"checkQuota should fail with EDQUOT when exceeding space limit, got: %s\", err)\n\t}\n}\n\n// testQuotaEdgeCasesComplex\nfunc testQuotaEdgeCasesComplex(t *testing.T, m Meta) {\n\tctx := Background()\n\tparent := RootInode\n\n\tfileOwnerUid := uint32(1001)\n\tfileOwnerGid := uint32(1001)\n\toperatorUid := uint32(1002)\n\toperatorGid := uint32(1002)\n\n\tformat := m.getBase().getFormat()\n\tformat.UserGroupQuota = true\n\n\tt.Log(\"Testing inodes-only quota limit...\")\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 0, MaxInodes: 2}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set inodes-only quota: %s\", err)\n\t}\n\tm.getBase().loadQuotas()\n\n\tvar fileInode Ino\n\tvar attr Attr\n\tif st := m.Create(ctx, parent, \"testfile_inodes\", 0644, 0, 0, &fileInode, &attr); st != 0 {\n\t\tt.Fatalf(\"Create testfile_inodes: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, fileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID: %s\", st)\n\t}\n\n\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\tfor i := 0; i < 5; i++ {\n\t\tvar sliceId uint64\n\t\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice %d: %s\", i, st)\n\t\t}\n\t\tslice := Slice{Id: sliceId, Size: 1024 * 1024, Len: 1024 * 1024}\n\t\tif st := m.Write(operatorCtx, fileInode, uint32(i), uint32(i*1024*1024), slice, time.Now()); st != 0 {\n\t\t\tt.Fatalf(\"Write %d should succeed (no space limit), got: %s\", i, st)\n\t\t}\n\t}\n\n\tvar newFileInode Ino\n\tif st := m.Create(ctx, parent, \"testfile_inodes2\", 0644, 0, 0, &newFileInode, &attr); st != syscall.EDQUOT {\n\t\tt.Fatalf(\"Create should fail with EDQUOT (inodes limit exceeded), got: %s\", st)\n\t}\n\n\tt.Log(\"Testing space-only quota limit...\")\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1024 * 1024, MaxInodes: 0}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set space-only quota: %s\", err)\n\t}\n\tm.getBase().loadQuotas()\n\n\tif st := m.Create(ctx, parent, \"testfile_space\", 0644, 0, 0, &fileInode, &attr); st != 0 {\n\t\tt.Fatalf(\"Create testfile_space: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, fileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID: %s\", st)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tvar newFileInode Ino\n\t\tif st := m.Create(ctx, parent, fmt.Sprintf(\"testfile_space_%d\", i), 0644, 0, 0, &newFileInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create file %d should succeed (no inodes limit), got: %s\", i, st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, newFileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for file %d: %s\", i, st)\n\t\t}\n\t}\n\n\tvar sliceId uint64\n\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\tt.Fatalf(\"NewSlice for space test: %s\", st)\n\t}\n\tslice := Slice{Id: sliceId, Size: 2 * 1024 * 1024, Len: 2 * 1024 * 1024}\n\tif st := m.Write(operatorCtx, fileInode, 0, 0, slice, time.Now()); st != syscall.EDQUOT {\n\t\tt.Fatalf(\"Write should fail with EDQUOT (space limit exceeded), got: %s\", st)\n\t}\n}\n\nfunc testCheckQuotaFileOwner(t *testing.T, m Meta) {\n\tif err := m.NewSession(true); err != nil {\n\t\tt.Fatalf(\"New session: %s\", err)\n\t}\n\tdefer m.CloseSession()\n\tctx := Background()\n\tvar parent Ino\n\tvar attr Attr\n\n\tif st := m.Mkdir(ctx, RootInode, \"checkquota\", 0755, 0, 0, &parent, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir checkquota: %s\", st)\n\t}\n\n\tfileOwnerUid := uint32(1001)\n\tfileOwnerGid := uint32(2001)\n\toperatorUid := uint32(1002)\n\toperatorGid := uint32(2002)\n\n\tt.Run(\"FileOwnerQuotaCheck\", func(t *testing.T) {\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1 << 20, MaxInodes: 5}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota set user quota for file owner uid %d: %s\", fileOwnerUid, err)\n\t\t}\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, fileOwnerGid, map[string]*Quota{fmt.Sprintf(\"gid:%d\", fileOwnerGid): {MaxSpace: 2 << 20, MaxInodes: 10}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota set group quota for file owner gid %d: %s\", fileOwnerGid, err)\n\t\t}\n\t\tm.getBase().loadQuotas()\n\n\t\tvar fileInode Ino\n\t\tif st := m.Create(ctx, parent, \"ownerfile\", 0644, 0, 0, &fileInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create ownerfile: %s\", st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, fileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for ownerfile: %s\", st)\n\t\t}\n\n\t\tvar checkAttr Attr\n\t\tif st := m.GetAttr(ctx, fileInode, &checkAttr); st != 0 {\n\t\t\tt.Fatalf(\"GetAttr for ownerfile: %s\", st)\n\t\t}\n\t\tif checkAttr.Uid != fileOwnerUid || checkAttr.Gid != fileOwnerGid {\n\t\t\tt.Fatalf(\"File owner not set correctly: expected uid=%d gid=%d, got uid=%d gid=%d\",\n\t\t\t\tfileOwnerUid, fileOwnerGid, checkAttr.Uid, checkAttr.Gid)\n\t\t}\n\n\t\tvar sliceId uint64\n\t\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice: %s\", st)\n\t\t}\n\t\ttestSlice := Slice{Id: sliceId, Size: 1024, Len: 1024}\n\n\t\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\t\tif st := m.Write(operatorCtx, fileInode, 0, 0, testSlice, time.Now()); st != 0 {\n\t\t\tt.Fatalf(\"Write to ownerfile by different user: %s\", st)\n\t\t}\n\n\t\tqs := make(map[string]*Quota)\n\t\tif err := m.HandleQuota(ctx, QuotaGet, \"\", fileOwnerUid, 0, qs, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota get user quota: %s\", err)\n\t\t}\n\t\tif q := qs[fmt.Sprintf(\"uid:%d\", fileOwnerUid)]; q.UsedSpace < 1024 {\n\t\t\tt.Fatalf(\"User quota used space should be >= 1024, got %d\", q.UsedSpace)\n\t\t}\n\n\t\tqs = make(map[string]*Quota)\n\t\tif err := m.HandleQuota(ctx, QuotaGet, \"\", 0, fileOwnerGid, qs, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota get group quota: %s\", err)\n\t\t}\n\t\tif q := qs[fmt.Sprintf(\"gid:%d\", fileOwnerGid)]; q.UsedSpace < 1024 {\n\t\t\tt.Fatalf(\"Group quota used space should be >= 1024, got %d\", q.UsedSpace)\n\t\t}\n\t})\n\n\tt.Run(\"QuotaExceededByFileOwner\", func(t *testing.T) {\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1024, MaxInodes: 2}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota set strict user quota: %s\", err)\n\t\t}\n\t\tm.getBase().loadQuotas()\n\n\t\tvar newFileInode Ino\n\t\tif st := m.Create(ctx, parent, \"strictfile\", 0644, 0, 0, &newFileInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create strictfile: %s\", st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, newFileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for strictfile: %s\", st)\n\t\t}\n\n\t\tvar smallSliceId uint64\n\t\tif st := m.NewSlice(ctx, &smallSliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice for small data: %s\", st)\n\t\t}\n\t\tsmallSlice := Slice{Id: smallSliceId, Size: 512, Len: 512}\n\t\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\t\tif st := m.Write(operatorCtx, newFileInode, 0, 0, smallSlice, time.Now()); st != 0 {\n\t\t\tt.Fatalf(\"Write small data: %s\", st)\n\t\t}\n\n\t\tvar largeSliceId uint64\n\t\tif st := m.NewSlice(ctx, &largeSliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice for large data: %s\", st)\n\t\t}\n\t\tlargeSlice := Slice{Id: largeSliceId, Size: 1024, Len: 1024}\n\t\tif st := m.Write(operatorCtx, newFileInode, 0, 512, largeSlice, time.Now()); st != syscall.EDQUOT {\n\t\t\tt.Fatalf(\"Write should fail with EDQUOT when exceeding file owner's quota, got: %s\", st)\n\t\t}\n\t})\n\n\tt.Run(\"TruncateQuotaCheck\", func(t *testing.T) {\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1 << 20, MaxInodes: 10}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota reset user quota: %s\", err)\n\t\t}\n\t\tm.getBase().loadQuotas()\n\n\t\tvar truncFileInode Ino\n\t\tif st := m.Create(ctx, parent, \"truncfile\", 0644, 0, 0, &truncFileInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create truncfile: %s\", st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, truncFileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for truncfile: %s\", st)\n\t\t}\n\n\t\tvar initialSliceId uint64\n\t\tif st := m.NewSlice(ctx, &initialSliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice for initial data: %s\", st)\n\t\t}\n\t\tinitialSlice := Slice{Id: initialSliceId, Size: 512, Len: 512}\n\t\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\t\tif st := m.Write(operatorCtx, truncFileInode, 0, 0, initialSlice, time.Now()); st != 0 {\n\t\t\tt.Fatalf(\"Initial write to truncfile: %s\", st)\n\t\t}\n\n\t\tfileOwnerCtx := &testContext{Context: context.Background(), uid: fileOwnerUid, gid: fileOwnerGid}\n\t\tif st := m.Truncate(fileOwnerCtx, truncFileInode, 0, 1024, &attr, false); st != 0 {\n\t\t\tt.Fatalf(\"Truncate truncfile by file owner: %s\", st)\n\t\t}\n\n\t\tif attr.Length != 1024 {\n\t\t\tt.Fatalf(\"Truncate failed: expected length 1024, got %d\", attr.Length)\n\t\t}\n\t})\n\n\tt.Run(\"MknodQuotaCheck\", func(t *testing.T) {\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1 << 20, MaxInodes: 10}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota reset user quota: %s\", err)\n\t\t}\n\t\tm.getBase().loadQuotas()\n\n\t\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\t\tvar deviceInode Ino\n\t\tif st := m.Mknod(operatorCtx, parent, \"device\", TypeFile, 0644, 0, 0, \"\", &deviceInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Mknod device by operator: %s\", st)\n\t\t}\n\n\t\tif attr.Uid != operatorUid || attr.Gid != operatorGid {\n\t\t\tt.Fatalf(\"Mknod file owner should be operator: expected uid=%d gid=%d, got uid=%d gid=%d\",\n\t\t\t\toperatorUid, operatorGid, attr.Uid, attr.Gid)\n\t\t}\n\n\t\tm.Unlink(ctx, parent, \"device\")\n\t})\n\n\tt.Run(\"CloneQuotaCheck\", func(t *testing.T) {\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1 << 20, MaxInodes: 10}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota reset user quota: %s\", err)\n\t\t}\n\t\tm.getBase().loadQuotas()\n\n\t\tvar srcInode Ino\n\t\tif st := m.Create(ctx, parent, \"srcfile\", 0644, 0, 0, &srcInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create srcfile: %s\", st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, srcInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for srcfile: %s\", st)\n\t\t}\n\n\t\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\t\tvar count, total uint64\n\t\tif st := m.Clone(operatorCtx, parent, srcInode, parent, \"clonefile\", 0, 0, 4, &count, &total); st != 0 {\n\t\t\tt.Fatalf(\"Clone srcfile by operator: %s\", st)\n\t\t}\n\n\t\tvar cloneInode Ino\n\t\tvar cloneAttr Attr\n\t\tif st := m.Lookup(ctx, parent, \"clonefile\", &cloneInode, &cloneAttr, false); st != 0 {\n\t\t\tt.Fatalf(\"Lookup clonefile: %s\", st)\n\t\t}\n\t\tif cloneAttr.Uid != operatorUid || cloneAttr.Gid != operatorGid {\n\t\t\tt.Fatalf(\"Clone file owner should be operator: expected uid=%d gid=%d, got uid=%d gid=%d\",\n\t\t\t\toperatorUid, operatorGid, cloneAttr.Uid, cloneAttr.Gid)\n\t\t}\n\n\t\tm.Unlink(ctx, parent, \"srcfile\")\n\t\tm.Unlink(ctx, parent, \"clonefile\")\n\t})\n\n\tt.Run(\"CrossUserOperations\", func(t *testing.T) {\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 1 << 20, MaxInodes: 10}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota set file owner quota: %s\", err)\n\t\t}\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", operatorUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", operatorUid): {MaxSpace: 512, MaxInodes: 2}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota set operator quota: %s\", err)\n\t\t}\n\t\tm.getBase().loadQuotas()\n\n\t\tvar crossFileInode Ino\n\t\tif st := m.Create(ctx, parent, \"crossfile\", 0644, 0, 0, &crossFileInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create crossfile: %s\", st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, crossFileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for crossfile: %s\", st)\n\t\t}\n\n\t\tvar crossSliceId uint64\n\t\tif st := m.NewSlice(ctx, &crossSliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice for cross data: %s\", st)\n\t\t}\n\t\tcrossSlice := Slice{Id: crossSliceId, Size: 1024, Len: 1024}\n\t\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\t\tif st := m.Write(operatorCtx, crossFileInode, 0, 0, crossSlice, time.Now()); st != 0 {\n\t\t\tt.Fatalf(\"Write to crossfile by operator: %s\", st)\n\t\t}\n\n\t\tqs := make(map[string]*Quota)\n\t\tif err := m.HandleQuota(ctx, QuotaGet, \"\", fileOwnerUid, 0, qs, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota get file owner quota: %s\", err)\n\t\t}\n\t\tif q := qs[fmt.Sprintf(\"uid:%d\", fileOwnerUid)]; q.UsedSpace < 1024 {\n\t\t\tt.Fatalf(\"File owner quota should be used: expected >= 1024, got %d\", q.UsedSpace)\n\t\t}\n\n\t\tqs = make(map[string]*Quota)\n\t\tif err := m.HandleQuota(ctx, QuotaGet, \"\", operatorUid, 0, qs, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota get operator quota: %s\", err)\n\t\t}\n\t\tif q := qs[fmt.Sprintf(\"uid:%d\", operatorUid)]; q.UsedSpace > 0 {\n\t\t\tt.Fatalf(\"Operator quota should not be used for file owner's file: got %d\", q.UsedSpace)\n\t\t}\n\n\t\tm.Unlink(ctx, parent, \"crossfile\")\n\t})\n\n\tt.Run(\"EdgeCases\", func(t *testing.T) {\n\t\tif err := m.HandleQuota(ctx, QuotaSet, \"\", fileOwnerUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", fileOwnerUid): {MaxSpace: 0, MaxInodes: 0}}, false, false, false); err != nil {\n\t\t\tt.Fatalf(\"HandleQuota set zero quota: %s\", err)\n\t\t}\n\t\tm.getBase().loadQuotas()\n\n\t\tvar edgeFileInode Ino\n\t\tif st := m.Create(ctx, parent, \"edgefile\", 0644, 0, 0, &edgeFileInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create edgefile: %s\", st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, edgeFileInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: fileOwnerUid, Gid: fileOwnerGid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for edgefile: %s\", st)\n\t\t}\n\n\t\tvar edgeSliceId uint64\n\t\tif st := m.NewSlice(ctx, &edgeSliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice for edge data: %s\", st)\n\t\t}\n\t\tedgeSlice := Slice{Id: edgeSliceId, Size: 1, Len: 1}\n\t\toperatorCtx := &testContext{Context: context.Background(), uid: operatorUid, gid: operatorGid}\n\t\tif st := m.Write(operatorCtx, edgeFileInode, 0, 0, edgeSlice, time.Now()); st != syscall.EDQUOT {\n\t\t\tt.Fatalf(\"Write should fail with EDQUOT when quota is zero, got: %s\", st)\n\t\t}\n\n\t\tm.Unlink(ctx, parent, \"edgefile\")\n\t})\n\n\tm.Unlink(ctx, parent, \"ownerfile\")\n\tm.Unlink(ctx, parent, \"strictfile\")\n\tm.Unlink(ctx, parent, \"truncfile\")\n}\n\n// testContext\ntype testContext struct {\n\tcontext.Context\n\tuid uint32\n\tgid uint32\n}\n\nfunc (c *testContext) Uid() uint32 { return c.uid }\nfunc (c *testContext) Gid() uint32 { return c.gid }\nfunc (c *testContext) Gids() []uint32 { return []uint32{c.gid} }\nfunc (c *testContext) Pid() uint32 { return 0 }\nfunc (c *testContext) WithValue(k, v interface{}) Context { return c }\nfunc (c *testContext) Cancel() {}\nfunc (c *testContext) Canceled() bool { return false }\nfunc (c *testContext) CheckPermission() bool { return true }\n\nfunc cleanupQuotaTest(ctx Context, m Meta, parent Ino, uid, gid uint32) {\n\tfor i := 0; i < 3; i++ {\n\t\tfilename := fmt.Sprintf(\"testfile%d\", i)\n\t\tm.Unlink(ctx, parent, filename)\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tfilename := fmt.Sprintf(\"writefile%d\", i)\n\t\tm.Unlink(ctx, parent, filename)\n\t}\n\tfor i := 0; i < 4; i++ {\n\t\tfilename := fmt.Sprintf(\"groupfile%d\", i)\n\t\tm.Unlink(ctx, parent, filename)\n\t}\n\n\tm.Unlink(ctx, parent, \"userfile\")\n\tm.Unlink(ctx, parent, \"groupfile\")\n\tm.Unlink(ctx, parent, \"hardlink\")\n\tm.Rmdir(ctx, RootInode, \"ugquota\")\n\n\tm.HandleQuota(ctx, QuotaDel, \"\", uid, 0, nil, false, false, false)\n\tm.HandleQuota(ctx, QuotaDel, \"\", 0, gid, nil, false, false, false)\n\tm.HandleQuota(ctx, QuotaDel, \"/path1\", uid, 0, nil, false, false, false)\n\tm.HandleQuota(ctx, QuotaDel, \"/path2\", 0, gid, nil, false, false, false)\n\tfor i := 0; i < 5; i++ {\n\t\ttestUid := uint32(3000 + i)\n\t\tm.HandleQuota(ctx, QuotaDel, \"\", testUid, 0, nil, false, false, false)\n\t}\n}\n\nfunc testBasicQuotaOperations(t *testing.T, m Meta, ctx Context, uid, gid uint32) {\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", uid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", uid): {MaxSpace: 1 << 30, MaxInodes: 10}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set user quota for uid %d: %s\", uid, err)\n\t}\n\tm.getBase().loadQuotas()\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, gid, map[string]*Quota{fmt.Sprintf(\"gid:%d\", gid): {MaxSpace: 2 << 30, MaxInodes: 20}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set group quota for gid %d: %s\", gid, err)\n\t}\n\tm.getBase().loadQuotas()\n\n\tqs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get user quota for uid %d: %s\", uid, err)\n\t} else if q := qs[fmt.Sprintf(\"uid:%d\", uid)]; q.MaxSpace != 1<<30 || q.MaxInodes != 10 {\n\t\tt.Fatalf(\"HandleQuota get user quota for uid %d: bad result %+v\", uid, q)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", 0, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get group quota for gid %d: %s\", gid, err)\n\t} else if q := qs[fmt.Sprintf(\"gid:%d\", gid)]; q.MaxSpace != 2<<30 || q.MaxInodes != 20 {\n\t\tt.Fatalf(\"HandleQuota get group quota for gid %d: bad result %+v\", gid, q)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaList, \"\", 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota list: %s\", err)\n\t} else {\n\t\tif len(qs) < 2 {\n\t\t\tt.Fatalf(\"HandleQuota list bad result: expected at least 2, got %d\", len(qs))\n\t\t}\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", uid, 0, nil, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota del user quota for uid %d: %s\", uid, err)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", 0, gid, nil, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota del group quota for gid %d: %s\", gid, err)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaList, \"\", 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota list after deletion: %s\", err)\n\t}\n\n\tm.getBase().loadQuotas()\n}\n\nfunc testQuotaFileOperations(t *testing.T, m Meta, ctx Context, parent Ino, uid, gid uint32) {\n\tvar userInode Ino\n\tvar attr Attr\n\tif st := m.Create(ctx, parent, \"userfile\", 0644, 0, 0, &userInode, &attr); st != 0 {\n\t\tt.Fatalf(\"Create ugquota/userfile: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, userInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID for userfile: %s\", st)\n\t}\n\n\tvar checkAttr Attr\n\tif st := m.GetAttr(ctx, userInode, &checkAttr); st != 0 {\n\t\tt.Fatalf(\"GetAttr for userfile: %s\", st)\n\t}\n\tif checkAttr.Uid != uid {\n\t\tt.Fatalf(\"SetAttr UID failed: expected %d, got %d\", uid, checkAttr.Uid)\n\t}\n\tif checkAttr.Gid != gid {\n\t\tt.Fatalf(\"SetAttr GID failed: expected %d, got %d\", gid, checkAttr.Gid)\n\t}\n\n\tvar groupInode Ino\n\tif st := m.Create(ctx, parent, \"groupfile\", 0644, 0, 0, &groupInode, &attr); st != 0 {\n\t\tt.Fatalf(\"Create ugquota/groupfile: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, groupInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID for groupfile: %s\", st)\n\t}\n\n\tm.FlushSession()\n\ttime.Sleep(time.Second * 2)\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", uid, 0, nil, false, false, false); err != nil {\n\t\tt.Logf(\"HandleQuota delete user quota (may not exist): %s\", err)\n\t}\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", uid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", uid): {MaxSpace: 1 << 30, MaxInodes: 10}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set user quota for uid %d: %s\", uid, err)\n\t}\n\n\tqs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get user quota after file creation: %s\", err)\n\t} else if q := qs[fmt.Sprintf(\"uid:%d\", uid)]; q.UsedInodes < 1 {\n\t\tt.Fatalf(\"HandleQuota get user quota: used inodes should be >= 1, got %d\", q.UsedInodes)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", 0, gid, nil, false, false, false); err != nil {\n\t\tt.Logf(\"HandleQuota delete group quota (may not exist): %s\", err)\n\t}\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, gid, map[string]*Quota{fmt.Sprintf(\"gid:%d\", gid): {MaxSpace: 2 << 30, MaxInodes: 20}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set group quota for gid %d: %s\", gid, err)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get user quota after file creation: %s\", err)\n\t} else if q := qs[fmt.Sprintf(\"uid:%d\", uid)]; q.UsedInodes < 1 {\n\t\tt.Fatalf(\"HandleQuota get user quota: used inodes should be >= 1, got %d\", q.UsedInodes)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", 0, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get group quota after file creation: %s\", err)\n\t} else if q := qs[fmt.Sprintf(\"gid:%d\", gid)]; q.UsedInodes < 1 {\n\t\tt.Fatalf(\"HandleQuota get group quota: used inodes should be >= 1, got %d\", q.UsedInodes)\n\t}\n\n\tm.getBase().doFlushQuotas()\n}\n\nfunc testQuotaErrorCases(t *testing.T, m Meta, ctx Context, uid, gid uint32) {\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, 0, map[string]*Quota{\"\": {MaxSpace: 1 << 30, MaxInodes: 10}}, false, false, false); err == nil {\n\t\tt.Fatalf(\"HandleQuota should fail for invalid quota type (no path, uid, or gid)\")\n\t}\n\n\tqs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, 99, \"\", uid, 0, qs, false, false, false); err == nil {\n\t\tt.Fatalf(\"HandleQuota should fail for invalid command\")\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", uid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", uid): {MaxSpace: 0, MaxInodes: 10}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set user quota with MaxSpace=0: %s\", err)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, gid, map[string]*Quota{fmt.Sprintf(\"gid:%d\", gid): {MaxSpace: 1 << 30, MaxInodes: 0}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set group quota with MaxInodes=0: %s\", err)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", uid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", uid): {MaxSpace: 1 << 62, MaxInodes: 1 << 30}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set user quota with large values: %s\", err)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", 9999, 0, nil, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota del non-existent user quota should not fail: %s\", err)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", 9999, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get non-existent user quota should not fail: %s\", err)\n\t}\n}\n\nfunc testQuotaConcurrentOperations(t *testing.T, m Meta, ctx Context) {\n\tvar wg sync.WaitGroup\n\n\tfor i := 0; i < 5; i++ {\n\t\twg.Add(1)\n\t\tgo func(id int) {\n\t\t\tdefer wg.Done()\n\t\t\ttestUid := uint32(3000 + id)\n\t\t\terr := m.HandleQuota(ctx, QuotaSet, \"\", testUid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", testUid): {MaxSpace: 1 << 20, MaxInodes: 5}}, false, false, false)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Concurrent HandleQuota set user quota for uid %d: %s\", testUid, err)\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\tfor i := 0; i < 5; i++ {\n\t\twg.Add(1)\n\t\tgo func(id int) {\n\t\t\tdefer wg.Done()\n\t\t\ttestUid := uint32(3000 + id)\n\t\t\tqs := make(map[string]*Quota)\n\t\t\terr := m.HandleQuota(ctx, QuotaGet, \"\", testUid, 0, qs, false, false, false)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Concurrent HandleQuota get user quota for uid %d: %s\", testUid, err)\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n}\n\nfunc testQuotaMixedTypes(t *testing.T, m Meta, ctx Context, uid, gid uint32) {\n\tvar attr Attr\n\n\tvar path1Inode Ino\n\tif st := m.Mkdir(ctx, RootInode, \"path1\", 0755, 0, 0, &path1Inode, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir path1: %s\", st)\n\t}\n\n\tvar path2Inode Ino\n\tif st := m.Mkdir(ctx, RootInode, \"path2\", 0755, 0, 0, &path2Inode, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir path2: %s\", st)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"/path1\", uid, 0, map[string]*Quota{\"/path1\": {MaxSpace: 100 << 20, MaxInodes: 20}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set path quota for uid %d: %s\", uid, err)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, \"/path2\", 0, gid, map[string]*Quota{\"/path2\": {MaxSpace: 200 << 20, MaxInodes: 30}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set path quota for gid %d: %s\", gid, err)\n\t}\n\n\tqs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaList, \"\", 0, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota list mixed quota types: %s\", err)\n\t}\n\tif len(qs) < 4 {\n\t\tt.Fatalf(\"HandleQuota list mixed quota types: expected at least 4, got %d\", len(qs))\n\t}\n}\n\nfunc testQuotaUsageStatistics(t *testing.T, m Meta, ctx Context, parent Ino, uid, gid uint32) {\n\tvar attr Attr\n\n\tfor i := 0; i < 3; i++ {\n\t\tfilename := fmt.Sprintf(\"testfile%d\", i)\n\t\tvar testInode Ino\n\t\tif st := m.Create(ctx, parent, filename, 0644, 0, 0, &testInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create %s: %s\", filename, st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, testInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for %s: %s\", filename, st)\n\t\t}\n\t}\n\n\tfor i := 0; i < 4; i++ {\n\t\tfilename := fmt.Sprintf(\"groupfile%d\", i)\n\t\tvar groupTestInode Ino\n\t\tif st := m.Create(ctx, parent, filename, 0644, 0, 0, &groupTestInode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create %s: %s\", filename, st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, groupTestInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for %s: %s\", filename, st)\n\t\t}\n\t}\n\n\t// Set parent directory attributes to be included in quotas\n\tif st := m.SetAttr(ctx, parent, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID for parent directory: %s\", st)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", uid, 0, nil, false, false, false); err != nil {\n\t\tt.Logf(\"HandleQuota delete user quota (may not exist): %s\", err)\n\t}\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", uid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", uid): {MaxSpace: 1 << 30, MaxInodes: 10}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set user quota for uid %d: %s\", uid, err)\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", 0, gid, nil, false, false, false); err != nil {\n\t\tt.Logf(\"HandleQuota delete group quota (may not exist): %s\", err)\n\t}\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, gid, map[string]*Quota{fmt.Sprintf(\"gid:%d\", gid): {MaxSpace: 2 << 30, MaxInodes: 20}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota set group quota for gid %d: %s\", gid, err)\n\t}\n\n\ttime.Sleep(time.Second * 2)\n\n\tqs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, 0, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get user quota for usage verification: %s\", err)\n\t} else if q := qs[fmt.Sprintf(\"uid:%d\", uid)]; q.UsedInodes < 4 {\n\t\tt.Fatalf(\"HandleQuota user quota usage: expected >= 4 inodes, got %d\", q.UsedInodes)\n\t}\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", 0, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"HandleQuota get group quota for usage verification: %s\", err)\n\t} else if q := qs[fmt.Sprintf(\"gid:%d\", gid)]; q.UsedInodes < 5 {\n\t\tt.Fatalf(\"HandleQuota group quota usage: expected >= 5 inodes, got %d\", q.UsedInodes)\n\t}\n}\n\nfunc testUserGroupQuota(t *testing.T, m Meta) {\n\tif err := m.NewSession(true); err != nil {\n\t\tt.Fatalf(\"New session: %s\", err)\n\t}\n\tdefer m.CloseSession()\n\tctx := Background()\n\tvar parent Ino\n\tvar attr Attr\n\n\tif st := m.Mkdir(ctx, RootInode, \"ugquota\", 0755, 0, 0, &parent, &attr); st != 0 {\n\t\tt.Fatalf(\"Mkdir ugquota: %s\", st)\n\t}\n\n\tuid := uint32(1001)\n\tgid := uint32(2001)\n\n\tt.Run(\"BasicQuotaOperations\", func(t *testing.T) {\n\t\ttestBasicQuotaOperations(t, m, ctx, uid, gid)\n\t})\n\n\tt.Run(\"QuotaFileOperations\", func(t *testing.T) {\n\t\ttestQuotaFileOperations(t, m, ctx, parent, uid, gid)\n\t})\n\n\tt.Run(\"QuotaErrorCases\", func(t *testing.T) {\n\t\ttestQuotaErrorCases(t, m, ctx, uid, gid)\n\t})\n\n\tt.Run(\"QuotaConcurrentOperations\", func(t *testing.T) {\n\t\ttestQuotaConcurrentOperations(t, m, ctx)\n\t})\n\n\tt.Run(\"QuotaMixedTypes\", func(t *testing.T) {\n\t\ttestQuotaMixedTypes(t, m, ctx, uid, gid)\n\t})\n\n\tt.Run(\"QuotaUsageStatistics\", func(t *testing.T) {\n\t\ttestQuotaUsageStatistics(t, m, ctx, parent, uid, gid)\n\t})\n\n\tt.Run(\"CheckQuotaFileOwner\", func(t *testing.T) {\n\t\ttestCheckQuotaFileOwnerSimple(t, m)\n\t})\n\n\tt.Run(\"QuotaEdgeCases\", func(t *testing.T) {\n\t\ttestQuotaEdgeCases(t, m)\n\t})\n\n\tt.Run(\"HardlinkQuota\", func(t *testing.T) {\n\t\ttestHardlinkQuota(t, m, ctx, parent, uid, gid)\n\t})\n\n\tt.Run(\"BatchUnlinkWithUserGroupQuota\", func(t *testing.T) {\n\t\ttestBatchUnlinkWithUserGroupQuota(t, m, ctx, parent, uid, gid)\n\t})\n\n\tcleanupQuotaTest(ctx, m, parent, uid, gid)\n\n}\n\nfunc testHardlinkQuota(t *testing.T, m Meta, ctx Context, parent Ino, uid, gid uint32) {\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", uid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", uid): {MaxSpace: 100 << 20, MaxInodes: 100}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"Set user quota: %s\", err)\n\t}\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, gid, map[string]*Quota{fmt.Sprintf(\"gid:%d\", gid): {MaxSpace: 100 << 20, MaxInodes: 100}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"Set group quota: %s\", err)\n\t}\n\n\tvar parentPath string\n\tif parent == RootInode {\n\t\tparentPath = \"/\"\n\t} else {\n\t\tparentPath = \"/ugquota\"\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaSet, parentPath, 0, 0, map[string]*Quota{parentPath: {MaxSpace: 200 << 20, MaxInodes: 200}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"Set directory quota for %s: %s\", parentPath, err)\n\t}\n\n\tm.getBase().loadQuotas()\n\n\tvar originalFile Ino\n\tvar attr Attr\n\tfileSize := uint64(8192) // 8KB 文件\n\tif st := m.Create(ctx, parent, \"test_original_file\", 0644, 0, 0, &originalFile, &attr); st != 0 {\n\t\tt.Fatalf(\"Create original file: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, originalFile, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID for original file: %s\", st)\n\t}\n\n\tvar sliceId uint64\n\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\tt.Fatalf(\"NewSlice: %s\", st)\n\t}\n\tslice := Slice{Id: sliceId, Size: uint32(fileSize), Len: uint32(fileSize)}\n\tif st := m.Write(ctx, originalFile, 0, 0, slice, time.Now()); st != 0 {\n\t\tt.Fatalf(\"Write data to original file: %s\", st)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(100 * time.Millisecond)\n\n\tqs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota after file creation: %s\", err)\n\t}\n\tugQuotaAfterFile := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaAfterFile == nil {\n\t\tt.Fatalf(\"User group quota not found after file creation\")\n\t}\n\n\tdirQs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, parentPath, 0, 0, dirQs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get directory quota after file creation: %s\", err)\n\t}\n\tdirQuotaAfterFile := dirQs[parentPath]\n\tif dirQuotaAfterFile == nil {\n\t\tt.Fatalf(\"Directory quota not found after file creation\")\n\t}\n\n\tif st := m.Link(ctx, originalFile, parent, \"test_hardlink_file\", &attr); st != 0 {\n\t\tt.Fatalf(\"Create hardlink: %s\", st)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(100 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota after hardlink creation: %s\", err)\n\t}\n\tugQuotaAfterHardlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaAfterHardlink == nil {\n\t\tt.Fatalf(\"User group quota not found after hardlink creation\")\n\t}\n\n\tdirQs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, parentPath, 0, 0, dirQs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get directory quota after hardlink creation: %s\", err)\n\t}\n\tdirQuotaAfterHardlink := dirQs[parentPath]\n\tif dirQuotaAfterHardlink == nil {\n\t\tt.Fatalf(\"Directory quota not found after hardlink creation\")\n\t}\n\t// After the new strategy, creating a hardlink does not increase user/group quota\n\t// because hardlink only creates a new directory entry, not a new file\n\texpectedSpaceIncrease := int64(0)\n\texpectedInodeIncrease := int64(0)\n\n\tactualSpaceIncrease := ugQuotaAfterHardlink.UsedSpace - ugQuotaAfterFile.UsedSpace\n\tactualInodeIncrease := ugQuotaAfterHardlink.UsedInodes - ugQuotaAfterFile.UsedInodes\n\n\tif actualSpaceIncrease != expectedSpaceIncrease {\n\t\tt.Fatalf(\"UG quota space increase mismatch: expected %d, got %d\", expectedSpaceIncrease, actualSpaceIncrease)\n\t}\n\tif actualInodeIncrease != expectedInodeIncrease {\n\t\tt.Fatalf(\"UG quota inode increase mismatch: expected %d, got %d\", expectedInodeIncrease, actualInodeIncrease)\n\t}\n\n\tdirExpectedSpaceIncrease := int64(8192)\n\tdirExpectedInodeIncrease := int64(1)\n\n\tdirActualSpaceIncrease := dirQuotaAfterHardlink.UsedSpace - dirQuotaAfterFile.UsedSpace\n\tdirActualInodeIncrease := dirQuotaAfterHardlink.UsedInodes - dirQuotaAfterFile.UsedInodes\n\n\tif dirActualSpaceIncrease != dirExpectedSpaceIncrease {\n\t\tt.Fatalf(\"Directory quota space increase mismatch: expected %d, got %d\", dirExpectedSpaceIncrease, dirActualSpaceIncrease)\n\t}\n\tif dirActualInodeIncrease != dirExpectedInodeIncrease {\n\t\tt.Fatalf(\"Directory quota inode increase mismatch: expected %d, got %d\", dirExpectedInodeIncrease, dirActualInodeIncrease)\n\t}\n\n\tif st := m.Unlink(ctx, parent, \"test_hardlink_file\", true); st != 0 {\n\t\tt.Fatalf(\"Unlink hardlink: %s\", st)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(100 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota after hardlink deletion: %s\", err)\n\t}\n\tugQuotaAfterUnlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaAfterUnlink == nil {\n\t\tt.Fatalf(\"User group quota not found after hardlink deletion\")\n\t}\n\n\tdirQs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, parentPath, 0, 0, dirQs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get directory quota after hardlink deletion: %s\", err)\n\t}\n\tdirQuotaAfterUnlink := dirQs[parentPath]\n\tif dirQuotaAfterUnlink == nil {\n\t\tt.Fatalf(\"Directory quota not found after hardlink deletion\")\n\t}\n\n\t// After the new strategy, deleting a hardlink does not decrease user/group quota\n\t// because hardlink only deletes a directory entry, not the actual file\n\texpectedSpaceDecrease := int64(0)\n\texpectedInodeDecrease := int64(0)\n\n\tactualSpaceDecrease := ugQuotaAfterHardlink.UsedSpace - ugQuotaAfterUnlink.UsedSpace\n\tactualInodeDecrease := ugQuotaAfterHardlink.UsedInodes - ugQuotaAfterUnlink.UsedInodes\n\n\tif actualSpaceDecrease != expectedSpaceDecrease {\n\t\tt.Fatalf(\"UG quota space decrease mismatch: expected %d, got %d\", expectedSpaceDecrease, actualSpaceDecrease)\n\t}\n\tif actualInodeDecrease != expectedInodeDecrease {\n\t\tt.Fatalf(\"UG quota inode decrease mismatch: expected %d, got %d\", expectedInodeDecrease, actualInodeDecrease)\n\t}\n\n\tdirExpectedSpaceDecrease := int64(8192)\n\tdirExpectedInodeDecrease := int64(1)\n\n\tdirActualSpaceDecrease := dirQuotaAfterHardlink.UsedSpace - dirQuotaAfterUnlink.UsedSpace\n\tdirActualInodeDecrease := dirQuotaAfterHardlink.UsedInodes - dirQuotaAfterUnlink.UsedInodes\n\n\tif dirActualSpaceDecrease != dirExpectedSpaceDecrease {\n\t\tt.Fatalf(\"Directory quota space decrease mismatch: expected %d, got %d\", dirExpectedSpaceDecrease, dirActualSpaceDecrease)\n\t}\n\tif dirActualInodeDecrease != dirExpectedInodeDecrease {\n\t\tt.Fatalf(\"Directory quota inode decrease mismatch: expected %d, got %d\", dirExpectedInodeDecrease, dirActualInodeDecrease)\n\t}\n\n\tm.Unlink(ctx, parent, \"test_original_file\")\n\tm.HandleQuota(ctx, QuotaDel, \"\", uid, gid, nil, false, false, false)\n\tm.HandleQuota(ctx, QuotaDel, parentPath, 0, 0, nil, false, false, false)\n}\n\nfunc testBatchUnlinkWithUserGroupQuota(t *testing.T, m Meta, ctx Context, parent Ino, uid, gid uint32) {\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", uid, 0, map[string]*Quota{fmt.Sprintf(\"uid:%d\", uid): {MaxSpace: 100 << 20, MaxInodes: 100}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"Set user quota: %s\", err)\n\t}\n\tif err := m.HandleQuota(ctx, QuotaSet, \"\", 0, gid, map[string]*Quota{fmt.Sprintf(\"gid:%d\", gid): {MaxSpace: 100 << 20, MaxInodes: 100}}, false, false, false); err != nil {\n\t\tt.Fatalf(\"Set group quota: %s\", err)\n\t}\n\tm.getBase().loadQuotas()\n\n\tvar fileInodes []Ino\n\tvar fileAttrs []Attr\n\tfileNames := []string{\"batch_file1\", \"batch_file2\", \"batch_file3\"}\n\tfileSize := uint64(4096) // 4KB per file\n\n\tfor _, fileName := range fileNames {\n\t\tvar inode Ino\n\t\tvar attr Attr\n\t\tif st := m.Create(ctx, parent, fileName, 0644, 0, 0, &inode, &attr); st != 0 {\n\t\t\tt.Fatalf(\"Create %s: %s\", fileName, st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, inode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for %s: %s\", fileName, st)\n\t\t}\n\t\tvar sliceId uint64\n\t\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\t\tt.Fatalf(\"NewSlice for %s: %s\", fileName, st)\n\t\t}\n\t\tslice := Slice{Id: sliceId, Size: uint32(fileSize), Len: uint32(fileSize)}\n\t\tif st := m.Write(ctx, inode, 0, 0, slice, time.Now()); st != 0 {\n\t\t\tt.Fatalf(\"Write data to %s: %s\", fileName, st)\n\t\t}\n\t\tfileInodes = append(fileInodes, inode)\n\t\tfileAttrs = append(fileAttrs, attr)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs := make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota before batch unlink: %s\", err)\n\t}\n\tugQuotaBefore := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaBefore == nil {\n\t\tt.Fatalf(\"User group quota not found before batch unlink\")\n\t}\n\n\tvar entries []*Entry\n\tfor i, fileName := range fileNames {\n\t\tvar attr Attr\n\t\tif st := m.GetAttr(ctx, fileInodes[i], &attr); st != 0 {\n\t\t\tt.Fatalf(\"GetAttr for %s: %s\", fileName, st)\n\t\t}\n\t\tentries = append(entries, &Entry{\n\t\t\tInode: fileInodes[i],\n\t\t\tName: []byte(fileName),\n\t\t\tAttr: &attr,\n\t\t})\n\t}\n\n\tvar count uint64\n\tif st := m.getBase().BatchUnlink(ctx, parent, entries, &count, false); st != 0 {\n\t\tt.Fatalf(\"BatchUnlink failed: %s\", st)\n\t}\n\n\tif count != uint64(len(fileNames)) {\n\t\tt.Fatalf(\"BatchUnlink count mismatch: expected %d, got %d\", len(fileNames), count)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota after batch unlink: %s\", err)\n\t}\n\tugQuotaAfter := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaAfter == nil {\n\t\tt.Fatalf(\"User group quota not found after batch unlink\")\n\t}\n\n\t// After the new strategy, files moved to trash do not decrease user/group quota\n\t// Only files permanently deleted from trash decrease quota\n\texpectedInodeDecrease := int64(0)\n\tactualInodeDecrease := ugQuotaBefore.UsedInodes - ugQuotaAfter.UsedInodes\n\n\tif actualInodeDecrease != expectedInodeDecrease {\n\t\tt.Fatalf(\"User group quota inode decrease mismatch: expected %d, got %d\", expectedInodeDecrease, actualInodeDecrease)\n\t}\n\n\texpectedSpaceDecrease := int64(0)\n\tactualSpaceDecrease := ugQuotaBefore.UsedSpace - ugQuotaAfter.UsedSpace\n\n\tif actualSpaceDecrease != expectedSpaceDecrease {\n\t\tt.Fatalf(\"User group quota space decrease mismatch: expected %d, got %d\", expectedSpaceDecrease, actualSpaceDecrease)\n\t}\n\n\tvar originalInode Ino\n\tvar originalAttr Attr\n\thardlinkFileSize := uint64(8192) // 8KB\n\thardlinkFileName := \"hardlink_original\"\n\tif st := m.Create(ctx, parent, hardlinkFileName, 0644, 0, 0, &originalInode, &originalAttr); st != 0 {\n\t\tt.Fatalf(\"Create original file for hardlink test: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, originalInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID for original file: %s\", st)\n\t}\n\tvar sliceId uint64\n\tif st := m.NewSlice(ctx, &sliceId); st != 0 {\n\t\tt.Fatalf(\"NewSlice for original file: %s\", st)\n\t}\n\tslice := Slice{Id: sliceId, Size: uint32(hardlinkFileSize), Len: uint32(hardlinkFileSize)}\n\tif st := m.Write(ctx, originalInode, 0, 0, slice, time.Now()); st != 0 {\n\t\tt.Fatalf(\"Write data to original file: %s\", st)\n\t}\n\n\thardlinkFileName2 := \"hardlink_link\"\n\tif st := m.Link(ctx, originalInode, parent, hardlinkFileName2, &originalAttr); st != 0 {\n\t\tt.Fatalf(\"Create hardlink: %s\", st)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota before hardlink unlink: %s\", err)\n\t}\n\tugQuotaBeforeHardlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaBeforeHardlink == nil {\n\t\tt.Fatalf(\"User group quota not found before hardlink unlink\")\n\t}\n\n\tvar hardlinkAttr Attr\n\tvar hardlinkInode Ino\n\tif st := m.Lookup(ctx, parent, hardlinkFileName2, &hardlinkInode, &hardlinkAttr, false); st != 0 {\n\t\tt.Fatalf(\"Lookup hardlink file: %s\", st)\n\t}\n\tif hardlinkInode != originalInode {\n\t\tt.Fatalf(\"Hardlink inode mismatch: expected %d, got %d\", originalInode, hardlinkInode)\n\t}\n\tif hardlinkAttr.Nlink < 2 {\n\t\tt.Fatalf(\"Expected Nlink >= 2 for hardlink, got %d\", hardlinkAttr.Nlink)\n\t}\n\n\tvar hardlinkEntry Attr\n\tif st := m.GetAttr(ctx, hardlinkInode, &hardlinkEntry); st != 0 {\n\t\tt.Fatalf(\"GetAttr for hardlink: %s\", st)\n\t}\n\thardlinkEntries := []*Entry{\n\t\t{\n\t\t\tInode: hardlinkInode,\n\t\t\tName: []byte(hardlinkFileName2),\n\t\t\tAttr: &hardlinkEntry,\n\t\t},\n\t}\n\n\tcount = 0\n\tif st := m.getBase().BatchUnlink(ctx, parent, hardlinkEntries, &count, false); st != 0 {\n\t\tt.Fatalf(\"BatchUnlink hardlink failed: %s\", st)\n\t}\n\n\tif count != 1 {\n\t\tt.Fatalf(\"BatchUnlink hardlink count mismatch: expected 1, got %d\", count)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota after hardlink unlink: %s\", err)\n\t}\n\tugQuotaAfterHardlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaAfterHardlink == nil {\n\t\tt.Fatalf(\"User group quota not found after hardlink unlink\")\n\t}\n\n\t// After the new strategy, hardlinks moved to trash do not decrease user/group quota\n\texpectedHardlinkInodeDecrease := int64(0)\n\texpectedHardlinkSpaceDecrease := int64(0)\n\n\tactualHardlinkInodeDecrease := ugQuotaBeforeHardlink.UsedInodes - ugQuotaAfterHardlink.UsedInodes\n\tactualHardlinkSpaceDecrease := ugQuotaBeforeHardlink.UsedSpace - ugQuotaAfterHardlink.UsedSpace\n\n\tif actualHardlinkInodeDecrease != expectedHardlinkInodeDecrease {\n\t\tt.Fatalf(\"Hardlink unlink: user group quota inode decrease mismatch: expected %d, got %d\", expectedHardlinkInodeDecrease, actualHardlinkInodeDecrease)\n\t}\n\tif actualHardlinkSpaceDecrease != expectedHardlinkSpaceDecrease {\n\t\tt.Fatalf(\"Hardlink unlink: user group quota space decrease mismatch: expected %d, got %d (should be 0 for hardlink deletion)\", expectedHardlinkSpaceDecrease, actualHardlinkSpaceDecrease)\n\t}\n\n\tvar checkAttr Attr\n\tif st := m.GetAttr(ctx, originalInode, &checkAttr); st != 0 {\n\t\tt.Fatalf(\"Original file should still exist after hardlink deletion: %s\", st)\n\t}\n\tif checkAttr.Nlink != hardlinkAttr.Nlink-1 {\n\t\tt.Fatalf(\"Original file Nlink should decrease by 1: expected %d, got %d\", hardlinkAttr.Nlink-1, checkAttr.Nlink)\n\t}\n\n\tm.Unlink(ctx, parent, hardlinkFileName)\n\n\t// Test: Batch unlink multiple hardlinks pointing to the same inode in one call\n\tvar multiHardlinkOriginal Ino\n\tvar multiHardlinkOriginalAttr Attr\n\tmultiHardlinkFileSize := uint64(12288) // 12KB\n\tmultiHardlinkOriginalName := \"multi_hardlink_original\"\n\tif st := m.Create(ctx, parent, multiHardlinkOriginalName, 0644, 0, 0, &multiHardlinkOriginal, &multiHardlinkOriginalAttr); st != 0 {\n\t\tt.Fatalf(\"Create original file for multi-hardlink test: %s\", st)\n\t}\n\tif st := m.SetAttr(ctx, multiHardlinkOriginal, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\tt.Fatalf(\"SetAttr UID and GID for multi-hardlink original file: %s\", st)\n\t}\n\tvar multiHardlinkSliceId uint64\n\tif st := m.NewSlice(ctx, &multiHardlinkSliceId); st != 0 {\n\t\tt.Fatalf(\"NewSlice for multi-hardlink original file: %s\", st)\n\t}\n\tmultiHardlinkSlice := Slice{Id: multiHardlinkSliceId, Size: uint32(multiHardlinkFileSize), Len: uint32(multiHardlinkFileSize)}\n\tif st := m.Write(ctx, multiHardlinkOriginal, 0, 0, multiHardlinkSlice, time.Now()); st != 0 {\n\t\tt.Fatalf(\"Write data to multi-hardlink original file: %s\", st)\n\t}\n\n\thardlinkNames := []string{\"multi_hardlink1\", \"multi_hardlink2\", \"multi_hardlink3\"}\n\tfor _, linkName := range hardlinkNames {\n\t\tif st := m.Link(ctx, multiHardlinkOriginal, parent, linkName, &multiHardlinkOriginalAttr); st != 0 {\n\t\t\tt.Fatalf(\"Create hardlink %s: %s\", linkName, st)\n\t\t}\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota before multi-hardlink batch unlink: %s\", err)\n\t}\n\tugQuotaBeforeMultiHardlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaBeforeMultiHardlink == nil {\n\t\tt.Fatalf(\"User group quota not found before multi-hardlink batch unlink\")\n\t}\n\n\tvar initialAttr Attr\n\tif st := m.GetAttr(ctx, multiHardlinkOriginal, &initialAttr); st != 0 {\n\t\tt.Fatalf(\"GetAttr for multi-hardlink original file: %s\", st)\n\t}\n\tinitialNlink := initialAttr.Nlink\n\texpectedFinalNlink := initialNlink - uint32(len(hardlinkNames))\n\tif initialNlink < uint32(len(hardlinkNames)+1) {\n\t\tt.Fatalf(\"Expected Nlink >= %d, got %d\", len(hardlinkNames)+1, initialNlink)\n\t}\n\n\tvar multiHardlinkEntries []*Entry\n\tfor _, linkName := range hardlinkNames {\n\t\tvar linkAttr Attr\n\t\tif st := m.GetAttr(ctx, multiHardlinkOriginal, &linkAttr); st != 0 {\n\t\t\tt.Fatalf(\"GetAttr for hardlink %s: %s\", linkName, st)\n\t\t}\n\t\tmultiHardlinkEntries = append(multiHardlinkEntries, &Entry{\n\t\t\tInode: multiHardlinkOriginal,\n\t\t\tName: []byte(linkName),\n\t\t\tAttr: &linkAttr,\n\t\t})\n\t}\n\n\tcount = 0\n\tif st := m.getBase().BatchUnlink(ctx, parent, multiHardlinkEntries, &count, false); st != 0 {\n\t\tt.Fatalf(\"BatchUnlink multiple hardlinks failed: %s\", st)\n\t}\n\n\tif count != uint64(len(hardlinkNames)) {\n\t\tt.Fatalf(\"BatchUnlink multiple hardlinks count mismatch: expected %d, got %d\", len(hardlinkNames), count)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota after multi-hardlink batch unlink: %s\", err)\n\t}\n\tugQuotaAfterMultiHardlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaAfterMultiHardlink == nil {\n\t\tt.Fatalf(\"User group quota not found after multi-hardlink batch unlink\")\n\t}\n\n\t// After the new strategy, hardlinks moved to trash do not decrease user/group quota\n\texpectedMultiHardlinkInodeDecrease := int64(0)\n\texpectedMultiHardlinkSpaceDecrease := int64(0)\n\n\tactualMultiHardlinkInodeDecrease := ugQuotaBeforeMultiHardlink.UsedInodes - ugQuotaAfterMultiHardlink.UsedInodes\n\tactualMultiHardlinkSpaceDecrease := ugQuotaBeforeMultiHardlink.UsedSpace - ugQuotaAfterMultiHardlink.UsedSpace\n\n\tif actualMultiHardlinkInodeDecrease != expectedMultiHardlinkInodeDecrease {\n\t\tt.Fatalf(\"Multi-hardlink batch unlink: user group quota inode decrease mismatch: expected %d, got %d\", expectedMultiHardlinkInodeDecrease, actualMultiHardlinkInodeDecrease)\n\t}\n\tif actualMultiHardlinkSpaceDecrease != expectedMultiHardlinkSpaceDecrease {\n\t\tt.Fatalf(\"Multi-hardlink batch unlink: user group quota space decrease mismatch: expected %d, got %d (should be 0 for hardlink deletion)\", expectedMultiHardlinkSpaceDecrease, actualMultiHardlinkSpaceDecrease)\n\t}\n\n\tvar finalAttr Attr\n\tif st := m.GetAttr(ctx, multiHardlinkOriginal, &finalAttr); st != 0 {\n\t\tt.Fatalf(\"Original file should still exist after multi-hardlink deletion: %s\", st)\n\t}\n\tif finalAttr.Nlink != expectedFinalNlink {\n\t\tt.Fatalf(\"Original file Nlink mismatch: expected %d, got %d (initial was %d, deleted %d links)\", expectedFinalNlink, finalAttr.Nlink, initialNlink, len(hardlinkNames))\n\t}\n\n\tfor _, linkName := range hardlinkNames {\n\t\tvar lookupInode Ino\n\t\tvar lookupAttr Attr\n\t\tif st := m.Lookup(ctx, parent, linkName, &lookupInode, &lookupAttr, false); st == 0 {\n\t\t\tt.Fatalf(\"Hardlink %s should have been deleted, but still exists\", linkName)\n\t\t}\n\t}\n\n\tvar originalLookupInode Ino\n\tvar originalLookupAttr Attr\n\tif st := m.Lookup(ctx, parent, multiHardlinkOriginalName, &originalLookupInode, &originalLookupAttr, false); st != 0 {\n\t\tt.Fatalf(\"Original file %s should still exist: %s\", multiHardlinkOriginalName, st)\n\t}\n\tif originalLookupInode != multiHardlinkOriginal {\n\t\tt.Fatalf(\"Original file inode mismatch: expected %d, got %d\", multiHardlinkOriginal, originalLookupInode)\n\t}\n\n\tm.Unlink(ctx, parent, multiHardlinkOriginalName)\n\n\t// Test: Batch unlink symlinks\n\tsymlinkNames := []string{\"symlink1\", \"symlink2\", \"symlink3\"}\n\tvar symlinkInodes []Ino\n\tvar symlinkAttrs []Attr\n\tfor _, symlinkName := range symlinkNames {\n\t\tvar symlinkInode Ino\n\t\tvar symlinkAttr Attr\n\t\ttarget := \"/target/\" + symlinkName\n\t\tif st := m.Symlink(ctx, parent, symlinkName, target, &symlinkInode, &symlinkAttr); st != 0 {\n\t\t\tt.Fatalf(\"Create symlink %s: %s\", symlinkName, st)\n\t\t}\n\t\tif st := m.SetAttr(ctx, symlinkInode, SetAttrUID|SetAttrGID, 0, &Attr{Uid: uid, Gid: gid}); st != 0 {\n\t\t\tt.Fatalf(\"SetAttr UID and GID for symlink %s: %s\", symlinkName, st)\n\t\t}\n\t\tsymlinkInodes = append(symlinkInodes, symlinkInode)\n\t\tsymlinkAttrs = append(symlinkAttrs, symlinkAttr)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota before symlink batch unlink: %s\", err)\n\t}\n\tugQuotaBeforeSymlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaBeforeSymlink == nil {\n\t\tt.Fatalf(\"User group quota not found before symlink batch unlink\")\n\t}\n\n\tvar symlinkEntries []*Entry\n\tfor i, symlinkName := range symlinkNames {\n\t\tvar symlinkAttr Attr\n\t\tif st := m.GetAttr(ctx, symlinkInodes[i], &symlinkAttr); st != 0 {\n\t\t\tt.Fatalf(\"GetAttr for symlink %s: %s\", symlinkName, st)\n\t\t}\n\t\tsymlinkEntries = append(symlinkEntries, &Entry{\n\t\t\tInode: symlinkInodes[i],\n\t\t\tName: []byte(symlinkName),\n\t\t\tAttr: &symlinkAttr,\n\t\t})\n\t}\n\n\tcount = 0\n\tif st := m.getBase().BatchUnlink(ctx, parent, symlinkEntries, &count, false); st != 0 {\n\t\tt.Fatalf(\"BatchUnlink symlinks failed: %s\", st)\n\t}\n\n\tif count != uint64(len(symlinkNames)) {\n\t\tt.Fatalf(\"BatchUnlink symlinks count mismatch: expected %d, got %d\", len(symlinkNames), count)\n\t}\n\n\tm.getBase().doFlushQuotas()\n\ttime.Sleep(200 * time.Millisecond)\n\n\tqs = make(map[string]*Quota)\n\tif err := m.HandleQuota(ctx, QuotaGet, \"\", uid, gid, qs, false, false, false); err != nil {\n\t\tt.Fatalf(\"Get user group quota after symlink batch unlink: %s\", err)\n\t}\n\tugQuotaAfterSymlink := qs[fmt.Sprintf(\"uid:%d\", uid)]\n\tif ugQuotaAfterSymlink == nil {\n\t\tt.Fatalf(\"User group quota not found after symlink batch unlink\")\n\t}\n\n\texpectedSymlinkInodeDecrease := int64(3)\n\texpectedSymlinkSpaceDecrease := 3 * align4K(0)\n\n\tactualSymlinkInodeDecrease := ugQuotaBeforeSymlink.UsedInodes - ugQuotaAfterSymlink.UsedInodes\n\tactualSymlinkSpaceDecrease := ugQuotaBeforeSymlink.UsedSpace - ugQuotaAfterSymlink.UsedSpace\n\n\tif actualSymlinkInodeDecrease != expectedSymlinkInodeDecrease {\n\t\tt.Fatalf(\"Symlink batch unlink: user group quota inode decrease mismatch: expected %d, got %d\", expectedSymlinkInodeDecrease, actualSymlinkInodeDecrease)\n\t}\n\tif actualSymlinkSpaceDecrease != expectedSymlinkSpaceDecrease {\n\t\tt.Fatalf(\"Symlink batch unlink: user group quota space decrease mismatch: expected %d, got %d (should be %d for symlink deletion)\", expectedSymlinkSpaceDecrease, actualSymlinkSpaceDecrease, expectedSymlinkSpaceDecrease)\n\t}\n\n\tfor _, symlinkName := range symlinkNames {\n\t\tvar lookupInode Ino\n\t\tvar lookupAttr Attr\n\t\tif st := m.Lookup(ctx, parent, symlinkName, &lookupInode, &lookupAttr, false); st == 0 {\n\t\t\tt.Fatalf(\"Symlink %s should have been deleted, but still exists\", symlinkName)\n\t\t}\n\t}\n\n\tif err := m.HandleQuota(ctx, QuotaDel, \"\", uid, gid, nil, false, false, false); err != nil {\n\t\tt.Fatalf(\"Delete user group quota: %s\", err)\n\t}\n}\n" }, { "path": "pkg/meta/benchmarks_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"fmt\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/sirupsen/logrus\"\n)\n\nconst (\n\tredisAddr = \"redis://127.0.0.1/1\"\n\tsqlAddr = \"sqlite3://juicefs.db\"\n\t// sqlAddr = \"mysql://root:@/juicefs\" // MySQL\n\t// sqlAddr = \"mysql://root:@tcp(127.0.0.1:4000)/juicefs\" // TiDB\n\ttkvAddr = \"badger://test_db\"\n\t// tkvAddr = \"tikv://127.0.0.1:2379/juicefs\"\n)\n\nfunc init() {\n\tutils.SetLogLevel(logrus.InfoLevel)\n\t// utils.SetOutFile(\"bench-test.log\")\n}\n\nfunc encodeSlices(size int) []string {\n\tw := utils.NewBuffer(24)\n\tw.Put32(0)\n\tw.Put64(1014)\n\tw.Put32(122)\n\tw.Put32(0)\n\tw.Put32(122)\n\tv := string(w.Bytes())\n\tvals := make([]string, size)\n\tfor i := range vals {\n\t\tvals[i] = v\n\t}\n\treturn vals\n}\n\nfunc encodeSlicesAsBuf(nSlices uint32) []byte {\n\tw := utils.NewBuffer(nSlices * sliceBytes)\n\tfor i := uint32(0); i < nSlices; i++ {\n\t\tw.Put32(0)\n\t\tw.Put64(1014)\n\t\tw.Put32(122)\n\t\tw.Put32(0)\n\t\tw.Put32(122)\n\t}\n\treturn w.Bytes()\n}\n\nfunc BenchmarkReadSlices(b *testing.B) {\n\tcases := []struct {\n\t\tdesc string\n\t\tsize int\n\t}{\n\t\t{\"small\", 4},\n\t\t{\"mid\", 64},\n\t\t{\"large\", 1024},\n\t}\n\tfor _, c := range cases {\n\t\tb.Run(c.desc, func(b *testing.B) {\n\t\t\tvals := encodeSlices(c.size)\n\t\t\tb.ResetTimer()\n\t\t\tvar slices []*slice\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tslices = readSlices(vals)\n\t\t\t}\n\t\t\tif len(slices) != len(vals) {\n\t\t\t\tb.Fail()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc BenchmarkReadSliceBuf(b *testing.B) {\n\tcases := []struct {\n\t\tdesc string\n\t\tsize uint32\n\t}{\n\t\t{\"small\", 4},\n\t\t{\"mid\", 64},\n\t\t{\"large\", 1024},\n\t}\n\tfor _, c := range cases {\n\t\tb.Run(c.desc, func(b *testing.B) {\n\t\t\tbuf := encodeSlicesAsBuf(c.size)\n\t\t\tb.ResetTimer()\n\t\t\tvar slices []*slice\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tslices = readSliceBuf(buf)\n\t\t\t}\n\t\t\tif len(slices) != int(c.size) {\n\t\t\t\tb.Fail()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc prepareParent(m Meta, name string, inode *Ino) error {\n\tctx := Background()\n\tif err := m.Remove(ctx, 1, name, true, RmrDefaultThreads, nil); err != 0 && err != syscall.ENOENT {\n\t\treturn fmt.Errorf(\"remove: %s\", err)\n\t}\n\tif err := m.Mkdir(ctx, 1, name, 0755, 0, 0, inode, nil); err != 0 {\n\t\treturn fmt.Errorf(\"mkdir: %s\", err)\n\t}\n\treturn nil\n}\n\nfunc benchMkdir(b *testing.B, m Meta) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchMkdir\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Mkdir(ctx, parent, fmt.Sprintf(\"d%d\", i), 0755, 0, 0, &inode, nil); err != 0 {\n\t\t\tb.Fatalf(\"mkdir: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchMvdir(b *testing.B, m Meta) { // rename dir\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchMvdir\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Mkdir(ctx, parent, \"d0\", 0755, 0, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"mkdir: %s\", err)\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Rename(ctx, parent, fmt.Sprintf(\"d%d\", i), parent, fmt.Sprintf(\"d%d\", i+1), 0, nil, nil); err != 0 {\n\t\t\tb.Fatalf(\"rename dir: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchRmdir(b *testing.B, m Meta) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchRmdir\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\tif err := m.Mkdir(ctx, parent, \"dir\", 0755, 0, 0, &inode, nil); err != 0 {\n\t\t\tb.Fatalf(\"mkdir: %s\", err)\n\t\t}\n\t\tb.StartTimer()\n\t\tif err := m.Rmdir(ctx, parent, \"dir\"); err != 0 {\n\t\t\tb.Fatalf(\"rmdir: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchResolve(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchResolve\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tvar child Ino = parent\n\tfor i := 0; i < 5; i++ {\n\t\tif err := m.Mkdir(ctx, child, \"d\", 0755, 0, 0, &child, nil); err != 0 {\n\t\t\tb.Fatalf(\"mkdir: %s\", err)\n\t\t}\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Resolve(ctx, parent, \"d/d/d/d/d\", nil, nil); err != 0 {\n\t\t\tif err == syscall.ENOTSUP {\n\t\t\t\tb.SkipNow()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tb.Fatalf(\"resolve: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchReaddir(b *testing.B, m Meta, n int) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchReaddir\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tfor j := 0; j < n; j++ {\n\t\tif err := m.Create(ctx, parent, fmt.Sprintf(\"f%d\", j), 0644, 022, 0, nil, nil); err != 0 {\n\t\t\tb.Fatalf(\"create: %s\", err)\n\t\t}\n\t}\n\tvar entries []*Entry\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tentries = entries[:0]\n\t\tif err := m.Readdir(ctx, parent, 1, &entries); err != 0 {\n\t\t\tb.Fatalf(\"readdir: %s\", err)\n\t\t}\n\t\tif len(entries) != n+2 {\n\t\t\tb.Fatalf(\"files: %d != %d\", len(entries), n+2)\n\t\t}\n\t}\n}\n\nfunc benchMknod(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchMknod\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Mknod(ctx, parent, fmt.Sprintf(\"f%d\", i), TypeFile, 0644, 022, 0, \"\", nil, nil); err != 0 {\n\t\t\tb.Fatalf(\"mknod: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchCreate(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchCreate\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Create(ctx, parent, fmt.Sprintf(\"f%d\", i), 0644, 022, 0, nil, nil); err != 0 {\n\t\t\tb.Fatalf(\"create: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchRename(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchRename\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"f0\", 0644, 022, 0, nil, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Rename(ctx, parent, fmt.Sprintf(\"f%d\", i), parent, fmt.Sprintf(\"f%d\", i+1), 0, nil, nil); err != 0 {\n\t\t\tb.Fatalf(\"rename file: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchUnlink(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchUnlink\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\tif err := m.Create(ctx, parent, \"file\", 0644, 022, 0, nil, nil); err != 0 {\n\t\t\tb.Fatalf(\"create: %s\", err)\n\t\t}\n\t\tb.StartTimer()\n\t\tif err := m.Unlink(ctx, parent, \"file\"); err != 0 {\n\t\t\tb.Fatalf(\"unlink: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchLookup(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchLookup\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"file\", 0644, 022, 0, nil, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tvar inode Ino\n\tvar attr Attr\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Lookup(ctx, parent, \"file\", &inode, &attr, false); err != 0 {\n\t\t\tb.Fatalf(\"lookup: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchGetAttr(b *testing.B, m Meta) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchGetAttr\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"file\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tvar attr Attr\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.GetAttr(ctx, inode, &attr); err != 0 {\n\t\t\tb.Fatalf(\"getattr: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchSetAttr(b *testing.B, m Meta) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchSetAttr\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"file\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tvar attr = Attr{Mode: 0644}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tattr.Mode ^= 1\n\t\tif err := m.SetAttr(ctx, inode, SetAttrMode, 0, &attr); err != 0 {\n\t\t\tb.Fatalf(\"setattr: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchAccess(b *testing.B, m Meta) { // contains a Getattr\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchAccess\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"file\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tmyCtx := NewContext(100, 1, []uint32{1})\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Access(myCtx, inode, 4, nil); err != 0 && err != syscall.EACCES {\n\t\t\tb.Fatalf(\"access: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchSetXattr(b *testing.B, m Meta) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchSetXattr\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"fxattr\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tb.ResetTimer()\n\tvalue := []byte(\"value0\")\n\tfor i := 0; i < b.N; i++ {\n\t\tvalue[5] = byte(i%10 + 48)\n\t\tif err := m.SetXattr(ctx, inode, \"key\", value, 0); err != 0 {\n\t\t\tb.Fatalf(\"setxattr: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchGetXattr(b *testing.B, m Meta) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchGetXattr\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"fxattr\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tif err := m.SetXattr(ctx, inode, \"key\", []byte(\"value\"), 0); err != 0 {\n\t\tb.Fatalf(\"setxattr: %s\", err)\n\t}\n\tvar buf []byte\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.GetXattr(ctx, inode, \"key\", &buf); err != 0 {\n\t\t\tb.Fatalf(\"getxattr: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchRemoveXattr(b *testing.B, m Meta) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchRemoveXattr\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"fxattr\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\tif err := m.SetXattr(ctx, inode, \"key\", []byte(\"value\"), 0); err != 0 {\n\t\t\tb.Fatalf(\"setxattr: %s\", err)\n\t\t}\n\t\tb.StartTimer()\n\t\tif err := m.RemoveXattr(ctx, inode, \"key\"); err != 0 {\n\t\t\tb.Fatalf(\"removexattr: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchListXattr(b *testing.B, m Meta, n int) {\n\tvar parent, inode Ino\n\tif err := prepareParent(m, \"benchListXattr\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tif err := m.Create(ctx, parent, \"fxattr\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tfor j := 0; j < n; j++ {\n\t\tif err := m.SetXattr(ctx, inode, fmt.Sprintf(\"key%d\", j), []byte(\"value\"), 0); err != 0 {\n\t\t\tb.Fatalf(\"setxattr: %s\", err)\n\t\t}\n\t}\n\tvar buf []byte\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.ListXattr(ctx, inode, &buf); err != 0 {\n\t\t\tb.Fatalf(\"removexattr: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchLink(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchLink\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tvar inode Ino\n\tif err := m.Create(ctx, parent, \"source\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Link(ctx, inode, parent, fmt.Sprintf(\"l%d\", i), nil); err != 0 {\n\t\t\tb.Fatalf(\"link: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchSymlink(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchSymlink\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tvar inode Ino\n\tif err := m.Create(ctx, parent, \"source\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Symlink(ctx, parent, fmt.Sprintf(\"s%d\", i), \"/benchSymlink/source\", nil, nil); err != 0 {\n\t\t\tb.Fatalf(\"symlink: %s\", err)\n\t\t}\n\t}\n}\n\n/*\nfunc benchReadlink(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchReadlink\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tvar inode Ino\n\tif err := m.Create(ctx, parent, \"source\", 0644, 022, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tif err := m.Symlink(ctx, parent, \"slink\", \"/benchReadlink/source\", &inode, nil); err != 0 {\n\t\tb.Fatalf(\"symlink: %s\", err)\n\t}\n\tvar buf []byte\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.ReadLink(ctx, inode, &buf); err != 0 {\n\t\t\tb.Fatalf(\"readlink: %s\", err)\n\t\t}\n\t}\n}\n*/\n\nfunc benchNewChunk(b *testing.B, m Meta) {\n\tctx := Background()\n\tvar sliceId uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.NewSlice(ctx, &sliceId); err != 0 {\n\t\t\tb.Fatalf(\"newchunk: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchWrite(b *testing.B, m Meta) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchWrite\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tvar inode Ino\n\tif err := m.Create(ctx, parent, \"file\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tvar (\n\t\tsliceId uint64\n\t\toffset uint32\n\t\tstep uint32 = 1024\n\t)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.NewSlice(ctx, &sliceId); err != 0 {\n\t\t\tb.Fatalf(\"newchunk: %s\", err)\n\t\t}\n\t\tif err := m.Write(ctx, inode, 0, offset, Slice{Id: sliceId, Size: step, Len: step}, time.Now()); err != 0 {\n\t\t\tb.Fatalf(\"write: %s\", err)\n\t\t}\n\t\toffset += step\n\t\tif offset+step > ChunkSize {\n\t\t\toffset = 0\n\t\t}\n\t}\n}\n\nfunc benchRead(b *testing.B, m Meta, n int) {\n\tvar parent Ino\n\tif err := prepareParent(m, \"benchRead\", &parent); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tctx := Background()\n\tvar inode Ino\n\tif err := m.Create(ctx, parent, \"file\", 0644, 022, 0, &inode, nil); err != 0 {\n\t\tb.Fatalf(\"create: %s\", err)\n\t}\n\tvar sliceId uint64\n\tvar step uint32 = 1024\n\tfor j := 0; j < n; j++ {\n\t\tif err := m.NewSlice(ctx, &sliceId); err != 0 {\n\t\t\tb.Fatalf(\"newchunk: %s\", err)\n\t\t}\n\t\tif err := m.Write(ctx, inode, 0, uint32(j)*step, Slice{Id: sliceId, Size: step, Len: step}, time.Now()); err != 0 {\n\t\t\tb.Fatalf(\"write: %s\", err)\n\t\t}\n\t}\n\tvar slices []Slice\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := m.Read(ctx, inode, 0, &slices); err != 0 {\n\t\t\tb.Fatalf(\"read: %s\", err)\n\t\t}\n\t}\n}\n\nfunc benchmarkDir(b *testing.B, m Meta) { // mkdir, rename dir, rmdir, readdir\n\tb.Run(\"mkdir\", func(b *testing.B) { benchMkdir(b, m) })\n\tb.Run(\"mvdir\", func(b *testing.B) { benchMvdir(b, m) })\n\tb.Run(\"rmdir\", func(b *testing.B) { benchRmdir(b, m) })\n\tb.Run(\"resolve\", func(b *testing.B) { benchResolve(b, m) })\n\tb.Run(\"readdir_10\", func(b *testing.B) { benchReaddir(b, m, 10) })\n\tb.Run(\"readdir_1k\", func(b *testing.B) { benchReaddir(b, m, 1000) })\n\t// b.Run(\"readdir_100k\", func(b *testing.B) { benchReaddir(b, m, 100000) })\n}\n\nfunc benchmarkFile(b *testing.B, m Meta) {\n\tb.Run(\"mknod\", func(b *testing.B) { benchMknod(b, m) })\n\tb.Run(\"create\", func(b *testing.B) { benchCreate(b, m) })\n\tb.Run(\"rename\", func(b *testing.B) { benchRename(b, m) })\n\tb.Run(\"unlink\", func(b *testing.B) { benchUnlink(b, m) })\n\tb.Run(\"lookup\", func(b *testing.B) { benchLookup(b, m) })\n\tb.Run(\"getattr\", func(b *testing.B) { benchGetAttr(b, m) })\n\tb.Run(\"setattr\", func(b *testing.B) { benchSetAttr(b, m) })\n\tb.Run(\"access\", func(b *testing.B) { benchAccess(b, m) })\n}\n\nfunc benchmarkXattr(b *testing.B, m Meta) {\n\tb.Run(\"setxattr\", func(b *testing.B) { benchSetXattr(b, m) })\n\tb.Run(\"getxattr\", func(b *testing.B) { benchGetXattr(b, m) })\n\tb.Run(\"removexattr\", func(b *testing.B) { benchRemoveXattr(b, m) })\n\tb.Run(\"listxattr_1\", func(b *testing.B) { benchListXattr(b, m, 1) })\n\tb.Run(\"listxattr_10\", func(b *testing.B) { benchListXattr(b, m, 10) })\n}\n\nfunc benchmarkLink(b *testing.B, m Meta) {\n\tb.Run(\"link\", func(b *testing.B) { benchLink(b, m) })\n\tb.Run(\"symlink\", func(b *testing.B) { benchSymlink(b, m) })\n\t// maybe meaningless since symlink would be cached\n\t// b.Run(\"readlink\", func(b *testing.B) { benchReadlink(b, m) })\n}\n\nfunc benchmarkData(b *testing.B, m Meta) {\n\tm.OnMsg(DeleteSlice, func(args ...interface{}) error { return nil })\n\tm.OnMsg(CompactChunk, func(args ...interface{}) error { return nil })\n\tb.Run(\"newchunk\", func(b *testing.B) { benchNewChunk(b, m) })\n\tb.Run(\"write\", func(b *testing.B) { benchWrite(b, m) })\n\tb.Run(\"read_1\", func(b *testing.B) { benchRead(b, m, 1) })\n\tb.Run(\"read_10\", func(b *testing.B) { benchRead(b, m, 10) })\n}\n\nfunc benchmarkAll(b *testing.B, m Meta) {\n\t_ = m.Init(&Format{Name: \"benchmarkAll\", DirStats: true}, true)\n\t_ = m.NewSession(false)\n\tbenchmarkDir(b, m)\n\tbenchmarkFile(b, m)\n\tbenchmarkXattr(b, m)\n\tbenchmarkLink(b, m)\n\tbenchmarkData(b, m)\n}\n\nfunc BenchmarkRedis(b *testing.B) {\n\tm := NewClient(redisAddr, nil)\n\tbenchmarkAll(b, m)\n}\n\nfunc BenchmarkSQL(b *testing.B) {\n\tm := NewClient(sqlAddr, nil)\n\tbenchmarkAll(b, m)\n}\n\nfunc BenchmarkTKV(b *testing.B) {\n\tm := NewClient(tkvAddr, nil)\n\tbenchmarkAll(b, m)\n}\n" }, { "path": "pkg/meta/config.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"crypto/md5\"\n\t\"crypto/rand\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"time\"\n\n\t\"github.com/emmansun/gmsm/sm3\"\n\t\"github.com/emmansun/gmsm/sm4\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/pkg/errors\"\n)\n\n// Config for clients.\ntype Config struct {\n\tRetries int\n\tMaxDeletes int\n\tSkipDirNlink int\n\tCaseInsensi bool\n\tReadOnly bool\n\tNoBGJob bool // disable background jobs like clean-up, backup, etc.\n\tOpenCache time.Duration\n\tOpenCacheLimit uint64 // max number of files to cache (soft limit)\n\tHeartbeat time.Duration\n\tMountPoint string\n\tSubdir string\n\tAtimeMode string\n\tDirStatFlushPeriod time.Duration\n\tSkipDirMtime time.Duration\n\tSid uint64\n\tSortDir bool\n\tFastStatfs bool\n\tNetworkInterfaces []string // list of network interfaces to use for IP discovery (empty means all)\n}\n\nfunc DefaultConf() *Config {\n\treturn &Config{Retries: 10, MaxDeletes: 2, Heartbeat: 12 * time.Second, AtimeMode: NoAtime, DirStatFlushPeriod: 1 * time.Second}\n}\n\nfunc (c *Config) SelfCheck() {\n\tif c.MaxDeletes == 0 {\n\t\tlogger.Warnf(\"Deleting object will be disabled since max-deletes is 0\")\n\t}\n\tif c.Heartbeat != 0 && c.Heartbeat < time.Second {\n\t\tlogger.Warnf(\"heartbeat should not be less than 1 second\")\n\t\tc.Heartbeat = time.Second\n\t}\n\tif c.Heartbeat > time.Minute*10 {\n\t\tlogger.Warnf(\"heartbeat should not be greater than 10 minutes\")\n\t\tc.Heartbeat = time.Minute * 10\n\t}\n}\n\ntype Format struct {\n\tName string\n\tUUID string\n\tStorage string\n\tStorageClass string `json:\",omitempty\"`\n\tBucket string\n\tAccessKey string `json:\",omitempty\"`\n\tSecretKey string `json:\",omitempty\"`\n\tSessionToken string `json:\",omitempty\"`\n\tBlockSize int\n\tCompression string `json:\",omitempty\"`\n\tShards int `json:\",omitempty\"`\n\tHashPrefix bool `json:\",omitempty\"`\n\tCapacity uint64 `json:\",omitempty\"`\n\tInodes uint64 `json:\",omitempty\"`\n\tEncryptKey string `json:\",omitempty\"`\n\tEncryptAlgo string `json:\",omitempty\"`\n\tKeyEncrypted bool `json:\",omitempty\"`\n\tUploadLimit int64 `json:\",omitempty\"` // Mbps\n\tDownloadLimit int64 `json:\",omitempty\"` // Mbps\n\tTrashDays int\n\tMetaVersion int `json:\",omitempty\"`\n\tMinClientVersion string `json:\",omitempty\"`\n\tMaxClientVersion string `json:\",omitempty\"`\n\tDirStats bool `json:\",omitempty\"`\n\tUserGroupQuota bool `json:\",omitempty\"`\n\tEnableACL bool\n\tRangerRestUrl string `json:\",omitempty\"`\n\tRangerService string `json:\",omitempty\"`\n\n\t//kerberos\n\tKerbConf string `json:\",omitempty\"`\n}\n\nfunc (f *Format) update(old *Format, force bool) error {\n\tif force {\n\t\tlogger.Warnf(\"Existing volume will be overwrited: %s\", old)\n\t} else {\n\t\tvar args []interface{}\n\t\tswitch {\n\t\tcase f.Name != old.Name:\n\t\t\targs = []interface{}{\"name\", old.Name, f.Name}\n\t\tcase f.BlockSize != old.BlockSize:\n\t\t\targs = []interface{}{\"block size\", old.BlockSize, f.BlockSize}\n\t\tcase f.Compression != old.Compression:\n\t\t\targs = []interface{}{\"compression\", old.Compression, f.Compression}\n\t\tcase f.Shards != old.Shards:\n\t\t\targs = []interface{}{\"shards\", old.Shards, f.Shards}\n\t\tcase f.HashPrefix != old.HashPrefix:\n\t\t\targs = []interface{}{\"hash prefix\", old.HashPrefix, f.HashPrefix}\n\t\tcase f.MetaVersion != old.MetaVersion:\n\t\t\targs = []interface{}{\"meta version\", old.MetaVersion, f.MetaVersion}\n\t\t}\n\t\tif args == nil {\n\t\t\tif f.UUID != old.UUID {\n\t\t\t\tif err := f.Decrypt(); err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"decrypt format: %s\", err)\n\t\t\t\t}\n\t\t\t\tf.UUID = old.UUID // UUID cannot be changed alone\n\t\t\t\tif err := f.Encrypt(); err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"encrypt format: %s\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"cannot update volume %s from %v to %v\", args...)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (f *Format) RemoveSecret() {\n\tif f.SecretKey != \"\" {\n\t\tf.SecretKey = \"removed\"\n\t}\n\tif f.SessionToken != \"\" {\n\t\tf.SessionToken = \"removed\"\n\t}\n\tif f.EncryptKey != \"\" {\n\t\tf.EncryptKey = \"removed\"\n\t}\n}\n\nfunc (f *Format) String() string {\n\tt := *f\n\tt.RemoveSecret()\n\ts, _ := json.MarshalIndent(t, \"\", \" \")\n\treturn string(s)\n}\n\nfunc (f *Format) CheckVersion() error {\n\tif f.MetaVersion > MaxVersion {\n\t\treturn fmt.Errorf(\"incompatible metadata version: %d; please upgrade the client\", f.MetaVersion)\n\t}\n\n\tver := version.GetVersion()\n\treturn f.CheckCliVersion(&ver)\n}\n\nfunc (f *Format) CheckCliVersion(ver *version.Semver) error {\n\tif ver == nil {\n\t\treturn errors.New(\"version is nil\")\n\t}\n\n\tif f.MinClientVersion != \"\" {\n\t\tminClientVer := version.Parse(f.MinClientVersion)\n\t\tr, err := version.CompareVersions(ver, minClientVer)\n\t\tif err == nil && r < 0 {\n\t\t\terr = fmt.Errorf(\"allowed minimum version: %s; please upgrade the client\", f.MinClientVersion)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif f.MaxClientVersion != \"\" {\n\t\tmaxClientVer := version.Parse(f.MaxClientVersion)\n\t\tr, err := version.CompareVersions(ver, maxClientVer)\n\t\tif err == nil && r > 0 {\n\t\t\terr = fmt.Errorf(\"allowed maximum version: %s; please use an older client\", f.MaxClientVersion)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc newCipher(algo string, key string) (cipher.AEAD, error) {\n\tswitch algo {\n\tcase object.SM4GCM:\n\t\tblock, err := sm4.NewCipher(sm3.Kdf([]byte(key), 16))\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"new sm4 cipher: %s\", err)\n\t\t}\n\t\taead, err := cipher.NewGCM(block)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"new sm4 GCM: %s\", err)\n\t\t}\n\t\treturn aead, nil\n\tdefault:\n\t\thashKey := md5.Sum([]byte(key))\n\t\tblock, err := aes.NewCipher(hashKey[:])\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"new cipher: %s\", err)\n\t\t}\n\t\taead, err := cipher.NewGCM(block)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"new GCM: %s\", err)\n\t\t}\n\t\treturn aead, nil\n\t}\n}\n\nfunc (f *Format) Encrypt() error {\n\tif f.KeyEncrypted || f.SecretKey == \"\" && f.EncryptKey == \"\" && f.SessionToken == \"\" {\n\t\treturn nil\n\t}\n\tci, err := newCipher(f.EncryptAlgo, f.UUID)\n\tif err != nil {\n\t\treturn err\n\t}\n\tencrypt := func(k *string) {\n\t\tif *k == \"\" {\n\t\t\treturn\n\t\t}\n\t\tnonce := make([]byte, ci.NonceSize())\n\t\tif _, err = io.ReadFull(rand.Reader, nonce); err != nil {\n\t\t\tlogger.Fatalf(\"generate nonce for secret key: %s\", err)\n\t\t}\n\t\tciphertext := ci.Seal(nil, nonce, []byte(*k), nil)\n\t\tbuf := make([]byte, ci.NonceSize()+len(ciphertext))\n\t\tcopy(buf, nonce)\n\t\tcopy(buf[ci.NonceSize():], ciphertext)\n\t\t*k = base64.StdEncoding.EncodeToString(buf)\n\t}\n\n\tencrypt(&f.SecretKey)\n\tencrypt(&f.SessionToken)\n\tencrypt(&f.EncryptKey)\n\tf.KeyEncrypted = true\n\treturn nil\n}\n\nfunc (f *Format) Decrypt() error {\n\tif !f.KeyEncrypted {\n\t\treturn nil\n\t}\n\n\tci, err := newCipher(f.EncryptAlgo, f.UUID)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdecrypt := func(k *string) {\n\t\tif *k == \"\" {\n\t\t\treturn\n\t\t}\n\t\tif *k == \"removed\" {\n\t\t\terr = fmt.Errorf(\"secret was removed; please correct it with `config` command\")\n\t\t\treturn\n\t\t}\n\t\tbuf, e := base64.StdEncoding.DecodeString(*k)\n\t\tif e != nil {\n\t\t\terr = fmt.Errorf(\"decode key: %s\", e)\n\t\t\treturn\n\t\t}\n\t\tplaintext, e := ci.Open(nil, buf[:ci.NonceSize()], buf[ci.NonceSize():], nil)\n\t\tif e != nil {\n\t\t\terr = fmt.Errorf(\"open cipher: %s\", e)\n\t\t\treturn\n\t\t}\n\t\t*k = string(plaintext)\n\t}\n\n\tdecrypt(&f.EncryptKey)\n\tdecrypt(&f.SecretKey)\n\tdecrypt(&f.SessionToken)\n\tf.KeyEncrypted = false\n\treturn err\n}\n" }, { "path": "pkg/meta/config_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRemoveSecret(t *testing.T) {\n\tformat := Format{Name: \"test\", SecretKey: \"testSecret\", EncryptKey: \"testEncrypt\", SessionToken: \"token\"}\n\tif err := format.Encrypt(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tformat.RemoveSecret()\n\tif format.SecretKey != \"removed\" || format.EncryptKey != \"removed\" || format.SessionToken != \"removed\" {\n\t\tt.Fatalf(\"invalid format: %+v\", format)\n\t}\n\n\tif err := format.Decrypt(); err != nil && !strings.Contains(err.Error(), \"secret was removed\") {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestEncrypt(t *testing.T) {\n\tcases := []struct {\n\t\talgo string\n\t}{\n\t\t{object.AES256GCM_RSA},\n\t\t{object.CHACHA20_RSA},\n\t\t{object.SM4GCM},\n\t}\n\tformat := Format{Name: \"test\", SecretKey: \"testSecret\", SessionToken: \"token\", EncryptKey: \"testEncrypt\"}\n\tfor _, c := range cases {\n\t\tformat.EncryptAlgo = c.algo\n\t\tt.Run(c.algo, func(t *testing.T) {\n\t\t\tif err := format.Encrypt(); err != nil {\n\t\t\t\tt.Fatalf(\"Format encrypt: %s\", err)\n\t\t\t}\n\t\t\tif format.SecretKey == \"testSecret\" || format.SessionToken == \"token\" || format.EncryptKey == \"testEncrypt\" {\n\t\t\t\tt.Fatalf(\"invalid format: %+v\", format)\n\t\t\t}\n\t\t\tif err := format.Decrypt(); err != nil {\n\t\t\t\tt.Fatalf(\"Format decrypt: %s\", err)\n\t\t\t}\n\t\t\tif format.SecretKey != \"testSecret\" || format.SessionToken != \"token\" || format.EncryptKey != \"testEncrypt\" {\n\t\t\t\tt.Fatalf(\"invalid format: %+v\", format)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestFormat_Update_KeyConflict(t *testing.T) {\n\toldFormat := Format{Name: \"test\", UUID: \"UUID-A\"}\n\n\tnewFormat := Format{Name: \"test\", UUID: \"UUID-B\", SecretKey: \"secret\"}\n\tif err := newFormat.Encrypt(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tassert.True(t, newFormat.KeyEncrypted)\n\n\tif err := newFormat.update(&oldFormat, false); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tassert.Equal(t, \"UUID-A\", newFormat.UUID)\n\tassert.True(t, newFormat.KeyEncrypted)\n\n\tif err := newFormat.Decrypt(); err != nil {\n\t\tt.Fatalf(\"failed to decrypt with new UUID (which is old UUID A): %s\", err)\n\t}\n\n\tassert.Equal(t, \"secret\", newFormat.SecretKey)\n}\n" }, { "path": "pkg/meta/context.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"time\"\n)\n\ntype CtxKey string\n\ntype Context interface {\n\tcontext.Context\n\tGid() uint32\n\tGids() []uint32\n\tUid() uint32\n\tPid() uint32\n\tWithValue(k, v interface{}) Context // should remain const semantics, so user can chain it\n\tCancel()\n\tCanceled() bool\n\tCheckPermission() bool\n}\n\nfunc Background() Context {\n\treturn WrapContext(context.Background())\n}\n\ntype wrapContext struct {\n\tcontext.Context\n\tcancel func()\n\tpid uint32\n\tuid uint32\n\tgids []uint32\n}\n\nfunc (c *wrapContext) Uid() uint32 {\n\treturn c.uid\n}\n\nfunc (c *wrapContext) Gid() uint32 {\n\treturn c.gids[0]\n}\n\nfunc (c *wrapContext) Gids() []uint32 {\n\treturn c.gids\n}\n\nfunc (c *wrapContext) Pid() uint32 {\n\treturn c.pid\n}\n\nfunc (c *wrapContext) Cancel() {\n\tif c.cancel != nil {\n\t\tc.cancel()\n\t}\n}\n\nfunc (c *wrapContext) Canceled() bool {\n\treturn c.Err() != nil\n}\n\nfunc (c *wrapContext) WithValue(k, v interface{}) Context {\n\twc := *c // gids is a const, so it's safe to shallow copy\n\twc.Context = context.WithValue(c.Context, k, v)\n\treturn &wc\n}\n\nfunc (c *wrapContext) CheckPermission() bool {\n\treturn true\n}\n\nfunc NewContext(pid, uid uint32, gids []uint32) Context {\n\treturn WrapWithCancel(context.Background(), pid, uid, gids)\n}\n\nfunc WrapContext(ctx context.Context) Context {\n\treturn WrapWithCancel(ctx, 0, 0, []uint32{0})\n}\n\nfunc WrapWithCancel(ctx context.Context, pid, uid uint32, gids []uint32) Context {\n\tc, cancel := context.WithCancel(ctx)\n\treturn &wrapContext{c, cancel, pid, uid, gids}\n}\n\nfunc WrapWithTimeout(ctx Context, timeout time.Duration) Context {\n\tc, cancel := context.WithTimeout(ctx, timeout)\n\treturn &wrapContext{c, cancel, ctx.Pid(), ctx.Uid(), ctx.Gids()}\n}\n\nfunc WrapWithoutCancel(ctx context.Context, pid, uid uint32, gids []uint32) Context {\n\treturn &wrapContext{ctx, nil, pid, uid, gids}\n}\n\nfunc containsGid(ctx Context, gid uint32) bool {\n\tfor _, g := range ctx.Gids() {\n\t\tif g == gid {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n" }, { "path": "pkg/meta/dump.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"strings\"\n\t\"sync\"\n\t\"unicode/utf8\"\n\n\t\"github.com/goccy/go-json\"\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nconst (\n\tjsonIndent = \" \"\n\tjsonWriteSize = 64 << 10\n)\n\ntype DumpedCounters struct {\n\tUsedSpace int64 `json:\"usedSpace\"`\n\tUsedInodes int64 `json:\"usedInodes\"`\n\tNextInode int64 `json:\"nextInodes\"`\n\tNextChunk int64 `json:\"nextChunk\"`\n\tNextSession int64 `json:\"nextSession\"`\n\tNextTrash int64 `json:\"nextTrash\"`\n\tNextCleanupSlices int64 `json:\"nextCleanupSlices,omitempty\"` // deprecated, always 0\n}\n\ntype DumpedDelFile struct {\n\tInode Ino `json:\"inode\"`\n\tLength uint64 `json:\"length\"`\n\tExpire int64 `json:\"expire\"`\n}\n\ntype DumpedSustained struct {\n\tSid uint64 `json:\"sid\"`\n\tInodes []Ino `json:\"inodes\"`\n}\n\ntype DumpedAttr struct {\n\tInode Ino `json:\"inode\"`\n\tFlags uint8 `json:\"flags,omitempty\"`\n\tType string `json:\"type\"`\n\tMode uint16 `json:\"mode\"`\n\tUid uint32 `json:\"uid\"`\n\tGid uint32 `json:\"gid\"`\n\tAtime int64 `json:\"atime\"`\n\tMtime int64 `json:\"mtime\"`\n\tCtime int64 `json:\"ctime\"`\n\tAtimensec uint32 `json:\"atimensec,omitempty\"`\n\tMtimensec uint32 `json:\"mtimensec,omitempty\"`\n\tCtimensec uint32 `json:\"ctimensec,omitempty\"`\n\tNlink uint32 `json:\"nlink\"`\n\tLength uint64 `json:\"length\"`\n\tRdev uint32 `json:\"rdev,omitempty\"`\n\tfull bool\n}\n\ntype DumpedSlice struct {\n\tChunkid uint64 `json:\"chunkid,omitempty\"`\n\tId uint64 `json:\"id\"`\n\tPos uint32 `json:\"pos,omitempty\"`\n\tSize uint32 `json:\"size\"`\n\tOff uint32 `json:\"off,omitempty\"`\n\tLen uint32 `json:\"len\"`\n}\n\ntype DumpedChunk struct {\n\tIndex uint32 `json:\"index\"`\n\tSlices []*DumpedSlice `json:\"slices\"`\n}\n\ntype DumpedXattr struct {\n\tName string `json:\"name\"`\n\tValue string `json:\"value\"`\n}\n\ntype DumpedQuota struct {\n\tMaxSpace int64 `json:\"maxSpace\"`\n\tMaxInodes int64 `json:\"maxInodes\"`\n\tUsedSpace int64 `json:\"-\"`\n\tUsedInodes int64 `json:\"-\"`\n}\n\ntype DumpedACLEntry struct {\n\tId uint32 `json:\"id\"`\n\tPerm uint16 `json:\"perm\"`\n}\n\ntype DumpedACL struct {\n\tOwner uint16 `json:\"owner\"`\n\tGroup uint16 `json:\"group\"`\n\tOther uint16 `json:\"other\"`\n\tMask uint16 `json:\"mask\"`\n\tUsers []DumpedACLEntry `json:\"users\"`\n\tGroups []DumpedACLEntry `json:\"groups\"`\n}\n\ntype DumpedEntry struct {\n\tName string `json:\"-\"`\n\tParents []Ino `json:\"-\"`\n\tAttr *DumpedAttr `json:\"attr,omitempty\"`\n\tSymlink string `json:\"symlink,omitempty\"`\n\tXattrs []*DumpedXattr `json:\"xattrs,omitempty\"`\n\tChunks []*DumpedChunk `json:\"chunks,omitempty\"`\n\tEntries map[string]*DumpedEntry `json:\"entries,omitempty\"`\n\tAccessACL *DumpedACL `json:\"posix_acl_access,omitempty\"`\n\tDefaultACL *DumpedACL `json:\"posix_acl_default,omitempty\"`\n}\n\ntype wrapEntryPool struct {\n\tsync.Pool\n}\n\nfunc (p *wrapEntryPool) Get() *DumpedEntry {\n\treturn p.Pool.Get().(*DumpedEntry)\n}\n\nfunc (p *wrapEntryPool) Put(de *DumpedEntry) {\n\tif de == nil {\n\t\treturn\n\t}\n\n\tde.Name = \"\"\n\tde.Xattrs = nil\n\tde.Chunks = nil\n\tde.Symlink = \"\"\n\tde.AccessACL = nil\n\tde.DefaultACL = nil\n\tde.Entries = nil\n\tp.Pool.Put(de)\n}\n\nvar entryPool = wrapEntryPool{\n\tPool: sync.Pool{\n\t\tNew: func() interface{} {\n\t\t\treturn &DumpedEntry{\n\t\t\t\tAttr: &DumpedAttr{},\n\t\t\t}\n\t\t},\n\t},\n}\n\nvar CHARS = []byte(\"0123456789ABCDEF\")\n\nfunc escape(original string) string {\n\t// similar to url.Escape but backward compatible if no '%' in it\n\tvar escValue = make([]byte, 0, len(original))\n\tfor i, r := range original {\n\t\tif r == utf8.RuneError || r < 32 || r == '%' || r == '\"' || r == '\\\\' {\n\t\t\tif escValue == nil {\n\t\t\t\tescValue = make([]byte, i, len(original)*2)\n\t\t\t\tfor j := 0; j < i; j++ {\n\t\t\t\t\tescValue[j] = original[j]\n\t\t\t\t}\n\t\t\t}\n\t\t\tc := byte(r)\n\t\t\tif r == utf8.RuneError {\n\t\t\t\tc = original[i]\n\t\t\t}\n\t\t\tescValue = append(escValue, '%')\n\t\t\tescValue = append(escValue, CHARS[(c>>4)&0xF])\n\t\t\tescValue = append(escValue, CHARS[c&0xF])\n\t\t} else if escValue != nil {\n\t\t\tn := utf8.RuneLen(r)\n\t\t\tescValue = append(escValue, original[i:i+n]...)\n\t\t}\n\t}\n\tif escValue == nil {\n\t\treturn original\n\t}\n\treturn string(escValue)\n}\n\nfunc parseHex(c byte) (byte, error) {\n\tif c >= '0' && c <= '9' {\n\t\treturn c - '0', nil\n\t} else if c >= 'A' && c <= 'F' {\n\t\treturn 10 + (c - 'A'), nil\n\t} else {\n\t\treturn 0, fmt.Errorf(\"hex expected: %c\", c)\n\t}\n}\n\nfunc unescape(s string) []byte {\n\tif !strings.ContainsRune(s, '%') {\n\t\treturn []byte(s)\n\t}\n\n\tp := []byte(s)\n\tn := 0\n\tfor i := 0; i < len(p); i++ {\n\t\tc := p[i]\n\t\tif c == '%' && i+2 < len(p) {\n\t\t\th, e1 := parseHex(p[i+1])\n\t\t\tl, e2 := parseHex(p[i+2])\n\t\t\tif e1 == nil && e2 == nil {\n\t\t\t\tc = h*16 + l\n\t\t\t\ti += 2\n\t\t\t}\n\t\t}\n\t\tp[n] = c\n\t\tn++\n\t}\n\treturn p[:n]\n}\n\nfunc (de *DumpedEntry) writeJSON(bw *bufio.Writer, depth int) error {\n\tprefix := strings.Repeat(jsonIndent, depth)\n\tfieldPrefix := prefix + jsonIndent\n\twrite := func(s string) {\n\t\tif _, err := bw.WriteString(s); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\twrite(fmt.Sprintf(\"\\n%s\\\"%s\\\": {\", prefix, escape(de.Name)))\n\tdata, err := json.Marshal(de.Attr)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\twrite(fmt.Sprintf(\"\\n%s\\\"attr\\\": %s\", fieldPrefix, data))\n\tif len(de.Symlink) > 0 {\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"symlink\\\": \\\"%s\\\"\", fieldPrefix, escape(de.Symlink)))\n\t}\n\tif len(de.Xattrs) > 0 {\n\t\tfor _, dumpedXattr := range de.Xattrs {\n\t\t\tdumpedXattr.Value = escape(dumpedXattr.Value)\n\t\t}\n\t\tif data, err = json.Marshal(de.Xattrs); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"xattrs\\\": %s\", fieldPrefix, data))\n\t}\n\tif de.AccessACL != nil {\n\t\tif data, err = json.Marshal(de.AccessACL); err != nil {\n\t\t\treturn err\n\t\t}\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"posix_acl_access\\\": %s\", fieldPrefix, data))\n\t}\n\tif de.DefaultACL != nil {\n\t\tif data, err = json.Marshal(de.DefaultACL); err != nil {\n\t\t\treturn err\n\t\t}\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"posix_acl_default\\\": %s\", fieldPrefix, data))\n\t}\n\tif len(de.Chunks) == 1 {\n\t\tif data, err = json.Marshal(de.Chunks); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"chunks\\\": %s\", fieldPrefix, data))\n\t} else if len(de.Chunks) > 1 {\n\t\tchunkPrefix := fieldPrefix + jsonIndent\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"chunks\\\": [\", fieldPrefix))\n\t\tfor i, c := range de.Chunks {\n\t\t\tif data, err = json.Marshal(c); err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t\twrite(fmt.Sprintf(\"\\n%s%s\", chunkPrefix, data))\n\t\t\tif i != len(de.Chunks)-1 {\n\t\t\t\twrite(\",\")\n\t\t\t}\n\t\t}\n\t\twrite(fmt.Sprintf(\"\\n%s]\", fieldPrefix))\n\t}\n\twrite(fmt.Sprintf(\"\\n%s}\", prefix))\n\treturn nil\n}\n\nfunc (de *DumpedEntry) writeJsonWithOutEntry(bw *bufio.Writer, depth int) error {\n\tprefix := strings.Repeat(jsonIndent, depth)\n\tfieldPrefix := prefix + jsonIndent\n\twrite := func(s string) {\n\t\tif _, err := bw.WriteString(s); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\twrite(fmt.Sprintf(\"\\n%s\\\"%s\\\": {\", prefix, escape(de.Name)))\n\tdata, err := json.Marshal(de.Attr)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\twrite(fmt.Sprintf(\"\\n%s\\\"attr\\\": %s\", fieldPrefix, data))\n\tif len(de.Xattrs) > 0 {\n\t\tfor _, dumpedXattr := range de.Xattrs {\n\t\t\tdumpedXattr.Value = escape(dumpedXattr.Value)\n\t\t}\n\t\tif data, err = json.Marshal(de.Xattrs); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"xattrs\\\": %s\", fieldPrefix, data))\n\t}\n\tif de.AccessACL != nil {\n\t\tif data, err = json.Marshal(de.AccessACL); err != nil {\n\t\t\treturn err\n\t\t}\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"posix_acl_access\\\": %s\", fieldPrefix, data))\n\t}\n\tif de.DefaultACL != nil {\n\t\tif data, err = json.Marshal(de.DefaultACL); err != nil {\n\t\t\treturn err\n\t\t}\n\t\twrite(fmt.Sprintf(\",\\n%s\\\"posix_acl_default\\\": %s\", fieldPrefix, data))\n\t}\n\twrite(fmt.Sprintf(\",\\n%s\\\"entries\\\": {\", fieldPrefix))\n\treturn nil\n}\n\ntype DumpedMeta struct {\n\tSetting Format\n\tCounters *DumpedCounters\n\tSustained []*DumpedSustained\n\tDelFiles []*DumpedDelFile\n\tQuotas map[Ino]*DumpedQuota `json:\",omitempty\"`\n\tFSTree *DumpedEntry `json:\",omitempty\"`\n\tTrash *DumpedEntry `json:\",omitempty\"`\n}\n\nfunc (dm *DumpedMeta) validate() error {\n\tif dm.Counters == nil {\n\t\treturn errors.New(\"invalid dumped meta: missing 'Counters'\")\n\t}\n\treturn nil\n}\n\nfunc (dm *DumpedMeta) writeJsonWithOutTree(w io.Writer) (*bufio.Writer, error) {\n\tif dm.FSTree != nil || dm.Trash != nil {\n\t\treturn nil, fmt.Errorf(\"invalid dumped meta\")\n\t}\n\tdata, err := json.MarshalIndent(dm, \"\", jsonIndent)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbw := bufio.NewWriterSize(w, jsonWriteSize)\n\tif _, err = bw.Write(append(data[:len(data)-2], ',')); err != nil { // delete \\n}\n\t\treturn nil, err\n\t}\n\treturn bw, nil\n}\n\nfunc (m *baseMeta) loadDumpedQuotas(ctx Context, quotas map[Ino]*DumpedQuota) {\n\t// update quota\n\tfor inode, q := range quotas {\n\t\tif _, err := m.en.doSetQuota(ctx, DirQuotaType, uint64(inode), &Quota{q.MaxSpace, q.MaxInodes, q.UsedSpace, q.UsedInodes, 0, 0}); err != nil {\n\t\t\tlogger.Warnf(\"reset quota of %d: %s\", inode, err)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc dumpAttr(a *Attr, d *DumpedAttr) {\n\tif a.Typ > 0 {\n\t\td.Type = typeToString(a.Typ)\n\t}\n\td.Flags = a.Flags\n\td.Mode = a.Mode\n\td.Uid = a.Uid\n\td.Gid = a.Gid\n\td.Atime = a.Atime\n\td.Mtime = a.Mtime\n\td.Ctime = a.Ctime\n\td.Atimensec = a.Atimensec\n\td.Mtimensec = a.Mtimensec\n\td.Ctimensec = a.Ctimensec\n\td.Nlink = a.Nlink\n\td.Rdev = a.Rdev\n\tif a.Typ == TypeFile {\n\t\td.Length = a.Length\n\t} else {\n\t\td.Length = 0\n\t}\n\td.full = a.Full\n}\n\nfunc loadAttr(d *DumpedAttr) *Attr {\n\treturn &Attr{\n\t\tFlags: d.Flags,\n\t\tTyp: typeFromString(d.Type),\n\t\tMode: d.Mode,\n\t\tUid: d.Uid,\n\t\tGid: d.Gid,\n\t\tAtime: d.Atime,\n\t\tMtime: d.Mtime,\n\t\tCtime: d.Ctime,\n\t\tAtimensec: d.Atimensec,\n\t\tMtimensec: d.Mtimensec,\n\t\tCtimensec: d.Ctimensec,\n\t\tNlink: d.Nlink,\n\t\tRdev: d.Rdev,\n\t\tFull: true,\n\t} // Length and Parent not set\n}\n\ntype chunkKey struct {\n\tid uint64\n\tsize uint32\n}\n\nfunc loadEntries(r io.Reader, load func(*DumpedEntry), addChunk func(*chunkKey)) (dm *DumpedMeta,\n\tcounters *DumpedCounters, parents map[Ino][]Ino, refs map[chunkKey]int64, err error) {\n\tlogger.Infoln(\"Loading from file ...\")\n\tdec := json.NewDecoder(r)\n\tif _, err = dec.Token(); err != nil {\n\t\treturn\n\t}\n\n\tprogress := utils.NewProgress(false)\n\tbar := progress.AddCountBar(\"Loaded entries\", 1) // with root\n\tdm = &DumpedMeta{}\n\tcounters = &DumpedCounters{ // rebuild counters\n\t\tNextInode: 2,\n\t\tNextChunk: 1,\n\t}\n\tparents = make(map[Ino][]Ino)\n\trefs = make(map[chunkKey]int64)\n\tvar name json.Token\n\tfor dec.More() {\n\t\tname, err = dec.Token()\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"parse name: %s\", err)\n\t\t\treturn\n\t\t}\n\t\tswitch name {\n\t\tcase \"Setting\":\n\t\t\tif err = dec.Decode(&dm.Setting); err == nil {\n\t\t\t\t_, err = json.MarshalIndent(dm.Setting, \"\", \"\")\n\t\t\t}\n\t\tcase \"Counters\":\n\t\t\tif err = dec.Decode(&dm.Counters); err == nil {\n\t\t\t\tbar.SetTotal(dm.Counters.UsedInodes) // TODO\n\t\t\t}\n\t\tcase \"Sustained\":\n\t\t\terr = dec.Decode(&dm.Sustained)\n\t\tcase \"DelFiles\":\n\t\t\terr = dec.Decode(&dm.DelFiles)\n\t\tcase \"Quotas\":\n\t\t\terr = dec.Decode(&dm.Quotas)\n\t\tcase \"FSTree\":\n\t\t\t_, err = decodeEntry(dec, 0, counters, parents, dm.Quotas, refs, bar, load, addChunk)\n\t\tcase \"Trash\":\n\t\t\t_, err = decodeEntry(dec, 1, counters, parents, nil, refs, bar, load, addChunk)\n\t\t}\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"load %v: %s\", name, err)\n\t\t\treturn\n\t\t}\n\t}\n\t_, _ = dec.Token() // }\n\tprogress.Done()\n\n\tif err = dm.validate(); err != nil {\n\t\treturn\n\t}\n\n\tlogger.Infof(\"Dumped counters: %+v\", *dm.Counters)\n\tlogger.Infof(\"Loaded counters: %+v\", *counters)\n\treturn\n}\n\nfunc decodeEntry(dec *json.Decoder, parent Ino, cs *DumpedCounters, parents map[Ino][]Ino, quotas map[Ino]*DumpedQuota,\n\trefs map[chunkKey]int64, bar *utils.Bar, load func(*DumpedEntry), addChunk func(*chunkKey)) (*DumpedEntry, error) {\n\tif _, err := dec.Token(); err != nil {\n\t\treturn nil, err\n\t}\n\tvar e = DumpedEntry{}\n\tfor dec.More() {\n\t\tname, err := dec.Token()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tswitch name {\n\t\tcase \"attr\":\n\t\t\terr = dec.Decode(&e.Attr)\n\t\t\tif err == nil {\n\t\t\t\tif parent == 0 {\n\t\t\t\t\tparent = 1\n\t\t\t\t\te.Attr.Inode = 1 // fix loading from subdir\n\t\t\t\t}\n\t\t\t\tinode := e.Attr.Inode\n\t\t\t\tif typeFromString(e.Attr.Type) == TypeDirectory {\n\t\t\t\t\te.Attr.Nlink = 2\n\t\t\t\t} else {\n\t\t\t\t\te.Attr.Nlink = 1\n\t\t\t\t}\n\t\t\t\te.Parents = append(parents[inode], parent)\n\t\t\t\tparents[inode] = e.Parents\n\t\t\t\tif len(e.Parents) == 1 {\n\t\t\t\t\tif inode > 1 && inode != TrashInode {\n\t\t\t\t\t\tcs.UsedSpace += align4K(e.Attr.Length)\n\t\t\t\t\t\tcs.UsedInodes += 1\n\t\t\t\t\t}\n\t\t\t\t\tif inode < TrashInode {\n\t\t\t\t\t\tif cs.NextInode <= int64(inode) {\n\t\t\t\t\t\t\tcs.NextInode = int64(inode) + 1\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif cs.NextTrash < int64(inode-TrashInode) {\n\t\t\t\t\t\t\tcs.NextTrash = int64(inode - TrashInode)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"chunks\":\n\t\t\terr = dec.Decode(&e.Chunks)\n\t\t\tif err == nil && len(e.Parents) == 1 {\n\t\t\t\tfor _, c := range e.Chunks {\n\t\t\t\t\tfor _, s := range c.Slices {\n\t\t\t\t\t\tif s.Chunkid != 0 && s.Id == 0 {\n\t\t\t\t\t\t\ts.Id = s.Chunkid\n\t\t\t\t\t\t\ts.Chunkid = 0\n\t\t\t\t\t\t}\n\t\t\t\t\t\tck := chunkKey{s.Id, s.Size}\n\t\t\t\t\t\trefs[ck]++\n\t\t\t\t\t\tif addChunk != nil && refs[ck] == 1 {\n\t\t\t\t\t\t\taddChunk(&ck)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif cs.NextChunk <= int64(s.Id) {\n\t\t\t\t\t\t\tcs.NextChunk = int64(s.Id) + 1\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"entries\":\n\t\t\te.Entries = make(map[string]*DumpedEntry)\n\t\t\t_, err = dec.Token()\n\t\t\tvar usedSpace, usedInodes int64\n\t\t\tif err == nil {\n\t\t\t\tfor dec.More() {\n\t\t\t\t\tvar n json.Token\n\t\t\t\t\tn, err = dec.Token()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tvar child *DumpedEntry\n\t\t\t\t\tchild, err = decodeEntry(dec, e.Attr.Inode, cs, parents, quotas, refs, bar, load, addChunk)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tif e.Attr.Inode < TrashInode && typeFromString(child.Attr.Type) == TypeDirectory {\n\t\t\t\t\t\te.Attr.Nlink++\n\t\t\t\t\t}\n\t\t\t\t\te.Entries[n.(string)] = &DumpedEntry{\n\t\t\t\t\t\tAttr: &DumpedAttr{\n\t\t\t\t\t\t\tInode: child.Attr.Inode,\n\t\t\t\t\t\t\tType: child.Attr.Type,\n\t\t\t\t\t\t\tLength: child.Attr.Length,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\tusedSpace += align4K(child.Attr.Length)\n\t\t\t\t\tusedInodes++\n\t\t\t\t}\n\t\t\t\tif err == nil {\n\t\t\t\t\ti := e.Attr.Inode\n\t\t\t\t\tfor {\n\t\t\t\t\t\tif q := quotas[i]; q != nil {\n\t\t\t\t\t\t\tq.UsedSpace += usedSpace\n\t\t\t\t\t\t\tq.UsedInodes += usedInodes\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif i <= 1 || len(parents[i]) == 0 {\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\ti = parents[i][0]\n\t\t\t\t\t}\n\n\t\t\t\t\tvar t json.Token\n\t\t\t\t\tt, err = dec.Token()\n\t\t\t\t\tif err == nil && t != json.Delim('}') {\n\t\t\t\t\t\terr = fmt.Errorf(\"unexpected %v\", t)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"symlink\":\n\t\t\terr = dec.Decode(&e.Symlink)\n\t\tcase \"xattrs\":\n\t\t\terr = dec.Decode(&e.Xattrs)\n\t\tcase \"posix_acl_access\":\n\t\t\terr = dec.Decode(&e.AccessACL)\n\t\tcase \"posix_acl_default\":\n\t\t\terr = dec.Decode(&e.DefaultACL)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"decode %v: %s\", name, err)\n\t\t}\n\t}\n\tif len(e.Parents) == 1 {\n\t\tload(&e)\n\t\tbar.Increment()\n\t}\n\tif _, err := dec.Token(); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &e, nil\n}\n\nfunc dumpACL(rule *aclAPI.Rule) *DumpedACL {\n\tif rule == nil {\n\t\treturn nil\n\t}\n\treturn &DumpedACL{\n\t\tOwner: rule.Owner,\n\t\tGroup: rule.Group,\n\t\tOther: rule.Other,\n\t\tMask: rule.Mask,\n\t\tUsers: dumpACLEntries(rule.NamedUsers),\n\t\tGroups: dumpACLEntries(rule.NamedGroups),\n\t}\n}\n\nfunc dumpACLEntries(entries aclAPI.Entries) []DumpedACLEntry {\n\tif len(entries) == 0 {\n\t\treturn nil\n\t}\n\tdumpedEnts := make([]DumpedACLEntry, len(entries))\n\tfor i, ent := range entries {\n\t\tdumpedEnts[i].Id = ent.Id\n\t\tdumpedEnts[i].Perm = ent.Perm\n\t}\n\treturn dumpedEnts\n}\n\nfunc loadACL(dumped *DumpedACL) *aclAPI.Rule {\n\tif dumped == nil {\n\t\treturn nil\n\t}\n\treturn &aclAPI.Rule{\n\t\tOwner: dumped.Owner,\n\t\tGroup: dumped.Group,\n\t\tMask: dumped.Mask,\n\t\tOther: dumped.Other,\n\t\tNamedUsers: loadACLEntries(dumped.Users),\n\t\tNamedGroups: loadACLEntries(dumped.Groups),\n\t}\n}\n\nfunc loadACLEntries(dumpedEnts []DumpedACLEntry) aclAPI.Entries {\n\tif len(dumpedEnts) == 0 {\n\t\treturn nil\n\t}\n\tents := make(aclAPI.Entries, len(dumpedEnts))\n\tfor i, d := range dumpedEnts {\n\t\tents[i].Id = d.Id\n\t\tents[i].Perm = d.Perm\n\t}\n\treturn ents\n}\n" }, { "path": "pkg/meta/info.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype redisVersion struct {\n\tver string\n\tmajor, minor int\n}\n\nvar oldestSupportedVer = redisVersion{\"4.0.x\", 4, 0}\n\nfunc parseRedisVersion(v string) (ver redisVersion, err error) {\n\tparts := strings.Split(v, \".\")\n\tif len(parts) < 2 {\n\t\terr = fmt.Errorf(\"invalid redisVersion: %v\", v)\n\t\treturn\n\t}\n\tver.ver = v\n\tver.major, err = strconv.Atoi(parts[0])\n\tif err != nil {\n\t\treturn\n\t}\n\tver.minor, err = strconv.Atoi(parts[1])\n\treturn\n}\n\nfunc (ver redisVersion) olderThan(v2 redisVersion) bool {\n\tif ver.major < v2.major {\n\t\treturn true\n\t}\n\tif ver.major > v2.major {\n\t\treturn false\n\t}\n\treturn ver.minor < v2.minor\n}\n\nfunc (ver redisVersion) String() string {\n\treturn ver.ver\n}\n\ntype redisInfo struct {\n\taofEnabled bool\n\tmaxMemoryPolicy string\n\tredisVersion string\n\tstorageProvider string // redis is \"\", keyDB is \"none\" or \"flash\"\n}\n\nfunc checkRedisInfo(rawInfo string) (info redisInfo, err error) {\n\tlines := strings.Split(strings.TrimSpace(rawInfo), \"\\n\")\n\tfor _, l := range lines {\n\t\tl = strings.TrimSpace(l)\n\t\tif l == \"\" || strings.HasPrefix(l, \"#\") {\n\t\t\tcontinue\n\t\t}\n\t\tkvPair := strings.SplitN(l, \":\", 2)\n\t\tif len(kvPair) < 2 {\n\t\t\tcontinue\n\t\t}\n\t\tkey, val := kvPair[0], kvPair[1]\n\t\tswitch key {\n\t\tcase \"aof_enabled\":\n\t\t\tinfo.aofEnabled = val == \"1\"\n\t\t\tif val == \"0\" {\n\t\t\t\tlogger.Warnf(\"AOF is not enabled, you may lose data if Redis is not shutdown properly.\")\n\t\t\t}\n\t\tcase \"maxmemory_policy\":\n\t\t\tinfo.maxMemoryPolicy = val\n\t\tcase \"redis_version\":\n\t\t\tinfo.redisVersion = val\n\t\t\tver, err := parseRedisVersion(val)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"Failed to parse Redis server version %q: %s\", ver, err)\n\t\t\t} else {\n\t\t\t\tif ver.olderThan(oldestSupportedVer) {\n\t\t\t\t\tlogger.Fatalf(\"Redis version should not be older than %s\", oldestSupportedVer)\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"storage_provider\":\n\t\t\t// if storage_provider is none reset it to \"\"\n\t\t\tif val == \"flash\" {\n\t\t\t\tinfo.storageProvider = val\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n" }, { "path": "pkg/meta/info_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport \"testing\"\n\nfunc TestOlderThan(t *testing.T) {\n\tv := redisVersion{\"2.2.10\", 2, 2}\n\tif !v.olderThan(redisVersion{\"6.2\", 6, 2}) {\n\t\tt.Fatal(\"Expect true, got false.\")\n\t}\n\tif !v.olderThan(redisVersion{\"2.3\", 2, 3}) {\n\t\tt.Fatal(\"Expect true, got false.\")\n\t}\n\tif v.olderThan(redisVersion{\"2.2\", 2, 2}) {\n\t\tt.Fatal(\"Expect false, got true.\")\n\t}\n\tif v.olderThan(redisVersion{\"2.1\", 2, 1}) {\n\t\tt.Fatal(\"Expect false, got true.\")\n\t}\n\tif v.olderThan(v) {\n\t\tt.Fatal(\"Expect false, got true.\")\n\t}\n\tif v.olderThan(redisVersion{}) {\n\t\tt.Fatal(\"Expect false, got true.\")\n\t}\n}\n\nfunc TestParseRedisVersion(t *testing.T) {\n\tt.Run(\"Should return error for invalid redisVersion\", func(t *testing.T) {\n\t\tinvalidVers := []string{\"\", \"2.sadf.1\", \"3\", \"t.3.4\"}\n\t\tfor _, v := range invalidVers {\n\t\t\t_, err := parseRedisVersion(v)\n\t\t\tif err == nil {\n\t\t\t\tt.Fail()\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"Should parse redisVersion\", func(t *testing.T) {\n\t\tver, err := parseRedisVersion(\"6.2.19\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Failed to parse a valid redisVersion: %s\", err)\n\t\t}\n\t\tif !(ver.major == 6 && ver.minor == 2) {\n\t\t\tt.Fatalf(\"Expect %s, got %s\", \"6.2\", ver)\n\t\t}\n\t\tif ver.String() != \"6.2.19\" {\n\t\t\tt.Fatalf(\"Expect %s, got %s\", \"6.2.19\", ver)\n\t\t}\n\t})\n}\n\nfunc TestParseRedisInfo(t *testing.T) {\n\tt.Run(\"Should parse the fields we are interested in\", func(t *testing.T) {\n\t\tinput := `# Server\n\tredis_version:6.1.240\n\tredis_git_sha1:00000000\n\tredis_git_dirty:0\n\tredis_build_id:a26db646ea64a07c\n\tredis_mode:standalone\n\tos:Linux 5.4.0-1017-aws x86_64\n\tarch_bits:64\n\tmultiplexing_api:epoll\n\tatomicvar_api:c11-builtin\n\tgcc_version:9.3.0\n\tprocess_id:2755423\n\tprocess_supervised:no\n\trun_id:d04b36ea49704b152d8ce82bf563d26bcd52e741\n\ttcp_port:6379\n\tserver_time_usec:1610404734862725\n\tuptime_in_seconds:2430194\n\tuptime_in_days:28\n\thz:10\n\tconfigured_hz:10\n\tlru_clock:16569214\n\texecutable:/usr/local/bin/redis-server\n\tconfig_file:/etc/redis/redis.conf\n\tio_threads_active:0\n\n\t\t# Clients\n\tconnected_clients:2\n\tcluster_connections:0\n\tmaxclients:10000\n\tclient_recent_max_input_buffer:24\n\tclient_recent_max_output_buffer:0\n\tblocked_clients:0\n\ttracking_clients:0\n\tclients_in_timeout_table:0\n\n\t\t# Memory\n\tused_memory:200001664\n\tused_memory_human:190.74M\n\tused_memory_rss:210456576\n\tused_memory_rss_human:200.71M\n\tused_memory_peak:200060312\n\tused_memory_peak_human:190.79M\n\tused_memory_peak_perc:99.97%\n\t\tused_memory_overhead:54246680\n\tused_memory_startup:803648\n\tused_memory_dataset:145754984\n\tused_memory_dataset_perc:73.17%\n\t\tallocator_allocated:199994624\n\tallocator_active:200847360\n\tallocator_resident:209551360\n\ttotal_system_memory:16596942848\n\ttotal_system_memory_human:15.46G\n\tused_memory_lua:37888\n\tused_memory_lua_human:37.00K\n\tused_memory_scripts:0\n\tused_memory_scripts_human:0B\n\tnumber_of_cached_scripts:0\n\tmaxmemory:200000000\n\tmaxmemory_human:190.73M\n\tmaxmemory_policy:allkeys-lru\n\tallocator_frag_ratio:1.00\n\tallocator_frag_bytes:852736\n\tallocator_rss_ratio:1.04\n\tallocator_rss_bytes:8704000\n\trss_overhead_ratio:1.00\n\trss_overhead_bytes:905216\n\tmem_fragmentation_ratio:1.05\n\tmem_fragmentation_bytes:10538760\n\tmem_not_counted_for_evict:0\n\tmem_replication_backlog:0\n\tmem_clients_slaves:0\n\tmem_clients_normal:41008\n\tmem_aof_buffer:0\n\tmem_allocator:jemalloc-5.1.0\n\tactive_defrag_running:0\n\tlazyfree_pending_objects:0\n\tlazyfreed_objects:0\n\n\t\t# Persistence\n\tloading:0\n\trdb_changes_since_last_save:6407091\n\trdb_bgsave_in_progress:0\n\trdb_last_save_time:1607974540\n\trdb_last_bgsave_status:ok\n\trdb_last_bgsave_time_sec:-1\n\trdb_current_bgsave_time_sec:-1\n\trdb_last_cow_size:0\n\taof_enabled:0\n\taof_rewrite_in_progress:0\n\taof_rewrite_scheduled:0\n\taof_last_rewrite_time_sec:-1\n\taof_current_rewrite_time_sec:-1\n\taof_last_bgrewrite_status:ok\n\taof_last_write_status:ok\n\taof_last_cow_size:0\n\tmodule_fork_in_progress:0\n\tmodule_fork_last_cow_size:0\n\n\t\t# Stats\n\ttotal_connections_received:127469\n\ttotal_commands_processed:15725530\n\tinstantaneous_ops_per_sec:8\n\ttotal_net_input_bytes:1305500885\n\ttotal_net_output_bytes:237264322\n\tinstantaneous_input_kbps:0.74\n\tinstantaneous_output_kbps:0.10\n\trejected_connections:0\n\tsync_full:0\n\tsync_partial_ok:0\n\tsync_partial_err:0\n\texpired_keys:41809\n\texpired_stale_perc:0.00\n\texpired_time_cap_reached_count:0\n\texpire_cycle_cpu_milliseconds:75107\n\tevicted_keys:182417\n\tkeyspace_hits:3627925\n\tkeyspace_misses:1661042\n\tpubsub_channels:0\n\tpubsub_patterns:0\n\tlatest_fork_usec:0\n\ttotal_forks:0\n\tmigrate_cached_sockets:0\n\tslave_expires_tracked_keys:0\n\tactive_defrag_hits:0\n\tactive_defrag_misses:0\n\tactive_defrag_key_hits:0\n\tactive_defrag_key_misses:0\n\ttracking_total_keys:0\n\ttracking_total_items:0\n\ttracking_total_prefixes:0\n\tunexpected_error_replies:0\n\tdump_payload_sanitizations:0\n\ttotal_reads_processed:15835400\n\ttotal_writes_processed:15835323\n\tio_threaded_reads_processed:0\n\tio_threaded_writes_processed:0\n\n\t\t# Replication\n\trole:master\n\tconnected_slaves:0\n\tmaster_replid:d4fc9b96fa0c5d3eb4c4444a394ba6e4e40cc0d5\n\tmaster_replid2:0000000000000000000000000000000000000000\n\tmaster_repl_offset:0\n\tsecond_repl_offset:-1\n\trepl_backlog_active:0\n\trepl_backlog_size:1048576\n\trepl_backlog_first_byte_offset:0\n\trepl_backlog_histlen:0\n\n\t\t# CPU\n\tused_cpu_sys:3574.527853\n\tused_cpu_user:13274.227145\n\tused_cpu_sys_children:0.000000\n\tused_cpu_user_children:0.000000\n\tused_cpu_sys_main_thread:3553.579738\n\tused_cpu_user_main_thread:13249.100447\n\n\t\t# Modules\n\n\t\t# Cluster\n\tcluster_enabled:0\n\n\t\t# Keyspace\n\tdb0:keys=1125326,expires=5,avg_ttl=321749445601195`\n\t\tinfo, err := checkRedisInfo(input)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Failed to parse redis info: %s\", err)\n\t\t}\n\t\tif info.redisVersion != \"6.1.240\" {\n\t\t\tt.Fatalf(\"Expect %s, got %q\", \"6.1.240\", info.redisVersion)\n\t\t}\n\t\tif info.aofEnabled {\n\t\t\tt.Fatalf(\"Expect %t, got %t\", false, true)\n\t\t}\n\t\tif info.maxMemoryPolicy != \"allkeys-lru\" {\n\t\t\tt.Fatalf(\"Expect %s, got %s\", \"allkeys-lru\", info.maxMemoryPolicy)\n\t\t}\n\t})\n}\n" }, { "path": "pkg/meta/interface.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nconst (\n\t// MaxVersion is the max of supported versions.\n\tMaxVersion = 1\n\t// ChunkBits is the size of a chunk.\n\tChunkBits = 26\n\t// ChunkSize is size of a chunk\n\tChunkSize = 1 << ChunkBits // 64M\n\t// DeleteSlice is a message to delete a slice from object store.\n\tDeleteSlice = 1000\n\t// CompactChunk is a message to compact a chunk in object store.\n\tCompactChunk = 1001\n\t// Rmr is a message to remove a directory recursively.\n\tRmr = 1002\n\t// LegacyInfo is a message to get the internal info for file or directory.\n\tLegacyInfo = 1003\n\t// FillCache is a message to build cache for target directories/files\n\tFillCache = 1004\n\t// InfoV2 is a message to get the internal info for file or directory.\n\tInfoV2 = 1005\n\t// Clone is a message to clone a file or dir from another.\n\tClone = 1006\n\t// OpSummary is a message to get tree summary of directories.\n\tOpSummary = 1007\n\t// CompactPath is a message to trigger compact\n\tCompactPath = 1008\n)\n\nconst (\n\tTypeFile = 1 // type for regular file\n\tTypeDirectory = 2 // type for directory\n\tTypeSymlink = 3 // type for symlink\n\tTypeFIFO = 4 // type for FIFO node\n\tTypeBlockDev = 5 // type for block device\n\tTypeCharDev = 6 // type for character device\n\tTypeSocket = 7 // type for socket\n)\n\nconst (\n\tRenameNoReplace = 1 << iota\n\tRenameExchange\n\tRenameWhiteout\n\t_renameReserved1\n\t_renameReserved2\n\tRenameRestore // internal\n)\n\nconst (\n\t// SetAttrMode is a mask to update a attribute of node\n\tSetAttrMode = 1 << iota\n\tSetAttrUID\n\tSetAttrGID\n\tSetAttrSize\n\tSetAttrAtime\n\tSetAttrMtime\n\tSetAttrCtime\n\tSetAttrAtimeNow\n\tSetAttrMtimeNow\n\tSetAttrCtimeNow\n\tSetAttrFlag = 1 << 15\n)\n\nconst (\n\tFlagImmutable = 1 << iota // same as Windows FILE_ATTRIBUTE_READONLY\n\tFlagAppend\n\tFlagWindowsHidden\n\tFlagWindowsSystem\n\tFlagWindowsArchive\n\tFlagSkipTrash // skip moving to .trash - Mapped to 's' in chattr\n)\n\nconst (\n\tQuotaSet uint8 = iota\n\tQuotaGet\n\tQuotaDel\n\tQuotaList\n\tQuotaCheck\n)\n\nconst MaxName = 255\nconst MaxSymlink = 4096\n\ntype Ino uint64\n\nconst RootInode Ino = 1\nconst TrashInode Ino = 0x7FFFFFFF10000000 // larger than vfs.minInternalNode\n\nconst RmrDefaultThreads = 50\n\nfunc (i Ino) String() string {\n\treturn strconv.FormatUint(uint64(i), 10)\n}\n\nfunc (i Ino) IsValid() bool {\n\treturn i >= RootInode\n}\n\nfunc (i Ino) IsTrash() bool {\n\treturn i >= TrashInode\n}\n\nfunc (i Ino) IsNormal() bool {\n\treturn i >= RootInode && i < TrashInode\n}\n\nvar TrashName = \".trash\"\n\ntype internalNode struct {\n\tinode Ino\n\tname string\n}\n\n// Type of control messages\nconst CPROGRESS = 0xFE // 16 bytes: progress increment\nconst CDATA = 0xFF // 4 bytes: data length\n\n// MsgCallback is a callback for messages from meta service.\ntype MsgCallback func(...interface{}) error\n\n// Attr represents attributes of a node.\ntype Attr struct {\n\tFlags uint8 // flags\n\tTyp uint8 // type of a node\n\tMode uint16 // permission mode\n\tUid uint32 // owner id\n\tGid uint32 // group id of owner\n\tRdev uint32 // device number\n\tAtime int64 // last access time\n\tMtime int64 // last modified time\n\tCtime int64 // last change time for meta\n\tAtimensec uint32 // nanosecond part of atime\n\tMtimensec uint32 // nanosecond part of mtime\n\tCtimensec uint32 // nanosecond part of ctime\n\tNlink uint32 // number of links (sub-directories or hardlinks)\n\tLength uint64 // length of regular file\n\n\tParent Ino // inode of parent; 0 means tracked by parentKey (for hardlinks)\n\tFull bool // the attributes are completed or not\n\tKeepCache bool // whether to keep the cached page or not\n\n\tAccessACL uint32 // access ACL id (identical ACL rules share the same access ACL ID.)\n\tDefaultACL uint32 // default ACL id (default ACL and the access ACL share the same cache and store)\n}\n\nfunc (attr *Attr) Marshal() []byte {\n\tsize := uint32(36 + 24 + 4 + 8)\n\tif attr.AccessACL|attr.DefaultACL != aclAPI.None {\n\t\tsize += 8\n\t}\n\tw := utils.NewBuffer(size)\n\tw.Put8(attr.Flags)\n\tw.Put16((uint16(attr.Typ) << 12) | (attr.Mode & 0xfff))\n\tw.Put32(attr.Uid)\n\tw.Put32(attr.Gid)\n\tw.Put64(uint64(attr.Atime))\n\tw.Put32(attr.Atimensec)\n\tw.Put64(uint64(attr.Mtime))\n\tw.Put32(attr.Mtimensec)\n\tw.Put64(uint64(attr.Ctime))\n\tw.Put32(attr.Ctimensec)\n\tw.Put32(attr.Nlink)\n\tw.Put64(attr.Length)\n\tw.Put32(attr.Rdev)\n\tw.Put64(uint64(attr.Parent))\n\tif attr.AccessACL+attr.DefaultACL > 0 {\n\t\tw.Put32(attr.AccessACL)\n\t\tw.Put32(attr.DefaultACL)\n\t}\n\tlogger.Tracef(\"attr: %+v -> %+v\", attr, w.Bytes())\n\treturn w.Bytes()\n}\n\nfunc (attr *Attr) Unmarshal(buf []byte) {\n\tif attr == nil || len(buf) == 0 {\n\t\treturn\n\t}\n\trb := utils.FromBuffer(buf)\n\tattr.Flags = rb.Get8()\n\tattr.Mode = rb.Get16()\n\tattr.Typ = uint8(attr.Mode >> 12)\n\tattr.Mode &= 0xfff\n\tattr.Uid = rb.Get32()\n\tattr.Gid = rb.Get32()\n\tattr.Atime = int64(rb.Get64())\n\tattr.Atimensec = rb.Get32()\n\tattr.Mtime = int64(rb.Get64())\n\tattr.Mtimensec = rb.Get32()\n\tattr.Ctime = int64(rb.Get64())\n\tattr.Ctimensec = rb.Get32()\n\tattr.Nlink = rb.Get32()\n\tattr.Length = rb.Get64()\n\tattr.Rdev = rb.Get32()\n\tif rb.Left() >= 8 {\n\t\tattr.Parent = Ino(rb.Get64())\n\t}\n\tattr.Full = true\n\tif rb.Left() >= 8 {\n\t\tattr.AccessACL = rb.Get32()\n\t\tattr.DefaultACL = rb.Get32()\n\t}\n\tlogger.Tracef(\"attr: %+v -> %+v\", buf, attr)\n}\n\nfunc typeToStatType(_type uint8) uint32 {\n\tswitch _type & 0x7F {\n\tcase TypeDirectory:\n\t\treturn syscall.S_IFDIR\n\tcase TypeSymlink:\n\t\treturn syscall.S_IFLNK\n\tcase TypeFile:\n\t\treturn syscall.S_IFREG\n\tcase TypeFIFO:\n\t\treturn syscall.S_IFIFO\n\tcase TypeSocket:\n\t\treturn syscall.S_IFSOCK\n\tcase TypeBlockDev:\n\t\treturn syscall.S_IFBLK\n\tcase TypeCharDev:\n\t\treturn syscall.S_IFCHR\n\tdefault:\n\t\tpanic(_type)\n\t}\n}\n\nfunc typeToString(_type uint8) string {\n\tswitch _type {\n\tcase TypeFile:\n\t\treturn \"regular\"\n\tcase TypeDirectory:\n\t\treturn \"directory\"\n\tcase TypeSymlink:\n\t\treturn \"symlink\"\n\tcase TypeFIFO:\n\t\treturn \"fifo\"\n\tcase TypeBlockDev:\n\t\treturn \"blockdev\"\n\tcase TypeCharDev:\n\t\treturn \"chardev\"\n\tcase TypeSocket:\n\t\treturn \"socket\"\n\tdefault:\n\t\treturn \"unknown\"\n\t}\n}\n\nfunc typeFromString(s string) uint8 {\n\tswitch s {\n\tcase \"regular\":\n\t\treturn TypeFile\n\tcase \"directory\":\n\t\treturn TypeDirectory\n\tcase \"symlink\":\n\t\treturn TypeSymlink\n\tcase \"fifo\":\n\t\treturn TypeFIFO\n\tcase \"blockdev\":\n\t\treturn TypeBlockDev\n\tcase \"chardev\":\n\t\treturn TypeCharDev\n\tcase \"socket\":\n\t\treturn TypeSocket\n\tdefault:\n\t\tpanic(s)\n\t}\n}\n\n// SMode is the file mode including type and unix permission.\nfunc (a *Attr) SMode() uint32 {\n\treturn typeToStatType(a.Typ) | uint32(a.Mode)\n}\n\n// Entry is an entry inside a directory.\ntype Entry struct {\n\tInode Ino\n\tName []byte\n\tAttr *Attr\n}\n\n// Slice is a slice of a chunk.\n// Multiple slices could be combined together as a chunk.\ntype Slice struct {\n\tId uint64\n\tSize uint32\n\tOff uint32\n\tLen uint32\n}\n\n// Summary represents the total number of files/directories and\n// total length of all files inside a directory.\ntype Summary struct {\n\tLength uint64\n\tSize uint64\n\tFiles uint64\n\tDirs uint64\n}\n\ntype TreeSummary struct {\n\tInode Ino\n\tPath string\n\tType uint8\n\tSize uint64\n\tFiles uint64\n\tDirs uint64\n\tChildren []*TreeSummary `json:\",omitempty\"`\n}\n\ntype SessionInfo struct {\n\tVersion string\n\tHostName string\n\tIPAddrs []string `json:\",omitempty\"`\n\tMountPoint string\n\tMountTime time.Time\n\tProcessID int\n}\n\ntype Flock struct {\n\tInode Ino\n\tOwner uint64\n\tLtype string\n}\n\ntype Plock struct {\n\tInode Ino\n\tOwner uint64\n\tRecords []plockRecord\n}\n\n// Session contains detailed information of a client session\ntype Session struct {\n\tSid uint64\n\tExpire time.Time\n\tSessionInfo\n\tSustained []Ino `json:\",omitempty\"`\n\tFlocks []Flock `json:\",omitempty\"`\n\tPlocks []Plock `json:\",omitempty\"`\n}\n\n// Meta is a interface for a meta service for file system.\ntype Meta interface {\n\t// Name of database\n\tName() string\n\t// Init is used to initialize a meta service.\n\tInit(format *Format, force bool) error\n\t// Shutdown close current database connections.\n\tShutdown() error\n\t// Reset cleans up all metadata, VERY DANGEROUS!\n\tReset() error\n\t// Load loads the existing setting of a formatted volume from meta service.\n\tLoad(checkVersion bool) (*Format, error)\n\t// NewSession creates or update client session.\n\tNewSession(record bool) error\n\t// CloseSession does cleanup and close the session.\n\tCloseSession() error\n\t// FlushSession flushes the status to meta service.\n\tFlushSession()\n\t// GetSession retrieves information of session with sid\n\tGetSession(sid uint64, detail bool) (*Session, error)\n\t// ListSessions returns all client sessions.\n\tListSessions() ([]*Session, error)\n\t// ScanDeletedObject scan deleted objects by customized scanner.\n\tScanDeletedObject(Context, trashSliceScan, pendingSliceScan, trashFileScan, pendingFileScan) error\n\t// ListLocks returns all locks of a inode.\n\tListLocks(ctx context.Context, inode Ino) ([]PLockItem, []FLockItem, error)\n\t// CleanStaleSessions cleans up sessions not active for more than 5 minutes\n\tCleanStaleSessions(ctx Context)\n\t// CleanupTrashBefore deletes all files in trash before the given time.\n\tCleanupTrashBefore(ctx Context, edge time.Time, increProgress func(int), stats *CleanupTrashStats) syscall.Errno\n\t// CleanupDetachedNodesBefore deletes all detached nodes before the given time.\n\tCleanupDetachedNodesBefore(ctx Context, edge time.Time, increProgress func())\n\n\t// StatFS returns summary statistics of a volume.\n\tStatFS(ctx Context, ino Ino, totalspace, availspace, iused, iavail *uint64) syscall.Errno\n\t// Access checks the access permission on given inode.\n\tAccess(ctx Context, inode Ino, modemask uint8, attr *Attr) syscall.Errno\n\t// Lookup returns the inode and attributes for the given entry in a directory.\n\tLookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr, checkPerm bool) syscall.Errno\n\t// Resolve fetches the inode and attributes for an entry identified by the given path.\n\t// ENOTSUP will be returned if there's no natural implementation for this operation or\n\t// if there are any symlink following involved.\n\tResolve(ctx Context, parent Ino, path string, inode *Ino, attr *Attr) syscall.Errno\n\t// GetAttr returns the attributes for given node.\n\tGetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno\n\t// SetAttr updates the attributes for given node.\n\tSetAttr(ctx Context, inode Ino, set uint16, sggidclearmode uint8, attr *Attr) syscall.Errno\n\t// Check setting attr is allowed or not\n\tCheckSetAttr(ctx Context, inode Ino, set uint16, attr Attr) syscall.Errno\n\t// Truncate changes the length for given file.\n\tTruncate(ctx Context, inode Ino, flags uint8, attrlength uint64, attr *Attr, skipPermCheck bool) syscall.Errno\n\t// Fallocate preallocate given space for given file.\n\tFallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, length *uint64) syscall.Errno\n\t// ReadLink returns the target of a symlink.\n\tReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno\n\t// Symlink creates a symlink in a directory with given name.\n\tSymlink(ctx Context, parent Ino, name string, path string, inode *Ino, attr *Attr) syscall.Errno\n\t// Mknod creates a node in a directory with given name, type and permissions.\n\tMknod(ctx Context, parent Ino, name string, _type uint8, mode uint16, cumask uint16, rdev uint32, path string, inode *Ino, attr *Attr) syscall.Errno\n\t// Mkdir creates a sub-directory with given name and mode.\n\tMkdir(ctx Context, parent Ino, name string, mode uint16, cumask uint16, copysgid uint8, inode *Ino, attr *Attr) syscall.Errno\n\t// Unlink removes a file entry from a directory.\n\t// The file will be deleted if it's not linked by any entries and not open by any sessions.\n\tUnlink(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno\n\t// BatchUnlink remove some file entries from the same directory\n\tBatchUnlink(ctx Context, parent Ino, entries []*Entry, count *uint64, skipCheckTrash bool) syscall.Errno\n\t// Rmdir removes an empty sub-directory.\n\tRmdir(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno\n\t// Rename move an entry from a source directory to another with given name.\n\t// The targeted entry will be overwrited if it's a file or empty directory.\n\t// For Hadoop, the target should not be overwritten.\n\tRename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode *Ino, attr *Attr) syscall.Errno\n\t// Link creates an entry for node.\n\tLink(ctx Context, inodeSrc, parent Ino, name string, attr *Attr) syscall.Errno\n\t// Readdir returns all entries for given directory, which include attributes if plus is true.\n\tReaddir(ctx Context, inode Ino, wantattr uint8, entries *[]*Entry) syscall.Errno\n\t// NewDirHandler returns a stream for directory entries.\n\tNewDirHandler(ctx Context, inode Ino, plus bool, initEntries []*Entry) (DirHandler, syscall.Errno)\n\t// Create creates a file in a directory with given name.\n\tCreate(ctx Context, parent Ino, name string, mode uint16, cumask uint16, flags uint32, inode *Ino, attr *Attr) syscall.Errno\n\t// Open checks permission on a node and track it as open.\n\tOpen(ctx Context, inode Ino, flags uint32, attr *Attr) syscall.Errno\n\t// Close a file.\n\tClose(ctx Context, inode Ino) syscall.Errno\n\t// Read returns the list of slices on the given chunk.\n\tRead(ctx Context, inode Ino, indx uint32, slices *[]Slice) syscall.Errno\n\t// NewSlice returns an id for new slice.\n\tNewSlice(ctx Context, id *uint64) syscall.Errno\n\t// Write put a slice of data on top of the given chunk.\n\tWrite(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno\n\t// InvalidateChunkCache invalidate chunk cache\n\tInvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno\n\t// CopyFileRange copies part of a file to another one.\n\tCopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno\n\t// GetParents returns a map of node parents (> 1 parents if hardlinked)\n\tGetParents(ctx Context, inode Ino) map[Ino]int\n\t// GetDirStat returns the space and inodes usage of a directory.\n\tGetDirStat(ctx Context, inode Ino) (stat *dirStat, st syscall.Errno)\n\n\t// GetXattr returns the value of extended attribute for given name.\n\tGetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno\n\t// ListXattr returns all extended attributes of a node.\n\tListXattr(ctx Context, inode Ino, dbuff *[]byte) syscall.Errno\n\t// SetXattr update the extended attribute of a node.\n\tSetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno\n\t// RemoveXattr removes the extended attribute of a node.\n\tRemoveXattr(ctx Context, inode Ino, name string) syscall.Errno\n\t// Flock tries to put a lock on given file.\n\tFlock(ctx Context, inode Ino, owner uint64, ltype uint32, block bool) syscall.Errno\n\t// Getlk returns the current lock owner for a range on a file.\n\tGetlk(ctx Context, inode Ino, owner uint64, ltype *uint32, start, end *uint64, pid *uint32) syscall.Errno\n\t// Setlk sets a file range lock on given file.\n\tSetlk(ctx Context, inode Ino, owner uint64, block bool, ltype uint32, start, end uint64, pid uint32) syscall.Errno\n\n\t// Compact all the chunks by merge small slices together\n\tCompactAll(ctx Context, threads int, bar *utils.Bar) syscall.Errno\n\t// Compact chunks for specified path\n\tCompact(ctx Context, inode Ino, concurrency int, preFunc, postFunc func()) syscall.Errno\n\n\t// ListSlices returns all slices used by all files.\n\tListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno\n\t// Remove all files and directories recursively.\n\t// count represents the number of attempted deletions of entries (even if failed).\n\tRemove(ctx Context, parent Ino, name string, skipTrash bool, numThreads int, count *uint64) syscall.Errno\n\t// Get summary of a node; for a directory it will accumulate all its child nodes\n\tGetSummary(ctx Context, inode Ino, summary *Summary, recursive bool, strict bool) syscall.Errno\n\t// GetTreeSummary returns a summary in tree structure\n\tGetTreeSummary(ctx Context, root *TreeSummary, depth, topN uint8, strict bool, updateProgress func(count uint64, bytes uint64)) syscall.Errno\n\t// Clone a file or directory\n\tClone(ctx Context, srcParentIno, srcIno, dstParentIno Ino, dstName string, cmode uint8, cumask uint16, concurrency uint8, count, total *uint64) syscall.Errno\n\t// GetPaths returns all paths of an inode\n\tGetPaths(ctx Context, inode Ino) []string\n\t// Check integrity of an absolute path and repair it if asked\n\tCheck(ctx Context, fpath string, opt *CheckOpt) error\n\t// Change root to a directory specified by subdir\n\tChroot(ctx Context, subdir string) syscall.Errno\n\t// chroot set the root directory by inode\n\tchroot(inode Ino)\n\t// Get a copy of the current format\n\tGetFormat() Format\n\n\t// OnMsg add a callback for the given message type.\n\tOnMsg(mtype uint32, cb MsgCallback)\n\t// OnReload register a callback for any change founded after reloaded.\n\tOnReload(func(new *Format))\n\n\tHandleQuota(ctx Context, cmd uint8, dpath string, uid uint32, gid uint32, quotas map[string]*Quota, strict, repair bool, create bool) error\n\t//Triggers a global user group quota scan\n\tScanUserGroupUsage(ctx Context) error\n\n\t// Dump the tree under root, which may be modified by checkRoot\n\tDumpMeta(w io.Writer, root Ino, threads int, keepSecret, fast, skipTrash bool) error\n\tLoadMeta(r io.Reader) error\n\n\tDumpMetaV2(ctx Context, w io.Writer, opt *DumpOption) error\n\tLoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error\n\n\t// getBase return the base engine.\n\tgetBase() *baseMeta\n\tInitMetrics(registerer prometheus.Registerer)\n\tInitSharedMetrics(registerer prometheus.Registerer)\n\n\tSetFacl(ctx Context, ino Ino, aclType uint8, n *aclAPI.Rule) syscall.Errno\n\tGetFacl(ctx Context, ino Ino, aclType uint8, n *aclAPI.Rule) syscall.Errno\n\n\t// kerberos\n\tStoreToken(ctx Context, token []byte) (id uint32, st syscall.Errno)\n\tUpdateToken(ctx Context, id uint32, token []byte) syscall.Errno\n\tLoadToken(ctx Context, id uint32) (token []byte, st syscall.Errno)\n\tDeleteTokens(ctx Context, ids []uint32) syscall.Errno\n\tListTokens(ctx Context) (tokens map[uint32][]byte, st syscall.Errno)\n}\n\ntype CheckOpt struct {\n\tRepair bool\n\tRecursive bool\n\tSyncDirStat bool\n\tRepairDirMode uint16\n\tShowProgress func(n int)\n\tSlices map[Ino][]Slice\n}\n\ntype CleanupTrashStats struct {\n\tDeletedFiles int64\n}\n\ntype Creator func(driver, addr string, conf *Config) (Meta, error)\n\nvar metaDrivers = make(map[string]Creator)\n\nfunc Register(name string, register Creator) {\n\tmetaDrivers[name] = register\n}\n\nfunc injectPasswordIntoURI(uri, password string) (string, error) {\n\tatIndex := strings.LastIndex(uri, \"@\")\n\tif atIndex == -1 {\n\t\treturn \"\", fmt.Errorf(\"invalid uri: %s\", uri)\n\t}\n\tdIndex := strings.Index(uri, \"://\") + 3\n\ts := strings.Split(uri[dIndex:atIndex], \":\")\n\n\tif len(s) > 2 {\n\t\treturn \"\", fmt.Errorf(\"invalid uri: %s\", uri)\n\t}\n\n\tif len(s) == 2 && s[1] != \"\" {\n\t\treturn uri, nil\n\t}\n\tpwd := url.UserPassword(\"\", password) // escape only password\n\treturn uri[:dIndex] + s[0] + pwd.String() + uri[atIndex:], nil\n}\n\nfunc readPasswordFromFile(filePath string) (string, error) {\n\tcontent, err := os.ReadFile(filePath)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to read password file %s: %w\", filePath, err)\n\t}\n\treturn strings.TrimSpace(string(content)), nil\n}\n\nfunc setPasswordFromEnv(uri string) (string, error) {\n\tvar password string\n\tvar err error\n\n\tif metaPassword := os.Getenv(\"META_PASSWORD\"); metaPassword != \"\" {\n\t\tpassword = metaPassword\n\t} else if passwordFile := os.Getenv(\"META_PASSWORD_FILE\"); passwordFile != \"\" {\n\t\tpassword, err = readPasswordFromFile(passwordFile)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t} else {\n\t\t// No password source available, return original URI\n\t\treturn uri, nil\n\t}\n\n\treturn injectPasswordIntoURI(uri, password)\n}\n\n// NewClient creates a Meta client for given uri.\nfunc NewClient(uri string, conf *Config) Meta {\n\tvar err error\n\tif !strings.Contains(uri, \"://\") {\n\t\turi = \"redis://\" + uri\n\t}\n\tp := strings.Index(uri, \"://\")\n\tif p < 0 {\n\t\tlogger.Fatalf(\"invalid uri: %s\", uri)\n\t}\n\tdriver := uri[:p]\n\tif driver == \"mysql\" || driver == \"postgres\" {\n\t\tif uri, err = setPasswordFromEnv(uri); err != nil {\n\t\t\tlogger.Fatalf(err.Error())\n\t\t}\n\t}\n\tlogger.Infof(\"Meta address: %s\", utils.RemovePassword(uri))\n\tf, ok := metaDrivers[driver]\n\tif !ok {\n\t\tlogger.Fatalf(\"Invalid meta driver: %s\", driver)\n\t}\n\tif conf == nil {\n\t\tconf = DefaultConf()\n\t} else {\n\t\tconf.SelfCheck()\n\t}\n\tm, err := f(driver, uri[p+3:], conf)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Meta %s is not available: %s\", utils.RemovePassword(uri), err)\n\t}\n\treturn m\n}\n" }, { "path": "pkg/meta/interface_test.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n)\n\nfunc Test_injectPasswordIntoURI(t *testing.T) {\n\tconst dbPasswd = \"dbPasswd\"\n\ttests := []struct {\n\t\turi string\n\t\twant string\n\t\twantErr bool\n\t}{\n\t\t//mysql\n\t\t{\n\t\t\turi: \"mysql://root:password@(127.0.0.1:3306)/juicefs\",\n\t\t\twant: \"mysql://root:password@(127.0.0.1:3306)/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"mysql://root:@(127.0.0.1:3306)/juicefs\",\n\t\t\twant: \"mysql://root:dbPasswd@(127.0.0.1:3306)/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"mysql://root@(127.0.0.1:3306)/juicefs\",\n\t\t\twant: \"mysql://root:dbPasswd@(127.0.0.1:3306)/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"mysql://root@@(127.0.0.1:3306)/juicefs\",\n\t\t\twant: \"mysql://root@:dbPasswd@(127.0.0.1:3306)/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t// no user is ok\n\t\t{\n\t\t\turi: \"mysql://:@(127.0.0.1:3306)/juicefs\",\n\t\t\twant: \"mysql://:dbPasswd@(127.0.0.1:3306)/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"mysql://:pwd@(127.0.0.1:3306)/juicefs\",\n\t\t\twant: \"mysql://:pwd@(127.0.0.1:3306)/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t//postgres\n\t\t{\n\t\t\turi: \"postgres://root:password@192.168.1.6:5432/juicefs\",\n\t\t\twant: \"postgres://root:password@192.168.1.6:5432/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"postgres://root:@192.168.1.6:5432/juicefs\",\n\t\t\twant: \"postgres://root:dbPasswd@192.168.1.6:5432/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"postgres://root@192.168.1.6:5432/juicefs\",\n\t\t\twant: \"postgres://root:dbPasswd@192.168.1.6:5432/juicefs\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"postgres://root@/pgtest?host=/tmp/pgsocket/&port=5433\",\n\t\t\twant: \"postgres://root:dbPasswd@/pgtest?host=/tmp/pgsocket/&port=5433\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\turi: \"postgres://@/pgtest?host=/tmp/pgsocket/&port=5433&user=pguser\",\n\t\t\twant: \"postgres://:dbPasswd@/pgtest?host=/tmp/pgsocket/&port=5433&user=pguser\",\n\t\t\twantErr: false,\n\t\t},\n\t\t// Error conditions\n\t\t{\n\t\t\turi: \"mysql://root(127.0.0.1:3306)/juicefs\", // missing @\n\t\t\twant: \"\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\turi: \"mysql://a:b:c:@(127.0.0.1:3306)/juicefs\",\n\t\t\twant: \"\",\n\t\t\twantErr: true,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tgot, err := injectPasswordIntoURI(tt.uri, dbPasswd)\n\n\t\t\tif tt.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"injectPasswordIntoURI() expected error but got none\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"injectPasswordIntoURI() unexpected error = %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif got != tt.want {\n\t\t\t\t\tt.Errorf(\"injectPasswordIntoURI() = %q, want %q\", got, tt.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc Test_setPasswordFromEnv(t *testing.T) {\n\ttempDir := t.TempDir()\n\tpasswordFile := filepath.Join(tempDir, \"password.txt\")\n\terr := os.WriteFile(passwordFile, []byte(\"filePassword\"), 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test password file: %v\", err)\n\t}\n\n\ttests := []struct {\n\t\tname string\n\t\tmetaPassword string\n\t\tmetaPasswordFile string\n\t\turi string\n\t\twant string\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"META_PASSWORD only\",\n\t\t\tmetaPassword: \"envPassword\",\n\t\t\turi: \"mysql://root@localhost/db\",\n\t\t\twant: \"mysql://root:envPassword@localhost/db\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"META_PASSWORD_FILE only\",\n\t\t\tmetaPasswordFile: passwordFile,\n\t\t\turi: \"mysql://root@localhost/db\",\n\t\t\twant: \"mysql://root:filePassword@localhost/db\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"META_PASSWORD takes precedence over META_PASSWORD_FILE\",\n\t\t\tmetaPassword: \"envPassword\",\n\t\t\tmetaPasswordFile: passwordFile,\n\t\t\turi: \"mysql://root@localhost/db\",\n\t\t\twant: \"mysql://root:envPassword@localhost/db\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"neither environment variable set\",\n\t\t\turi: \"mysql://root@localhost/db\",\n\t\t\twant: \"mysql://root@localhost/db\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"META_PASSWORD_FILE points to non-existent file\",\n\t\t\tmetaPasswordFile: \"/non/existent/file\",\n\t\t\turi: \"mysql://root@localhost/db\",\n\t\t\twant: \"\",\n\t\t\twantErr: true,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Clean environment\n\t\t\tdefer os.Unsetenv(\"META_PASSWORD\")\n\t\t\tdefer os.Unsetenv(\"META_PASSWORD_FILE\")\n\t\t\t// Just to be safe\n\t\t\tos.Unsetenv(\"META_PASSWORD\")\n\t\t\tos.Unsetenv(\"META_PASSWORD_FILE\")\n\n\t\t\t// Set environment variables as needed\n\t\t\tif tt.metaPassword != \"\" {\n\t\t\t\tos.Setenv(\"META_PASSWORD\", tt.metaPassword)\n\t\t\t}\n\t\t\tif tt.metaPasswordFile != \"\" {\n\t\t\t\tos.Setenv(\"META_PASSWORD_FILE\", tt.metaPasswordFile)\n\t\t\t}\n\n\t\t\tgot, err := setPasswordFromEnv(tt.uri)\n\n\t\t\tif tt.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"setPasswordFromEnv() expected error but got none\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"setPasswordFromEnv() unexpected error = %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif got != tt.want {\n\t\t\t\t\tt.Errorf(\"setPasswordFromEnv() = %q, want %q\", got, tt.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc Test_readPasswordFromFile(t *testing.T) {\n\t// Create temporary directory for test files\n\ttempDir := t.TempDir()\n\n\ttests := []struct {\n\t\tname string\n\t\tcontent string\n\t\tfilename string\n\t\tcreateFile bool\n\t\twant string\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"valid password file\",\n\t\t\tcontent: \"mypassword\",\n\t\t\tfilename: \"password.txt\",\n\t\t\tcreateFile: true,\n\t\t\twant: \"mypassword\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"password with leading and trailing whitespace\",\n\t\t\tcontent: \"\\n mypassword \\n\\t\",\n\t\t\tfilename: \"password_with_spaces.txt\",\n\t\t\tcreateFile: true,\n\t\t\twant: \"mypassword\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"empty file\",\n\t\t\tcontent: \"\",\n\t\t\tfilename: \"empty.txt\",\n\t\t\tcreateFile: true,\n\t\t\twant: \"\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"complex password with special characters\",\n\t\t\tcontent: \"pa$$w0rd!@#\",\n\t\t\tfilename: \"complex.txt\",\n\t\t\tcreateFile: true,\n\t\t\twant: \"pa$$w0rd!@#\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"file does not exist\",\n\t\t\tcontent: \"\",\n\t\t\tfilename: \"nonexistent.txt\",\n\t\t\tcreateFile: false,\n\t\t\twant: \"\",\n\t\t\twantErr: true,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar filePath string\n\t\t\tif tt.createFile {\n\t\t\t\tfilePath = filepath.Join(tempDir, tt.filename)\n\t\t\t\terr := os.WriteFile(filePath, []byte(tt.content), 0600)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Failed to create test file %s: %v\", filePath, err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfilePath = filepath.Join(tempDir, tt.filename)\n\t\t\t}\n\n\t\t\tgot, err := readPasswordFromFile(filePath)\n\n\t\t\tif tt.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"readPasswordFromFile() expected error but got none\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"readPasswordFromFile() unexpected error = %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif got != tt.want {\n\t\t\t\t\tt.Errorf(\"readPasswordFromFile() = %q, want %q\", got, tt.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "pkg/meta/load_dump_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/sirupsen/logrus\"\n\t\"golang.org/x/text/encoding/simplifiedchinese\"\n\t\"golang.org/x/text/transform\"\n)\n\nconst sampleFile = \"metadata.sample\"\nconst subSampleFile = \"metadata-sub.sample\"\n\nfunc TestEscape(t *testing.T) {\n\tcases := []struct {\n\t\tvalue []rune\n\t\tgbkStart, gbkEnd int\n\t}{\n\t\t{value: []rune(\"%1F果汁数据科技有限公司%2B\"), gbkStart: 0, gbkEnd: 0},\n\t\t{value: []rune(\"果汁数据科技有限公司%1F\"), gbkStart: 0, gbkEnd: 1},\n\t\t{value: []rune(\"果汁数据科技有限公司\"), gbkStart: 1, gbkEnd: 2},\n\t\t{value: []rune(\"果汁数据科技有限公司\"), gbkStart: 1, gbkEnd: 4},\n\t\t{value: []rune(\"果汁数据科技有限公司\"), gbkStart: 5, gbkEnd: 10},\n\t\t{value: []rune(\"果汁数据科技有限公司\"), gbkStart: 0, gbkEnd: 10},\n\t\t{value: []rune(\"GBK果汁数据科技有限公司文件\"), gbkStart: 0, gbkEnd: 15},\n\t\t{value: []rune(\"%果汁数据科%技有限公司%\"), gbkStart: 1, gbkEnd: 4},\n\t\t{value: []rune(\"\\\"果汁数据科\\\"技有限公司%\"), gbkStart: 1, gbkEnd: 4},\n\t\t{value: []rune(\"\\\\果汁数\\\\据科技有限公司\"), gbkStart: 1, gbkEnd: 4},\n\t}\n\tfor _, c := range cases {\n\t\tvar v []byte\n\t\tprefix := c.value[:c.gbkStart]\n\t\tmiddle := c.value[c.gbkStart:c.gbkEnd]\n\t\tsuffix := c.value[c.gbkEnd:]\n\t\tgbk, err := Utf8ToGbk([]byte(string(middle)))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Utf8ToGbk error: %v\", err)\n\t\t}\n\t\tv = append(v, []byte(string(prefix))...)\n\t\tv = append(v, gbk...)\n\t\tv = append(v, []byte(string(suffix))...)\n\t\ts := escape(string(v))\n\t\tt.Log(\"escape value: \", s)\n\t\tr := unescape(s)\n\t\tif !bytes.Equal(r, v) {\n\t\t\tt.Fatalf(\"expected %v, but got %v\", v, r)\n\t\t}\n\t}\n}\n\nfunc Utf8ToGbk(s []byte) ([]byte, error) {\n\treader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())\n\td, e := io.ReadAll(reader)\n\tif e != nil {\n\t\treturn nil, e\n\t}\n\treturn d, nil\n}\n\nfunc GbkToUtf8(s []byte) ([]byte, error) {\n\treader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())\n\td, e := io.ReadAll(reader)\n\tif e != nil {\n\t\treturn nil, e\n\t}\n\treturn d, nil\n}\n\nfunc checkMeta(t *testing.T, m Meta) {\n\tif _, err := m.Load(true); err != nil {\n\t\tt.Fatalf(\"load setting: %s\", err)\n\t}\n\n\tcounters := map[string]int64{\n\t\t\"usedSpace\": 115392512,\n\t\t\"totalInodes\": 14,\n\t\t\"nextInode\": 35,\n\t\t\"nextChunk\": 9,\n\t\t\"nextSession\": 0,\n\t\t\"nextTrash\": 1,\n\t}\n\tfor name, expect := range counters {\n\t\tval, err := m.getBase().en.getCounter(name)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"get counter %s: %s\", name, err)\n\t\t}\n\t\tif m.Name() == \"redis\" && (name == \"nextChunk\" || name == \"nextInode\") {\n\t\t\texpect--\n\t\t}\n\t\tif val != expect {\n\t\t\tt.Fatalf(\"counter %s: %d != %d\", name, val, expect)\n\t\t}\n\t}\n\n\tctx := Background()\n\tvar entries []*Entry\n\tif st := m.Readdir(ctx, 1, 1, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t} else if len(entries) != 11 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\n\tvar expectedStat dirStat\n\tfor _, entry := range entries {\n\t\tfname := string(entry.Name)\n\t\tif strings.HasPrefix(fname, \"GBK\") {\n\t\t\tif utf8, err := GbkToUtf8(entry.Name); err != nil || string(utf8) != \"GBK果汁数据科技有限公司文件\" {\n\t\t\t\tt.Fatalf(\"load GBK file error: %s\", string(utf8))\n\t\t\t}\n\t\t}\n\t\tif strings.HasPrefix(fname, \"UTF8\") && fname != \"UTF8果汁数据科技有限公司目录\" && fname != \"UTF8果汁数据科技有限公司文件\" {\n\t\t\tt.Fatalf(\"load entries error: %s\", fname)\n\t\t}\n\t\tif string(entry.Name) != \".\" && string(entry.Name) != \"..\" {\n\t\t\tvar length uint64\n\t\t\tif entry.Attr.Typ == TypeFile {\n\t\t\t\tlength = entry.Attr.Length\n\t\t\t}\n\t\t\texpectedStat.inodes++\n\t\t\texpectedStat.length += int64(length)\n\t\t\texpectedStat.space += align4K(length)\n\t\t}\n\t}\n\n\tstat, st := m.(engine).doGetDirStat(ctx, 1, false)\n\tif st != 0 {\n\t\tt.Fatalf(\"get dir stat: %s\", st)\n\t}\n\tif stat == nil {\n\t\tt.Fatalf(\"get dir stat: nil\")\n\t}\n\tif *stat != expectedStat {\n\t\tt.Fatalf(\"expected: %v, but got: %v\", expectedStat, *stat)\n\t}\n\n\tvar summary Summary\n\tif st = m.GetSummary(ctx, 1, &summary, true, true); st != 0 {\n\t\tt.Fatalf(\"get summary: %s\", st)\n\t}\n\texpectedQuota := Quota{\n\t\tMaxInodes: 100,\n\t\tMaxSpace: 1 << 30,\n\t\tUsedSpace: int64(summary.Size) - align4K(0),\n\t\tUsedInodes: int64(summary.Dirs+summary.Files) - 1,\n\t}\n\n\tquota, err := m.(engine).doGetQuota(ctx, DirQuotaType, 1)\n\tif err != nil {\n\t\tt.Fatalf(\"get quota: %s\", err)\n\t}\n\tif quota == nil {\n\t\tt.Fatalf(\"get quota: nil\")\n\t}\n\tif *quota != expectedQuota {\n\t\tt.Fatalf(\"expected: %v, but got: %v\", expectedQuota, *quota)\n\t}\n\n\tattr := &Attr{}\n\tif st := m.GetAttr(ctx, 2, attr); st != 0 {\n\t\tt.Fatalf(\"getattr: %s\", st)\n\t}\n\tif attr.Nlink != 1 || attr.Length != 24 {\n\t\tt.Fatalf(\"nlink: %d, length: %d\", attr.Nlink, attr.Length)\n\t}\n\n\tif attr.Flags != 128 {\n\t\tt.Fatalf(\"expect the flags euqal 128, but actual is: %d\", attr.Flags)\n\t}\n\n\tif attr.AccessACL == 0 || attr.DefaultACL == 0 {\n\t\tt.Fatalf(\"expect ACL not 0, but actual is: %d, %d\", attr.AccessACL, attr.DefaultACL)\n\t}\n\n\tar := &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, 2, aclAPI.TypeAccess, ar); st != 0 {\n\t\tt.Fatalf(\"get access acl: %s\", st)\n\t}\n\tar2 := &aclAPI.Rule{\n\t\tOwner: 6,\n\t\tGroup: 4,\n\t\tMask: 4,\n\t\tOther: 4,\n\t\tNamedUsers: []aclAPI.Entry{\n\t\t\t{Id: 1, Perm: 6},\n\t\t\t{Id: 2, Perm: 7},\n\t\t},\n\t\tNamedGroups: nil,\n\t}\n\tif !bytes.Equal(ar.Encode(), ar2.Encode()) {\n\t\tt.Fatalf(\"access acl: %v != %v\", ar, ar2)\n\t}\n\n\tdr := &aclAPI.Rule{}\n\tif st := m.GetFacl(ctx, 2, aclAPI.TypeDefault, dr); st != 0 {\n\t\tt.Fatalf(\"get default acl: %s\", st)\n\t}\n\tdr2 := &aclAPI.Rule{\n\t\tOwner: 7,\n\t\tGroup: 5,\n\t\tMask: 5,\n\t\tOther: 5,\n\t\tNamedUsers: nil,\n\t\tNamedGroups: []aclAPI.Entry{\n\t\t\t{Id: 3, Perm: 6},\n\t\t\t{Id: 4, Perm: 7},\n\t\t},\n\t}\n\tif !bytes.Equal(dr.Encode(), dr2.Encode()) {\n\t\tt.Fatalf(\"default acl: %v != %v\", dr, dr2)\n\t}\n\n\tvar slices []Slice\n\tif st := m.Read(ctx, 2, 0, &slices); st != 0 {\n\t\tt.Fatalf(\"read chunk: %s\", st)\n\t}\n\tif len(slices) != 1 || slices[0].Id != 4 || slices[0].Size != 24 {\n\t\tt.Fatalf(\"slices: %v\", slices)\n\t}\n\tif st := m.GetAttr(ctx, 4, attr); st != 0 || attr.Nlink != 2 { // hard link\n\t\tt.Fatalf(\"getattr: %s, %d\", st, attr.Nlink)\n\t}\n\tif ps := m.GetParents(ctx, 4); len(ps) != 2 || ps[1] != 1 || ps[3] != 1 {\n\t\tt.Fatalf(\"getparents: %+v != {1:1, 3:1}\", ps)\n\t}\n\tvar target []byte\n\n\tif st := m.ReadLink(ctx, 5, &target); st == 0 { // symlink\n\t\tif utf8, err := GbkToUtf8(target); err != nil || string(utf8) != \"GBK果汁数据科技有限公司文件\" {\n\t\t\tt.Fatalf(\"readlink: %s, %s\", st, target)\n\t\t}\n\t} else {\n\t\tt.Fatalf(\"readlink: %s, %s\", st, target)\n\t}\n\n\tvar value []byte\n\tif st := m.GetXattr(ctx, 2, \"k\", &value); st != 0 || string(value) != \"v\" {\n\t\tt.Fatalf(\"getxattr: %s %v\", st, value)\n\t}\n\tif st := m.GetXattr(ctx, 3, \"dk\", &value); st != 0 || string(value) != \"果汁%25\" {\n\t\tt.Fatalf(\"getxattr: %s %v\", st, value)\n\t}\n}\n\nfunc testLoadSub(t *testing.T, uri, fname string) {\n\tm := NewClient(uri, nil)\n\tif err := m.Reset(); err != nil {\n\t\tt.Fatalf(\"reset meta: %s\", err)\n\t}\n\tfp, err := os.Open(fname)\n\tif err != nil {\n\t\tt.Fatalf(\"open file: %s\", fname)\n\t}\n\tdefer fp.Close()\n\tif err = m.LoadMeta(fp); err != nil {\n\t\tt.Fatalf(\"load meta: %s\", err)\n\t}\n\n\tvar entries []*Entry\n\tif st := m.Readdir(Background(), 1, 0, &entries); st != 0 {\n\t\tt.Fatalf(\"readdir: %s\", st)\n\t} else if len(entries) != 4 {\n\t\tt.Fatalf(\"entries: %d\", len(entries))\n\t}\n\tfor _, entry := range entries {\n\t\tfname := string(entry.Name)\n\t\tif fname != \".\" && fname != \"..\" && fname != \"big\" && fname != \"f11\" {\n\t\t\tt.Fatalf(\"invalid entry name: %s\", fname)\n\t\t}\n\t}\n}\n\nfunc testDump(t *testing.T, m Meta, root Ino, expect, result string) {\n\tfp, err := os.OpenFile(result, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)\n\tif err != nil {\n\t\tt.Fatalf(\"open file %s: %s\", result, err)\n\t}\n\tdefer fp.Close()\n\tif _, err = m.Load(true); err != nil {\n\t\tt.Fatalf(\"load setting: %s\", err)\n\t}\n\tif err = m.DumpMeta(fp, root, 1, false, true, false); err != nil {\n\t\tt.Fatalf(\"dump meta: %s\", err)\n\t}\n\tcmd := exec.Command(\"diff\", expect, result)\n\tif out, err := cmd.Output(); err != nil {\n\t\tt.Fatalf(\"diff %s %s: %s\", expect, result, out)\n\t}\n\tfp.Seek(0, 0)\n\tif err = m.DumpMeta(fp, root, 10, false, false, false); err != nil {\n\t\tt.Fatalf(\"dump meta: %s\", err)\n\t}\n\tcmd = exec.Command(\"diff\", expect, result)\n\tif out, err := cmd.Output(); err != nil {\n\t\tt.Fatalf(\"diff %s %s: %s\", expect, result, out)\n\t}\n}\n\nfunc testLoadDump(t *testing.T, name, addr string) {\n\tt.Run(\"Metadata Engine: \"+name, func(t *testing.T) {\n\t\tm := testLoad(t, addr, sampleFile, false)\n\t\ttestDump(t, m, 1, sampleFile, \"test.dump\")\n\t\tm.Shutdown()\n\t\tconf := DefaultConf()\n\t\tconf.Subdir = \"d1\"\n\t\tm = NewClient(addr, conf)\n\t\t_ = m.Chroot(Background(), \"d1\")\n\t\ttestDump(t, m, 1, subSampleFile, \"test_subdir.dump\")\n\t\ttestDump(t, m, 0, sampleFile, \"test.dump\")\n\t\t_ = m.Shutdown()\n\t\ttestLoadSub(t, addr, subSampleFile)\n\t})\n}\n\nfunc TestLoadDump(t *testing.T) { //skip mutate\n\ttestLoadDump(t, \"redis\", \"redis://127.0.0.1/10\")\n\t// testLoadDump(t, \"mysql\", \"mysql://root:@/dev\")\n\ttestLoadDump(t, \"badger\", \"badger://jfs-load-dump\")\n\ttestLoadDump(t, \"tikv\", \"tikv://127.0.0.1:2379/jfs-load-dump\")\n}\n\nfunc testDumpV2(t *testing.T, m Meta, result string, opt *DumpOption) {\n\tif opt == nil {\n\t\topt = &DumpOption{Threads: 10, KeepSecret: true}\n\t}\n\tfp, err := os.OpenFile(result, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)\n\tif err != nil {\n\t\tt.Fatalf(\"open file %s: %s\", result, err)\n\t}\n\tdefer fp.Close()\n\tif _, err = m.Load(true); err != nil {\n\t\tt.Fatalf(\"load setting: %s\", err)\n\t}\n\tif err = m.DumpMetaV2(Background(), fp, opt); err != nil {\n\t\tt.Fatalf(\"dump meta: %s\", err)\n\t}\n\tfp.Sync()\n}\n\nfunc testLoad(t *testing.T, uri, fname string, v2 bool) Meta {\n\tm := NewClient(uri, nil)\n\tif err := m.Reset(); err != nil {\n\t\tt.Fatalf(\"reset meta: %s\", err)\n\t}\n\tfp, err := os.Open(fname)\n\tif err != nil {\n\t\tt.Fatalf(\"open file: %s\", fname)\n\t}\n\tdefer fp.Close()\n\tif v2 {\n\t\tif err = m.LoadMetaV2(Background(), fp, &LoadOption{Threads: 10}); err != nil {\n\t\t\tt.Fatalf(\"load meta: %s\", err)\n\t\t}\n\t} else {\n\t\tif err = m.LoadMeta(fp); err != nil {\n\t\t\tt.Fatalf(\"load meta: %s\", err)\n\t\t}\n\t}\n\tcheckMeta(t, m)\n\treturn m\n}\n\nfunc testLoadDumpV2(t *testing.T, name, addr1, addr2 string) {\n\tt.Run(\"Metadata Engine: \"+name, func(t *testing.T) {\n\t\tstart := time.Now()\n\t\tm := testLoad(t, addr1, sampleFile, false)\n\t\tt.Logf(\"load meta: %v\", time.Since(start))\n\t\tstart = time.Now()\n\t\ttestDumpV2(t, m, fmt.Sprintf(\"%s.dump\", name), nil)\n\t\tm.Shutdown()\n\t\tt.Logf(\"dump meta v2: %v\", time.Since(start))\n\t\tstart = time.Now()\n\t\tm = testLoad(t, addr2, fmt.Sprintf(\"%s.dump\", name), true)\n\t\tm.Shutdown()\n\t\tt.Logf(\"load meta v2: %v\", time.Since(start))\n\t})\n}\n\nfunc testLoadOtherEngine(t *testing.T, src, dst, dstAddr string) {\n\tt.Run(fmt.Sprintf(\"Load %s to %s\", src, dst), func(t *testing.T) {\n\t\tm := testLoad(t, dstAddr, fmt.Sprintf(\"%s.dump\", src), true)\n\t\tm.Shutdown()\n\t})\n}\n\nfunc TestLoadDumpV2(t *testing.T) {\n\tlogger.SetLevel(logrus.DebugLevel)\n\n\tengines := map[string][]string{\n\t\t\"sqlite3\": {\"sqlite3://dev.db\", \"sqlite3://dev2.db\"},\n\t\t// \"mysql\": {\"mysql://root:@/dev\", \"mysql://root:@/dev2\"},\n\t\t\"redis\": {\"redis://127.0.0.1:6379/2\", \"redis://127.0.0.1:6379/3\"},\n\t\t\"badger\": {\"badger://\" + path.Join(t.TempDir(), \"jfs-load-duimp-testdb-bk1\"), \"badger://\" + path.Join(t.TempDir(), \"jfs-load-duimp-testdb-bk2\")},\n\t\t// \"tikv\": {\"tikv://127.0.0.1:2379/jfs-load-dump-1\", \"tikv://127.0.0.1:2379/jfs-load-dump-2\"},\n\t}\n\n\tfor name, addrs := range engines {\n\t\ttestLoadDumpV2(t, name, addrs[0], addrs[1])\n\t\ttestSecretAndTrash(t, addrs[0], addrs[1])\n\t}\n\n\tfor src := range engines {\n\t\tfor dst, dstAddr := range engines {\n\t\t\tif src == dst {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttestLoadOtherEngine(t, src, dst, dstAddr[1])\n\t\t}\n\t}\n}\n\nfunc TestLoadDumpSlow(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SKIP_NON_CORE\") == \"true\" {\n\t\tt.Skipf(\"skip non-core test\")\n\t}\n\ttestLoadDump(t, \"redis cluster\", \"redis://127.0.0.1:7001/10\")\n\ttestLoadDump(t, \"sqlite\", \"sqlite3://\"+path.Join(t.TempDir(), \"jfs-load-dump-test.db\"))\n\ttestLoadDump(t, \"badger\", \"badger://\"+path.Join(t.TempDir(), \"jfs-load-duimp-testdb\"))\n\ttestLoadDump(t, \"etcd\", fmt.Sprintf(\"etcd://%s/jfs-load-dump\", os.Getenv(\"ETCD_ADDR\")))\n\ttestLoadDump(t, \"postgres\", \"postgres://localhost:5432/test?sslmode=disable\")\n}\n\nfunc TestLoadDump_MemKV(t *testing.T) {\n\tt.Run(\"Metadata Engine: memkv\", func(t *testing.T) {\n\t\t_ = os.Remove(settingPath)\n\t\tm := testLoad(t, \"memkv://test/jfs\", sampleFile, false)\n\t\ttestDump(t, m, 1, sampleFile, \"test.dump\")\n\t})\n\tt.Run(\"Metadata Engine: memkv; --SubDir d1 \", func(t *testing.T) {\n\t\t_ = os.Remove(settingPath)\n\t\tm := testLoad(t, \"memkv://user:pass@test/jfs\", sampleFile, false)\n\t\tif kvm, ok := m.(*kvMeta); ok { // memkv will be empty if created again\n\t\t\tif st := kvm.Chroot(Background(), \"d1\"); st != 0 {\n\t\t\t\tt.Fatalf(\"Chroot to subdir d1: %s\", st)\n\t\t\t}\n\t\t}\n\t\ttestDump(t, m, 1, subSampleFile, \"test_subdir.dump\")\n\t\ttestDump(t, m, 0, sampleFile, \"test.dump\")\n\t\t_ = os.Remove(settingPath)\n\t\ttestLoadSub(t, \"memkv://user:pass@test/jfs\", subSampleFile)\n\t})\n}\n\nfunc testSecretAndTrash(t *testing.T, addr, addr2 string) {\n\tm := testLoad(t, addr, sampleFile, false)\n\ttestDumpV2(t, m, \"sqlite-secret.dump\", &DumpOption{Threads: 10, KeepSecret: true})\n\tm2 := testLoad(t, addr2, \"sqlite-secret.dump\", true)\n\tif m2.GetFormat().EncryptKey != m.GetFormat().EncryptKey {\n\t\tt.Fatalf(\"encrypt key not valid: %s\", m2.GetFormat().EncryptKey)\n\t}\n\ttestDumpV2(t, m, \"sqlite-non-secret.dump\", &DumpOption{Threads: 10, KeepSecret: false})\n\tm2.Shutdown()\n\n\tm2 = testLoad(t, addr2, \"sqlite-non-secret.dump\", true)\n\tif m2.GetFormat().EncryptKey != \"removed\" {\n\t\tt.Fatalf(\"encrypt key not valid: %s\", m2.GetFormat().EncryptKey)\n\t}\n\n\t// trash\n\ttrashs := map[Ino]uint64{\n\t\t27: 11,\n\t\t29: 10485760,\n\t}\n\tcnt := 0\n\tm2.getBase().scanTrashFiles(Background(), func(inode Ino, size uint64, ts time.Time) (clean bool, err error) {\n\t\tcnt++\n\t\tif tSize, ok := trashs[inode]; !ok || size != tSize {\n\t\t\tt.Fatalf(\"trash file: %d %d\", inode, size)\n\t\t}\n\t\treturn false, nil\n\t})\n\tif cnt != len(trashs) {\n\t\tt.Fatalf(\"trash count: %d != %d\", cnt, len(trashs))\n\t}\n\n\tm.Shutdown()\n\tm2.Shutdown()\n}\n\n/*\nfunc BenchmarkLoadDumpV2(b *testing.B) {\n\tlogrus.SetLevel(logrus.DebugLevel)\n\tb.ReportAllocs()\n\tengines := map[string]string{\n\t\t\"mysql\": \"mysql://root:@/dev\",\n\t\t\"redis\": \"redis://127.0.0.1:6379/2\",\n\t\t\"tikv\": \"tikv://127.0.0.1:2379/jfs-load-dump-1\",\n\t}\n\n\tsample := \"../../1M_files_in_one_dir.dump\"\n\tfor name, addr := range engines {\n\t\tm := NewClient(addr, nil)\n\t\tdefer func() {\n\t\t\tm.Reset()\n\t\t\tm.Shutdown()\n\t\t}()\n\t\tb.Run(\"Load \"+name, func(b *testing.B) {\n\t\t\tif err := m.Reset(); err != nil {\n\t\t\t\tb.Fatalf(\"reset meta: %s\", err)\n\t\t\t}\n\t\t\tfp, err := os.Open(sample)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"open file: %s\", sample)\n\t\t\t}\n\t\t\tdefer fp.Close()\n\n\t\t\tb.ResetTimer()\n\t\t\tif err = m.LoadMeta(fp); err != nil {\n\t\t\t\tb.Fatalf(\"load meta: %s\", err)\n\t\t\t}\n\t\t})\n\n\t\tb.Run(\"Dump \"+name, func(b *testing.B) {\n\t\t\tpath := fmt.Sprintf(\"%s.v1.dump\", name)\n\t\t\tfp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"open file %s: %s\", path, err)\n\t\t\t}\n\t\t\tdefer fp.Close()\n\t\t\tif _, err = m.Load(true); err != nil {\n\t\t\t\tb.Fatalf(\"load setting: %s\", err)\n\t\t\t}\n\n\t\t\tb.ResetTimer()\n\t\t\tif err = m.DumpMeta(fp, RootInode, 10, true, true, false); err != nil {\n\t\t\t\tb.Fatalf(\"dump meta: %s\", err)\n\t\t\t}\n\t\t\tfp.Sync()\n\t\t})\n\n\t\tb.Run(\"DumpV2 \"+name, func(b *testing.B) {\n\t\t\tpath := fmt.Sprintf(\"%s.v2.dump\", name)\n\t\t\tfp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"open file %s: %s\", path, err)\n\t\t\t}\n\t\t\tdefer fp.Close()\n\n\t\t\tb.ResetTimer()\n\t\t\tif err = m.DumpMetaV2(Background(), fp, &DumpOption{Threads: 10}); err != nil {\n\t\t\t\tb.Fatalf(\"dump meta: %s\", err)\n\t\t\t}\n\t\t\tfp.Sync()\n\n\t\t\tb.StopTimer()\n\t\t\tbak := &bakFormat{}\n\t\t\tfp2, err := os.Open(path)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"open file: %s\", path)\n\t\t\t}\n\t\t\tdefer fp2.Close()\n\t\t\tfooter, err := bak.readFooter(fp2)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"read footer: %s\", err)\n\t\t\t}\n\t\t\tfor name, info := range footer.msg.Infos {\n\t\t\t\tb.Logf(\"segment: %s, num: %d\", name, info.Num)\n\t\t\t}\n\t\t\tb.StartTimer()\n\t\t})\n\n\t\tb.Run(\"LoadV2 \"+name, func(b *testing.B) {\n\t\t\tpath := fmt.Sprintf(\"%s.v2.dump\", name)\n\t\t\tif err := m.Reset(); err != nil {\n\t\t\t\tb.Fatalf(\"reset meta: %s\", err)\n\t\t\t}\n\t\t\tfp, err := os.Open(path)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"open file: %s\", path)\n\t\t\t}\n\t\t\tdefer fp.Close()\n\n\t\t\tb.ResetTimer()\n\t\t\tif err = m.LoadMetaV2(Background(), fp, &LoadOption{Threads: 10}); err != nil {\n\t\t\t\tb.Fatalf(\"load meta: %s\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n*/\n" }, { "path": "pkg/meta/lua_scripts.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n// nolint\npackage meta\n\nconst scriptLookup = `\nlocal buf = redis.call('HGET', KEYS[1], KEYS[2])\nif not buf then\n error(\"ENOENT\")\nend\nlocal ino = struct.unpack(\">I8\", string.sub(buf, 2))\n-- double float has 52 significant bits\nif ino > 4503599627370495 then\n error(\"ENOTSUP\")\nend\nreturn {ino, redis.call('GET', \"i\" .. string.format(\"%.f\", ino))}\n`\n\nconst scriptResolve = `\nlocal function unpack_attr(buf)\n local x = {}\n x.flags, x.mode, x.uid, x.gid = struct.unpack(\">BHI4I4\", string.sub(buf, 0, 11))\n x.type = math.floor(x.mode / 4096) % 8\n x.mode = x.mode % 4096\n return x\nend\n\nlocal function get_attr(ino)\n local encoded_attr = redis.call('GET', \"i\" .. string.format(\"%.f\", ino))\n if not encoded_attr then\n error(\"ENOENT\")\n end\n return unpack_attr(encoded_attr)\nend\n\nlocal function lookup(parent, name)\n local buf = redis.call('HGET', \"d\" .. string.format(\"%.f\", parent), name)\n if not buf then\n error(\"ENOENT\")\n end\n return struct.unpack(\">BI8\", buf)\nend\n\nlocal function has_value(tab, val)\n for index, value in ipairs(tab) do\n if value == val then\n return true\n end\n end\n return false\nend\n\nlocal function can_access(ino, uid, gids)\n if uid == 0 then\n return true\n end\n\n local attr = get_attr(ino)\n local mode = 0\n if attr.uid == uid then\n mode = math.floor(attr.mode / 64) % 8\n elseif has_value(gids, tostring(attr.gid)) then\n mode = math.floor(attr.mode / 8) % 8\n else\n mode = attr.mode % 8\n end\n return mode % 2 == 1\nend\n\nlocal function resolve(parent, path, uid, gids)\n local _maxIno = 4503599627370495\n local _type = 2\n for name in string.gmatch(path, \"[^/]+\") do\n if _type == 3 or parent > _maxIno then\n error(\"ENOTSUP\")\n elseif _type ~= 2 then\n error(\"ENOTDIR\")\n elseif parent > 1 and not can_access(parent, uid, gids) then \n error(\"EACCESS\")\n end\n _type, parent = lookup(parent, name)\n end\n if parent > _maxIno then\n error(\"ENOTSUP\")\n end\n return {parent, redis.call('GET', \"i\" .. string.format(\"%.f\", parent))}\nend\n\nreturn resolve(tonumber(KEYS[1]), KEYS[2], tonumber(KEYS[3]), ARGV)\n`\n" }, { "path": "pkg/meta/metadata-sub.sample", "content": "{\n \"Setting\": {\n \"Name\": \"load-dump-test\",\n \"UUID\": \"faa27c8f-edab-4791-a4e0-1620b732b343\",\n \"Storage\": \"file\",\n \"Bucket\": \"/Users/juicefs/.juicefs/local/\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"EncryptKey\": \"AQSttslKOSE/hQT/gmaMniCsdPF8JdPRfoYK6zFkdUOnifYwBA==\",\n \"KeyEncrypted\": true,\n \"TrashDays\": 1,\n \"MetaVersion\": 1,\n \"EnableACL\": true\n },\n \"Counters\": {\n \"usedSpace\": 115392512,\n \"usedInodes\": 14,\n \"nextInodes\": 35,\n \"nextChunk\": 9,\n \"nextSession\": 0,\n \"nextTrash\": 1\n },\n \"Sustained\": [],\n \"DelFiles\": [\n {\n \"inode\": 23,\n \"length\": 0,\n \"expire\": 1637664458\n }\n ],\n \"Quotas\": {\n \"1\": {\n \"maxSpace\": 1073741824,\n \"maxInodes\": 100\n }\n },\n \"FSTree\": {\n \"attr\": {\"inode\":3,\"type\":\"directory\",\"mode\":493,\"uid\":501,\"gid\":20,\"atime\":1623746591,\"mtime\":1623746610,\"ctime\":1623746610,\"atimensec\":959224111,\"mtimensec\":959224111,\"ctimensec\":959224111,\"nlink\":2,\"length\":0},\n \"xattrs\": [{\"name\":\"dk\",\"value\":\"果汁%2525\"}],\n \"entries\": {\n \"big\": {\n \"attr\": {\"inode\":6,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":0,\"atime\":1637150857,\"mtime\":1637150858,\"ctime\":1637150878,\"atimensec\":961503222,\"mtimensec\":961503222,\"ctimensec\":961503222,\"nlink\":1,\"length\":104857600},\n \"posix_acl_default\": {\"owner\":7,\"group\":5,\"other\":5,\"mask\":5,\"users\":null,\"groups\":[{\"id\":5,\"perm\":4},{\"id\":6,\"perm\":4}]},\n \"chunks\": [\n {\"index\":0,\"slices\":[{\"id\":5,\"size\":67108864,\"len\":67108864}]},\n {\"index\":1,\"slices\":[{\"id\":6,\"size\":37748736,\"len\":37748736}]}\n ]\n },\n \"f11\": {\n \"attr\": {\"inode\":4,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":20,\"atime\":1623746610,\"mtime\":1623746610,\"ctime\":1623746639,\"atimensec\":591590333,\"mtimensec\":591590333,\"ctimensec\":591590333,\"nlink\":2,\"length\":12},\n \"chunks\": [{\"index\":0,\"slices\":[{\"id\":2,\"size\":12,\"len\":12}]}]\n }\n }\n }\n}\n" }, { "path": "pkg/meta/metadata.sample", "content": "{\n \"Setting\": {\n \"Name\": \"load-dump-test\",\n \"UUID\": \"faa27c8f-edab-4791-a4e0-1620b732b343\",\n \"Storage\": \"file\",\n \"Bucket\": \"/Users/juicefs/.juicefs/local/\",\n \"SecretKey\": \"removed\",\n \"BlockSize\": 4096,\n \"Compression\": \"none\",\n \"EncryptKey\": \"AQSttslKOSE/hQT/gmaMniCsdPF8JdPRfoYK6zFkdUOnifYwBA==\",\n \"KeyEncrypted\": true,\n \"TrashDays\": 1,\n \"MetaVersion\": 1,\n \"EnableACL\": true\n },\n \"Counters\": {\n \"usedSpace\": 115392512,\n \"usedInodes\": 14,\n \"nextInodes\": 35,\n \"nextChunk\": 9,\n \"nextSession\": 0,\n \"nextTrash\": 1\n },\n \"Sustained\": [],\n \"DelFiles\": [\n {\n \"inode\": 23,\n \"length\": 0,\n \"expire\": 1637664458\n }\n ],\n \"Quotas\": {\n \"1\": {\n \"maxSpace\": 1073741824,\n \"maxInodes\": 100\n }\n },\n \"FSTree\": {\n \"attr\": {\"inode\":1,\"type\":\"directory\",\"mode\":511,\"uid\":0,\"gid\":0,\"atime\":1623745101,\"mtime\":1638437879,\"ctime\":1638437879,\"nlink\":5,\"length\":0},\n \"xattrs\": [{\"name\":\"lastBackup\",\"value\":\"2021-11-23T18:29:54+08:00\"}],\n \"posix_acl_access\": {\"owner\":6,\"group\":4,\"other\":4,\"mask\":4,\"users\":null,\"groups\":null},\n \"entries\": {\n \"GBK%B9%FB֭%CA%FD%BEݿƼ%BC%D3%D0%CF޹%AB˾%CEļ%FE\": {\n \"attr\": {\"inode\":34,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":0,\"atime\":1648717321,\"mtime\":1648717321,\"ctime\":1648717321,\"atimensec\":401146141,\"mtimensec\":401146141,\"ctimensec\":401146141,\"nlink\":1,\"length\":0},\n \"posix_acl_access\": {\"owner\":6,\"group\":4,\"other\":4,\"mask\":4,\"users\":[{\"id\":1,\"perm\":6},{\"id\":2,\"perm\":7}],\"groups\":null},\n \"posix_acl_default\": {\"owner\":7,\"group\":5,\"other\":5,\"mask\":5,\"users\":null,\"groups\":[{\"id\":3,\"perm\":6},{\"id\":4,\"perm\":7}]}\n },\n \"UTF8果汁数据科技有限公司文件\": {\n \"attr\": {\"inode\":33,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":0,\"atime\":1648717211,\"mtime\":1648717211,\"ctime\":1648717211,\"atimensec\":36325414,\"mtimensec\":36325414,\"ctimensec\":36325414,\"nlink\":1,\"length\":0}\n },\n \"UTF8果汁数据科技有限公司目录\": {\n \"attr\": {\"inode\":32,\"type\":\"directory\",\"mode\":493,\"uid\":501,\"gid\":0,\"atime\":1648717173,\"mtime\":1648717173,\"ctime\":1648717173,\"atimensec\":605897411,\"mtimensec\":605897411,\"ctimensec\":605897411,\"nlink\":2,\"length\":0},\n \"entries\": {\n }\n },\n \"d\": {\n \"attr\": {\"inode\":25,\"type\":\"directory\",\"mode\":493,\"uid\":1,\"gid\":0,\"atime\":1637664458,\"mtime\":1637664458,\"ctime\":1637664458,\"atimensec\":862381233,\"mtimensec\":862381233,\"ctimensec\":862381233,\"nlink\":2,\"length\":0},\n \"entries\": {\n }\n },\n \"d1\": {\n \"attr\": {\"inode\":3,\"type\":\"directory\",\"mode\":493,\"uid\":501,\"gid\":20,\"atime\":1623746591,\"mtime\":1623746610,\"ctime\":1623746610,\"atimensec\":959224111,\"mtimensec\":959224111,\"ctimensec\":959224111,\"nlink\":2,\"length\":0},\n \"xattrs\": [{\"name\":\"dk\",\"value\":\"果汁%2525\"}],\n \"entries\": {\n \"big\": {\n \"attr\": {\"inode\":6,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":0,\"atime\":1637150857,\"mtime\":1637150858,\"ctime\":1637150878,\"atimensec\":961503222,\"mtimensec\":961503222,\"ctimensec\":961503222,\"nlink\":1,\"length\":104857600},\n \"posix_acl_default\": {\"owner\":7,\"group\":5,\"other\":5,\"mask\":5,\"users\":null,\"groups\":[{\"id\":5,\"perm\":4},{\"id\":6,\"perm\":4}]},\n \"chunks\": [\n {\"index\":0,\"slices\":[{\"id\":5,\"size\":67108864,\"len\":67108864}]},\n {\"index\":1,\"slices\":[{\"id\":6,\"size\":37748736,\"len\":37748736}]}\n ]\n },\n \"f11\": {\n \"attr\": {\"inode\":4,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":20,\"atime\":1623746610,\"mtime\":1623746610,\"ctime\":1623746639,\"atimensec\":591590333,\"mtimensec\":591590333,\"ctimensec\":591590333,\"nlink\":2,\"length\":12},\n \"chunks\": [{\"index\":0,\"slices\":[{\"id\":2,\"size\":12,\"len\":12}]}]\n }\n }\n },\n \"f1\": {\n \"attr\": {\"inode\":2,\"flags\":128,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":20,\"atime\":1623746580,\"mtime\":1623746661,\"ctime\":1623746661,\"atimensec\":219686444,\"mtimensec\":219686444,\"ctimensec\":219686444,\"nlink\":1,\"length\":24},\n \"xattrs\": [{\"name\":\"k\",\"value\":\"v\"}],\n \"posix_acl_access\": {\"owner\":6,\"group\":4,\"other\":4,\"mask\":4,\"users\":[{\"id\":1,\"perm\":6},{\"id\":2,\"perm\":7}],\"groups\":null},\n \"posix_acl_default\": {\"owner\":7,\"group\":5,\"other\":5,\"mask\":5,\"users\":null,\"groups\":[{\"id\":3,\"perm\":6},{\"id\":4,\"perm\":7}]},\n \"chunks\": [{\"index\":0,\"slices\":[{\"id\":1,\"size\":6,\"len\":6},{\"id\":2,\"size\":12,\"len\":12},{\"id\":4,\"size\":24,\"len\":24}]}]\n },\n \"l1\": {\n \"attr\": {\"inode\":4,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":20,\"atime\":1623746610,\"mtime\":1623746610,\"ctime\":1623746639,\"atimensec\":591590333,\"mtimensec\":591590333,\"ctimensec\":591590333,\"nlink\":2,\"length\":12},\n \"chunks\": [{\"index\":0,\"slices\":[{\"id\":2,\"size\":12,\"len\":12}]}]\n },\n \"s1\": {\n \"attr\": {\"inode\":5,\"type\":\"symlink\",\"mode\":420,\"uid\":501,\"gid\":20,\"atime\":1623746645,\"mtime\":1623746645,\"ctime\":1623746645,\"atimensec\":984144666,\"mtimensec\":984144666,\"ctimensec\":984144666,\"nlink\":1,\"length\":0},\n \"symlink\": \"GBK%B9%FB֭%CA%FD%BEݿƼ%BC%D3%D0%CF޹%AB˾%CEļ%FE\"\n },\n \"sd\": {\n \"attr\": {\"inode\":26,\"type\":\"symlink\",\"mode\":420,\"uid\":1,\"gid\":0,\"atime\":1637664458,\"mtime\":1637664458,\"ctime\":1637664458,\"atimensec\":873647777,\"mtimensec\":873647777,\"ctimensec\":873647777,\"nlink\":1,\"length\":0},\n \"symlink\": \"d\"\n }\n }\n },\n \"Trash\": {\n \"attr\": {\"inode\":9223372032828243968,\"type\":\"directory\",\"mode\":365,\"uid\":0,\"gid\":0,\"atime\":1623745101,\"mtime\":1638437877,\"ctime\":1638437877,\"nlink\":2,\"length\":0},\n \"entries\": {\n \"2021-12-02-09\": {\n \"attr\": {\"inode\":9223372032828243969,\"type\":\"directory\",\"mode\":365,\"uid\":0,\"gid\":0,\"atime\":1638437877,\"mtime\":1638437877,\"ctime\":1638437877,\"atimensec\":598277000,\"mtimensec\":598277000,\"ctimensec\":598277000,\"nlink\":2,\"length\":0},\n \"entries\": {\n \"1-27-tf1\": {\n \"attr\": {\"inode\":27,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":0,\"atime\":1638437852,\"mtime\":1638437852,\"ctime\":1638437877,\"atimensec\":28186000,\"mtimensec\":28186000,\"ctimensec\":28186000,\"nlink\":1,\"length\":11},\n \"chunks\": [{\"index\":0,\"slices\":[{\"id\":7,\"size\":11,\"len\":11}]}]\n },\n \"1-28-td1\": {\n \"attr\": {\"inode\":28,\"type\":\"directory\",\"mode\":493,\"uid\":501,\"gid\":0,\"atime\":1638437856,\"mtime\":1638437879,\"ctime\":1638437879,\"atimensec\":59246000,\"mtimensec\":59246000,\"ctimensec\":59246000,\"nlink\":2,\"length\":0},\n \"entries\": {\n }\n },\n \"28-29-tdf1\": {\n \"attr\": {\"inode\":29,\"type\":\"regular\",\"mode\":420,\"uid\":501,\"gid\":0,\"atime\":1638437873,\"mtime\":1638437873,\"ctime\":1638437879,\"atimensec\":449880000,\"mtimensec\":449880000,\"ctimensec\":449880000,\"nlink\":1,\"length\":10485760},\n \"chunks\": [{\"index\":0,\"slices\":[{\"id\":8,\"size\":10485760,\"len\":10485760}]}]\n }\n }\n }\n }\n }\n}\n" }, { "path": "pkg/meta/openfile.go", "content": "package meta\n\nimport (\n\t\"sync\"\n\t\"time\"\n)\n\nconst (\n\tinvalidateAllChunks = 0xFFFFFFFF\n\tinvalidateAttrOnly = 0xFFFFFFFE\n)\n\nvar ofPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn &openFile{}\n\t},\n}\n\ntype openFile struct {\n\tsync.RWMutex\n\tattr Attr\n\trefs int\n\tlastCheck int64\n\tfirst []Slice\n\tchunks map[uint32][]Slice\n}\n\nfunc (o *openFile) invalidateChunk() {\n\to.first = nil\n\tfor c := range o.chunks {\n\t\tdelete(o.chunks, c)\n\t}\n}\n\nfunc (o *openFile) release() {\n\to.attr = Attr{}\n\to.refs = 0\n\to.lastCheck = 0\n\to.first = nil\n\to.chunks = nil\n\tofPool.Put(o)\n}\n\ntype openfiles struct {\n\tsync.Mutex\n\texpire time.Duration\n\tlimit uint64\n\tfiles map[Ino]*openFile\n}\n\nfunc newOpenFiles(expire time.Duration, limit uint64) *openfiles {\n\tof := &openfiles{\n\t\texpire: expire,\n\t\tlimit: limit,\n\t\tfiles: make(map[Ino]*openFile),\n\t}\n\tgo of.cleanup()\n\treturn of\n}\n\nfunc (o *openfiles) cleanup() {\n\tfor {\n\t\tvar (\n\t\t\tcnt, deleted, todel int\n\t\t\tcandidateIno Ino\n\t\t\tcandidateOf *openFile\n\t\t)\n\t\to.Lock()\n\t\tif o.limit > 0 && len(o.files) > int(o.limit) {\n\t\t\ttodel = len(o.files) - int(o.limit)\n\t\t}\n\t\tnow := time.Now().Unix()\n\t\tfor ino, of := range o.files {\n\t\t\tcnt++\n\t\t\tif cnt > 1e3 || todel > 0 && deleted >= todel {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif of.refs <= 0 {\n\t\t\t\tif now-of.lastCheck > 3600*12 {\n\t\t\t\t\tof.release()\n\t\t\t\t\tdelete(o.files, ino)\n\t\t\t\t\tdeleted++\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif todel == 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif candidateIno == 0 {\n\t\t\t\t\tcandidateIno = ino\n\t\t\t\t\tcandidateOf = of\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif of.lastCheck < candidateOf.lastCheck {\n\t\t\t\t\tcandidateIno = ino\n\t\t\t\t\tcandidateOf = of\n\t\t\t\t}\n\t\t\t\tcandidateOf.release()\n\t\t\t\tdelete(o.files, candidateIno)\n\t\t\t\tdeleted++\n\t\t\t\tcandidateIno = 0\n\t\t\t}\n\t\t}\n\t\to.Unlock()\n\t\ttime.Sleep(time.Millisecond * time.Duration(1000*(cnt+1-deleted*2)/(cnt+1)))\n\t}\n}\n\nfunc (o *openfiles) OpenCheck(ino Ino, attr *Attr) bool {\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif ok && time.Second*time.Duration(time.Now().Unix()-of.lastCheck) < o.expire {\n\t\tif attr != nil {\n\t\t\t*attr = of.attr\n\t\t}\n\t\tof.refs++\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (o *openfiles) Open(ino Ino, attr *Attr) {\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif !ok {\n\t\tof = ofPool.Get().(*openFile)\n\t\to.files[ino] = of\n\t} else if attr != nil && attr.Mtime == of.attr.Mtime && attr.Mtimensec == of.attr.Mtimensec {\n\t\tattr.KeepCache = of.attr.KeepCache\n\t} else {\n\t\tof.invalidateChunk()\n\t}\n\tif attr != nil {\n\t\tof.attr = *attr\n\t}\n\t// next open can keep cache if not modified\n\tof.attr.KeepCache = true\n\tof.refs++\n\tof.lastCheck = time.Now().Unix()\n}\n\nfunc (o *openfiles) Close(ino Ino) bool {\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif ok {\n\t\tof.refs--\n\t\treturn of.refs <= 0\n\t}\n\treturn true\n}\n\nfunc (o *openfiles) Check(ino Ino, attr *Attr) bool {\n\tif attr == nil {\n\t\tpanic(\"attr is nil\")\n\t}\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif ok && time.Second*time.Duration(time.Now().Unix()-of.lastCheck) < o.expire {\n\t\t*attr = of.attr\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (o *openfiles) Update(ino Ino, attr *Attr) bool {\n\tif attr == nil {\n\t\treturn false\n\t}\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif ok {\n\t\tif attr.Mtime != of.attr.Mtime || attr.Mtimensec != of.attr.Mtimensec {\n\t\t\tof.invalidateChunk()\n\t\t} else {\n\t\t\tattr.KeepCache = of.attr.KeepCache\n\t\t}\n\t\tof.attr = *attr\n\t\tof.lastCheck = time.Now().Unix()\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (o *openfiles) IsOpen(ino Ino) bool {\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\treturn ok && of.refs > 0\n}\n\nfunc (o *openfiles) ReadChunk(ino Ino, indx uint32) ([]Slice, bool) {\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tif indx == 0 {\n\t\treturn of.first, of.first != nil\n\t} else {\n\t\tcs, ok := of.chunks[indx]\n\t\treturn cs, ok\n\t}\n}\n\nfunc (o *openfiles) CacheChunk(ino Ino, indx uint32, cs []Slice) {\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif !ok {\n\t\treturn\n\t}\n\tif indx == 0 {\n\t\tof.first = cs\n\t} else {\n\t\tif of.chunks == nil {\n\t\t\tof.chunks = make(map[uint32][]Slice)\n\t\t}\n\t\tof.chunks[indx] = cs\n\t}\n}\n\nfunc (o *openfiles) InvalidateChunk(ino Ino, indx uint32) {\n\to.Lock()\n\tdefer o.Unlock()\n\tof, ok := o.files[ino]\n\tif ok {\n\t\tif indx == invalidateAllChunks {\n\t\t\tof.invalidateChunk()\n\t\t} else if indx == 0 {\n\t\t\tof.first = nil\n\t\t} else {\n\t\t\tdelete(of.chunks, indx)\n\t\t}\n\t\tof.lastCheck = 0\n\t}\n}\n\nfunc (o *openfiles) find(ino Ino) *openFile {\n\to.Lock()\n\tdefer o.Unlock()\n\treturn o.files[ino]\n}\n" }, { "path": "pkg/meta/pb/backup.pb.go", "content": "// Code generated by protoc-gen-go. DO NOT EDIT.\n// versions:\n// \tprotoc-gen-go v1.35.2\n// \tprotoc v5.29.0\n// source: pkg/meta/pb/backup.proto\n\npackage pb\n\nimport (\n\tprotoreflect \"google.golang.org/protobuf/reflect/protoreflect\"\n\tprotoimpl \"google.golang.org/protobuf/runtime/protoimpl\"\n\treflect \"reflect\"\n\tsync \"sync\"\n)\n\nconst (\n\t// Verify that this generated code is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)\n\t// Verify that runtime/protoimpl is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)\n)\n\ntype Format struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tData []byte `protobuf:\"bytes,1,opt,name=data,proto3\" json:\"data,omitempty\"` // meta.Format's json format\n}\n\nfunc (x *Format) Reset() {\n\t*x = Format{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[0]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Format) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Format) ProtoMessage() {}\n\nfunc (x *Format) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[0]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Format.ProtoReflect.Descriptor instead.\nfunc (*Format) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{0}\n}\n\nfunc (x *Format) GetData() []byte {\n\tif x != nil {\n\t\treturn x.Data\n\t}\n\treturn nil\n}\n\ntype Counter struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tKey string `protobuf:\"bytes,1,opt,name=key,proto3\" json:\"key,omitempty\"`\n\tValue int64 `protobuf:\"varint,2,opt,name=value,proto3\" json:\"value,omitempty\"`\n}\n\nfunc (x *Counter) Reset() {\n\t*x = Counter{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[1]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Counter) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Counter) ProtoMessage() {}\n\nfunc (x *Counter) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[1]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Counter.ProtoReflect.Descriptor instead.\nfunc (*Counter) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{1}\n}\n\nfunc (x *Counter) GetKey() string {\n\tif x != nil {\n\t\treturn x.Key\n\t}\n\treturn \"\"\n}\n\nfunc (x *Counter) GetValue() int64 {\n\tif x != nil {\n\t\treturn x.Value\n\t}\n\treturn 0\n}\n\ntype Sustained struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tSid uint64 `protobuf:\"varint,1,opt,name=sid,proto3\" json:\"sid,omitempty\"`\n\tInodes []uint64 `protobuf:\"varint,2,rep,packed,name=inodes,proto3\" json:\"inodes,omitempty\"`\n}\n\nfunc (x *Sustained) Reset() {\n\t*x = Sustained{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[2]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Sustained) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Sustained) ProtoMessage() {}\n\nfunc (x *Sustained) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[2]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Sustained.ProtoReflect.Descriptor instead.\nfunc (*Sustained) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{2}\n}\n\nfunc (x *Sustained) GetSid() uint64 {\n\tif x != nil {\n\t\treturn x.Sid\n\t}\n\treturn 0\n}\n\nfunc (x *Sustained) GetInodes() []uint64 {\n\tif x != nil {\n\t\treturn x.Inodes\n\t}\n\treturn nil\n}\n\ntype DelFile struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tLength uint64 `protobuf:\"varint,2,opt,name=length,proto3\" json:\"length,omitempty\"`\n\tExpire int64 `protobuf:\"varint,3,opt,name=expire,proto3\" json:\"expire,omitempty\"`\n}\n\nfunc (x *DelFile) Reset() {\n\t*x = DelFile{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[3]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *DelFile) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*DelFile) ProtoMessage() {}\n\nfunc (x *DelFile) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[3]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use DelFile.ProtoReflect.Descriptor instead.\nfunc (*DelFile) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{3}\n}\n\nfunc (x *DelFile) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *DelFile) GetLength() uint64 {\n\tif x != nil {\n\t\treturn x.Length\n\t}\n\treturn 0\n}\n\nfunc (x *DelFile) GetExpire() int64 {\n\tif x != nil {\n\t\treturn x.Expire\n\t}\n\treturn 0\n}\n\ntype SliceRef struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tId uint64 `protobuf:\"varint,1,opt,name=id,proto3\" json:\"id,omitempty\"`\n\tSize uint32 `protobuf:\"varint,2,opt,name=size,proto3\" json:\"size,omitempty\"`\n\tRefs int64 `protobuf:\"varint,3,opt,name=refs,proto3\" json:\"refs,omitempty\"`\n}\n\nfunc (x *SliceRef) Reset() {\n\t*x = SliceRef{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[4]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *SliceRef) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*SliceRef) ProtoMessage() {}\n\nfunc (x *SliceRef) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[4]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use SliceRef.ProtoReflect.Descriptor instead.\nfunc (*SliceRef) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{4}\n}\n\nfunc (x *SliceRef) GetId() uint64 {\n\tif x != nil {\n\t\treturn x.Id\n\t}\n\treturn 0\n}\n\nfunc (x *SliceRef) GetSize() uint32 {\n\tif x != nil {\n\t\treturn x.Size\n\t}\n\treturn 0\n}\n\nfunc (x *SliceRef) GetRefs() int64 {\n\tif x != nil {\n\t\treturn x.Refs\n\t}\n\treturn 0\n}\n\ntype Acl struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tId uint32 `protobuf:\"varint,1,opt,name=id,proto3\" json:\"id,omitempty\"`\n\tData []byte `protobuf:\"bytes,2,opt,name=data,proto3\" json:\"data,omitempty\"` // acl.Rule's binary format\n}\n\nfunc (x *Acl) Reset() {\n\t*x = Acl{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[5]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Acl) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Acl) ProtoMessage() {}\n\nfunc (x *Acl) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[5]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Acl.ProtoReflect.Descriptor instead.\nfunc (*Acl) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{5}\n}\n\nfunc (x *Acl) GetId() uint32 {\n\tif x != nil {\n\t\treturn x.Id\n\t}\n\treturn 0\n}\n\nfunc (x *Acl) GetData() []byte {\n\tif x != nil {\n\t\treturn x.Data\n\t}\n\treturn nil\n}\n\ntype Xattr struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tName string `protobuf:\"bytes,2,opt,name=name,proto3\" json:\"name,omitempty\"`\n\tValue []byte `protobuf:\"bytes,3,opt,name=value,proto3\" json:\"value,omitempty\"`\n}\n\nfunc (x *Xattr) Reset() {\n\t*x = Xattr{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[6]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Xattr) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Xattr) ProtoMessage() {}\n\nfunc (x *Xattr) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[6]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Xattr.ProtoReflect.Descriptor instead.\nfunc (*Xattr) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{6}\n}\n\nfunc (x *Xattr) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Xattr) GetName() string {\n\tif x != nil {\n\t\treturn x.Name\n\t}\n\treturn \"\"\n}\n\nfunc (x *Xattr) GetValue() []byte {\n\tif x != nil {\n\t\treturn x.Value\n\t}\n\treturn nil\n}\n\ntype Quota struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tMaxSpace int64 `protobuf:\"varint,2,opt,name=maxSpace,proto3\" json:\"maxSpace,omitempty\"`\n\tMaxInodes int64 `protobuf:\"varint,3,opt,name=maxInodes,proto3\" json:\"maxInodes,omitempty\"`\n\tUsedSpace int64 `protobuf:\"varint,4,opt,name=usedSpace,proto3\" json:\"usedSpace,omitempty\"`\n\tUsedInodes int64 `protobuf:\"varint,5,opt,name=usedInodes,proto3\" json:\"usedInodes,omitempty\"`\n}\n\nfunc (x *Quota) Reset() {\n\t*x = Quota{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[7]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Quota) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Quota) ProtoMessage() {}\n\nfunc (x *Quota) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[7]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Quota.ProtoReflect.Descriptor instead.\nfunc (*Quota) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{7}\n}\n\nfunc (x *Quota) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Quota) GetMaxSpace() int64 {\n\tif x != nil {\n\t\treturn x.MaxSpace\n\t}\n\treturn 0\n}\n\nfunc (x *Quota) GetMaxInodes() int64 {\n\tif x != nil {\n\t\treturn x.MaxInodes\n\t}\n\treturn 0\n}\n\nfunc (x *Quota) GetUsedSpace() int64 {\n\tif x != nil {\n\t\treturn x.UsedSpace\n\t}\n\treturn 0\n}\n\nfunc (x *Quota) GetUsedInodes() int64 {\n\tif x != nil {\n\t\treturn x.UsedInodes\n\t}\n\treturn 0\n}\n\ntype Stat struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tDataLength int64 `protobuf:\"varint,2,opt,name=dataLength,proto3\" json:\"dataLength,omitempty\"`\n\tUsedSpace int64 `protobuf:\"varint,3,opt,name=usedSpace,proto3\" json:\"usedSpace,omitempty\"`\n\tUsedInodes int64 `protobuf:\"varint,4,opt,name=usedInodes,proto3\" json:\"usedInodes,omitempty\"`\n}\n\nfunc (x *Stat) Reset() {\n\t*x = Stat{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[8]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Stat) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Stat) ProtoMessage() {}\n\nfunc (x *Stat) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[8]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Stat.ProtoReflect.Descriptor instead.\nfunc (*Stat) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{8}\n}\n\nfunc (x *Stat) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Stat) GetDataLength() int64 {\n\tif x != nil {\n\t\treturn x.DataLength\n\t}\n\treturn 0\n}\n\nfunc (x *Stat) GetUsedSpace() int64 {\n\tif x != nil {\n\t\treturn x.UsedSpace\n\t}\n\treturn 0\n}\n\nfunc (x *Stat) GetUsedInodes() int64 {\n\tif x != nil {\n\t\treturn x.UsedInodes\n\t}\n\treturn 0\n}\n\ntype Node struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tData []byte `protobuf:\"bytes,2,opt,name=data,proto3\" json:\"data,omitempty\"` // meta.Attr's binary format\n}\n\nfunc (x *Node) Reset() {\n\t*x = Node{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[9]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Node) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Node) ProtoMessage() {}\n\nfunc (x *Node) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[9]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Node.ProtoReflect.Descriptor instead.\nfunc (*Node) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{9}\n}\n\nfunc (x *Node) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Node) GetData() []byte {\n\tif x != nil {\n\t\treturn x.Data\n\t}\n\treturn nil\n}\n\ntype Edge struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tParent uint64 `protobuf:\"varint,1,opt,name=parent,proto3\" json:\"parent,omitempty\"`\n\tInode uint64 `protobuf:\"varint,2,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tName []byte `protobuf:\"bytes,3,opt,name=name,proto3\" json:\"name,omitempty\"`\n\tType uint32 `protobuf:\"varint,4,opt,name=type,proto3\" json:\"type,omitempty\"`\n}\n\nfunc (x *Edge) Reset() {\n\t*x = Edge{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[10]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Edge) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Edge) ProtoMessage() {}\n\nfunc (x *Edge) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[10]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Edge.ProtoReflect.Descriptor instead.\nfunc (*Edge) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{10}\n}\n\nfunc (x *Edge) GetParent() uint64 {\n\tif x != nil {\n\t\treturn x.Parent\n\t}\n\treturn 0\n}\n\nfunc (x *Edge) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Edge) GetName() []byte {\n\tif x != nil {\n\t\treturn x.Name\n\t}\n\treturn nil\n}\n\nfunc (x *Edge) GetType() uint32 {\n\tif x != nil {\n\t\treturn x.Type\n\t}\n\treturn 0\n}\n\n// for redis and tikv only\ntype Parent struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tParent uint64 `protobuf:\"varint,2,opt,name=parent,proto3\" json:\"parent,omitempty\"`\n\tCnt int64 `protobuf:\"varint,3,opt,name=cnt,proto3\" json:\"cnt,omitempty\"`\n}\n\nfunc (x *Parent) Reset() {\n\t*x = Parent{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[11]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Parent) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Parent) ProtoMessage() {}\n\nfunc (x *Parent) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[11]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Parent.ProtoReflect.Descriptor instead.\nfunc (*Parent) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{11}\n}\n\nfunc (x *Parent) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Parent) GetParent() uint64 {\n\tif x != nil {\n\t\treturn x.Parent\n\t}\n\treturn 0\n}\n\nfunc (x *Parent) GetCnt() int64 {\n\tif x != nil {\n\t\treturn x.Cnt\n\t}\n\treturn 0\n}\n\ntype Chunk struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tIndex uint32 `protobuf:\"varint,2,opt,name=index,proto3\" json:\"index,omitempty\"`\n\tSlices []byte `protobuf:\"bytes,3,opt,name=slices,proto3\" json:\"slices,omitempty\"` // array of meta.slice\n}\n\nfunc (x *Chunk) Reset() {\n\t*x = Chunk{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[12]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Chunk) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Chunk) ProtoMessage() {}\n\nfunc (x *Chunk) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[12]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Chunk.ProtoReflect.Descriptor instead.\nfunc (*Chunk) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{12}\n}\n\nfunc (x *Chunk) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Chunk) GetIndex() uint32 {\n\tif x != nil {\n\t\treturn x.Index\n\t}\n\treturn 0\n}\n\nfunc (x *Chunk) GetSlices() []byte {\n\tif x != nil {\n\t\treturn x.Slices\n\t}\n\treturn nil\n}\n\ntype Symlink struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tInode uint64 `protobuf:\"varint,1,opt,name=inode,proto3\" json:\"inode,omitempty\"`\n\tTarget []byte `protobuf:\"bytes,2,opt,name=target,proto3\" json:\"target,omitempty\"`\n}\n\nfunc (x *Symlink) Reset() {\n\t*x = Symlink{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[13]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Symlink) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Symlink) ProtoMessage() {}\n\nfunc (x *Symlink) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[13]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Symlink.ProtoReflect.Descriptor instead.\nfunc (*Symlink) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{13}\n}\n\nfunc (x *Symlink) GetInode() uint64 {\n\tif x != nil {\n\t\treturn x.Inode\n\t}\n\treturn 0\n}\n\nfunc (x *Symlink) GetTarget() []byte {\n\tif x != nil {\n\t\treturn x.Target\n\t}\n\treturn nil\n}\n\ntype Batch struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tNodes []*Node `protobuf:\"bytes,1,rep,name=nodes,proto3\" json:\"nodes,omitempty\"`\n\tEdges []*Edge `protobuf:\"bytes,2,rep,name=edges,proto3\" json:\"edges,omitempty\"`\n\tChunks []*Chunk `protobuf:\"bytes,3,rep,name=chunks,proto3\" json:\"chunks,omitempty\"`\n\tSliceRefs []*SliceRef `protobuf:\"bytes,4,rep,name=sliceRefs,proto3\" json:\"sliceRefs,omitempty\"`\n\tXattrs []*Xattr `protobuf:\"bytes,5,rep,name=xattrs,proto3\" json:\"xattrs,omitempty\"`\n\tParents []*Parent `protobuf:\"bytes,6,rep,name=parents,proto3\" json:\"parents,omitempty\"`\n\tSymlinks []*Symlink `protobuf:\"bytes,7,rep,name=symlinks,proto3\" json:\"symlinks,omitempty\"`\n\tSustained []*Sustained `protobuf:\"bytes,8,rep,name=sustained,proto3\" json:\"sustained,omitempty\"`\n\tDelfiles []*DelFile `protobuf:\"bytes,9,rep,name=delfiles,proto3\" json:\"delfiles,omitempty\"`\n\tDirstats []*Stat `protobuf:\"bytes,10,rep,name=dirstats,proto3\" json:\"dirstats,omitempty\"`\n\tQuotas []*Quota `protobuf:\"bytes,11,rep,name=quotas,proto3\" json:\"quotas,omitempty\"`\n\tAcls []*Acl `protobuf:\"bytes,12,rep,name=acls,proto3\" json:\"acls,omitempty\"`\n\tCounters []*Counter `protobuf:\"bytes,13,rep,name=counters,proto3\" json:\"counters,omitempty\"`\n}\n\nfunc (x *Batch) Reset() {\n\t*x = Batch{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[14]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Batch) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Batch) ProtoMessage() {}\n\nfunc (x *Batch) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[14]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Batch.ProtoReflect.Descriptor instead.\nfunc (*Batch) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{14}\n}\n\nfunc (x *Batch) GetNodes() []*Node {\n\tif x != nil {\n\t\treturn x.Nodes\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetEdges() []*Edge {\n\tif x != nil {\n\t\treturn x.Edges\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetChunks() []*Chunk {\n\tif x != nil {\n\t\treturn x.Chunks\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetSliceRefs() []*SliceRef {\n\tif x != nil {\n\t\treturn x.SliceRefs\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetXattrs() []*Xattr {\n\tif x != nil {\n\t\treturn x.Xattrs\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetParents() []*Parent {\n\tif x != nil {\n\t\treturn x.Parents\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetSymlinks() []*Symlink {\n\tif x != nil {\n\t\treturn x.Symlinks\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetSustained() []*Sustained {\n\tif x != nil {\n\t\treturn x.Sustained\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetDelfiles() []*DelFile {\n\tif x != nil {\n\t\treturn x.Delfiles\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetDirstats() []*Stat {\n\tif x != nil {\n\t\treturn x.Dirstats\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetQuotas() []*Quota {\n\tif x != nil {\n\t\treturn x.Quotas\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetAcls() []*Acl {\n\tif x != nil {\n\t\treturn x.Acls\n\t}\n\treturn nil\n}\n\nfunc (x *Batch) GetCounters() []*Counter {\n\tif x != nil {\n\t\treturn x.Counters\n\t}\n\treturn nil\n}\n\ntype Footer struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tMagic uint32 `protobuf:\"varint,1,opt,name=magic,proto3\" json:\"magic,omitempty\"`\n\tVersion uint32 `protobuf:\"varint,2,opt,name=version,proto3\" json:\"version,omitempty\"`\n\tInfos map[string]*Footer_SegInfo `protobuf:\"bytes,3,rep,name=infos,proto3\" json:\"infos,omitempty\" protobuf_key:\"bytes,1,opt,name=key,proto3\" protobuf_val:\"bytes,2,opt,name=value,proto3\"`\n}\n\nfunc (x *Footer) Reset() {\n\t*x = Footer{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[15]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Footer) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Footer) ProtoMessage() {}\n\nfunc (x *Footer) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[15]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Footer.ProtoReflect.Descriptor instead.\nfunc (*Footer) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{15}\n}\n\nfunc (x *Footer) GetMagic() uint32 {\n\tif x != nil {\n\t\treturn x.Magic\n\t}\n\treturn 0\n}\n\nfunc (x *Footer) GetVersion() uint32 {\n\tif x != nil {\n\t\treturn x.Version\n\t}\n\treturn 0\n}\n\nfunc (x *Footer) GetInfos() map[string]*Footer_SegInfo {\n\tif x != nil {\n\t\treturn x.Infos\n\t}\n\treturn nil\n}\n\ntype Footer_SegInfo struct {\n\tstate protoimpl.MessageState\n\tsizeCache protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tOffset []uint64 `protobuf:\"varint,1,rep,packed,name=offset,proto3\" json:\"offset,omitempty\"`\n\tNum uint64 `protobuf:\"varint,2,opt,name=num,proto3\" json:\"num,omitempty\"`\n}\n\nfunc (x *Footer_SegInfo) Reset() {\n\t*x = Footer_SegInfo{}\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[16]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *Footer_SegInfo) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*Footer_SegInfo) ProtoMessage() {}\n\nfunc (x *Footer_SegInfo) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_meta_pb_backup_proto_msgTypes[16]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use Footer_SegInfo.ProtoReflect.Descriptor instead.\nfunc (*Footer_SegInfo) Descriptor() ([]byte, []int) {\n\treturn file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{15, 0}\n}\n\nfunc (x *Footer_SegInfo) GetOffset() []uint64 {\n\tif x != nil {\n\t\treturn x.Offset\n\t}\n\treturn nil\n}\n\nfunc (x *Footer_SegInfo) GetNum() uint64 {\n\tif x != nil {\n\t\treturn x.Num\n\t}\n\treturn 0\n}\n\nvar File_pkg_meta_pb_backup_proto protoreflect.FileDescriptor\n\nvar file_pkg_meta_pb_backup_proto_rawDesc = []byte{\n\t0x0a, 0x18, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61,\n\t0x63, 0x6b, 0x75, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x70, 0x62, 0x22, 0x1c,\n\t0x0a, 0x06, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61,\n\t0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x31, 0x0a, 0x07,\n\t0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,\n\t0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,\n\t0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22,\n\t0x35, 0x0a, 0x09, 0x53, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03,\n\t0x73, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x73, 0x69, 0x64, 0x12, 0x16,\n\t0x0a, 0x06, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x04, 0x52, 0x06,\n\t0x69, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, 0x4f, 0x0a, 0x07, 0x44, 0x65, 0x6c, 0x46, 0x69, 0x6c,\n\t0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04,\n\t0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74,\n\t0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12,\n\t0x16, 0x0a, 0x06, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52,\n\t0x06, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x22, 0x42, 0x0a, 0x08, 0x53, 0x6c, 0x69, 0x63, 0x65,\n\t0x52, 0x65, 0x66, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52,\n\t0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,\n\t0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x66, 0x73, 0x18,\n\t0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x72, 0x65, 0x66, 0x73, 0x22, 0x29, 0x0a, 0x03, 0x41,\n\t0x63, 0x6c, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02,\n\t0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c,\n\t0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x47, 0x0a, 0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12,\n\t0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05,\n\t0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20,\n\t0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,\n\t0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22,\n\t0x95, 0x01, 0x0a, 0x05, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f,\n\t0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12,\n\t0x1a, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,\n\t0x03, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6d,\n\t0x61, 0x78, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09,\n\t0x6d, 0x61, 0x78, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x73, 0x65,\n\t0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x75, 0x73,\n\t0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x49,\n\t0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x75, 0x73, 0x65,\n\t0x64, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, 0x7a, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12,\n\t0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05,\n\t0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x4c, 0x65, 0x6e,\n\t0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x4c,\n\t0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x53, 0x70, 0x61,\n\t0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x75, 0x73, 0x65, 0x64, 0x53, 0x70,\n\t0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x49, 0x6e, 0x6f, 0x64, 0x65,\n\t0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x49, 0x6e, 0x6f,\n\t0x64, 0x65, 0x73, 0x22, 0x30, 0x0a, 0x04, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69,\n\t0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64,\n\t0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52,\n\t0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x5c, 0x0a, 0x04, 0x45, 0x64, 0x67, 0x65, 0x12, 0x16, 0x0a,\n\t0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x70,\n\t0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x02,\n\t0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e,\n\t0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,\n\t0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74,\n\t0x79, 0x70, 0x65, 0x22, 0x48, 0x0a, 0x06, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a,\n\t0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e,\n\t0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20,\n\t0x01, 0x28, 0x04, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x63,\n\t0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x63, 0x6e, 0x74, 0x22, 0x4b, 0x0a,\n\t0x05, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18,\n\t0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05,\n\t0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x69, 0x6e, 0x64,\n\t0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01,\n\t0x28, 0x0c, 0x52, 0x06, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x73, 0x22, 0x37, 0x0a, 0x07, 0x53, 0x79,\n\t0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01,\n\t0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74,\n\t0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x72,\n\t0x67, 0x65, 0x74, 0x22, 0xed, 0x03, 0x0a, 0x05, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1e, 0x0a,\n\t0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70,\n\t0x62, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x1e, 0x0a,\n\t0x05, 0x65, 0x64, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70,\n\t0x62, 0x2e, 0x45, 0x64, 0x67, 0x65, 0x52, 0x05, 0x65, 0x64, 0x67, 0x65, 0x73, 0x12, 0x21, 0x0a,\n\t0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e,\n\t0x70, 0x62, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73,\n\t0x12, 0x2a, 0x0a, 0x09, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x52, 0x65, 0x66, 0x73, 0x18, 0x04, 0x20,\n\t0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x6c, 0x69, 0x63, 0x65, 0x52, 0x65,\n\t0x66, 0x52, 0x09, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x52, 0x65, 0x66, 0x73, 0x12, 0x21, 0x0a, 0x06,\n\t0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70,\n\t0x62, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x12,\n\t0x24, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b,\n\t0x32, 0x0a, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x07, 0x70, 0x61,\n\t0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b,\n\t0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x79, 0x6d,\n\t0x6c, 0x69, 0x6e, 0x6b, 0x52, 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x2b,\n\t0x0a, 0x09, 0x73, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x64, 0x18, 0x08, 0x20, 0x03, 0x28,\n\t0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x64,\n\t0x52, 0x09, 0x73, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x64, 0x12, 0x27, 0x0a, 0x08, 0x64,\n\t0x65, 0x6c, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e,\n\t0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x08, 0x64, 0x65, 0x6c, 0x66,\n\t0x69, 0x6c, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x64, 0x69, 0x72, 0x73, 0x74, 0x61, 0x74, 0x73,\n\t0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74,\n\t0x52, 0x08, 0x64, 0x69, 0x72, 0x73, 0x74, 0x61, 0x74, 0x73, 0x12, 0x21, 0x0a, 0x06, 0x71, 0x75,\n\t0x6f, 0x74, 0x61, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e,\n\t0x51, 0x75, 0x6f, 0x74, 0x61, 0x52, 0x06, 0x71, 0x75, 0x6f, 0x74, 0x61, 0x73, 0x12, 0x1b, 0x0a,\n\t0x04, 0x61, 0x63, 0x6c, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x70, 0x62,\n\t0x2e, 0x41, 0x63, 0x6c, 0x52, 0x04, 0x61, 0x63, 0x6c, 0x73, 0x12, 0x27, 0x0a, 0x08, 0x63, 0x6f,\n\t0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70,\n\t0x62, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74,\n\t0x65, 0x72, 0x73, 0x22, 0xe8, 0x01, 0x0a, 0x06, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x12, 0x14,\n\t0x0a, 0x05, 0x6d, 0x61, 0x67, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d,\n\t0x61, 0x67, 0x69, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18,\n\t0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2b,\n\t0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e,\n\t0x70, 0x62, 0x2e, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45,\n\t0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x33, 0x0a, 0x07, 0x53,\n\t0x65, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74,\n\t0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x10,\n\t0x0a, 0x03, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6e, 0x75, 0x6d,\n\t0x1a, 0x4c, 0x0a, 0x0a, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,\n\t0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,\n\t0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,\n\t0x12, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x67, 0x49,\n\t0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x06,\n\t0x5a, 0x04, 0x2e, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,\n}\n\nvar (\n\tfile_pkg_meta_pb_backup_proto_rawDescOnce sync.Once\n\tfile_pkg_meta_pb_backup_proto_rawDescData = file_pkg_meta_pb_backup_proto_rawDesc\n)\n\nfunc file_pkg_meta_pb_backup_proto_rawDescGZIP() []byte {\n\tfile_pkg_meta_pb_backup_proto_rawDescOnce.Do(func() {\n\t\tfile_pkg_meta_pb_backup_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_meta_pb_backup_proto_rawDescData)\n\t})\n\treturn file_pkg_meta_pb_backup_proto_rawDescData\n}\n\nvar file_pkg_meta_pb_backup_proto_msgTypes = make([]protoimpl.MessageInfo, 18)\nvar file_pkg_meta_pb_backup_proto_goTypes = []any{\n\t(*Format)(nil), // 0: pb.Format\n\t(*Counter)(nil), // 1: pb.Counter\n\t(*Sustained)(nil), // 2: pb.Sustained\n\t(*DelFile)(nil), // 3: pb.DelFile\n\t(*SliceRef)(nil), // 4: pb.SliceRef\n\t(*Acl)(nil), // 5: pb.Acl\n\t(*Xattr)(nil), // 6: pb.Xattr\n\t(*Quota)(nil), // 7: pb.Quota\n\t(*Stat)(nil), // 8: pb.Stat\n\t(*Node)(nil), // 9: pb.Node\n\t(*Edge)(nil), // 10: pb.Edge\n\t(*Parent)(nil), // 11: pb.Parent\n\t(*Chunk)(nil), // 12: pb.Chunk\n\t(*Symlink)(nil), // 13: pb.Symlink\n\t(*Batch)(nil), // 14: pb.Batch\n\t(*Footer)(nil), // 15: pb.Footer\n\t(*Footer_SegInfo)(nil), // 16: pb.Footer.SegInfo\n\tnil, // 17: pb.Footer.InfosEntry\n}\nvar file_pkg_meta_pb_backup_proto_depIdxs = []int32{\n\t9, // 0: pb.Batch.nodes:type_name -> pb.Node\n\t10, // 1: pb.Batch.edges:type_name -> pb.Edge\n\t12, // 2: pb.Batch.chunks:type_name -> pb.Chunk\n\t4, // 3: pb.Batch.sliceRefs:type_name -> pb.SliceRef\n\t6, // 4: pb.Batch.xattrs:type_name -> pb.Xattr\n\t11, // 5: pb.Batch.parents:type_name -> pb.Parent\n\t13, // 6: pb.Batch.symlinks:type_name -> pb.Symlink\n\t2, // 7: pb.Batch.sustained:type_name -> pb.Sustained\n\t3, // 8: pb.Batch.delfiles:type_name -> pb.DelFile\n\t8, // 9: pb.Batch.dirstats:type_name -> pb.Stat\n\t7, // 10: pb.Batch.quotas:type_name -> pb.Quota\n\t5, // 11: pb.Batch.acls:type_name -> pb.Acl\n\t1, // 12: pb.Batch.counters:type_name -> pb.Counter\n\t17, // 13: pb.Footer.infos:type_name -> pb.Footer.InfosEntry\n\t16, // 14: pb.Footer.InfosEntry.value:type_name -> pb.Footer.SegInfo\n\t15, // [15:15] is the sub-list for method output_type\n\t15, // [15:15] is the sub-list for method input_type\n\t15, // [15:15] is the sub-list for extension type_name\n\t15, // [15:15] is the sub-list for extension extendee\n\t0, // [0:15] is the sub-list for field type_name\n}\n\nfunc init() { file_pkg_meta_pb_backup_proto_init() }\nfunc file_pkg_meta_pb_backup_proto_init() {\n\tif File_pkg_meta_pb_backup_proto != nil {\n\t\treturn\n\t}\n\ttype x struct{}\n\tout := protoimpl.TypeBuilder{\n\t\tFile: protoimpl.DescBuilder{\n\t\t\tGoPackagePath: reflect.TypeOf(x{}).PkgPath(),\n\t\t\tRawDescriptor: file_pkg_meta_pb_backup_proto_rawDesc,\n\t\t\tNumEnums: 0,\n\t\t\tNumMessages: 18,\n\t\t\tNumExtensions: 0,\n\t\t\tNumServices: 0,\n\t\t},\n\t\tGoTypes: file_pkg_meta_pb_backup_proto_goTypes,\n\t\tDependencyIndexes: file_pkg_meta_pb_backup_proto_depIdxs,\n\t\tMessageInfos: file_pkg_meta_pb_backup_proto_msgTypes,\n\t}.Build()\n\tFile_pkg_meta_pb_backup_proto = out.File\n\tfile_pkg_meta_pb_backup_proto_rawDesc = nil\n\tfile_pkg_meta_pb_backup_proto_goTypes = nil\n\tfile_pkg_meta_pb_backup_proto_depIdxs = nil\n}\n" }, { "path": "pkg/meta/pb/backup.proto", "content": "syntax = \"proto3\";\npackage pb;\noption go_package = \"./pb\";\n\n/*\n1. install protocol buffer compiler\n2. install Go protoc plugin (protoc-gen-go)\n3. exec: protoc --go_out=pkg/meta pkg/meta/pb/backup.proto in main directory\n*/\n\nmessage Format {\n bytes data = 1; // meta.Format's json format\n}\n\nmessage Counter {\n string key = 1;\n int64 value = 2;\n}\n\nmessage Sustained {\n uint64 sid = 1;\n repeated uint64 inodes = 2;\n}\n\nmessage DelFile {\n uint64 inode = 1;\n uint64 length = 2;\n int64 expire = 3;\n}\n\nmessage SliceRef {\n uint64 id = 1;\n uint32 size = 2;\n int64 refs = 3;\n}\n\nmessage Acl {\n uint32 id = 1;\n bytes data = 2; // acl.Rule's binary format\n}\n\nmessage Xattr {\n uint64 inode = 1;\n string name = 2;\n bytes value = 3;\n}\n\nmessage Quota {\n uint64 inode = 1;\n int64 maxSpace = 2;\n int64 maxInodes = 3;\n int64 usedSpace = 4;\n int64 usedInodes = 5;\n}\n\nmessage Stat {\n uint64 inode = 1;\n int64 dataLength = 2;\n int64 usedSpace = 3;\n int64 usedInodes = 4;\n}\n\nmessage Node {\n uint64 inode = 1;\n bytes data = 2; // meta.Attr's binary format\n}\n\nmessage Edge {\n uint64 parent = 1;\n uint64 inode = 2;\n bytes name = 3;\n uint32 type = 4;\n}\n\n// for redis and tikv only\nmessage Parent {\n uint64 inode = 1;\n uint64 parent = 2 ;\n int64 cnt = 3;\n}\n\nmessage Chunk {\n uint64 inode = 1;\n uint32 index = 2;\n bytes slices = 3; // array of meta.slice\n}\n\nmessage Symlink {\n uint64 inode = 1;\n bytes target = 2;\n}\n\nmessage Batch {\n repeated Node nodes = 1;\n repeated Edge edges = 2;\n repeated Chunk chunks = 3;\n repeated SliceRef sliceRefs = 4;\n repeated Xattr xattrs = 5;\n repeated Parent parents = 6;\n repeated Symlink symlinks = 7;\n repeated Sustained sustained = 8;\n repeated DelFile delfiles = 9;\n repeated Stat dirstats = 10;\n repeated Quota quotas = 11;\n repeated Acl acls = 12;\n repeated Counter counters = 13;\n}\n\nmessage Footer {\n message SegInfo {\n repeated uint64 offset = 1;\n uint64 num = 2;\n }\n\n uint32 magic = 1;\n uint32 version = 2;\n map infos = 3;\n}" }, { "path": "pkg/meta/quota.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/pkg/errors\"\n)\n\n// stat of dir\ntype dirStat struct {\n\tlength int64\n\tspace int64\n\tinodes int64\n}\n\nconst (\n\tDirQuotaType = iota\n\tUserQuotaType\n\tGroupQuotaType\n)\n\ntype Quota struct {\n\tMaxSpace, MaxInodes int64\n\tUsedSpace, UsedInodes int64\n\tnewSpace, newInodes int64\n}\n\ntype iQuota struct {\n\tqtype uint32\n\tqkey uint64 // ino/uid/gid\n\tquota *Quota\n}\n\n// Returns true if it will exceed the quota limit\nfunc (q *Quota) check(space, inodes int64) bool {\n\tif space > 0 {\n\t\tmax := atomic.LoadInt64(&q.MaxSpace)\n\t\tif max > 0 && atomic.LoadInt64(&q.UsedSpace)+atomic.LoadInt64(&q.newSpace)+space > max {\n\t\t\treturn true\n\t\t}\n\t}\n\tif inodes > 0 {\n\t\tmax := atomic.LoadInt64(&q.MaxInodes)\n\t\tif max > 0 && atomic.LoadInt64(&q.UsedInodes)+atomic.LoadInt64(&q.newInodes)+inodes > max {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (q *Quota) update(space, inodes int64) {\n\tatomic.AddInt64(&q.newSpace, space)\n\tatomic.AddInt64(&q.newInodes, inodes)\n}\n\nfunc (q *Quota) snap() Quota {\n\treturn Quota{\n\t\tMaxSpace: atomic.LoadInt64(&q.MaxSpace),\n\t\tMaxInodes: atomic.LoadInt64(&q.MaxInodes),\n\t\tUsedSpace: atomic.LoadInt64(&q.UsedSpace),\n\t\tUsedInodes: atomic.LoadInt64(&q.UsedInodes),\n\t\tnewSpace: atomic.LoadInt64(&q.newSpace),\n\t\tnewInodes: atomic.LoadInt64(&q.newInodes),\n\t}\n}\n\n// not thread safe\nfunc (q *Quota) sanitize() {\n\tif q.UsedSpace < 0 {\n\t\tq.UsedSpace = 0\n\t}\n\tif q.MaxSpace > 0 && q.MaxSpace < q.UsedSpace {\n\t\tq.MaxSpace = q.UsedSpace\n\t}\n\tif q.UsedInodes < 0 {\n\t\tq.UsedInodes = 0\n\t}\n\tif q.MaxInodes > 0 && q.MaxInodes < q.UsedInodes {\n\t\tq.MaxInodes = q.UsedInodes\n\t}\n}\n\nfunc (m *baseMeta) parallelSyncDirStat(ctx Context, inos map[Ino]bool) *sync.WaitGroup {\n\tvar wg sync.WaitGroup\n\tfor i := range inos {\n\t\twg.Add(1)\n\t\tgo func(ino Ino) {\n\t\t\tdefer wg.Done()\n\t\t\t_, st := m.en.doSyncDirStat(ctx, ino)\n\t\t\tif st != 0 && st != syscall.ENOENT {\n\t\t\t\tlogger.Warnf(\"sync dir stat for %d: %s\", ino, st)\n\t\t\t}\n\t\t}(i)\n\t}\n\treturn &wg\n}\n\nfunc (m *baseMeta) groupBatch(batch map[Ino]dirStat, size int) [][]Ino {\n\tvar inos []Ino\n\tfor ino := range batch {\n\t\tinos = append(inos, ino)\n\t}\n\tsort.Slice(inos, func(i, j int) bool {\n\t\treturn inos[i] < inos[j]\n\t})\n\tvar batches [][]Ino\n\tfor i := 0; i < len(inos); i += size {\n\t\tend := i + size\n\t\tif end > len(inos) {\n\t\t\tend = len(inos)\n\t\t}\n\t\tbatches = append(batches, inos[i:end])\n\t}\n\treturn batches\n}\n\nfunc (m *baseMeta) calcDirStat(ctx Context, ino Ino) (*dirStat, syscall.Errno) {\n\tvar entries []*Entry\n\tif eno := m.en.doReaddir(ctx, ino, 1, &entries, -1); eno != 0 {\n\t\treturn nil, eno\n\t}\n\n\tstat := new(dirStat)\n\tfor _, e := range entries {\n\t\tif ctx.Canceled() {\n\t\t\treturn nil, syscall.EINTR\n\t\t}\n\t\tstat.inodes += 1\n\t\tvar l uint64\n\t\tif e.Attr.Typ == TypeFile {\n\t\t\tl = e.Attr.Length\n\t\t}\n\t\tstat.length += int64(l)\n\t\tstat.space += align4K(l)\n\t}\n\treturn stat, 0\n}\n\nfunc (m *baseMeta) GetDirStat(ctx Context, inode Ino) (stat *dirStat, st syscall.Errno) {\n\tstat, st = m.en.doGetDirStat(ctx, m.checkRoot(inode), !m.conf.ReadOnly)\n\tif st != 0 {\n\t\treturn\n\t}\n\tif stat == nil {\n\t\tstat, st = m.calcDirStat(ctx, inode)\n\t}\n\treturn\n}\n\nfunc (m *baseMeta) updateDirStat(ctx Context, ino Ino, length, space, inodes int64) {\n\tif !m.getFormat().DirStats {\n\t\treturn\n\t}\n\tm.dirStatsLock.Lock()\n\tdefer m.dirStatsLock.Unlock()\n\tstat := m.dirStats[ino]\n\tstat.length += length\n\tstat.inodes += inodes\n\tstat.space += space\n\tm.dirStats[ino] = stat\n}\n\nfunc (m *baseMeta) updateParentStat(ctx Context, inode, parent Ino, length, space int64) {\n\tif length == 0 && space == 0 {\n\t\treturn\n\t}\n\tm.en.updateStats(space, 0)\n\tif !m.getFormat().DirStats {\n\t\treturn\n\t}\n\tif parent > 0 {\n\t\tm.updateDirStat(ctx, parent, length, space, 0)\n\t\tm.updateDirQuota(ctx, parent, space, 0)\n\t} else {\n\t\tgo func() {\n\t\t\tfor p, v := range m.en.doGetParents(ctx, inode) {\n\t\t\t\tm.updateDirStat(ctx, p, length*int64(v), space*int64(v), 0)\n\t\t\t\tm.updateDirQuota(ctx, p, space*int64(v), 0)\n\t\t\t}\n\t\t}()\n\t}\n}\n\nfunc (m *baseMeta) flushDirStat(ctx Context) {\n\tdefer m.sessWG.Done()\n\tperiod := 1 * time.Second\n\tif m.conf.DirStatFlushPeriod != 0 {\n\t\tperiod = m.conf.DirStatFlushPeriod\n\t}\n\n\tticker := time.NewTicker(period)\n\tdefer ticker.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-ticker.C:\n\t\t\tm.doFlushDirStat()\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) doFlushDirStat() {\n\tif !m.getFormat().DirStats {\n\t\treturn\n\t}\n\tm.dirStatsLock.Lock()\n\tif len(m.dirStats) == 0 {\n\t\tm.dirStatsLock.Unlock()\n\t\treturn\n\t}\n\tstats := m.dirStats\n\tm.dirStats = make(map[Ino]dirStat)\n\tm.dirStatsLock.Unlock()\n\terr := m.en.doUpdateDirStat(Background(), stats)\n\tif err != nil {\n\t\tlogger.Errorf(\"update dir stat failed: %v\", err)\n\t}\n}\n\nfunc (m *baseMeta) flushStats(ctx Context) {\n\tdefer m.sessWG.Done()\n\tticker := time.NewTicker(time.Second)\n\tdefer ticker.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-ticker.C:\n\t\t\tm.doFlushStats()\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) doFlushStats() {\n\tm.fsStatsLock.Lock()\n\tm.en.doFlushStats()\n\tm.fsStatsLock.Unlock()\n}\n\nfunc (m *baseMeta) syncVolumeStat(ctx Context) error {\n\treturn m.en.doSyncVolumeStat(ctx)\n}\n\nfunc (m *baseMeta) checkQuota(ctx Context, space, inodes int64, uid, gid uint32, parents ...Ino) syscall.Errno {\n\tif space <= 0 && inodes <= 0 {\n\t\treturn 0\n\t}\n\tif m.checkUserQuota(ctx, uint64(uid), space, inodes) {\n\t\treturn syscall.EDQUOT\n\t}\n\tif m.checkGroupQuota(ctx, uint64(gid), space, inodes) {\n\t\treturn syscall.EDQUOT\n\t}\n\n\tformat := m.getFormat()\n\tif space > 0 && format.Capacity > 0 && atomic.LoadInt64(&m.usedSpace)+atomic.LoadInt64(&m.newSpace)+space > int64(format.Capacity) {\n\t\treturn syscall.ENOSPC\n\t}\n\tif inodes > 0 && format.Inodes > 0 && atomic.LoadInt64(&m.usedInodes)+atomic.LoadInt64(&m.newInodes)+inodes > int64(format.Inodes) {\n\t\treturn syscall.ENOSPC\n\t}\n\tif !format.DirStats {\n\t\treturn 0\n\t}\n\tfor _, ino := range parents {\n\t\tif m.checkDirQuota(ctx, ino, space, inodes) {\n\t\t\treturn syscall.EDQUOT\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) loadQuotas() {\n\tformat := m.getFormat()\n\tif !format.DirStats && !format.UserGroupQuota {\n\t\treturn\n\t}\n\n\tdirQuotas, userQuotas, groupQuotas, err := m.en.doLoadQuotas(Background())\n\tif err != nil {\n\t\tlogger.Warnf(\"Load quotas: %s\", err)\n\t\treturn\n\t}\n\tm.quotaMu.Lock()\n\tdefer m.quotaMu.Unlock()\n\n\tm.syncQuotaMaps(m.dirQuotas, dirQuotas, \"inode\")\n\tm.syncQuotaMaps(m.userQuotas, userQuotas, \"user\")\n\tm.syncQuotaMaps(m.groupQuotas, groupQuotas, \"group\")\n}\n\nfunc (m *baseMeta) syncQuotaMaps(existing map[uint64]*Quota, loaded map[uint64]*Quota, quotaType string) {\n\t// add new or update existing\n\tfor key, q := range loaded {\n\t\tlogger.Debugf(\"Load quotas got %s %d -> %+v\", quotaType, key, q)\n\t\tif quota, ok := existing[key]; ok {\n\t\t\tatomic.SwapInt64("a.MaxSpace, q.MaxSpace)\n\t\t\tatomic.SwapInt64("a.MaxInodes, q.MaxInodes)\n\t\t\tatomic.SwapInt64("a.UsedSpace, q.UsedSpace)\n\t\t\tatomic.SwapInt64("a.UsedInodes, q.UsedInodes)\n\t\t} else {\n\t\t\texisting[key] = q\n\t\t}\n\t}\n\t// delete that are not in loaded\n\tif quotaType == \"inode\" {\n\t\tfor key := range existing {\n\t\t\tif _, ok := loaded[key]; !ok {\n\t\t\t\tlogger.Infof(\"Quota for %s %d is deleted\", quotaType, key)\n\t\t\t\tdelete(existing, key)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) getDirParent(ctx Context, inode Ino) (Ino, syscall.Errno) {\n\tm.parentMu.Lock()\n\tparent, ok := m.dirParents[inode]\n\tm.parentMu.Unlock()\n\tif ok {\n\t\treturn parent, 0\n\t}\n\tlogger.Debugf(\"Get directory parent of inode %d: cache miss\", inode)\n\tvar attr Attr\n\tst := m.GetAttr(ctx, inode, &attr)\n\treturn attr.Parent, st\n}\n\n// get inode of the first parent (or myself) with quota\nfunc (m *baseMeta) getQuotaParent(ctx Context, inode Ino) (Ino, *Quota) {\n\tif !m.getFormat().DirStats {\n\t\treturn 0, nil\n\t}\n\tvar q *Quota\n\tvar st syscall.Errno\n\tfor {\n\t\tm.quotaMu.RLock()\n\t\tq = m.dirQuotas[uint64(inode)]\n\t\tm.quotaMu.RUnlock()\n\t\tif q != nil {\n\t\t\treturn inode, q\n\t\t}\n\t\tif inode <= RootInode {\n\t\t\tbreak\n\t\t}\n\t\tlastInode := inode\n\t\tif inode, st = m.getDirParent(ctx, inode); st != 0 {\n\t\t\tlogger.Warnf(\"Get directory parent of inode %d: %s\", lastInode, st)\n\t\t\tbreak\n\t\t}\n\t}\n\treturn 0, nil\n}\n\nfunc (m *baseMeta) checkDirQuota(ctx Context, inode Ino, space, inodes int64) bool {\n\tif !m.getFormat().DirStats {\n\t\treturn false\n\t}\n\tvar q *Quota\n\tvar st syscall.Errno\n\tfor {\n\t\tm.quotaMu.RLock()\n\t\tq = m.dirQuotas[uint64(inode)]\n\t\tm.quotaMu.RUnlock()\n\t\tif q != nil && q.check(space, inodes) {\n\t\t\treturn true\n\t\t}\n\t\tif inode <= RootInode {\n\t\t\tbreak\n\t\t}\n\t\tlastInode := inode\n\t\tif inode, st = m.getDirParent(ctx, inode); st != 0 {\n\t\t\tlogger.Warnf(\"Get directory parent of inode %d: %s\", lastInode, st)\n\t\t\tbreak\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (m *baseMeta) checkUserQuota(ctx Context, uid uint64, space, inodes int64) bool {\n\tif !m.getFormat().UserGroupQuota {\n\t\treturn false\n\t}\n\n\tvar q *Quota\n\tm.quotaMu.RLock()\n\tq, ok := m.userQuotas[uid]\n\tm.quotaMu.RUnlock()\n\n\tif !ok {\n\t\treturn false\n\t}\n\treturn q.check(space, inodes)\n}\n\nfunc (m *baseMeta) checkGroupQuota(ctx Context, gid uint64, space, inodes int64) bool {\n\tif !m.getFormat().UserGroupQuota {\n\t\treturn false\n\t}\n\n\tvar q *Quota\n\tm.quotaMu.RLock()\n\tq, ok := m.groupQuotas[gid]\n\tm.quotaMu.RUnlock()\n\n\tif !ok {\n\t\treturn false\n\t}\n\treturn q.check(space, inodes)\n}\n\nfunc (m *baseMeta) updateDirQuota(ctx Context, inode Ino, space, inodes int64) {\n\tif !m.getFormat().DirStats {\n\t\treturn\n\t}\n\tvar q *Quota\n\tvar st syscall.Errno\n\tfor {\n\t\tm.quotaMu.RLock()\n\t\tq = m.dirQuotas[uint64(inode)]\n\t\tm.quotaMu.RUnlock()\n\t\tif q != nil {\n\t\t\tq.update(space, inodes)\n\t\t}\n\t\tif inode <= RootInode {\n\t\t\tbreak\n\t\t}\n\t\tlastInode := inode\n\t\tif inode, st = m.getDirParent(ctx, inode); st != 0 {\n\t\t\tlogger.Warnf(\"Get directory parent of inode %d: %s\", lastInode, st)\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) updateUserGroupStat(ctx Context, uid, gid uint32, space, inodes int64) {\n\tif !m.getFormat().UserGroupQuota {\n\t\treturn\n\t}\n\tif (uid == 0 && gid == 0) || (space == 0 && inodes == 0) {\n\t\treturn\n\t}\n\tm.quotaMu.Lock()\n\tif uid > 0 {\n\t\tif uq := m.userQuotas[uint64(uid)]; uq != nil {\n\t\t\tuq.update(space, inodes)\n\t\t} else {\n\t\t\tm.userQuotas[uint64(uid)] = &Quota{\n\t\t\t\tUsedSpace: 0,\n\t\t\t\tUsedInodes: 0,\n\t\t\t\tMaxSpace: -1, // No limit\n\t\t\t\tMaxInodes: -1,\n\t\t\t\tnewSpace: space,\n\t\t\t\tnewInodes: inodes,\n\t\t\t}\n\t\t}\n\t}\n\tif gid > 0 {\n\t\tif gq := m.groupQuotas[uint64(gid)]; gq != nil {\n\t\t\tgq.update(space, inodes)\n\t\t} else {\n\t\t\tm.groupQuotas[uint64(gid)] = &Quota{\n\t\t\t\tUsedSpace: 0,\n\t\t\t\tUsedInodes: 0,\n\t\t\t\tMaxSpace: -1, // No limit\n\t\t\t\tMaxInodes: -1,\n\t\t\t\tnewSpace: space,\n\t\t\t\tnewInodes: inodes,\n\t\t\t}\n\t\t}\n\t}\n\tm.quotaMu.Unlock()\n}\n\nfunc (m *baseMeta) flushQuotas(ctx Context) {\n\tdefer m.sessWG.Done()\n\tticker := time.NewTicker(3 * time.Second)\n\tdefer ticker.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-ticker.C:\n\t\t\tm.doFlushQuotas()\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) collectQuotas(qtype uint32, quotas map[uint64]*Quota) []*iQuota {\n\tvar result []*iQuota\n\tfor key, q := range quotas {\n\t\tnewSpace := atomic.LoadInt64(&q.newSpace)\n\t\tnewInodes := atomic.LoadInt64(&q.newInodes)\n\t\tif newSpace != 0 || newInodes != 0 {\n\t\t\tresult = append(result, &iQuota{\n\t\t\t\tqtype: qtype,\n\t\t\t\tqkey: key,\n\t\t\t\tquota: &Quota{newSpace: newSpace, newInodes: newInodes},\n\t\t\t})\n\t\t}\n\t}\n\treturn result\n}\n\nfunc (m *baseMeta) updateQuota(q *Quota, newSpace, newInodes int64) {\n\tatomic.AddInt64(&q.newSpace, -newSpace)\n\tatomic.AddInt64(&q.UsedSpace, newSpace)\n\tatomic.AddInt64(&q.newInodes, -newInodes)\n\tatomic.AddInt64(&q.UsedInodes, newInodes)\n}\n\nfunc (m *baseMeta) doFlushQuotas() {\n\tif !m.getFormat().DirStats && !m.getFormat().UserGroupQuota {\n\t\treturn\n\t}\n\n\tvar allQuotas []*iQuota\n\tm.quotaMu.RLock()\n\tallQuotas = append(allQuotas, m.collectQuotas(DirQuotaType, m.dirQuotas)...)\n\tallQuotas = append(allQuotas, m.collectQuotas(UserQuotaType, m.userQuotas)...)\n\tallQuotas = append(allQuotas, m.collectQuotas(GroupQuotaType, m.groupQuotas)...)\n\tm.quotaMu.RUnlock()\n\n\tif len(allQuotas) == 0 {\n\t\treturn\n\t}\n\tif err := m.en.doFlushQuotas(Background(), allQuotas); err != nil {\n\t\tlogger.Warnf(\"Flush quotas: %s\", err)\n\t\treturn\n\t}\n\tm.quotaMu.Lock()\n\tfor _, snap := range allQuotas {\n\t\tvar q *Quota\n\t\tswitch snap.qtype {\n\t\tcase DirQuotaType:\n\t\t\tq = m.dirQuotas[snap.qkey]\n\t\tcase UserQuotaType:\n\t\t\tq = m.userQuotas[snap.qkey]\n\t\tcase GroupQuotaType:\n\t\t\tq = m.groupQuotas[snap.qkey]\n\t\t}\n\t\tif q != nil {\n\t\t\tm.updateQuota(q, snap.quota.newSpace, snap.quota.newInodes)\n\t\t}\n\t}\n\tm.quotaMu.Unlock()\n\n}\n\nfunc (m *baseMeta) HandleQuota(ctx Context, cmd uint8, dpath string, uid uint32, gid uint32, quotas map[string]*Quota, strict, repair bool, create bool) error {\n\tvar inode Ino\n\tif cmd != QuotaList && uid == 0 && gid == 0 {\n\t\tif st := m.resolve(ctx, dpath, &inode, create); st != 0 {\n\t\t\treturn fmt.Errorf(\"resolve dir %s: %s\", dpath, st)\n\t\t}\n\t\tif inode.IsTrash() {\n\t\t\treturn errors.New(\"no quota for any trash directory\")\n\t\t}\n\t}\n\n\tvar key uint64\n\tvar qtype uint32\n\tqtype = 0xffffffff\n\tif uid != 0 {\n\t\tqtype = UserQuotaType\n\t\tkey = uint64(uid)\n\t} else if gid != 0 {\n\t\tqtype = GroupQuotaType\n\t\tkey = uint64(gid)\n\t} else if dpath != \"\" {\n\t\tqtype = DirQuotaType\n\t\tkey = uint64(inode)\n\t}\n\n\tif cmd != QuotaList && qtype == 0xffffffff {\n\t\treturn fmt.Errorf(\"invalid quota type\")\n\t}\n\n\tswitch cmd {\n\tcase QuotaSet:\n\t\treturn m.handleQuotaSet(ctx, qtype, key, dpath, quotas, strict)\n\tcase QuotaGet:\n\t\treturn m.handleQuotaGet(ctx, qtype, key, dpath, quotas)\n\tcase QuotaDel:\n\t\treturn m.en.doDelQuota(ctx, qtype, key)\n\tcase QuotaList:\n\t\treturn m.handleQuotaList(ctx, qtype, key, quotas)\n\tcase QuotaCheck:\n\t\treturn m.handleQuotaCheck(ctx, qtype, key, dpath, strict, repair, quotas)\n\tdefault:\n\t\treturn fmt.Errorf(\"invalid quota command: %d\", cmd)\n\t}\n}\n\nfunc (m *baseMeta) handleQuotaSet(ctx Context, qtype uint32, key uint64, dpath string, quotas map[string]*Quota, strict bool) error {\n\tformat := m.getFormat()\n\tvar quota *Quota\n\tvar scan bool = false\n\tswitch qtype {\n\tcase DirQuotaType:\n\t\tif !format.DirStats {\n\t\t\tformat.DirStats = true\n\t\t\terr := m.en.doInit(format, false)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"init dir stats: %s\", err)\n\t\t\t}\n\t\t}\n\t\tquota = quotas[dpath]\n\tcase UserQuotaType:\n\t\tif !format.UserGroupQuota {\n\t\t\tformat.UserGroupQuota = true\n\t\t\tscan = true\n\t\t\terr := m.en.doInit(format, false)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"init user group quota: %s\", err)\n\t\t\t}\n\t\t}\n\t\tquota = quotas[fmt.Sprintf(\"uid:%d\", key)]\n\tcase GroupQuotaType:\n\t\tif !format.UserGroupQuota {\n\t\t\tformat.UserGroupQuota = true\n\t\t\tscan = true\n\t\t\terr := m.en.doInit(format, false)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"init user group quota: %s\", err)\n\t\t\t}\n\t\t}\n\t\tquota = quotas[fmt.Sprintf(\"gid:%d\", key)]\n\t}\n\tif quota == nil {\n\t\treturn nil\n\t}\n\n\tcreated, err := m.en.doSetQuota(ctx, qtype, uint64(key), &Quota{\n\t\tMaxSpace: quota.MaxSpace,\n\t\tMaxInodes: quota.MaxInodes,\n\t\tUsedSpace: -1,\n\t\tUsedInodes: -1,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !created {\n\t\treturn nil\n\t}\n\treturn m.initializeQuotaUsage(ctx, qtype, key, dpath, strict, scan)\n}\n\nfunc (m *baseMeta) initializeQuotaUsage(ctx Context, qtype uint32, key uint64, dpath string, strict bool, scan bool) error {\n\tswitch qtype {\n\tcase DirQuotaType:\n\t\twrapErr := func(e error) error {\n\t\t\treturn errors.Wrapf(e, \"set quota usage for file(%s), please repair it later\", dpath)\n\t\t}\n\n\t\tvar sum Summary\n\t\tif st := m.GetSummary(ctx, Ino(key), &sum, true, strict); st != 0 {\n\t\t\treturn wrapErr(st)\n\t\t}\n\n\t\t_, err := m.en.doSetQuota(ctx, DirQuotaType, key, &Quota{\n\t\t\tUsedSpace: int64(sum.Size) - align4K(0),\n\t\t\tUsedInodes: int64(sum.Dirs+sum.Files) - 1,\n\t\t\tMaxSpace: -1,\n\t\t\tMaxInodes: -1,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn wrapErr(err)\n\t\t}\n\t\treturn nil\n\tcase UserQuotaType, GroupQuotaType:\n\t\tif scan {\n\t\t\treturn m.ScanUserGroupUsage(ctx)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *baseMeta) ScanUserGroupUsage(ctx Context) error {\n\tuserUsage, groupUsage, err := m.scanGlobalUserGroupUsage(ctx)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"scan global user group usage: %v\", err)\n\t}\n\n\tvar userQuotasSnapshot map[uint64]*Quota\n\tvar groupQuotasSnapshot map[uint64]*Quota\n\n\tm.quotaMu.Lock()\n\t// Reset user and group quotas\n\tm.userQuotas = make(map[uint64]*Quota)\n\tm.groupQuotas = make(map[uint64]*Quota)\n\tfor uid, usage := range userUsage {\n\t\tm.userQuotas[uid] = &Quota{\n\t\t\tMaxSpace: -1,\n\t\t\tMaxInodes: -1,\n\t\t\tUsedSpace: int64(usage.Size),\n\t\t\tUsedInodes: int64(usage.Files),\n\t\t}\n\t}\n\tfor gid, usage := range groupUsage {\n\t\tm.groupQuotas[gid] = &Quota{\n\t\t\tMaxSpace: -1,\n\t\t\tMaxInodes: -1,\n\t\t\tUsedSpace: int64(usage.Size),\n\t\t\tUsedInodes: int64(usage.Files),\n\t\t}\n\t}\n\n\tuserQuotasSnapshot = make(map[uint64]*Quota)\n\tfor uid, quota := range m.userQuotas {\n\t\tuserQuotasSnapshot[uid] = &Quota{\n\t\t\tMaxSpace: atomic.LoadInt64("a.MaxSpace),\n\t\t\tMaxInodes: atomic.LoadInt64("a.MaxInodes),\n\t\t\tUsedSpace: atomic.LoadInt64("a.UsedSpace),\n\t\t\tUsedInodes: atomic.LoadInt64("a.UsedInodes),\n\t\t}\n\t}\n\n\tgroupQuotasSnapshot = make(map[uint64]*Quota)\n\tfor gid, quota := range m.groupQuotas {\n\t\tgroupQuotasSnapshot[gid] = &Quota{\n\t\t\tMaxSpace: atomic.LoadInt64("a.MaxSpace),\n\t\t\tMaxInodes: atomic.LoadInt64("a.MaxInodes),\n\t\t\tUsedSpace: atomic.LoadInt64("a.UsedSpace),\n\t\t\tUsedInodes: atomic.LoadInt64("a.UsedInodes),\n\t\t}\n\t}\n\tm.quotaMu.Unlock()\n\n\tfor uid, quota := range userQuotasSnapshot {\n\t\t_, err := m.en.doSetQuota(ctx, UserQuotaType, uid, quota)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"Failed to save user quota for uid %d: %v\", uid, err)\n\t\t}\n\t}\n\n\tfor gid, quota := range groupQuotasSnapshot {\n\t\t_, err := m.en.doSetQuota(ctx, GroupQuotaType, gid, quota)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"Failed to save group quota for gid %d: %v\", gid, err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (m *baseMeta) scanGlobalUserGroupUsage(ctx Context) (map[uint64]*Summary, map[uint64]*Summary, error) {\n\tuserUsage := make(map[uint64]*Summary)\n\tgroupUsage := make(map[uint64]*Summary)\n\n\tprocessedFiles := make(map[Ino]bool)\n\tvisitedDirs := make(map[Ino]bool)\n\n\tdirQueue := []Ino{RootInode}\n\tif m.getFormat().TrashDays > 0 {\n\t\tvar trashAttr Attr\n\t\tif st := m.en.doGetAttr(ctx, TrashInode, &trashAttr); st == 0 {\n\t\t\tdirQueue = append(dirQueue, TrashInode)\n\t\t}\n\t}\n\n\tfor len(dirQueue) > 0 {\n\t\tcurrentDir := dirQueue[0]\n\t\tdirQueue = dirQueue[1:]\n\n\t\tvar entries []*Entry\n\t\tvisitedDirs[currentDir] = true\n\t\terr := m.en.doReaddir(ctx, currentDir, 1, &entries, -1)\n\t\tif err != 0 {\n\t\t\tlogger.Warnf(\"readdir %d: %s\", currentDir, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor _, e := range entries {\n\t\t\tif string(e.Name) == \".\" || string(e.Name) == \"..\" {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tuid, gid := uint64(e.Attr.Uid), uint64(e.Attr.Gid)\n\t\t\tif (uid == 0 || gid == 0) && e.Attr.Typ == TypeFile {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif userUsage[uid] == nil {\n\t\t\t\tuserUsage[uid] = &Summary{}\n\t\t\t}\n\t\t\tif groupUsage[gid] == nil {\n\t\t\t\tgroupUsage[gid] = &Summary{}\n\t\t\t}\n\n\t\t\tvar space int64\n\t\t\tvar inodes int64\n\t\t\tif e.Attr.Typ == TypeFile {\n\t\t\t\tif e.Attr.Nlink > 1 {\n\t\t\t\t\tif processedFiles[e.Inode] {\n\t\t\t\t\t\tspace = 0\n\t\t\t\t\t\tinodes = 0\n\t\t\t\t\t} else {\n\t\t\t\t\t\tspace = align4K(e.Attr.Length)\n\t\t\t\t\t\tinodes = 1\n\t\t\t\t\t\tprocessedFiles[e.Inode] = true\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tspace = align4K(e.Attr.Length)\n\t\t\t\t\tinodes = 1\n\t\t\t\t}\n\t\t\t} else if e.Attr.Typ == TypeDirectory {\n\t\t\t\tspace = align4K(0)\n\t\t\t\tinodes = 1\n\t\t\t\tuserUsage[uid].Dirs++\n\t\t\t\tgroupUsage[gid].Dirs++\n\t\t\t\tif !visitedDirs[e.Inode] {\n\t\t\t\t\tdirQueue = append(dirQueue, e.Inode)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tuserUsage[uid].Size += uint64(space)\n\t\t\tuserUsage[uid].Files += uint64(inodes)\n\t\t\tgroupUsage[gid].Size += uint64(space)\n\t\t\tgroupUsage[gid].Files += uint64(inodes)\n\n\t\t}\n\t}\n\treturn userUsage, groupUsage, nil\n}\n\nfunc (m *baseMeta) handleQuotaGet(ctx Context, qtype uint32, key uint64, dpath string, quotas map[string]*Quota) error {\n\tq, err := m.en.doGetQuota(ctx, qtype, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif q == nil {\n\t\treturn nil\n\t}\n\tswitch qtype {\n\tcase DirQuotaType:\n\t\tquotas[dpath] = q\n\tcase UserQuotaType:\n\t\tquotas[fmt.Sprintf(\"uid:%d\", key)] = q\n\tcase GroupQuotaType:\n\t\tquotas[fmt.Sprintf(\"gid:%d\", key)] = q\n\t}\n\treturn nil\n}\n\nfunc (m *baseMeta) handleQuotaList(ctx Context, qtype uint32, key uint64, quotas map[string]*Quota) error {\n\tdirQuotas, userQuotas, groupQuotas, err := m.en.doLoadQuotas(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmatch := func(targetType uint32, k uint64, v *Quota) bool {\n\t\tif v.MaxInodes == -1 && v.MaxSpace == -1 {\n\t\t\treturn false\n\t\t}\n\t\treturn qtype == 0xffffffff || (qtype == targetType && k == key)\n\t}\n\n\tfor ino, quota := range dirQuotas {\n\t\tif !match(DirQuotaType, ino, quota) {\n\t\t\tcontinue\n\t\t}\n\t\tif ps := m.GetPaths(ctx, Ino(ino)); len(ps) > 0 {\n\t\t\tquotas[ps[0]] = quota\n\t\t} else {\n\t\t\tquotas[fmt.Sprintf(\"inode:%d\", ino)] = quota\n\t\t}\n\t}\n\tfor uid, quota := range userQuotas {\n\t\tif match(UserQuotaType, uid, quota) {\n\t\t\tquotas[fmt.Sprintf(\"uid:%d\", uid)] = quota\n\t\t}\n\t}\n\tfor gid, quota := range groupQuotas {\n\t\tif match(GroupQuotaType, gid, quota) {\n\t\t\tquotas[fmt.Sprintf(\"gid:%d\", gid)] = quota\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *baseMeta) handleQuotaCheck(ctx Context, qtype uint32, key uint64, dpath string, strict, repair bool, quotas map[string]*Quota) error {\n\tq, err := m.en.doGetQuota(ctx, qtype, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif q == nil {\n\t\treturn fmt.Errorf(\"no quota for inode %d path %s\", key, dpath)\n\t}\n\n\tvar sum Summary\n\tif st := m.GetSummary(ctx, Ino(key), &sum, true, strict); st != 0 {\n\t\treturn st\n\t}\n\n\tusedInodes := int64(sum.Dirs+sum.Files) - 1\n\tusedSpace := int64(sum.Size) - align4K(0) // quota ignore root dir\n\n\tif q.UsedInodes == usedInodes && q.UsedSpace == usedSpace {\n\t\tlogger.Infof(\"quota of %s is consistent\", dpath)\n\t\tquotas[dpath] = q\n\t\treturn nil\n\t}\n\n\tlogger.Warnf(\n\t\t\"%s: quota(%s, %s) != summary(%s, %s)\", dpath,\n\t\thumanize.Comma(q.UsedInodes), humanize.IBytes(uint64(q.UsedSpace)),\n\t\thumanize.Comma(usedInodes), humanize.IBytes(uint64(usedSpace)),\n\t)\n\n\tif repair {\n\t\tq.UsedInodes = usedInodes\n\t\tq.UsedSpace = usedSpace\n\t\tquotas[dpath] = q\n\t\tlogger.Info(\"repairing...\")\n\t\t_, err = m.en.doSetQuota(ctx, qtype, key, &Quota{\n\t\t\tMaxInodes: -1,\n\t\t\tMaxSpace: -1,\n\t\t\tUsedInodes: q.UsedInodes,\n\t\t\tUsedSpace: q.UsedSpace,\n\t\t})\n\t\treturn err\n\t}\n\n\treturn fmt.Errorf(\"quota of %s is inconsistent, please repair it with --repair flag\", dpath)\n}\n\nfunc (m *baseMeta) updateQuotaMetrics() {\n\tm.quotaMu.RLock()\n\tdirQuotasSnapshot := make(map[uint64]Quota)\n\tfor inode, quota := range m.dirQuotas {\n\t\tdirQuotasSnapshot[inode] = quota.snap()\n\t}\n\n\tuserQuotasSnapshot := make(map[uint64]Quota)\n\tfor uid, quota := range m.userQuotas {\n\t\tuserQuotasSnapshot[uid] = quota.snap()\n\t}\n\n\tgroupQuotasSnapshot := make(map[uint64]Quota)\n\tfor gid, quota := range m.groupQuotas {\n\t\tgroupQuotasSnapshot[gid] = quota.snap()\n\t}\n\tm.quotaMu.RUnlock()\n\tm.updateDirQuotaMetrics(dirQuotasSnapshot)\n\tm.updateUserQuotaMetrics(userQuotasSnapshot)\n\tm.updateGroupQuotaMetrics(groupQuotasSnapshot)\n}\n\nfunc (m *baseMeta) updateDirQuotaMetrics(dirQuotas map[uint64]Quota) {\n\tm.quotaMetricMu.Lock()\n\tdefer m.quotaMetricMu.Unlock()\n\tfor inode, quota := range dirQuotas {\n\t\tinodeStr := strconv.FormatUint(inode, 10)\n\t\tm.dirQuotaMaxSpaceG.WithLabelValues(inodeStr).Set(float64(quota.MaxSpace))\n\t\tm.dirQuotaMaxInodesG.WithLabelValues(inodeStr).Set(float64(quota.MaxInodes))\n\t\tm.dirQuotaUsedSpaceG.WithLabelValues(inodeStr).Set(float64(quota.UsedSpace))\n\t\tm.dirQuotaUsedInodesG.WithLabelValues(inodeStr).Set(float64(quota.UsedInodes))\n\t\tm.dirQuotaMetricKeys[inode] = true\n\t}\n}\n\nfunc (m *baseMeta) updateUserQuotaMetrics(userQuotas map[uint64]Quota) {\n\tm.quotaMetricMu.Lock()\n\tdefer m.quotaMetricMu.Unlock()\n\tfor uid, quota := range userQuotas {\n\t\tuidStr := strconv.FormatUint(uid, 10)\n\t\tm.userQuotaMaxSpaceG.WithLabelValues(uidStr).Set(float64(quota.MaxSpace))\n\t\tm.userQuotaMaxInodesG.WithLabelValues(uidStr).Set(float64(quota.MaxInodes))\n\t\tm.userQuotaUsedSpaceG.WithLabelValues(uidStr).Set(float64(quota.UsedSpace))\n\t\tm.userQuotaUsedInodesG.WithLabelValues(uidStr).Set(float64(quota.UsedInodes))\n\t\tm.userQuotaMetricKeys[uid] = true\n\t}\n}\n\nfunc (m *baseMeta) updateGroupQuotaMetrics(groupQuotas map[uint64]Quota) {\n\tm.quotaMetricMu.Lock()\n\tdefer m.quotaMetricMu.Unlock()\n\tfor gid, quota := range groupQuotas {\n\t\tgidStr := strconv.FormatUint(gid, 10)\n\t\tm.groupQuotaMaxSpaceG.WithLabelValues(gidStr).Set(float64(quota.MaxSpace))\n\t\tm.groupQuotaMaxInodesG.WithLabelValues(gidStr).Set(float64(quota.MaxInodes))\n\t\tm.groupQuotaUsedSpaceG.WithLabelValues(gidStr).Set(float64(quota.UsedSpace))\n\t\tm.groupQuotaUsedInodesG.WithLabelValues(gidStr).Set(float64(quota.UsedInodes))\n\t\tm.groupQuotaMetricKeys[gid] = true\n\t}\n}\n\nfunc (m *baseMeta) cleanupQuotaMetrics() {\n\tm.quotaMu.RLock()\n\tdirQuotas := make(map[uint64]Quota, len(m.dirQuotas))\n\tfor inode, q := range m.dirQuotas {\n\t\tdirQuotas[inode] = q.snap()\n\t}\n\tuserQuotas := make(map[uint64]Quota, len(m.userQuotas))\n\tfor uid, q := range m.userQuotas {\n\t\tuserQuotas[uid] = q.snap()\n\t}\n\tgroupQuotas := make(map[uint64]Quota, len(m.groupQuotas))\n\tfor gid, q := range m.groupQuotas {\n\t\tgroupQuotas[gid] = q.snap()\n\t}\n\tm.quotaMu.RUnlock()\n\n\tm.quotaMetricMu.Lock()\n\tdefer m.quotaMetricMu.Unlock()\n\n\t// directory quotas\n\tfor inode := range m.dirQuotaMetricKeys {\n\t\tq, ok := dirQuotas[inode]\n\t\tif !ok || (q.MaxSpace <= 0 && q.MaxInodes <= 0) {\n\t\t\tinodeStr := strconv.FormatUint(inode, 10)\n\t\t\tm.dirQuotaMaxSpaceG.DeleteLabelValues(inodeStr)\n\t\t\tm.dirQuotaMaxInodesG.DeleteLabelValues(inodeStr)\n\t\t\tm.dirQuotaUsedSpaceG.DeleteLabelValues(inodeStr)\n\t\t\tm.dirQuotaUsedInodesG.DeleteLabelValues(inodeStr)\n\t\t\tdelete(m.dirQuotaMetricKeys, inode)\n\t\t}\n\t}\n\n\t// user quotas\n\tfor uid := range m.userQuotaMetricKeys {\n\t\tq, ok := userQuotas[uid]\n\t\tif !ok || (q.MaxSpace <= 0 && q.MaxInodes <= 0) {\n\t\t\tuidStr := strconv.FormatUint(uid, 10)\n\t\t\tm.userQuotaMaxSpaceG.DeleteLabelValues(uidStr)\n\t\t\tm.userQuotaMaxInodesG.DeleteLabelValues(uidStr)\n\t\t\tm.userQuotaUsedSpaceG.DeleteLabelValues(uidStr)\n\t\t\tm.userQuotaUsedInodesG.DeleteLabelValues(uidStr)\n\t\t\tdelete(m.userQuotaMetricKeys, uid)\n\t\t}\n\t}\n\n\t// group quotas\n\tfor gid := range m.groupQuotaMetricKeys {\n\t\tq, ok := groupQuotas[gid]\n\t\tif !ok || (q.MaxSpace <= 0 && q.MaxInodes <= 0) {\n\t\t\tgidStr := strconv.FormatUint(gid, 10)\n\t\t\tm.groupQuotaMaxSpaceG.DeleteLabelValues(gidStr)\n\t\t\tm.groupQuotaMaxInodesG.DeleteLabelValues(gidStr)\n\t\t\tm.groupQuotaUsedSpaceG.DeleteLabelValues(gidStr)\n\t\t\tm.groupQuotaUsedInodesG.DeleteLabelValues(gidStr)\n\t\t\tdelete(m.groupQuotaMetricKeys, gid)\n\t\t}\n\t}\n}\n" }, { "path": "pkg/meta/random_test.go", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"math\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\t\"unicode\"\n\n\t\"pgregory.net/rapid\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/sirupsen/logrus\"\n)\n\ntype tSlice struct {\n\tpos uint32\n\tid uint64\n\tclen uint32\n\toff uint32\n\tlen uint32\n}\n\ntype tQuota struct {\n\tsize uint64\n\tinodes uint64\n}\n\ntype tNode struct {\n\tname string\n\tinode Ino\n\t_type uint8\n\tmode uint16\n\tuid uint32\n\tgid uint32\n\tatime int64\n\tmtime int64\n\tctime int64\n\tiflags uint8\n\tlength uint64\n\tparents []*tNode\n\thardlink bool\n\tchunks map[uint32][]tSlice\n\tchildren map[string]*tNode\n\ttarget string\n\txattrs map[string][]byte\n\tquota *tQuota\n\tflocks map[ownerKey]byte\n\tplocks map[ownerKey][]plockRecord\n\taccACL *aclAPI.Rule\n\tdefACL *aclAPI.Rule\n}\n\nfunc (n *tNode) accessMode(uid uint32, gids []uint32) uint8 {\n\tif uid == 0 {\n\t\treturn 0x7\n\t}\n\tmode := n.mode\n\tif uid == n.uid {\n\t\treturn uint8(mode>>6) & 7\n\t}\n\tfor _, gid := range gids {\n\t\tif gid == n.gid {\n\t\t\treturn uint8(mode>>3) & 7\n\t\t}\n\t}\n\treturn uint8(mode & 7)\n}\n\nfunc (n *tNode) access(ctx Context, mask uint8) bool {\n\tif ctx.Uid() == 0 {\n\t\treturn true\n\t}\n\n\tif n.accACL != nil && (n.mode&00070) != 0 {\n\t\treturn n.accACL.CanAccess(ctx.Uid(), ctx.Gids(), n.uid, n.gid, mask)\n\t}\n\n\tmode := n.accessMode(ctx.Uid(), ctx.Gids())\n\tif mode&mask != mask {\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc (n *tNode) stickyAccess(child *tNode, uid uint32) bool {\n\tif uid == 0 || n.mode&01000 == 0 {\n\t\treturn true\n\t}\n\tif uid == n.uid || uid == child.uid {\n\t\treturn true\n\t}\n\treturn false\n}\n\ntype fsMachine struct {\n\tnodes map[Ino]*tNode\n\tmeta Meta\n\tsid uint64\n\tctx Context\n}\n\nvar metaType string\nvar tCounter uint64\n\nfunc (m *fsMachine) Init(t *rapid.T) {\n\tm.sid = uint64(rapid.UintRange(1, math.MaxUint32-1).Draw(t, \"sid\"))\n\tm.nodes = make(map[Ino]*tNode)\n\tm.nodes[1] = &tNode{\n\t\t_type: TypeDirectory,\n\t\tmode: 0777,\n\t\tinode: RootInode,\n\t\tlength: 4096,\n\t\txattrs: make(map[string][]byte),\n\t\tchildren: make(map[string]*tNode),\n\t\tparents: []*tNode{{inode: RootInode, _type: TypeDirectory}},\n\t}\n\t_ = os.Remove(settingPath)\n\tm.meta = NewClient(metaURL, testConfig())\n\tm.meta.Reset()\n\tif err := m.meta.Init(testFormat(), true); err != nil {\n\t\tt.Fatalf(\"initialize failed: %s\", err)\n\t}\n\tm.meta.getBase().sessCtx = Background()\n\tm.meta.getBase().sid = m.sid\n\tregistry := prometheus.NewRegistry() // replace default so only JuiceFS metrics are exposed\n\tregisterer := prometheus.WrapRegistererWithPrefix(\"juicefs_\",\n\t\tprometheus.WrapRegistererWith(prometheus.Labels{\"mp\": \"virtual-mp\", \"vol_name\": \"test-vol\"}, registry))\n\tm.meta.InitMetrics(registerer)\n\n\tswitch m.meta.(type) {\n\tcase *dbMeta:\n\t\tmetaType = \"db\"\n\tcase *redisMeta:\n\t\tmetaType = \"redis\"\n\tcase *kvMeta:\n\t\tmetaType = \"tkv\"\n\t}\n\n\ttCounter++\n\tif tCounter%50 == 0 {\n\t\tfmt.Println(\"current counter: \", tCounter)\n\t}\n}\n\nfunc (m *fsMachine) genName(t *rapid.T) string {\n\tname := rapid.StringN(1, 200, 255).Draw(t, \"name\")\n\tname = strings.ReplaceAll(name, \"|\", \"a\") // FIXME: name can't contain '|'\n\tname = strings.ReplaceAll(name, \".#\", \"aa\")\n\tname = strings.ReplaceAll(name, \"\\n\", \"a\")\n\treturn name\n}\n\nfunc (m *fsMachine) Cleanup() {\n\tm.meta.CloseSession()\n\tm.meta.Reset()\n\tm.meta.Shutdown()\n}\n\nfunc (m *fsMachine) prepare(t *rapid.T) {\n\t// m.ctx.ts++\n\tuid := rapid.Uint32Range(0, 5).Draw(t, \"uid\")\n\tgid := rapid.Uint32Range(0, 5).Draw(t, \"gid\")\n\tm.ctx = NewContext(1, uid, []uint32{gid})\n\t// t.Logf(\"time: %d\", m.ctx.ts)\n}\n\nfunc (m *fsMachine) pickNode(t *rapid.T) Ino {\n\tm.prepare(t)\n\tvar inodes []Ino\n\tfor inode := range m.nodes {\n\t\tinodes = append(inodes, Ino(inode))\n\t}\n\tsort.Slice(inodes, func(i, j int) bool { return inodes[i] < inodes[j] })\n\treturn rapid.SampledFrom(inodes).Draw(t, \"node\")\n}\n\nfunc (m *fsMachine) create(_type uint8, parent Ino, name string, mode, umask uint16, inode Ino) syscall.Errno {\n\tif _type < TypeFile || _type == TypeSymlink {\n\t\treturn syscall.EINVAL\n\t}\n\tif err := checkFSNodeName(name); err != 0 {\n\t\treturn err\n\t}\n\tp := m.nodes[parent]\n\tif p == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif p.children == nil {\n\t\treturn syscall.ENOTDIR\n\t}\n\n\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\treturn syscall.EACCES\n\t}\n\tif p.children[name] != nil {\n\t\treturn syscall.EEXIST\n\t}\n\tn := &tNode{\n\t\tname: name,\n\t\t_type: _type,\n\t\tmode: mode &^ umask,\n\t\tinode: inode,\n\t\tuid: m.ctx.Uid(),\n\t\tgid: m.ctx.Gids()[0],\n\t\tparents: []*tNode{p},\n\t\txattrs: make(map[string][]byte),\n\t}\n\n\tif runtime.GOOS == \"darwin\" {\n\t\tn.gid = p.gid\n\t} else if runtime.GOOS == \"linux\" && p.mode&02000 != 0 {\n\t\tn.gid = p.gid\n\t\tif _type == TypeDirectory {\n\t\t\tp.mode |= 02000\n\t\t} else if n.mode&02010 == 02010 && m.ctx.Uid() != 0 {\n\t\t\tvar found bool\n\t\t\tfor _, gid := range m.ctx.Gids() {\n\t\t\t\tif gid == p.gid {\n\t\t\t\t\tfound = true\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !found {\n\t\t\t\tn.mode &= ^uint16(02000)\n\t\t\t}\n\t\t}\n\t}\n\n\tmode &= 07777\n\tif p.defACL != nil && _type != TypeSymlink {\n\t\t// inherit default acl\n\t\tif _type == TypeDirectory {\n\t\t\tn.defACL = p.defACL\n\t\t}\n\n\t\t// set access acl by parent's default acl\n\t\trule := p.defACL\n\n\t\tif rule.IsMinimal() {\n\t\t\t// simple acl as default\n\t\t\tn.mode = mode & (0xFE00 | rule.GetMode())\n\t\t} else {\n\t\t\tcRule := rule.ChildAccessACL(mode)\n\t\t\tn.accACL = cRule\n\t\t\tn.mode = (mode & 0xFE00) | cRule.GetMode()\n\t\t}\n\t} else {\n\t\tn.mode = mode & ^umask\n\t}\n\n\tswitch _type {\n\tcase TypeDirectory:\n\t\tn.children = make(map[string]*tNode)\n\t\tn.length = 4 << 10\n\tcase TypeFile:\n\t\tn.chunks = make(map[uint32][]tSlice)\n\tcase TypeSymlink:\n\t\tn.length = uint64(len(name))\n\tdefault:\n\t\tn.length = 0\n\t}\n\n\t// p.mtime = m.ctx.ts\n\t// p.ctime = m.ctx.ts\n\tm.nodes[inode] = n\n\tp.children[name] = n\n\treturn 0\n}\n\nfunc checkFSNodeName(name string) syscall.Errno {\n\tlen := len(name)\n\tif len == 0 {\n\t\treturn syscall.EINVAL\n\t}\n\tif len > MaxName {\n\t\treturn syscall.ENAMETOOLONG\n\t}\n\tif name[0] == '.' {\n\t\tif len == 1 {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif len == 2 && name[1] == '.' {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t}\n\tif strings.ContainsAny(name, \"/\\x00\") {\n\t\treturn syscall.EINVAL\n\t}\n\treturn 0\n}\n\nfunc (m *fsMachine) link(parent Ino, name string, inode Ino) syscall.Errno {\n\tif name == \".\" || name == \"..\" {\n\t\treturn syscall.EEXIST\n\t}\n\tif err := checkFSNodeName(name); err != 0 {\n\t\treturn err\n\t}\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif n.children != nil {\n\t\treturn syscall.EPERM\n\t}\n\tp := m.nodes[parent]\n\tif p == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif p.children == nil {\n\t\treturn syscall.ENOTDIR\n\t}\n\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\treturn syscall.EACCES\n\t}\n\tif p.children[name] != nil {\n\t\treturn syscall.EEXIST\n\t}\n\t// n.ctime = m.ctx.ts\n\t// p.mtime = m.ctx.ts\n\t// p.ctime = m.ctx.ts\n\tn.parents = append(n.parents, p)\n\tn.hardlink = true\n\tp.children[name] = n\n\treturn 0\n}\n\nfunc (m *fsMachine) symlink(parent Ino, name string, inode Ino, target string) syscall.Errno {\n\tif len(target) == 0 || len(target) > SymlinkMax {\n\t\treturn syscall.EINVAL\n\t}\n\tfor _, c := range target {\n\t\tif c == 0 {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t}\n\tif err := checkFSNodeName(name); err != 0 {\n\t\treturn err\n\t}\n\tp := m.nodes[parent]\n\tif p == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif p.children == nil {\n\t\treturn syscall.ENOTDIR\n\t}\n\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\treturn syscall.EACCES\n\t}\n\tif p.children[name] != nil {\n\t\treturn syscall.EEXIST\n\t}\n\tn := &tNode{\n\t\tname: name,\n\t\t_type: TypeSymlink,\n\t\tinode: inode,\n\t\tmode: 0777,\n\t\tuid: m.ctx.Uid(),\n\t\tgid: m.ctx.Gids()[0],\n\t\t// atime: m.ctx.ts,\n\t\t// mtime: m.ctx.ts,\n\t\t// ctime: m.ctx.ts,\n\t\tparents: []*tNode{p},\n\t\ttarget: target,\n\t\txattrs: make(map[string][]byte),\n\t}\n\n\t_type := TypeSymlink\n\tif runtime.GOOS == \"darwin\" {\n\t\tn.gid = p.gid\n\t} else if runtime.GOOS == \"linux\" && p.mode&02000 != 0 {\n\t\tn.gid = p.gid\n\t\tif _type == TypeDirectory {\n\t\t\tp.mode |= 02000\n\t\t} else if n.mode&02010 == 02010 && m.ctx.Uid() != 0 {\n\t\t\tvar found bool\n\t\t\tfor _, gid := range m.ctx.Gids() {\n\t\t\t\tif gid == p.gid {\n\t\t\t\t\tfound = true\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !found {\n\t\t\t\tn.mode &= ^uint16(02000)\n\t\t\t}\n\t\t}\n\t}\n\n\tn.length = uint64(len(target))\n\t// p.mtime = m.ctx.ts\n\t// p.ctime = m.ctx.ts\n\tm.nodes[inode] = n\n\tp.children[name] = n\n\treturn 0\n}\n\nfunc (m *fsMachine) readlink(inode Ino) (string, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn \"\", syscall.ENOENT\n\t}\n\tif n.target == \"\" {\n\t\treturn \"\", syscall.EINVAL\n\t}\n\treturn n.target, 0\n}\n\nfunc (m *fsMachine) pickChild(parent Ino, t *rapid.T) string {\n\tn := m.nodes[parent]\n\tif len(n.children) == 0 {\n\t\treturn \"\"\n\t}\n\tvar names []string\n\tfor name := range n.children {\n\t\tnames = append(names, name)\n\t}\n\tsort.Slice(names, func(i, j int) bool { return names[i] < names[j] })\n\treturn rapid.SampledFrom(names).Draw(t, \"child\")\n}\n\nfunc (m *fsMachine) unlink(parent Ino, name string) syscall.Errno {\n\tp := m.nodes[parent]\n\tif p == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif p._type != TypeDirectory {\n\t\treturn syscall.ENOTDIR\n\t}\n\n\tif metaType == \"db\" {\n\t\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\t\treturn syscall.EACCES\n\t\t}\n\t}\n\tc := p.children[name]\n\n\tif c._type == TypeDirectory {\n\t\treturn syscall.EPERM\n\t}\n\n\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\treturn syscall.EACCES\n\t}\n\n\tif _, ok := p.children[name]; !ok {\n\t\treturn syscall.ENOENT\n\t}\n\tif err := checkFSNodeName(name); err != 0 {\n\t\treturn err\n\t}\n\n\tif !p.stickyAccess(c, m.ctx.Uid()) {\n\t\treturn syscall.EACCES\n\t}\n\n\tdelete(p.children, name)\n\tfor i, tp := range c.parents {\n\t\tif tp == p {\n\t\t\tc.parents = append(c.parents[:i], c.parents[i+1:]...)\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(c.parents) == 0 {\n\t\tdelete(m.nodes, c.inode)\n\t} else {\n\t\t// c.ctime = m.ctx.ts\n\t}\n\t// p.mtime = m.ctx.ts\n\t// p.ctime = m.ctx.ts\n\treturn 0\n}\n\nfunc (m *fsMachine) rmdir(parent Ino, name string) syscall.Errno {\n\tp := m.nodes[parent]\n\tif p == nil {\n\t\treturn syscall.ENOENT\n\t}\n\n\tif metaType == \"db\" {\n\t\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\t\treturn syscall.EACCES\n\t\t}\n\t}\n\tc := p.children[name]\n\n\tif c._type != TypeDirectory {\n\t\treturn syscall.ENOTDIR\n\t}\n\n\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\treturn syscall.EACCES\n\t}\n\tif _, ok := p.children[name]; !ok {\n\t\treturn syscall.ENOENT\n\t}\n\tif err := checkFSNodeName(name); err != 0 {\n\t\treturn err\n\t}\n\n\tif len(c.children) != 0 {\n\t\treturn syscall.ENOTEMPTY\n\t}\n\n\tif !p.stickyAccess(c, m.ctx.Uid()) {\n\t\treturn syscall.EACCES\n\t}\n\n\tdelete(p.children, name)\n\tfor i, tp := range c.parents {\n\t\tif tp == p {\n\t\t\tc.parents = append(c.parents[:i], c.parents[i+1:]...)\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(c.parents) == 0 {\n\t\tdelete(m.nodes, c.inode)\n\t} else {\n\t\t// c.ctime = m.ctx.ts\n\t}\n\t// p.mtime = m.ctx.ts\n\t// p.ctime = m.ctx.ts\n\treturn 0\n}\n\nfunc (m *fsMachine) lookup(parent Ino, name string, checkPerm bool) (Ino, syscall.Errno) {\n\tp := m.nodes[parent]\n\tif checkPerm {\n\t\tif !p.access(m.ctx, MODE_MASK_X) {\n\t\t\treturn 0, syscall.EACCES\n\t\t}\n\t}\n\tif _, ok := p.children[name]; !ok {\n\t\treturn 0, syscall.ENOENT\n\t}\n\tif p == nil {\n\t\treturn 0, syscall.ENOENT\n\t}\n\tif err := checkFSNodeName(name); err != 0 {\n\t\treturn 0, err\n\t}\n\t//if p.children == nil {\n\t//\treturn 0, syscall.ENOENT\n\t//}\n\tif !p.access(m.ctx, MODE_MASK_X) {\n\t\treturn 0, syscall.EACCES\n\t}\n\tc := p.children[name]\n\tif c == nil {\n\t\treturn 0, syscall.ENOENT\n\t}\n\treturn c.inode, 0\n}\n\nfunc (m *fsMachine) getattr(inode Ino) (*tNode, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn nil, syscall.ENOENT\n\t}\n\treturn n, 0\n}\n\nfunc (m *fsMachine) doMknod(inode Ino) (*tNode, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn nil, syscall.ENOENT\n\t}\n\treturn n, 0\n}\n\nfunc (m *fsMachine) setattr(inode Ino, attr Attr) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\t// FIXME: check attr\n\treturn 0\n}\n\nfunc (m *fsMachine) truncate(inode Ino, length uint64) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif n._type != TypeFile {\n\t\treturn syscall.EPERM\n\t}\n\tif !n.access(m.ctx, MODE_MASK_W) {\n\t\treturn syscall.EACCES\n\t}\n\tfor i := range n.chunks {\n\t\tif uint64(i)*ChunkSize >= length {\n\t\t\tdelete(n.chunks, i)\n\t\t} else if uint64(i)*ChunkSize+ChunkSize > length {\n\t\t\tvar slices []tSlice\n\t\t\tfor _, s := range n.chunks[i] {\n\t\t\t\tif s.pos < uint32(length-uint64(i)*ChunkSize) {\n\t\t\t\t\tif s.pos+s.len > uint32(length-uint64(i)*ChunkSize) {\n\t\t\t\t\t\ts.len = uint32(length-uint64(i)*ChunkSize) - s.pos\n\t\t\t\t\t}\n\t\t\t\t\tslices = append(slices, tSlice{s.pos, s.id, s.clen, s.off, s.len})\n\t\t\t\t}\n\t\t\t}\n\t\t\tn.chunks[i] = slices\n\t\t}\n\t}\n\tn.length = length\n\t// n.mtime = m.ctx.ts\n\t// n.ctime = m.ctx.ts\n\treturn 0\n}\n\nfunc (m *fsMachine) fallocate(inode Ino, mode uint8, offset uint64, size uint64) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif n._type != TypeFile {\n\t\treturn syscall.EPERM\n\t}\n\tif !n.access(m.ctx, MODE_MASK_W) {\n\t\treturn syscall.EACCES\n\t}\n\tif offset+size > n.length {\n\t\tn.length = offset + size\n\t}\n\t// n.mtime = m.ctx.ts\n\t// n.ctime = m.ctx.ts\n\treturn 0\n}\n\nfunc (m *fsMachine) copy_file_range(srcinode Ino, srcoff uint64, dstinode Ino, dstoff uint64, size uint64, flags uint64) syscall.Errno {\n\t//if srcinode == dstinode && (size == 0 || srcoff <= dstoff && dstoff < srcoff+size || dstoff < srcoff && srcoff < dstoff+size) {\n\t//\treturn syscall.EINVAL // overlap\n\t//}\n\tsrc := m.nodes[srcinode]\n\tif src == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif src._type != TypeFile {\n\t\treturn syscall.EINVAL\n\t}\n\tdst := m.nodes[dstinode]\n\tif dst == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif dst._type != TypeFile {\n\t\treturn syscall.EINVAL\n\t}\n\t//if !src.access(m.ctx, MODE_MASK_R) {\n\t//\treturn syscall.EACCES\n\t//}\n\t//if !dst.access(m.ctx, MODE_MASK_W) {\n\t//\treturn syscall.EACCES\n\t//}\n\tupdateChunk := func(off uint64, s tSlice) {\n\t\tfor s.len > 0 {\n\t\t\tindx := uint32(off / ChunkSize)\n\t\t\tpos := uint32(off % ChunkSize)\n\t\t\tlen := uint32(ChunkSize - pos)\n\t\t\tif len > s.len {\n\t\t\t\tlen = s.len\n\t\t\t}\n\t\t\tdst.chunks[indx] = append(dst.chunks[indx], tSlice{pos, s.id, s.clen, s.off, len})\n\t\t\ts.off += len\n\t\t\ts.len -= len\n\t\t\toff += uint64(len)\n\t\t}\n\t}\n\tif srcoff >= src.length {\n\t\treturn 0\n\t}\n\tif srcoff+size > src.length {\n\t\tsize = src.length - srcoff\n\t}\n\tif dstoff+size > dst.length {\n\t\tdst.length = dstoff + size\n\t}\n\tfor size > 0 {\n\t\tindx := uint32(srcoff / ChunkSize)\n\t\tpos := uint32(srcoff % ChunkSize)\n\t\tl := uint32(ChunkSize - pos)\n\t\tif srcoff < src.length && srcoff+uint64(l) > src.length {\n\t\t\tl = uint32(src.length - srcoff)\n\t\t}\n\t\tif uint64(l) > size {\n\t\t\tl = uint32(size)\n\t\t}\n\n\t\tupdateChunk(dstoff, tSlice{0, 0, 0, 0, l})\n\t\tvar cs []tSlice\n\t\tcs = append(cs, src.chunks[indx]...) // copy\n\t\tfor _, s := range cs {\n\t\t\tif s.pos+s.len <= pos || s.pos >= pos+l {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif s.pos+s.len > pos+l {\n\t\t\t\ts.len = pos + l - s.pos\n\t\t\t}\n\t\t\tif s.pos < pos {\n\t\t\t\tdiff := pos - s.pos\n\t\t\t\ts.off += diff\n\t\t\t\ts.len -= diff\n\t\t\t\ts.pos = pos\n\t\t\t}\n\t\t\tupdateChunk(dstoff+uint64(s.pos-pos), s)\n\t\t}\n\t\tsrcoff += uint64(l)\n\t\tdstoff += uint64(l)\n\t\tsize -= uint64(l)\n\t}\n\t// dst.mtime = m.ctx.ts\n\t// dst.ctime = m.ctx.ts\n\treturn 0\n}\n\n// rmr Hint: Unlike the Rmr with the meta interface.\nfunc (m *fsMachine) rmr(parent Ino, name string, removed *uint64) syscall.Errno {\n\tp := m.nodes[parent]\n\tif p == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\treturn syscall.EACCES\n\t}\n\tif p.children == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif err := checkFSNodeName(name); err != 0 {\n\t\treturn err\n\t}\n\n\tc := p.children[name]\n\tif c == nil {\n\t\treturn syscall.ENOENT\n\t}\n\n\tif !p.stickyAccess(c, m.ctx.Uid()) {\n\t\treturn syscall.EPERM\n\t}\n\tfor n := range c.children {\n\t\tif eno := m.rmr(c.inode, n, removed); eno != 0 {\n\t\t\treturn eno\n\t\t}\n\t}\n\n\tif !p.access(m.ctx, MODE_MASK_W|MODE_MASK_X) {\n\t\treturn syscall.EACCES\n\t}\n\n\tvar st syscall.Errno\n\tif c._type == TypeDirectory {\n\t\tst = m.rmdir(parent, name)\n\t} else {\n\t\tst = m.unlink(parent, name)\n\t}\n\tif st == 0 && removed != nil {\n\t\t*removed++\n\t}\n\treturn 0\n}\n\nfunc (m *fsMachine) isancestor(a, b *tNode) bool {\n\tif a == b {\n\t\treturn true\n\t}\n\tfor _, p := range b.parents {\n\t\tif m.isancestor(a, p) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (m *fsMachine) rename(srcparent Ino, srcname string, dstparent Ino, dstname string, flag uint8) syscall.Errno {\n\tif dstparent == srcparent && dstname == srcname {\n\t\treturn 0\n\t}\n\tif err := checkFSNodeName(dstname); err != 0 {\n\t\treturn err\n\t}\n\t// todo: The order of condition checks in different metadata engines is inconsistent\n\tif metaType == \"db\" {\n\t\tsrc := m.nodes[srcparent]\n\t\tif src == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tdst := m.nodes[dstparent]\n\t\tif dst == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif src._type != TypeDirectory || dst._type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif !src.access(m.ctx, MODE_MASK_X|MODE_MASK_W) {\n\t\t\treturn syscall.EACCES\n\t\t}\n\t\tif !dst.access(m.ctx, MODE_MASK_X|MODE_MASK_W) {\n\t\t\treturn syscall.EACCES\n\t\t}\n\t}\n\n\tif metaType == \"redis\" {\n\t\tsrc := m.nodes[srcparent]\n\t\tif src == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tdst := m.nodes[dstparent]\n\t\tif dst == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tsrcnode := src.children[srcname]\n\t\tif srcnode == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tc := dst.children[dstname]\n\t\tif c != nil {\n\t\t\tif srcnode._type == TypeDirectory && c._type != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t} else if srcnode._type != TypeDirectory && c._type == TypeDirectory {\n\t\t\t\treturn syscall.EISDIR\n\t\t\t}\n\t\t}\n\n\t}\n\tsrc := m.nodes[srcparent]\n\tif src == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif src.children == nil {\n\t\treturn syscall.ENOTDIR\n\t}\n\tif !src.access(m.ctx, MODE_MASK_X|MODE_MASK_W) {\n\t\treturn syscall.EACCES\n\t}\n\n\tdst := m.nodes[dstparent]\n\tif dst == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif dst.children == nil {\n\t\treturn syscall.ENOTDIR\n\t}\n\tif !dst.access(m.ctx, MODE_MASK_X|MODE_MASK_W) {\n\t\treturn syscall.EACCES\n\t}\n\n\tsrcnode := src.children[srcname]\n\tif srcnode == nil {\n\t\treturn syscall.ENOENT\n\t}\n\n\tif metaType == \"tkv\" {\n\t\tc := dst.children[dstname]\n\t\tif c != nil {\n\t\t\tif srcnode._type == TypeDirectory && c._type != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t} else if srcnode._type != TypeDirectory && c._type == TypeDirectory {\n\t\t\t\treturn syscall.EISDIR\n\t\t\t}\n\t\t}\n\t}\n\n\tif !src.stickyAccess(srcnode, m.ctx.Uid()) {\n\t\treturn syscall.EACCES\n\t}\n\n\t// owner of a directory cannot rename subdirectories owned by other users.\n\tuid := m.ctx.Uid()\n\tif src != dst && src.mode&0o1000 != 0 && uid != 0 &&\n\t\tuid != srcnode.uid && (uid != src.uid || srcnode._type == TypeDirectory) {\n\t\treturn syscall.EACCES\n\t}\n\n\tif c := dst.children[dstname]; c != nil {\n\t\tif c == srcnode {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif flag != 0 {\n\t\t} else if srcnode._type == TypeDirectory && c._type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t} else if srcnode._type != TypeDirectory && c._type == TypeDirectory {\n\t\t\treturn syscall.EISDIR\n\t\t}\n\t\tif len(c.children) != 0 {\n\t\t\treturn syscall.ENOTEMPTY\n\t\t}\n\t\tif dst != src || dstname != srcname {\n\t\t\tif !dst.stickyAccess(c, m.ctx.Uid()) {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t\tif st := m.rmr(dst.inode, dstname, nil); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t}\n\t}\n\tfor i, tp := range srcnode.parents {\n\t\tif tp == src {\n\t\t\tsrcnode.parents[i] = dst\n\t\t\tbreak\n\t\t}\n\t}\n\tdelete(src.children, srcname)\n\tsrcnode.name = dstname\n\tdst.children[dstname] = srcnode\n\t// srcnode.ctime = m.ctx.ts\n\t// src.mtime = m.ctx.ts\n\t// src.ctime = m.ctx.ts\n\t// dst.mtime = m.ctx.ts\n\t// dst.ctime = m.ctx.ts\n\treturn 0\n}\n\ntype tEntry struct {\n\tname string\n\tnode *tNode\n}\n\nfunc (m *fsMachine) readdir(inode Ino) ([]*tEntry, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn nil, syscall.ENOENT\n\t}\n\tif !n.access(m.ctx, MODE_MASK_R) {\n\t\treturn nil, syscall.EACCES\n\t}\n\tvar result []*tEntry\n\tresult = append(result, &tEntry{\n\t\tname: \".\",\n\t\tnode: n,\n\t}, &tEntry{\n\t\tname: \"..\",\n\t\tnode: n.parents[0],\n\t})\n\n\tfor name, node := range n.children {\n\t\tresult = append(result, &tEntry{\n\t\t\tname: name,\n\t\t\tnode: node,\n\t\t})\n\t}\n\tsort.Slice(result, func(i, j int) bool { return result[i].name < result[j].name })\n\treturn result, 0\n}\n\nfunc (m *fsMachine) write(inode Ino, indx uint32, pos uint32, chunkid uint64, cleng uint32, off, len uint32) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif n._type != TypeFile {\n\t\treturn syscall.EPERM\n\t}\n\tif len == 0 {\n\t\treturn 0\n\t}\n\t//pos = pos % ChunkSize // fix invalid pos\n\t//if chunkid == 0 || cleng == 0 || len == 0 || pos+len > ChunkSize || off+len > cleng {\n\t//\treturn syscall.EINVAL\n\t//}\n\tn.chunks[indx] = append(n.chunks[indx], tSlice{pos, chunkid, cleng, off, len})\n\tif uint64(indx)*ChunkSize+uint64(pos+len) > n.length {\n\t\tn.length = uint64(indx)*ChunkSize + uint64(pos) + uint64(len)\n\t}\n\treturn 0\n}\n\nfunc (m *fsMachine) read(inode Ino, indx uint32) (uint64, []tSlice, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn 0, nil, syscall.ENOENT\n\t}\n\tif n._type != TypeFile {\n\t\treturn 0, nil, syscall.EPERM\n\t}\n\t// if !n.access(m.ctx, MODE_MASK_R) {\n\t// \treturn 0, nil, \"\", syscall.EACCES\n\t// }\n\tvar ss []*slice\n\tvar clen = make(map[uint64]uint32)\n\tfor _, s := range n.chunks[indx] {\n\t\tss = append(ss, &slice{id: s.id, off: s.off, len: s.len, pos: s.pos})\n\t\tclen[s.id] = s.clen\n\t}\n\tcs := buildSlice2(ss)\n\tfor i := range cs {\n\t\tif _, ok := clen[cs[i].id]; ok {\n\t\t\tcs[i].clen = clen[cs[i].id]\n\t\t}\n\t}\n\treturn n.length, cs, 0\n}\n\nfunc buildSlice2(ss []*slice) []tSlice {\n\tif len(ss) == 0 {\n\t\treturn nil\n\t}\n\tvar root *slice\n\tfor i := range ss {\n\t\ts := new(slice)\n\t\t*s = *ss[i]\n\t\tvar right *slice\n\t\ts.left, right = root.cut(s.pos)\n\t\t_, s.right = right.cut(s.pos + s.len)\n\t\troot = s\n\t}\n\t// root.optimize(1)\n\tvar pos uint32\n\tvar chunk []tSlice\n\troot.visit(func(s *slice) {\n\t\tif s.pos > pos {\n\t\t\tchunk = append(chunk, tSlice{pos: pos, len: s.pos - pos, clen: s.pos - pos})\n\t\t\tpos = s.pos\n\t\t}\n\t\tchunk = append(chunk, tSlice{pos: pos, id: s.id, off: s.off, len: s.len})\n\t\tpos += s.len\n\t})\n\treturn chunk\n}\n\nfunc (m *fsMachine) setxattr(inode Ino, name string, value []byte, mode uint8) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\t// if !xattr.check(name) {\n\t// \treturn syscall.EINVAL\n\t// }\n\tswitch mode {\n\tcase XattrCreate:\n\t\tif n.xattrs[name] != nil {\n\t\t\treturn syscall.EEXIST\n\t\t}\n\t\tn.xattrs[name] = value\n\tcase XattrReplace:\n\t\tif n.xattrs[name] == nil {\n\t\t\treturn ENOATTR\n\t\t}\n\t\tn.xattrs[name] = value\n\tcase XattrCreateOrReplace:\n\t\tn.xattrs[name] = value\n\tdefault:\n\t\treturn syscall.EINVAL\n\t}\n\t// n.ctime = m.ctx.ts\n\treturn 0\n}\n\nfunc (m *fsMachine) removexattr(inode Ino, name string) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\t// if !xattr.check(name) {\n\t// \treturn syscall.EINVAL\n\t// }\n\tif n.xattrs[name] == nil {\n\t\treturn ENOATTR\n\t}\n\t// n.ctime = m.ctx.ts\n\tdelete(n.xattrs, name)\n\treturn 0\n}\n\nfunc (m *fsMachine) getxattr(inode Ino, name string) ([]byte, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn nil, syscall.ENOENT\n\t}\n\t// if !xattr.check(name) {\n\t// \treturn nil, syscall.EINVAL\n\t// }\n\tif v, ok := n.xattrs[name]; ok {\n\t\treturn v, 0\n\t}\n\treturn nil, ENOATTR\n}\n\nfunc (m *fsMachine) listxattr(inode Ino) ([]byte, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn nil, syscall.ENOENT\n\t}\n\tvar names []string\n\tfor name := range n.xattrs {\n\t\tnames = append(names, name+\"\\x00\")\n\t}\n\n\tif n.accACL != nil {\n\t\tnames = append(names, \"system.posix_acl_access\"+\"\\x00\")\n\t}\n\tif n.defACL != nil {\n\t\tnames = append(names, \"system.posix_acl_default\"+\"\\x00\")\n\t}\n\n\tsort.Slice(names, func(i, j int) bool { return names[i] < names[j] })\n\tr := []byte(strings.Join(names, \"\"))\n\tif len(r) > 65536 {\n\t\treturn nil, syscall.ERANGE\n\t}\n\treturn r, 0\n}\n\nfunc (m *fsMachine) Mkdir(t *rapid.T) {\n\tparent := m.pickNode(t)\n\tname := rapid.StringN(1, 200, 255).Draw(t, \"name\")\n\tmode := rapid.Uint16Range(0, 01777).Draw(t, \"mode\")\n\tif name == \".\" || name == \"..\" {\n\t\tt.Skipf(\"skip mkdir %s\", name)\n\t}\n\tvar inode Ino\n\tvar attr Attr\n\tst := m.meta.Mkdir(m.ctx, parent, name, mode, 0, 0, &inode, &attr)\n\tt.Logf(\"parent ino %d, dir ino %d\", parent, inode)\n\t//var attr2 Attr\n\t//m.meta.GetAttr(m.ctx, inode, &attr2)\n\tst2 := m.create(TypeDirectory, parent, name, mode, 0, inode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) Mknod(t *rapid.T) {\n\tparent := m.pickNode(t)\n\tname := rapid.StringN(1, 200, 255).Draw(t, \"name\")\n\tif name == \".\" || name == \"..\" {\n\t\tt.Skipf(\"skip mknod %s\", name)\n\t}\n\t_type := rapid.Uint8Range(0, TypeDirectory).Draw(t, \"type\")\n\tmode := rapid.Uint16Range(0, 01777).Draw(t, \"mode\")\n\tvar inode Ino\n\tvar attr Attr\n\tst := m.meta.Mknod(m.ctx, parent, name, _type, mode, 0, 0, \"\", &inode, &attr)\n\tst2 := m.create(_type, parent, name, mode, 0, inode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) Link(t *rapid.T) {\n\tparent := m.pickNode(t)\n\tname := m.genName(t)\n\tinode := m.pickNode(t)\n\tst := m.meta.Link(m.ctx, inode, parent, name, nil)\n\tst2 := m.link(parent, name, inode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\nfunc (m *fsMachine) Rmdir(t *rapid.T) {\n\tparent := m.pickNode(t)\n\tname := m.pickChild(parent, t)\n\tif name == \"\" {\n\t\treturn\n\t}\n\tst := m.meta.Rmdir(m.ctx, parent, name)\n\tst2 := m.rmdir(parent, name)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) Unlink(t *rapid.T) {\n\tparent := m.pickNode(t)\n\tname := m.pickChild(parent, t)\n\tif name == \"\" {\n\t\treturn\n\t}\n\tst := m.meta.Unlink(m.ctx, parent, name)\n\tst2 := m.unlink(parent, name)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nconst SymlinkMax = 65536\n\nfunc (m *fsMachine) Symlink(t *rapid.T) {\n\tparent := m.pickNode(t)\n\tname := rapid.StringN(1, 200, 255).Draw(t, \"name\")\n\ttarget := rapid.StringN(1, 1000, SymlinkMax+1).Draw(t, \"target\")\n\tif name == \".\" || name == \"..\" {\n\t\tt.Skipf(\"skip symlink %s\", name)\n\t}\n\tif target == \".\" || target == \"..\" {\n\t\tt.Skipf(\"skip symlink %s\", target)\n\t}\n\tvar ti Ino\n\tst := m.meta.Symlink(m.ctx, parent, name, target, &ti, nil)\n\tst2 := m.symlink(parent, name, ti, target)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) Readlink(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tvar target []byte\n\tst := m.meta.ReadLink(m.ctx, inode, &target)\n\ttarget2, st2 := m.readlink(inode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tif st == 0 && string(target) != target2 {\n\t\tt.Fatalf(\"expect %s but got %s\", target2, target)\n\t}\n}\n\nfunc (m *fsMachine) Lookup(t *rapid.T) {\n\tparent := m.pickNode(t)\n\tname := m.pickChild(parent, t)\n\tif name == \"\" {\n\t\treturn\n\t}\n\tvar inode Ino\n\tvar attr Attr\n\tst := m.meta.Lookup(m.ctx, parent, name, &inode, &attr, true)\n\tinode2, st2 := m.lookup(parent, name, true)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tif st == 0 && inode != inode2 {\n\t\tt.Fatalf(\"expect %d but got %d\", inode2, inode)\n\t}\n}\n\nfunc (m *fsMachine) Getattr(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tvar attr Attr\n\tst := m.meta.GetAttr(m.ctx, inode, &attr)\n\tt.Logf(\"attr %#v\", attr)\n\tvar n *tNode\n\tif st == 0 {\n\t\tn = new(tNode)\n\t\tn._type = attr.Typ\n\t\tn.mode = attr.Mode\n\t\tn.uid = attr.Uid\n\t\tn.gid = attr.Gid\n\t\t// n.atime = attr.Atime\n\t\t// n.mtime = attr.Mtime\n\t\t// n.ctime = attr.Ctime\n\t\tn.length = attr.Length\n\t}\n\tn2, st2 := m.getattr(inode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tif n2 != nil {\n\t\tif n2._type != n._type || n2.mode != n.mode ||\n\t\t\tn2.uid != n.uid || n2.gid != n.gid ||\n\t\t\t// n2.atime != n.atime || n2.mtime != n.mtime || n2.ctime != n.ctime ||\n\t\t\tn2.length != n.length {\n\t\t\tt.Logf(\"expect %+v but got %+v\", n2, n)\n\t\t\tt.Fatalf(\"attr not matched\")\n\t\t}\n\t}\n}\n\nfunc (m *fsMachine) Rename(t *rapid.T) {\n\tdstName := rapid.StringN(1, 200, 255).Draw(t, \"name\")\n\tif dstName == \".\" || dstName == \"..\" {\n\t\tt.Skipf(\"skip name . and ..\")\n\t}\n\n\tsrcParent := m.pickNode(t)\n\tsrcName := m.pickChild(srcParent, t)\n\tif srcName == \"\" {\n\t\treturn\n\t}\n\tvar srcIno Ino\n\tfor name, n := range m.nodes[srcParent].children {\n\t\t// When the node is a hard link, name is not equal to n.name\n\t\tif srcName == name {\n\t\t\tsrcIno = n.inode\n\t\t}\n\t}\n\tdstParent := m.pickNode(t)\n\n\tif srcIno == dstParent {\n\t\tt.Skipf(\"skip rename srcIno is dstParent\")\n\t}\n\t// hard link\n\tif n, ok := m.nodes[dstParent].children[dstName]; ok && n.inode == srcIno {\n\t\tt.Skipf(\"skip rename srcIno is dstParent\")\n\t}\n\ttmp := m.nodes[dstParent].inode\n\tfor {\n\t\tif tmp == RootInode {\n\t\t\tbreak\n\t\t}\n\t\tif tmp == srcIno {\n\t\t\tt.Skipf(\"skip rename dstParent is subdir of srcIno\")\n\t\t} else {\n\t\t\ttmp = m.nodes[tmp].parents[0].inode\n\t\t}\n\t}\n\n\tvar inode Ino\n\tvar attr Attr\n\tst := m.rename(srcParent, srcName, dstParent, dstName, 0)\n\tst2 := m.meta.Rename(m.ctx, srcParent, srcName, dstParent, dstName, 0, &inode, &attr)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st, st2)\n\t}\n}\n\n/*\nDue to concurrency issues, the execution result of rmr is unpredictable.\n\n\tfunc (m *fsMachine) Rmr(t *rapid.T) {\n\t\tparent := m.pickNode(t)\n\t\tt.Logf(\"rmr parent ino %d\", parent)\n\t\tname := m.pickChild(parent, t)\n\t\tvar removed, removed2 uint64\n\t\tst := m.meta.Remove(m.ctx, parent, name, &removed)\n\t\tst2 := m.rmr(parent, name, &removed2)\n\t\tif st != st2 {\n\t\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t\t}\n\t\tif removed != removed2 {\n\t\t\tt.Fatalf(\"expect removed %d but got %d\", removed2, removed)\n\t\t}\n\t}\n*/\n\nfunc (m *fsMachine) Readdir(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tvar names []string\n\tvar result []*Entry\n\tst := m.meta.Readdir(m.ctx, inode, 0, &result)\n\tif st == 0 {\n\t\tfor _, e := range result {\n\t\t\tnames = append(names, string(e.Name))\n\t\t}\n\t\tsort.Strings(names)\n\t}\n\tstdRes, st2 := m.readdir(inode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tvar names2 []string\n\tfor _, entry := range stdRes {\n\t\tnames2 = append(names2, entry.name)\n\t}\n\tif st == 0 && !reflect.DeepEqual(names, names2) {\n\t\tt.Fatalf(\"expect %+v but got %+v\", names2, names)\n\t}\n}\n\n// Truncate is currently disabled.\n// FIXME: The comparison of the truncate results requires compacting all slices,\n// and some tricky processing are required on results.\n//func (m *fsMachine) Truncate(t *rapid.T) {\n//\tinode := m.pickNode(t)\n//\tlength := rapid.Uint64Range(0, 500<<20).Draw(t, \"length\")\n//\tvar attr Attr\n//\tst := m.meta.Truncate(m.ctx, inode, 0, length, &attr, false)\n//\tst2 := m.truncate(inode, length)\n//\tif st != st2 {\n//\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n//\t}\n//}\n\nfunc (m *fsMachine) Fallocate(t *rapid.T) {\n\tinode := m.pickNode(t)\n\toffset := rapid.Uint64Range(0, 500<<20).Draw(t, \"offset\")\n\tlength := rapid.Uint64Range(1, 500<<20).Draw(t, \"length\")\n\tst := m.meta.Fallocate(m.ctx, inode, 0, offset, length, nil)\n\tst2 := m.fallocate(inode, 0, offset, length)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\n// CopyFileRange is currently disabled, same reason as Truncate.\n//func (m *fsMachine) CopyFileRange(t *rapid.T) {\n//\tsrcinode := m.pickNode(t)\n//\tsrcoff := rapid.Uint64Max(m.nodes[srcinode].length).Draw(t, \"srcoff\")\n//\tdstinode := m.pickNode(t)\n//\tdstoff := rapid.Uint64Max(m.nodes[dstinode].length).Draw(t, \"dstoff\")\n//\tsize := rapid.Uint64Max(m.nodes[srcinode].length).Draw(t, \"size\")\n//\tvar copied uint64\n//\tst := m.meta.CopyFileRange(m.ctx, srcinode, srcoff, dstinode, dstoff, size, 0, &copied)\n//\tst2 := m.copy_file_range(srcinode, srcoff, dstinode, dstoff, size, 0)\n//\tif st != st2 {\n//\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n//\t}\n//}\n\nfunc (m *fsMachine) getPath(inode Ino) string {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn \"\"\n\t}\n\tif len(n.parents) == 0 {\n\t\treturn \"/\"\n\t}\n\tp := n.parents[0]\n\tfor name, t := range p.children {\n\t\tif t == n {\n\t\t\treturn m.getPath(p.inode) + \"/\" + name\n\t\t}\n\t}\n\tpanic(\"unreachable\")\n}\n\nfunc (m *fsMachine) Write(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tindx := rapid.Uint32Range(0, 10).Draw(t, \"indx\")\n\tpos := rapid.Uint32Range(0, ChunkSize).Draw(t, \"pos\")\n\tvar chunkid uint64\n\tm.meta.NewSlice(m.ctx, &chunkid)\n\tcleng := rapid.Uint32Range(1, ChunkSize).Draw(t, \"cleng\")\n\toff := rapid.Uint32Range(0, cleng-1).Draw(t, \"off\")\n\tlen := rapid.Uint32Range(1, cleng-off).Draw(t, \"len\")\n\tst := m.meta.Write(m.ctx, inode, indx, pos, Slice{chunkid, cleng, off, len}, time.Time{})\n\tst2 := m.write(inode, indx, pos, chunkid, cleng, off, len)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) Read(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tindx := rapid.Uint32Range(0, 10).Draw(t, \"indx\")\n\tvar result []Slice\n\tst := m.meta.Read(m.ctx, inode, indx, &result)\n\tvar slices []tSlice\n\tif st == 0 {\n\t\tvar pos uint32\n\t\tfor _, so := range result {\n\t\t\ts := tSlice{pos, so.Id, so.Size, so.Off, so.Len}\n\t\t\tslices = append(slices, s)\n\t\t\tpos += slices[len(slices)-1].len\n\t\t}\n\t}\n\t_, slices2, st2 := m.read(inode, indx)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tif st == 0 && !reflect.DeepEqual(cleanupSlices(slices), cleanupSlices(slices2)) {\n\t\tt.Fatalf(\"expect %+v but got %+v\", slices2, slices)\n\t}\n}\n\nfunc cleanupSlices(ss []tSlice) []tSlice {\n\tfor i := 0; i < len(ss); i++ {\n\t\ts := ss[i]\n\t\tif s.id == 0 && s.off > 0 {\n\t\t\ts.off = 0\n\t\t\tss[i] = s\n\t\t}\n\t\tif ss[i].id == 0 && i > 0 && ss[i-1].id == 0 {\n\t\t\tss[i-1].len += ss[i].len\n\t\t\tss = append(ss[:i], ss[i+1:]...)\n\t\t\ti--\n\t\t}\n\t}\n\tfor len(ss) > 0 && ss[len(ss)-1].id == 0 {\n\t\tss = ss[:len(ss)-1]\n\t}\n\tif len(ss) == 0 {\n\t\tss = nil\n\t}\n\treturn ss\n}\n\nfunc (m *fsMachine) SetXAttr(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tname := string(rapid.SliceOfN(rapid.RuneFrom(nil, unicode.Lu), 1, XATTR_NAME_MAX).Draw(t, \"name\"))\n\tvalue := rapid.SliceOfN(rapid.Byte(), 1, XATTR_SIZE_MAX+1).Draw(t, \"value\")\n\tmode := rapid.Uint8Range(0, XATTR_REMOVE).Draw(t, \"mode\")\n\tst := m.meta.SetXattr(m.ctx, inode, name, value, uint32(mode))\n\tst2 := m.setxattr(inode, name, value, mode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) RemoveXattr(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tname := rapid.StringN(1, 200, XATTR_NAME_MAX+1).Draw(t, \"name\")\n\tst := m.meta.RemoveXattr(m.ctx, inode, name)\n\tst2 := m.removexattr(inode, name)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nconst XATTR_REMOVE = 5\nconst XATTR_NAME_MAX = 255\nconst XATTR_SIZE_MAX = 65536\n\nfunc (m *fsMachine) GetXAttr(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tname := string(rapid.SliceOfN(rapid.RuneFrom(nil, unicode.Lu), 1, XATTR_NAME_MAX+1).Draw(t, \"name\"))\n\tvar value []byte\n\tst := m.meta.GetXattr(m.ctx, inode, name, &value)\n\tvalue2, st2 := m.getxattr(inode, name)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tif st == 0 && string(value) != string(value2) {\n\t\tt.Fatalf(\"expect %s but got %s\", string(value2), string(value))\n\t}\n}\n\nfunc (m *fsMachine) ListXAttr(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tvar attrs []byte\n\tst := m.meta.ListXattr(m.ctx, inode, &attrs)\n\tattrs2, st2 := m.listxattr(inode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tas := strings.Split(string(attrs), \"\\x00\")\n\tsort.Strings(as)\n\tas2 := strings.Split(string(attrs2), \"\\x00\")\n\tsort.Strings(as2)\n\tif st == 0 && !reflect.DeepEqual(as, as2) {\n\t\tt.Fatalf(\"expect %s but got %s\", string(attrs2), string(attrs))\n\t}\n}\n\nfunc (m *fsMachine) Check(t *rapid.T) {\n\tm.ctx = NewContext(0, 0, []uint32{0})\n\tif err := m.checkFSTree(RootInode); err != nil {\n\t\tt.Fatalf(\"check FSTree error %s\", err)\n\t}\n}\n\nfunc (m *fsMachine) checkFSTree(root Ino) error {\n\tvar result []*Entry\n\tif st := m.meta.Readdir(m.ctx, root, 1, &result); st != 0 {\n\t\treturn fmt.Errorf(\"meta readdir error %s\", st)\n\t}\n\tsort.Slice(result, func(i, j int) bool { return string(result[i].Name) < string(result[j].Name) })\n\n\tstdResult, st := m.readdir(root)\n\tif st != 0 {\n\t\treturn fmt.Errorf(\"standard meta readdir error %d\", st)\n\t}\n\tif len(result) != len(stdResult) {\n\t\treturn fmt.Errorf(\"the results of reading the directory should have equal lengths. standard meta: %#v test meta: %#v\", stdResult, result)\n\t}\n\tfor i := 0; i < len(result); i++ {\n\t\tstdEntry := stdResult[i]\n\t\tstdNode := stdEntry.node\n\t\tentry := result[i]\n\t\tif stdEntry.name == \".\" || stdEntry.name == \"..\" {\n\t\t\tcontinue\n\t\t}\n\t\tif stdEntry.name != string(entry.Name) {\n\t\t\treturn fmt.Errorf(\"name should equal. ino %d standard meta: %s, test meta %s\", stdNode.inode, stdNode.name, string(entry.Name))\n\t\t}\n\t\tif stdNode._type != entry.Attr.Typ {\n\t\t\treturn fmt.Errorf(\"type should equal ino: %d, standard meta: %d, test meta %d\", entry.Inode, stdNode._type, entry.Attr.Typ)\n\t\t}\n\t\tswitch entry.Attr.Typ {\n\t\tcase TypeDirectory:\n\t\t\tif err := m.checkFSTree(entry.Inode); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tdefault:\n\t\t\tif stdNode.inode != entry.Inode {\n\t\t\t\treturn fmt.Errorf(\"inode should equal. standard meta: %d, test meta %d\", stdNode.inode, entry.Inode)\n\t\t\t}\n\t\t\tif stdNode.gid != entry.Attr.Gid {\n\t\t\t\treturn fmt.Errorf(\"gid should equal. ino %d standard meta: %d, test meta %d\", stdNode.inode, stdNode.gid, entry.Attr.Gid)\n\t\t\t}\n\t\t\tif stdNode.uid != entry.Attr.Uid {\n\t\t\t\treturn fmt.Errorf(\"uid should equal. ino %d standard meta: %d, test meta %d\", stdNode.inode, stdNode.uid, entry.Attr.Uid)\n\t\t\t}\n\t\t\tif stdNode.length != entry.Attr.Length {\n\t\t\t\treturn fmt.Errorf(\"length should equal. ino %d standard meta: %d, test meta %d\", stdNode.inode, stdNode.length, entry.Attr.Length)\n\t\t\t}\n\t\t\tif stdNode.iflags != entry.Attr.Flags {\n\t\t\t\treturn fmt.Errorf(\"flags should equal. ino %d standard meta: %d, test meta %d\", stdNode.inode, stdNode.iflags, entry.Attr.Flags)\n\t\t\t}\n\t\t\tif stdNode.mode != entry.Attr.Mode {\n\t\t\t\treturn fmt.Errorf(\"mode should equal. ino %d standard meta: %d, test meta %d\", stdNode.inode, stdNode.mode, entry.Attr.Mode)\n\t\t\t}\n\t\t\t// If a hard link has been set, the parent will be cleared.\n\t\t\tif !stdNode.hardlink {\n\t\t\t\tif stdNode.parents[0].inode != entry.Attr.Parent {\n\t\t\t\t\treturn fmt.Errorf(\"parent should equal. ino %d standard meta: %d, test meta %d\", stdNode.inode, stdNode.parents[0].inode, entry.Attr.Parent)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// check chunks\n\t\t\tfor indx := range stdNode.chunks {\n\t\t\t\tvar rs []Slice\n\t\t\t\tst := m.meta.Read(m.ctx, stdNode.inode, indx, &rs)\n\t\t\t\tvar slices []tSlice\n\t\t\t\tif st == 0 {\n\t\t\t\t\tvar pos uint32\n\t\t\t\t\tfor _, so := range rs {\n\t\t\t\t\t\ts := tSlice{pos, so.Id, so.Size, so.Off, so.Len}\n\t\t\t\t\t\tslices = append(slices, s)\n\t\t\t\t\t\tpos += slices[len(slices)-1].len\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t_, slices2, st2 := m.read(stdNode.inode, indx)\n\t\t\t\tif st != st2 {\n\t\t\t\t\treturn fmt.Errorf(\"read eno should equal. standard meta ino %d ,indx %d std meta eno %d test meta eno %d\", stdNode.inode, indx, st2, st)\n\t\t\t\t}\n\t\t\t\tif st == 0 && !reflect.DeepEqual(cleanupSlices(slices), cleanupSlices(slices2)) {\n\t\t\t\t\treturn fmt.Errorf(\"slice should equal. standard meta %+v test meta %+v\", slices2, slices)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// check symlink\n\t\t\tvar target []byte\n\t\t\tst := m.meta.ReadLink(m.ctx, stdNode.inode, &target)\n\t\t\ttarget2, st2 := m.readlink(stdNode.inode)\n\t\t\tif st != st2 {\n\t\t\t\treturn fmt.Errorf(\"readlink eno should equal. standard meta ino %d stadndard meta %d test meta %d\", stdNode.inode, st2, st)\n\t\t\t}\n\t\t\tif st == 0 && string(target) != target2 {\n\t\t\t\treturn fmt.Errorf(\"symlink should equal. standard meta ino %d stadndard meta %s test meta %s\", stdNode.inode, target2, string(target))\n\t\t\t}\n\n\t\t\t// check xattr\n\t\t\tvar attrs []byte\n\t\t\tst = m.meta.ListXattr(m.ctx, stdNode.inode, &attrs)\n\t\t\tattrs2, st2 := m.listxattr(stdNode.inode)\n\t\t\tif st != st2 {\n\t\t\t\treturn fmt.Errorf(\"listxattr eno should equal. standard meta ino %d stadndard meta %d test meta %d\", stdNode.inode, st2, st)\n\t\t\t}\n\t\t\tas := strings.Split(string(attrs), \"\\x00\")\n\t\t\tsort.Strings(as)\n\t\t\tas2 := strings.Split(string(attrs2), \"\\x00\")\n\t\t\tsort.Strings(as2)\n\t\t\tif st == 0 && !reflect.DeepEqual(as, as2) {\n\t\t\t\treturn fmt.Errorf(\"listxattr should equal. standard meta ino %d stadndard meta %s test meta %s\", stdNode.inode, as2, as)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *fsMachine) setfacl(inode Ino, atype uint8, rule *aclAPI.Rule) syscall.Errno {\n\tif atype != aclAPI.TypeAccess && atype != aclAPI.TypeDefault {\n\t\treturn syscall.EINVAL\n\t}\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif m.ctx.Uid() != 0 && m.ctx.Uid() != n.uid {\n\t\treturn syscall.EPERM\n\t}\n\n\tif rule.IsEmpty() {\n\t\tif atype == aclAPI.TypeDefault {\n\t\t\tn.defACL = nil\n\t\t\tm.removexattr(inode, \"system.posix_acl_default\")\n\t\t}\n\t\t// TODO: update ctime\n\t\treturn 0\n\t}\n\n\tif rule.IsMinimal() && atype == aclAPI.TypeAccess {\n\t\tn.accACL = nil\n\t\tn.mode &= 07000\n\t\tn.mode |= ((rule.Owner & 7) << 6) | ((rule.Group & 7) << 3) | (rule.Other & 7)\n\t\treturn 0\n\t}\n\n\trule.InheritPerms(n.mode)\n\tif atype == aclAPI.TypeAccess {\n\t\tn.accACL = rule\n\t\tif n.accACL.GetMode() != n.mode&0777 {\n\t\t\tn.mode = n.mode&07000 | n.accACL.GetMode()\n\t\t}\n\t} else {\n\t\tn.defACL = rule\n\t}\n\treturn 0\n}\n\nfunc (m *fsMachine) Setfacl(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tatype := rapid.Uint8Range(1, 2).Draw(t, \"atype\")\n\tuser := rapid.Uint16Range(0, 7).Draw(t, \"user\")\n\tgroup := rapid.Uint16Range(0, 7).Draw(t, \"group\")\n\tother := rapid.Uint16Range(0, 7).Draw(t, \"other\")\n\tmask := rapid.Uint16Range(0, 7).Draw(t, \"mask\")\n\tvar users aclAPI.Entries\n\tvar groups aclAPI.Entries\n\n\tus := rapid.IntRange(0, 3).Draw(t, \"users\")\n\tfor i := 0; i < us; i++ {\n\t\tusers = append(users, aclAPI.Entry{Id: rapid.Uint32Range(1, 5).Draw(t, \"uid\"), Perm: rapid.Uint16Range(0, 7).Draw(t, \"perm\")})\n\t}\n\tgs := rapid.IntRange(0, 3).Draw(t, \"groups\")\n\tfor i := 0; i < gs; i++ {\n\t\tgroups = append(groups, aclAPI.Entry{Id: rapid.Uint32Range(1, 5).Draw(t, \"gid\"), Perm: rapid.Uint16Range(0, 7).Draw(t, \"perm\")})\n\t}\n\trule := &aclAPI.Rule{\n\t\tOwner: user,\n\t\tGroup: group,\n\t\tMask: mask,\n\t\tOther: other,\n\t\tNamedUsers: users,\n\t\tNamedGroups: groups,\n\t}\n\n\tst := m.meta.SetFacl(m.ctx, inode, atype, rule)\n\tst2 := m.setfacl(inode, atype, rule)\n\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) getfacl(inode Ino, atype uint8) (*aclAPI.Rule, syscall.Errno) {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn nil, syscall.ENOENT\n\t}\n\tswitch atype {\n\tcase aclAPI.TypeAccess:\n\t\tif n.accACL == nil {\n\t\t\treturn nil, ENOATTR\n\t\t}\n\t\treturn n.accACL, 0\n\tcase aclAPI.TypeDefault:\n\t\tif n.defACL == nil {\n\t\t\treturn nil, ENOATTR\n\t\t}\n\t\treturn n.defACL, 0\n\tdefault:\n\t\treturn nil, syscall.EINVAL\n\t}\n}\n\nfunc (m *fsMachine) GetACL(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tatype := rapid.Uint8Range(1, 2).Draw(t, \"atype\")\n\n\trule := &aclAPI.Rule{}\n\tst := m.meta.GetFacl(m.ctx, inode, atype, rule)\n\trule2, st2 := m.getfacl(inode, atype)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tif st == 0 && !rule.IsEqual(rule2) {\n\t\tt.Fatalf(\"expect %+v but got %+v, %t\", rule2, rule, reflect.DeepEqual(rule, *rule2))\n\t}\n}\n\nfunc (m *fsMachine) RemoveACL(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tatype := rapid.Uint8Range(1, 2).Draw(t, \"atype\")\n\n\tvar rule *aclAPI.Rule\n\tif atype == aclAPI.TypeAccess {\n\t\trule = &aclAPI.Rule{\n\t\t\tMask: 0xFFFF,\n\t\t}\n\t\trule.InheritPerms(m.nodes[inode].mode)\n\t} else {\n\t\trule = aclAPI.EmptyRule()\n\t}\n\n\tst := m.meta.SetFacl(m.ctx, inode, atype, rule)\n\tst2 := m.setfacl(inode, atype, rule)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (n *tNode) stat(visited map[Ino]struct{}) (uint64, uint64) {\n\tif _, ok := visited[n.inode]; ok {\n\t\treturn 0, 0\n\t}\n\tvisited[n.inode] = struct{}{}\n\n\tvar size uint64 = uint64(align4K(n.length))\n\tvar inodes uint64 = 1\n\tif n._type == TypeDirectory {\n\t\tfor _, c := range n.children {\n\t\t\ts, i := c.stat(visited)\n\t\t\tsize += s\n\t\t\tinodes += i\n\t\t}\n\t}\n\treturn size, inodes\n}\n\nfunc (m *fsMachine) statfs(format Format) (uint64, uint64, uint64, uint64) {\n\tn := m.nodes[RootInode]\n\tvisited := make(map[Ino]struct{})\n\tused, iused := n.stat(visited)\n\tused -= uint64(align4K(0))\n\tiused -= 1\n\tvar avail, iavail uint64\n\tavail = 1<<50 - used\n\tiavail = 10 << 20\n\t// if inodes is not limited in Format, iavail always keep the same number of inodes\n\tif format.Inodes > 0 {\n\t\tiavail -= iused\n\t}\n\tif n.quota != nil {\n\t\tif n.quota.size > 0 {\n\t\t\tif used > n.quota.size {\n\t\t\t\tavail = 0\n\t\t\t} else {\n\t\t\t\tavail = n.quota.size - used\n\t\t\t}\n\t\t}\n\t\tif n.quota.inodes > 0 {\n\t\t\tif iused > n.quota.inodes {\n\t\t\t\tiavail = 0\n\t\t\t} else {\n\t\t\t\tiavail = uint64(n.quota.inodes) - iused\n\t\t\t}\n\t\t}\n\t}\n\treturn used + avail, avail, iused, iavail\n}\n\nfunc (m *fsMachine) StatFS(t *rapid.T) {\n\tvar totalsize, availspace, iused, iavail uint64\n\tm.meta.StatFS(m.ctx, RootInode, &totalsize, &availspace, &iused, &iavail)\n\ttotal2, avail2, iused2, iavail2 := m.statfs(m.meta.GetFormat())\n\tif totalsize != total2 || availspace != avail2 || iused != iused2 || iavail != iavail2 {\n\t\tt.Fatalf(\"expect %d %d %d %d but got %d %d %d %d\", total2, avail2, iused2, iavail2, totalsize, availspace, iused, iavail)\n\t}\n}\n\nfunc (m *fsMachine) amtime(inode Ino, flag uint16, atime, mtime int64, oattr *Attr) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\n\tchanged := false\n\tif flag&SetAttrAtime != 0 {\n\t\tn.atime = atime\n\t\tchanged = changed || oattr.Atime != atime\n\t}\n\tif flag&SetAttrMtime != 0 {\n\t\tn.mtime = mtime\n\t\tchanged = changed || oattr.Mtime != mtime\n\t}\n\n\tif changed {\n\t\tif n.uid == 0 && m.ctx.Uid() != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif ok := n.access(m.ctx, MODE_MASK_W); !ok && n.uid != m.ctx.Uid() {\n\t\t\treturn syscall.EACCES\n\t\t}\n\t}\n\t// TODO ctime\n\treturn 0\n}\n\nfunc (m *fsMachine) SetAmtime(t *rapid.T) {\n\tinode := m.pickNode(t)\n\n\toattr := &Attr{}\n\tif st := m.meta.GetAttr(m.ctx, inode, oattr); st != 0 {\n\t\treturn\n\t}\n\n\tatime := rapid.Int64Range(0, 1e8).Draw(t, \"atime\")\n\tmtime := rapid.Int64Range(0, 1e8).Draw(t, \"mtime\")\n\tvar flag uint16\n\tattr := &Attr{\n\t\tAtime: atime,\n\t\tMtime: mtime,\n\t}\n\n\tif atime > 0 {\n\t\tflag |= SetAttrAtime\n\t}\n\tif mtime > 0 {\n\t\tflag |= SetAttrMtime\n\t}\n\tst2 := m.amtime(inode, flag, atime, mtime, oattr)\n\tst := m.meta.SetAttr(m.ctx, inode, flag, 0, attr)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\n\tif st == 0 {\n\t\t// validate time only here\n\t\tnode := m.nodes[inode]\n\t\tif flag&SetAttrAtime != 0 && attr.Atime != node.atime {\n\t\t\tt.Fatalf(\"expect %d but got %d\", node.atime, attr.Atime)\n\t\t}\n\t\tif flag&SetAttrMtime != 0 && attr.Mtime != node.mtime {\n\t\t\tt.Fatalf(\"expect %d but got %d\", node.mtime, attr.Mtime)\n\t\t}\n\t}\n}\n\nfunc (m *fsMachine) chmod(inode Ino, mode uint16) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif n.accACL != nil {\n\t\tn.accACL.SetMode(mode)\n\t\tn.mode = mode&07000 | n.accACL.GetMode()\n\t} else {\n\t\tif m.ctx.Uid() != 0 && m.ctx.Uid() != n.uid &&\n\t\t\t(n.mode&01777 != mode&01777 || mode&02000 > n.mode&02000 || mode&04000 > n.mode&04000) {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tn.mode = mode\n\t}\n\t// n.ctime = m.ctx.ts\n\treturn 0\n}\n\nfunc (m *fsMachine) Chmod(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tmode := rapid.Uint16Range(0, 01777).Draw(t, \"mode\")\n\tst := m.meta.SetAttr(m.ctx, inode, SetAttrMode, 0, &Attr{Mode: mode})\n\tst2 := m.chmod(inode, mode)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) chown(inode Ino, flag uint16, uid, gid uint32) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif flag&SetAttrUID != 0 && n.uid != uid {\n\t\tif m.ctx.Uid() != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tn.uid = uid\n\t}\n\tif flag&SetAttrGID != 0 {\n\t\tif m.ctx.Uid() != 0 && m.ctx.Uid() != n.uid {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif n.gid != gid {\n\t\t\tif m.ctx.CheckPermission() && m.ctx.Uid() != 0 && !containsGid(m.ctx, gid) {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tn.gid = gid\n\t\t}\n\t}\n\t// n.ctime = m.ctx.ts\n\treturn 0\n}\n\nfunc (m *fsMachine) Chown(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tuid := rapid.Uint32Range(0, 10).Draw(t, \"uid\")\n\tgid := rapid.Uint32Range(0, 10).Draw(t, \"gid\")\n\tvar flag uint16\n\tif uid < 10 {\n\t\tflag |= SetAttrUID\n\t}\n\tif gid < 10 {\n\t\tflag |= SetAttrGID\n\t}\n\tst := m.meta.SetAttr(m.ctx, inode, flag, 0, &Attr{Uid: uid, Gid: gid})\n\tst2 := m.chown(inode, flag, uid, gid)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nfunc (m *fsMachine) flock(inode Ino, owner uint64, typ uint32) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\t// m.openfiles[inode] = true\n\tif n.flocks == nil {\n\t\tn.flocks = make(map[ownerKey]byte)\n\t}\n\tlowner := ownerKey{Sid: m.sid, Owner: owner}\n\tswitch typ {\n\tcase F_UNLCK:\n\t\tdelete(n.flocks, lowner)\n\tcase F_RDLCK:\n\t\tfor o, l := range n.flocks {\n\t\t\tif l == 'W' && o != lowner {\n\t\t\t\treturn syscall.EAGAIN\n\t\t\t}\n\t\t}\n\t\tn.flocks[lowner] = 'R'\n\tcase F_WRLCK:\n\t\tfor o := range n.flocks {\n\t\t\tif o == lowner {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn syscall.EAGAIN\n\t\t}\n\t\tn.flocks[lowner] = 'W'\n\tdefault:\n\t\treturn syscall.EINVAL\n\t}\n\treturn 0\n}\n\nfunc (m *fsMachine) Flock(t *rapid.T) {\n\tinode := m.pickNode(t)\n\towner := rapid.Uint64().Draw(t, \"owner\")\n\ttyp := rapid.SampledFrom([]uint32{F_WRLCK, F_RDLCK, F_UNLCK}).Draw(t, \"typ\")\n\tst := m.flock(inode, owner, typ)\n\tst2 := m.meta.Flock(m.ctx, inode, owner, typ, false)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st, st2)\n\t}\n\n\tif st == 0 {\n\t\tplocks1, flocks1, err1 := m.meta.ListLocks(m.ctx, inode)\n\t\tplocks2, flocks2, err2 := m.listLocks(inode)\n\t\tif err1 != nil && err2 == nil || err1 == nil && err2 != nil {\n\t\t\tt.Fatalf(\"expect %s but got %s\", err2, err1)\n\t\t}\n\t\tif err1 == nil {\n\t\t\tsort.Slice(flocks1, func(i, j int) bool {\n\t\t\t\treturn flocks1[i].Owner < flocks1[j].Owner\n\t\t\t})\n\t\t\tsort.Slice(flocks2, func(i, j int) bool {\n\t\t\t\treturn flocks2[i].Owner < flocks2[j].Owner\n\t\t\t})\n\t\t\tif !compareLocks(flocks1, flocks2) {\n\t\t\t\tt.Fatalf(\"expect %+v but got %+v\", flocks2, flocks1)\n\t\t\t}\n\t\t\tsort.Slice(plocks1, func(i, j int) bool {\n\t\t\t\tif plocks1[i].Owner != plocks1[j].Owner {\n\t\t\t\t\treturn plocks1[i].Owner < plocks1[j].Owner\n\t\t\t\t}\n\t\t\t\tif plocks1[i].Start != plocks1[j].Start {\n\t\t\t\t\treturn plocks1[i].Start < plocks1[j].Start\n\t\t\t\t}\n\t\t\t\treturn plocks1[i].End < plocks1[j].End\n\t\t\t})\n\t\t\tsort.Slice(plocks2, func(i, j int) bool {\n\t\t\t\tif plocks2[i].Owner != plocks2[j].Owner {\n\t\t\t\t\treturn plocks2[i].Owner < plocks2[j].Owner\n\t\t\t\t}\n\t\t\t\tif plocks2[i].Start != plocks2[j].Start {\n\t\t\t\t\treturn plocks2[i].Start < plocks2[j].Start\n\t\t\t\t}\n\t\t\t\treturn plocks2[i].End < plocks2[j].End\n\t\t\t})\n\t\t\tif !compareLocks(plocks1, plocks2) {\n\t\t\t\tt.Fatalf(\"expect %+v but got %+v\", plocks2, plocks1)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (m *fsMachine) listLocks(inode Ino) ([]PLockItem, []FLockItem, error) {\n\tvar flocks []FLockItem\n\tvar plocks []PLockItem\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn plocks, flocks, syscall.ENOENT\n\t}\n\tfor o, l := range n.flocks {\n\t\tflocks = append(flocks, FLockItem{ownerKey: ownerKey{\n\t\t\tSid: o.Sid,\n\t\t\tOwner: o.Owner,\n\t\t}, Type: string(l)})\n\t}\n\tfor o, ls := range n.plocks {\n\t\tfor _, l := range ls {\n\t\t\tplocks = append(plocks, PLockItem{ownerKey: ownerKey{\n\t\t\t\tSid: o.Sid,\n\t\t\t\tOwner: o.Owner,\n\t\t\t}, plockRecord: l})\n\t\t}\n\t}\n\treturn plocks, flocks, nil\n}\n\nfunc compareLocks[T any](l1, l2 []T) bool {\n\tif len(l1) == 0 && len(l2) == 0 {\n\t\treturn true\n\t}\n\treturn reflect.DeepEqual(l1, l2)\n}\n\nfunc (m *fsMachine) ListLocks(t *rapid.T) {\n\tinode := m.pickNode(t)\n\tplocks1, flocks1, err1 := m.meta.ListLocks(m.ctx, inode)\n\tplocks2, flocks2, err2 := m.listLocks(inode)\n\tif err1 != nil && err2 == nil || err1 == nil && err2 != nil {\n\t\tt.Fatalf(\"expect %s but got %s\", err2, err1)\n\t}\n\tif err1 == nil {\n\t\t// sort flocks by owner\n\t\tsort.Slice(flocks1, func(i, j int) bool {\n\t\t\treturn flocks1[i].Owner < flocks1[j].Owner\n\t\t})\n\t\tsort.Slice(flocks2, func(i, j int) bool {\n\t\t\treturn flocks2[i].Owner < flocks2[j].Owner\n\t\t})\n\t\tif !compareLocks(flocks1, flocks2) {\n\t\t\tt.Fatalf(\"expect %+v but got %+v\", flocks2, flocks1)\n\t\t}\n\t\t// sort plocks by owner\n\t\tsort.Slice(plocks1, func(i, j int) bool {\n\t\t\treturn plocks1[i].Owner < plocks1[j].Owner\n\t\t})\n\t\tsort.Slice(plocks2, func(i, j int) bool {\n\t\t\treturn plocks2[i].Owner < plocks2[j].Owner\n\t\t})\n\t\tif !compareLocks(plocks1, plocks2) {\n\t\t\tt.Fatalf(\"expect %+v but got %+v\", plocks2, plocks1)\n\t\t}\n\t}\n}\n\nfunc (m *fsMachine) getlk(inode Ino, owner uint64, ltype *uint32, start *uint64, end *uint64, pid *uint32) syscall.Errno {\n\tif *ltype == F_UNLCK {\n\t\t*start = 0\n\t\t*end = 0\n\t\t*pid = 0\n\t\treturn 0\n\t}\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tfor o, ls := range n.plocks {\n\t\tfor _, l := range ls {\n\t\t\tif o.Owner != owner && (*ltype == F_WRLCK || l.Type == F_WRLCK) && *end >= l.Start && *start <= l.End {\n\t\t\t\t*ltype = l.Type\n\t\t\t\t*start = l.Start\n\t\t\t\t*end = l.End\n\t\t\t\tif o.Sid == m.sid {\n\t\t\t\t\t*pid = l.Pid\n\t\t\t\t} else {\n\t\t\t\t\t*pid = 0\n\t\t\t\t}\n\t\t\t\treturn 0\n\t\t\t}\n\t\t}\n\t}\n\t*ltype = F_UNLCK\n\t*start = 0\n\t*end = 0\n\t*pid = 0\n\treturn 0\n}\n\nfunc (m *fsMachine) Getlk(t *rapid.T) {\n\tinode := m.pickNode(t)\n\towner := rapid.Uint64().Draw(t, \"owner\")\n\tltype := rapid.Uint32Range(0, 2).Draw(t, \"ltype\")\n\tstart := rapid.Uint64Range(0, 500<<20).Draw(t, \"start\")\n\tlength := rapid.Uint64Range(1, 500<<20).Draw(t, \"len\")\n\tend := start + length - 1\n\n\tvar pid1, pid2 uint32\n\tftype1, ftype2 := ltype, ltype\n\tfstart1, fstart2 := start, start\n\tfend1, fend2 := end, end\n\tst := m.getlk(inode, owner, &ftype1, &fstart1, &fend1, &pid1)\n\tst2 := m.meta.Getlk(m.ctx, inode, owner, &ftype2, &fstart2, &fend2, &pid2)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n\tif st == 0 && ltype != F_UNLCK && (ftype1 == F_UNLCK && ftype2 != F_UNLCK || ftype1 != F_UNLCK && ftype2 == F_UNLCK) {\n\t\tt.Fatalf(\"status not right, %d %d\", ftype1, ftype2)\n\t}\n}\n\nfunc (m *fsMachine) setlk(inode Ino, owner uint64, ltype uint32, start uint64, end uint64, pid uint32) syscall.Errno {\n\tn := m.nodes[inode]\n\tif n == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif ltype != F_UNLCK {\n\t\t// m.openfiles[inode] = true\n\t}\n\tif n.plocks == nil {\n\t\tn.plocks = make(map[ownerKey][]plockRecord)\n\t}\n\tlowner := ownerKey{Sid: m.sid, Owner: owner}\n\tif ltype == F_UNLCK {\n\t\tif n.plocks[lowner] == nil {\n\t\t\treturn 0\n\t\t}\n\t} else {\n\t\tfor o, ls := range n.plocks {\n\t\t\tfor _, l := range ls {\n\t\t\t\tif o != lowner && (ltype == F_WRLCK || l.Type == F_WRLCK) && end >= l.Start && start <= l.End {\n\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tls := updateLocks(n.plocks[lowner], plockRecord{ltype, pid, start, end})\n\tif len(ls) == 0 {\n\t\tdelete(n.plocks, lowner)\n\t} else {\n\t\tn.plocks[lowner] = ls\n\t}\n\treturn 0\n}\n\nfunc (m *fsMachine) Setlk(t *rapid.T) {\n\tinode := m.pickNode(t)\n\towner := rapid.Uint64().Draw(t, \"owner\")\n\tltype := rapid.SampledFrom([]uint32{F_WRLCK, F_RDLCK, F_UNLCK}).Draw(t, \"ltype\")\n\tstart := rapid.Uint64Range(0, 500<<20).Draw(t, \"start\")\n\tlen := rapid.Uint64Range(1, 500<<20).Draw(t, \"len\")\n\tpid := rapid.Uint32Range(1, 10000).Draw(t, \"pid\")\n\tvar end = start + len - 1\n\tst := m.meta.Setlk(m.ctx, inode, owner, false, ltype, start, end, pid)\n\tst2 := m.setlk(inode, owner, ltype, start, end, pid)\n\tif st != st2 {\n\t\tt.Fatalf(\"expect %s but got %s\", st2, st)\n\t}\n}\n\nvar metaURL string\n\nfunc init() {\n\tflag.StringVar(&metaURL, \"rapid.meta\", \"memkv://jfs-unit-test\", \"meta URL\")\n\t// flag.StringVar(&metaURL, \"rapid.meta\", \"sqlite3://test.db\", \"meta URL\")\n\t// flag.StringVar(&metaURL, \"rapid.meta\", \"redis://localhost:6379\", \"meta URL\")\n}\n\nfunc defaultFlag(name string, value string) func() {\n\tif f := flag.Lookup(name); f.Value.String() == f.DefValue {\n\t\tflag.Set(name, value)\n\t\treturn func() {\n\t\t\tflag.Set(name, f.DefValue)\n\t\t}\n\t}\n\treturn func() {}\n}\n\nfunc TestFSOps(t *testing.T) {\n\tlogger.SetLevel(logrus.ErrorLevel)\n\tdefer logger.SetLevel(logrus.InfoLevel)\n\tdefer defaultFlag(\"rapid.shrinktime\", \"1h\")()\n\tdefer defaultFlag(\"rapid.steps\", \"200\")()\n\tdefer defaultFlag(\"rapid.checks\", \"5000\")()\n\t//defer defaultFlag(\"rapid.seed\", \"1\")()\n\t//defer defaultFlag(\"rapid.v\", \"true\")()\n\t//defer defaultFlag(\"rapid.debug\", \"true\")()\n\t//defer defaultFlag(\"rapid.debugvis\", \"true\")()\n\t//defer defaultFlag(\"rapid.failfile\", \"testdata/rapid/TestFSOps/TestFSOps-20250228114121-68323.fail\")()\n\n\trapid.Check(t, rapid.Run[*fsMachine]())\n}\n" }, { "path": "pkg/meta/redis.go", "content": "//go:build !noredis\n// +build !noredis\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/binary\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"hash/fnv\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"github.com/redis/go-redis/v9/maintnotifications\"\n\t\"golang.org/x/sync/errgroup\"\n)\n\n/*\n\tNode: i$inode -> Attribute{type,mode,uid,gid,atime,mtime,ctime,nlink,length,rdev}\n\tDir: d$inode -> {name -> {inode,type}}\n\tParent: p$inode -> {parent -> count} // for hard links\n\tFile: c$inode_$indx -> [Slice{pos,id,length,off,len}]\n\tSymlink: s$inode -> target\n\tXattr: x$inode -> {name -> value}\n\tFlock: lockf$inode -> { $sid_$owner -> ltype }\n\tPOSIX lock: lockp$inode -> { $sid_$owner -> Plock(pid,ltype,start,end) }\n\tSessions: sessions -> [ $sid -> heartbeat ]\n\tsustained: session$sid -> [$inode]\n\tlocked: locked$sid -> { lockf$inode or lockp$inode }\n\n\tRemoved files: delfiles -> [$inode:$length -> seconds]\n\tdetached nodes: detachedNodes -> [$inode -> seconds]\n\tSlices refs: sliceRef -> {k$sliceId_$size -> refcount}\n\n\tDir data length: dirDataLength -> { $inode -> length }\n\tDir used space: dirUsedSpace -> { $inode -> usedSpace }\n\tDir used inodes: dirUsedInodes -> { $inode -> usedInodes }\n\tQuota: dirQuota -> { $inode -> {maxSpace, maxInodes} }\n\tQuota used space: dirQuotaUsedSpace -> { $inode -> usedSpace }\n\tQuota used inodes: dirQuotaUsedInodes -> { $inode -> usedInodes }\n\tAcl: acl -> { $acl_id -> acl }\n\tKrbToken: krbToken -> { $token_id -> token }\n\n\tRedis features:\n\t Sorted Set: 1.2+\n\t Hash Set: 4.0+\n\t Transaction: 2.2+\n\t Scripting: 2.6+\n\t Scan: 2.8+\n*/\n\ntype redisMeta struct {\n\t*baseMeta\n\trdb redis.UniversalClient\n\tprefix string\n\tshaLookup string // The SHA returned by Redis for the loaded `scriptLookup`\n\tshaResolve string // The SHA returned by Redis for the loaded `scriptResolve`\n\tcache *redisCache\n}\n\nvar _ Meta = (*redisMeta)(nil)\nvar _ engine = (*redisMeta)(nil)\n\nfunc init() {\n\tRegister(\"redis\", newRedisMeta)\n\tRegister(\"rediss\", newRedisMeta)\n\tRegister(\"unix\", newRedisMeta)\n}\n\n// newRedisMeta return a meta store using Redis.\nfunc newRedisMeta(driver, addr string, conf *Config) (Meta, error) {\n\turi := driver + \"://\" + addr\n\tu, err := url.Parse(uri)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"url parse %s: %s\", uri, err)\n\t}\n\tvalues := u.Query()\n\tquery := queryMap{&values}\n\tminRetryBackoff := query.duration(\"min-retry-backoff\", \"min_retry_backoff\", time.Millisecond*20)\n\tmaxRetryBackoff := query.duration(\"max-retry-backoff\", \"max_retry_backoff\", time.Second*10)\n\treadTimeout := query.duration(\"read-timeout\", \"read_timeout\", time.Second*30)\n\twriteTimeout := query.duration(\"write-timeout\", \"write_timeout\", time.Second*5)\n\trouteRead := query.pop(\"route-read\")\n\tskipVerify := query.pop(\"insecure-skip-verify\")\n\tcertFile := query.pop(\"tls-cert-file\")\n\tkeyFile := query.pop(\"tls-key-file\")\n\tcaCertFile := query.pop(\"tls-ca-cert-file\")\n\ttlsServerName := query.pop(\"tls-server-name\")\n\n\t// Client-side caching options\n\tclientCacheStr := query.pop(\"client-cache\")\n\tclientCache := clientCacheStr != \"false\" && clientCacheStr != \"\"\n\tclientCacheSize := query.getInt(\"client-cache-size\", \"client_cache_size\", 12800)\n\t// Default TTL to prevent reading stale cache for a long time when the connection fails.\n\tclientCacheExpiry := query.duration(\"client-cache-expire\", \"client_cache_expire\", time.Minute)\n\tclientCachePreload := query.getInt(\"client-cache-preload\", \"client_cache_preload\", 0) // may cause conflict\n\tu.RawQuery = values.Encode()\n\n\thosts := u.Host\n\topt, err := redis.ParseURL(u.String())\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"redis parse %s: %s\", uri, err)\n\t}\n\tif opt.TLSConfig != nil {\n\t\topt.TLSConfig.ServerName = tlsServerName // use the host of each connection as ServerName\n\t\topt.TLSConfig.InsecureSkipVerify = skipVerify != \"\"\n\t\tif certFile != \"\" {\n\t\t\tcert, err := tls.LoadX509KeyPair(certFile, keyFile)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"get certificate error certFile:%s keyFile:%s error:%s\", certFile, keyFile, err)\n\t\t\t}\n\t\t\topt.TLSConfig.Certificates = []tls.Certificate{cert}\n\t\t}\n\t\tif caCertFile != \"\" {\n\t\t\tcaCert, err := os.ReadFile(caCertFile)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"read ca cert file error path:%s error:%s\", caCertFile, err)\n\t\t\t}\n\t\t\tcaCertPool := x509.NewCertPool()\n\t\t\tcaCertPool.AppendCertsFromPEM(caCert)\n\t\t\topt.TLSConfig.RootCAs = caCertPool\n\t\t}\n\t}\n\tif opt.Password == \"\" {\n\t\topt.Password = os.Getenv(\"REDIS_PASSWORD\")\n\t}\n\tif opt.Password == \"\" {\n\t\topt.Password = os.Getenv(\"META_PASSWORD\")\n\t}\n\tif opt.Password == \"\" {\n\t\tif passwordFile := os.Getenv(\"META_PASSWORD_FILE\"); passwordFile != \"\" {\n\t\t\tpassword, err := readPasswordFromFile(passwordFile)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"%v\", err)\n\t\t\t} else {\n\t\t\t\topt.Password = password\n\t\t\t}\n\t\t}\n\t}\n\topt.MaxRetries = conf.Retries\n\tif opt.MaxRetries == 0 {\n\t\topt.MaxRetries = -1 // Redis use -1 to disable retries\n\t}\n\topt.MinRetryBackoff = minRetryBackoff\n\topt.MaxRetryBackoff = maxRetryBackoff\n\topt.ReadTimeout = readTimeout\n\topt.WriteTimeout = writeTimeout\n\topt.MaintNotificationsConfig = &maintnotifications.Config{Mode: maintnotifications.ModeDisabled}\n\tvar prefix string\n\tvar rdb redis.UniversalClient\n\n\tif strings.Contains(hosts, \",\") && strings.Index(hosts, \",\") < strings.Index(hosts, \":\") {\n\t\tvar fopt redis.FailoverOptions\n\t\tps := strings.Split(hosts, \",\")\n\t\tfopt.MasterName = ps[0]\n\t\tfopt.SentinelAddrs = ps[1:]\n\t\t_, port, _ := net.SplitHostPort(fopt.SentinelAddrs[len(fopt.SentinelAddrs)-1])\n\t\tif port == \"\" {\n\t\t\tport = \"26379\"\n\t\t}\n\t\tfor i, addr := range fopt.SentinelAddrs {\n\t\t\th, p, e := net.SplitHostPort(addr)\n\t\t\tif e != nil {\n\t\t\t\tfopt.SentinelAddrs[i] = net.JoinHostPort(addr, port)\n\t\t\t} else if p == \"\" {\n\t\t\t\tfopt.SentinelAddrs[i] = net.JoinHostPort(h, port)\n\t\t\t}\n\t\t}\n\t\tfopt.SentinelPassword = os.Getenv(\"SENTINEL_PASSWORD\")\n\t\tfopt.DB = opt.DB\n\t\tfopt.Username = opt.Username\n\t\tfopt.Password = opt.Password\n\t\tfopt.TLSConfig = opt.TLSConfig\n\t\tfopt.MaxRetries = opt.MaxRetries\n\t\tfopt.MinRetryBackoff = opt.MinRetryBackoff\n\t\tfopt.MaxRetryBackoff = opt.MaxRetryBackoff\n\t\tfopt.DialTimeout = opt.DialTimeout\n\t\tfopt.ReadTimeout = opt.ReadTimeout\n\t\tfopt.WriteTimeout = opt.WriteTimeout\n\t\tfopt.PoolFIFO = opt.PoolFIFO // default: false\n\t\tfopt.PoolSize = opt.PoolSize // default: GOMAXPROCS * 10\n\t\tfopt.PoolTimeout = opt.PoolTimeout // default: ReadTimeout + 1 second.\n\t\tfopt.MinIdleConns = opt.MinIdleConns // disable by default\n\t\tfopt.MaxIdleConns = opt.MaxIdleConns // disable by default\n\t\tfopt.MaxActiveConns = opt.MaxActiveConns // default: 0, no limit\n\t\tfopt.ConnMaxIdleTime = opt.ConnMaxIdleTime // default: 30 minutes\n\t\tfopt.ConnMaxLifetime = opt.ConnMaxLifetime // disable by default\n\t\tif conf.ReadOnly {\n\t\t\t// NOTE: RouteByLatency and RouteRandomly are not supported since they require cluster client\n\t\t\tfopt.ReplicaOnly = routeRead == \"replica\"\n\t\t}\n\t\trdb = redis.NewFailoverClient(&fopt)\n\t} else {\n\t\tif !strings.Contains(hosts, \",\") {\n\t\t\tc := redis.NewClient(opt)\n\t\t\tinfo, err := c.ClusterInfo(Background()).Result()\n\t\t\tif err != nil && strings.Contains(err.Error(), \"cluster mode\") || err == nil && strings.Contains(info, \"cluster_state:\") {\n\t\t\t\tlogger.Infof(\"redis %s is in cluster mode\", hosts)\n\t\t\t} else {\n\t\t\t\trdb = c\n\t\t\t}\n\t\t}\n\t\tif rdb == nil {\n\t\t\tvar copt redis.ClusterOptions\n\t\t\tcopt.Addrs = strings.Split(hosts, \",\")\n\t\t\tcopt.MaxRedirects = 1\n\t\t\tcopt.Username = opt.Username\n\t\t\tcopt.Password = opt.Password\n\t\t\tcopt.TLSConfig = opt.TLSConfig\n\t\t\tcopt.MaxRetries = opt.MaxRetries\n\t\t\tcopt.MinRetryBackoff = opt.MinRetryBackoff\n\t\t\tcopt.MaxRetryBackoff = opt.MaxRetryBackoff\n\t\t\tcopt.DialTimeout = opt.DialTimeout\n\t\t\tcopt.ReadTimeout = opt.ReadTimeout\n\t\t\tcopt.WriteTimeout = opt.WriteTimeout\n\t\t\tcopt.PoolFIFO = opt.PoolFIFO // default: false\n\t\t\tcopt.PoolSize = opt.PoolSize // default: GOMAXPROCS * 10\n\t\t\tcopt.PoolTimeout = opt.PoolTimeout // default: ReadTimeout + 1 second.\n\t\t\tcopt.MinIdleConns = opt.MinIdleConns // disable by default\n\t\t\tcopt.MaxIdleConns = opt.MaxIdleConns // disable by default\n\t\t\tcopt.MaxActiveConns = opt.MaxActiveConns // default: 0, no limit\n\t\t\tcopt.ConnMaxIdleTime = opt.ConnMaxIdleTime // default: 30 minutes\n\t\t\tcopt.ConnMaxLifetime = opt.ConnMaxLifetime // disable by default\n\t\t\tif conf.ReadOnly {\n\t\t\t\tswitch routeRead {\n\t\t\t\tcase \"random\":\n\t\t\t\t\tcopt.RouteRandomly = true\n\t\t\t\tcase \"latency\":\n\t\t\t\t\tcopt.RouteByLatency = true\n\t\t\t\tcase \"replica\":\n\t\t\t\t\tcopt.ReadOnly = true\n\t\t\t\tdefault:\n\t\t\t\t\t// route to primary\n\t\t\t\t}\n\t\t\t}\n\t\t\trdb = redis.NewClusterClient(&copt)\n\t\t\tprefix = fmt.Sprintf(\"{%d}\", opt.DB)\n\t\t}\n\t}\n\n\tm := &redisMeta{\n\t\tbaseMeta: newBaseMeta(addr, conf),\n\t\trdb: rdb,\n\t\tprefix: prefix,\n\t}\n\tif clientCache {\n\t\tm.cache = newRedisCache(prefix, clientCacheSize, clientCacheExpiry, clientCachePreload)\n\t\tif err = m.cache.init(m.rdb); err != nil {\n\t\t\tlogger.Warnf(\"Failed to setup client-side caching: %v\", err)\n\t\t\tm.cache = nil\n\t\t}\n\t}\n\tm.en = m\n\tm.checkServerConfig()\n\treturn m, nil\n}\n\nfunc (m *redisMeta) Shutdown() error {\n\tif m.cache != nil {\n\t\tm.cache.close()\n\t\tm.cache = nil\n\t}\n\treturn m.rdb.Close()\n}\n\n// Override NewSession to initialize client-side cache after session is created\nfunc (m *redisMeta) NewSession(record bool) error {\n\t// First, create the session normally\n\terr := m.baseMeta.NewSession(record)\n\tif err != nil {\n\t\treturn err\n\t}\n\tgo m.preloadCache()\n\treturn nil\n}\n\nfunc (m *redisMeta) doDeleteSlice(id uint64, size uint32) error {\n\treturn m.rdb.HDel(Background(), m.sliceRefs(), m.sliceKey(id, size)).Err()\n}\n\nfunc (m *redisMeta) Name() string {\n\treturn \"redis\"\n}\n\nfunc (m *redisMeta) doInit(format *Format, force bool) error {\n\tctx := Background()\n\tbody, err := m.rdb.Get(ctx, m.setting()).Bytes()\n\tif err != nil && err != redis.Nil {\n\t\treturn err\n\t}\n\tif err == nil {\n\t\tvar old Format\n\t\terr = json.Unmarshal(body, &old)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"existing format is broken: %s\", err)\n\t\t}\n\t\tif !old.DirStats && format.DirStats {\n\t\t\t// remove dir stats as they are outdated\n\t\t\terr := m.rdb.Del(ctx, m.dirUsedInodesKey(), m.dirUsedSpaceKey()).Err()\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"remove dir stats\")\n\t\t\t}\n\t\t}\n\t\tif !old.UserGroupQuota && format.UserGroupQuota {\n\t\t\t// remove user group quota as they are outdated\n\t\t\terr := m.rdb.Del(ctx, m.userQuotaKey(), m.userQuotaUsedSpaceKey(), m.userQuotaUsedInodesKey(),\n\t\t\t\tm.groupQuotaKey(), m.groupQuotaUsedSpaceKey(), m.groupQuotaUsedInodesKey()).Err()\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"remove user group quota\")\n\t\t\t}\n\t\t}\n\t\tif err = format.update(&old, force); err != nil {\n\t\t\treturn errors.Wrap(err, \"update format\")\n\t\t}\n\t}\n\n\tdata, err := json.MarshalIndent(format, \"\", \"\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"json: %s\", err)\n\t}\n\tts := time.Now().Unix()\n\tattr := &Attr{\n\t\tTyp: TypeDirectory,\n\t\tAtime: ts,\n\t\tMtime: ts,\n\t\tCtime: ts,\n\t\tNlink: 2,\n\t\tLength: 4 << 10,\n\t\tParent: RootInode,\n\t}\n\tif format.TrashDays > 0 {\n\t\tattr.Mode = 0555\n\t\tif err = m.rdb.SetNX(ctx, m.inodeKey(TrashInode), m.marshal(attr), 0).Err(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err = m.rdb.Set(ctx, m.setting(), data, 0).Err(); err != nil {\n\t\treturn err\n\t}\n\tm.fmt = format\n\tif body != nil {\n\t\treturn nil\n\t}\n\n\t// root inode\n\tattr.Mode = 0777\n\treturn m.rdb.Set(ctx, m.inodeKey(RootInode), m.marshal(attr), 0).Err()\n}\n\nfunc (m *redisMeta) cacheACLs(ctx Context) error {\n\tif !m.getFormat().EnableACL {\n\t\treturn nil\n\t}\n\n\tvals, err := m.rdb.HGetAll(ctx, m.aclKey()).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor k, v := range vals {\n\t\tid, _ := strconv.ParseUint(k, 10, 32)\n\t\ttmpRule := &aclAPI.Rule{}\n\t\ttmpRule.Decode([]byte(v))\n\t\tm.aclCache.Put(uint32(id), tmpRule)\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) Reset() error {\n\tif m.prefix != \"\" {\n\t\treturn m.scan(Background(), \"*\", func(keys []string) error {\n\t\t\treturn m.rdb.Del(Background(), keys...).Err()\n\t\t})\n\t}\n\treturn m.rdb.FlushDB(Background()).Err()\n}\n\nfunc (m *redisMeta) doLoad() ([]byte, error) {\n\tbody, err := m.rdb.Get(Background(), m.setting()).Bytes()\n\tif err == redis.Nil {\n\t\treturn nil, nil\n\t}\n\treturn body, err\n}\n\nfunc (m *redisMeta) doNewSession(sinfo []byte, update bool) error {\n\terr := m.rdb.ZAdd(Background(), m.allSessions(), redis.Z{\n\t\tScore: float64(m.expireTime()),\n\t\tMember: strconv.FormatUint(m.sid, 10)}).Err()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"set session ID %d: %s\", m.sid, err)\n\t}\n\tif err = m.rdb.HSet(Background(), m.sessionInfos(), m.sid, sinfo).Err(); err != nil {\n\t\treturn fmt.Errorf(\"set session info: %s\", err)\n\t}\n\n\tif m.shaLookup, err = m.rdb.ScriptLoad(Background(), scriptLookup).Result(); err != nil {\n\t\tlogger.Warnf(\"load scriptLookup: %v\", err)\n\t\tm.shaLookup = \"\"\n\t}\n\tif m.shaResolve, err = m.rdb.ScriptLoad(Background(), scriptResolve).Result(); err != nil {\n\t\tlogger.Warnf(\"load scriptResolve: %v\", err)\n\t\tm.shaResolve = \"\"\n\t}\n\n\tif !m.conf.NoBGJob {\n\t\tgo m.cleanupLegacies()\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) getCounter(name string) (int64, error) {\n\tv, err := m.rdb.Get(Background(), m.counterKey(name)).Int64()\n\tif err == redis.Nil {\n\t\terr = nil\n\t}\n\treturn v, err\n}\n\nfunc (m *redisMeta) incrCounter(name string, value int64) (int64, error) {\n\tif m.conf.ReadOnly {\n\t\treturn 0, syscall.EROFS\n\t}\n\tkey := m.counterKey(name)\n\tif name == \"nextInode\" || name == \"nextChunk\" {\n\t\t// for nextinode, nextchunk\n\t\t// the current one is already used\n\t\tv, err := m.rdb.IncrBy(Background(), key, value).Result()\n\t\treturn v + 1, err\n\t}\n\treturn m.rdb.IncrBy(Background(), key, value).Result()\n}\n\nfunc (m *redisMeta) setIfSmall(name string, value, diff int64) (bool, error) {\n\tvar changed bool\n\tctx := Background()\n\tname = m.prefix + name\n\terr := m.txn(ctx.WithValue(txMethodKey{}, \"setIfSmall:\"+name), func(tx *redis.Tx) error {\n\t\tchanged = false\n\t\told, err := tx.Get(ctx, name).Int64()\n\t\tif err != nil && err != redis.Nil {\n\t\t\treturn err\n\t\t}\n\t\tif old > value-diff {\n\t\t\treturn nil\n\t\t} else {\n\t\t\tchanged = true\n\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\tpipe.Set(ctx, name, value, 0)\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn err\n\t\t}\n\t}, name)\n\n\treturn changed, err\n}\n\nfunc (m *redisMeta) getSession(sid string, detail bool) (*Session, error) {\n\tctx := Background()\n\tinfo, err := m.rdb.HGet(ctx, m.sessionInfos(), sid).Bytes()\n\tif err == redis.Nil { // legacy client has no info\n\t\tinfo = []byte(\"{}\")\n\t} else if err != nil {\n\t\treturn nil, fmt.Errorf(\"HGet sessionInfos %s: %s\", sid, err)\n\t}\n\tvar s Session\n\tif err := json.Unmarshal(info, &s); err != nil {\n\t\treturn nil, fmt.Errorf(\"corrupted session info; json error: %s\", err)\n\t}\n\ts.Sid, _ = strconv.ParseUint(sid, 10, 64)\n\tif detail {\n\t\tinodes, err := m.rdb.SMembers(ctx, m.sustained(s.Sid)).Result()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"SMembers %s: %s\", sid, err)\n\t\t}\n\t\ts.Sustained = make([]Ino, 0, len(inodes))\n\t\tfor _, sinode := range inodes {\n\t\t\tinode, _ := strconv.ParseUint(sinode, 10, 64)\n\t\t\ts.Sustained = append(s.Sustained, Ino(inode))\n\t\t}\n\n\t\tlocks, err := m.rdb.SMembers(ctx, m.lockedKey(s.Sid)).Result()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"SMembers %s: %s\", sid, err)\n\t\t}\n\t\ts.Flocks = make([]Flock, 0, len(locks)) // greedy\n\t\ts.Plocks = make([]Plock, 0, len(locks))\n\t\tfor _, lock := range locks {\n\t\t\towners, err := m.rdb.HGetAll(ctx, lock).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"HGetAll %s: %s\", lock, err)\n\t\t\t}\n\t\t\tisFlock := strings.HasPrefix(lock, m.prefix+\"lockf\")\n\t\t\tinode, _ := strconv.ParseUint(lock[len(m.prefix)+5:], 10, 64)\n\t\t\tfor k, v := range owners {\n\t\t\t\tparts := strings.Split(k, \"_\")\n\t\t\t\tif parts[0] != sid {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\towner, _ := strconv.ParseUint(parts[1], 16, 64)\n\t\t\t\tif isFlock {\n\t\t\t\t\ts.Flocks = append(s.Flocks, Flock{Ino(inode), owner, v})\n\t\t\t\t} else {\n\t\t\t\t\ts.Plocks = append(s.Plocks, Plock{Ino(inode), owner, loadLocks([]byte(v))})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn &s, nil\n}\n\nfunc (m *redisMeta) GetSession(sid uint64, detail bool) (*Session, error) {\n\tvar legacy bool\n\tkey := strconv.FormatUint(sid, 10)\n\tscore, err := m.rdb.ZScore(Background(), m.allSessions(), key).Result()\n\tif err == redis.Nil {\n\t\tlegacy = true\n\t\tscore, err = m.rdb.ZScore(Background(), legacySessions, key).Result()\n\t}\n\tif err == redis.Nil {\n\t\terr = fmt.Errorf(\"session not found: %d\", sid)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ts, err := m.getSession(key, detail)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ts.Expire = time.Unix(int64(score), 0)\n\tif legacy {\n\t\ts.Expire = s.Expire.Add(time.Minute * 5)\n\t}\n\treturn s, nil\n}\n\nfunc (m *redisMeta) ListSessions() ([]*Session, error) {\n\tkeys, err := m.rdb.ZRangeWithScores(Background(), m.allSessions(), 0, -1).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsessions := make([]*Session, 0, len(keys))\n\tfor _, k := range keys {\n\t\ts, err := m.getSession(k.Member.(string), false)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"get session: %s\", err)\n\t\t\tcontinue\n\t\t}\n\t\ts.Expire = time.Unix(int64(k.Score), 0)\n\t\tsessions = append(sessions, s)\n\t}\n\n\t// add clients with version before 1.0-beta3 as well\n\tkeys, err = m.rdb.ZRangeWithScores(Background(), legacySessions, 0, -1).Result()\n\tif err != nil {\n\t\tlogger.Errorf(\"Scan legacy sessions: %s\", err)\n\t\treturn sessions, nil\n\t}\n\tfor _, k := range keys {\n\t\ts, err := m.getSession(k.Member.(string), false)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Get legacy session: %s\", err)\n\t\t\tcontinue\n\t\t}\n\t\ts.Expire = time.Unix(int64(k.Score), 0).Add(time.Minute * 5)\n\t\tsessions = append(sessions, s)\n\t}\n\treturn sessions, nil\n}\n\nfunc (m *redisMeta) sustained(sid uint64) string {\n\treturn m.prefix + \"session\" + strconv.FormatUint(sid, 10)\n}\n\nfunc (m *redisMeta) lockedKey(sid uint64) string {\n\treturn m.prefix + \"locked\" + strconv.FormatUint(sid, 10)\n}\n\nfunc (m *redisMeta) symKey(inode Ino) string {\n\treturn m.prefix + \"s\" + inode.String()\n}\n\nfunc (m *redisMeta) inodeKey(inode Ino) string {\n\treturn m.prefix + \"i\" + inode.String()\n}\n\nfunc (m *redisMeta) entryKey(parent Ino) string {\n\treturn m.prefix + \"d\" + parent.String()\n}\n\nfunc (m *redisMeta) parentKey(inode Ino) string {\n\treturn m.prefix + \"p\" + inode.String()\n}\n\nfunc (m *redisMeta) chunkKey(inode Ino, indx uint32) string {\n\treturn m.prefix + \"c\" + inode.String() + \"_\" + strconv.FormatInt(int64(indx), 10)\n}\n\nfunc (m *redisMeta) sliceKey(id uint64, size uint32) string {\n\t// inside hashset\n\treturn \"k\" + strconv.FormatUint(id, 10) + \"_\" + strconv.FormatUint(uint64(size), 10)\n}\n\nfunc (m *redisMeta) xattrKey(inode Ino) string {\n\treturn m.prefix + \"x\" + inode.String()\n}\n\nfunc (m *redisMeta) flockKey(inode Ino) string {\n\treturn m.prefix + \"lockf\" + inode.String()\n}\n\nfunc (m *redisMeta) ownerKey(owner uint64) string {\n\treturn fmt.Sprintf(\"%d_%016X\", m.sid, owner)\n}\n\nfunc (m *redisMeta) plockKey(inode Ino) string {\n\treturn m.prefix + \"lockp\" + inode.String()\n}\n\nfunc (m *redisMeta) setting() string {\n\treturn m.prefix + \"setting\"\n}\n\nfunc (m *redisMeta) usedSpaceKey() string {\n\treturn m.prefix + usedSpace\n}\n\nfunc (m *redisMeta) nextTrashKey() string {\n\treturn m.prefix + \"nextTrash\"\n}\n\nfunc (m *redisMeta) counterKey(name string) string {\n\tif name == \"nextInode\" || name == \"nextChunk\" || name == \"nextSession\" {\n\t\tname = strings.ToLower(name)\n\t}\n\treturn m.prefix + name\n}\n\nfunc (m *redisMeta) dirDataLengthKey() string {\n\treturn m.prefix + \"dirDataLength\"\n}\n\nfunc (m *redisMeta) dirUsedSpaceKey() string {\n\treturn m.prefix + \"dirUsedSpace\"\n}\n\nfunc (m *redisMeta) dirUsedInodesKey() string {\n\treturn m.prefix + \"dirUsedInodes\"\n}\n\nfunc (m *redisMeta) dirQuotaUsedSpaceKey() string {\n\treturn m.prefix + \"dirQuotaUsedSpace\"\n}\n\nfunc (m *redisMeta) dirQuotaUsedInodesKey() string {\n\treturn m.prefix + \"dirQuotaUsedInodes\"\n}\n\nfunc (m *redisMeta) dirQuotaKey() string {\n\treturn m.prefix + \"dirQuota\"\n}\n\nfunc (m *redisMeta) userQuotaUsedSpaceKey() string {\n\treturn m.prefix + \"userQuotaUsedSpace\"\n}\n\nfunc (m *redisMeta) userQuotaUsedInodesKey() string {\n\treturn m.prefix + \"userQuotaUsedInodes\"\n}\n\nfunc (m *redisMeta) userQuotaKey() string {\n\treturn m.prefix + \"userQuota\"\n}\n\nfunc (m *redisMeta) groupQuotaUsedSpaceKey() string {\n\treturn m.prefix + \"groupQuotaUsedSpace\"\n}\n\nfunc (m *redisMeta) groupQuotaUsedInodesKey() string {\n\treturn m.prefix + \"groupQuotaUsedInodes\"\n}\n\nfunc (m *redisMeta) groupQuotaKey() string {\n\treturn m.prefix + \"groupQuota\"\n}\n\nfunc (m *redisMeta) totalInodesKey() string {\n\treturn m.prefix + totalInodes\n}\n\nfunc (m *redisMeta) aclKey() string {\n\treturn m.prefix + \"acl\"\n}\n\nfunc (m *redisMeta) krbTokenKey() string {\n\treturn m.prefix + \"krbToken\"\n}\n\nfunc (m *redisMeta) delfiles() string {\n\treturn m.prefix + \"delfiles\"\n}\n\nfunc (m *redisMeta) detachedNodes() string {\n\treturn m.prefix + \"detachedNodes\"\n}\n\nfunc (r *redisMeta) delSlices() string {\n\treturn r.prefix + \"delSlices\"\n}\n\nfunc (r *redisMeta) allSessions() string {\n\treturn r.prefix + \"allSessions\"\n}\n\nfunc (m *redisMeta) sessionInfos() string {\n\treturn m.prefix + \"sessionInfos\"\n}\n\nfunc (m *redisMeta) sliceRefs() string {\n\treturn m.prefix + \"sliceRef\"\n}\n\nfunc (m *redisMeta) packQuota(space, inodes int64) []byte {\n\twb := utils.NewBuffer(16)\n\twb.Put64(uint64(space))\n\twb.Put64(uint64(inodes))\n\treturn wb.Bytes()\n}\n\nfunc (m *redisMeta) parseQuota(buf []byte) (space, inodes int64) {\n\tif len(buf) == 0 {\n\t\treturn 0, 0\n\t}\n\tif len(buf) != 16 {\n\t\tlogger.Errorf(\"invalid quota value: %v\", buf)\n\t\treturn 0, 0\n\t}\n\trb := utils.ReadBuffer(buf)\n\treturn int64(rb.Get64()), int64(rb.Get64())\n}\n\nfunc (m *redisMeta) packEntry(_type uint8, inode Ino) []byte {\n\twb := utils.NewBuffer(9)\n\twb.Put8(_type)\n\twb.Put64(uint64(inode))\n\treturn wb.Bytes()\n}\n\nfunc (m *redisMeta) parseEntry(buf []byte) (uint8, Ino) {\n\tif len(buf) != 9 {\n\t\tpanic(\"invalid entry\")\n\t}\n\treturn buf[0], Ino(binary.BigEndian.Uint64(buf[1:]))\n}\n\nfunc (m *redisMeta) updateStats(space int64, inodes int64) {\n\tatomic.AddInt64(&m.usedSpace, space)\n\tatomic.AddInt64(&m.usedInodes, inodes)\n}\n\nfunc (m *redisMeta) doSyncVolumeStat(ctx Context) error {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tvar used, inodes int64\n\tif err := m.hscan(ctx, m.dirUsedSpaceKey(), func(keys []string) error {\n\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\tv, err := strconv.ParseInt(keys[i+1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"invalid used space: %s->%s\", keys[i], keys[i+1])\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tused += v\n\t\t}\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\tif err := m.hscan(ctx, m.dirUsedInodesKey(), func(keys []string) error {\n\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\tv, err := strconv.ParseInt(keys[i+1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"invalid used inode: %s->%s\", keys[i], keys[i+1])\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tinodes += v\n\t\t}\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tvar inoKeys []string\n\tif err := m.scan(ctx, m.prefix+\"session*\", func(keys []string) error {\n\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\tkey := keys[i]\n\t\t\tif key == \"sessions\" {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tinodes, err := m.rdb.SMembers(ctx, key).Result()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"SMembers %s: %s\", key, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, sinode := range inodes {\n\t\t\t\tino, err := strconv.ParseInt(sinode, 10, 64)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Warnf(\"invalid sustained: %s->%s\", key, sinode)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tinoKeys = append(inoKeys, m.inodeKey(Ino(ino)))\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tbatch := 1000\n\tfor i := 0; i < len(inoKeys); i += batch {\n\t\tend := i + batch\n\t\tif end > len(inoKeys) {\n\t\t\tend = len(inoKeys)\n\t\t}\n\t\tvalues, err := m.rdb.MGet(ctx, inoKeys[i:end]...).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvar attr Attr\n\t\tfor _, v := range values {\n\t\t\tif v != nil {\n\t\t\t\tm.parseAttr([]byte(v.(string)), &attr)\n\t\t\t\tused += align4K(attr.Length)\n\t\t\t\tinodes += 1\n\t\t\t}\n\t\t}\n\t}\n\tif err := m.scanTrashEntry(ctx, func(_ Ino, length uint64) {\n\t\tused += align4K(length)\n\t\tinodes += 1\n\t}); err != nil {\n\t\treturn err\n\t}\n\tlogger.Debugf(\"Used space: %s, inodes: %d\", humanize.IBytes(uint64(used)), inodes)\n\tif err := m.rdb.Set(ctx, m.totalInodesKey(), strconv.FormatInt(inodes, 10), 0).Err(); err != nil {\n\t\treturn fmt.Errorf(\"set total inodes: %s\", err)\n\t}\n\treturn m.rdb.Set(ctx, m.usedSpaceKey(), strconv.FormatInt(used, 10), 0).Err()\n}\n\n// redisMeta updates the usage in each transaction\nfunc (m *redisMeta) doFlushStats() {}\n\nfunc (m *redisMeta) handleLuaResult(op string, res interface{}, err error, returnedIno *int64, returnedAttr *string) syscall.Errno {\n\tif err != nil {\n\t\tmsg := err.Error()\n\t\tif strings.Contains(msg, \"NOSCRIPT\") {\n\t\t\tvar err2 error\n\t\t\tswitch op {\n\t\t\tcase \"lookup\":\n\t\t\t\tm.shaLookup, err2 = m.rdb.ScriptLoad(Background(), scriptLookup).Result()\n\t\t\tcase \"resolve\":\n\t\t\t\tm.shaResolve, err2 = m.rdb.ScriptLoad(Background(), scriptResolve).Result()\n\t\t\tdefault:\n\t\t\t\treturn syscall.ENOTSUP\n\t\t\t}\n\t\t\tif err2 == nil {\n\t\t\t\tlogger.Infof(\"loaded script succeed for %s\", op)\n\t\t\t\treturn syscall.EAGAIN\n\t\t\t} else {\n\t\t\t\tlogger.Warnf(\"load script %s: %s\", op, err2)\n\t\t\t\treturn syscall.ENOTSUP\n\t\t\t}\n\t\t} else if strings.Contains(msg, \"ENOENT\") {\n\t\t\treturn syscall.ENOENT\n\t\t} else if strings.Contains(msg, \"EACCESS\") {\n\t\t\treturn syscall.EACCES\n\t\t} else if strings.Contains(msg, \"ENOTDIR\") {\n\t\t\treturn syscall.ENOTDIR\n\t\t} else if strings.Contains(msg, \"ENOTSUP\") {\n\t\t\treturn syscall.ENOTSUP\n\t\t} else {\n\t\t\tlogger.Warnf(\"unexpected error for %s: %s\", op, msg)\n\t\t\tswitch op {\n\t\t\tcase \"lookup\":\n\t\t\t\tm.shaLookup = \"\"\n\t\t\tcase \"resolve\":\n\t\t\t\tm.shaResolve = \"\"\n\t\t\t}\n\t\t\treturn syscall.ENOTSUP\n\t\t}\n\t}\n\tvals, ok := res.([]interface{})\n\tif !ok {\n\t\tlogger.Errorf(\"invalid script result: %v\", res)\n\t\treturn syscall.ENOTSUP\n\t}\n\t*returnedIno, ok = vals[0].(int64)\n\tif !ok {\n\t\tlogger.Errorf(\"invalid script result: %v\", res)\n\t\treturn syscall.ENOTSUP\n\t}\n\tif vals[1] == nil {\n\t\treturn syscall.ENOTSUP\n\t}\n\t*returnedAttr, ok = vals[1].(string)\n\tif !ok {\n\t\tlogger.Errorf(\"invalid script result: %v\", res)\n\t\treturn syscall.ENOTSUP\n\t}\n\treturn 0\n}\n\nfunc (m *redisMeta) doLookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr) syscall.Errno {\n\tvar foundIno Ino\n\tvar foundType uint8\n\tvar encodedAttr []byte\n\tvar err error\n\tentryKey := m.entryKey(parent)\n\tif m.cache != nil {\n\t\tif entry, ok := m.cache.entryCache.Get(m.cache.entryName(parent, name)); ok {\n\t\t\tif !entry.isMark() {\n\t\t\t\t*inode = entry.ino\n\t\t\t\tif attr != nil {\n\t\t\t\t\t*attr = entry.Attr\n\t\t\t\t}\n\t\t\t\treturn 0\n\t\t\t}\n\t\t\tm.cache.entryCache.AddIf(m.cache.entryName(parent, name), &entryMark, func(oldEntry *cachedEntry, exists bool) bool {\n\t\t\t\treturn exists\n\t\t\t})\n\t\t}\n\t}\n\tif len(m.shaLookup) > 0 && attr != nil && !m.conf.CaseInsensi && m.prefix == \"\" {\n\t\tvar res interface{}\n\t\tvar returnedIno int64\n\t\tvar returnedAttr string\n\t\tres, err = m.rdb.EvalSha(ctx, m.shaLookup, []string{entryKey, name}).Result()\n\t\tif st := m.handleLuaResult(\"lookup\", res, err, &returnedIno, &returnedAttr); st == 0 {\n\t\t\tfoundIno = Ino(returnedIno)\n\t\t\tencodedAttr = []byte(returnedAttr)\n\t\t} else if st == syscall.EAGAIN {\n\t\t\treturn m.doLookup(ctx, parent, name, inode, attr)\n\t\t} else if st != syscall.ENOTSUP {\n\t\t\treturn st\n\t\t}\n\t}\n\tif foundIno == 0 || len(encodedAttr) == 0 {\n\t\tvar buf []byte\n\t\tbuf, err = m.rdb.HGet(ctx, entryKey, name).Bytes()\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\t\tfoundType, foundIno = m.parseEntry(buf)\n\t\tencodedAttr, err = m.rdb.Get(ctx, m.inodeKey(foundIno)).Bytes()\n\t}\n\n\tif err == nil {\n\t\tm.parseAttr(encodedAttr, attr)\n\t\tm.of.Update(foundIno, attr)\n\t\tif m.cache != nil {\n\t\t\tce := &cachedEntry{ino: foundIno}\n\t\t\tm.parseAttr(encodedAttr, &ce.Attr)\n\t\t\t_, _ = m.cache.entryCache.AddIf(m.cache.entryName(parent, name), ce, func(oldEntry *cachedEntry, exists bool) bool {\n\t\t\t\treturn exists && oldEntry.isMark()\n\t\t\t})\n\t\t}\n\t} else if err == redis.Nil { // corrupt entry\n\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", foundIno, parent, name)\n\t\t*attr = Attr{Typ: foundType}\n\t\terr = nil\n\t}\n\t*inode = foundIno\n\treturn errno(err)\n}\n\nfunc (m *redisMeta) Resolve(ctx Context, parent Ino, path string, inode *Ino, attr *Attr) syscall.Errno {\n\tif len(m.shaResolve) == 0 || m.conf.CaseInsensi || m.prefix != \"\" {\n\t\treturn syscall.ENOTSUP\n\t}\n\tdefer m.timeit(\"Resolve\", time.Now())\n\tparent = m.checkRoot(parent)\n\tkeys := []string{parent.String(), path,\n\t\tstrconv.FormatUint(uint64(ctx.Uid()), 10)}\n\tvar gids []interface{}\n\tfor _, gid := range ctx.Gids() {\n\t\tgids = append(gids, strconv.FormatUint(uint64(gid), 10))\n\t}\n\tres, err := m.rdb.EvalSha(ctx, m.shaResolve, keys, gids...).Result()\n\tvar returnedIno int64\n\tvar returnedAttr string\n\tst := m.handleLuaResult(\"resolve\", res, err, &returnedIno, &returnedAttr)\n\tif st == 0 {\n\t\tif inode != nil {\n\t\t\t*inode = Ino(returnedIno)\n\t\t}\n\t\tm.parseAttr([]byte(returnedAttr), attr)\n\t} else if st == syscall.EAGAIN {\n\t\treturn m.Resolve(ctx, parent, path, inode, attr)\n\t}\n\treturn st\n}\n\nfunc (m *redisMeta) doGetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno {\n\ta, err := m.rdb.Get(ctx, m.inodeKey(inode)).Bytes()\n\tif err == nil {\n\t\tm.parseAttr(a, attr)\n\t}\n\treturn errno(err)\n}\n\ntype timeoutError interface {\n\tTimeout() bool\n}\n\nfunc (m *redisMeta) shouldRetry(err error, retryOnFailure bool) bool {\n\tswitch err {\n\tcase redis.TxFailedErr:\n\t\treturn true\n\tcase io.EOF, io.ErrUnexpectedEOF:\n\t\treturn retryOnFailure\n\tcase nil, context.Canceled, context.DeadlineExceeded:\n\t\treturn false\n\t}\n\n\tif v, ok := err.(timeoutError); ok && v.Timeout() {\n\t\treturn retryOnFailure\n\t}\n\n\ts := err.Error()\n\tif s == \"ERR max number of clients reached\" ||\n\t\tstrings.Contains(s, \"Conn is in a bad state\") ||\n\t\tstrings.Contains(s, \"EXECABORT\") {\n\t\treturn true\n\t}\n\tps := strings.SplitN(s, \" \", 3)\n\tswitch ps[0] {\n\tcase \"LOADING\":\n\tcase \"READONLY\":\n\tcase \"CLUSTERDOWN\":\n\tcase \"TRYAGAIN\":\n\tcase \"MOVED\":\n\tcase \"ASK\":\n\tcase \"ERR\":\n\t\tif len(ps) > 1 {\n\t\t\tswitch ps[1] {\n\t\t\tcase \"DISABLE\":\n\t\t\t\tfallthrough\n\t\t\tcase \"NOWRITE\":\n\t\t\t\tfallthrough\n\t\t\tcase \"NOREAD\":\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\tdefault:\n\t\treturn false\n\t}\n\treturn true\n}\n\n// errNo is an alias to syscall.Errno to disable retry in Redis Cluster\ntype errNo uintptr\n\nfunc (e errNo) Error() string {\n\treturn syscall.Errno(e).Error()\n}\n\n// replaceErrno replace returned syscall.Errno as errNo\nfunc replaceErrno(txf func(tx *redis.Tx) error) func(tx *redis.Tx) error {\n\treturn func(tx *redis.Tx) error {\n\t\terr := txf(tx)\n\t\tif eno, ok := err.(syscall.Errno); ok {\n\t\t\terr = errNo(eno)\n\t\t}\n\t\treturn err\n\t}\n}\n\nfunc (m *redisMeta) txn(ctx Context, txf func(tx *redis.Tx) error, keys ...string) error {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tfor _, k := range keys {\n\t\tif !strings.HasPrefix(k, m.prefix) {\n\t\t\tpanic(fmt.Sprintf(\"Invalid key %s not starts with prefix %s\", k, m.prefix))\n\t\t}\n\t}\n\tvar khash = fnv.New32()\n\t_, _ = khash.Write([]byte(keys[0]))\n\th := uint(khash.Sum32())\n\n\tstart := time.Now()\n\tdefer func() { m.txDist.Observe(time.Since(start).Seconds()) }()\n\n\tm.txLock(h)\n\tdefer m.txUnlock(h)\n\t// TODO: enable retry for some of idempotent transactions\n\tvar (\n\t\tretryOnFailure = false\n\t\tlastErr error\n\t\tmethod txMethod\n\t)\n\tfor i := 0; i < 50; i++ {\n\t\tif ctx.Canceled() {\n\t\t\tlogger.Warnf(\"Transaction %s interrupted after %s, tried %d, keys: %v\", method.name(ctx), time.Since(start), i+1, keys)\n\t\t\treturn syscall.EINTR\n\t\t}\n\t\terr := m.rdb.Watch(ctx, replaceErrno(txf), keys...)\n\t\tif eno, ok := err.(errNo); ok {\n\t\t\tif eno == 0 {\n\t\t\t\terr = nil\n\t\t\t} else {\n\t\t\t\terr = syscall.Errno(eno)\n\t\t\t}\n\t\t}\n\t\tif err != nil && m.shouldRetry(err, retryOnFailure) {\n\t\t\tm.txRestart.WithLabelValues(method.name(ctx)).Add(1)\n\t\t\tlogger.Debugf(\"Transaction failed, restart it (tried %d): %s\", i+1, err)\n\t\t\tlastErr = err\n\t\t\ttime.Sleep(time.Millisecond * time.Duration(rand.Int()%((i+1)*(i+1))))\n\t\t\tcontinue\n\t\t} else if err == nil && i > 1 {\n\t\t\tlogger.Warnf(\"Transaction succeeded after %d tries (%s), keys: %v, method: %s, last error: %s\", i+1, time.Since(start), keys, method.name(ctx), lastErr)\n\t\t}\n\t\treturn err\n\t}\n\tlogger.Warnf(\"Already tried 50 times, returning: %s\", lastErr)\n\treturn lastErr\n}\n\nfunc (m *redisMeta) doTruncate(ctx Context, inode Ino, flags uint8, length uint64, delta *dirStat, attr *Attr, skipPermCheck bool) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\t*delta = dirStat{}\n\t\tvar t Attr\n\t\ta, err := tx.Get(ctx, m.inodeKey(inode)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, &t)\n\t\tif t.Typ != TypeFile || t.Flags&(FlagImmutable|FlagAppend) != 0 || (flags == 0 && t.Parent > TrashInode) {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif !skipPermCheck {\n\t\t\tif st := m.Access(ctx, inode, MODE_MASK_W, &t); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t}\n\t\tif length == t.Length {\n\t\t\t*attr = t\n\t\t\treturn nil\n\t\t}\n\t\tdelta.length = int64(length) - int64(t.Length)\n\t\tdelta.space = align4K(length) - align4K(t.Length)\n\t\tif err := m.checkQuota(ctx, delta.space, 0, t.Uid, t.Gid, m.getParents(ctx, tx, inode, t.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tvar zeroChunks []uint32\n\t\tvar left, right = t.Length, length\n\t\tif left > right {\n\t\t\tright, left = left, right\n\t\t}\n\t\tif (right-left)/ChunkSize >= 10000 {\n\t\t\t// super large\n\t\t\tvar cursor uint64\n\t\t\tvar keys []string\n\t\t\tfor {\n\t\t\t\tkeys, cursor, err = tx.Scan(ctx, cursor, m.prefix+fmt.Sprintf(\"c%d_*\", inode), 10000).Result()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tfor _, key := range keys {\n\t\t\t\t\tindx, err := strconv.Atoi(strings.Split(key[len(m.prefix):], \"_\")[1])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlogger.Errorf(\"parse %s: %s\", key, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif uint64(indx) > left/ChunkSize && uint64(indx) < right/ChunkSize {\n\t\t\t\t\t\tzeroChunks = append(zeroChunks, uint32(indx))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif cursor <= 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor i := left/ChunkSize + 1; i < right/ChunkSize; i++ {\n\t\t\t\tzeroChunks = append(zeroChunks, uint32(i))\n\t\t\t}\n\t\t}\n\t\tt.Length = length\n\t\tnow := time.Now()\n\t\tt.Mtime = now.Unix()\n\t\tt.Mtimensec = uint32(now.Nanosecond())\n\t\tt.Ctime = now.Unix()\n\t\tt.Ctimensec = uint32(now.Nanosecond())\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(&t), 0)\n\t\t\t// zero out from left to right\n\t\t\tvar l = uint32(right - left)\n\t\t\tif right > (left/ChunkSize+1)*ChunkSize {\n\t\t\t\tl = ChunkSize - uint32(left%ChunkSize)\n\t\t\t}\n\t\t\tpipe.RPush(ctx, m.chunkKey(inode, uint32(left/ChunkSize)), marshalSlice(uint32(left%ChunkSize), 0, 0, 0, l))\n\t\t\tbuf := marshalSlice(0, 0, 0, 0, ChunkSize)\n\t\t\tfor _, indx := range zeroChunks {\n\t\t\t\tpipe.RPushX(ctx, m.chunkKey(inode, indx), buf)\n\t\t\t}\n\t\t\tif right > (left/ChunkSize+1)*ChunkSize && right%ChunkSize > 0 {\n\t\t\t\tpipe.RPush(ctx, m.chunkKey(inode, uint32(right/ChunkSize)), marshalSlice(0, 0, 0, 0, uint32(right%ChunkSize)))\n\t\t\t}\n\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), delta.space)\n\t\t\treturn nil\n\t\t})\n\t\tif err == nil {\n\t\t\t*attr = t\n\t\t}\n\t\treturn err\n\t}, m.inodeKey(inode)))\n}\n\nfunc (m *redisMeta) doFallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, delta *dirStat, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\t*delta = dirStat{}\n\t\tt := Attr{}\n\t\ta, err := tx.Get(ctx, m.inodeKey(inode)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, &t)\n\t\tif t.Typ == TypeFIFO {\n\t\t\treturn syscall.EPIPE\n\t\t}\n\t\tif t.Typ != TypeFile || (t.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W, &t); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (t.Flags&FlagAppend) != 0 && (mode&^fallocKeepSize) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tlength := t.Length\n\t\tif off+size > t.Length {\n\t\t\tif mode&fallocKeepSize == 0 {\n\t\t\t\tlength = off + size\n\t\t\t}\n\t\t}\n\n\t\told := t.Length\n\t\tdelta.length = int64(length) - int64(old)\n\t\tdelta.space = align4K(length) - align4K(old)\n\t\tif err := m.checkQuota(ctx, delta.space, 0, t.Uid, t.Gid, m.getParents(ctx, tx, inode, t.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tt.Length = length\n\t\tnow := time.Now()\n\t\tt.Mtime = now.Unix()\n\t\tt.Mtimensec = uint32(now.Nanosecond())\n\t\tt.Ctime = now.Unix()\n\t\tt.Ctimensec = uint32(now.Nanosecond())\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(&t), 0)\n\t\t\tif mode&(fallocZeroRange|fallocPunchHole) != 0 && off < old {\n\t\t\t\toff, size := off, size\n\t\t\t\tif off+size > old {\n\t\t\t\t\tsize = old - off\n\t\t\t\t}\n\t\t\t\tfor size > 0 {\n\t\t\t\t\tindx := uint32(off / ChunkSize)\n\t\t\t\t\tcoff := off % ChunkSize\n\t\t\t\t\tl := size\n\t\t\t\t\tif coff+size > ChunkSize {\n\t\t\t\t\t\tl = ChunkSize - coff\n\t\t\t\t\t}\n\t\t\t\t\tpipe.RPush(ctx, m.chunkKey(inode, indx), marshalSlice(uint32(coff), 0, 0, 0, uint32(l)))\n\t\t\t\t\toff += l\n\t\t\t\t\tsize -= l\n\t\t\t\t}\n\t\t\t}\n\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), align4K(length)-align4K(old))\n\t\t\treturn nil\n\t\t})\n\t\tif err == nil {\n\t\t\t*attr = t\n\t\t}\n\t\treturn err\n\t}, m.inodeKey(inode)))\n}\n\nfunc (m *redisMeta) doSetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr, oldAttr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tvar cur Attr\n\t\ta, err := tx.Get(ctx, m.inodeKey(inode)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, &cur)\n\t\tif oldAttr != nil {\n\t\t\t*oldAttr = cur\n\t\t}\n\t\tif cur.Parent > TrashInode {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tnow := time.Now()\n\n\t\trule, err := m.getACL(ctx, tx, cur.AccessACL)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\trule = rule.Dup()\n\t\tdirtyAttr, st := m.mergeAttr(ctx, inode, set, &cur, attr, now, rule)\n\t\tif st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif dirtyAttr == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\tdirtyAttr.AccessACL, err = m.insertACL(ctx, tx, rule)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tdirtyAttr.Ctime = now.Unix()\n\t\tdirtyAttr.Ctimensec = uint32(now.Nanosecond())\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(dirtyAttr), 0)\n\t\t\treturn nil\n\t\t})\n\t\tif err == nil {\n\t\t\t*attr = *dirtyAttr\n\t\t}\n\t\treturn err\n\t}, m.inodeKey(inode)))\n}\n\nfunc (m *redisMeta) doReadlink(ctx Context, inode Ino, noatime bool) (atime int64, target []byte, err error) {\n\tif noatime {\n\t\ttarget, err = m.rdb.Get(ctx, m.symKey(inode)).Bytes()\n\t\tif err == redis.Nil {\n\t\t\terr = nil\n\t\t}\n\t\treturn\n\t}\n\n\tattr := &Attr{}\n\tnow := time.Now()\n\terr = m.txn(ctx, func(tx *redis.Tx) error {\n\t\trs, e := tx.MGet(ctx, m.inodeKey(inode), m.symKey(inode)).Result()\n\t\tif e != nil {\n\t\t\treturn e\n\t\t}\n\t\tif rs[0] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr([]byte(rs[0].(string)), attr)\n\t\tif attr.Typ != TypeSymlink {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif rs[1] == nil {\n\t\t\treturn syscall.EIO\n\t\t}\n\t\ttarget = []byte(rs[1].(string))\n\t\tif !m.atimeNeedsUpdate(attr, now) {\n\t\t\tatime = attr.Atime*int64(time.Second) + int64(attr.Atimensec)\n\t\t\treturn nil\n\t\t}\n\t\tattr.Atime = now.Unix()\n\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\tatime = now.UnixNano()\n\t\t_, e = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\t\t\treturn nil\n\t\t})\n\t\treturn e\n\t}, m.inodeKey(inode))\n\treturn\n}\n\nfunc (m *redisMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode, cumask uint16, path string, inode *Ino, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tvar pattr Attr\n\t\ta, err := tx.Get(ctx, m.inodeKey(parent)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pattr.Flags & FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif (pattr.Flags & FlagSkipTrash) != 0 {\n\t\t\tattr.Flags |= FlagSkipTrash\n\t\t}\n\n\t\tbuf, err := tx.HGet(ctx, m.entryKey(parent), name).Bytes()\n\t\tif err != nil && err != redis.Nil {\n\t\t\treturn err\n\t\t}\n\t\tvar foundIno Ino\n\t\tvar foundType uint8\n\t\tif err == nil {\n\t\t\tfoundType, foundIno = m.parseEntry(buf)\n\t\t} else if m.conf.CaseInsensi { // err == redis.Nil\n\t\t\tif entry := m.resolveCase(ctx, parent, name); entry != nil {\n\t\t\t\tfoundType, foundIno = entry.Attr.Typ, entry.Inode\n\t\t\t}\n\t\t}\n\t\tif foundIno != 0 {\n\t\t\tif _type == TypeFile || _type == TypeDirectory { // file for create, directory for subTrash\n\t\t\t\ta, err = tx.Get(ctx, m.inodeKey(foundIno)).Bytes()\n\t\t\t\tif err == nil {\n\t\t\t\t\tm.parseAttr(a, attr)\n\t\t\t\t} else if err == redis.Nil {\n\t\t\t\t\t*attr = Attr{Typ: foundType, Parent: parent} // corrupt entry\n\t\t\t\t} else {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\t*inode = foundIno\n\t\t\t}\n\t\t\treturn syscall.EEXIST\n\t\t} else if parent == TrashInode {\n\t\t\tif next, err := tx.Incr(ctx, m.nextTrashKey()).Result(); err != nil { // Some inode will be wasted if conflict happens\n\t\t\t\treturn err\n\t\t\t} else {\n\t\t\t\t*inode = TrashInode + Ino(next)\n\t\t\t}\n\t\t}\n\n\t\tmode &= 07777\n\t\tif pattr.DefaultACL != aclAPI.None && _type != TypeSymlink {\n\t\t\t// inherit default acl\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tattr.DefaultACL = pattr.DefaultACL\n\t\t\t}\n\n\t\t\t// set access acl by parent's default acl\n\t\t\trule, err := m.getACL(ctx, tx, pattr.DefaultACL)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif rule.IsMinimal() {\n\t\t\t\t// simple acl as default\n\t\t\t\tattr.Mode = mode & (0xFE00 | rule.GetMode())\n\t\t\t} else {\n\t\t\t\tcRule := rule.ChildAccessACL(mode)\n\t\t\t\tid, err := m.insertACL(ctx, tx, cRule)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tattr.AccessACL = id\n\t\t\t\tattr.Mode = (mode & 0xFE00) | cRule.GetMode()\n\t\t\t}\n\t\t} else {\n\t\t\tattr.Mode = mode & ^cumask\n\t\t}\n\n\t\tvar updateParent bool\n\t\tnow := time.Now()\n\t\tif parent != TrashInode {\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tpattr.Nlink++\n\t\t\t\tupdateParent = true\n\t\t\t}\n\t\t\tif updateParent || now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime {\n\t\t\t\tpattr.Mtime = now.Unix()\n\t\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\t\tpattr.Ctime = now.Unix()\n\t\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tupdateParent = true\n\t\t\t}\n\t\t}\n\t\tattr.Atime = now.Unix()\n\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\tattr.Mtime = now.Unix()\n\t\tattr.Mtimensec = uint32(now.Nanosecond())\n\t\tattr.Ctime = now.Unix()\n\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\tif ctx.Value(CtxKey(\"behavior\")) == \"Hadoop\" || runtime.GOOS == \"darwin\" {\n\t\t\tattr.Gid = pattr.Gid\n\t\t} else if runtime.GOOS == \"linux\" && pattr.Mode&02000 != 0 {\n\t\t\tattr.Gid = pattr.Gid\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tattr.Mode |= 02000\n\t\t\t} else if attr.Mode&02010 == 02010 && ctx.Uid() != 0 {\n\t\t\t\tvar found bool\n\t\t\t\tfor _, gid := range ctx.Gids() {\n\t\t\t\t\tif gid == pattr.Gid {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tattr.Mode &= ^uint16(02000)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, m.inodeKey(*inode), m.marshal(attr), 0)\n\t\t\tif updateParent {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)\n\t\t\t}\n\t\t\tif _type == TypeSymlink {\n\t\t\t\tpipe.Set(ctx, m.symKey(*inode), path, 0)\n\t\t\t}\n\t\t\tpipe.HSet(ctx, m.entryKey(parent), name, m.packEntry(_type, *inode))\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tfield := (*inode).String()\n\t\t\t\tpipe.HSet(ctx, m.dirUsedInodesKey(), field, \"0\")\n\t\t\t\tpipe.HSet(ctx, m.dirDataLengthKey(), field, \"0\")\n\t\t\t\tpipe.HSet(ctx, m.dirUsedSpaceKey(), field, \"0\")\n\t\t\t}\n\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), align4K(0))\n\t\t\tpipe.Incr(ctx, m.totalInodesKey())\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(parent), m.entryKey(parent)))\n}\n\nfunc (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno {\n\tvar trash, inode Ino\n\tif !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\tif trash == 0 {\n\t\tdefer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }()\n\t}\n\tif attr == nil {\n\t\tattr = &Attr{}\n\t}\n\tvar _type uint8\n\tvar opened bool\n\tvar newSpace, newInode int64\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\topened = false\n\t\t*attr = Attr{}\n\t\tnewSpace, newInode = 0, 0\n\t\tbuf, err := tx.HGet(ctx, m.entryKey(parent), name).Bytes()\n\t\tif err == redis.Nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parent, name); e != nil {\n\t\t\t\tname = string(e.Name)\n\t\t\t\tbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t\terr = nil\n\t\t\t}\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_type, inode = m.parseEntry(buf)\n\t\tif _type == TypeDirectory {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif err := tx.Watch(ctx, m.inodeKey(inode)).Err(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\trs, err := tx.MGet(ctx, m.inodeKey(parent), m.inodeKey(inode)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif rs[0] == nil {\n\t\t\treturn redis.Nil\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr([]byte(rs[0].(string)), &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar updateParent bool\n\t\tnow := time.Now()\n\t\tif !parent.IsTrash() && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime {\n\t\t\tpattr.Mtime = now.Unix()\n\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tpattr.Ctime = now.Unix()\n\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tupdateParent = true\n\t\t}\n\t\tif rs[1] != nil {\n\t\t\tm.parseAttr([]byte(rs[1].(string)), attr)\n\t\t\tif ctx.Uid() != 0 && pattr.Mode&01000 != 0 && ctx.Uid() != pattr.Uid && ctx.Uid() != attr.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t\tif (attr.Flags&FlagAppend) != 0 || (attr.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif (attr.Flags&FlagSkipTrash) != 0 && trash > 0 {\n\t\t\t\ttrash = 0\n\t\t\t\tdefer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }()\n\t\t\t}\n\t\t\tif trash > 0 && attr.Nlink > 1 && tx.HExists(ctx, m.entryKey(trash), m.trashEntry(parent, inode, name)).Val() {\n\t\t\t\ttrash = 0\n\t\t\t\tdefer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }()\n\t\t\t}\n\t\t\tattr.Ctime = now.Unix()\n\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tif trash == 0 {\n\t\t\t\tattr.Nlink--\n\t\t\t\tif _type == TypeFile && attr.Nlink == 0 && m.sid > 0 {\n\t\t\t\t\topened = m.of.IsOpen(inode)\n\t\t\t\t}\n\t\t\t} else if attr.Parent > 0 {\n\t\t\t\tattr.Parent = trash\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", inode, parent, name)\n\t\t\ttrash = 0\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.HDel(ctx, m.entryKey(parent), name)\n\t\t\tif updateParent {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)\n\t\t\t}\n\t\t\tif attr.Nlink > 0 {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\t\t\t\tif trash > 0 {\n\t\t\t\t\tpipe.HSet(ctx, m.entryKey(trash), m.trashEntry(parent, inode, name), buf)\n\t\t\t\t\tif attr.Parent == 0 {\n\t\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(inode), trash.String(), 1)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif attr.Parent == 0 {\n\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(inode), parent.String(), -1)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tswitch _type {\n\t\t\t\tcase TypeFile:\n\t\t\t\t\tif opened {\n\t\t\t\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\t\t\t\t\t\tpipe.SAdd(ctx, m.sustained(m.sid), strconv.Itoa(int(inode)))\n\t\t\t\t\t} else {\n\t\t\t\t\t\tpipe.ZAdd(ctx, m.delfiles(), redis.Z{Score: float64(now.Unix()), Member: m.toDelete(inode, attr.Length)})\n\t\t\t\t\t\tpipe.Del(ctx, m.inodeKey(inode))\n\t\t\t\t\t\tnewSpace, newInode = -align4K(attr.Length), -1\n\t\t\t\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), newSpace)\n\t\t\t\t\t\tpipe.Decr(ctx, m.totalInodesKey())\n\t\t\t\t\t}\n\t\t\t\tcase TypeSymlink:\n\t\t\t\t\tpipe.Del(ctx, m.symKey(inode))\n\t\t\t\t\tfallthrough\n\t\t\t\tdefault:\n\t\t\t\t\tpipe.Del(ctx, m.inodeKey(inode))\n\t\t\t\t\tnewSpace, newInode = -align4K(0), -1\n\t\t\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), newSpace)\n\t\t\t\t\tpipe.Decr(ctx, m.totalInodesKey())\n\t\t\t\t}\n\t\t\t\tpipe.Del(ctx, m.xattrKey(inode))\n\t\t\t\tif attr.Parent == 0 {\n\t\t\t\t\tpipe.Del(ctx, m.parentKey(inode))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\treturn err\n\t}, m.inodeKey(parent), m.entryKey(parent))\n\tif err == nil && trash == 0 {\n\t\tif _type == TypeFile && attr.Nlink == 0 {\n\t\t\tm.fileDeleted(opened, parent.IsTrash(), inode, attr.Length)\n\t\t}\n\t\tm.updateStats(newSpace, newInode)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, newSpace, newInode)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *redisMeta) doBatchUnlink(ctx Context, parent Ino, entries []*Entry, delta *dirStat, skipCheckTrash ...bool) syscall.Errno {\n\tif len(entries) == 0 {\n\t\treturn 0\n\t}\n\tvar trash Ino\n\tif len(skipCheckTrash) == 0 || !skipCheckTrash[0] {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\n\ttype entryInfo struct {\n\t\tname string\n\t\tinode Ino\n\t\ttyp uint8\n\t\ttrash Ino\n\t\tattr *Attr\n\t\ttrashName string\n\t\tbuf []byte\n\t}\n\ttype dNode struct {\n\t\topened bool\n\t\tlength uint64\n\t}\n\n\t// Each entry averages ~4 tx operations, so batch size should be 1000/4\n\tbatchSize := 1000 / 4\n\tfor len(entries) > 0 {\n\t\tif batchSize > len(entries) {\n\t\t\tbatchSize = len(entries)\n\t\t}\n\t\tbatch := entries[:batchSize]\n\t\tentries = entries[batchSize:]\n\n\t\tvar entryInfos []*entryInfo\n\t\tvar batchDirLength, batchDirSpace, batchDirInodes int64\n\t\tvar batchFsSpace, batchFsInodes int64\n\t\tvar deltas ugQuotaDeltas\n\t\tvar delNodes map[Ino]*dNode\n\t\twatchKeys := []string{m.inodeKey(parent), m.entryKey(parent)}\n\n\t\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\t\tbatchDirLength, batchDirSpace, batchDirInodes = 0, 0, 0\n\t\t\tbatchFsSpace, batchFsInodes = 0, 0\n\t\t\tdeltas = make(ugQuotaDeltas)\n\t\t\tdelNodes = make(map[Ino]*dNode)\n\n\t\t\trs, err := tx.Get(ctx, m.inodeKey(parent)).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvar pattr Attr\n\t\t\tm.parseAttr([]byte(rs), &pattr)\n\t\t\tif pattr.Typ != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t\tif (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\n\t\t\tentryKey := m.entryKey(parent)\n\t\t\tentryInfos = make([]*entryInfo, 0, len(batch))\n\t\t\tnow := time.Now()\n\t\t\tenames := make([]string, 0, len(batch))\n\t\t\tfor _, entry := range batch {\n\t\t\t\tenames = append(enames, string(entry.Name))\n\t\t\t}\n\t\t\tvals, err := tx.HMGet(ctx, entryKey, enames...).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor idx, entry := range batch {\n\t\t\t\tval := vals[idx]\n\t\t\t\tif val == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tbuf := []byte(val.(string))\n\t\t\t\ttyp, ino := m.parseEntry(buf)\n\t\t\t\tif entry.Inode != ino || typ == TypeDirectory || (entry.Attr != nil && entry.Attr.Typ != typ) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tentryInfos = append(entryInfos, &entryInfo{\n\t\t\t\t\tname: string(entry.Name),\n\t\t\t\t\tinode: ino,\n\t\t\t\t\ttyp: typ,\n\t\t\t\t\ttrash: trash,\n\t\t\t\t\tbuf: buf,\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tinodesSet := make(map[Ino]struct{}, len(entryInfos))\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif _, ok := inodesSet[info.inode]; !ok {\n\t\t\t\t\tinodesSet[info.inode] = struct{}{}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// load inode attrs for all distinct inodes\n\t\t\tif len(inodesSet) > 0 {\n\t\t\t\tinodesList := make([]Ino, 0, len(inodesSet))\n\t\t\t\tkeys := make([]string, 0, len(inodesSet))\n\t\t\t\tfor ino := range inodesSet {\n\t\t\t\t\tinodesList = append(inodesList, ino)\n\t\t\t\t\tkeys = append(keys, m.inodeKey(ino))\n\t\t\t\t}\n\t\t\t\tif err := tx.Watch(ctx, keys...).Err(); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\trs, err := tx.MGet(ctx, keys...).Result()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tnodeMap := make(map[Ino]*Attr, len(inodesList))\n\t\t\t\tfor i, v := range rs {\n\t\t\t\t\tif v == nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tvar a Attr\n\t\t\t\t\tm.parseAttr([]byte(v.(string)), &a)\n\t\t\t\t\tnodeMap[inodesList[i]] = &a\n\t\t\t\t}\n\n\t\t\t\t// iterate all target entries, apply basic checks and build info\n\t\t\t\tfor _, info := range entryInfos {\n\t\t\t\t\tattr, ok := nodeMap[info.inode]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t\tinfo.attr = nil\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif ctx.Uid() != 0 && pattr.Mode&01000 != 0 && ctx.Uid() != pattr.Uid && ctx.Uid() != attr.Uid {\n\t\t\t\t\t\treturn syscall.EACCES\n\t\t\t\t\t}\n\t\t\t\t\tif (attr.Flags&FlagAppend) != 0 || (attr.Flags&FlagImmutable) != 0 {\n\t\t\t\t\t\treturn syscall.EPERM\n\t\t\t\t\t}\n\t\t\t\t\tif (attr.Flags & FlagSkipTrash) != 0 {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t}\n\t\t\t\t\tinfo.attr = attr\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// check trash entries for hard links\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif info.attr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif info.trash > 0 && info.attr.Nlink > 1 {\n\t\t\t\t\tinfo.trashName = m.trashEntry(parent, info.inode, info.name)\n\t\t\t\t\texists, err := tx.HExists(ctx, m.entryKey(info.trash), info.trashName).Result()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif exists {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// update ctime\n\t\t\t\tinfo.attr.Ctime = now.Unix()\n\t\t\t\tinfo.attr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tif info.trash > 0 && info.attr.Parent > 0 {\n\t\t\t\t\tinfo.attr.Parent = info.trash\n\t\t\t\t}\n\t\t\t\tif info.trash == 0 && info.attr.Nlink > 0 {\n\t\t\t\t\tinfo.attr.Nlink--\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// check opened status for all inodes with Nlink == 0 after all decrements\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif info.attr != nil && info.trash == 0 && info.attr.Nlink == 0 && info.typ == TypeFile {\n\t\t\t\t\topened := false\n\t\t\t\t\tif m.sid > 0 {\n\t\t\t\t\t\topened = m.of.IsOpen(info.inode)\n\t\t\t\t\t}\n\t\t\t\t\tdelNodes[info.inode] = &dNode{opened, info.attr.Length}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar updateParent bool\n\t\t\tif !parent.IsTrash() && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime {\n\t\t\t\tpattr.Mtime = now.Unix()\n\t\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\t\tpattr.Ctime = now.Unix()\n\t\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tupdateParent = true\n\t\t\t}\n\n\t\t\tnowUnix := now.Unix()\n\t\t\tvisited := make(map[Ino]bool)\n\t\t\tvisited[0] = true // skip dummyNode\n\n\t\t\t// collect data for batch operations\n\t\t\tvar names []string\n\t\t\tvar keys []string\n\t\t\tvar sustained []interface{}\n\t\t\tvar delfiles []redis.Z\n\t\t\tvar inodes map[Ino]*Attr\n\t\t\tparentOps := make(map[string]map[string]int64) // key -> field -> incr\n\t\t\ttrashOps := make(map[string]map[string]interface{}) // key -> field -> value\n\t\t\tstats := make(map[string]int64) // key -> delta\n\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tnames = append(names, info.name)\n\t\t\t\tif info.attr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif info.typ == TypeFile {\n\t\t\t\t\tbatchDirLength -= int64(info.attr.Length)\n\t\t\t\t\tbatchDirSpace -= align4K(info.attr.Length)\n\t\t\t\t} else {\n\t\t\t\t\tbatchDirSpace -= align4K(0)\n\t\t\t\t}\n\t\t\t\tbatchDirInodes--\n\n\t\t\t\tif !visited[info.inode] {\n\t\t\t\t\tif info.attr.Nlink > 0 {\n\t\t\t\t\t\tif inodes == nil {\n\t\t\t\t\t\t\tinodes = make(map[Ino]*Attr)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tinodes[info.inode] = info.attr\n\t\t\t\t\t} else {\n\t\t\t\t\t\tswitch info.typ {\n\t\t\t\t\t\tcase TypeFile:\n\t\t\t\t\t\t\tif dnode, ok := delNodes[info.inode]; ok && dnode.opened {\n\t\t\t\t\t\t\t\tif inodes == nil {\n\t\t\t\t\t\t\t\t\tinodes = make(map[Ino]*Attr)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tinodes[info.inode] = info.attr\n\t\t\t\t\t\t\t\tsustained = append(sustained, strconv.Itoa(int(info.inode)))\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tdelfiles = append(delfiles, redis.Z{\n\t\t\t\t\t\t\t\t\tScore: float64(nowUnix),\n\t\t\t\t\t\t\t\t\tMember: m.toDelete(info.inode, info.attr.Length),\n\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t\tkeys = append(keys, m.inodeKey(info.inode))\n\t\t\t\t\t\t\t\tbatchFsSpace -= align4K(info.attr.Length)\n\t\t\t\t\t\t\t\tbatchFsInodes--\n\t\t\t\t\t\t\t\tstats[m.usedSpaceKey()] -= align4K(info.attr.Length)\n\t\t\t\t\t\t\t\tstats[m.totalInodesKey()]--\n\t\t\t\t\t\t\t\tdeltas.add(&ugQuotaDelta{\n\t\t\t\t\t\t\t\t\tUid: info.attr.Uid,\n\t\t\t\t\t\t\t\t\tGid: info.attr.Gid,\n\t\t\t\t\t\t\t\t\tSpace: -align4K(info.attr.Length),\n\t\t\t\t\t\t\t\t\tInodes: -1,\n\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase TypeSymlink:\n\t\t\t\t\t\t\tkeys = append(keys, m.symKey(info.inode))\n\t\t\t\t\t\t\tfallthrough\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tkeys = append(keys, m.inodeKey(info.inode))\n\t\t\t\t\t\t\tbatchFsSpace -= align4K(0)\n\t\t\t\t\t\t\tbatchFsInodes--\n\t\t\t\t\t\t\tstats[m.usedSpaceKey()] -= align4K(0)\n\t\t\t\t\t\t\tstats[m.totalInodesKey()]--\n\t\t\t\t\t\t\tdeltas.add(&ugQuotaDelta{\n\t\t\t\t\t\t\t\tUid: info.attr.Uid,\n\t\t\t\t\t\t\t\tGid: info.attr.Gid,\n\t\t\t\t\t\t\t\tSpace: -align4K(0),\n\t\t\t\t\t\t\t\tInodes: -1,\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t}\n\t\t\t\t\t\tkeys = append(keys, m.xattrKey(info.inode))\n\t\t\t\t\t\tif info.attr.Parent == 0 {\n\t\t\t\t\t\t\tkeys = append(keys, m.parentKey(info.inode))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tm.of.InvalidateChunk(info.inode, invalidateAttrOnly)\n\t\t\t\t}\n\t\t\t\tif info.attr.Nlink > 0 && info.attr.Parent == 0 {\n\t\t\t\t\tkey := m.parentKey(info.inode)\n\t\t\t\t\tif parentOps[key] == nil {\n\t\t\t\t\t\tparentOps[key] = make(map[string]int64)\n\t\t\t\t\t}\n\t\t\t\t\tparentOps[key][parent.String()]--\n\t\t\t\t}\n\n\t\t\t\tif info.attr.Nlink > 0 && info.trash > 0 {\n\t\t\t\t\tif info.trashName == \"\" {\n\t\t\t\t\t\tinfo.trashName = m.trashEntry(parent, info.inode, info.name)\n\t\t\t\t\t}\n\t\t\t\t\tkey := m.entryKey(info.trash)\n\t\t\t\t\tif trashOps[key] == nil {\n\t\t\t\t\t\ttrashOps[key] = make(map[string]interface{})\n\t\t\t\t\t}\n\t\t\t\t\ttrashOps[key][info.trashName] = info.buf\n\t\t\t\t\tif info.attr.Parent == 0 {\n\t\t\t\t\t\tkey := m.parentKey(info.inode)\n\t\t\t\t\t\tif parentOps[key] == nil {\n\t\t\t\t\t\t\tparentOps[key] = make(map[string]int64)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tparentOps[key][info.trash.String()]++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tvisited[info.inode] = true\n\t\t\t}\n\n\t\t\t// execute batched operations using pipeline\n\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\tif len(names) > 0 {\n\t\t\t\t\tpipe.HDel(ctx, m.entryKey(parent), names...)\n\t\t\t\t}\n\t\t\t\tfor inode, attr := range inodes {\n\t\t\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\t\t\t\t}\n\t\t\t\tif len(sustained) > 0 {\n\t\t\t\t\tpipe.SAdd(ctx, m.sustained(m.sid), sustained...)\n\t\t\t\t}\n\t\t\t\tif len(delfiles) > 0 {\n\t\t\t\t\tpipe.ZAdd(ctx, m.delfiles(), delfiles...)\n\t\t\t\t}\n\t\t\t\tif len(keys) > 0 {\n\t\t\t\t\tpipe.Del(ctx, keys...)\n\t\t\t\t}\n\t\t\t\tfor key, delta := range stats {\n\t\t\t\t\tif delta != 0 {\n\t\t\t\t\t\tpipe.IncrBy(ctx, key, delta)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor key, fields := range parentOps {\n\t\t\t\t\tfor field, incr := range fields {\n\t\t\t\t\t\tif incr != 0 {\n\t\t\t\t\t\t\tpipe.HIncrBy(ctx, key, field, incr)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor key, fields := range trashOps {\n\t\t\t\t\tfor field, value := range fields {\n\t\t\t\t\t\tpipe.HSet(ctx, key, field, value)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif updateParent {\n\t\t\t\t\tpipe.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn err\n\t\t}, watchKeys...)\n\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\n\t\t// outside of transaction: trigger data deletion callbacks\n\t\tfor inode, info := range delNodes {\n\t\t\tm.fileDeleted(info.opened, parent.IsTrash(), inode, info.length)\n\t\t}\n\n\t\tdelta.length += batchDirLength\n\t\tdelta.space += batchDirSpace\n\t\tdelta.inodes += batchDirInodes\n\t\tm.updateStats(batchFsSpace, batchFsInodes)\n\t\tfor _, q := range deltas {\n\t\t\tm.updateUserGroupStat(ctx, q.Uid, q.Gid, q.Space, q.Inodes)\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *redisMeta) doRmdir(ctx Context, parent Ino, name string, pinode *Ino, oldAttr *Attr, skipCheckTrash ...bool) syscall.Errno {\n\tvar trash Ino\n\tif !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\tvar attr Attr\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\tbuf, err := tx.HGet(ctx, m.entryKey(parent), name).Bytes()\n\t\tif err == redis.Nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parent, name); e != nil {\n\t\t\t\tname = string(e.Name)\n\t\t\t\tbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t\terr = nil\n\t\t\t}\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttyp, inode := m.parseEntry(buf)\n\t\tif typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pinode != nil {\n\t\t\t*pinode = inode\n\t\t}\n\t\tif err = tx.Watch(ctx, m.inodeKey(inode), m.entryKey(inode)).Err(); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\trs, err := tx.MGet(ctx, m.inodeKey(parent), m.inodeKey(inode)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif rs[0] == nil {\n\t\t\treturn redis.Nil\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr([]byte(rs[0].(string)), &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tnow := time.Now()\n\t\tpattr.Nlink--\n\t\tpattr.Mtime = now.Unix()\n\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\tpattr.Ctime = now.Unix()\n\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\n\t\tcnt, err := tx.HLen(ctx, m.entryKey(inode)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif cnt > 0 {\n\t\t\treturn syscall.ENOTEMPTY\n\t\t}\n\t\tif rs[1] != nil {\n\t\t\tm.parseAttr([]byte(rs[1].(string)), &attr)\n\t\t\tif oldAttr != nil {\n\t\t\t\t*oldAttr = attr\n\t\t\t}\n\t\t\tif ctx.Uid() != 0 && pattr.Mode&01000 != 0 && ctx.Uid() != pattr.Uid && ctx.Uid() != attr.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t\tif (attr.Flags & FlagSkipTrash) != 0 {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tif trash > 0 {\n\t\t\t\tattr.Ctime = now.Unix()\n\t\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tattr.Parent = trash\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", inode, parent, name)\n\t\t\ttrash = 0\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.HDel(ctx, m.entryKey(parent), name)\n\t\t\tif !parent.IsTrash() {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)\n\t\t\t}\n\t\t\tif trash > 0 {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(&attr), 0)\n\t\t\t\tpipe.HSet(ctx, m.entryKey(trash), m.trashEntry(parent, inode, name), buf)\n\t\t\t} else {\n\t\t\t\tpipe.Del(ctx, m.inodeKey(inode))\n\t\t\t\tpipe.Del(ctx, m.xattrKey(inode))\n\t\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), -align4K(0))\n\t\t\t\tpipe.Decr(ctx, m.totalInodesKey())\n\t\t\t}\n\n\t\t\tfield := inode.String()\n\t\t\tpipe.HDel(ctx, m.dirDataLengthKey(), field)\n\t\t\tpipe.HDel(ctx, m.dirUsedSpaceKey(), field)\n\t\t\tpipe.HDel(ctx, m.dirUsedInodesKey(), field)\n\t\t\tpipe.HDel(ctx, m.dirQuotaKey(), field)\n\t\t\tpipe.HDel(ctx, m.dirQuotaUsedSpaceKey(), field)\n\t\t\tpipe.HDel(ctx, m.dirQuotaUsedInodesKey(), field)\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(parent), m.entryKey(parent))\n\tif err == nil && trash == 0 {\n\t\tm.updateStats(-align4K(0), -1)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, -align4K(0), -1)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *redisMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode, tInode *Ino, attr, tAttr *Attr) syscall.Errno {\n\texchange := flags == RenameExchange\n\tvar opened bool\n\tvar trash, dino Ino\n\tvar dtyp uint8\n\tvar tattr Attr\n\tvar newSpace, newInode int64\n\tkeys := []string{m.inodeKey(parentSrc), m.entryKey(parentSrc), m.inodeKey(parentDst), m.entryKey(parentDst)}\n\tif parentSrc.IsTrash() {\n\t\t// lock the parentDst\n\t\tkeys[0], keys[2] = keys[2], keys[0]\n\t}\n\tif !exchange {\n\t\tif st := m.checkTrash(parentDst, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\topened = false\n\t\tdino, dtyp = 0, 0\n\t\ttattr = Attr{}\n\t\tnewSpace, newInode = 0, 0\n\t\tbuf, err := tx.HGet(ctx, m.entryKey(parentSrc), nameSrc).Bytes()\n\t\tif err == redis.Nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parentSrc, nameSrc); e != nil {\n\t\t\t\tnameSrc = string(e.Name)\n\t\t\t\tbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t\terr = nil\n\t\t\t}\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttyp, ino := m.parseEntry(buf)\n\t\tif parentSrc == parentDst && nameSrc == nameDst {\n\t\t\tif inode != nil {\n\t\t\t\t*inode = ino\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tkeys := []string{m.inodeKey(ino)}\n\n\t\tdbuf, err := tx.HGet(ctx, m.entryKey(parentDst), nameDst).Bytes()\n\t\tif err == redis.Nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parentDst, nameDst); e != nil {\n\t\t\t\tif (nameSrc != string(e.Name)) || parentDst != parentSrc {\n\t\t\t\t\tnameDst = string(e.Name)\n\t\t\t\t\tdbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t\t\terr = nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif err != nil && err != redis.Nil {\n\t\t\treturn err\n\t\t}\n\t\tif err == nil {\n\t\t\tif flags&RenameNoReplace != 0 {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\tdtyp, dino = m.parseEntry(dbuf)\n\t\t\tkeys = append(keys, m.inodeKey(dino))\n\t\t\tif dtyp == TypeDirectory {\n\t\t\t\tkeys = append(keys, m.entryKey(dino))\n\t\t\t}\n\t\t\tif !exchange {\n\t\t\t\tif st := m.checkTrash(parentDst, &trash); st != 0 {\n\t\t\t\t\treturn st\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif err := tx.Watch(ctx, keys...).Err(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif dino > 0 {\n\t\t\tif ino == dino {\n\t\t\t\treturn errno(nil)\n\t\t\t}\n\t\t\tif exchange {\n\t\t\t} else if typ == TypeDirectory && dtyp != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t} else if typ != TypeDirectory && dtyp == TypeDirectory {\n\t\t\t\treturn syscall.EISDIR\n\t\t\t}\n\t\t}\n\n\t\tkeys = []string{m.inodeKey(parentSrc), m.inodeKey(parentDst), m.inodeKey(ino)}\n\t\tif dino > 0 {\n\t\t\tkeys = append(keys, m.inodeKey(dino))\n\t\t}\n\t\trs, err := tx.MGet(ctx, keys...).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif rs[0] == nil || rs[1] == nil || rs[2] == nil {\n\t\t\treturn redis.Nil\n\t\t}\n\t\tvar sattr, dattr, iattr Attr\n\t\tm.parseAttr([]byte(rs[0].(string)), &sattr)\n\t\tif sattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif st := m.Access(ctx, parentSrc, MODE_MASK_W|MODE_MASK_X, &sattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tm.parseAttr([]byte(rs[1].(string)), &dattr)\n\t\tif dattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif flags&RenameRestore == 0 && dattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parentDst, MODE_MASK_W|MODE_MASK_X, &dattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\t// TODO: check parentDst is a subdir of source node\n\t\tif ino == parentDst || ino == dattr.Parent {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tm.parseAttr([]byte(rs[2].(string)), &iattr)\n\t\tif (sattr.Flags&FlagAppend) != 0 || (sattr.Flags&FlagImmutable) != 0 || (dattr.Flags&FlagImmutable) != 0 || (iattr.Flags&FlagAppend) != 0 || (iattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif parentSrc != parentDst && sattr.Mode&0o1000 != 0 && ctx.Uid() != 0 &&\n\t\t\tctx.Uid() != iattr.Uid && (ctx.Uid() != sattr.Uid || iattr.Typ == TypeDirectory) {\n\t\t\treturn syscall.EACCES\n\t\t}\n\n\t\tvar supdate, dupdate bool\n\t\tnow := time.Now()\n\t\tif dino > 0 {\n\t\t\tif rs[3] == nil {\n\t\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", dino, parentDst, nameDst)\n\t\t\t\ttrash = 0\n\t\t\t} else {\n\t\t\t\tm.parseAttr([]byte(rs[3].(string)), &tattr)\n\t\t\t}\n\t\t\tif (tattr.Flags&FlagAppend) != 0 || (tattr.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif (tattr.Flags & FlagSkipTrash) != 0 {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\ttattr.Ctime = now.Unix()\n\t\t\ttattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tif exchange {\n\t\t\t\tif parentSrc != parentDst {\n\t\t\t\t\tif dtyp == TypeDirectory {\n\t\t\t\t\t\ttattr.Parent = parentSrc\n\t\t\t\t\t\tdattr.Nlink--\n\t\t\t\t\t\tsattr.Nlink++\n\t\t\t\t\t\tsupdate, dupdate = true, true\n\t\t\t\t\t} else if tattr.Parent > 0 {\n\t\t\t\t\t\ttattr.Parent = parentSrc\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif dtyp == TypeDirectory {\n\t\t\t\t\tcnt, err := tx.HLen(ctx, m.entryKey(dino)).Result()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif cnt != 0 {\n\t\t\t\t\t\treturn syscall.ENOTEMPTY\n\t\t\t\t\t}\n\t\t\t\t\tdattr.Nlink--\n\t\t\t\t\tdupdate = true\n\t\t\t\t\tif trash > 0 {\n\t\t\t\t\t\ttattr.Parent = trash\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif trash == 0 {\n\t\t\t\t\t\ttattr.Nlink--\n\t\t\t\t\t\tif dtyp == TypeFile && tattr.Nlink == 0 && m.sid > 0 {\n\t\t\t\t\t\t\topened = m.of.IsOpen(dino)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tdefer func() { m.of.InvalidateChunk(dino, invalidateAttrOnly) }()\n\t\t\t\t\t} else if tattr.Parent > 0 {\n\t\t\t\t\t\ttattr.Parent = trash\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif ctx.Uid() != 0 && dattr.Mode&01000 != 0 && ctx.Uid() != dattr.Uid && ctx.Uid() != tattr.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t} else {\n\t\t\tif exchange {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t}\n\t\tif ctx.Uid() != 0 && sattr.Mode&01000 != 0 && ctx.Uid() != sattr.Uid && ctx.Uid() != iattr.Uid {\n\t\t\treturn syscall.EACCES\n\t\t}\n\n\t\tif parentSrc != parentDst {\n\t\t\tif typ == TypeDirectory {\n\t\t\t\tiattr.Parent = parentDst\n\t\t\t\tsattr.Nlink--\n\t\t\t\tdattr.Nlink++\n\t\t\t\tsupdate, dupdate = true, true\n\t\t\t} else if iattr.Parent > 0 {\n\t\t\t\tiattr.Parent = parentDst\n\t\t\t}\n\t\t}\n\t\tif supdate || now.Sub(time.Unix(sattr.Mtime, int64(sattr.Mtimensec))) >= m.conf.SkipDirMtime {\n\t\t\tsattr.Mtime = now.Unix()\n\t\t\tsattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tsattr.Ctime = now.Unix()\n\t\t\tsattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tsupdate = true\n\t\t}\n\t\tif dupdate || now.Sub(time.Unix(dattr.Mtime, int64(dattr.Mtimensec))) >= m.conf.SkipDirMtime {\n\t\t\tdattr.Mtime = now.Unix()\n\t\t\tdattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tdattr.Ctime = now.Unix()\n\t\t\tdattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tdupdate = true\n\t\t}\n\t\tiattr.Ctime = now.Unix()\n\t\tiattr.Ctimensec = uint32(now.Nanosecond())\n\t\tif inode != nil {\n\t\t\t*inode = ino\n\t\t}\n\t\tif attr != nil {\n\t\t\t*attr = iattr\n\t\t}\n\t\tif dino > 0 {\n\t\t\t*tInode = dino\n\t\t\t*tAttr = tattr\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tif exchange { // dbuf, tattr are valid\n\t\t\t\tpipe.Set(ctx, m.inodeKey(dino), m.marshal(&tattr), 0)\n\t\t\t\tpipe.HSet(ctx, m.entryKey(parentSrc), nameSrc, dbuf)\n\t\t\t\tif parentSrc != parentDst && tattr.Parent == 0 {\n\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(dino), parentSrc.String(), 1)\n\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(dino), parentDst.String(), -1)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tpipe.HDel(ctx, m.entryKey(parentSrc), nameSrc)\n\t\t\t\tif dino > 0 {\n\t\t\t\t\tif trash > 0 {\n\t\t\t\t\t\tpipe.Set(ctx, m.inodeKey(dino), m.marshal(&tattr), 0)\n\t\t\t\t\t\tpipe.HSet(ctx, m.entryKey(trash), m.trashEntry(parentDst, dino, nameDst), dbuf)\n\t\t\t\t\t\tif tattr.Parent == 0 {\n\t\t\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(dino), trash.String(), 1)\n\t\t\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(dino), parentDst.String(), -1)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if dtyp != TypeDirectory && tattr.Nlink > 0 {\n\t\t\t\t\t\tpipe.Set(ctx, m.inodeKey(dino), m.marshal(&tattr), 0)\n\t\t\t\t\t\tif tattr.Parent == 0 {\n\t\t\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(dino), parentDst.String(), -1)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif dtyp == TypeFile {\n\t\t\t\t\t\t\tif opened {\n\t\t\t\t\t\t\t\tpipe.Set(ctx, m.inodeKey(dino), m.marshal(&tattr), 0)\n\t\t\t\t\t\t\t\tpipe.SAdd(ctx, m.sustained(m.sid), strconv.Itoa(int(dino)))\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tpipe.ZAdd(ctx, m.delfiles(), redis.Z{Score: float64(now.Unix()), Member: m.toDelete(dino, tattr.Length)})\n\t\t\t\t\t\t\t\tpipe.Del(ctx, m.inodeKey(dino))\n\t\t\t\t\t\t\t\tnewSpace, newInode = -align4K(tattr.Length), -1\n\t\t\t\t\t\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), newSpace)\n\t\t\t\t\t\t\t\tpipe.Decr(ctx, m.totalInodesKey())\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tif dtyp == TypeSymlink {\n\t\t\t\t\t\t\t\tpipe.Del(ctx, m.symKey(dino))\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tpipe.Del(ctx, m.inodeKey(dino))\n\t\t\t\t\t\t\tnewSpace, newInode = -align4K(0), -1\n\t\t\t\t\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), newSpace)\n\t\t\t\t\t\t\tpipe.Decr(ctx, m.totalInodesKey())\n\t\t\t\t\t\t}\n\t\t\t\t\t\tpipe.Del(ctx, m.xattrKey(dino))\n\t\t\t\t\t\tif tattr.Parent == 0 {\n\t\t\t\t\t\t\tpipe.Del(ctx, m.parentKey(dino))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif dtyp == TypeDirectory {\n\t\t\t\t\t\tfield := dino.String()\n\t\t\t\t\t\tpipe.HDel(ctx, m.dirQuotaKey(), field)\n\t\t\t\t\t\tpipe.HDel(ctx, m.dirQuotaUsedSpaceKey(), field)\n\t\t\t\t\t\tpipe.HDel(ctx, m.dirQuotaUsedInodesKey(), field)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif parentDst != parentSrc {\n\t\t\t\tif !parentSrc.IsTrash() && supdate {\n\t\t\t\t\tpipe.Set(ctx, m.inodeKey(parentSrc), m.marshal(&sattr), 0)\n\t\t\t\t}\n\t\t\t\tif iattr.Parent == 0 {\n\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(ino), parentDst.String(), 1)\n\t\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(ino), parentSrc.String(), -1)\n\t\t\t\t}\n\t\t\t}\n\t\t\tpipe.Set(ctx, m.inodeKey(ino), m.marshal(&iattr), 0)\n\t\t\tpipe.HSet(ctx, m.entryKey(parentDst), nameDst, buf)\n\t\t\tif dupdate {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(parentDst), m.marshal(&dattr), 0)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, keys...)\n\tif err == nil && !exchange && trash == 0 {\n\t\tif dino > 0 && dtyp == TypeFile && tattr.Nlink == 0 {\n\t\t\tm.fileDeleted(opened, false, dino, tattr.Length)\n\t\t}\n\t\tm.updateStats(newSpace, newInode)\n\t\tm.updateUserGroupStat(ctx, tattr.Uid, tattr.Gid, newSpace, newInode)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *redisMeta) doLink(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\trs, err := tx.MGet(ctx, m.inodeKey(parent), m.inodeKey(inode)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif rs[0] == nil || rs[1] == nil {\n\t\t\treturn redis.Nil\n\t\t}\n\t\tvar pattr, iattr Attr\n\t\tm.parseAttr([]byte(rs[0].(string)), &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif pattr.Flags&FlagImmutable != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar updateParent bool\n\t\tnow := time.Now()\n\t\tif now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime {\n\t\t\tpattr.Mtime = now.Unix()\n\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tpattr.Ctime = now.Unix()\n\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tupdateParent = true\n\t\t}\n\t\tm.parseAttr([]byte(rs[1].(string)), &iattr)\n\t\tif iattr.Typ == TypeDirectory {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif (iattr.Flags&FlagAppend) != 0 || (iattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\toldParent := iattr.Parent\n\t\tiattr.Parent = 0\n\t\tiattr.Ctime = now.Unix()\n\t\tiattr.Ctimensec = uint32(now.Nanosecond())\n\t\tiattr.Nlink++\n\n\t\terr = tx.HGet(ctx, m.entryKey(parent), name).Err()\n\t\tif err != nil && err != redis.Nil {\n\t\t\treturn err\n\t\t} else if err == nil {\n\t\t\treturn syscall.EEXIST\n\t\t} else if err == redis.Nil && m.conf.CaseInsensi && m.resolveCase(ctx, parent, name) != nil {\n\t\t\treturn syscall.EEXIST\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.HSet(ctx, m.entryKey(parent), name, m.packEntry(iattr.Typ, inode))\n\t\t\tif updateParent {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)\n\t\t\t}\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(&iattr), 0)\n\t\t\tif oldParent > 0 {\n\t\t\t\tpipe.HIncrBy(ctx, m.parentKey(inode), oldParent.String(), 1)\n\t\t\t}\n\t\t\tpipe.HIncrBy(ctx, m.parentKey(inode), parent.String(), 1)\n\t\t\treturn nil\n\t\t})\n\t\tif err == nil && attr != nil {\n\t\t\t*attr = iattr\n\t\t}\n\t\treturn err\n\t}, m.inodeKey(parent), m.entryKey(parent), m.inodeKey(inode)))\n}\n\nfunc (m *redisMeta) fillAttr(ctx Context, es []*Entry) error {\n\tif len(es) == 0 {\n\t\treturn nil\n\t}\n\tvar keys = make([]string, len(es))\n\tfor i, e := range es {\n\t\tkeys[i] = m.inodeKey(e.Inode)\n\t}\n\trs, err := m.rdb.MGet(ctx, keys...).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor j, re := range rs {\n\t\tif re != nil {\n\t\t\tif a, ok := re.(string); ok {\n\t\t\t\tm.parseAttr([]byte(a), es[j].Attr)\n\t\t\t\tm.of.Update(es[j].Inode, es[j].Attr)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry, limit int) syscall.Errno {\n\tvar stop = errors.New(\"stop\")\n\terr := m.hscan(ctx, m.entryKey(inode), func(keys []string) error {\n\t\tnewEntries := make([]Entry, len(keys)/2)\n\t\tnewAttrs := make([]Attr, len(keys)/2)\n\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\ttyp, ino := m.parseEntry([]byte(keys[i+1]))\n\t\t\tif keys[i] == \"\" {\n\t\t\t\tlogger.Errorf(\"Corrupt entry with empty name: inode %d parent %d\", ino, inode)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tent := &newEntries[i/2]\n\t\t\tent.Inode = ino\n\t\t\tent.Name = []byte(keys[i])\n\t\t\tent.Attr = &newAttrs[i/2]\n\t\t\tent.Attr.Typ = typ\n\t\t\t*entries = append(*entries, ent)\n\t\t\tif limit > 0 && len(*entries) >= limit {\n\t\t\t\treturn stop\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif errors.Is(err, stop) {\n\t\terr = nil\n\t}\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\n\tif plus != 0 && len(*entries) != 0 {\n\t\tbatchSize := 4096\n\t\tnEntries := len(*entries)\n\t\tif nEntries <= batchSize {\n\t\t\terr = m.fillAttr(ctx, *entries)\n\t\t} else {\n\t\t\tindexCh := make(chan []*Entry, 10)\n\t\t\tvar wg sync.WaitGroup\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tfor es := range indexCh {\n\t\t\t\t\t\te := m.fillAttr(ctx, es)\n\t\t\t\t\t\tif e != nil {\n\t\t\t\t\t\t\terr = e\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\t\t\tfor i := 0; i < nEntries; i += batchSize {\n\t\t\t\tif i+batchSize > nEntries {\n\t\t\t\t\tindexCh <- (*entries)[i:]\n\t\t\t\t} else {\n\t\t\t\t\tindexCh <- (*entries)[i : i+batchSize]\n\t\t\t\t}\n\t\t\t}\n\t\t\tclose(indexCh)\n\t\t\twg.Wait()\n\t\t}\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *redisMeta) doCleanStaleSession(sid uint64) error {\n\tvar fail bool\n\t// release locks\n\tvar ctx = Background()\n\tssid := strconv.FormatInt(int64(sid), 10)\n\tkey := m.lockedKey(sid)\n\tif inodes, err := m.rdb.SMembers(ctx, key).Result(); err == nil {\n\t\tfor _, k := range inodes {\n\t\t\towners, err := m.rdb.HKeys(ctx, k).Result()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"HKeys %s: %s\", k, err)\n\t\t\t\tfail = true\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvar fields []string\n\t\t\tfor _, o := range owners {\n\t\t\t\tif strings.Split(o, \"_\")[0] == ssid {\n\t\t\t\t\tfields = append(fields, o)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(fields) > 0 {\n\t\t\t\tif err = m.rdb.HDel(ctx, k, fields...).Err(); err != nil {\n\t\t\t\t\tlogger.Warnf(\"HDel %s %s: %s\", k, fields, err)\n\t\t\t\t\tfail = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err = m.rdb.SRem(ctx, key, k).Err(); err != nil {\n\t\t\t\tlogger.Warnf(\"SRem %s %s: %s\", key, k, err)\n\t\t\t\tfail = true\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlogger.Warnf(\"SMembers %s: %s\", key, err)\n\t\tfail = true\n\t}\n\n\tkey = m.sustained(sid)\n\tif inodes, err := m.rdb.SMembers(ctx, key).Result(); err == nil {\n\t\tfor _, sinode := range inodes {\n\t\t\tinode, _ := strconv.ParseUint(sinode, 10, 64)\n\t\t\tif err = m.doDeleteSustainedInode(sid, Ino(inode)); err != nil {\n\t\t\t\tlogger.Warnf(\"Delete sustained inode %d of sid %d: %s\", inode, sid, err)\n\t\t\t\tfail = true\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlogger.Warnf(\"SMembers %s: %s\", key, err)\n\t\tfail = true\n\t}\n\n\tif !fail {\n\t\tif err := m.rdb.HDel(ctx, m.sessionInfos(), ssid).Err(); err != nil {\n\t\t\tlogger.Warnf(\"HDel sessionInfos %s: %s\", ssid, err)\n\t\t\tfail = true\n\t\t}\n\t}\n\tif fail {\n\t\treturn fmt.Errorf(\"failed to clean up sid %d\", sid)\n\t} else {\n\t\tif n, err := m.rdb.ZRem(ctx, m.allSessions(), ssid).Result(); err != nil {\n\t\t\treturn err\n\t\t} else if n == 1 {\n\t\t\treturn nil\n\t\t}\n\t\treturn m.rdb.ZRem(ctx, legacySessions, ssid).Err()\n\t}\n}\n\nfunc (m *redisMeta) doFindStaleSessions(limit int) ([]uint64, error) {\n\tvals, err := m.rdb.ZRangeByScore(Background(), m.allSessions(), &redis.ZRangeBy{\n\t\tMin: \"-inf\",\n\t\tMax: strconv.FormatInt(time.Now().Unix(), 10),\n\t\tCount: int64(limit)}).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsids := make([]uint64, len(vals))\n\tfor i, v := range vals {\n\t\tsids[i], _ = strconv.ParseUint(v, 10, 64)\n\t}\n\tlimit -= len(sids)\n\tif limit <= 0 {\n\t\treturn sids, nil\n\t}\n\n\t// check clients with version before 1.0-beta3 as well\n\tvals, err = m.rdb.ZRangeByScore(Background(), legacySessions, &redis.ZRangeBy{\n\t\tMin: \"-inf\",\n\t\tMax: strconv.FormatInt(time.Now().Add(time.Minute*-5).Unix(), 10),\n\t\tCount: int64(limit)}).Result()\n\tif err != nil {\n\t\tlogger.Errorf(\"Scan stale legacy sessions: %s\", err)\n\t\treturn sids, nil\n\t}\n\tfor _, v := range vals {\n\t\tsid, _ := strconv.ParseUint(v, 10, 64)\n\t\tsids = append(sids, sid)\n\t}\n\treturn sids, nil\n}\n\nfunc (m *redisMeta) doRefreshSession() error {\n\tctx := Background()\n\tssid := strconv.FormatUint(m.sid, 10)\n\t// we have to check sessionInfo here because the operations are not within a transaction\n\tok, err := m.rdb.HExists(ctx, m.sessionInfos(), ssid).Result()\n\tif err == nil && !ok {\n\t\tlogger.Warnf(\"Session %d was stale and cleaned up, but now it comes back again\", m.sid)\n\t\terr = m.rdb.HSet(ctx, m.sessionInfos(), m.sid, m.newSessionInfo()).Err()\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn m.rdb.ZAdd(ctx, m.allSessions(), redis.Z{\n\t\tScore: float64(m.expireTime()),\n\t\tMember: ssid}).Err()\n}\n\nfunc (m *redisMeta) doDeleteSustainedInode(sid uint64, inode Ino) error {\n\tvar attr Attr\n\tvar ctx = Background()\n\tvar newSpace int64\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\tnewSpace = 0\n\t\ta, err := tx.Get(ctx, m.inodeKey(inode)).Bytes()\n\t\tif err == redis.Nil {\n\t\t\treturn nil\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, &attr)\n\t\tnewSpace = -align4K(attr.Length)\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.ZAdd(ctx, m.delfiles(), redis.Z{Score: float64(time.Now().Unix()), Member: m.toDelete(inode, attr.Length)})\n\t\t\tpipe.Del(ctx, m.inodeKey(inode))\n\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), newSpace)\n\t\t\tpipe.Decr(ctx, m.totalInodesKey())\n\t\t\tpipe.SRem(ctx, m.sustained(sid), strconv.Itoa(int(inode)))\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(inode))\n\tif err == nil && newSpace < 0 {\n\t\tm.updateStats(newSpace, -1)\n\t\tm.tryDeleteFileData(inode, attr.Length, false)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, newSpace, 0)\n\t}\n\treturn err\n}\n\nfunc (m *redisMeta) doRead(ctx Context, inode Ino, indx uint32) ([]*slice, syscall.Errno) {\n\tvals, err := m.rdb.LRange(ctx, m.chunkKey(inode, indx), 0, -1).Result()\n\tif err != nil {\n\t\treturn nil, errno(err)\n\t}\n\treturn readSlices(vals), 0\n}\n\nfunc (m *redisMeta) doList(ctx Context, inode Ino) ([]*slice, syscall.Errno) {\n\tvar attr Attr\n\terr := m.doGetAttr(ctx, inode, &attr)\n\tif err != 0 {\n\t\treturn nil, err\n\t}\n\tp := m.rdb.Pipeline()\n\tvar slices []*slice\n\tvar indx uint32\n\tfor uint64(indx)*ChunkSize < attr.Length {\n\t\tfor i := 0; uint64(indx)*ChunkSize < attr.Length && i < 1000; i++ {\n\t\t\t_ = p.LRange(ctx, m.chunkKey(inode, indx), 0, -1)\n\t\t\tindx++\n\t\t}\n\t\tcmds, err := p.Exec(ctx)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"list of inode %d: %s\", inode, err)\n\t\t\treturn nil, errno(err)\n\t\t}\n\t\tfor _, cmd := range cmds {\n\t\t\tval := cmd.(*redis.StringSliceCmd).Val()\n\t\t\tif len(val) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tss := readSlices(val)\n\t\t\tif ss == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tslices = append(slices, ss...)\n\t\t}\n\t}\n\n\treturn slices, 0\n}\n\nfunc (m *redisMeta) doWrite(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time, numSlices *int, delta *dirStat, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\t*delta = dirStat{}\n\t\t*attr = Attr{}\n\t\ta, err := tx.Get(ctx, m.inodeKey(inode)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, attr)\n\t\tif attr.Typ != TypeFile {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tnewleng := uint64(indx)*ChunkSize + uint64(off) + uint64(slice.Len)\n\t\tif newleng > attr.Length {\n\t\t\tdelta.length = int64(newleng - attr.Length)\n\t\t\tdelta.space = align4K(newleng) - align4K(attr.Length)\n\t\t\tattr.Length = newleng\n\t\t}\n\t\tif err := m.checkQuota(ctx, delta.space, 0, attr.Uid, attr.Gid, m.getParents(ctx, tx, inode, attr.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tnow := time.Now()\n\t\tattr.Mtime = mtime.Unix()\n\t\tattr.Mtimensec = uint32(mtime.Nanosecond())\n\t\tattr.Ctime = now.Unix()\n\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\n\t\tvar rpush *redis.IntCmd\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\trpush = pipe.RPush(ctx, m.chunkKey(inode, indx), marshalSlice(off, slice.Id, slice.Size, slice.Off, slice.Len))\n\t\t\t// most of chunk are used by single inode, so use that as the default (1 == not exists)\n\t\t\t// pipe.Incr(ctx, r.sliceKey(slice.ID, slice.Size))\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\t\t\tif delta.space > 0 {\n\t\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), delta.space)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err == nil {\n\t\t\t*numSlices = int(rpush.Val())\n\t\t}\n\t\treturn err\n\t}, m.inodeKey(inode)))\n}\n\nfunc (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno {\n\tdefer m.timeit(\"CopyFileRange\", time.Now())\n\tf := m.of.find(fout)\n\tif f != nil {\n\t\tf.Lock()\n\t\tdefer f.Unlock()\n\t}\n\tvar newLength, newSpace int64\n\tvar sattr, attr Attr\n\tdefer func() { m.of.InvalidateChunk(fout, invalidateAllChunks) }()\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\tnewLength, newSpace = 0, 0\n\t\trs, err := tx.MGet(ctx, m.inodeKey(fin), m.inodeKey(fout)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif rs[0] == nil || rs[1] == nil {\n\t\t\treturn redis.Nil\n\t\t}\n\t\tsattr = Attr{}\n\t\tm.parseAttr([]byte(rs[0].(string)), &sattr)\n\t\tif sattr.Typ != TypeFile {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif offIn >= sattr.Length {\n\t\t\tif copied != nil {\n\t\t\t\t*copied = 0\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tsize := size\n\t\tif offIn+size > sattr.Length {\n\t\t\tsize = sattr.Length - offIn\n\t\t}\n\t\tattr = Attr{}\n\t\tm.parseAttr([]byte(rs[1].(string)), &attr)\n\t\tif attr.Typ != TypeFile {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif (attr.Flags&FlagImmutable) != 0 || (attr.Flags&FlagAppend) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tnewleng := offOut + size\n\t\tif newleng > attr.Length {\n\t\t\tnewLength = int64(newleng - attr.Length)\n\t\t\tnewSpace = align4K(newleng) - align4K(attr.Length)\n\t\t\tattr.Length = newleng\n\t\t}\n\t\tif err := m.checkQuota(ctx, newSpace, 0, attr.Uid, attr.Gid, m.getParents(ctx, tx, fout, attr.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tnow := time.Now()\n\t\tattr.Mtime = now.Unix()\n\t\tattr.Mtimensec = uint32(now.Nanosecond())\n\t\tattr.Ctime = now.Unix()\n\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\tif outLength != nil {\n\t\t\t*outLength = attr.Length\n\t\t}\n\n\t\tvar vals [][]string\n\t\tfor i := offIn / ChunkSize; i <= (offIn+size)/ChunkSize; i++ {\n\t\t\tval, err := tx.LRange(ctx, m.chunkKey(fin, uint32(i)), 0, -1).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvals = append(vals, val)\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tcoff := offIn / ChunkSize * ChunkSize\n\t\t\tfor _, sv := range vals {\n\t\t\t\t// Add a zero chunk for hole\n\t\t\t\tss := readSlices(sv)\n\t\t\t\tif ss == nil {\n\t\t\t\t\treturn syscall.EIO\n\t\t\t\t}\n\t\t\t\tss = append([]*slice{{len: ChunkSize}}, ss...)\n\t\t\t\tcs := buildSlice(ss)\n\t\t\t\ttpos := coff\n\t\t\t\tfor _, s := range cs {\n\t\t\t\t\tpos := tpos\n\t\t\t\t\ttpos += uint64(s.Len)\n\t\t\t\t\tif pos < offIn+size && pos+uint64(s.Len) > offIn {\n\t\t\t\t\t\tif pos < offIn {\n\t\t\t\t\t\t\tdec := offIn - pos\n\t\t\t\t\t\t\ts.Off += uint32(dec)\n\t\t\t\t\t\t\tpos += dec\n\t\t\t\t\t\t\ts.Len -= uint32(dec)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif pos+uint64(s.Len) > offIn+size {\n\t\t\t\t\t\t\tdec := pos + uint64(s.Len) - (offIn + size)\n\t\t\t\t\t\t\ts.Len -= uint32(dec)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tdoff := pos - offIn + offOut\n\t\t\t\t\t\tindx := uint32(doff / ChunkSize)\n\t\t\t\t\t\tdpos := uint32(doff % ChunkSize)\n\t\t\t\t\t\tif dpos+s.Len > ChunkSize {\n\t\t\t\t\t\t\tpipe.RPush(ctx, m.chunkKey(fout, indx), marshalSlice(dpos, s.Id, s.Size, s.Off, ChunkSize-dpos))\n\t\t\t\t\t\t\tif s.Id > 0 {\n\t\t\t\t\t\t\t\tpipe.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.Id, s.Size), 1)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tskip := ChunkSize - dpos\n\t\t\t\t\t\t\tpipe.RPush(ctx, m.chunkKey(fout, indx+1), marshalSlice(0, s.Id, s.Size, s.Off+skip, s.Len-skip))\n\t\t\t\t\t\t\tif s.Id > 0 {\n\t\t\t\t\t\t\t\tpipe.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.Id, s.Size), 1)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tpipe.RPush(ctx, m.chunkKey(fout, indx), marshalSlice(dpos, s.Id, s.Size, s.Off, s.Len))\n\t\t\t\t\t\t\tif s.Id > 0 {\n\t\t\t\t\t\t\t\tpipe.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.Id, s.Size), 1)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tcoff += ChunkSize\n\t\t\t}\n\t\t\tpipe.Set(ctx, m.inodeKey(fout), m.marshal(&attr), 0)\n\t\t\tif newSpace > 0 {\n\t\t\t\tpipe.IncrBy(ctx, m.usedSpaceKey(), newSpace)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err == nil {\n\t\t\tif copied != nil {\n\t\t\t\t*copied = size\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}, m.inodeKey(fout), m.inodeKey(fin))\n\tif err == nil {\n\t\tm.updateParentStat(ctx, fout, attr.Parent, newLength, newSpace)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, newSpace, 0)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *redisMeta) getParents(ctx Context, tx *redis.Tx, inode, parent Ino) []Ino {\n\tif parent > 0 {\n\t\treturn []Ino{parent}\n\t}\n\tvals, err := tx.HGetAll(ctx, m.parentKey(inode)).Result()\n\tif err != nil {\n\t\tlogger.Warnf(\"Scan parent key of inode %d: %s\", inode, err)\n\t\treturn nil\n\t}\n\tps := make([]Ino, 0, len(vals))\n\tfor k, v := range vals {\n\t\tif n, _ := strconv.Atoi(v); n > 0 {\n\t\t\tino, _ := strconv.ParseUint(k, 10, 64)\n\t\t\tps = append(ps, Ino(ino))\n\t\t}\n\t}\n\treturn ps\n}\n\nfunc (m *redisMeta) doGetParents(ctx Context, inode Ino) map[Ino]int {\n\tvals, err := m.rdb.HGetAll(ctx, m.parentKey(inode)).Result()\n\tif err != nil {\n\t\tlogger.Warnf(\"Scan parent key of inode %d: %s\", inode, err)\n\t\treturn nil\n\t}\n\tps := make(map[Ino]int)\n\tfor k, v := range vals {\n\t\tif n, _ := strconv.Atoi(v); n > 0 {\n\t\t\tino, _ := strconv.ParseUint(k, 10, 64)\n\t\t\tps[Ino(ino)] = n\n\t\t}\n\t}\n\treturn ps\n}\n\nfunc (m *redisMeta) doSyncDirStat(ctx Context, ino Ino) (*dirStat, syscall.Errno) {\n\tif m.conf.ReadOnly {\n\t\treturn nil, syscall.EROFS\n\t}\n\tfield := ino.String()\n\tstat, st := m.calcDirStat(ctx, ino)\n\tif st != 0 {\n\t\treturn nil, st\n\t}\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\tn, err := tx.Exists(ctx, m.inodeKey(ino)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif n <= 0 {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.HSet(ctx, m.dirDataLengthKey(), field, stat.length)\n\t\t\tpipe.HSet(ctx, m.dirUsedSpaceKey(), field, stat.space)\n\t\t\tpipe.HSet(ctx, m.dirUsedInodesKey(), field, stat.inodes)\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(ino))\n\treturn stat, errno(err)\n}\n\nfunc (m *redisMeta) doUpdateDirStat(ctx Context, batch map[Ino]dirStat) error {\n\tspaceKey := m.dirUsedSpaceKey()\n\tlengthKey := m.dirDataLengthKey()\n\tinodesKey := m.dirUsedInodesKey()\n\tnonexist := make(map[Ino]bool, 0)\n\tstatList := make([]Ino, 0, len(batch))\n\tpipeline := m.rdb.Pipeline()\n\tfor ino := range batch {\n\t\tpipeline.HExists(ctx, spaceKey, ino.String())\n\t\tstatList = append(statList, ino)\n\t}\n\trets, err := pipeline.Exec(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor i, ret := range rets {\n\t\tif ret.Err() != nil {\n\t\t\treturn ret.Err()\n\t\t}\n\t\tif exist, _ := ret.(*redis.BoolCmd).Result(); !exist {\n\t\t\tnonexist[statList[i]] = true\n\t\t}\n\t}\n\tif len(nonexist) > 0 {\n\t\twg := m.parallelSyncDirStat(ctx, nonexist)\n\t\tdefer wg.Wait()\n\t}\n\n\tfor _, group := range m.groupBatch(batch, 1000) {\n\t\t_, err := m.rdb.Pipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tfor _, ino := range group {\n\t\t\t\tfield := ino.String()\n\t\t\t\tif nonexist[ino] {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tstat := batch[ino]\n\t\t\t\tif stat.length != 0 {\n\t\t\t\t\tpipe.HIncrBy(ctx, lengthKey, field, stat.length)\n\t\t\t\t}\n\t\t\t\tif stat.space != 0 {\n\t\t\t\t\tpipe.HIncrBy(ctx, spaceKey, field, stat.space)\n\t\t\t\t}\n\t\t\t\tif stat.inodes != 0 {\n\t\t\t\t\tpipe.HIncrBy(ctx, inodesKey, field, stat.inodes)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) doGetDirStat(ctx Context, ino Ino, trySync bool) (*dirStat, syscall.Errno) {\n\tfield := ino.String()\n\tdataLength, errLength := m.rdb.HGet(ctx, m.dirDataLengthKey(), field).Int64()\n\tif errLength != nil && errLength != redis.Nil {\n\t\treturn nil, errno(errLength)\n\t}\n\tusedSpace, errSpace := m.rdb.HGet(ctx, m.dirUsedSpaceKey(), field).Int64()\n\tif errSpace != nil && errSpace != redis.Nil {\n\t\treturn nil, errno(errSpace)\n\t}\n\tusedInodes, errInodes := m.rdb.HGet(ctx, m.dirUsedInodesKey(), field).Int64()\n\tif errInodes != nil && errSpace != redis.Nil {\n\t\treturn nil, errno(errInodes)\n\t}\n\tif errLength != redis.Nil && errSpace != redis.Nil && errInodes != redis.Nil {\n\t\tif trySync && (dataLength < 0 || usedSpace < 0 || usedInodes < 0) {\n\t\t\treturn m.doSyncDirStat(ctx, ino)\n\t\t}\n\t\treturn &dirStat{dataLength, usedSpace, usedInodes}, 0\n\t}\n\n\tif trySync {\n\t\treturn m.doSyncDirStat(ctx, ino)\n\t}\n\treturn nil, 0\n}\n\n// For now only deleted files\nfunc (m *redisMeta) cleanupLegacies() {\n\tfor {\n\t\tutils.SleepWithJitter(time.Minute)\n\t\trng := &redis.ZRangeBy{Min: \"-inf\", Max: strconv.FormatInt(time.Now().Add(-time.Hour).Unix(), 10), Count: 1000}\n\t\tvals, err := m.rdb.ZRangeByScore(Background(), m.delfiles(), rng).Result()\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tvar count int\n\t\tfor _, v := range vals {\n\t\t\tps := strings.Split(v, \":\")\n\t\t\tif len(ps) != 2 {\n\t\t\t\tinode, _ := strconv.ParseUint(ps[0], 10, 64)\n\t\t\t\tvar length uint64 = 1 << 30\n\t\t\t\tif len(ps) > 2 {\n\t\t\t\t\tlength, _ = strconv.ParseUint(ps[2], 10, 64)\n\t\t\t\t}\n\t\t\t\tlogger.Infof(\"cleanup legacy delfile inode %d with %d bytes (%s)\", inode, length, v)\n\t\t\t\tm.doDeleteFileData_(Ino(inode), length, v)\n\t\t\t\tcount++\n\t\t\t}\n\t\t}\n\t\tif count == 0 {\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (m *redisMeta) doFindDeletedFiles(ts int64, limit int) (map[Ino]uint64, error) {\n\trng := &redis.ZRangeBy{Min: \"-inf\", Max: strconv.FormatInt(ts, 10), Count: int64(limit)}\n\tvals, err := m.rdb.ZRangeByScore(Background(), m.delfiles(), rng).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfiles := make(map[Ino]uint64, len(vals))\n\tfor _, v := range vals {\n\t\tps := strings.Split(v, \":\")\n\t\tif len(ps) != 2 { // will be cleaned up as legacy\n\t\t\tcontinue\n\t\t}\n\t\tinode, _ := strconv.ParseUint(ps[0], 10, 64)\n\t\tfiles[Ino(inode)], _ = strconv.ParseUint(ps[1], 10, 64)\n\t}\n\treturn files, nil\n}\n\nfunc (m *redisMeta) doCleanupSlices(ctx Context, count *uint64) error {\n\treturn m.hscan(ctx, m.sliceRefs(), func(keys []string) error {\n\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\tkey, val := keys[i], keys[i+1]\n\t\t\tif strings.HasPrefix(val, \"-\") { // < 0\n\t\t\t\tps := strings.Split(key, \"_\")\n\t\t\t\tif len(ps) == 2 {\n\t\t\t\t\tid, _ := strconv.ParseUint(ps[0][1:], 10, 64)\n\t\t\t\t\tsize, _ := strconv.ParseUint(ps[1], 10, 32)\n\t\t\t\t\tif id > 0 && size > 0 {\n\t\t\t\t\t\tm.deleteSlice(id, uint32(size))\n\t\t\t\t\t\tif count != nil {\n\t\t\t\t\t\t\t*count++\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if val == \"0\" {\n\t\t\t\tm.cleanupZeroRef(key)\n\t\t\t}\n\t\t\tif ctx.Canceled() {\n\t\t\t\treturn ctx.Err()\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *redisMeta) cleanupZeroRef(key string) {\n\tvar ctx = Background()\n\t_ = m.txn(ctx, func(tx *redis.Tx) error {\n\t\tv, err := tx.HGet(ctx, m.sliceRefs(), key).Int()\n\t\tif err != nil && err != redis.Nil {\n\t\t\treturn err\n\t\t}\n\t\tif v != 0 {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\t_, err = tx.TxPipelined(ctx, func(p redis.Pipeliner) error {\n\t\t\tp.HDel(ctx, m.sliceRefs(), key)\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.sliceRefs())\n}\n\nfunc (m *redisMeta) cleanupLeakedChunks(delete bool) {\n\tvar ctx = Background()\n\tprefix := len(m.prefix)\n\t_ = m.scan(ctx, \"c*\", func(ckeys []string) error {\n\t\tvar ikeys []string\n\t\tvar rs []*redis.IntCmd\n\t\tp := m.rdb.Pipeline()\n\t\tfor _, k := range ckeys {\n\t\t\tps := strings.Split(k, \"_\")\n\t\t\tif len(ps) != 2 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tino, _ := strconv.ParseUint(ps[0][prefix+1:], 10, 64)\n\t\t\tikeys = append(ikeys, k)\n\t\t\trs = append(rs, p.Exists(ctx, m.inodeKey(Ino(ino))))\n\t\t}\n\t\tif len(rs) > 0 {\n\t\t\tcmds, err := p.Exec(ctx)\n\t\t\tif err != nil {\n\t\t\t\tfor _, c := range cmds {\n\t\t\t\t\tif c.Err() != nil {\n\t\t\t\t\t\tlogger.Errorf(\"Check inodes with command %s: %s\", c.String(), c.Err())\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor i, rr := range rs {\n\t\t\t\tif rr.Val() == 0 {\n\t\t\t\t\tkey := ikeys[i]\n\t\t\t\t\tlogger.Infof(\"found leaked chunk %s\", key)\n\t\t\t\t\tif delete {\n\t\t\t\t\t\tps := strings.Split(key, \"_\")\n\t\t\t\t\t\tino, _ := strconv.ParseUint(ps[0][prefix+1:], 10, 64)\n\t\t\t\t\t\tindx, _ := strconv.Atoi(ps[1])\n\t\t\t\t\t\t_ = m.deleteChunk(Ino(ino), uint32(indx))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *redisMeta) cleanupOldSliceRefs(delete bool) {\n\tvar ctx = Background()\n\t_ = m.scan(ctx, \"k*\", func(ckeys []string) error {\n\t\tvalues, err := m.rdb.MGet(ctx, ckeys...).Result()\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"mget slices: %s\", err)\n\t\t\treturn err\n\t\t}\n\t\tvar todel []string\n\t\tfor i, v := range values {\n\t\t\tif v == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif strings.HasPrefix(v.(string), m.prefix+\"-\") || v == \"0\" { // < 0\n\t\t\t\t// the objects will be deleted by gc\n\t\t\t\ttodel = append(todel, ckeys[i])\n\t\t\t} else {\n\t\t\t\tvv, _ := strconv.Atoi(v.(string))\n\t\t\t\tm.rdb.HIncrBy(ctx, m.sliceRefs(), ckeys[i], int64(vv))\n\t\t\t\tm.rdb.DecrBy(ctx, ckeys[i], int64(vv))\n\t\t\t\tlogger.Infof(\"move refs %d for slice %s\", vv, ckeys[i])\n\t\t\t}\n\t\t}\n\t\tif delete && len(todel) > 0 {\n\t\t\tm.rdb.Del(ctx, todel...)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *redisMeta) toDelete(inode Ino, length uint64) string {\n\treturn inode.String() + \":\" + strconv.Itoa(int(length))\n}\n\nfunc (m *redisMeta) deleteChunk(inode Ino, indx uint32) error {\n\tvar ctx = Background()\n\tkey := m.chunkKey(inode, indx)\n\tvar todel []*slice\n\tvar rs []*redis.IntCmd\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\ttodel = todel[:0]\n\t\trs = rs[:0]\n\t\tvals, err := tx.LRange(ctx, key, 0, -1).Result()\n\t\tif err != nil || len(vals) == 0 {\n\t\t\treturn err\n\t\t}\n\t\tslices := readSlices(vals)\n\t\tif slices == nil {\n\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d, use `gc` to clean up leaked slices\", inode, indx)\n\t\t}\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Del(ctx, key)\n\t\t\tfor _, s := range slices {\n\t\t\t\tif s.id > 0 {\n\t\t\t\t\ttodel = append(todel, s)\n\t\t\t\t\trs = append(rs, pipe.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.id, s.size), -1))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, key)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"delete slice from chunk %s fail: %s, retry later\", key, err)\n\t}\n\tfor i, s := range todel {\n\t\tif rs[i].Val() < 0 {\n\t\t\tm.deleteSlice(s.id, s.size)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) doDeleteFileData(inode Ino, length uint64) {\n\tm.doDeleteFileData_(inode, length, \"\")\n}\n\nfunc (m *redisMeta) doDeleteFileData_(inode Ino, length uint64, tracking string) {\n\tvar ctx = Background()\n\tvar indx uint32\n\tp := m.rdb.Pipeline()\n\tfor uint64(indx)*ChunkSize < length {\n\t\tvar keys []string\n\t\tfor i := 0; uint64(indx)*ChunkSize < length && i < 1000; i++ {\n\t\t\tkey := m.chunkKey(inode, indx)\n\t\t\tkeys = append(keys, key)\n\t\t\t_ = p.LLen(ctx, key)\n\t\t\tindx++\n\t\t}\n\t\tcmds, err := p.Exec(ctx)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"delete chunks of inode %d: %s\", inode, err)\n\t\t\treturn\n\t\t}\n\t\tfor i, cmd := range cmds {\n\t\t\tval, err := cmd.(*redis.IntCmd).Result()\n\t\t\tif err == redis.Nil || val == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tidx, _ := strconv.Atoi(strings.Split(keys[i][len(m.prefix):], \"_\")[1])\n\t\t\terr = m.deleteChunk(inode, uint32(idx))\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"delete chunk %s: %s\", keys[i], err)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\tif tracking == \"\" {\n\t\ttracking = inode.String() + \":\" + strconv.FormatInt(int64(length), 10)\n\t}\n\t_ = m.rdb.ZRem(ctx, m.delfiles(), tracking)\n}\n\nfunc (r *redisMeta) doCleanupDelayedSlices(ctx Context, edge int64) (int, error) {\n\tvar count int\n\tvar ss []Slice\n\tvar rs []*redis.IntCmd\n\terr := r.hscan(ctx, r.delSlices(), func(keys []string) error {\n\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\tif ctx.Canceled() {\n\t\t\t\treturn ctx.Err()\n\t\t\t}\n\t\t\tkey := keys[i]\n\t\t\tps := strings.Split(key, \"_\")\n\t\t\tif len(ps) != 2 {\n\t\t\t\tlogger.Warnf(\"Invalid key %s\", key)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif ts, e := strconv.ParseUint(ps[1], 10, 64); e != nil {\n\t\t\t\tlogger.Warnf(\"Invalid key %s\", key)\n\t\t\t\tcontinue\n\t\t\t} else if ts >= uint64(edge) {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif err := r.txn(ctx, func(tx *redis.Tx) error {\n\t\t\t\tss, rs = ss[:0], rs[:0]\n\t\t\t\tval, e := tx.HGet(ctx, r.delSlices(), key).Result()\n\t\t\t\tif e == redis.Nil {\n\t\t\t\t\treturn nil\n\t\t\t\t} else if e != nil {\n\t\t\t\t\treturn e\n\t\t\t\t}\n\t\t\t\tbuf := []byte(val)\n\t\t\t\tr.decodeDelayedSlices(buf, &ss)\n\t\t\t\tif len(ss) == 0 {\n\t\t\t\t\treturn fmt.Errorf(\"invalid value for delSlices %s: %v\", key, buf)\n\t\t\t\t}\n\t\t\t\t_, e = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\t\tfor _, s := range ss {\n\t\t\t\t\t\trs = append(rs, pipe.HIncrBy(ctx, r.sliceRefs(), r.sliceKey(s.Id, s.Size), -1))\n\t\t\t\t\t}\n\t\t\t\t\tpipe.HDel(ctx, r.delSlices(), key)\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\treturn e\n\t\t\t}, r.delSlices()); err != nil {\n\t\t\t\tlogger.Warnf(\"Cleanup delSlices %s: %s\", key, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor i, s := range ss {\n\t\t\t\tif rs[i].Err() == nil && rs[i].Val() < 0 {\n\t\t\t\t\tr.deleteSlice(s.Id, s.Size)\n\t\t\t\t\tcount++\n\t\t\t\t}\n\t\t\t\tif ctx.Canceled() {\n\t\t\t\t\treturn ctx.Err()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\treturn count, err\n}\n\nfunc (m *redisMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno {\n\tvar rs []*redis.IntCmd // trash disabled: check reference of slices\n\tif delayed == nil {\n\t\trs = make([]*redis.IntCmd, len(ss))\n\t}\n\tkey := m.chunkKey(inode, indx)\n\tctx := Background()\n\tst := errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tn := len(origin) / sliceBytes\n\t\tvals2, err := tx.LRange(ctx, key, 0, int64(n-1)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(vals2) != n {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tfor i, val := range vals2 {\n\t\t\tif val != string(origin[i*sliceBytes:(i+1)*sliceBytes]) {\n\t\t\t\treturn syscall.EINVAL\n\t\t\t}\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.LTrim(ctx, key, int64(n), -1)\n\t\t\tpipe.LPush(ctx, key, marshalSlice(pos, id, size, 0, size))\n\t\t\tfor i := skipped; i > 0; i-- {\n\t\t\t\tpipe.LPush(ctx, key, origin[(i-1)*sliceBytes:i*sliceBytes])\n\t\t\t}\n\t\t\tpipe.HSet(ctx, m.sliceRefs(), m.sliceKey(id, size), \"0\") // create the key to tracking it\n\t\t\tif delayed != nil {\n\t\t\t\tif len(delayed) > 0 {\n\t\t\t\t\tpipe.HSet(ctx, m.delSlices(), fmt.Sprintf(\"%d_%d\", id, time.Now().Unix()), delayed)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\tif s.id > 0 {\n\t\t\t\t\t\trs[i] = pipe.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.id, s.size), -1)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, key))\n\t// there could be false-negative that the compaction is successful, double-check\n\tif st != 0 && st != syscall.EINVAL {\n\t\tif e := m.rdb.HGet(ctx, m.sliceRefs(), m.sliceKey(id, size)).Err(); e == nil {\n\t\t\tst = 0 // successful\n\t\t} else if e == redis.Nil {\n\t\t\tlogger.Infof(\"compacted chunk %d was not used\", id)\n\t\t\tst = syscall.EINVAL // failed\n\t\t}\n\t}\n\n\tif st == syscall.EINVAL {\n\t\tm.rdb.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(id, size), -1)\n\t} else if st == 0 {\n\t\tm.cleanupZeroRef(m.sliceKey(id, size))\n\t\tif delayed == nil {\n\t\t\tfor i, s := range ss {\n\t\t\t\tif s.id > 0 && rs[i].Err() == nil && rs[i].Val() < 0 {\n\t\t\t\t\tm.deleteSlice(s.id, s.size)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn st\n}\n\nfunc (m *redisMeta) scanAllChunks(ctx Context, ch chan<- cchunk, bar *utils.Bar) error {\n\tp := m.rdb.Pipeline()\n\treturn m.scan(ctx, \"c*_*\", func(keys []string) error {\n\t\tfor _, key := range keys {\n\t\t\t_ = p.LLen(ctx, key)\n\t\t}\n\t\tcmds, err := p.Exec(ctx)\n\t\tif err != nil {\n\t\t\tfor _, c := range cmds {\n\t\t\t\tif c.Err() != nil {\n\t\t\t\t\tlogger.Warnf(\"Scan chunks with command %s: %s\", c.String(), c.Err())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tfor i, cmd := range cmds {\n\t\t\tcnt := cmd.(*redis.IntCmd).Val()\n\t\t\tif cnt > 1 {\n\t\t\t\tvar inode uint64\n\t\t\t\tvar indx uint32\n\t\t\t\tn, err := fmt.Sscanf(keys[i], m.prefix+\"c%d_%d\", &inode, &indx)\n\t\t\t\tif err == nil && n == 2 {\n\t\t\t\t\tbar.IncrTotal(1)\n\t\t\t\t\tch <- cchunk{Ino(inode), indx, int(cnt)}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *redisMeta) cleanupLeakedInodes(delete bool) {\n\tvar ctx = Background()\n\tvar foundInodes = make(map[Ino]struct{})\n\tfoundInodes[RootInode] = struct{}{}\n\tfoundInodes[TrashInode] = struct{}{}\n\tcutoff := time.Now().Add(time.Hour * -1)\n\tprefix := len(m.prefix)\n\n\t_ = m.scan(ctx, \"d[0-9]*\", func(keys []string) error {\n\t\tfor _, key := range keys {\n\t\t\tino, _ := strconv.Atoi(key[prefix+1:])\n\t\t\tvar entries []*Entry\n\t\t\teno := m.doReaddir(ctx, Ino(ino), 0, &entries, 0)\n\t\t\tif eno != syscall.ENOENT && eno != 0 {\n\t\t\t\tlogger.Errorf(\"readdir %d: %s\", ino, eno)\n\t\t\t\treturn eno\n\t\t\t}\n\t\t\tfor _, e := range entries {\n\t\t\t\tfoundInodes[e.Inode] = struct{}{}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\t_ = m.scan(ctx, \"i*\", func(keys []string) error {\n\t\tvalues, err := m.rdb.MGet(ctx, keys...).Result()\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"mget inodes: %s\", err)\n\t\t\treturn nil\n\t\t}\n\t\tfor i, v := range values {\n\t\t\tif v == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvar attr Attr\n\t\t\tm.parseAttr([]byte(v.(string)), &attr)\n\t\t\tino, _ := strconv.Atoi(keys[i][prefix+1:])\n\t\t\tif _, ok := foundInodes[Ino(ino)]; !ok && time.Unix(attr.Ctime, 0).Before(cutoff) {\n\t\t\t\tlogger.Infof(\"found dangling inode: %s %+v\", keys[i], attr)\n\t\t\t\tif delete {\n\t\t\t\t\terr = m.doDeleteSustainedInode(0, Ino(ino))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlogger.Errorf(\"delete leaked inode %d : %s\", ino, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *redisMeta) scan(ctx context.Context, pattern string, f func([]string) error) error {\n\tvar rdb *redis.Client\n\tif c, ok := m.rdb.(*redis.ClusterClient); ok {\n\t\tvar err error\n\t\trdb, err = c.MasterForKey(ctx, m.prefix)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\trdb = m.rdb.(*redis.Client)\n\t}\n\tvar cursor uint64\n\tfor {\n\t\tkeys, c, err := rdb.Scan(ctx, cursor, m.prefix+pattern, 10000).Result()\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"scan %s: %s\", pattern, err)\n\t\t\treturn err\n\t\t}\n\t\tif len(keys) > 0 {\n\t\t\terr = f(keys)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif c == 0 {\n\t\t\tbreak\n\t\t}\n\t\tcursor = c\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) hscan(ctx context.Context, key string, f func([]string) error) error {\n\tvar cursor uint64\n\tfor {\n\t\tkeys, c, err := m.rdb.HScan(ctx, key, cursor, \"*\", 10000).Result()\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"HSCAN %s: %s\", key, err)\n\t\t\treturn err\n\t\t}\n\t\tif len(keys) > 0 {\n\t\t\tif err = f(keys); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif c == 0 {\n\t\t\tbreak\n\t\t}\n\t\tcursor = c\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno {\n\tlogger.Debugf(\"start cleanup...\")\n\tm.cleanupLeakedInodes(delete)\n\tm.cleanupLeakedChunks(delete)\n\tm.cleanupOldSliceRefs(delete)\n\tif delete {\n\t\t_ = m.doCleanupSlices(ctx, nil)\n\t}\n\tlogger.Debugf(\"start listing slices...\")\n\n\tp := m.rdb.Pipeline()\n\terr := m.scan(ctx, \"c*_*\", func(keys []string) error {\n\t\tfor _, key := range keys {\n\t\t\t_ = p.LRange(ctx, key, 0, -1)\n\t\t}\n\t\tcmds, err := p.Exec(ctx)\n\t\tif err != nil {\n\t\t\tfor _, c := range cmds {\n\t\t\t\tif c.Err() != nil {\n\t\t\t\t\tlogger.Warnf(\"List slices with command %s: %s\", c.String(), c.Err())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tfor _, cmd := range cmds {\n\t\t\tkey := cmd.(*redis.StringSliceCmd).Args()[1].(string)\n\t\t\tinode, _ := strconv.Atoi(strings.Split(key[len(m.prefix)+1:], \"_\")[0])\n\t\t\tvals := cmd.(*redis.StringSliceCmd).Val()\n\t\t\tss := readSlices(vals)\n\t\t\tif ss == nil {\n\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk key %s\", inode, key)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, s := range ss {\n\t\t\t\tif s.id > 0 {\n\t\t\t\t\tslices[Ino(inode)] = append(slices[Ino(inode)], Slice{Id: s.id, Size: s.size})\n\t\t\t\t\tif showProgress != nil {\n\t\t\t\t\t\tshowProgress()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"scan chunks: %s\", err)\n\t\treturn errno(err)\n\t}\n\n\tif scanPending {\n\t\t_ = m.hscan(Background(), m.sliceRefs(), func(keys []string) error {\n\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\tkey, val := keys[i], keys[i+1]\n\t\t\t\tif strings.HasPrefix(val, \"-\") { // < 0\n\t\t\t\t\tps := strings.Split(key, \"_\")\n\t\t\t\t\tif len(ps) == 2 {\n\t\t\t\t\t\tid, _ := strconv.ParseUint(ps[0][1:], 10, 64)\n\t\t\t\t\t\tsize, _ := strconv.ParseUint(ps[1], 10, 32)\n\t\t\t\t\t\tif id > 0 && size > 0 {\n\t\t\t\t\t\t\tslices[0] = append(slices[0], Slice{Id: id, Size: uint32(size)})\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tif m.getFormat().TrashDays == 0 {\n\t\treturn 0\n\t}\n\treturn errno(m.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) {\n\t\tslices[1] = append(slices[1], ss...)\n\t\tif showProgress != nil {\n\t\t\tfor range ss {\n\t\t\t\tshowProgress()\n\t\t\t}\n\t\t}\n\t\treturn false, nil\n\t}))\n}\n\nfunc (m *redisMeta) scanTrashSlices(ctx Context, scan trashSliceScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\n\tdelKeys := make(chan string, 1000)\n\tc, cancel := context.WithCancel(ctx)\n\tdefer cancel()\n\tgo func() {\n\t\t_ = m.hscan(c, m.delSlices(), func(keys []string) error {\n\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\tdelKeys <- keys[i]\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tclose(delKeys)\n\t}()\n\n\tvar ss []Slice\n\tvar rs []*redis.IntCmd\n\tfor key := range delKeys {\n\t\tvar clean bool\n\t\ttask := func(tx *redis.Tx) error {\n\t\t\tss = ss[:0]\n\t\t\trs = rs[:0]\n\t\t\tval, err := tx.HGet(ctx, m.delSlices(), key).Result()\n\t\t\tif err == redis.Nil {\n\t\t\t\treturn nil\n\t\t\t} else if err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tps := strings.Split(key, \"_\")\n\t\t\tif len(ps) != 2 {\n\t\t\t\treturn fmt.Errorf(\"invalid key %s\", key)\n\t\t\t}\n\t\t\tts, err := strconv.ParseInt(ps[1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"invalid key %s, fail to parse timestamp\", key)\n\t\t\t}\n\n\t\t\tm.decodeDelayedSlices([]byte(val), &ss)\n\t\t\tclean, err = scan(ss, ts)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif clean {\n\t\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\t\tfor _, s := range ss {\n\t\t\t\t\t\trs = append(rs, pipe.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.Id, s.Size), -1))\n\t\t\t\t\t}\n\t\t\t\t\tpipe.HDel(ctx, m.delSlices(), key)\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\terr := m.txn(ctx, task, m.delSlices())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif clean && len(rs) == len(ss) {\n\t\t\tfor i, s := range ss {\n\t\t\t\tif rs[i].Err() == nil && rs[i].Val() < 0 {\n\t\t\t\t\tm.deleteSlice(s.Id, s.Size)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (m *redisMeta) scanPendingSlices(ctx Context, scan pendingSliceScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\n\tpendingKeys := make(chan string, 1000)\n\tc, cancel := context.WithCancel(ctx)\n\tdefer cancel()\n\tgo func() {\n\t\t_ = m.hscan(c, m.sliceRefs(), func(keys []string) error {\n\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\tval := keys[i+1]\n\t\t\t\trefs, err := strconv.ParseInt(val, 10, 64)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// ignored\n\t\t\t\t\tlogger.Warn(errors.Wrapf(err, \"parse slice ref: %s\", val))\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tif refs < 0 {\n\t\t\t\t\tpendingKeys <- keys[i]\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tclose(pendingKeys)\n\t}()\n\n\tfor key := range pendingKeys {\n\t\tps := strings.Split(key[1:], \"_\")\n\t\tif len(ps) != 2 {\n\t\t\treturn fmt.Errorf(\"invalid key %s\", key)\n\t\t}\n\t\tid, err := strconv.ParseUint(ps[0], 10, 64)\n\t\tif err != nil {\n\t\t\treturn errors.Wrapf(err, \"invalid key %s, fail to parse id\", key)\n\t\t}\n\t\tsize, err := strconv.ParseUint(ps[1], 10, 64)\n\t\tif err != nil {\n\t\t\treturn errors.Wrapf(err, \"invalid key %s, fail to parse size\", key)\n\t\t}\n\t\tclean, err := scan(id, uint32(size))\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"scan pending slices\")\n\t\t}\n\t\tif clean {\n\t\t\t// TODO: m.deleteSlice(id, uint32(size))\n\t\t\t// avoid lint warning\n\t\t\t_ = clean\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) scanPendingFiles(ctx Context, scan pendingFileScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\n\tvisited := make(map[Ino]bool)\n\tstart := int64(0)\n\tconst batchSize = 1000\n\n\tfor {\n\t\tpairs, err := m.rdb.ZRangeWithScores(Background(), m.delfiles(), start, start+batchSize).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor _, p := range pairs {\n\t\t\tv := p.Member.(string)\n\t\t\tps := strings.Split(v, \":\")\n\t\t\tif len(ps) != 2 { // will be cleaned up as legacy\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tinode, _ := strconv.ParseUint(ps[0], 10, 64)\n\t\t\tif visited[Ino(inode)] {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvisited[Ino(inode)] = true\n\t\t\tsize, _ := strconv.ParseUint(ps[1], 10, 64)\n\t\t\tif _, err := scan(Ino(inode), size, int64(p.Score)); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tstart += batchSize\n\t\tif len(pairs) < batchSize {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (m *redisMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tattr.Nlink = 2\n\t\tvals, err := tx.HGetAll(ctx, m.entryKey(inode)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, v := range vals {\n\t\t\ttyp, _ := m.parseEntry([]byte(v))\n\t\t\tif typ == TypeDirectory {\n\t\t\t\tattr.Nlink++\n\t\t\t}\n\t\t}\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(inode), m.entryKey(inode)))\n}\n\nfunc (m *redisMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno {\n\tdefer m.timeit(\"GetXattr\", time.Now())\n\tinode = m.checkRoot(inode)\n\tvar err error\n\t*vbuff, err = m.rdb.HGet(ctx, m.xattrKey(inode), name).Bytes()\n\tif err == redis.Nil {\n\t\terr = ENOATTR\n\t}\n\treturn errno(err)\n}\n\nfunc (m *redisMeta) ListXattr(ctx Context, inode Ino, names *[]byte) syscall.Errno {\n\tdefer m.timeit(\"ListXattr\", time.Now())\n\tinode = m.checkRoot(inode)\n\tvals, err := m.rdb.HKeys(ctx, m.xattrKey(inode)).Result()\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\t*names = nil\n\tfor _, name := range vals {\n\t\t*names = append(*names, []byte(name)...)\n\t\t*names = append(*names, 0)\n\t}\n\n\tval, err := m.rdb.Get(ctx, m.inodeKey(inode)).Bytes()\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tattr := &Attr{}\n\tm.parseAttr(val, attr)\n\tsetXAttrACL(names, attr.AccessACL, attr.DefaultACL)\n\treturn 0\n}\n\nfunc (m *redisMeta) doSetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno {\n\tkey := m.xattrKey(inode)\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tswitch flags {\n\t\tcase XattrCreate:\n\t\t\tok, err := tx.HSetNX(ctx, key, name, value).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif !ok {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\treturn nil\n\t\tcase XattrReplace:\n\t\t\tif ok, err := tx.HExists(ctx, key, name).Result(); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if !ok {\n\t\t\t\treturn ENOATTR\n\t\t\t}\n\t\t\t_, err := tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\tpipe.HSet(ctx, key, name, value)\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn err\n\t\tdefault: // XattrCreateOrReplace\n\t\t\t_, err := tx.HSet(ctx, key, name, value).Result()\n\t\t\treturn err\n\t\t}\n\t}, key))\n}\n\nfunc (m *redisMeta) doRemoveXattr(ctx Context, inode Ino, name string) syscall.Errno {\n\tn, err := m.rdb.HDel(ctx, m.xattrKey(inode), name).Result()\n\tif err != nil {\n\t\treturn errno(err)\n\t} else if n == 0 {\n\t\treturn ENOATTR\n\t} else {\n\t\treturn 0\n\t}\n}\n\ntype quotaKeys struct {\n\tquotaKey string\n\tusedSpaceKey string\n\tusedInodesKey string\n}\n\nfunc (m *redisMeta) getQuotaKeys(qtype uint32) (*quotaKeys, error) {\n\tswitch qtype {\n\tcase DirQuotaType:\n\t\treturn "aKeys{\n\t\t\tquotaKey: m.dirQuotaKey(),\n\t\t\tusedSpaceKey: m.dirQuotaUsedSpaceKey(),\n\t\t\tusedInodesKey: m.dirQuotaUsedInodesKey(),\n\t\t}, nil\n\tcase UserQuotaType:\n\t\treturn "aKeys{\n\t\t\tquotaKey: m.userQuotaKey(),\n\t\t\tusedSpaceKey: m.userQuotaUsedSpaceKey(),\n\t\t\tusedInodesKey: m.userQuotaUsedInodesKey(),\n\t\t}, nil\n\tcase GroupQuotaType:\n\t\treturn "aKeys{\n\t\t\tquotaKey: m.groupQuotaKey(),\n\t\t\tusedSpaceKey: m.groupQuotaUsedSpaceKey(),\n\t\t\tusedInodesKey: m.groupQuotaUsedInodesKey(),\n\t\t}, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unknown quota type: %d\", qtype)\n\t}\n}\n\nfunc (m *redisMeta) doGetQuota(ctx Context, qtype uint32, key uint64) (*Quota, error) {\n\tconfig, err := m.getQuotaKeys(qtype)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfield := strconv.FormatUint(key, 10)\n\tcmds, err := m.rdb.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\tpipe.HGet(ctx, config.quotaKey, field)\n\t\tpipe.HGet(ctx, config.usedSpaceKey, field)\n\t\tpipe.HGet(ctx, config.usedInodesKey, field)\n\t\treturn nil\n\t})\n\tif err == redis.Nil {\n\t\treturn nil, nil\n\t} else if err != nil {\n\t\treturn nil, err\n\t}\n\n\tbuf, _ := cmds[0].(*redis.StringCmd).Bytes()\n\tif len(buf) != 16 {\n\t\treturn nil, fmt.Errorf(\"invalid quota value: %v\", buf)\n\t}\n\n\tvar quota Quota\n\tquota.MaxSpace, quota.MaxInodes = m.parseQuota(buf)\n\tif quota.UsedSpace, err = cmds[1].(*redis.StringCmd).Int64(); err != nil {\n\t\treturn nil, err\n\t}\n\tif quota.UsedInodes, err = cmds[2].(*redis.StringCmd).Int64(); err != nil {\n\t\treturn nil, err\n\t}\n\treturn "a, nil\n}\n\nfunc (m *redisMeta) doSetQuota(ctx Context, qtype uint32, key uint64, quota *Quota) (bool, error) {\n\tconfig, err := m.getQuotaKeys(qtype)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tvar created bool\n\terr = m.txn(ctx, func(tx *redis.Tx) error {\n\t\torigin := &Quota{MaxSpace: -1, MaxInodes: -1}\n\t\tfield := strconv.FormatUint(key, 10)\n\n\t\tbuf, e := tx.HGet(ctx, config.quotaKey, field).Bytes()\n\t\tif e == nil {\n\t\t\tcreated = false\n\t\t\torigin.MaxSpace, origin.MaxInodes = m.parseQuota(buf)\n\t\t} else if e == redis.Nil {\n\t\t\tcreated = true\n\t\t} else {\n\t\t\treturn e\n\t\t}\n\n\t\tif quota.MaxSpace >= 0 {\n\t\t\torigin.MaxSpace = quota.MaxSpace\n\t\t}\n\t\tif quota.MaxInodes >= 0 {\n\t\t\torigin.MaxInodes = quota.MaxInodes\n\t\t}\n\n\t\t_, e = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.HSet(ctx, config.quotaKey, field, m.packQuota(origin.MaxSpace, origin.MaxInodes))\n\t\t\tif quota.UsedSpace >= 0 {\n\t\t\t\tpipe.HSet(ctx, config.usedSpaceKey, field, quota.UsedSpace)\n\t\t\t} else if created {\n\t\t\t\tpipe.HSet(ctx, config.usedSpaceKey, field, 0)\n\t\t\t}\n\t\t\tif quota.UsedInodes >= 0 {\n\t\t\t\tpipe.HSet(ctx, config.usedInodesKey, field, quota.UsedInodes)\n\t\t\t} else if created {\n\t\t\t\tpipe.HSet(ctx, config.usedInodesKey, field, 0)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn e\n\t}, m.inodeKey(Ino(key)))\n\treturn created, err\n}\n\nfunc (m *redisMeta) doDelQuota(ctx Context, qtype uint32, key uint64) error {\n\tconfig, err := m.getQuotaKeys(qtype)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfield := strconv.FormatUint(key, 10)\n\t_, err = m.rdb.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\tif qtype == UserQuotaType || qtype == GroupQuotaType {\n\t\t\tquotaData := m.packQuota(-1, -1) // -1 means unlimited\n\t\t\tpipe.HSet(ctx, config.quotaKey, field, quotaData)\n\t\t} else {\n\t\t\tpipe.HDel(ctx, config.quotaKey, field)\n\t\t\tpipe.HDel(ctx, config.usedSpaceKey, field)\n\t\t\tpipe.HDel(ctx, config.usedInodesKey, field)\n\t\t}\n\t\treturn nil\n\t})\n\treturn err\n}\n\nfunc (m *redisMeta) doLoadQuotas(ctx Context) (map[uint64]*Quota, map[uint64]*Quota, map[uint64]*Quota, error) {\n\tquotaTypes := []struct {\n\t\tqtype uint32\n\t\tname string\n\t}{\n\t\t{DirQuotaType, \"dir\"},\n\t\t{UserQuotaType, \"user\"},\n\t\t{GroupQuotaType, \"group\"},\n\t}\n\n\tquotaMaps := make([]map[uint64]*Quota, 3)\n\tfor i, qt := range quotaTypes {\n\t\tconfig, err := m.getQuotaKeys(qt.qtype)\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, fmt.Errorf(\"failed to load %s quotas: %w\", qt.name, err)\n\t\t}\n\n\t\tquotas := make(map[uint64]*Quota)\n\t\tif err := m.hscan(ctx, config.usedInodesKey, func(keys []string) error {\n\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\tkey := keys[i]\n\t\t\t\tid, err := strconv.ParseUint(key, 10, 64)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"invalid key in %s: %s\", qt.name, key)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tusedInodes, err := strconv.ParseInt(keys[i+1], 10, 64)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"invalid usedInodes for %s %s: %s\", qt.name, key, keys[i+1])\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tusedSpace, err := m.rdb.HGet(ctx, config.usedSpaceKey, key).Int64()\n\t\t\t\tif err != nil && err != redis.Nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tvar maxSpace, maxInodes int64 = -1, -1\n\t\t\t\tif buf, err := m.rdb.HGet(ctx, config.quotaKey, key).Bytes(); err == nil {\n\t\t\t\t\tif len(buf) != 16 {\n\t\t\t\t\t\tlogger.Errorf(\"invalid quota value for %s %s: len=%d\", qt.name, key, len(buf))\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tmaxSpace, maxInodes = m.parseQuota(buf)\n\t\t\t\t} else if err != redis.Nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tquotas[id] = &Quota{\n\t\t\t\t\tMaxSpace: maxSpace,\n\t\t\t\t\tMaxInodes: maxInodes,\n\t\t\t\t\tUsedSpace: usedSpace,\n\t\t\t\t\tUsedInodes: usedInodes,\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}); err != nil {\n\t\t\treturn nil, nil, nil, err\n\t\t}\n\t\tquotaMaps[i] = quotas\n\t}\n\n\treturn quotaMaps[0], quotaMaps[1], quotaMaps[2], nil\n}\n\nfunc (m *redisMeta) doFlushQuotas(ctx Context, quotas []*iQuota) error {\n\t_, err := m.rdb.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\tfor _, q := range quotas {\n\t\t\tconfig, err := m.getQuotaKeys(q.qtype)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tkey := strconv.FormatUint(q.qkey, 10)\n\t\t\tpipe.HSetNX(ctx, config.quotaKey, key, m.packQuota(-1, -1))\n\t\t\tpipe.HIncrBy(ctx, config.usedSpaceKey, key, q.quota.newSpace)\n\t\t\tpipe.HIncrBy(ctx, config.usedInodesKey, key, q.quota.newInodes)\n\t\t}\n\t\treturn nil\n\t})\n\treturn err\n}\n\nfunc (m *redisMeta) checkServerConfig() {\n\trawInfo, err := m.rdb.Info(Background()).Result()\n\tif err != nil {\n\t\tlogger.Warnf(\"parse info: %s\", err)\n\t\treturn\n\t}\n\trInfo, err := checkRedisInfo(rawInfo)\n\tif err != nil {\n\t\tlogger.Warnf(\"parse info: %s\", err)\n\t}\n\tif rInfo.storageProvider == \"\" && rInfo.maxMemoryPolicy != \"\" && rInfo.maxMemoryPolicy != \"noeviction\" {\n\t\tlogger.Warnf(\"maxmemory_policy is %q, we will try to reconfigure it to 'noeviction'.\", rInfo.maxMemoryPolicy)\n\t\tif _, err := m.rdb.ConfigSet(Background(), \"maxmemory-policy\", \"noeviction\").Result(); err != nil {\n\t\t\tlogger.Errorf(\"try to reconfigure maxmemory-policy to 'noeviction' failed: %s\", err)\n\t\t} else if result, err := m.rdb.ConfigGet(Background(), \"maxmemory-policy\").Result(); err != nil {\n\t\t\tlogger.Warnf(\"get config maxmemory-policy failed: %s\", err)\n\t\t} else if len(result) == 1 && result[\"maxmemory-policy\"] != \"noeviction\" {\n\t\t\tlogger.Warnf(\"reconfigured maxmemory-policy to 'noeviction', but it's still %s\", result[\"maxmemory-policy\"])\n\t\t} else {\n\t\t\tlogger.Infof(\"set maxmemory-policy to 'noeviction' successfully\")\n\t\t}\n\t}\n\tstart := time.Now()\n\t_, err = m.rdb.Ping(Background()).Result()\n\tif err != nil {\n\t\tlogger.Errorf(\"Ping redis: %s\", err.Error())\n\t\treturn\n\t}\n\tlogger.Infof(\"Ping redis latency: %s\", time.Since(start))\n}\n\nfunc (m *redisMeta) dumpEntries(es ...*DumpedEntry) error {\n\tctx := Background()\n\tvar keys []string\n\tfor _, e := range es {\n\t\tkeys = append(keys, m.inodeKey(e.Attr.Inode))\n\t}\n\treturn m.txn(ctx, func(tx *redis.Tx) error {\n\t\tp := tx.Pipeline()\n\t\tvar ar = make([]*redis.StringCmd, len(es))\n\t\tvar xr = make([]*redis.MapStringStringCmd, len(es))\n\t\tvar sr = make([]*redis.StringCmd, len(es))\n\t\tvar cr = make([]*redis.StringSliceCmd, len(es))\n\t\tvar dr = make([]*redis.ScanCmd, len(es))\n\t\tfor i, e := range es {\n\t\t\tinode := e.Attr.Inode\n\t\t\tar[i] = p.Get(ctx, m.inodeKey(inode))\n\t\t\txr[i] = p.HGetAll(ctx, m.xattrKey(inode))\n\t\t\tswitch e.Attr.Type {\n\t\t\tcase \"regular\":\n\t\t\t\tcr[i] = p.LRange(ctx, m.chunkKey(inode, 0), 0, -1)\n\t\t\tcase \"directory\":\n\t\t\t\tdr[i] = p.HScan(ctx, m.entryKey(inode), 0, \"*\", 1000)\n\t\t\tcase \"symlink\":\n\t\t\t\tsr[i] = p.Get(ctx, m.symKey(inode))\n\t\t\t}\n\t\t}\n\t\tif _, err := p.Exec(ctx); err != nil && err != redis.Nil {\n\t\t\treturn err\n\t\t}\n\n\t\ttype lchunk struct {\n\t\t\tinode Ino\n\t\t\tindx uint32\n\t\t\ti uint32\n\t\t}\n\t\tvar lcs []*lchunk\n\t\tfor i, e := range es {\n\t\t\tinode := e.Attr.Inode\n\t\t\ttyp := typeFromString(e.Attr.Type)\n\t\t\ta, err := ar[i].Bytes()\n\t\t\tif err != nil {\n\t\t\t\tif err != redis.Nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif inode != TrashInode {\n\t\t\t\t\tlogger.Warnf(\"Corrupt inode: %d, missing attribute\", inode)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar attr Attr\n\t\t\tattr.Typ = typ\n\t\t\tattr.Nlink = 1\n\t\t\tm.parseAttr(a, &attr)\n\t\t\tif attr.Typ != typ {\n\t\t\t\te.Attr.Type = typeToString(attr.Typ)\n\t\t\t\treturn redis.TxFailedErr // retry\n\t\t\t}\n\t\t\tif err == redis.Nil && attr.Typ == TypeDirectory {\n\t\t\t\tattr.Nlink = 2\n\t\t\t}\n\t\t\tdumpAttr(&attr, e.Attr)\n\n\t\t\tkeys, err := xr[i].Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif len(keys) > 0 {\n\t\t\t\txattrs := make([]*DumpedXattr, 0, len(keys))\n\t\t\t\tfor k, v := range keys {\n\t\t\t\t\txattrs = append(xattrs, &DumpedXattr{k, v})\n\t\t\t\t}\n\t\t\t\tsort.Slice(xattrs, func(i, j int) bool { return xattrs[i].Name < xattrs[j].Name })\n\t\t\t\te.Xattrs = xattrs\n\t\t\t}\n\n\t\t\taccessACl, err := m.getACL(ctx, tx, attr.AccessACL)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\te.AccessACL = dumpACL(accessACl)\n\t\t\tdefaultACL, err := m.getACL(ctx, tx, attr.DefaultACL)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\te.DefaultACL = dumpACL(defaultACL)\n\n\t\t\tswitch typ {\n\t\t\tcase TypeFile:\n\t\t\t\te.Chunks = e.Chunks[:0]\n\t\t\t\tif attr.Length > 0 {\n\t\t\t\t\tvals, err := cr[i].Result()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif len(vals) > 0 {\n\t\t\t\t\t\tss := readSlices(vals)\n\t\t\t\t\t\tif ss == nil {\n\t\t\t\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d\", inode, 0)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tslices := make([]*DumpedSlice, 0, len(ss))\n\t\t\t\t\t\tfor _, s := range ss {\n\t\t\t\t\t\t\tslices = append(slices, &DumpedSlice{Id: s.id, Pos: s.pos, Size: s.size, Off: s.off, Len: s.len})\n\t\t\t\t\t\t}\n\t\t\t\t\t\te.Chunks = append(e.Chunks, &DumpedChunk{0, slices})\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif attr.Length > ChunkSize {\n\t\t\t\t\tfor indx := uint32(1); uint64(indx)*ChunkSize < attr.Length; indx++ {\n\t\t\t\t\t\tlcs = append(lcs, &lchunk{inode, indx, uint32(i)})\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase TypeDirectory:\n\t\t\t\tkeys, cursor, err := dr[i].Result()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif cursor == 0 {\n\t\t\t\t\te.Entries = make(map[string]*DumpedEntry)\n\t\t\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\t\t\tname := keys[i]\n\t\t\t\t\t\tt, inode := m.parseEntry([]byte(keys[i+1]))\n\t\t\t\t\t\tce := entryPool.Get()\n\t\t\t\t\t\tce.Name = name\n\t\t\t\t\t\tce.Attr.Inode = inode\n\t\t\t\t\t\tce.Attr.Type = typeToString(t)\n\t\t\t\t\t\te.Entries[name] = ce\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase TypeSymlink:\n\t\t\t\tif e.Symlink, err = sr[i].Result(); err != nil {\n\t\t\t\t\tif err != redis.Nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tlogger.Warnf(\"The symlink of inode %d is not found\", inode)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcr = make([]*redis.StringSliceCmd, len(es)*3)\n\t\tfor len(lcs) > 0 {\n\t\t\tif len(cr) > len(lcs) {\n\t\t\t\tcr = cr[:len(lcs)]\n\t\t\t}\n\t\t\tfor i := range cr {\n\t\t\t\tc := lcs[i]\n\t\t\t\tcr[i] = p.LRange(ctx, m.chunkKey(c.inode, c.indx), 0, -1)\n\t\t\t}\n\t\t\tif _, err := p.Exec(ctx); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor i := range cr {\n\t\t\t\tvals, err := cr[i].Result()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif len(vals) > 0 {\n\t\t\t\t\te := es[lcs[i].i]\n\t\t\t\t\tss := readSlices(vals)\n\t\t\t\t\tif ss == nil {\n\t\t\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d\", e.Attr.Inode, lcs[i].indx)\n\t\t\t\t\t}\n\t\t\t\t\tslices := make([]*DumpedSlice, 0, len(ss))\n\t\t\t\t\tfor _, s := range ss {\n\t\t\t\t\t\tslices = append(slices, &DumpedSlice{Id: s.id, Pos: s.pos, Size: s.size, Off: s.off, Len: s.len})\n\t\t\t\t\t}\n\t\t\t\t\te.Chunks = append(e.Chunks, &DumpedChunk{lcs[i].indx, slices})\n\t\t\t\t}\n\t\t\t}\n\t\t\tlcs = lcs[len(cr):]\n\t\t}\n\t\treturn nil\n\t}, keys...)\n}\n\nfunc (m *redisMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth, threads int, showProgress func(totalIncr, currentIncr int64)) error {\n\tbwWrite := func(s string) {\n\t\tif _, err := bw.WriteString(s); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\n\tif tree.Entries == nil {\n\t\ttree.Entries = make(map[string]*DumpedEntry)\n\t\terr := m.hscan(Background(), m.entryKey(inode), func(keys []string) error {\n\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\tname := keys[i]\n\t\t\t\tt, inode := m.parseEntry([]byte(keys[i+1]))\n\t\t\t\te := entryPool.Get()\n\t\t\t\te.Name = name\n\t\t\t\te.Attr.Inode = inode\n\t\t\t\te.Attr.Type = typeToString(t)\n\t\t\t\ttree.Entries[name] = e\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tvar err error\n\tif err = tree.writeJsonWithOutEntry(bw, depth); err != nil {\n\t\treturn err\n\t}\n\tentries := make([]*DumpedEntry, 0, len(tree.Entries))\n\tfor _, e := range tree.Entries {\n\t\tentries = append(entries, e)\n\t}\n\tsort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })\n\tif showProgress != nil {\n\t\tshowProgress(int64(len(entries)), 0)\n\t}\n\n\tvar batch = 100\n\tms := make([]sync.Mutex, threads)\n\tconds := make([]*sync.Cond, threads)\n\tready := make([]int, threads)\n\tfor c := 0; c < threads; c++ {\n\t\tconds[c] = sync.NewCond(&ms[c])\n\t\tif c*batch < len(entries) {\n\t\t\tgo func(c int) {\n\t\t\t\tfor i := c * batch; i < len(entries) && err == nil; i += threads * batch {\n\t\t\t\t\tes := entries[i:]\n\t\t\t\t\tif len(es) > batch {\n\t\t\t\t\t\tes = es[:batch]\n\t\t\t\t\t}\n\t\t\t\t\te := m.dumpEntries(es...)\n\t\t\t\t\tms[c].Lock()\n\t\t\t\t\tready[c] = len(es)\n\t\t\t\t\tif e != nil {\n\t\t\t\t\t\terr = e\n\t\t\t\t\t}\n\t\t\t\t\tconds[c].Signal()\n\t\t\t\t\tfor ready[c] > 0 && err == nil {\n\t\t\t\t\t\tconds[c].Wait()\n\t\t\t\t\t}\n\t\t\t\t\tms[c].Unlock()\n\t\t\t\t}\n\t\t\t}(c)\n\t\t}\n\t}\n\tfor i, e := range entries {\n\t\tb := i / batch\n\t\tc := b % threads\n\t\tms[c].Lock()\n\t\tfor ready[c] == 0 && err == nil {\n\t\t\tconds[c].Wait()\n\t\t}\n\t\tready[c]--\n\t\tif ready[c] == 0 {\n\t\t\tconds[c].Signal()\n\t\t}\n\t\tms[c].Unlock()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif e.Attr.Type == \"directory\" {\n\t\t\terr = m.dumpDir(e.Attr.Inode, e, bw, depth+2, threads, showProgress)\n\t\t} else {\n\t\t\terr = e.writeJSON(bw, depth+2)\n\t\t}\n\t\tentries[i] = nil\n\t\tentryPool.Put(e)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif i != len(entries)-1 {\n\t\t\tbwWrite(\",\")\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(0, 1)\n\t\t}\n\t}\n\tbwWrite(fmt.Sprintf(\"\\n%s}\\n%s}\", strings.Repeat(jsonIndent, depth+1), strings.Repeat(jsonIndent, depth)))\n\treturn nil\n}\n\nfunc (m *redisMeta) DumpMeta(w io.Writer, root Ino, threads int, keepSecret, fast, skipTrash bool) (err error) {\n\tdefer func() {\n\t\tif p := recover(); p != nil {\n\t\t\tdebug.PrintStack()\n\t\t\tif e, ok := p.(error); ok {\n\t\t\t\terr = e\n\t\t\t} else {\n\t\t\t\terr = errors.Errorf(\"DumpMeta error: %v\", p)\n\t\t\t}\n\t\t}\n\t}()\n\tctx := Background()\n\tzs, err := m.rdb.ZRangeWithScores(ctx, m.delfiles(), 0, -1).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdels := make([]*DumpedDelFile, 0, len(zs))\n\tfor _, z := range zs {\n\t\tparts := strings.Split(z.Member.(string), \":\")\n\t\tif len(parts) != 2 {\n\t\t\tlogger.Warnf(\"invalid delfile string: %s\", z.Member.(string))\n\t\t\tcontinue\n\t\t}\n\t\tinode, _ := strconv.ParseUint(parts[0], 10, 64)\n\t\tlength, _ := strconv.ParseUint(parts[1], 10, 64)\n\t\tdels = append(dels, &DumpedDelFile{Ino(inode), length, int64(z.Score)})\n\t}\n\n\tnames := []string{usedSpace, totalInodes, \"nextinode\", \"nextchunk\", \"nextsession\", \"nextTrash\"}\n\tfor i := range names {\n\t\tnames[i] = m.prefix + names[i]\n\t}\n\trs, _ := m.rdb.MGet(ctx, names...).Result()\n\tcs := make([]int64, len(rs))\n\tfor i, r := range rs {\n\t\tif r != nil {\n\t\t\tcs[i], _ = strconv.ParseInt(r.(string), 10, 64)\n\t\t}\n\t}\n\n\tkeys, err := m.rdb.ZRange(ctx, m.allSessions(), 0, -1).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\tsessions := make([]*DumpedSustained, 0, len(keys))\n\tfor _, k := range keys {\n\t\tsid, _ := strconv.ParseUint(k, 10, 64)\n\t\tvar ss []string\n\t\tss, err = m.rdb.SMembers(ctx, m.sustained(sid)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(ss) > 0 {\n\t\t\tinodes := make([]Ino, 0, len(ss))\n\t\t\tfor _, s := range ss {\n\t\t\t\tinode, _ := strconv.ParseUint(s, 10, 64)\n\t\t\t\tinodes = append(inodes, Ino(inode))\n\t\t\t}\n\t\t\tsessions = append(sessions, &DumpedSustained{sid, inodes})\n\t\t}\n\t}\n\tquotas := make(map[Ino]*DumpedQuota)\n\tfor k, v := range m.rdb.HGetAll(ctx, m.dirQuotaKey()).Val() {\n\t\tinode, err := strconv.ParseUint(k, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse inode: %s: %v\", k, err)\n\t\t\tcontinue\n\t\t}\n\t\tif len(v) != 16 {\n\t\t\tlogger.Warnf(\"invalid quota string: %s\", hex.EncodeToString([]byte(v)))\n\t\t\tcontinue\n\t\t}\n\t\tvar quota DumpedQuota\n\t\tquota.MaxSpace, quota.MaxInodes = m.parseQuota([]byte(v))\n\t\tquotas[Ino(inode)] = "a\n\t}\n\n\tdm := &DumpedMeta{\n\t\tSetting: *m.getFormat(),\n\t\tCounters: &DumpedCounters{\n\t\t\tUsedSpace: cs[0],\n\t\t\tUsedInodes: cs[1],\n\t\t\tNextInode: cs[2] + 1, // Redis nextInode/nextChunk is 1 smaller than sql/tkv\n\t\t\tNextChunk: cs[3] + 1,\n\t\t\tNextSession: cs[4],\n\t\t\tNextTrash: cs[5],\n\t\t},\n\t\tSustained: sessions,\n\t\tDelFiles: dels,\n\t\tQuotas: quotas,\n\t}\n\tif !keepSecret && dm.Setting.SecretKey != \"\" {\n\t\tdm.Setting.SecretKey = \"removed\"\n\t\tlogger.Warnf(\"Secret key is removed for the sake of safety\")\n\t}\n\tif !keepSecret && dm.Setting.SessionToken != \"\" {\n\t\tdm.Setting.SessionToken = \"removed\"\n\t\tlogger.Warnf(\"Session token is removed for the sake of safety\")\n\t}\n\tbw, err := dm.writeJsonWithOutTree(w)\n\tif err != nil {\n\t\treturn err\n\t}\n\troot = m.checkRoot(root)\n\tprogress := utils.NewProgress(false)\n\tbar := progress.AddCountBar(\"Dumped entries\", 1) // with root\n\tuseTotal := root == RootInode && !skipTrash\n\tif useTotal {\n\t\tbar.SetTotal(dm.Counters.UsedInodes)\n\t}\n\n\tshowProgress := func(totalIncr, currentIncr int64) {\n\t\tif !useTotal {\n\t\t\tbar.IncrTotal(totalIncr)\n\t\t}\n\t\tbar.IncrInt64(currentIncr)\n\t}\n\n\tvar tree = &DumpedEntry{\n\t\tName: \"FSTree\",\n\t\tAttr: &DumpedAttr{\n\t\t\tInode: root,\n\t\t\tType: typeToString(TypeDirectory),\n\t\t},\n\t}\n\tif err = m.dumpEntries(tree); err != nil {\n\t\treturn err\n\t}\n\tbar.Increment()\n\tif err = m.dumpDir(root, tree, bw, 1, threads, showProgress); err != nil {\n\t\treturn err\n\t}\n\tif root == RootInode && !skipTrash {\n\t\ttrash := &DumpedEntry{\n\t\t\tName: \"Trash\",\n\t\t\tAttr: &DumpedAttr{\n\t\t\t\tInode: TrashInode,\n\t\t\t\tType: typeToString(TypeDirectory),\n\t\t\t},\n\t\t}\n\t\tif err = m.dumpEntries(trash); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err = bw.WriteString(\",\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err = m.dumpDir(TrashInode, trash, bw, 1, threads, showProgress); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif _, err = bw.WriteString(\"\\n}\\n\"); err != nil {\n\t\treturn err\n\t}\n\tprogress.Done()\n\n\treturn bw.Flush()\n}\n\nfunc (m *redisMeta) loadEntry(e *DumpedEntry, p redis.Pipeliner, tryExec func(), aclMaxId *uint32) {\n\tctx := Background()\n\tinode := e.Attr.Inode\n\tattr := loadAttr(e.Attr)\n\tattr.Parent = e.Parents[0]\n\tbatch := 100\n\tif attr.Typ == TypeFile {\n\t\tattr.Length = e.Attr.Length\n\t\tfor _, c := range e.Chunks {\n\t\t\tif len(c.Slices) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tslices := make([]string, 0, len(c.Slices))\n\t\t\tfor _, s := range c.Slices {\n\t\t\t\tslices = append(slices, string(marshalSlice(s.Pos, s.Id, s.Size, s.Off, s.Len)))\n\t\t\t\tif len(slices) > batch {\n\t\t\t\t\tp.RPush(ctx, m.chunkKey(inode, c.Index), slices)\n\t\t\t\t\ttryExec()\n\t\t\t\t\tslices = slices[:0]\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(slices) > 0 {\n\t\t\t\tp.RPush(ctx, m.chunkKey(inode, c.Index), slices)\n\t\t\t}\n\t\t}\n\t} else if attr.Typ == TypeDirectory {\n\t\tattr.Length = 4 << 10\n\t\tdentries := make(map[string]interface{}, batch)\n\t\tvar stat dirStat\n\t\tfor name, c := range e.Entries {\n\t\t\tlength := uint64(0)\n\t\t\tif typeFromString(c.Attr.Type) == TypeFile {\n\t\t\t\tlength = c.Attr.Length\n\t\t\t}\n\t\t\tstat.length += int64(length)\n\t\t\tstat.space += align4K(length)\n\t\t\tstat.inodes++\n\n\t\t\tdentries[string(unescape(name))] = m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode)\n\t\t\tif len(dentries) >= batch {\n\t\t\t\tp.HSet(ctx, m.entryKey(inode), dentries)\n\t\t\t\ttryExec()\n\t\t\t\tdentries = make(map[string]interface{}, batch)\n\t\t\t}\n\t\t}\n\t\tif len(dentries) > 0 {\n\t\t\tp.HSet(ctx, m.entryKey(inode), dentries)\n\t\t}\n\t\tfield := inode.String()\n\t\tp.HSet(ctx, m.dirDataLengthKey(), field, stat.length)\n\t\tp.HSet(ctx, m.dirUsedSpaceKey(), field, stat.space)\n\t\tp.HSet(ctx, m.dirUsedInodesKey(), field, stat.inodes)\n\t} else if attr.Typ == TypeSymlink {\n\t\tsymL := unescape(e.Symlink)\n\t\tattr.Length = uint64(len(symL))\n\t\tp.Set(ctx, m.symKey(inode), symL, 0)\n\t}\n\n\tif len(e.Xattrs) > 0 {\n\t\txattrs := make(map[string]interface{})\n\t\tfor _, x := range e.Xattrs {\n\t\t\txattrs[x.Name] = unescape(x.Value)\n\t\t}\n\t\tp.HSet(ctx, m.xattrKey(inode), xattrs)\n\t}\n\n\tattr.AccessACL = m.saveACL(loadACL(e.AccessACL), aclMaxId)\n\tattr.DefaultACL = m.saveACL(loadACL(e.DefaultACL), aclMaxId)\n\n\tp.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\ttryExec()\n}\n\nfunc (m *redisMeta) LoadMeta(r io.Reader) (err error) {\n\tctx := Background()\n\tif _, ok := m.rdb.(*redis.ClusterClient); ok {\n\t\terr = m.scan(ctx, \"*\", func(keys []string) error {\n\t\t\treturn fmt.Errorf(\"found key with same prefix: %s\", keys[0])\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tdbsize, err := m.rdb.DBSize(ctx).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif dbsize > 0 {\n\t\t\treturn fmt.Errorf(\"Database redis://%s is not empty\", m.addr)\n\t\t}\n\t}\n\n\tp := m.rdb.TxPipeline()\n\ttryExec := func() {\n\t\tif p.Len() > 1000 {\n\t\t\tif rs, err := p.Exec(ctx); err != nil {\n\t\t\t\tfor i, r := range rs {\n\t\t\t\t\tif r.Err() != nil {\n\t\t\t\t\t\tlogger.Errorf(\"failed command %d %+v: %s\", i, r, r.Err())\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t}\n\t}\n\tdefer func() {\n\t\tif e := recover(); e != nil {\n\t\t\tif ee, ok := e.(error); ok {\n\t\t\t\terr = ee\n\t\t\t} else {\n\t\t\t\tpanic(e)\n\t\t\t}\n\t\t}\n\t}()\n\n\tvar aclMaxId uint32\n\tdm, counters, parents, refs, err := loadEntries(r, func(e *DumpedEntry) { m.loadEntry(e, p, tryExec, &aclMaxId) }, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tm.loadDumpedQuotas(ctx, dm.Quotas)\n\tif err = m.loadDumpedACLs(ctx); err != nil {\n\t\treturn err\n\t}\n\tformat, _ := json.MarshalIndent(dm.Setting, \"\", \"\")\n\tp.Set(ctx, m.setting(), format, 0)\n\tcs := make(map[string]interface{})\n\tcs[m.prefix+usedSpace] = counters.UsedSpace\n\tcs[m.prefix+totalInodes] = counters.UsedInodes\n\tcs[m.prefix+\"nextinode\"] = counters.NextInode - 1\n\tcs[m.prefix+\"nextchunk\"] = counters.NextChunk - 1\n\tcs[m.prefix+\"nextsession\"] = counters.NextSession\n\tcs[m.prefix+\"nextTrash\"] = counters.NextTrash\n\tp.MSet(ctx, cs)\n\tif l := len(dm.DelFiles); l > 0 {\n\t\tif l > 100 {\n\t\t\tl = 100\n\t\t}\n\t\tzs := make([]redis.Z, 0, l)\n\t\tfor _, d := range dm.DelFiles {\n\t\t\tif len(zs) >= 100 {\n\t\t\t\tp.ZAdd(ctx, m.delfiles(), zs...)\n\t\t\t\ttryExec()\n\t\t\t\tzs = zs[:0]\n\t\t\t}\n\t\t\tzs = append(zs, redis.Z{\n\t\t\t\tScore: float64(d.Expire),\n\t\t\t\tMember: m.toDelete(d.Inode, d.Length),\n\t\t\t})\n\t\t}\n\t\tp.ZAdd(ctx, m.delfiles(), zs...)\n\t}\n\tslices := make(map[string]interface{})\n\tfor k, v := range refs {\n\t\tif v > 1 {\n\t\t\tif len(slices) > 100 {\n\t\t\t\tp.HSet(ctx, m.sliceRefs(), slices)\n\t\t\t\ttryExec()\n\t\t\t\tslices = make(map[string]interface{})\n\t\t\t}\n\t\t\tslices[m.sliceKey(k.id, k.size)] = v - 1\n\t\t}\n\t}\n\tif len(slices) > 0 {\n\t\tp.HSet(ctx, m.sliceRefs(), slices)\n\t}\n\tif _, err = p.Exec(ctx); err != nil {\n\t\treturn err\n\t}\n\n\t// update nlinks and parents for hardlinks\n\tst := make(map[Ino]int64)\n\tfor i, ps := range parents {\n\t\tif len(ps) > 1 {\n\t\t\ta, _ := m.rdb.Get(ctx, m.inodeKey(i)).Bytes()\n\t\t\t// reset nlink and parent\n\t\t\tbinary.BigEndian.PutUint32(a[47:51], uint32(len(ps))) // nlink\n\t\t\tbinary.BigEndian.PutUint64(a[63:71], 0)\n\t\t\tp.Set(ctx, m.inodeKey(i), a, 0)\n\t\t\tfor k := range st {\n\t\t\t\tdelete(st, k)\n\t\t\t}\n\t\t\tfor _, p := range ps {\n\t\t\t\tst[p] = st[p] + 1\n\t\t\t}\n\t\t\tfor parent, c := range st {\n\t\t\t\tp.HIncrBy(ctx, m.parentKey(i), parent.String(), c)\n\t\t\t}\n\t\t}\n\t}\n\t_, err = p.Exec(ctx)\n\treturn err\n}\n\nfunc (m *redisMeta) doCloneEntry(ctx Context, srcIno Ino, parent Ino, name string, ino Ino, originAttr *Attr, cmode uint8, cumask uint16, top bool) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\ta, err := tx.Get(ctx, m.inodeKey(srcIno)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, originAttr)\n\t\tattr := *originAttr\n\t\tif eno := m.Access(ctx, srcIno, MODE_MASK_R, &attr); eno != 0 {\n\t\t\treturn eno\n\t\t}\n\t\tattr.Parent = parent\n\t\tnow := time.Now()\n\t\tif cmode&CLONE_MODE_PRESERVE_ATTR == 0 {\n\t\t\tattr.Uid = ctx.Uid()\n\t\t\tattr.Gid = ctx.Gid()\n\t\t\tattr.Mode &= ^cumask\n\t\t\tattr.Atime = now.Unix()\n\t\t\tattr.Mtime = now.Unix()\n\t\t\tattr.Ctime = now.Unix()\n\t\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\t\tattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t}\n\t\t// TODO: preserve hardlink\n\t\tif attr.Typ == TypeFile && attr.Nlink > 1 {\n\t\t\tattr.Nlink = 1\n\t\t}\n\t\tsrcXattr, err := tx.HGetAll(ctx, m.xattrKey(srcIno)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tvar pattr Attr\n\t\tif top {\n\t\t\tif a, err := tx.Get(ctx, m.inodeKey(parent)).Bytes(); err != nil {\n\t\t\t\treturn err\n\t\t\t} else {\n\t\t\t\tm.parseAttr(a, &pattr)\n\t\t\t}\n\t\t\tif pattr.Typ != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t\tif (pattr.Flags & FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif exist, err := tx.HExists(ctx, m.entryKey(parent), name).Result(); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if exist {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\tif eno := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); eno != 0 {\n\t\t\t\treturn eno\n\t\t\t}\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(p redis.Pipeliner) error {\n\t\t\tp.Set(ctx, m.inodeKey(ino), m.marshal(&attr), 0)\n\t\t\tp.IncrBy(ctx, m.usedSpaceKey(), align4K(attr.Length))\n\t\t\tp.Incr(ctx, m.totalInodesKey())\n\t\t\tif len(srcXattr) > 0 {\n\t\t\t\tp.HMSet(ctx, m.xattrKey(ino), srcXattr)\n\t\t\t}\n\t\t\tif top && attr.Typ == TypeDirectory {\n\t\t\t\tp.ZAdd(ctx, m.detachedNodes(), redis.Z{Member: ino.String(), Score: float64(time.Now().Unix())})\n\t\t\t} else {\n\t\t\t\tp.HSet(ctx, m.entryKey(parent), name, m.packEntry(attr.Typ, ino))\n\t\t\t\tif top {\n\t\t\t\t\tnow := time.Now()\n\t\t\t\t\tpattr.Mtime = now.Unix()\n\t\t\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\t\t\tpattr.Ctime = now.Unix()\n\t\t\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\t\tp.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tswitch attr.Typ {\n\t\t\tcase TypeDirectory:\n\t\t\t\tsfield := srcIno.String()\n\t\t\t\tfield := ino.String()\n\t\t\t\tif v, err := tx.HGet(ctx, m.dirUsedInodesKey(), sfield).Result(); err == nil {\n\t\t\t\t\tp.HSet(ctx, m.dirUsedInodesKey(), field, v)\n\t\t\t\t\tp.HSet(ctx, m.dirDataLengthKey(), field, tx.HGet(ctx, m.dirDataLengthKey(), sfield).Val())\n\t\t\t\t\tp.HSet(ctx, m.dirUsedSpaceKey(), field, tx.HGet(ctx, m.dirUsedSpaceKey(), sfield).Val())\n\t\t\t\t}\n\t\t\tcase TypeFile:\n\t\t\t\t// copy chunks\n\t\t\t\tif attr.Length != 0 {\n\t\t\t\t\tvar vals [][]string\n\t\t\t\t\tfor i := 0; i <= int(attr.Length/ChunkSize); i++ {\n\t\t\t\t\t\tval, err := tx.LRange(ctx, m.chunkKey(srcIno, uint32(i)), 0, -1).Result()\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tvals = append(vals, val)\n\t\t\t\t\t}\n\n\t\t\t\t\tfor i, sv := range vals {\n\t\t\t\t\t\tif len(sv) == 0 {\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tss := readSlices(sv)\n\t\t\t\t\t\tif ss == nil {\n\t\t\t\t\t\t\treturn syscall.EIO\n\t\t\t\t\t\t}\n\t\t\t\t\t\tp.RPush(ctx, m.chunkKey(ino, uint32(i)), sv)\n\t\t\t\t\t\tfor _, s := range ss {\n\t\t\t\t\t\t\tif s.id > 0 {\n\t\t\t\t\t\t\t\tp.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.id, s.size), 1)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase TypeSymlink:\n\t\t\t\tpath, err := tx.Get(ctx, m.symKey(srcIno)).Result()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tp.Set(ctx, m.symKey(ino), path, 0)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(srcIno), m.xattrKey(srcIno)))\n}\n\nfunc (m *redisMeta) doBatchClone(ctx Context, srcParent Ino, dstParent Ino, entries []*Entry, cmode uint8, cumask uint16, result *batchCloneResult) syscall.Errno {\n\t// TODO: Implement batch clone for Redis backend\n\treturn syscall.ENOTSUP\n}\n\nfunc (m *redisMeta) doCleanupDetachedNode(ctx Context, ino Ino) syscall.Errno {\n\texists, err := m.rdb.Exists(ctx, m.inodeKey(ino)).Result()\n\tif err != nil || exists == 0 {\n\t\treturn errno(err)\n\t}\n\trmConcurrent := make(chan int, 10)\n\tif eno := m.emptyDir(ctx, ino, true, nil, rmConcurrent); eno != 0 {\n\t\treturn eno\n\t}\n\tm.updateStats(-align4K(0), -1)\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\t_, err := tx.TxPipelined(ctx, func(p redis.Pipeliner) error {\n\t\t\tp.Del(ctx, m.inodeKey(ino))\n\t\t\tp.Del(ctx, m.xattrKey(ino))\n\t\t\tp.DecrBy(ctx, m.usedSpaceKey(), align4K(0))\n\t\t\tp.Decr(ctx, m.totalInodesKey())\n\t\t\tfield := ino.String()\n\t\t\tp.HDel(ctx, m.dirUsedInodesKey(), field)\n\t\t\tp.HDel(ctx, m.dirDataLengthKey(), field)\n\t\t\tp.HDel(ctx, m.dirUsedSpaceKey(), field)\n\t\t\tp.ZRem(ctx, m.detachedNodes(), field)\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(ino), m.xattrKey(ino)))\n}\n\nfunc (m *redisMeta) doFindDetachedNodes(t time.Time) []Ino {\n\tvar inodes []Ino\n\tvals, err := m.rdb.ZRangeByScore(Background(), m.detachedNodes(), &redis.ZRangeBy{Min: \"-inf\", Max: strconv.FormatInt(t.Unix(), 10)}).Result()\n\tif err != nil {\n\t\tlogger.Errorf(\"Scan detached nodes error: %s\", err)\n\t\treturn nil\n\t}\n\tfor _, node := range vals {\n\t\tinode, _ := strconv.ParseUint(node, 10, 64)\n\t\tinodes = append(inodes, Ino(inode))\n\t}\n\treturn inodes\n}\n\nfunc (m *redisMeta) doAttachDirNode(ctx Context, parent Ino, dstIno Ino, name string) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tvar pattr Attr\n\t\ta, err := tx.Get(ctx, m.inodeKey(parent)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif (pattr.Flags & FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif tx.HExists(ctx, m.entryKey(parent), name).Val() {\n\t\t\treturn syscall.EEXIST\n\t\t}\n\n\t\t_, err = tx.TxPipelined(ctx, func(p redis.Pipeliner) error {\n\t\t\tp.HSet(ctx, m.entryKey(parent), name, m.packEntry(TypeDirectory, dstIno))\n\t\t\tpattr.Nlink++\n\t\t\tnow := time.Now()\n\t\t\tpattr.Mtime = now.Unix()\n\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tpattr.Ctime = now.Unix()\n\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tp.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)\n\t\t\tp.ZRem(ctx, m.detachedNodes(), dstIno.String())\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}, m.inodeKey(parent), m.entryKey(parent)))\n}\n\nfunc (m *redisMeta) doTouchAtime(ctx Context, inode Ino, attr *Attr, now time.Time) (bool, error) {\n\tvar updated bool\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\ta, err := tx.Get(ctx, m.inodeKey(inode)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(a, attr)\n\t\tif !m.atimeNeedsUpdate(attr, now) {\n\t\t\treturn nil\n\t\t}\n\t\tattr.Atime = now.Unix()\n\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\tif _, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)\n\t\t\treturn nil\n\t\t}); err == nil {\n\t\t\tupdated = true\n\t\t}\n\t\treturn err\n\t}, m.inodeKey(inode))\n\treturn updated, err\n}\n\nfunc (m *redisMeta) doSetFacl(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tval, err := tx.Get(ctx, m.inodeKey(ino)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tattr := &Attr{}\n\t\tm.parseAttr(val, attr)\n\n\t\tif ctx.Uid() != 0 && ctx.Uid() != attr.Uid {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tif attr.Flags&FlagImmutable != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\toriACL, oriMode := getAttrACLId(attr, aclType), attr.Mode\n\n\t\t// https://github.com/torvalds/linux/blob/480e035fc4c714fb5536e64ab9db04fedc89e910/fs/fuse/acl.c#L143-L151\n\t\t// TODO: check linux capabilities\n\t\tif ctx.Uid() != 0 && !inGroup(ctx, attr.Gid) {\n\t\t\t// clear sgid\n\t\t\tattr.Mode &= 05777\n\t\t}\n\n\t\tif rule.IsEmpty() {\n\t\t\t// remove acl\n\t\t\tsetAttrACLId(attr, aclType, aclAPI.None)\n\t\t} else if rule.IsMinimal() && aclType == aclAPI.TypeAccess {\n\t\t\t// remove acl\n\t\t\tsetAttrACLId(attr, aclType, aclAPI.None)\n\t\t\t// set mode\n\t\t\tattr.Mode &= 07000\n\t\t\tattr.Mode |= ((rule.Owner & 7) << 6) | ((rule.Group & 7) << 3) | (rule.Other & 7)\n\t\t} else {\n\t\t\trule.InheritPerms(attr.Mode)\n\t\t\taclId, err := m.insertACL(ctx, tx, rule)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsetAttrACLId(attr, aclType, aclId)\n\n\t\t\t// set mode\n\t\t\tif aclType == aclAPI.TypeAccess {\n\t\t\t\tattr.Mode &= 07000\n\t\t\t\tattr.Mode |= ((rule.Owner & 7) << 6) | ((rule.Mask & 7) << 3) | (rule.Other & 7)\n\t\t\t}\n\t\t}\n\n\t\t// update attr\n\t\tif oriACL != getAttrACLId(attr, aclType) || oriMode != attr.Mode {\n\t\t\tnow := time.Now()\n\t\t\tattr.Ctime = now.Unix()\n\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\tpipe.Set(ctx, m.inodeKey(ino), m.marshal(attr), 0)\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}, m.inodeKey(ino)))\n}\n\nfunc (m *redisMeta) doGetFacl(ctx Context, ino Ino, aclType uint8, aclId uint32, rule *aclAPI.Rule) syscall.Errno {\n\tif aclId == aclAPI.None {\n\t\tval, err := m.rdb.Get(ctx, m.inodeKey(ino)).Bytes()\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\t\tattr := &Attr{}\n\t\tm.parseAttr(val, attr)\n\t\tm.of.Update(ino, attr)\n\n\t\taclId = getAttrACLId(attr, aclType)\n\t}\n\n\ta, err := m.getACL(ctx, nil, aclId)\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tif a == nil {\n\t\treturn ENOATTR\n\t}\n\t*rule = *a\n\treturn 0\n}\n\nfunc (m *redisMeta) getACL(ctx Context, tx *redis.Tx, id uint32) (*aclAPI.Rule, error) {\n\tif id == aclAPI.None {\n\t\treturn nil, nil\n\t}\n\tif cRule := m.aclCache.Get(id); cRule != nil {\n\t\treturn cRule, nil\n\t}\n\n\tvar val []byte\n\tvar err error\n\tif tx != nil {\n\t\tval, err = tx.HGet(ctx, m.aclKey(), strconv.FormatUint(uint64(id), 10)).Bytes()\n\t} else {\n\t\tval, err = m.rdb.HGet(ctx, m.aclKey(), strconv.FormatUint(uint64(id), 10)).Bytes()\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif val == nil {\n\t\treturn nil, syscall.EIO\n\t}\n\n\trule := &aclAPI.Rule{}\n\trule.Decode(val)\n\tm.aclCache.Put(id, rule)\n\treturn rule, nil\n}\n\nfunc (m *redisMeta) insertACL(ctx Context, tx *redis.Tx, rule *aclAPI.Rule) (uint32, error) {\n\tif rule == nil || rule.IsEmpty() {\n\t\treturn aclAPI.None, nil\n\t}\n\n\tif err := m.tryLoadMissACLs(ctx, tx); err != nil {\n\t\tlogger.Warnf(\"SetFacl: load miss acls error: %s\", err)\n\t}\n\n\t// set acl\n\tvar aclId uint32\n\tif aclId = m.aclCache.GetId(rule); aclId == aclAPI.None {\n\t\t// TODO failures may result in some id wastage.\n\t\tnewId, err := m.incrCounter(aclCounter, 1)\n\t\tif err != nil {\n\t\t\treturn aclAPI.None, err\n\t\t}\n\t\taclId = uint32(newId)\n\n\t\tif err = tx.HSetNX(ctx, m.aclKey(), strconv.FormatUint(uint64(aclId), 10), rule.Encode()).Err(); err != nil {\n\t\t\treturn aclAPI.None, err\n\t\t}\n\t\tm.aclCache.Put(aclId, rule)\n\t}\n\treturn aclId, nil\n}\n\nfunc (m *redisMeta) tryLoadMissACLs(ctx Context, tx *redis.Tx) error {\n\tmissIds := m.aclCache.GetMissIds()\n\tif len(missIds) > 0 {\n\t\tmissKeys := make([]string, len(missIds))\n\t\tfor i, id := range missIds {\n\t\t\tmissKeys[i] = strconv.FormatUint(uint64(id), 10)\n\t\t}\n\n\t\tvals, err := tx.HMGet(ctx, m.aclKey(), missKeys...).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor i, data := range vals {\n\t\t\tvar rule aclAPI.Rule\n\t\t\tif data != nil {\n\t\t\t\trule.Decode([]byte(data.(string)))\n\t\t\t}\n\t\t\tm.aclCache.Put(missIds[i], &rule)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) loadDumpedACLs(ctx Context) error {\n\tid2Rule := m.aclCache.GetAll()\n\tif len(id2Rule) == 0 {\n\t\treturn nil\n\t}\n\n\treturn m.txn(ctx, func(tx *redis.Tx) error {\n\t\tmaxId := uint32(0)\n\t\tacls := make(map[string]interface{}, len(id2Rule))\n\t\tfor id, rule := range id2Rule {\n\t\t\tif id > maxId {\n\t\t\t\tmaxId = id\n\t\t\t}\n\t\t\tacls[strconv.FormatUint(uint64(id), 10)] = rule.Encode()\n\t\t}\n\t\tif err := tx.HSet(ctx, m.aclKey(), acls).Err(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn tx.Set(ctx, m.prefix+aclCounter, maxId, 0).Err()\n\t}, m.inodeKey(RootInode))\n}\n\nfunc (m *redisMeta) doStoreToken(ctx Context, token []byte) (id uint32, st syscall.Errno) {\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\tnewId, err := m.incrCounter(krbTokenCounter, 1)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\terr = tx.HSet(ctx, m.krbTokenKey(), strconv.FormatUint(uint64(newId), 10), token).Err()\n\t\tif err == nil {\n\t\t\tid = uint32(newId)\n\t\t}\n\t\treturn err\n\t}, m.krbTokenKey())\n\treturn id, errno(err)\n}\n\nfunc (m *redisMeta) doUpdateToken(ctx Context, id uint32, token []byte) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\texist, err := tx.HExists(ctx, m.krbTokenKey(), strconv.FormatUint(uint64(id), 10)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !exist {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\treturn tx.HSet(ctx, m.krbTokenKey(), strconv.FormatUint(uint64(id), 10), token).Err()\n\t}, m.krbTokenKey()))\n}\n\nfunc (m *redisMeta) doLoadToken(ctx Context, id uint32) (token []byte, st syscall.Errno) {\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\tval, err := tx.HGet(ctx, m.krbTokenKey(), strconv.FormatUint(uint64(id), 10)).Bytes()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif val == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\ttoken = val\n\t\treturn nil\n\t}, m.krbTokenKey())\n\treturn token, errno(err)\n}\n\nfunc (m *redisMeta) doDeleteTokens(ctx Context, ids []uint32) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *redis.Tx) error {\n\t\tstrIds := make([]string, len(ids))\n\t\tfor i, id := range ids {\n\t\t\tstrIds[i] = strconv.FormatUint(uint64(id), 10)\n\t\t}\n\t\treturn tx.HDel(ctx, m.krbTokenKey(), strIds...).Err()\n\t}, m.krbTokenKey()))\n}\n\nfunc (m *redisMeta) doListTokens(ctx Context) (tokens map[uint32][]byte, st syscall.Errno) {\n\ttokens = make(map[uint32][]byte)\n\terr := m.txn(ctx, func(tx *redis.Tx) error {\n\t\tvals, err := tx.HGetAll(ctx, m.krbTokenKey()).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor k, v := range vals {\n\t\t\tid, err := strconv.ParseUint(k, 10, 32)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"parse token id: %s: %v\", k, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttokens[uint32(id)] = []byte(v)\n\t\t}\n\t\treturn nil\n\t}, m.krbTokenKey())\n\treturn tokens, errno(err)\n}\n\nfunc (m *redisMeta) newDirHandler(inode Ino, plus bool, entries []*Entry) DirHandler {\n\treturn &redisDirHandler{\n\t\ten: m,\n\t\tinode: inode,\n\t\tplus: plus,\n\t\tinitEntries: entries,\n\t\tbatchNum: DirBatchNum[\"redis\"],\n\t}\n}\n\ntype redisDirHandler struct {\n\tsync.Mutex\n\tinode Ino\n\tplus bool\n\ten *redisMeta\n\tinitEntries []*Entry\n\tentries []*Entry\n\tindexes map[string]int\n\treadOff int\n\tbatchNum int\n}\n\nfunc (s *redisDirHandler) Close() {\n\ts.Lock()\n\ts.entries = nil\n\ts.readOff = 0\n\ts.Unlock()\n}\n\nfunc (s *redisDirHandler) Delete(name string) {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\tif len(s.entries) == 0 {\n\t\treturn\n\t}\n\n\tif idx, ok := s.indexes[name]; ok && idx >= s.readOff {\n\t\tdelete(s.indexes, name)\n\t\tn := len(s.entries)\n\t\tif idx < n-1 {\n\t\t\t// TODO: sorted\n\t\t\ts.entries[idx] = s.entries[n-1]\n\t\t\ts.indexes[string(s.entries[idx].Name)] = idx\n\t\t}\n\t\ts.entries = s.entries[:n-1]\n\t}\n}\n\nfunc (s *redisDirHandler) Insert(inode Ino, name string, attr *Attr) {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\tif len(s.entries) == 0 {\n\t\treturn\n\t}\n\n\t// TODO: sorted\n\ts.entries = append(s.entries, &Entry{Inode: inode, Name: []byte(name), Attr: attr})\n\ts.indexes[name] = len(s.entries) - 1\n}\n\nfunc (s *redisDirHandler) List(ctx Context, offset int) ([]*Entry, syscall.Errno) {\n\tvar prefix []*Entry\n\tif offset < len(s.initEntries) {\n\t\tprefix = s.initEntries[offset:]\n\t\toffset = 0\n\t} else {\n\t\toffset -= len(s.initEntries)\n\t}\n\n\ts.Lock()\n\tdefer s.Unlock()\n\tif s.entries == nil {\n\t\tvar entries []*Entry\n\t\terr := s.en.hscan(ctx, s.en.entryKey(s.inode), func(keys []string) error {\n\t\t\tnewEntries := make([]Entry, len(keys)/2)\n\t\t\tnewAttrs := make([]Attr, len(keys)/2)\n\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\ttyp, ino := s.en.parseEntry([]byte(keys[i+1]))\n\t\t\t\tif keys[i] == \"\" {\n\t\t\t\t\tlogger.Errorf(\"Corrupt entry with empty name: inode %d parent %d\", ino, s.inode)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tent := &newEntries[i/2]\n\t\t\t\tent.Inode = ino\n\t\t\t\tent.Name = []byte(keys[i])\n\t\t\t\tent.Attr = &newAttrs[i/2]\n\t\t\t\tent.Attr.Typ = typ\n\t\t\t\tentries = append(entries, ent)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, errno(err)\n\t\t}\n\n\t\tif s.en.conf.SortDir {\n\t\t\tsort.Slice(entries, func(i, j int) bool {\n\t\t\t\treturn string(entries[i].Name) < string(entries[j].Name)\n\t\t\t})\n\t\t}\n\t\tif s.plus {\n\t\t\tnEntries := len(entries)\n\t\t\tif nEntries <= s.batchNum {\n\t\t\t\terr = s.en.fillAttr(ctx, entries)\n\t\t\t} else {\n\t\t\t\teg := errgroup.Group{}\n\t\t\t\teg.SetLimit(2)\n\t\t\t\tfor i := 0; i < nEntries; i += s.batchNum {\n\t\t\t\t\tvar es []*Entry\n\t\t\t\t\tif i+s.batchNum > nEntries {\n\t\t\t\t\t\tes = entries[i:]\n\n\t\t\t\t\t} else {\n\t\t\t\t\t\tes = entries[i : i+s.batchNum]\n\t\t\t\t\t}\n\t\t\t\t\teg.Go(func() error {\n\t\t\t\t\t\treturn s.en.fillAttr(ctx, es)\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t\terr = eg.Wait()\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errno(err)\n\t\t\t}\n\t\t}\n\t\ts.entries = entries\n\n\t\tindexes := make(map[string]int, len(entries))\n\t\tfor i, e := range entries {\n\t\t\tindexes[string(e.Name)] = i\n\t\t}\n\t\ts.indexes = indexes\n\t}\n\n\tsize := len(s.entries) - offset\n\tif size > s.batchNum {\n\t\tsize = s.batchNum\n\t}\n\ts.readOff = offset + size\n\tentries := s.entries[offset : offset+size]\n\tif len(prefix) > 0 {\n\t\tentries = append(prefix, entries...)\n\t}\n\treturn entries, 0\n}\n\nfunc (s *redisDirHandler) Read(offset int) {\n\ts.readOff = offset - len(s.initEntries)\n}\n" }, { "path": "pkg/meta/redis_bak.go", "content": "//go:build !noredis\n// +build !noredis\n\n/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta/pb\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"golang.org/x/sync/errgroup\"\n\t\"google.golang.org/protobuf/proto\"\n)\n\nvar (\n\tredisBatchSize = 10000\n\tredisPipeLimit = 1000\n)\n\nfunc (m *redisMeta) dump(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar dumps = []func(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error{\n\t\tm.dumpFormat,\n\t\tm.dumpCounters,\n\t\tm.dumpMix, // node, edge, chunk, symlink, xattr, parent\n\t\tm.dumpSustained,\n\t\tm.dumpDelFiles,\n\t\tm.dumpSliceRef,\n\t\tm.dumpACL,\n\t\tm.dumpQuota,\n\t\tm.dumpDirStat,\n\t}\n\tfor _, f := range dumps {\n\t\terr := f(ctx, opt, ch)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *redisMeta) dumpCounters(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tcounters := make([]*pb.Counter, 0, len(counterNames))\n\tfor _, name := range counterNames {\n\t\tcnt, err := m.getCounter(name)\n\t\tif err != nil {\n\t\t\treturn errors.Wrapf(err, \"get counter %s\", name)\n\t\t}\n\t\tif name == \"nextInode\" || name == \"nextChunk\" {\n\t\t\tcnt++ // Redis nextInode/nextChunk is one smaller than db\n\t\t}\n\t\tcounters = append(counters, &pb.Counter{Key: name, Value: cnt})\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Counters: counters}})\n}\n\nfunc (m *redisMeta) dumpMix(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tlogger.Warnf(\"please make sure the redis server is readonly, otherwise the dumped metadata will be inconsistent\")\n\tpools := map[int][]*sync.Pool{\n\t\tsegTypeNode: {{New: func() interface{} { return &pb.Node{} }}},\n\t\tsegTypeEdge: {{New: func() interface{} { return &pb.Edge{} }}},\n\t\tsegTypeChunk: {{New: func() interface{} { return &pb.Chunk{} }}, {New: func() interface{} { return make([]byte, 8*sliceBytes) }}},\n\t\tsegTypeSymlink: {{New: func() interface{} { return &pb.Symlink{} }}},\n\t\tsegTypeXattr: {{New: func() interface{} { return &pb.Xattr{} }}},\n\t\tsegTypeParent: {{New: func() interface{} { return &pb.Parent{} }}},\n\t}\n\trelease := func(p proto.Message) {\n\t\tb := p.(*pb.Batch)\n\t\tfor _, n := range b.Nodes {\n\t\t\tpools[segTypeNode][0].Put(n)\n\t\t}\n\t\tfor _, e := range b.Edges {\n\t\t\tpools[segTypeEdge][0].Put(e)\n\t\t}\n\t\tfor _, c := range b.Chunks {\n\t\t\tpools[segTypeChunk][1].Put(c.Slices) // nolint:staticcheck\n\t\t\tc.Slices = nil\n\t\t\tpools[segTypeChunk][0].Put(c)\n\t\t}\n\t\tfor _, s := range b.Symlinks {\n\t\t\tpools[segTypeSymlink][0].Put(s)\n\t\t}\n\t\tfor _, x := range b.Xattrs {\n\t\t\tpools[segTypeXattr][0].Put(x)\n\t\t}\n\t\tfor _, p := range b.Parents {\n\t\t\tpools[segTypeParent][0].Put(p)\n\t\t}\n\t}\n\tchar2Typ := map[byte]int{\n\t\t'i': segTypeNode,\n\t\t'd': segTypeEdge,\n\t\t'c': segTypeChunk,\n\t\t's': segTypeSymlink,\n\t\t'x': segTypeXattr,\n\t\t'p': segTypeParent,\n\t}\n\ttyp2Limit := map[int]int{\n\t\tsegTypeNode: redisBatchSize,\n\t\tsegTypeEdge: redisBatchSize,\n\t\tsegTypeChunk: redisPipeLimit,\n\t\tsegTypeSymlink: redisBatchSize,\n\t\tsegTypeXattr: redisPipeLimit,\n\t\tsegTypeParent: redisPipeLimit,\n\t}\n\tvar typ2Keys = make(map[int][]string, len(typ2Limit))\n\tfor typ, limit := range typ2Limit {\n\t\ttyp2Keys[typ] = make([]string, 0, limit)\n\t}\n\n\tvar sums = map[int]*atomic.Uint64{\n\t\tsegTypeNode: {},\n\t\tsegTypeEdge: {},\n\t\tsegTypeChunk: {},\n\t\tsegTypeSymlink: {},\n\t\tsegTypeXattr: {},\n\t\tsegTypeParent: {},\n\t}\n\ttyp2Handles := map[int]func(ctx context.Context, ch chan<- *dumpedResult, keys []string, pools []*sync.Pool, rel func(p proto.Message), sum *atomic.Uint64) error{\n\t\tsegTypeNode: m.dumpNodes,\n\t\tsegTypeEdge: m.dumpEdges,\n\t\tsegTypeChunk: m.dumpChunks,\n\t\tsegTypeSymlink: m.dumpSymlinks,\n\t\tsegTypeXattr: m.dumpXattrs,\n\t\tsegTypeParent: m.dumpParents,\n\t}\n\n\teg, egCtx := errgroup.WithContext(ctx)\n\teg.SetLimit(opt.Threads)\n\n\tkeyCh := make(chan []string, opt.Threads*2)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tvar keys []string\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase keys = <-keyCh:\n\t\t\t}\n\t\t\tif keys == nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tfor _, key := range keys {\n\t\t\t\tif typ, ok := char2Typ[key[len(m.prefix)]]; ok {\n\t\t\t\t\ttyp2Keys[typ] = append(typ2Keys[typ], key)\n\t\t\t\t\tif len(typ2Keys[typ]) >= typ2Limit[typ] {\n\t\t\t\t\t\tiPools, sum, keys := pools[typ], sums[typ], typ2Keys[typ]\n\t\t\t\t\t\teg.Go(func() error {\n\t\t\t\t\t\t\treturn typ2Handles[typ](ctx, ch, keys, iPools, release, sum)\n\t\t\t\t\t\t})\n\t\t\t\t\t\ttyp2Keys[typ] = make([]string, 0, typ2Limit[typ])\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor typ, keys := range typ2Keys {\n\t\t\tif len(keys) > 0 {\n\t\t\t\tiKeys, iTyp := keys, typ\n\t\t\t\teg.Go(func() error {\n\t\t\t\t\treturn typ2Handles[iTyp](ctx, ch, iKeys, pools[iTyp], release, sums[iTyp])\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}()\n\n\tif err := m.scan(egCtx, \"*\", func(sKeys []string) error {\n\t\tkeyCh <- sKeys\n\t\treturn nil\n\t}); err != nil {\n\t\tctx.Cancel()\n\t\twg.Wait()\n\t\t_ = eg.Wait()\n\t\treturn err\n\t}\n\n\tclose(keyCh)\n\twg.Wait()\n\tif err := eg.Wait(); err != nil {\n\t\treturn err\n\t}\n\n\tlogger.Debugf(\"dump result: %s\", printSums(sums))\n\treturn nil\n}\n\nfunc (m *redisMeta) dumpSustained(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tkeys, err := m.rdb.ZRange(ctx, m.allSessions(), 0, -1).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tsustained := make([]*pb.Sustained, 0, len(keys))\n\tfor _, k := range keys {\n\t\tsid, _ := strconv.ParseUint(k, 10, 64)\n\t\tvar ss []string\n\t\tss, err = m.rdb.SMembers(ctx, m.sustained(sid)).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(ss) > 0 {\n\t\t\tinodes := make([]uint64, 0, len(ss))\n\t\t\tfor _, s := range ss {\n\t\t\t\tinode, _ := strconv.ParseUint(s, 10, 64)\n\t\t\t\tinodes = append(inodes, inode)\n\t\t\t}\n\t\t\tsustained = append(sustained, &pb.Sustained{Sid: sid, Inodes: inodes})\n\t\t}\n\t}\n\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Sustained: sustained}})\n}\n\nfunc (m *redisMeta) dumpDelFiles(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tzs, err := m.rdb.ZRangeWithScores(ctx, m.delfiles(), 0, -1).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdelFiles := make([]*pb.DelFile, 0, min(len(zs), redisBatchSize))\n\tfor i, z := range zs {\n\t\tparts := strings.Split(z.Member.(string), \":\")\n\t\tif len(parts) != 2 {\n\t\t\tlogger.Warnf(\"invalid delfile string: %s\", z.Member.(string))\n\t\t\tcontinue\n\t\t}\n\t\tinode, _ := strconv.ParseUint(parts[0], 10, 64)\n\t\tlength, _ := strconv.ParseUint(parts[1], 10, 64)\n\t\tdelFiles = append(delFiles, &pb.DelFile{Inode: inode, Length: length, Expire: int64(z.Score)})\n\t\tif len(delFiles) >= redisBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Delfiles: delFiles}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdelFiles = make([]*pb.DelFile, 0, min(len(zs)-i-1, redisBatchSize))\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Delfiles: delFiles}})\n}\n\nfunc (m *redisMeta) dumpSliceRef(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tsliceRefs := make([]*pb.SliceRef, 0, 1024)\n\tvar key string\n\tvar val int\n\tvar inErr error\n\tif err := m.hscan(ctx, m.sliceRefs(), func(keys []string) error {\n\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\tkey = keys[i]\n\t\t\tval, inErr = strconv.Atoi(keys[i+1])\n\t\t\tif inErr != nil {\n\t\t\t\tlogger.Errorf(\"invalid value: %s\", keys[i+1])\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif val >= 1 {\n\t\t\t\tps := strings.Split(key, \"_\")\n\t\t\t\tif len(ps) == 2 {\n\t\t\t\t\tid, _ := strconv.ParseUint(ps[0][1:], 10, 64)\n\t\t\t\t\tsize, _ := strconv.ParseUint(ps[1], 10, 32)\n\t\t\t\t\tsr := &pb.SliceRef{Id: id, Size: uint32(size), Refs: int64(val) + 1} // Redis sliceRef is one smaller than sql\n\t\t\t\t\tsliceRefs = append(sliceRefs, sr)\n\t\t\t\t\tif len(sliceRefs) >= redisBatchSize {\n\t\t\t\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{SliceRefs: sliceRefs}}); err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsliceRefs = make([]*pb.SliceRef, 0, 1024)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{SliceRefs: sliceRefs}})\n}\n\nfunc (m *redisMeta) dumpACL(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvals, err := m.rdb.HGetAll(ctx, m.aclKey()).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tacls := make([]*pb.Acl, 0, len(vals))\n\tfor k, v := range vals {\n\t\tid, _ := strconv.ParseUint(k, 10, 32)\n\t\tacls = append(acls, &pb.Acl{\n\t\t\tId: uint32(id),\n\t\t\tData: []byte(v),\n\t\t})\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Acls: acls}})\n}\n\nfunc (m *redisMeta) dumpQuota(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tquotas := make(map[Ino]*pb.Quota)\n\tvals, err := m.rdb.HGetAll(ctx, m.dirQuotaKey()).Result()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"get dirQuotaKey err: %w\", err)\n\t}\n\tfor k, v := range vals {\n\t\tinode, err := strconv.ParseUint(k, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse quota inode: %s: %v\", k, err)\n\t\t\tcontinue\n\t\t}\n\t\tif len(v) != 16 {\n\t\t\tlogger.Warnf(\"invalid quota string: %s\", hex.EncodeToString([]byte(v)))\n\t\t\tcontinue\n\t\t}\n\t\tspace, inodes := m.parseQuota([]byte(v))\n\t\tquotas[Ino(inode)] = &pb.Quota{\n\t\t\tInode: inode,\n\t\t\tMaxSpace: space,\n\t\t\tMaxInodes: inodes,\n\t\t}\n\t}\n\n\tvals, err = m.rdb.HGetAll(ctx, m.dirQuotaUsedInodesKey()).Result()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"get dirQuotaUsedInodesKey err: %w\", err)\n\t}\n\tfor k, v := range vals {\n\t\tinode, err := strconv.ParseUint(k, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse used inodes inode: %s: %v\", k, err)\n\t\t\tcontinue\n\t\t}\n\t\tif q, ok := quotas[Ino(inode)]; !ok {\n\t\t\tlogger.Warnf(\"quota for used inodes not found: %d\", inode)\n\t\t} else {\n\t\t\tq.UsedInodes, _ = strconv.ParseInt(v, 10, 64)\n\t\t}\n\t}\n\n\tvals, err = m.rdb.HGetAll(ctx, m.dirQuotaUsedSpaceKey()).Result()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"get dirQuotaUsedSpaceKey err: %w\", err)\n\t}\n\tfor k, v := range vals {\n\t\tinode, err := strconv.ParseUint(k, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse used space inode: %s: %v\", k, err)\n\t\t\tcontinue\n\t\t}\n\t\tif q, ok := quotas[Ino(inode)]; !ok {\n\t\t\tlogger.Warnf(\"quota for used space not found: %d\", inode)\n\t\t} else {\n\t\t\tq.UsedSpace, _ = strconv.ParseInt(v, 10, 64)\n\t\t}\n\t}\n\n\tqs := make([]*pb.Quota, 0, len(quotas))\n\tfor _, q := range quotas {\n\t\tqs = append(qs, q)\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Quotas: qs}})\n}\n\nfunc (m *redisMeta) dumpDirStat(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tstats := make(map[Ino]*pb.Stat)\n\tvals, err := m.rdb.HGetAll(ctx, m.dirDataLengthKey()).Result()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"get dirDataLengthKey err: %w\", err)\n\t}\n\tfor k, v := range vals {\n\t\tinode, err := strconv.ParseUint(k, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse length stat inode: %s: %v\", k, err)\n\t\t\tcontinue\n\t\t}\n\t\tlength, _ := strconv.ParseInt(v, 10, 64)\n\t\tstats[Ino(inode)] = &pb.Stat{\n\t\t\tInode: inode,\n\t\t\tDataLength: length,\n\t\t}\n\t}\n\n\tvals, err = m.rdb.HGetAll(ctx, m.dirUsedInodesKey()).Result()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"get dirUsedInodesKey err: %w\", err)\n\t}\n\tfor k, v := range vals {\n\t\tinode, err := strconv.ParseUint(k, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse inodes stat inode: %s: %v\", k, err)\n\t\t\tcontinue\n\t\t}\n\t\tinodes, _ := strconv.ParseInt(v, 10, 64)\n\t\tif q, ok := stats[Ino(inode)]; !ok {\n\t\t\tlogger.Warnf(\"stat for used inodes not found: %d\", inode)\n\t\t} else {\n\t\t\tq.UsedInodes = inodes\n\t\t}\n\t}\n\n\tvals, err = m.rdb.HGetAll(ctx, m.dirUsedSpaceKey()).Result()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"get dirUsedSpaceKey err: %w\", err)\n\t}\n\tfor k, v := range vals {\n\t\tinode, err := strconv.ParseUint(k, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse space stat inode: %s: %v\", k, err)\n\t\t\tcontinue\n\t\t}\n\t\tspace, _ := strconv.ParseInt(v, 10, 64)\n\t\tif q, ok := stats[Ino(inode)]; !ok {\n\t\t\tlogger.Warnf(\"stat for used space not found: %d\", inode)\n\t\t} else {\n\t\t\tq.UsedSpace = space\n\t\t}\n\t}\n\n\tss := make([]*pb.Stat, 0, min(len(stats), redisBatchSize))\n\tcnt := 0\n\tfor _, s := range stats {\n\t\tcnt++\n\t\tss = append(ss, s)\n\t\tif len(ss) >= redisBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Dirstats: ss}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tss = make([]*pb.Stat, 0, min(len(stats)-cnt, redisBatchSize))\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Dirstats: ss}})\n}\n\nfunc (m *redisMeta) dumpNodes(ctx context.Context, ch chan<- *dumpedResult, keys []string, pools []*sync.Pool, rel func(p proto.Message), sum *atomic.Uint64) error {\n\tvals, err := m.rdb.MGet(ctx, keys...).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\tnodes := make([]*pb.Node, 0, len(vals))\n\tvar inode uint64\n\tfor idx, v := range vals {\n\t\tif v == nil {\n\t\t\tcontinue\n\t\t}\n\t\tinode, _ = strconv.ParseUint(keys[idx][len(m.prefix)+1:], 10, 64)\n\t\tnode := pools[0].Get().(*pb.Node)\n\t\tnode.Inode = inode\n\t\tnode.Data = []byte(v.(string))\n\t\tnodes = append(nodes, node)\n\t}\n\tsum.Add(uint64(len(nodes)))\n\treturn dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Nodes: nodes}, rel})\n}\n\nfunc (m *redisMeta) dumpEdges(ctx context.Context, ch chan<- *dumpedResult, keys []string, pools []*sync.Pool, rel func(p proto.Message), sum *atomic.Uint64) error {\n\tedges := make([]*pb.Edge, 0, redisBatchSize)\n\tfor _, key := range keys {\n\t\tparent, _ := strconv.ParseUint(key[len(m.prefix)+1:], 10, 64)\n\t\tvar pe *pb.Edge\n\t\tif err := m.hscan(ctx, m.entryKey(Ino(parent)), func(keys []string) error {\n\t\t\tfor i := 0; i < len(keys); i += 2 {\n\t\t\t\tpe = pools[0].Get().(*pb.Edge)\n\t\t\t\tpe.Parent = parent\n\t\t\t\tpe.Name = []byte(keys[i])\n\t\t\t\ttyp, ino := m.parseEntry([]byte(keys[i+1]))\n\t\t\t\tpe.Type, pe.Inode = uint32(typ), uint64(ino)\n\t\t\t\tedges = append(edges, pe)\n\n\t\t\t\tif len(edges) >= redisBatchSize {\n\t\t\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Edges: edges}, rel}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tsum.Add(uint64(len(edges)))\n\t\t\t\t\tedges = make([]*pb.Edge, 0, redisBatchSize)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tsum.Add(uint64(len(edges)))\n\treturn dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Edges: edges}, rel})\n}\n\nfunc (m *redisMeta) dumpChunks(ctx context.Context, ch chan<- *dumpedResult, keys []string, pools []*sync.Pool, rel func(p proto.Message), sum *atomic.Uint64) error {\n\tpipe := m.rdb.Pipeline()\n\tinos := make([]uint64, 0, len(keys))\n\tidxs := make([]uint32, 0, len(keys))\n\tfor _, key := range keys {\n\t\tps := strings.Split(key, \"_\")\n\t\tif len(ps) != 2 {\n\t\t\tlogger.Warnf(\"invalid chunk key: %s\", key)\n\t\t\tcontinue\n\t\t}\n\t\tino, _ := strconv.ParseUint(ps[0][len(m.prefix)+1:], 10, 64)\n\t\tidx, _ := strconv.ParseUint(ps[1], 10, 32)\n\t\tpipe.LRange(ctx, m.chunkKey(Ino(ino), uint32(idx)), 0, -1)\n\t\tinos = append(inos, ino)\n\t\tidxs = append(idxs, uint32(idx))\n\t}\n\n\tcmds, err := pipe.Exec(ctx)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"chunk pipeline exec err: %w\", err)\n\t}\n\n\tchunks := make([]*pb.Chunk, 0, len(cmds))\n\tfor k, cmd := range cmds {\n\t\tvals, err := cmd.(*redis.StringSliceCmd).Result()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"get chunk result err: %w\", err)\n\t\t}\n\t\tif len(vals) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpc := pools[0].Get().(*pb.Chunk)\n\t\tpc.Inode = inos[k]\n\t\tpc.Index = idxs[k]\n\n\t\tpc.Slices = pools[1].Get().([]byte)\n\t\tif len(pc.Slices) < len(vals)*sliceBytes {\n\t\t\tpc.Slices = make([]byte, len(vals)*sliceBytes)\n\t\t}\n\t\tpc.Slices = pc.Slices[:len(vals)*sliceBytes]\n\n\t\tfor i, val := range vals {\n\t\t\tif len(val) != sliceBytes {\n\t\t\t\tlogger.Errorf(\"corrupt slice: len=%d, val=%v\", len(val), []byte(val))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcopy(pc.Slices[i*sliceBytes:], []byte(val))\n\t\t}\n\t\tchunks = append(chunks, pc)\n\t}\n\tsum.Add(uint64(len(chunks)))\n\treturn dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Chunks: chunks}, rel})\n}\n\nfunc (m *redisMeta) dumpSymlinks(ctx context.Context, ch chan<- *dumpedResult, keys []string, pools []*sync.Pool, rel func(p proto.Message), sum *atomic.Uint64) error {\n\tvals, err := m.rdb.MGet(ctx, keys...).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\tsyms := make([]*pb.Symlink, 0, len(vals))\n\tvar ps *pb.Symlink\n\tfor idx, v := range vals {\n\t\tif v == nil {\n\t\t\tcontinue\n\t\t}\n\t\tps = pools[0].Get().(*pb.Symlink)\n\t\tps.Inode, err = strconv.ParseUint(keys[idx][len(m.prefix)+1:], 10, 64)\n\t\tif err != nil {\n\t\t\tcontinue // key \"setting\"\n\t\t}\n\t\tps.Target = unescape(v.(string))\n\t\tsyms = append(syms, ps)\n\t}\n\n\tsum.Add(uint64(len(syms)))\n\treturn dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Symlinks: syms}, rel})\n}\n\nfunc (m *redisMeta) dumpXattrs(ctx context.Context, ch chan<- *dumpedResult, keys []string, pools []*sync.Pool, rel func(p proto.Message), sum *atomic.Uint64) error {\n\txattrs := make([]*pb.Xattr, 0, len(keys))\n\tpipe := m.rdb.Pipeline()\n\tfor _, key := range keys {\n\t\tpipe.HGetAll(ctx, key)\n\t}\n\tcmds, err := pipe.Exec(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar xattr *pb.Xattr\n\tfor idx, cmd := range cmds {\n\t\tinode, _ := strconv.ParseUint(keys[idx][len(m.prefix)+1:], 10, 64)\n\t\tres, err := cmd.(*redis.MapStringStringCmd).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor k, v := range res {\n\t\t\txattr = pools[0].Get().(*pb.Xattr)\n\t\t\txattr.Inode = inode\n\t\t\txattr.Name = k\n\t\t\txattr.Value = []byte(v)\n\t\t\txattrs = append(xattrs, xattr)\n\t\t}\n\t}\n\tsum.Add(uint64(len(xattrs)))\n\treturn dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Xattrs: xattrs}, rel})\n}\n\nfunc (m *redisMeta) dumpParents(ctx context.Context, ch chan<- *dumpedResult, keys []string, pools []*sync.Pool, rel func(p proto.Message), sum *atomic.Uint64) error {\n\tparents := make([]*pb.Parent, 0, len(keys))\n\tpipe := m.rdb.Pipeline()\n\tfor _, key := range keys {\n\t\tpipe.HGetAll(ctx, key)\n\t}\n\tcmds, err := pipe.Exec(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar pp *pb.Parent\n\tfor idx, cmd := range cmds {\n\t\tinode, _ := strconv.ParseUint(keys[idx][len(m.prefix)+1:], 10, 64)\n\t\tres, err := cmd.(*redis.MapStringStringCmd).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor k, v := range res {\n\t\t\tpp = pools[0].Get().(*pb.Parent)\n\t\t\tparent, _ := strconv.ParseUint(k, 10, 64)\n\t\t\tcnt, _ := strconv.ParseInt(v, 10, 64)\n\n\t\t\tpp.Inode = inode\n\t\t\tpp.Parent = parent\n\t\t\tpp.Cnt = cnt\n\t\t\tparents = append(parents, pp)\n\t\t}\n\t}\n\tsum.Add(uint64(len(parents)))\n\treturn dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Parents: parents}, rel})\n}\n\nfunc (m *redisMeta) load(ctx Context, typ int, opt *LoadOption, val proto.Message) error {\n\tswitch typ {\n\tcase segTypeFormat:\n\t\treturn m.loadFormat(ctx, val)\n\tcase segTypeCounter:\n\t\treturn m.loadCounters(ctx, val)\n\tcase segTypeNode:\n\t\treturn m.loadNodes(ctx, val)\n\tcase segTypeChunk:\n\t\treturn m.loadChunks(ctx, val)\n\tcase segTypeEdge:\n\t\treturn m.loadEdges(ctx, val)\n\tcase segTypeSymlink:\n\t\treturn m.loadSymlinks(ctx, val)\n\tcase segTypeSustained:\n\t\treturn m.loadSustained(ctx, val)\n\tcase segTypeDelFile:\n\t\treturn m.loadDelFiles(ctx, val)\n\tcase segTypeSliceRef:\n\t\treturn m.loadSliceRefs(ctx, val)\n\tcase segTypeAcl:\n\t\treturn m.loadAcl(ctx, val)\n\tcase segTypeXattr:\n\t\treturn m.loadXattrs(ctx, val)\n\tcase segTypeQuota:\n\t\treturn m.loadQuota(ctx, val)\n\tcase segTypeStat:\n\t\treturn m.loadDirStats(ctx, val)\n\tcase segTypeParent:\n\t\treturn m.loadParents(ctx, val)\n\tdefault:\n\t\tlogger.Warnf(\"skip segment type %d\", typ)\n\t\treturn nil\n\t}\n}\n\nfunc execPipe(ctx context.Context, pipe redis.Pipeliner) error {\n\tif pipe.Len() == 0 {\n\t\treturn nil\n\t}\n\tcmds, err := pipe.Exec(ctx)\n\tif err != nil {\n\t\tfor i, cmd := range cmds {\n\t\t\tif cmd.Err() != nil {\n\t\t\t\treturn fmt.Errorf(\"failed command %d %+v: %w\", i, cmd, cmd.Err())\n\t\t\t}\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *redisMeta) loadFormat(ctx Context, msg proto.Message) error {\n\treturn m.rdb.Set(ctx, m.setting(), msg.(*pb.Format).Data, 0).Err()\n}\n\nfunc (m *redisMeta) loadCounters(ctx Context, msg proto.Message) error {\n\tcs := make(map[string]interface{})\n\n\tfor _, c := range msg.(*pb.Batch).Counters {\n\t\tif c.Key == \"nextInode\" || c.Key == \"nextChunk\" {\n\t\t\tcs[m.counterKey(c.Key)] = c.Value - 1\n\t\t} else {\n\t\t\tcs[m.counterKey(c.Key)] = c.Value\n\t\t}\n\t}\n\treturn m.rdb.MSet(ctx, cs).Err()\n}\n\nfunc (m *redisMeta) loadNodes(ctx Context, msg proto.Message) error {\n\tbatch := msg.(*pb.Batch)\n\tpipe := m.rdb.Pipeline()\n\tfor _, pn := range batch.Nodes {\n\t\tpipe.Set(ctx, m.inodeKey(Ino(pn.Inode)), pn.Data, 0)\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) loadEdges(ctx Context, msg proto.Message) error {\n\tbatch := msg.(*pb.Batch)\n\tpipe := m.rdb.Pipeline()\n\tfor _, edge := range batch.Edges {\n\t\tbuff := utils.NewBuffer(9)\n\t\tbuff.Put8(uint8(edge.Type))\n\t\tbuff.Put64(edge.Inode)\n\t\tpipe.HSet(ctx, m.entryKey(Ino(edge.Parent)), edge.Name, buff.Bytes())\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) loadChunks(ctx Context, msg proto.Message) error {\n\tbatch := msg.(*pb.Batch)\n\tpipe := m.rdb.Pipeline()\n\tfor _, chk := range batch.Chunks {\n\t\tslices := make([]string, 0, len(chk.Slices))\n\t\tfor off := 0; off < len(chk.Slices); off += sliceBytes {\n\t\t\tslices = append(slices, string(chk.Slices[off:off+sliceBytes]))\n\t\t}\n\t\tpipe.RPush(ctx, m.chunkKey(Ino(chk.Inode), chk.Index), slices)\n\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) loadSymlinks(ctx Context, msg proto.Message) error {\n\tsyms := make(map[string]interface{}, redisBatchSize)\n\tfor _, ps := range msg.(*pb.Batch).Symlinks {\n\t\tsyms[m.symKey(Ino(ps.Inode))] = ps.Target\n\n\t\tif len(syms) >= redisBatchSize {\n\t\t\tif err := m.rdb.MSet(ctx, syms).Err(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor k := range syms {\n\t\t\t\tdelete(syms, k)\n\t\t\t}\n\t\t}\n\t}\n\tif len(syms) == 0 {\n\t\treturn nil\n\t}\n\treturn m.rdb.MSet(ctx, syms).Err()\n}\n\nfunc (m *redisMeta) loadSustained(ctx Context, msg proto.Message) error {\n\tpipe := m.rdb.Pipeline()\n\tfor _, ps := range msg.(*pb.Batch).Sustained {\n\t\tinodes := make([]interface{}, len(ps.Inodes))\n\t\tfor i, inode := range ps.Inodes {\n\t\t\tinodes[i] = inode\n\t\t}\n\t\tpipe.SAdd(ctx, m.sustained(ps.Sid), inodes...)\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) loadDelFiles(ctx Context, msg proto.Message) error {\n\tbatch := msg.(*pb.Batch)\n\tmbs := make([]redis.Z, 0, len(batch.Delfiles))\n\tfor _, pd := range batch.Delfiles {\n\t\tmbs = append(mbs, redis.Z{\n\t\t\tScore: float64(pd.Expire),\n\t\t\tMember: m.toDelete(Ino(pd.Inode), pd.Length),\n\t\t})\n\t}\n\tif len(mbs) == 0 {\n\t\treturn nil\n\t}\n\treturn m.rdb.ZAdd(ctx, m.delfiles(), mbs...).Err()\n}\n\nfunc (m *redisMeta) loadSliceRefs(ctx Context, msg proto.Message) error {\n\tslices := make(map[string]interface{})\n\tfor _, p := range msg.(*pb.Batch).SliceRefs {\n\t\tslices[m.sliceKey(p.Id, p.Size)] = strconv.Itoa(int(p.Refs - 1))\n\t}\n\tif len(slices) == 0 {\n\t\treturn nil\n\t}\n\treturn m.rdb.HSet(ctx, m.sliceRefs(), slices).Err()\n}\n\nvar loadLock sync.Mutex\nvar maxAclId uint32\n\nfunc (m *redisMeta) loadAcl(ctx Context, msg proto.Message) error {\n\tbatch := msg.(*pb.Batch)\n\tacls := make(map[string]interface{}, len(batch.Acls))\n\tfor _, pa := range batch.Acls {\n\t\tloadLock.Lock()\n\t\tif pa.Id > maxAclId {\n\t\t\tmaxAclId = pa.Id\n\t\t}\n\t\tloadLock.Unlock()\n\t\tacls[strconv.FormatUint(uint64(pa.Id), 10)] = pa.Data\n\t}\n\tif len(acls) == 0 {\n\t\treturn nil\n\t}\n\n\tif err := m.rdb.HSet(ctx, m.aclKey(), acls).Err(); err != nil {\n\t\treturn err\n\t}\n\treturn m.rdb.Set(ctx, m.counterKey(aclCounter), maxAclId, 0).Err()\n}\n\nfunc (m *redisMeta) loadXattrs(ctx Context, msg proto.Message) error {\n\tpipe := m.rdb.Pipeline()\n\txm := make(map[uint64]map[string]interface{}) // {inode: {name: value}}\n\tfor _, px := range msg.(*pb.Batch).Xattrs {\n\t\tif _, ok := xm[px.Inode]; !ok {\n\t\t\txm[px.Inode] = make(map[string]interface{})\n\t\t}\n\t\txm[px.Inode][px.Name] = px.Value\n\t}\n\n\tfor inode, xattrs := range xm {\n\t\tpipe.HSet(ctx, m.xattrKey(Ino(inode)), xattrs)\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) loadQuota(ctx Context, msg proto.Message) error {\n\tpipe := m.rdb.Pipeline()\n\tvar inodeKey string\n\tfor _, pq := range msg.(*pb.Batch).Quotas {\n\t\tinodeKey = Ino(pq.Inode).String()\n\t\tpipe.HSet(ctx, m.dirQuotaKey(), inodeKey, m.packQuota(pq.MaxSpace, pq.MaxInodes))\n\t\tpipe.HSet(ctx, m.dirQuotaUsedInodesKey(), inodeKey, pq.UsedInodes)\n\t\tpipe.HSet(ctx, m.dirQuotaUsedSpaceKey(), inodeKey, pq.UsedSpace)\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) loadDirStats(ctx Context, msg proto.Message) error {\n\tpipe := m.rdb.Pipeline()\n\tvar inodeKey string\n\tfor _, ps := range msg.(*pb.Batch).Dirstats {\n\t\tinodeKey = Ino(ps.Inode).String()\n\t\tpipe.HSet(ctx, m.dirDataLengthKey(), inodeKey, ps.DataLength)\n\t\tpipe.HSet(ctx, m.dirUsedInodesKey(), inodeKey, ps.UsedInodes)\n\t\tpipe.HSet(ctx, m.dirUsedSpaceKey(), inodeKey, ps.UsedSpace)\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) loadParents(ctx Context, msg proto.Message) error {\n\tpipe := m.rdb.Pipeline()\n\tfor _, p := range msg.(*pb.Batch).Parents {\n\t\tpipe.HIncrBy(ctx, m.parentKey(Ino(p.Inode)), Ino(p.Parent).String(), p.Cnt)\n\t\tif pipe.Len() >= redisPipeLimit {\n\t\t\tif err := execPipe(ctx, pipe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn execPipe(ctx, pipe)\n}\n\nfunc (m *redisMeta) prepareLoad(ctx Context, opt *LoadOption) error {\n\topt.check()\n\tif _, ok := m.rdb.(*redis.ClusterClient); ok {\n\t\terr := m.scan(ctx, \"*\", func(keys []string) error {\n\t\t\treturn fmt.Errorf(\"found key with same prefix: %s\", keys[0])\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tdbsize, err := m.rdb.DBSize(ctx).Result()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif dbsize > 0 {\n\t\t\treturn fmt.Errorf(\"database redis://%s is not empty\", m.addr)\n\t\t}\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/meta/redis_csc.go", "content": "//go:build !noredis\r\n// +build !noredis\r\n\r\n/*\r\n * JuiceFS, Copyright 2020 Juicedata, Inc.\r\n *\r\n * Licensed under the Apache License, Version 2.0 (the \"License\");\r\n * you may not use this file except in compliance with the License.\r\n * You may obtain a copy of the License at\r\n *\r\n * http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an \"AS IS\" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\n\r\npackage meta\r\n\r\nimport (\r\n\t\"context\"\r\n\t\"fmt\"\r\n\t\"os\"\r\n\t\"strconv\"\r\n\t\"strings\"\r\n\t\"time\"\r\n\t\"unsafe\"\r\n\r\n\t\"github.com/hashicorp/golang-lru/v2/expirable\"\r\n\t\"github.com/redis/go-redis/v9\"\r\n\t\"github.com/redis/go-redis/v9/push\"\r\n)\r\n\r\nvar entryMark cachedEntry\r\n\r\ntype cachedEntry struct {\r\n\tino Ino\r\n\tAttr\r\n}\r\n\r\nfunc (e *cachedEntry) isMark() bool {\r\n\treturn e.ino == 0\r\n}\r\n\r\n// redisCache support bcast mode client-side cache\r\n// cache attrs and entries only, chunks are already cached in OpenCache\r\ntype redisCache struct {\r\n\tcli *redis.Client\r\n\tprefix string\r\n\tcap int\r\n\texpiry time.Duration\r\n\tpreload int\r\n\tsubscription *redis.PubSub\r\n\r\n\tinodeCache *expirable.LRU[Ino, []byte]\r\n\tentryCache *expirable.LRU[string, *cachedEntry]\r\n}\r\n\r\nfunc newRedisCache(prefix string, cap int, expiry time.Duration, preload int) *redisCache {\r\n\tlogger.Infof(\"Initializing Redis client-side cache with size %d and expiry %+v\", cap, expiry)\r\n\treturn &redisCache{\r\n\t\tprefix: prefix,\r\n\t\tcap: cap,\r\n\t\texpiry: expiry,\r\n\t\tpreload: preload,\r\n\t\tinodeCache: expirable.NewLRU[Ino, []byte](cap, nil, expiry),\r\n\t\tentryCache: expirable.NewLRU[string, *cachedEntry](cap, nil, expiry),\r\n\t}\r\n}\r\n\r\nfunc (c *redisCache) init(cli redis.UniversalClient) error {\r\n\tctx := context.WithValue(context.Background(), invalidConnKey{}, true)\r\n\tvar err error\r\n\tif rc, ok := cli.(*redis.Client); ok {\r\n\t\tc.cli = rc\r\n\t} else if cc, ok := cli.(*redis.ClusterClient); ok {\r\n\t\t// For cluster mode, we should get the master node for our key\r\n\t\tif c.cli, err = cc.MasterForKey(ctx, c.prefix); err != nil {\r\n\t\t\treturn err\r\n\t\t}\r\n\t}\r\n\tc.cli.Options().OnConnect = c.onInvalidateConnect\r\n\t// under the RESP3 protocol, \"__redis__:invalidate\" actually has no effect.\r\n\t// we use Pubsub channel to simplify connection management and receiving PUSH messages.\r\n\tc.subscription = c.cli.Subscribe(ctx, \"__redis__:invalidate\")\r\n\t_ = c.subscription.Channel()\r\n\t// handle PUSH notifications for invalidation in c.HandlePushNotification\r\n\tif err = c.cli.RegisterPushNotificationHandler(\"invalidate\", c, true); err != nil {\r\n\t\tc.close()\r\n\t\treturn err\r\n\t}\r\n\t// handle client cmd to avoid race conditions\r\n\tc.cli.AddHook(c)\r\n\treturn nil\r\n}\r\n\r\nconst (\r\n\tkeyTypOther = iota\r\n\tkeyTypInode\r\n\tkeyTypEntry\r\n)\r\n\r\nfunc (c *redisCache) parse(key string) int {\r\n\tif strings.HasPrefix(key, c.prefix+\"i\") {\r\n\t\treturn keyTypInode\r\n\t}\r\n\tif strings.HasPrefix(key, c.prefix+\"d\") {\r\n\t\treturn keyTypEntry\r\n\t}\r\n\treturn keyTypOther\r\n}\r\n\r\nfunc (c *redisCache) entryName(parent Ino, name string) string {\r\n\treturn fmt.Sprintf(\"%d%d%s\", parent, os.PathSeparator, name)\r\n}\r\n\r\nfunc (c *redisCache) HandlePushNotification(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {\r\n\tif len(notification) != 2 || notification[0] == nil || notification[1] == nil {\r\n\t\treturn nil\r\n\t}\r\n\tif typ, ok := notification[0].(string); !ok || typ != \"invalidate\" {\r\n\t\treturn nil\r\n\t}\r\n\tiKeys := notification[1].([]interface{})\r\n\tvar key string\r\n\tfor _, iKey := range iKeys {\r\n\t\tkey = iKey.(string)\r\n\t\ttyp := c.parse(key)\r\n\t\tswitch typ {\r\n\t\tcase keyTypInode:\r\n\t\t\tinodeStr := key[len(c.prefix)+1:]\r\n\t\t\tinode, err := strconv.ParseUint(inodeStr, 10, 64)\r\n\t\t\tif err == nil {\r\n\t\t\t\tc.inodeCache.Remove(Ino(inode))\r\n\t\t\t}\r\n\t\tcase keyTypEntry:\r\n\t\t\tparentStr := key[len(c.prefix)+1:]\r\n\t\t\t// invalidate all entries related to this directory\r\n\t\t\tprefix := fmt.Sprintf(\"%s%d\", parentStr, os.PathSeparator)\r\n\t\t\tfor _, k := range c.entryCache.Keys() {\r\n\t\t\t\tif strings.HasPrefix(k, prefix) {\r\n\t\t\t\t\tc.entryCache.Remove(k)\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\treturn nil\r\n}\r\n\r\nfunc (c *redisCache) DialHook(next redis.DialHook) redis.DialHook { return nil }\r\n\r\nvar inodeMark []byte\r\n\r\nfunc (c *redisCache) beforeProcess(cmd redis.Cmder, skip bool) bool {\r\n\tname, args := cmd.Name(), cmd.Args()\r\n\tvar key string\r\n\tvar ok bool\r\n\tif len(args) < 2 {\r\n\t\treturn true\r\n\t}\r\n\tif key, ok = args[1].(string); !ok {\r\n\t\treturn true\r\n\t}\r\n\ttyp := c.parse(key)\r\n\r\n\tif name == \"get\" && typ == keyTypInode {\r\n\t\tnum, err := strconv.ParseUint(key[len(c.prefix)+1:], 10, 64)\r\n\t\tif err == nil {\r\n\t\t\tinode := Ino(num)\r\n\t\t\tif data, ok := c.inodeCache.Get(inode); ok {\r\n\t\t\t\tif !skip && len(data) > 0 {\r\n\t\t\t\t\trsp := cmd.(*redis.StringCmd)\r\n\t\t\t\t\trsp.SetErr(nil)\r\n\t\t\t\t\trsp.SetVal(bytesToString(data))\r\n\t\t\t\t\treturn false\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tc.inodeCache.AddIf(inode, inodeMark, func(oldVal []byte, exists bool) bool {\r\n\t\t\t\treturn !exists\r\n\t\t\t})\r\n\t\t\t// request to Redis server\r\n\t\t}\r\n\t}\r\n\treturn true\r\n}\r\n\r\nfunc (c *redisCache) afterProcess(cmd redis.Cmder) {\r\n\tname, args := cmd.Name(), cmd.Args()\r\n\tvar key string\r\n\tvar ok bool\r\n\tif len(args) < 2 {\r\n\t\treturn\r\n\t}\r\n\tif key, ok = args[1].(string); !ok {\r\n\t\treturn\r\n\t}\r\n\ttyp := c.parse(key)\r\n\r\n\tswitch name {\r\n\tcase \"get\":\r\n\t\tif typ == keyTypInode {\r\n\t\t\tif data, err := cmd.(*redis.StringCmd).Bytes(); err == nil {\r\n\t\t\t\tnum, err := strconv.ParseUint(key[len(c.prefix)+1:], 10, 64)\r\n\t\t\t\tif err != nil {\r\n\t\t\t\t\treturn\r\n\t\t\t\t}\r\n\t\t\t\t_, _ = c.inodeCache.AddIf(Ino(num), data, func(oldVal []byte, exists bool) bool {\r\n\t\t\t\t\treturn exists && len(oldVal) == 0\r\n\t\t\t\t})\r\n\t\t\t}\r\n\t\t}\r\n\tcase \"set\":\r\n\t\tif typ == keyTypInode {\r\n\t\t\tif cmd.(*redis.StatusCmd).Err() == nil {\r\n\t\t\t\tif num, err := strconv.ParseUint(key[len(c.prefix)+1:], 10, 64); err == nil {\r\n\t\t\t\t\t_ = c.inodeCache.Remove(Ino(num))\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\tcase \"hdel\":\r\n\t\tif typ == keyTypEntry {\r\n\t\t\tif err := cmd.(*redis.IntCmd).Err(); err == nil {\r\n\t\t\t\tfor i := 2; i < len(args); i++ {\r\n\t\t\t\t\t_ = c.entryCache.Remove(fmt.Sprintf(\"%s%d%s\", key[len(c.prefix)+1:], os.PathSeparator, args[i]))\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\tcase \"hset\":\r\n\t\tif typ == keyTypEntry {\r\n\t\t\tif err := cmd.(*redis.IntCmd).Err(); err == nil {\r\n\t\t\t\tfor i := 2; i < len(args); i += 2 {\r\n\t\t\t\t\t_ = c.entryCache.Remove(fmt.Sprintf(\"%s%d%s\", key[len(c.prefix)+1:], os.PathSeparator, args[i]))\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n}\r\n\r\nfunc (c *redisCache) ProcessHook(next redis.ProcessHook) redis.ProcessHook {\r\n\treturn func(ctx context.Context, cmd redis.Cmder) error {\r\n\t\tif !c.beforeProcess(cmd, false) {\r\n\t\t\treturn nil\r\n\t\t}\r\n\t\terr := next(ctx, cmd)\r\n\t\tc.afterProcess(cmd)\r\n\t\treturn err\r\n\t}\r\n}\r\n\r\nfunc (c *redisCache) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {\r\n\treturn func(ctx context.Context, cmds []redis.Cmder) error {\r\n\t\tfor _, cmd := range cmds {\r\n\t\t\t_ = c.beforeProcess(cmd, true)\r\n\t\t}\r\n\t\terr := next(ctx, cmds)\r\n\t\tfor _, cmd := range cmds {\r\n\t\t\tc.afterProcess(cmd)\r\n\t\t}\r\n\t\treturn err\r\n\t}\r\n}\r\n\r\nfunc (c *redisCache) close() {\r\n\tif c.subscription != nil {\r\n\t\tif err := c.subscription.Close(); err != nil {\r\n\t\t\tlogger.Warnf(\"failed closing Redis cache subscription: %v\", err)\r\n\t\t}\r\n\t\tc.subscription = nil\r\n\t}\r\n\tif c.cli != nil {\r\n\t\tc.cli.Options().OnConnect = nil\r\n\t}\r\n\tc.cli = nil\r\n}\r\n\r\ntype invalidConnKey struct{}\r\n\r\nfunc (c *redisCache) onInvalidateConnect(ctx context.Context, cn *redis.Conn) error {\r\n\tif ctx.Value(invalidConnKey{}) == nil {\r\n\t\treturn nil\r\n\t}\r\n\t// clear all caches on reconnect\r\n\tc.inodeCache.Purge()\r\n\tc.entryCache.Purge()\r\n\t// use the pubsub connection to handle tracking and invalidate\r\n\t_ = cn.Do(ctx, \"CLIENT\", \"TRACKING\", \"OFF\").Err()\r\n\tif err := cn.Do(ctx, \"CLIENT\", \"TRACKING\", \"ON\", \"BCAST\", \"PREFIX\", c.prefix+\"i\", \"PREFIX\", c.prefix+\"d\").Err(); err != nil {\r\n\t\tlogger.Warnf(\"Failed to enable Redis client-side caching on new connection: %v\", err)\r\n\t\treturn err\r\n\t}\r\n\treturn nil\r\n}\r\n\r\nfunc (m *redisMeta) preloadCache() {\r\n\tif m.cache == nil {\r\n\t\treturn\r\n\t}\r\n\tif m.cache.preload <= 0 {\r\n\t\treturn\r\n\t}\r\n\tstart := time.Now()\r\n\tctx := Background()\r\n\tattr := &Attr{}\r\n\tif eno := m.doGetAttr(ctx, m.root, attr); eno != 0 {\r\n\t\tlogger.Warnf(\"failed to get root inode %d attribute: %d\", m.root, eno)\r\n\t\treturn\r\n\t}\r\n\r\n\tvar entries []*Entry\r\n\tif eno := m.doReaddir(ctx, m.root, 1, &entries, m.cache.preload); eno != 0 {\r\n\t\tlogger.Warnf(\"failed to read root %d directory: %d\", m.root, eno)\r\n\t\treturn\r\n\t}\r\n\tfor _, entry := range entries {\r\n\t\tm.cache.entryCache.Add(m.cache.entryName(m.root, string(entry.Name)), &cachedEntry{\r\n\t\t\tino: entry.Inode,\r\n\t\t\tAttr: *entry.Attr,\r\n\t\t})\r\n\t}\r\n\tlogger.Infof(\"preload %d inodes in %v\", m.cache.inodeCache.Len(), time.Since(start))\r\n}\r\n\r\nfunc bytesToString(b []byte) string {\r\n\treturn *(*string)(unsafe.Pointer(&b))\r\n}\r\n" }, { "path": "pkg/meta/redis_csc_test.go", "content": "//go:build !noredis\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc mockRedisCSCMeta(t *testing.T) *redisMeta {\n\tm, err := newRedisMeta(\"redis\", \"127.0.0.1:6379/10?client-cache=true\", testConfig())\n\trequire.NoError(t, err, \"failed to create redis meta\")\n\trequire.Equal(t, \"redis\", m.Name(), \"meta name should be redis\")\n\treturn m.(*redisMeta)\n}\n\nfunc TestRedisCache(t *testing.T) {\n\tctx := context.Background()\n\tm := mockRedisCSCMeta(t)\n\t_ = m.rdb.FlushAll(ctx)\n\tdefer m.Shutdown()\n\tdefer m.cache.close()\n\n\tvar err error\n\tt.Run(\"invalidation handling\", func(t *testing.T) {\n\t\tcache := m.cache\n\t\tino := Ino(100)\n\t\tattr := &Attr{Typ: TypeFile, Mode: 0644}\n\t\tcache.inodeCache.Add(ino, attr.Marshal())\n\t\tif _, ok := cache.inodeCache.Get(ino); !ok {\n\t\t\tt.Fatal(\"inode should be in cache\")\n\t\t}\n\n\t\terr = m.rdb.Set(ctx, m.inodeKey(ino), m.marshal(&Attr{Mode: 0755}), 0).Err()\n\t\trequire.NoError(t, err, \"failed to set key %d\", ino)\n\t\tdumIno := Ino(101)\n\t\terr = m.rdb.Set(ctx, m.inodeKey(dumIno), m.marshal(&Attr{Mode: 0755}), 0).Err()\n\t\trequire.NoError(t, err, \"failed to set key %d\", dumIno)\n\t\ttime.Sleep(3 * time.Second)\n\t\tif _, ok := cache.inodeCache.Get(Ino(100)); ok {\n\t\t\tt.Fatal(\"inode should be invalidated and removed from cache\")\n\t\t}\n\n\t\tcache.entryCache.Add(cache.entryName(101, \"file\"), &cachedEntry{})\n\t\tm.rdb.HSet(ctx, m.entryKey(100), \"file\", \"content\").Err()\n\t})\n\tt.Run(\"cache expiration\", func(t *testing.T) {\n\t\tshortExpiry := 50 * time.Millisecond\n\t\tcache := newRedisCache(\"jfs\", 1000, shortExpiry, 0)\n\t\tattr := &Attr{Typ: TypeFile, Mode: 0644}\n\t\tcache.inodeCache.Add(Ino(102), attr.Marshal())\n\t\ttime.Sleep(3 * shortExpiry)\n\t\tif _, ok := cache.inodeCache.Get(Ino(102)); ok {\n\t\t\tt.Fatal(\"inode should be expired\")\n\t\t}\n\t})\n\n\tt.Run(\"inode hook\", func(t *testing.T) {\n\t\tcache := m.cache\n\t\tino := Ino(103)\n\t\tattr := &Attr{Typ: TypeFile, Length: 10}\n\t\tcache.inodeCache.Add(ino, attr.Marshal())\n\n\t\tdata, err := m.rdb.Get(ctx, m.inodeKey(ino)).Bytes()\n\t\trequire.NoError(t, err, \"failed to get inode\")\n\t\tattr2 := &Attr{}\n\t\tattr2.Unmarshal(data)\n\t\tattr2.Full = false\n\t\trequire.Equal(t, *attr, *attr2)\n\n\t\tattr3 := &Attr{Typ: TypeFile, Length: 20}\n\t\terr = m.rdb.Set(ctx, m.inodeKey(ino), attr3.Marshal(), 0).Err()\n\t\trequire.NoError(t, err)\n\t\t_, ok := cache.inodeCache.Get(ino)\n\t\trequire.False(t, ok)\n\t})\n\n\tt.Run(\"entry hook\", func(t *testing.T) {\n\t\tcache := m.cache\n\t\tino := Ino(104)\n\t\tname1, name2 := cache.entryName(ino, \"f1\"), cache.entryName(ino, \"f2\")\n\t\tcache.entryCache.Add(name1, &cachedEntry{})\n\t\tcache.entryCache.Add(name2, &cachedEntry{})\n\n\t\terr := m.rdb.HSet(ctx, m.entryKey(ino), \"f1\", \"c1\", \"f2\", \"c2\").Err()\n\t\trequire.NoError(t, err)\n\n\t\t_, ok := cache.entryCache.Get(name1)\n\t\trequire.False(t, ok)\n\t\t_, ok = cache.entryCache.Get(name2)\n\t\trequire.False(t, ok)\n\n\t\tcache.entryCache.Add(name1, &cachedEntry{})\n\t\tcache.entryCache.Add(name2, &cachedEntry{})\n\t\terr = m.rdb.HDel(ctx, m.entryKey(ino), \"f1\", \"f2\").Err()\n\t\trequire.NoError(t, err)\n\n\t\t_, ok = cache.entryCache.Get(name1)\n\t\trequire.False(t, ok)\n\t\t_, ok = cache.entryCache.Get(name2)\n\t\trequire.False(t, ok)\n\t})\n}\n" }, { "path": "pkg/meta/redis_lock.go", "content": "//go:build !noredis\n// +build !noredis\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/redis/go-redis/v9\"\n)\n\nfunc (r *redisMeta) Flock(ctx Context, inode Ino, owner uint64, ltype uint32, block bool) syscall.Errno {\n\tikey := r.flockKey(inode)\n\tlkey := r.ownerKey(owner)\n\tctx = ctx.WithValue(txMethodKey{}, \"Flock\"+strconv.Itoa(int(ltype)))\n\tif ltype == F_UNLCK {\n\t\treturn errno(r.txn(ctx, func(tx *redis.Tx) error {\n\t\t\tlkeys, err := tx.HKeys(ctx, ikey).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\tpipe.HDel(ctx, ikey, lkey)\n\t\t\t\tif len(lkeys) == 1 && lkeys[0] == lkey {\n\t\t\t\t\tpipe.SRem(ctx, r.lockedKey(r.sid), ikey)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn err\n\t\t}, ikey))\n\t}\n\tvar err error\n\tfor {\n\t\terr = r.txn(ctx, func(tx *redis.Tx) error {\n\t\t\towners, err := tx.HGetAll(ctx, ikey).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdelete(owners, lkey)\n\t\t\tif ltype == F_RDLCK {\n\t\t\t\tfor _, v := range owners {\n\t\t\t\t\tif v == \"W\" {\n\t\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\t\tpipe.HSet(ctx, ikey, lkey, \"R\")\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif len(owners) > 0 {\n\t\t\t\treturn syscall.EAGAIN\n\t\t\t}\n\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\tpipe.HSet(ctx, ikey, lkey, \"W\")\n\t\t\t\tpipe.SAdd(ctx, r.lockedKey(r.sid), ikey)\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn err\n\t\t}, ikey)\n\n\t\tif !block || err != syscall.EAGAIN {\n\t\t\tbreak\n\t\t}\n\t\tif ltype == F_WRLCK {\n\t\t\ttime.Sleep(time.Millisecond * 1)\n\t\t} else {\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t}\n\treturn errno(err)\n}\n\nfunc (r *redisMeta) Getlk(ctx Context, inode Ino, owner uint64, ltype *uint32, start, end *uint64, pid *uint32) syscall.Errno {\n\tif *ltype == F_UNLCK {\n\t\t*start = 0\n\t\t*end = 0\n\t\t*pid = 0\n\t\treturn 0\n\t}\n\tlkey := r.ownerKey(owner)\n\towners, err := r.rdb.HGetAll(ctx, r.plockKey(inode)).Result()\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tdelete(owners, lkey) // exclude itself\n\tfor k, d := range owners {\n\t\tls := loadLocks([]byte(d))\n\t\tfor _, l := range ls {\n\t\t\t// find conflicted locks\n\t\t\tif (*ltype == F_WRLCK || l.Type == F_WRLCK) && *end >= l.Start && *start <= l.End {\n\t\t\t\t*ltype = l.Type\n\t\t\t\t*start = l.Start\n\t\t\t\t*end = l.End\n\t\t\t\tsid, _ := strconv.Atoi(strings.Split(k, \"_\")[0])\n\t\t\t\tif uint64(sid) == r.sid {\n\t\t\t\t\t*pid = l.Pid\n\t\t\t\t} else {\n\t\t\t\t\t*pid = 0\n\t\t\t\t}\n\t\t\t\treturn 0\n\t\t\t}\n\t\t}\n\t}\n\t*ltype = F_UNLCK\n\t*start = 0\n\t*end = 0\n\t*pid = 0\n\treturn 0\n}\n\nfunc (r *redisMeta) Setlk(ctx Context, inode Ino, owner uint64, block bool, ltype uint32, start, end uint64, pid uint32) syscall.Errno {\n\tikey := r.plockKey(inode)\n\tlkey := r.ownerKey(owner)\n\tctx = ctx.WithValue(txMethodKey{}, \"Setlk\"+strconv.Itoa(int(ltype)))\n\tvar err error\n\tlock := plockRecord{ltype, pid, start, end}\n\tfor {\n\t\terr = r.txn(ctx, func(tx *redis.Tx) error {\n\t\t\tif ltype == F_UNLCK {\n\t\t\t\td, err := tx.HGet(ctx, ikey, lkey).Result()\n\t\t\t\tif err != nil && err != redis.Nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tls := loadLocks([]byte(d))\n\t\t\t\tif len(ls) == 0 {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tls = updateLocks(ls, lock)\n\t\t\t\tvar lkeys []string\n\t\t\t\tif len(ls) == 0 {\n\t\t\t\t\tlkeys, err = tx.HKeys(ctx, ikey).Result()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\t\tif len(ls) == 0 {\n\t\t\t\t\t\tpipe.HDel(ctx, ikey, lkey)\n\t\t\t\t\t\tif len(lkeys) == 1 && lkeys[0] == lkey {\n\t\t\t\t\t\t\tpipe.SRem(ctx, r.lockedKey(r.sid), ikey)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tpipe.HSet(ctx, ikey, lkey, dumpLocks(ls))\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\treturn err\n\t\t\t}\n\t\t\towners, err := tx.HGetAll(ctx, ikey).Result()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tls := loadLocks([]byte(owners[lkey]))\n\t\t\tdelete(owners, lkey)\n\t\t\tfor _, d := range owners {\n\t\t\t\tls := loadLocks([]byte(d))\n\t\t\t\tfor _, l := range ls {\n\t\t\t\t\t// find conflicted locks\n\t\t\t\t\tif (ltype == F_WRLCK || l.Type == F_WRLCK) && end >= l.Start && start <= l.End {\n\t\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tls = updateLocks(ls, lock)\n\t\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\t\tpipe.HSet(ctx, ikey, lkey, dumpLocks(ls))\n\t\t\t\tpipe.SAdd(ctx, r.lockedKey(r.sid), ikey)\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn err\n\t\t}, ikey)\n\n\t\tif !block || err != syscall.EAGAIN {\n\t\t\tbreak\n\t\t}\n\t\tif ltype == F_WRLCK {\n\t\t\ttime.Sleep(time.Millisecond * 1)\n\t\t} else {\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t}\n\treturn errno(err)\n}\n\nfunc (r *redisMeta) ListLocks(ctx context.Context, inode Ino) ([]PLockItem, []FLockItem, error) {\n\tfKey := r.flockKey(inode)\n\tpKey := r.plockKey(inode)\n\n\trawFLocks, err := r.rdb.HGetAll(ctx, fKey).Result()\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tflocks := make([]FLockItem, 0, len(rawFLocks))\n\tfor k, v := range rawFLocks {\n\t\towner, err := parseOwnerKey(k)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tflocks = append(flocks, FLockItem{*owner, v})\n\t}\n\n\trawPLocks, err := r.rdb.HGetAll(ctx, pKey).Result()\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tplocks := make([]PLockItem, 0)\n\tfor k, d := range rawPLocks {\n\t\towner, err := parseOwnerKey(k)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tls := loadLocks([]byte(d))\n\t\tfor _, l := range ls {\n\t\t\tplocks = append(plocks, PLockItem{*owner, l})\n\t\t}\n\t}\n\treturn plocks, flocks, nil\n}\n" }, { "path": "pkg/meta/slice.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport \"github.com/juicedata/juicefs/pkg/utils\"\n\ntype slice struct {\n\tid uint64\n\tsize uint32\n\toff uint32\n\tlen uint32\n\tpos uint32\n\tleft *slice\n\tright *slice\n}\n\nfunc newSlice(pos uint32, id uint64, cleng, off, len uint32) *slice {\n\tif len == 0 {\n\t\treturn nil\n\t}\n\ts := &slice{}\n\ts.pos = pos\n\ts.id = id\n\ts.size = cleng\n\ts.off = off\n\ts.len = len\n\ts.left = nil\n\ts.right = nil\n\treturn s\n}\n\nfunc (s *slice) read(buf []byte) {\n\trb := utils.ReadBuffer(buf)\n\ts.pos = rb.Get32()\n\ts.id = rb.Get64()\n\ts.size = rb.Get32()\n\ts.off = rb.Get32()\n\ts.len = rb.Get32()\n}\n\nfunc (s *slice) cut(pos uint32) (left, right *slice) {\n\tif s == nil {\n\t\treturn nil, nil\n\t}\n\tif pos <= s.pos {\n\t\tif s.left == nil {\n\t\t\ts.left = newSlice(pos, 0, 0, 0, s.pos-pos)\n\t\t}\n\t\tleft, s.left = s.left.cut(pos)\n\t\treturn left, s\n\t} else if pos < s.pos+s.len {\n\t\tl := pos - s.pos\n\t\tright = newSlice(pos, s.id, s.size, s.off+l, s.len-l)\n\t\tright.right = s.right\n\t\ts.len = l\n\t\ts.right = nil\n\t\treturn s, right\n\t} else {\n\t\tif s.right == nil {\n\t\t\ts.right = newSlice(s.pos+s.len, 0, 0, 0, pos-s.pos-s.len)\n\t\t}\n\t\ts.right, right = s.right.cut(pos)\n\t\treturn s, right\n\t}\n}\n\nfunc (s *slice) visit(f func(*slice)) {\n\tif s == nil {\n\t\treturn\n\t}\n\ts.left.visit(f)\n\tright := s.right\n\tf(s) // s could be freed\n\tright.visit(f)\n}\n\nconst sliceBytes = 24\n\nfunc marshalSlice(pos uint32, id uint64, size, off, len uint32) []byte {\n\tw := utils.NewBuffer(sliceBytes)\n\tw.Put32(pos)\n\tw.Put64(id)\n\tw.Put32(size)\n\tw.Put32(off)\n\tw.Put32(len)\n\treturn w.Bytes()\n}\n\nfunc readSlices(vals []string) []*slice {\n\tslices := make([]slice, len(vals))\n\tss := make([]*slice, len(vals))\n\tfor i, val := range vals {\n\t\tif len(val) != sliceBytes {\n\t\t\tlogger.Errorf(\"corrupt slice: len=%d, val=%v\", len(val), []byte(val))\n\t\t\treturn nil\n\t\t}\n\t\ts := &slices[i]\n\t\ts.read([]byte(val))\n\t\tss[i] = s\n\t}\n\treturn ss\n}\n\nfunc readSliceBuf(buf []byte) []*slice {\n\tif len(buf)%sliceBytes != 0 {\n\t\tlogger.Errorf(\"corrupt slices: len=%d\", len(buf))\n\t\treturn nil\n\t}\n\tnSlices := len(buf) / sliceBytes\n\tslices := make([]slice, nSlices)\n\tss := make([]*slice, nSlices)\n\tfor i := 0; i < len(buf); i += sliceBytes {\n\t\ts := &slices[i/sliceBytes]\n\t\ts.read(buf[i:])\n\t\tss[i/sliceBytes] = s\n\t}\n\treturn ss\n}\n\nfunc buildSlice(ss []*slice) []Slice {\n\tvar root *slice\n\tfor i := range ss {\n\t\ts := new(slice)\n\t\t*s = *ss[i]\n\t\tvar right *slice\n\t\ts.left, right = root.cut(s.pos)\n\t\t_, s.right = right.cut(s.pos + s.len)\n\t\troot = s\n\t}\n\tvar pos uint32\n\tvar chunk []Slice\n\troot.visit(func(s *slice) {\n\t\tif s.pos > pos {\n\t\t\tchunk = append(chunk, Slice{Size: s.pos - pos, Len: s.pos - pos})\n\t\t\tpos = s.pos\n\t\t}\n\t\tchunk = append(chunk, Slice{Id: s.id, Size: s.size, Off: s.off, Len: s.len})\n\t\tpos += s.len\n\t})\n\treturn chunk\n}\n\nfunc compactChunk(ss []*slice) (uint32, uint32, []Slice) {\n\tvar chunk = buildSlice(ss)\n\tvar pos uint32\n\tn := len(chunk)\n\tfor n > 1 {\n\t\tif chunk[0].Id == 0 {\n\t\t\tpos += chunk[0].Len\n\t\t\tchunk = chunk[1:]\n\t\t\tn--\n\t\t} else if chunk[n-1].Id == 0 {\n\t\t\tchunk = chunk[:n-1]\n\t\t\tn--\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\tif n == 1 && chunk[0].Id == 0 {\n\t\tchunk[0].Len = 1\n\t}\n\tvar size uint32\n\tfor _, c := range chunk {\n\t\tsize += c.Len\n\t}\n\treturn pos, size, chunk\n}\n\nfunc skipSome(chunk []*slice) int {\n\tvar skipped int\n\tvar total = len(chunk)\nOUT:\n\tfor skipped < total {\n\t\tss := chunk[skipped:]\n\t\tpos, size, c := compactChunk(ss)\n\t\tfirst := ss[0]\n\t\tif first.len < (1<<20) || first.len*5 < size || size == 0 {\n\t\t\t// it's too small\n\t\t\tbreak\n\t\t}\n\t\tisFirst := func(pos uint32, s Slice) bool {\n\t\t\treturn pos == first.pos && s.Id == first.id && s.Off == first.off && s.Len == first.len\n\t\t}\n\t\tif !isFirst(pos, c[0]) {\n\t\t\t// it's not the first slice, compact it\n\t\t\tbreak\n\t\t}\n\t\tfor _, s := range ss[1:] {\n\t\t\tif *s == *first {\n\t\t\t\tbreak OUT\n\t\t\t}\n\t\t}\n\t\tskipped++\n\t}\n\treturn skipped\n}\n" }, { "path": "pkg/meta/sql.go", "content": "//go:build !nosqlite || !nomysql || !nopg\n// +build !nosqlite !nomysql !nopg\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"database/sql\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"slices\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"xorm.io/xorm\"\n\t\"xorm.io/xorm/log\"\n\t\"xorm.io/xorm/names\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/sirupsen/logrus\"\n)\n\nconst MaxFieldsCountOfTable = 18 // node table\n\ntype setting struct {\n\tName string `xorm:\"pk\"`\n\tValue string `xorm:\"varchar(4096) notnull\"`\n}\n\ntype counter struct {\n\tName string `xorm:\"pk\"`\n\tValue int64 `xorm:\"notnull\"`\n}\n\ntype edge struct {\n\tId int64 `xorm:\"pk bigserial\"`\n\tParent Ino `xorm:\"unique(edge) notnull\"`\n\tName []byte `xorm:\"unique(edge) varbinary(255) notnull\"`\n\tInode Ino `xorm:\"index notnull\"`\n\tType uint8 `xorm:\"notnull\"`\n}\n\ntype node struct {\n\tInode Ino `xorm:\"pk\"`\n\tType uint8 `xorm:\"notnull\"`\n\tFlags uint8 `xorm:\"notnull\"`\n\tMode uint16 `xorm:\"notnull\"`\n\tUid uint32 `xorm:\"notnull\"`\n\tGid uint32 `xorm:\"notnull\"`\n\tAtime int64 `xorm:\"notnull\"`\n\tMtime int64 `xorm:\"notnull\"`\n\tCtime int64 `xorm:\"notnull\"`\n\tAtimensec int16 `xorm:\"notnull default 0\"`\n\tMtimensec int16 `xorm:\"notnull default 0\"`\n\tCtimensec int16 `xorm:\"notnull default 0\"`\n\tNlink uint32 `xorm:\"notnull\"`\n\tLength uint64 `xorm:\"notnull\"`\n\tRdev uint32\n\tParent Ino\n\tAccessACLId uint32 `xorm:\"'access_acl_id'\"`\n\tDefaultACLId uint32 `xorm:\"'default_acl_id'\"`\n}\n\nfunc (n *node) setAtime(ns int64) {\n\tn.Atime = ns / 1e3\n\tn.Atimensec = int16(ns % 1e3)\n}\n\nfunc (n *node) getMtime() int64 {\n\treturn n.Mtime*1e3 + int64(n.Mtimensec)\n}\n\nfunc (n *node) setMtime(ns int64) {\n\tn.Mtime = ns / 1e3\n\tn.Mtimensec = int16(ns % 1e3)\n}\n\nfunc (n *node) setCtime(ns int64) {\n\tn.Ctime = ns / 1e3\n\tn.Ctimensec = int16(ns % 1e3)\n}\n\nfunc getACLIdColName(aclType uint8) string {\n\tswitch aclType {\n\tcase aclAPI.TypeAccess:\n\t\treturn \"access_acl_id\"\n\tcase aclAPI.TypeDefault:\n\t\treturn \"default_acl_id\"\n\t}\n\treturn \"\"\n}\n\ntype acl struct {\n\tId uint32 `xorm:\"pk autoincr\"`\n\tOwner uint16\n\tGroup uint16\n\tMask uint16\n\tOther uint16\n\tNamedUsers []byte\n\tNamedGroups []byte\n}\n\nfunc newSQLAcl(r *aclAPI.Rule) *acl {\n\ta := &acl{\n\t\tOwner: r.Owner,\n\t\tGroup: r.Group,\n\t\tMask: r.Mask,\n\t\tOther: r.Other,\n\t}\n\ta.NamedUsers = r.NamedUsers.Encode()\n\ta.NamedGroups = r.NamedGroups.Encode()\n\treturn a\n}\n\nfunc (a *acl) toRule() *aclAPI.Rule {\n\tr := &aclAPI.Rule{}\n\tr.Owner = a.Owner\n\tr.Group = a.Group\n\tr.Other = a.Other\n\tr.Mask = a.Mask\n\tr.NamedUsers.Decode(a.NamedUsers)\n\tr.NamedGroups.Decode(a.NamedGroups)\n\treturn r\n}\n\ntype delegationToken struct {\n\tId uint32 `xorm:\"pk autoincr\"`\n\tToken []byte\n}\n\ntype namedNode struct {\n\tnode `xorm:\"extends\"`\n\tName []byte `xorm:\"varbinary(255)\"`\n}\n\ntype chunk struct {\n\tId int64 `xorm:\"pk bigserial\"`\n\tInode Ino `xorm:\"unique(chunk) notnull\"`\n\tIndx uint32 `xorm:\"unique(chunk) notnull\"`\n\tSlices []byte `xorm:\"blob notnull\"`\n}\n\ntype sliceRef struct {\n\tId uint64 `xorm:\"pk chunkid\"`\n\tSize uint32 `xorm:\"notnull\"`\n\tRefs int `xorm:\"index notnull\"`\n}\n\ntype delslices struct {\n\tId uint64 `xorm:\"pk chunkid\"`\n\tDeleted int64 `xorm:\"notnull\"` // timestamp\n\tSlices []byte `xorm:\"blob notnull\"`\n}\n\ntype symlink struct {\n\tInode Ino `xorm:\"pk\"`\n\tTarget []byte `xorm:\"varbinary(4096) notnull\"`\n}\n\ntype xattr struct {\n\tId int64 `xorm:\"pk bigserial\"`\n\tInode Ino `xorm:\"unique(name) notnull\"`\n\tName string `xorm:\"unique(name) notnull\"`\n\tValue []byte `xorm:\"blob notnull\"`\n}\n\ntype flock struct {\n\tId int64 `xorm:\"pk bigserial\"`\n\tInode Ino `xorm:\"notnull unique(flock)\"`\n\tSid uint64 `xorm:\"notnull unique(flock)\"`\n\tOwner int64 `xorm:\"notnull unique(flock)\"`\n\tLtype byte `xorm:\"notnull\"`\n}\n\ntype plock struct {\n\tId int64 `xorm:\"pk bigserial\"`\n\tInode Ino `xorm:\"notnull unique(plock)\"`\n\tSid uint64 `xorm:\"notnull unique(plock)\"`\n\tOwner int64 `xorm:\"notnull unique(plock)\"`\n\tRecords []byte `xorm:\"blob notnull\"`\n}\n\ntype session struct {\n\tSid uint64 `xorm:\"pk\"`\n\tHeartbeat int64 `xorm:\"notnull\"`\n\tInfo []byte `xorm:\"blob\"`\n}\n\ntype session2 struct {\n\tSid uint64 `xorm:\"pk\"`\n\tExpire int64 `xorm:\"notnull\"`\n\tInfo []byte `xorm:\"blob\"`\n}\n\ntype sustained struct {\n\tId int64 `xorm:\"pk bigserial\"`\n\tSid uint64 `xorm:\"unique(sustained) notnull\"`\n\tInode Ino `xorm:\"unique(sustained) notnull\"`\n}\n\ntype delfile struct {\n\tInode Ino `xorm:\"pk notnull\"`\n\tLength uint64 `xorm:\"notnull\"`\n\tExpire int64 `xorm:\"notnull\"`\n}\n\ntype dirStats struct {\n\tInode Ino `xorm:\"pk notnull\"`\n\tDataLength int64 `xorm:\"notnull\"`\n\tUsedSpace int64 `xorm:\"notnull\"`\n\tUsedInodes int64 `xorm:\"notnull\"`\n}\n\ntype detachedNode struct {\n\tInode Ino `xorm:\"pk notnull\"`\n\tAdded int64 `xorm:\"notnull\"`\n}\n\ntype dirQuota struct {\n\tInode Ino `xorm:\"pk\"`\n\tMaxSpace int64 `xorm:\"notnull\"`\n\tMaxInodes int64 `xorm:\"notnull\"`\n\tUsedSpace int64 `xorm:\"notnull\"`\n\tUsedInodes int64 `xorm:\"notnull\"`\n}\n\ntype userGroupQuota struct {\n\tQtype uint32 `xorm:\"pk notnull\"` // 1 for user, 2 for group\n\tQkey uint64 `xorm:\"pk notnull\"` // uid or gid\n\tMaxSpace int64 `xorm:\"notnull\"`\n\tMaxInodes int64 `xorm:\"notnull\"`\n\tUsedSpace int64 `xorm:\"notnull\"`\n\tUsedInodes int64 `xorm:\"notnull\"`\n}\n\ntype dbMeta struct {\n\t*baseMeta\n\tdb *xorm.Engine\n\tspool *sync.Pool\n\tsnap *dbSnap\n\n\tnoReadOnlyTxn bool\n\tstatement map[string]string\n\ttablePrefix string\n}\n\nvar _ Meta = (*dbMeta)(nil)\nvar _ engine = (*dbMeta)(nil)\n\ntype dbSnap struct {\n\tnode map[Ino]*node\n\tsymlink map[Ino]*symlink\n\txattr map[Ino][]*xattr\n\tedges map[Ino][]*edge\n\tchunk map[string]*chunk\n}\n\nfunc recoveryMysqlPwd(addr string) string {\n\tcolonIndex := strings.Index(addr, \":\")\n\tatIndex := strings.LastIndex(addr, \"@\")\n\tif colonIndex != -1 && colonIndex < atIndex {\n\t\tpwd := addr[colonIndex+1 : atIndex]\n\t\tif parse, err := url.Parse(\"mysql://root:\" + pwd + \"@127.0.0.1\"); err == nil {\n\t\t\tif originPwd, ok := parse.User.Password(); ok {\n\t\t\t\taddr = fmt.Sprintf(\"%s:%s%s\", addr[:colonIndex], originPwd, addr[atIndex:])\n\t\t\t}\n\t\t}\n\t}\n\treturn addr\n}\n\nfunc extractCustomConfig[T string | int](value *url.Values, key string, defaultV T) (T, error) {\n\tif value == nil {\n\t\treturn defaultV, nil\n\t}\n\tif v := value.Get(key); v != \"\" {\n\t\tvalue.Del(key)\n\t\tvar result T\n\t\tswitch any(defaultV).(type) {\n\t\tcase int:\n\t\t\tparsedInt, err := strconv.Atoi(v)\n\t\t\tif err != nil {\n\t\t\t\treturn defaultV, fmt.Errorf(\"failed to parse value as int: %v\", err)\n\t\t\t}\n\t\t\tresult = any(parsedInt).(T)\n\t\tcase string:\n\t\t\tresult = any(v).(T)\n\t\tdefault:\n\t\t\treturn defaultV, fmt.Errorf(\"unsupported type: %T\", defaultV)\n\t\t}\n\t\treturn result, nil\n\t} else {\n\t\treturn defaultV, nil\n\t}\n}\n\nvar setTransactionIsolation func(dns string) (string, error)\n\ntype prefixMapper struct {\n\tmapper names.Mapper\n\tprefix string\n}\n\nfunc (m prefixMapper) Obj2Table(name string) string {\n\tif name == \"sliceRef\" {\n\t\treturn m.prefix + \"chunk_ref\"\n\t}\n\treturn m.prefix + m.mapper.Obj2Table(name)\n}\n\nfunc (m prefixMapper) Table2Obj(name string) string {\n\tif name == m.prefix+\"chunk_ref\" {\n\t\treturn \"sliceRef\"\n\t}\n\treturn m.mapper.Table2Obj(name[len(m.prefix):])\n}\nfunc (m *dbMeta) sqlConv(sql string) string {\n\treturn m.statement[sql]\n}\n\nfunc (m *dbMeta) initStatement() {\n\tm.statement[\"SELECT length FROM node WHERE inode IN (SELECT inode FROM sustained)\"] =\n\t\tfmt.Sprintf(\"SELECT length FROM %snode WHERE inode IN (SELECT inode FROM %ssustained)\", m.tablePrefix, m.tablePrefix)\n\tm.statement[\"update counter set value=value + ? where name='totalInodes'\"] =\n\t\tfmt.Sprintf(\"update %scounter set value=value + ? where name='totalInodes'\", m.tablePrefix)\n\tm.statement[\"update counter set value= value + ? where name='usedSpace'\"] =\n\t\tfmt.Sprintf(\"update %scounter set value= value + ? where name='usedSpace'\", m.tablePrefix)\n\tm.statement[\"update chunk set slices=slices || ? where inode=? AND indx=?\"] =\n\t\tfmt.Sprintf(\"update %schunk set slices=slices || ? where inode=? AND indx=?\", m.tablePrefix)\n\tm.statement[\"update chunk set slices=concat(slices, ?) where inode=? AND indx=?\"] =\n\t\tfmt.Sprintf(\"update %schunk set slices=concat(slices, ?) where inode=? AND indx=?\", m.tablePrefix)\n\tm.statement[\"update chunk_ref set refs=refs+1 where chunkid = ? AND size = ?\"] =\n\t\tfmt.Sprintf(\"update %schunk_ref set refs=refs+1 where chunkid = ? AND size = ?\", m.tablePrefix)\n\tm.statement[\"update chunk_ref set refs=refs-1 where chunkid=? AND size=?\"] =\n\t\tfmt.Sprintf(\"update %schunk_ref set refs=refs-1 where chunkid=? AND size=?\", m.tablePrefix)\n\tm.statement[\"update dir_quota set used_space=used_space+?, used_inodes=used_inodes+? where inode=?\"] =\n\t\tfmt.Sprintf(\"update %sdir_quota set used_space=used_space+?, used_inodes=used_inodes+? where inode=?\", m.tablePrefix)\n\tm.statement[\"update user_group_quota set used_space=used_space+?, used_inodes=used_inodes+? where qtype=? and qkey=?\"] =\n\t\tfmt.Sprintf(\"update %suser_group_quota set used_space=used_space+?, used_inodes=used_inodes+? where qtype=? and qkey=?\", m.tablePrefix)\n\n\tm.statement[`\n\t\t\t INSERT INTO chunk (inode, indx, slices)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON CONFLICT (inode, indx)\n\t\t\t DO UPDATE SET slices=chunk.slices || ?`] =\n\t\tfmt.Sprintf(`\n\t\t\t INSERT INTO %schunk (inode, indx, slices)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON CONFLICT (inode, indx)\n\t\t\t DO UPDATE SET slices=%schunk.slices || ?`, m.tablePrefix, m.tablePrefix)\n\tm.statement[`\n\t\t\t INSERT INTO chunk (inode, indx, slices)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON DUPLICATE KEY UPDATE\n\t\t\t slices=concat(slices, ?)`] =\n\t\tfmt.Sprintf(`\n\t\t\t INSERT INTO %schunk (inode, indx, slices)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON DUPLICATE KEY UPDATE\n\t\t\t slices=concat(slices, ?)`, m.tablePrefix)\n\tm.statement[`\n\t\t\t INSERT INTO chunk_ref (chunkid, size, refs)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON CONFLICT (chunkid)\n\t\t\t DO UPDATE SET size=?, refs=?`] =\n\t\tfmt.Sprintf(`\n\t\t\t INSERT INTO %schunk_ref (chunkid, size, refs)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON CONFLICT (chunkid)\n\t\t\t DO UPDATE SET size=?, refs=?`, m.tablePrefix)\n\tm.statement[`\n\t\t\t INSERT INTO chunk_ref (chunkid, size, refs)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON DUPLICATE KEY UPDATE\n\t\t\t size=?, refs=?`] =\n\t\tfmt.Sprintf(`\n\t\t\t INSERT INTO %schunk_ref (chunkid, size, refs)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON DUPLICATE KEY UPDATE\n\t\t\t size=?, refs=?`, m.tablePrefix)\n\tm.statement[\"edge.inode=node.inode\"] = fmt.Sprintf(\"%sedge.inode=%snode.inode\", m.tablePrefix, m.tablePrefix)\n\tm.statement[\"edge.id\"] = fmt.Sprintf(\"%sedge.id\", m.tablePrefix)\n\tm.statement[\"edge.name\"] = fmt.Sprintf(\"%sedge.name\", m.tablePrefix)\n\tm.statement[\"edge.type\"] = fmt.Sprintf(\"%sedge.type\", m.tablePrefix)\n\tm.statement[\"edge.*\"] = fmt.Sprintf(\"%sedge.*\", m.tablePrefix)\n\tm.statement[\"node.*\"] = fmt.Sprintf(\"%snode.*\", m.tablePrefix)\n\tm.statement[`INSERT INTO chunk_ref (chunkid, size, refs) VALUES (?,?,?) ON CONFLICT DO NOTHING`] =\n\t\tfmt.Sprintf(`INSERT INTO %schunk_ref (chunkid, size, refs) VALUES (?,?,?) ON CONFLICT DO NOTHING`, m.tablePrefix)\n\tm.statement[`INSERT IGNORE INTO chunk_ref (chunkid, size, refs) VALUES (?,?,?)`] =\n\t\tfmt.Sprintf(`INSERT IGNORE INTO %schunk_ref (chunkid, size, refs) VALUES (?,?,?)`, m.tablePrefix)\n}\n\nfunc newSQLMeta(driver, addr string, conf *Config) (Meta, error) {\n\tvar searchPath string\n\n\tbaseUrl, queryStr, _ := strings.Cut(addr, \"?\")\n\tvar query url.Values\n\tvar err error\n\tquery, err = url.ParseQuery(queryStr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar vOpenConns, vIdleConns, vIdleTime, vLifeTime int\n\tif vOpenConns, err = extractCustomConfig(&query, \"max_open_conns\", 0); err != nil {\n\t\treturn nil, err\n\t}\n\tif vIdleConns, err = extractCustomConfig(&query, \"max_idle_conns\", runtime.GOMAXPROCS(-1)*2); err != nil {\n\t\treturn nil, err\n\t}\n\tif vIdleTime, err = extractCustomConfig(&query, \"max_idle_time\", 300); err != nil {\n\t\treturn nil, err\n\t}\n\tif vLifeTime, err = extractCustomConfig(&query, \"max_life_time\", 0); err != nil {\n\t\treturn nil, err\n\t}\n\tvar tablePrefix string\n\tif tablePrefix, err = extractCustomConfig(&query, \"table_prefix\", \"\"); err != nil {\n\t\treturn nil, err\n\t}\n\tif tablePrefix == \"\" {\n\t\ttablePrefix = \"jfs_\"\n\t} else {\n\t\ttablePrefix = \"jfs_\" + tablePrefix + \"_\"\n\t}\n\n\tif driver == \"sqlite3\" {\n\t\tif !query.Has(\"cache\") {\n\t\t\tquery.Add(\"cache\", \"shared\")\n\t\t}\n\t\tif !query.Has(\"_journal\") && !query.Has(\"_journal_mode\") {\n\t\t\tquery.Add(\"_journal\", \"WAL\")\n\t\t}\n\t\tif !query.Has(\"_timeout\") && !query.Has(\"_busy_timeout\") {\n\t\t\tquery.Add(\"_timeout\", \"5000\")\n\t\t}\n\t}\n\n\tif encode := query.Encode(); encode != \"\" {\n\t\taddr = fmt.Sprintf(\"%s?%s\", baseUrl, encode)\n\t} else {\n\t\taddr = baseUrl\n\t}\n\n\tif driver == \"postgres\" {\n\t\taddr = driver + \"://\" + addr\n\t\tdriver = \"pgx\"\n\n\t\tparse, err := url.Parse(addr)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"parse url %s failed: %s\", addr, err)\n\t\t}\n\t\tsearchPath = parse.Query().Get(\"search_path\")\n\t\tif searchPath != \"\" {\n\t\t\tif len(strings.Split(searchPath, \",\")) > 1 {\n\t\t\t\treturn nil, fmt.Errorf(\"currently, only one schema is supported in search_path\")\n\t\t\t}\n\t\t}\n\t}\n\n\t// escaping is not necessary for mysql password https://github.com/go-sql-driver/mysql#password\n\tif driver == \"mysql\" && setTransactionIsolation != nil {\n\t\taddr = recoveryMysqlPwd(addr)\n\t\tvar err error\n\t\tif addr, err = setTransactionIsolation(addr); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif driver == \"sqlite3\" {\n\t\tDirBatchNum[\"db\"] = 4096 // SQLITE_MAX_VARIABLE_NUMBER limit\n\t}\n\n\tengine, err := xorm.NewEngine(driver, addr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to use data source %s: %s\", driver, err)\n\t}\n\tswitch logger.Level { // make xorm less verbose\n\tcase logrus.TraceLevel:\n\t\tengine.SetLogLevel(log.LOG_DEBUG)\n\tcase logrus.DebugLevel:\n\t\tengine.SetLogLevel(log.LOG_INFO)\n\tcase logrus.InfoLevel, logrus.WarnLevel:\n\t\tengine.SetLogLevel(log.LOG_WARNING)\n\tcase logrus.ErrorLevel:\n\t\tengine.SetLogLevel(log.LOG_ERR)\n\tdefault:\n\t\tengine.SetLogLevel(log.LOG_OFF)\n\t}\n\tstart := time.Now()\n\tif err = engine.Ping(); err != nil {\n\t\treturn nil, fmt.Errorf(\"ping database: %s\", err)\n\t}\n\tif time.Since(start) > time.Millisecond*5 {\n\t\tlogger.Warnf(\"The latency to database is too high: %s\", time.Since(start))\n\t}\n\tif searchPath != \"\" {\n\t\tengine.SetSchema(searchPath)\n\t}\n\tif vOpenConns > 0 {\n\t\tengine.DB().SetMaxOpenConns(vOpenConns)\n\t}\n\tif vLifeTime > 0 {\n\t\tengine.DB().SetConnMaxLifetime(time.Second * time.Duration(vLifeTime))\n\t}\n\tengine.DB().SetMaxIdleConns(vIdleConns)\n\tengine.DB().SetConnMaxIdleTime(time.Second * time.Duration(vIdleTime))\n\tengine.SetTableMapper(prefixMapper{mapper: engine.GetTableMapper(), prefix: tablePrefix})\n\tm := &dbMeta{\n\t\tbaseMeta: newBaseMeta(addr, conf),\n\t\tdb: engine,\n\t\tstatement: make(map[string]string),\n\t\ttablePrefix: tablePrefix,\n\t}\n\tm.initStatement()\n\tm.spool = &sync.Pool{\n\t\tNew: func() interface{} {\n\t\t\ts := engine.NewSession()\n\t\t\truntime.SetFinalizer(s, func(s *xorm.Session) {\n\t\t\t\t_ = s.Close()\n\t\t\t})\n\t\t\treturn s\n\t\t},\n\t}\n\tm.en = m\n\treturn m, nil\n}\n\nfunc (m *dbMeta) Shutdown() error {\n\treturn m.db.Close()\n}\n\nfunc (m *dbMeta) Name() string {\n\tname := m.db.DriverName()\n\tif name == \"pgx\" {\n\t\tname = \"postgres\"\n\t}\n\treturn name\n}\n\nfunc (m *dbMeta) doDeleteSlice(id uint64, size uint32) error {\n\treturn m.txn(func(s *xorm.Session) error {\n\t\t_, err := s.Delete(&sliceRef{Id: id})\n\t\treturn err\n\t})\n}\n\nfunc (m *dbMeta) syncTable(beans ...interface{}) error {\n\terr := m.db.Sync2(beans...)\n\tif err != nil && strings.Contains(err.Error(), \"Duplicate key\") {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (m *dbMeta) syncAllTables() error {\n\tif err := m.syncTable(new(setting), new(counter)); err != nil {\n\t\treturn fmt.Errorf(\"create table setting, counter: %s\", err)\n\t}\n\tif err := m.syncTable(new(edge)); err != nil {\n\t\treturn fmt.Errorf(\"create table edge: %s\", err)\n\t}\n\tif err := m.syncTable(new(node), new(symlink), new(xattr)); err != nil {\n\t\treturn fmt.Errorf(\"create table node, symlink, xattr: %s\", err)\n\t}\n\tif err := m.syncTable(new(chunk), new(sliceRef), new(delslices)); err != nil {\n\t\treturn fmt.Errorf(\"create table chunk, chunk_ref, delslices: %s\", err)\n\t}\n\tif err := m.syncTable(new(session2), new(sustained), new(delfile)); err != nil {\n\t\treturn fmt.Errorf(\"create table session2, sustaind, delfile: %s\", err)\n\t}\n\tif err := m.syncTable(new(flock), new(plock), new(dirQuota), new(userGroupQuota)); err != nil {\n\t\treturn fmt.Errorf(\"create table flock, plock, dirQuota, userGroupQuota: %s\", err)\n\t}\n\tif err := m.syncTable(new(dirStats)); err != nil {\n\t\treturn fmt.Errorf(\"create table dirStats: %s\", err)\n\t}\n\tif err := m.syncTable(new(detachedNode)); err != nil {\n\t\treturn fmt.Errorf(\"create table detachedNode: %s\", err)\n\t}\n\tif err := m.syncTable(new(acl)); err != nil {\n\t\treturn fmt.Errorf(\"create table acl: %s\", err)\n\t}\n\tif err := m.syncTable(new(delegationToken)); err != nil {\n\t\treturn fmt.Errorf(\"create table delegationToken: %s\", err)\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) doInit(format *Format, force bool) error {\n\tif err := m.syncAllTables(); err != nil {\n\t\treturn err\n\t}\n\tvar s = setting{Name: \"format\"}\n\tvar ok bool\n\terr := m.simpleTxn(Background(), func(ses *xorm.Session) (err error) {\n\t\tok, err = ses.Get(&s)\n\t\treturn err\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif ok {\n\t\tvar old Format\n\t\terr = json.Unmarshal([]byte(s.Value), &old)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"json: %s\", err)\n\t\t}\n\t\tif !old.DirStats && format.DirStats {\n\t\t\t// remove dir stats as they are outdated\n\t\t\t_, err = m.db.Where(\"TRUE\").Delete(new(dirStats))\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"drop table dirStats\")\n\t\t\t}\n\t\t}\n\t\tif !old.UserGroupQuota && format.UserGroupQuota {\n\t\t\t// remove user group quota as they are outdated\n\t\t\t_, err = m.db.Where(\"TRUE\").Delete(new(userGroupQuota))\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"drop table userGroupQuota\")\n\t\t\t}\n\t\t}\n\t\tif err = format.update(&old, force); err != nil {\n\t\t\treturn errors.Wrap(err, \"update format\")\n\t\t}\n\t}\n\n\tdata, err := json.MarshalIndent(format, \"\", \"\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"json: %s\", err)\n\t}\n\n\tm.fmt = format\n\tn := &node{\n\t\tType: TypeDirectory,\n\t\tNlink: 2,\n\t\tLength: 4 << 10,\n\t\tParent: RootInode,\n\t}\n\tnow := time.Now().UnixNano()\n\tn.setAtime(now)\n\tn.setMtime(now)\n\tn.setCtime(now)\n\treturn m.txn(func(s *xorm.Session) error {\n\t\tif format.TrashDays > 0 {\n\t\t\tok2, err := s.ForUpdate().Get(&node{Inode: TrashInode})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif !ok2 {\n\t\t\t\tn.Inode = TrashInode\n\t\t\t\tn.Mode = 0555\n\t\t\t\tif err = mustInsert(s, n); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif ok {\n\t\t\t_, err = s.Update(&setting{\"format\", string(data)}, &setting{Name: \"format\"})\n\t\t\treturn err\n\t\t} else {\n\t\t\tvar set = &setting{\"format\", string(data)}\n\t\t\tif n, err := s.Insert(set); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if n == 0 {\n\t\t\t\treturn fmt.Errorf(\"format is not inserted\")\n\t\t\t}\n\t\t}\n\n\t\tn.Inode = RootInode\n\t\tn.Mode = 0777\n\t\tvar cs = []counter{\n\t\t\t{\"nextInode\", 2}, // 1 is root\n\t\t\t{\"nextChunk\", 1},\n\t\t\t{\"nextSession\", 0},\n\t\t\t{\"usedSpace\", 0},\n\t\t\t{\"totalInodes\", 0},\n\t\t\t{\"nextCleanupSlices\", 0},\n\t\t}\n\t\treturn mustInsert(s, n, &cs)\n\t})\n}\n\nfunc (m *dbMeta) cacheACLs(ctx Context) error {\n\tif !m.getFormat().EnableACL {\n\t\treturn nil\n\t}\n\treturn m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Table(&acl{}).Iterate(new(acl), func(idx int, bean interface{}) error {\n\t\t\ta := bean.(*acl)\n\t\t\tm.aclCache.Put(a.Id, a.toRule())\n\t\t\treturn nil\n\t\t})\n\t})\n}\n\nfunc (m *dbMeta) Reset() error {\n\tm.Lock()\n\tdefer m.Unlock()\n\treturn m.db.DropTables(&setting{}, &counter{},\n\t\t&node{}, &edge{}, &symlink{}, &xattr{},\n\t\t&chunk{}, &sliceRef{}, &delslices{},\n\t\t&session{}, &session2{}, &sustained{}, &delfile{},\n\t\t&flock{}, &plock{}, &dirStats{}, &dirQuota{}, &userGroupQuota{}, &detachedNode{}, &acl{}, &delegationToken{})\n}\n\nfunc (m *dbMeta) doLoad() (data []byte, err error) {\n\terr = m.simpleTxn(Background(), func(ses *xorm.Session) error {\n\t\tif ok, err := ses.IsTableExist(&setting{}); err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn nil\n\t\t}\n\t\ts := setting{Name: \"format\"}\n\t\tok, err := ses.Get(&s)\n\t\tif err == nil && ok {\n\t\t\tdata = []byte(s.Value)\n\t\t}\n\t\treturn err\n\t})\n\treturn\n}\n\nfunc (m *dbMeta) doNewSession(sinfo []byte, update bool) error {\n\t// add new table\n\terr := m.syncTable(new(session2), new(delslices), new(dirStats), new(detachedNode), new(dirQuota), new(userGroupQuota), new(acl), new(delegationToken))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"update table session2, delslices, dirstats, detachedNode, dirQuota, userGroupQuota, acl: %s\", err)\n\t}\n\t// add node table\n\tif err = m.syncTable(new(node)); err != nil {\n\t\treturn fmt.Errorf(\"update table node: %s\", err)\n\t}\n\t// add primary key\n\tif err = m.syncTable(new(edge), new(chunk), new(xattr), new(sustained)); err != nil {\n\t\treturn fmt.Errorf(\"update table edge, chunk, xattr, sustained: %s\", err)\n\t}\n\t// update the owner from uint64 to int64\n\tif err = m.syncTable(new(flock), new(plock)); err != nil {\n\t\treturn fmt.Errorf(\"update table flock, plock: %s\", err)\n\t}\n\n\tfor {\n\t\tbeans := session2{Sid: m.sid, Expire: m.expireTime(), Info: sinfo}\n\t\tif update {\n\t\t\treturn m.txn(func(s *xorm.Session) error {\n\t\t\t\t_, err = s.Cols(\"expire\", \"info\").Update(&beans, &session2{Sid: beans.Sid})\n\t\t\t\treturn err\n\t\t\t})\n\t\t} else {\n\t\t\tif err = m.txn(func(s *xorm.Session) error {\n\t\t\t\treturn mustInsert(s, &beans)\n\t\t\t}); err == nil {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tif isDuplicateEntryErr(err) {\n\t\t\t\tlogger.Warnf(\"session id %d is already used\", m.sid)\n\t\t\t\tif v, e := m.incrCounter(\"nextSession\", 1); e == nil {\n\t\t\t\t\tm.sid = uint64(v)\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\treturn fmt.Errorf(\"get session ID: %s\", e)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn fmt.Errorf(\"insert new session %d: %s\", m.sid, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) getSession(row interface{}, detail bool) (*Session, error) {\n\tvar s Session\n\tvar info []byte\n\tswitch row := row.(type) {\n\tcase *session2:\n\t\ts.Sid = row.Sid\n\t\ts.Expire = time.Unix(row.Expire, 0)\n\t\tinfo = row.Info\n\tcase *session:\n\t\ts.Sid = row.Sid\n\t\ts.Expire = time.Unix(row.Heartbeat, 0).Add(time.Minute * 5)\n\t\tinfo = row.Info\n\t\tif info == nil { // legacy client has no info\n\t\t\tinfo = []byte(\"{}\")\n\t\t}\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"invalid type: %T\", row)\n\t}\n\tif err := json.Unmarshal(info, &s); err != nil {\n\t\treturn nil, fmt.Errorf(\"corrupted session info; json error: %s\", err)\n\t}\n\tif detail {\n\t\tvar (\n\t\t\tsrows []sustained\n\t\t\tfrows []flock\n\t\t\tprows []plock\n\t\t)\n\t\terr := m.roTxn(Background(), func(ses *xorm.Session) error {\n\t\t\tif err := ses.Find(&srows, &sustained{Sid: s.Sid}); err != nil {\n\t\t\t\treturn fmt.Errorf(\"find sustained %d: %s\", s.Sid, err)\n\t\t\t}\n\t\t\ts.Sustained = make([]Ino, 0, len(srows))\n\t\t\tfor _, srow := range srows {\n\t\t\t\ts.Sustained = append(s.Sustained, srow.Inode)\n\t\t\t}\n\n\t\t\tif err := ses.Find(&frows, &flock{Sid: s.Sid}); err != nil {\n\t\t\t\treturn fmt.Errorf(\"find flock %d: %s\", s.Sid, err)\n\t\t\t}\n\t\t\ts.Flocks = make([]Flock, 0, len(frows))\n\t\t\tfor _, frow := range frows {\n\t\t\t\ts.Flocks = append(s.Flocks, Flock{frow.Inode, uint64(frow.Owner), string(frow.Ltype)})\n\t\t\t}\n\n\t\t\tif err := ses.Find(&prows, &plock{Sid: s.Sid}); err != nil {\n\t\t\t\treturn fmt.Errorf(\"find plock %d: %s\", s.Sid, err)\n\t\t\t}\n\t\t\ts.Plocks = make([]Plock, 0, len(prows))\n\t\t\tfor _, prow := range prows {\n\t\t\t\ts.Plocks = append(s.Plocks, Plock{prow.Inode, uint64(prow.Owner), loadLocks(prow.Records)})\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn &s, nil\n}\n\nfunc (m *dbMeta) GetSession(sid uint64, detail bool) (s *Session, err error) {\n\terr = m.roTxn(Background(), func(ses *xorm.Session) error {\n\t\tif ok, err := ses.IsTableExist(&session2{}); err != nil {\n\t\t\treturn err\n\t\t} else if ok {\n\t\t\trow := session2{Sid: sid}\n\t\t\tif ok, err = ses.Get(&row); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if ok {\n\t\t\t\ts, err = m.getSession(&row, detail)\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif ok, err := ses.IsTableExist(&session{}); err != nil {\n\t\t\treturn err\n\t\t} else if ok {\n\t\t\trow := session{Sid: sid}\n\t\t\tif ok, err = ses.Get(&row); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if ok {\n\t\t\t\ts, err = m.getSession(&row, detail)\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"session not found: %d\", sid)\n\t})\n\treturn\n}\n\nfunc (m *dbMeta) ListSessions() ([]*Session, error) {\n\tvar sessions []*Session\n\terr := m.roTxn(Background(), func(ses *xorm.Session) error {\n\t\tif ok, err := ses.IsTableExist(&session2{}); err != nil {\n\t\t\treturn err\n\t\t} else if ok {\n\t\t\tvar rows []session2\n\t\t\tif err = ses.Find(&rows); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsessions = make([]*Session, 0, len(rows))\n\t\t\tfor i := range rows {\n\t\t\t\ts, err := m.getSession(&rows[i], false)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"get session: %s\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsessions = append(sessions, s)\n\t\t\t}\n\t\t}\n\t\tif ok, err := ses.IsTableExist(&session{}); err != nil {\n\t\t\tlogger.Errorf(\"Check legacy session table: %s\", err)\n\t\t} else if ok {\n\t\t\tvar lrows []session\n\t\t\tif err = ses.Find(&lrows); err != nil {\n\t\t\t\tlogger.Errorf(\"Scan legacy sessions: %s\", err)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tfor i := range lrows {\n\t\t\t\ts, err := m.getSession(&lrows[i], false)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"Get legacy session: %s\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsessions = append(sessions, s)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\treturn sessions, err\n}\n\nfunc (m *dbMeta) getCounter(name string) (v int64, err error) {\n\terr = m.simpleTxn(Background(), func(s *xorm.Session) error {\n\t\tc := counter{Name: name}\n\t\t_, err := s.Get(&c)\n\t\tif err == nil {\n\t\t\tv = c.Value\n\t\t}\n\t\treturn err\n\t})\n\treturn\n}\n\nfunc (m *dbMeta) incrCounter(name string, value int64) (v int64, err error) {\n\terr = m.txn(func(s *xorm.Session) error {\n\t\tv, err = m.incrSessionCounter(s, name, value)\n\t\treturn err\n\t})\n\treturn\n}\n\nfunc (m *dbMeta) incrSessionCounter(s *xorm.Session, name string, value int64) (v int64, err error) {\n\tvar c = counter{Name: name}\n\tok, err := s.ForUpdate().Get(&c)\n\tif err != nil {\n\t\treturn\n\t}\n\tv = c.Value + value\n\tif value > 0 {\n\t\tc.Value = v\n\t\tif ok {\n\t\t\t_, err = s.Cols(\"value\").Update(&c, &counter{Name: name})\n\t\t} else {\n\t\t\terr = mustInsert(s, &c)\n\t\t}\n\t}\n\treturn\n}\n\nfunc (m *dbMeta) setIfSmall(name string, value, diff int64) (bool, error) {\n\tvar changed bool\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tchanged = false\n\t\tc := counter{Name: name}\n\t\tok, err := s.ForUpdate().Get(&c)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif c.Value > value-diff {\n\t\t\treturn nil\n\t\t} else {\n\t\t\tchanged = true\n\t\t\tc.Value = value\n\t\t\tif ok {\n\t\t\t\t_, err = s.Cols(\"value\").Update(&c, &counter{Name: name})\n\t\t\t} else {\n\t\t\t\terr = mustInsert(s, &c)\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t})\n\n\treturn changed, err\n}\n\nfunc mustInsert(s *xorm.Session, beans ...interface{}) error {\n\tfor start, end, size := 0, 0, len(beans); end < size; start = end {\n\t\tend = start + 200\n\t\tif end > size {\n\t\t\tend = size\n\t\t}\n\t\tif n, err := s.Insert(beans[start:end]...); err != nil {\n\t\t\treturn err\n\t\t} else if d := end - start - int(n); d > 0 {\n\t\t\treturn fmt.Errorf(\"%d records not inserted: %+v\", d, beans[start:end])\n\t\t}\n\t}\n\treturn nil\n}\n\nvar errBusy error\n\nfunc (m *dbMeta) shouldRetry(err error) bool {\n\tif m.Name() == \"mysql\" && err == syscall.EBUSY {\n\t\t// Retry transaction when parent node update return 0 rows in MySQL\n\t\treturn true\n\t}\n\n\tmsg := strings.ToLower(err.Error())\n\tif strings.Contains(msg, \"too many connections\") || strings.Contains(msg, \"too many clients\") {\n\t\tlogger.Warnf(\"transaction failed: %s, will retry it. please increase the max number of connections in your database, or use a connection pool.\", msg)\n\t\treturn true\n\t}\n\tswitch m.Name() {\n\tcase \"sqlite3\":\n\t\treturn errors.Is(err, errBusy) || strings.Contains(msg, \"database is locked\")\n\tcase \"mysql\":\n\t\t// MySQL, MariaDB or TiDB\n\t\t// error 1020 for MariaDB when conflict\n\t\treturn strings.Contains(msg, \"try restarting transaction\") || strings.Contains(msg, \"try again later\") ||\n\t\t\tstrings.Contains(msg, \"duplicate entry\") || strings.Contains(msg, \"error 1020 (hy000)\") ||\n\t\t\tstrings.Contains(msg, \"invalid connection\") || strings.Contains(msg, \"bad connection\") || errors.Is(err, io.EOF) // could not send data to client: No buffer space available\n\tcase \"postgres\":\n\t\tif e, ok := err.(interface{ SafeToRetry() bool }); ok {\n\t\t\treturn e.SafeToRetry()\n\t\t}\n\t\treturn strings.Contains(msg, \"current transaction is aborted\") || strings.Contains(msg, \"deadlock detected\") ||\n\t\t\tstrings.Contains(msg, \"duplicate key value\") || strings.Contains(msg, \"could not serialize access\") ||\n\t\t\tstrings.Contains(msg, \"bad connection\") || errors.Is(err, io.EOF) // could not send data to client: No buffer space available\n\tdefault:\n\t\treturn false\n\t}\n}\n\nfunc (m *dbMeta) txn(f func(s *xorm.Session) error, inodes ...Ino) error {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tstart := time.Now()\n\tdefer func() { m.txDist.Observe(time.Since(start).Seconds()) }()\n\n\tif m.Name() == \"sqlite3\" {\n\t\t// sqlite only allow one writer at a time\n\t\tinodes = []Ino{1}\n\t}\n\n\tdefer m.txBatchLock(inodes...)()\n\tvar (\n\t\tlastErr error\n\t\tmethod txMethod\n\t)\n\tfor i := 0; i < 50; i++ {\n\t\t_, err := m.db.Transaction(func(s *xorm.Session) (interface{}, error) {\n\t\t\treturn nil, f(s)\n\t\t})\n\t\tif eno, ok := err.(syscall.Errno); ok && eno == 0 {\n\t\t\terr = nil\n\t\t}\n\t\tif err != nil && m.shouldRetry(err) {\n\t\t\tm.txRestart.WithLabelValues(method.name(context.TODO())).Add(1)\n\t\t\tlogger.Debugf(\"Transaction failed, restart it (tried %d): %s\", i+1, err)\n\t\t\tlastErr = err\n\t\t\ttime.Sleep(time.Millisecond * time.Duration(i*i))\n\t\t\tcontinue\n\t\t} else if err == nil && i > 1 {\n\t\t\tlogger.Warnf(\"Transaction succeeded after %d tries (%s), inodes: %v, method: %s, last error: %s\", i+1, time.Since(start), inodes, method.name(context.TODO()), lastErr)\n\t\t}\n\t\treturn err\n\t}\n\tlogger.Warnf(\"Already tried 50 times, returning: %s\", lastErr)\n\treturn lastErr\n}\n\nfunc (m *dbMeta) roTxn(ctx context.Context, f func(s *xorm.Session) error) error {\n\tstart := time.Now()\n\tdefer func() { m.txDist.Observe(time.Since(start).Seconds()) }()\n\ts := m.db.NewSession()\n\tdefer s.Close()\n\tvar opt sql.TxOptions\n\tif !m.noReadOnlyTxn {\n\t\topt.ReadOnly = true\n\t\topt.Isolation = sql.LevelRepeatableRead\n\t}\n\n\tvar maxRetry int\n\tval := ctx.Value(txMaxRetryKey{})\n\tif val == nil {\n\t\tmaxRetry = 50\n\t} else {\n\t\tmaxRetry = val.(int)\n\t}\n\tvar (\n\t\tlastErr error\n\t\tmethod txMethod\n\t)\n\tfor i := 0; i < maxRetry; i++ {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn ctx.Err()\n\t\tdefault:\n\t\t}\n\t\terr := s.BeginTx(&opt)\n\t\tif err != nil && opt.ReadOnly && (strings.Contains(err.Error(), \"READ\") || strings.Contains(err.Error(), \"driver does not support read-only transactions\")) {\n\t\t\tlogger.Warnf(\"the database does not support read-only transaction\")\n\t\t\tm.noReadOnlyTxn = true\n\t\t\topt = sql.TxOptions{} // use default level\n\t\t\terr = s.BeginTx(&opt)\n\t\t}\n\t\tif err != nil {\n\t\t\tlogger.Debugf(\"Start transaction failed, try again (tried %d): %s\", i+1, err)\n\t\t\tlastErr = err\n\t\t\ttime.Sleep(time.Millisecond * time.Duration(i*i))\n\t\t\tcontinue\n\t\t}\n\t\terr = f(s)\n\t\tif eno, ok := err.(syscall.Errno); ok && eno == 0 {\n\t\t\terr = nil\n\t\t}\n\t\t_ = s.Rollback()\n\t\tif err != nil && m.shouldRetry(err) {\n\t\t\tm.txRestart.WithLabelValues(method.name(ctx)).Add(1)\n\t\t\tlogger.Debugf(\"Read transaction failed, restart it (tried %d): %s\", i+1, err)\n\t\t\tlastErr = err\n\t\t\ttime.Sleep(time.Millisecond * time.Duration(i*i))\n\t\t\tcontinue\n\t\t} else if err == nil && i > 1 {\n\t\t\tlogger.Warnf(\"Read transaction succeeded after %d tries (%s), method: %s, last error: %s\", i+1, time.Since(start), method.name(ctx), lastErr)\n\t\t}\n\t\treturn err\n\t}\n\tlogger.Warnf(\"Already tried %d times, returning: %s\", maxRetry, lastErr)\n\treturn lastErr\n}\n\nfunc (m *dbMeta) simpleTxn(ctx context.Context, f func(s *xorm.Session) error) error {\n\tstart := time.Now()\n\tdefer func() { m.txDist.Observe(time.Since(start).Seconds()) }()\n\ts := m.spool.Get().(*xorm.Session)\n\tdefer m.spool.Put(s)\n\n\tvar (\n\t\tmaxRetry = 50\n\t\tlastErr error\n\t\tmethod txMethod\n\t)\n\tfor i := 0; i < maxRetry; i++ {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn ctx.Err()\n\t\tdefault:\n\t\t}\n\t\terr := f(s)\n\t\tif eno, ok := err.(syscall.Errno); ok && eno == 0 {\n\t\t\terr = nil\n\t\t}\n\t\tif err != nil && m.shouldRetry(err) {\n\t\t\tm.txRestart.WithLabelValues(method.name(ctx)).Add(1)\n\t\t\tlogger.Debugf(\"Read transaction failed, restart it (tried %d): %s\", i+1, err)\n\t\t\tlastErr = err\n\t\t\ttime.Sleep(time.Millisecond * time.Duration(i*i))\n\t\t\tcontinue\n\t\t} else if err == nil && i > 1 {\n\t\t\tlogger.Warnf(\"Simple transaction succeeded after %d tries (%s), method: %s, last error: %s\", i+1, time.Since(start), method.name(ctx), lastErr)\n\t\t}\n\t\treturn err\n\t}\n\tlogger.Warnf(\"Already tried %d times, returning: %s\", maxRetry, lastErr)\n\treturn lastErr\n}\n\nfunc (m *dbMeta) parseAttr(n *node, attr *Attr) {\n\tif attr == nil || n == nil {\n\t\treturn\n\t}\n\tattr.Typ = n.Type\n\tattr.Mode = n.Mode\n\tattr.Flags = n.Flags\n\tattr.Uid = n.Uid\n\tattr.Gid = n.Gid\n\tattr.Atime = n.Atime / 1e6\n\tattr.Atimensec = uint32(n.Atime%1e6*1000) + uint32(n.Atimensec)\n\tattr.Mtime = n.Mtime / 1e6\n\tattr.Mtimensec = uint32(n.Mtime%1e6*1000) + uint32(n.Mtimensec)\n\tattr.Ctime = n.Ctime / 1e6\n\tattr.Ctimensec = uint32(n.Ctime%1e6*1000) + uint32(n.Ctimensec)\n\tattr.Nlink = n.Nlink\n\tattr.Length = n.Length\n\tattr.Rdev = n.Rdev\n\tattr.Parent = n.Parent\n\tattr.Full = true\n\tattr.AccessACL = n.AccessACLId\n\tattr.DefaultACL = n.DefaultACLId\n}\n\nfunc (m *dbMeta) parseNode(attr *Attr, n *node) {\n\tif attr == nil || n == nil {\n\t\treturn\n\t}\n\tn.Type = attr.Typ\n\tn.Mode = attr.Mode\n\tn.Flags = attr.Flags\n\tn.Uid = attr.Uid\n\tn.Gid = attr.Gid\n\tn.setAtime(attr.Atime*1e9 + int64(attr.Atimensec))\n\tn.setMtime(attr.Mtime*1e9 + int64(attr.Mtimensec))\n\tn.setCtime(attr.Ctime*1e9 + int64(attr.Ctimensec))\n\tn.Nlink = attr.Nlink\n\tn.Length = attr.Length\n\tn.Rdev = attr.Rdev\n\tn.Parent = attr.Parent\n\tn.AccessACLId = attr.AccessACL\n\tn.DefaultACLId = attr.DefaultACL\n}\n\nfunc (m *dbMeta) updateStats(space int64, inodes int64) {\n\tatomic.AddInt64(&m.newSpace, space)\n\tatomic.AddInt64(&m.newInodes, inodes)\n}\n\nfunc (m *dbMeta) doSyncVolumeStat(ctx Context) error {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tvar used, inode int64\n\tif err := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\ttotal, err := s.SumsInt(&dirStats{}, \"used_space\", \"used_inodes\")\n\t\tused += total[0]\n\t\tinode += total[1]\n\t\treturn err\n\t}); err != nil {\n\t\treturn err\n\t}\n\tif err := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tqueryResultMap, err := s.QueryString(m.sqlConv(\"SELECT length FROM node WHERE inode IN (SELECT inode FROM sustained)\"))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, v := range queryResultMap {\n\t\t\tvalue, err := strconv.ParseInt(v[\"length\"], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"parse sustained length: %s err: %s\", v[\"length\"], err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tused += align4K(uint64(value))\n\t\t\tinode += 1\n\t\t}\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tif err := m.scanTrashEntry(ctx, func(_ Ino, length uint64) {\n\t\tused += align4K(length)\n\t\tinode += 1\n\t}); err != nil {\n\t\treturn err\n\t}\n\tlogger.Debugf(\"Used space: %s, inodes: %d\", humanize.IBytes(uint64(used)), inode)\n\treturn m.txn(func(s *xorm.Session) error {\n\t\tif _, err := s.Cols(\"value\").Update(&counter{Value: inode}, &counter{Name: totalInodes}); err != nil {\n\t\t\treturn fmt.Errorf(\"update totalInodes: %s\", err)\n\t\t}\n\t\t_, err := s.Cols(\"value\").Update(&counter{Value: used}, &counter{Name: usedSpace})\n\t\treturn err\n\t})\n}\n\nfunc (m *dbMeta) doFlushStats() {\n\tnewSpace := atomic.LoadInt64(&m.newSpace)\n\tnewInodes := atomic.LoadInt64(&m.newInodes)\n\tif newSpace != 0 || newInodes != 0 {\n\t\terr := m.txn(func(s *xorm.Session) error {\n\t\t\tif _, err := s.Exec(m.sqlConv(\"update counter set value=value + ? where name='totalInodes'\"), newInodes); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t_, err := s.Exec(m.sqlConv(\"update counter set value= value + ? where name='usedSpace'\"), newSpace)\n\t\t\treturn err\n\t\t})\n\t\tif err != nil && !strings.Contains(err.Error(), \"attempt to write a readonly database\") {\n\t\t\tlogger.Warnf(\"update stats: %s\", err)\n\t\t}\n\t\tif err == nil {\n\t\t\tatomic.AddInt64(&m.newSpace, -newSpace)\n\t\t\tatomic.AddInt64(&m.usedSpace, newSpace)\n\t\t\tatomic.AddInt64(&m.newInodes, -newInodes)\n\t\t\tatomic.AddInt64(&m.usedInodes, newInodes)\n\t\t}\n\t}\n}\n\nfunc (m *dbMeta) doLookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr) syscall.Errno {\n\treturn errno(m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\ts = s.Table(&edge{})\n\t\tnn := namedNode{node: node{Parent: parent}, Name: []byte(name)}\n\t\tvar exist bool\n\t\tvar err error\n\t\tif attr != nil {\n\t\t\ts = s.Join(\"INNER\", &node{}, m.sqlConv(\"edge.inode=node.inode\"))\n\t\t\texist, err = s.Select(m.sqlConv(\"node.*\")).Get(&nn)\n\t\t} else {\n\t\t\texist, err = s.Select(\"*\").Get(&nn)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !exist {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\t*inode = nn.Inode\n\t\tm.parseAttr(&nn.node, attr)\n\t\tm.of.Update(nn.Inode, attr)\n\t\treturn nil\n\t}))\n}\n\nfunc (m *dbMeta) doGetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno {\n\treturn errno(m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tvar n = node{Inode: inode}\n\t\tok, err := s.Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(&n, attr)\n\t\treturn nil\n\t}))\n}\n\nfunc (m *dbMeta) doSetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr, oldAttr *Attr) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tvar cur = node{Inode: inode}\n\t\tok, err := s.ForUpdate().Get(&cur)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar curAttr Attr\n\t\tm.parseAttr(&cur, &curAttr)\n\t\tif oldAttr != nil {\n\t\t\t*oldAttr = curAttr\n\t\t}\n\t\tif curAttr.Parent > TrashInode {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tnow := time.Now()\n\n\t\trule, err := m.getACL(s, curAttr.AccessACL)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\trule = rule.Dup()\n\t\tdirtyAttr, st := m.mergeAttr(ctx, inode, set, &curAttr, attr, now, rule)\n\t\tif st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif dirtyAttr == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\tdirtyAttr.AccessACL, err = m.insertACL(s, rule)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tvar dirtyNode node\n\t\tm.parseNode(dirtyAttr, &dirtyNode)\n\t\tdirtyNode.setCtime(now.UnixNano())\n\t\t_, err = s.Cols(\"flags\", \"mode\", \"uid\", \"gid\", \"atime\", \"mtime\", \"ctime\",\n\t\t\t\"atimensec\", \"mtimensec\", \"ctimensec\", \"access_acl_id\", \"default_acl_id\").\n\t\t\tUpdate(&dirtyNode, &node{Inode: inode})\n\t\tif err == nil {\n\t\t\tm.parseAttr(&dirtyNode, attr)\n\t\t}\n\t\treturn err\n\t}, inode))\n}\n\nfunc (m *dbMeta) appendSlice(s *xorm.Session, inode Ino, indx uint32, buf []byte) error {\n\tvar r sql.Result\n\tvar err error\n\tdriver := m.Name()\n\tif driver == \"sqlite3\" || driver == \"postgres\" {\n\t\tr, err = s.Exec(m.sqlConv(\"update chunk set slices=slices || ? where inode=? AND indx=?\"), buf, inode, indx)\n\t} else {\n\t\tr, err = s.Exec(m.sqlConv(\"update chunk set slices=concat(slices, ?) where inode=? AND indx=?\"), buf, inode, indx)\n\t}\n\tif err == nil {\n\t\tif n, _ := r.RowsAffected(); n == 0 {\n\t\t\terr = mustInsert(s, &chunk{Inode: inode, Indx: indx, Slices: buf})\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *dbMeta) upsertSlice(s *xorm.Session, inode Ino, indx uint32, buf []byte, insert *bool) error {\n\tvar err error\n\tdriver := m.Name()\n\tif driver == \"sqlite3\" || driver == \"postgres\" {\n\t\t_, err = s.Exec(m.sqlConv(`\n\t\t\t INSERT INTO chunk (inode, indx, slices)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON CONFLICT (inode, indx)\n\t\t\t DO UPDATE SET slices=chunk.slices || ?`), inode, indx, buf, buf)\n\t} else {\n\t\tvar r sql.Result\n\t\tr, err = s.Exec(m.sqlConv(`\n\t\t\t INSERT INTO chunk (inode, indx, slices)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON DUPLICATE KEY UPDATE\n\t\t\t slices=concat(slices, ?)`), inode, indx, buf, buf)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tn, _ := r.RowsAffected()\n\t\t*insert = n == 1 // https://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html\n\t}\n\treturn err\n}\n\nfunc (m *dbMeta) doTruncate(ctx Context, inode Ino, flags uint8, length uint64, delta *dirStat, attr *Attr, skipPermCheck bool) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\t*delta = dirStat{}\n\t\tnodeAttr := node{Inode: inode}\n\t\tok, err := s.ForUpdate().Get(&nodeAttr)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif nodeAttr.Type != TypeFile || nodeAttr.Flags&(FlagImmutable|FlagAppend) != 0 || (flags == 0 && nodeAttr.Parent > TrashInode) {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tm.parseAttr(&nodeAttr, attr)\n\t\tif !skipPermCheck {\n\t\t\tif st := m.Access(ctx, inode, MODE_MASK_W, attr); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t}\n\t\tif length == nodeAttr.Length {\n\t\t\treturn nil\n\t\t}\n\t\tdelta.length = int64(length) - int64(nodeAttr.Length)\n\t\tdelta.space = align4K(length) - align4K(nodeAttr.Length)\n\t\tif err := m.checkQuota(ctx, delta.space, 0, nodeAttr.Uid, nodeAttr.Gid, m.getParents(s, inode, nodeAttr.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tvar zeroChunks []chunk\n\t\tvar left, right = nodeAttr.Length, length\n\t\tif left > right {\n\t\t\tright, left = left, right\n\t\t}\n\t\tif right/ChunkSize-left/ChunkSize > 1 {\n\t\t\terr := s.Where(\"inode = ? AND indx > ? AND indx < ?\", inode, left/ChunkSize, right/ChunkSize).Cols(\"indx\").ForUpdate().Find(&zeroChunks)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tl := uint32(right - left)\n\t\tif right > (left/ChunkSize+1)*ChunkSize {\n\t\t\tl = ChunkSize - uint32(left%ChunkSize)\n\t\t}\n\t\tif err = m.appendSlice(s, inode, uint32(left/ChunkSize), marshalSlice(uint32(left%ChunkSize), 0, 0, 0, l)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tbuf := marshalSlice(0, 0, 0, 0, ChunkSize)\n\t\tfor _, c := range zeroChunks {\n\t\t\tif err = m.appendSlice(s, inode, c.Indx, buf); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif right > (left/ChunkSize+1)*ChunkSize && right%ChunkSize > 0 {\n\t\t\tif err = m.appendSlice(s, inode, uint32(right/ChunkSize), marshalSlice(0, 0, 0, 0, uint32(right%ChunkSize))); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tnodeAttr.Length = length\n\t\tnow := time.Now().UnixNano()\n\t\tnodeAttr.setMtime(now)\n\t\tnodeAttr.setCtime(now)\n\t\tif _, err = s.Cols(\"length\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&nodeAttr, &node{Inode: nodeAttr.Inode}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.parseAttr(&nodeAttr, attr)\n\t\treturn nil\n\t}, inode))\n}\n\nfunc (m *dbMeta) doFallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, delta *dirStat, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\t*delta = dirStat{}\n\t\tnodeAttr := node{Inode: inode}\n\t\tok, err := s.ForUpdate().Get(&nodeAttr)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif nodeAttr.Type == TypeFIFO {\n\t\t\treturn syscall.EPIPE\n\t\t}\n\t\tif nodeAttr.Type != TypeFile || (nodeAttr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar t Attr\n\t\tm.parseAttr(&nodeAttr, &t)\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W, &t); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (nodeAttr.Flags&FlagAppend) != 0 && (mode&^fallocKeepSize) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tlength := nodeAttr.Length\n\t\tif off+size > nodeAttr.Length {\n\t\t\tif mode&fallocKeepSize == 0 {\n\t\t\t\tlength = off + size\n\t\t\t}\n\t\t}\n\n\t\told := nodeAttr.Length\n\t\tdelta.length = int64(length) - int64(old)\n\t\tdelta.space = align4K(length) - align4K(old)\n\t\tif err := m.checkQuota(ctx, delta.space, 0, nodeAttr.Uid, nodeAttr.Gid, m.getParents(s, inode, nodeAttr.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tnow := time.Now().UnixNano()\n\t\tnodeAttr.Length = length\n\t\tnodeAttr.setMtime(now)\n\t\tnodeAttr.setCtime(now)\n\t\tif _, err := s.Cols(\"length\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&nodeAttr, &node{Inode: inode}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif mode&(fallocZeroRange|fallocPunchHole) != 0 && off < old {\n\t\t\toff, size := off, size\n\t\t\tif off+size > old {\n\t\t\t\tsize = old - off\n\t\t\t}\n\t\t\tfor size > 0 {\n\t\t\t\tindx := uint32(off / ChunkSize)\n\t\t\t\tcoff := off % ChunkSize\n\t\t\t\tl := size\n\t\t\t\tif coff+size > ChunkSize {\n\t\t\t\t\tl = ChunkSize - coff\n\t\t\t\t}\n\t\t\t\terr = m.appendSlice(s, inode, indx, marshalSlice(uint32(coff), 0, 0, 0, uint32(l)))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\toff += l\n\t\t\t\tsize -= l\n\t\t\t}\n\t\t}\n\t\tm.parseAttr(&nodeAttr, attr)\n\t\treturn nil\n\t}, inode))\n}\n\nfunc (m *dbMeta) doReadlink(ctx Context, inode Ino, noatime bool) (atime int64, target []byte, err error) {\n\tif noatime {\n\t\terr = m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\t\tvar l = symlink{Inode: inode}\n\t\t\tok, err := s.Get(&l)\n\t\t\tif err == nil && ok {\n\t\t\t\ttarget = l.Target\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t\treturn\n\t}\n\n\tattr := &Attr{}\n\tnow := time.Now()\n\terr = m.txn(func(s *xorm.Session) error {\n\t\tnodeAttr := node{Inode: inode}\n\t\tok, e := s.ForUpdate().Get(&nodeAttr)\n\t\tif e != nil {\n\t\t\treturn e\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif nodeAttr.Type != TypeSymlink {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tl := symlink{Inode: inode}\n\t\tok, e = s.Get(&l)\n\t\tif e != nil {\n\t\t\treturn e\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.EIO\n\t\t}\n\t\tm.parseAttr(&nodeAttr, attr)\n\t\ttarget = l.Target\n\t\tif !m.atimeNeedsUpdate(attr, now) {\n\t\t\tatime = attr.Atime*int64(time.Second) + int64(attr.Atimensec)\n\t\t\treturn nil\n\t\t}\n\t\tnodeAttr.setAtime(now.UnixNano())\n\t\tatime = now.UnixNano()\n\t\t_, e = s.Cols(\"atime\", \"atimensec\").Update(&nodeAttr, &node{Inode: inode})\n\t\treturn e\n\t}, inode)\n\treturn\n}\n\nfunc (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode, cumask uint16, path string, inode *Ino, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tvar pn = node{Inode: parent}\n\t\tok, err := s.Get(&pn)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif pn.Type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr(&pn, &pattr)\n\t\tif pattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pn.Flags & FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar e = edge{Parent: parent, Name: []byte(name)}\n\t\tok, err = s.Get(&e)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvar foundIno Ino\n\t\tvar foundType uint8\n\t\tif ok {\n\t\t\tfoundType, foundIno = e.Type, e.Inode\n\t\t} else if m.conf.CaseInsensi {\n\t\t\tif entry := m.resolveCase(ctx, parent, name); entry != nil {\n\t\t\t\tfoundType, foundIno = entry.Attr.Typ, entry.Inode\n\t\t\t}\n\t\t}\n\t\tif foundIno != 0 {\n\t\t\tif _type == TypeFile || _type == TypeDirectory {\n\t\t\t\tfoundNode := node{Inode: foundIno}\n\t\t\t\tok, err = s.Get(&foundNode)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t} else if ok {\n\t\t\t\t\tm.parseAttr(&foundNode, attr)\n\t\t\t\t} else if attr != nil {\n\t\t\t\t\t*attr = Attr{Typ: foundType, Parent: parent} // corrupt entry\n\t\t\t\t}\n\t\t\t\t*inode = foundIno\n\t\t\t}\n\t\t\treturn syscall.EEXIST\n\t\t} else if parent == TrashInode {\n\t\t\tif next, err := m.incrSessionCounter(s, \"nextTrash\", 1); err != nil {\n\t\t\t\treturn err\n\t\t\t} else {\n\t\t\t\t*inode = TrashInode + Ino(next)\n\t\t\t}\n\t\t}\n\n\t\tn := node{Inode: *inode}\n\t\tm.parseNode(attr, &n)\n\t\tmode &= 07777\n\t\tif pattr.DefaultACL != aclAPI.None && _type != TypeSymlink {\n\t\t\t// inherit default acl\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tn.DefaultACLId = pattr.DefaultACL\n\t\t\t}\n\n\t\t\t// set access acl by parent's default acl\n\t\t\trule, err := m.getACL(s, pattr.DefaultACL)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif rule.IsMinimal() {\n\t\t\t\t// simple acl as default\n\t\t\t\tn.Mode = mode & (0xFE00 | rule.GetMode())\n\t\t\t} else {\n\t\t\t\tcRule := rule.ChildAccessACL(mode)\n\t\t\t\tid, err := m.insertACL(s, cRule)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tn.AccessACLId = id\n\t\t\t\tn.Mode = (mode & 0xFE00) | cRule.GetMode()\n\t\t\t}\n\t\t} else {\n\t\t\tn.Mode = mode & ^cumask\n\t\t}\n\t\tif (pn.Flags & FlagSkipTrash) != 0 {\n\t\t\tn.Flags |= FlagSkipTrash\n\t\t}\n\n\t\tvar updateParent bool\n\t\tvar nlinkAdjust int32\n\t\tnow := time.Now().UnixNano()\n\t\tif parent != TrashInode {\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tpn.Nlink++\n\t\t\t\tupdateParent = true\n\t\t\t\tnlinkAdjust++\n\t\t\t}\n\t\t\tif updateParent || time.Duration(now-pn.getMtime()) >= m.conf.SkipDirMtime {\n\t\t\t\tpn.setMtime(now)\n\t\t\t\tpn.setCtime(now)\n\t\t\t\tupdateParent = true\n\t\t\t}\n\t\t}\n\t\tn.setAtime(now)\n\t\tn.setMtime(now)\n\t\tn.setCtime(now)\n\t\tif ctx.Value(CtxKey(\"behavior\")) == \"Hadoop\" || runtime.GOOS == \"darwin\" {\n\t\t\tn.Gid = pn.Gid\n\t\t} else if runtime.GOOS == \"linux\" && pn.Mode&02000 != 0 {\n\t\t\tn.Gid = pn.Gid\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tn.Mode |= 02000\n\t\t\t} else if n.Mode&02010 == 02010 && ctx.Uid() != 0 {\n\t\t\t\tvar found bool\n\t\t\t\tfor _, gid := range ctx.Gids() {\n\t\t\t\t\tif gid == pn.Gid {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tn.Mode &= ^uint16(02000)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif err = mustInsert(s, &edge{Parent: parent, Name: []byte(name), Inode: *inode, Type: _type}, &n); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _type == TypeSymlink {\n\t\t\tif err = mustInsert(s, &symlink{Inode: *inode, Target: []byte(path)}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif _type == TypeDirectory {\n\t\t\tif err = mustInsert(s, &dirStats{Inode: *inode}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif updateParent {\n\t\t\tif _n, err := s.SetExpr(\"nlink\", fmt.Sprintf(\"nlink + (%d)\", nlinkAdjust)).Cols(\"nlink\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&pn, &node{Inode: pn.Inode}); err != nil || _n == 0 {\n\t\t\t\tif err == nil {\n\t\t\t\t\tlogger.Infof(\"Update parent node affected rows = %d should be 1 for inode = %d .\", _n, pn.Inode)\n\t\t\t\t\tif m.Name() == \"mysql\" {\n\t\t\t\t\t\terr = syscall.EBUSY\n\t\t\t\t\t} else {\n\t\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tm.parseAttr(&n, attr)\n\t\treturn nil\n\t}))\n}\n\nfunc (m *dbMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno {\n\tvar trash Ino\n\tif !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\tvar n node\n\tvar opened bool\n\tvar newSpace, newInode int64\n\terr := m.txn(func(s *xorm.Session) error {\n\t\topened = false\n\t\tnewSpace, newInode = 0, 0\n\t\tvar pn = node{Inode: parent}\n\t\tok, err := s.Get(&pn)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif pn.Type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr(&pn, &pattr)\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pn.Flags&FlagAppend) != 0 || (pn.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar e = edge{Parent: parent, Name: []byte(name)}\n\t\tok, err = s.Get(&e)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok && m.conf.CaseInsensi {\n\t\t\tif ee := m.resolveCase(ctx, parent, name); ee != nil {\n\t\t\t\tok = true\n\t\t\t\te.Name = ee.Name\n\t\t\t\te.Inode = ee.Inode\n\t\t\t\te.Type = ee.Attr.Typ\n\t\t\t}\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif e.Type == TypeDirectory {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tn = node{Inode: e.Inode}\n\t\tok, err = s.ForUpdate().Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tnow := time.Now().UnixNano()\n\t\tif ok {\n\t\t\tif ctx.Uid() != 0 && pn.Mode&01000 != 0 && ctx.Uid() != pn.Uid && ctx.Uid() != n.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t\tif (n.Flags&FlagAppend) != 0 || (n.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif (n.Flags & FlagSkipTrash) != 0 {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tif trash > 0 && n.Nlink > 1 {\n\t\t\t\tif o, e := s.Get(&edge{Parent: trash, Name: []byte(m.trashEntry(parent, e.Inode, string(e.Name))), Inode: e.Inode, Type: e.Type}); e == nil && o {\n\t\t\t\t\ttrash = 0\n\t\t\t\t}\n\t\t\t}\n\t\t\tn.setCtime(now)\n\t\t\tif trash == 0 {\n\t\t\t\tn.Nlink--\n\t\t\t\tif n.Type == TypeFile && n.Nlink == 0 && m.sid > 0 {\n\t\t\t\t\topened = m.of.IsOpen(e.Inode)\n\t\t\t\t}\n\t\t\t} else if n.Parent > 0 {\n\t\t\t\tn.Parent = trash\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", e.Inode, parent, name)\n\t\t\ttrash = 0\n\t\t}\n\t\tdefer func() { m.of.InvalidateChunk(e.Inode, invalidateAttrOnly) }()\n\n\t\tvar updateParent bool\n\t\tif !parent.IsTrash() && time.Duration(now-pn.getMtime()) >= m.conf.SkipDirMtime {\n\t\t\tpn.setMtime(now)\n\t\t\tpn.setCtime(now)\n\t\t\tupdateParent = true\n\t\t}\n\n\t\tif _, err := s.Delete(&edge{Parent: parent, Name: e.Name}); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif n.Nlink > 0 {\n\t\t\tif _, err := s.Cols(\"nlink\", \"ctime\", \"ctimensec\", \"parent\").Update(&n, &node{Inode: e.Inode}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif trash > 0 {\n\t\t\t\tif err = mustInsert(s, &edge{Parent: trash, Name: []byte(m.trashEntry(parent, e.Inode, string(e.Name))), Inode: e.Inode, Type: e.Type}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tswitch e.Type {\n\t\t\tcase TypeFile:\n\t\t\t\tif opened {\n\t\t\t\t\tif err = mustInsert(s, sustained{Sid: m.sid, Inode: e.Inode}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif _, err := s.Cols(\"nlink\", \"ctime\", \"ctimensec\").Update(&n, &node{Inode: e.Inode}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif err = mustInsert(s, delfile{e.Inode, n.Length, time.Now().Unix()}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif _, err := s.Delete(&node{Inode: e.Inode}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tnewSpace, newInode = -align4K(n.Length), -1\n\t\t\t\t}\n\t\t\tcase TypeSymlink:\n\t\t\t\tif _, err := s.Delete(&symlink{Inode: e.Inode}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tfallthrough\n\t\t\tdefault:\n\t\t\t\tif _, err := s.Delete(&node{Inode: e.Inode}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tnewSpace, newInode = -align4K(0), -1\n\t\t\t}\n\t\t\tif _, err := s.Delete(&xattr{Inode: e.Inode}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif updateParent {\n\t\t\tvar _n int64\n\t\t\tif _n, err = s.Cols(\"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&pn, &node{Inode: pn.Inode}); err != nil || _n == 0 {\n\t\t\t\tif err == nil {\n\t\t\t\t\tlogger.Infof(\"Update parent node affected rows = %d should be 1 for inode = %d .\", _n, pn.Inode)\n\t\t\t\t\tif m.Name() == \"mysql\" {\n\t\t\t\t\t\terr = syscall.EBUSY\n\t\t\t\t\t} else {\n\t\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn err\n\t})\n\tif err == nil && trash == 0 {\n\t\tif n.Type == TypeFile && n.Nlink == 0 {\n\t\t\tm.fileDeleted(opened, parent.IsTrash(), n.Inode, n.Length)\n\t\t}\n\t\tm.updateStats(newSpace, newInode)\n\t\tm.updateUserGroupStat(ctx, n.Uid, n.Gid, newSpace, newInode)\n\t}\n\tif err == nil && attr != nil {\n\t\tm.parseAttr(&n, attr)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *dbMeta) doRmdir(ctx Context, parent Ino, name string, pinode *Ino, attr *Attr, skipCheckTrash ...bool) syscall.Errno {\n\tvar trash Ino\n\tif !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\tvar n node\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tvar pn = node{Inode: parent}\n\t\tok, err := s.Get(&pn)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif pn.Type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr(&pn, &pattr)\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif pn.Flags&FlagImmutable != 0 || pn.Flags&FlagAppend != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar e = edge{Parent: parent, Name: []byte(name)}\n\t\tok, err = s.Get(&e)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok && m.conf.CaseInsensi {\n\t\t\tif ee := m.resolveCase(ctx, parent, name); ee != nil {\n\t\t\t\tok = true\n\t\t\t\te.Inode = ee.Inode\n\t\t\t\te.Name = ee.Name\n\t\t\t\te.Type = ee.Attr.Typ\n\t\t\t}\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif e.Type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pinode != nil {\n\t\t\t*pinode = e.Inode\n\t\t}\n\t\tn = node{Inode: e.Inode}\n\t\tok, err = s.ForUpdate().Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif ok && attr != nil {\n\t\t\tm.parseAttr(&n, attr)\n\t\t}\n\t\texist, err := s.Exist(&edge{Parent: e.Inode})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif exist {\n\t\t\treturn syscall.ENOTEMPTY\n\t\t}\n\t\tif (n.Flags & FlagSkipTrash) != 0 {\n\t\t\ttrash = 0\n\t\t}\n\t\tnow := time.Now().UnixNano()\n\t\tif ok {\n\t\t\tif ctx.Uid() != 0 && pn.Mode&01000 != 0 && ctx.Uid() != pn.Uid && ctx.Uid() != n.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t\tif trash > 0 {\n\t\t\t\tn.setCtime(now)\n\t\t\t\tn.Parent = trash\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", e.Inode, parent, name)\n\t\t\ttrash = 0\n\t\t}\n\t\tpn.Nlink--\n\t\tpn.setMtime(now)\n\t\tpn.setCtime(now)\n\n\t\tif _, err := s.Delete(&edge{Parent: parent, Name: e.Name}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := s.Delete(&dirStats{Inode: e.Inode}); err != nil {\n\t\t\tlogger.Warnf(\"remove dir usage of ino(%d): %s\", e.Inode, err)\n\t\t\treturn err\n\t\t}\n\t\tif _, err = s.Delete(&dirQuota{Inode: e.Inode}); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif trash > 0 {\n\t\t\tif _, err = s.Cols(\"ctime\", \"ctimensec\", \"parent\").Update(&n, &node{Inode: n.Inode}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err = mustInsert(s, &edge{Parent: trash, Name: []byte(m.trashEntry(parent, e.Inode, string(e.Name))), Inode: e.Inode, Type: e.Type}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tif _, err := s.Delete(&node{Inode: e.Inode}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif _, err := s.Delete(&xattr{Inode: e.Inode}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif !parent.IsTrash() {\n\t\t\t_, err = s.SetExpr(\"nlink\", \"nlink - 1\").Cols(\"nlink\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&pn, &node{Inode: pn.Inode})\n\t\t}\n\t\treturn err\n\t})\n\tif err == nil && trash == 0 {\n\t\tm.updateStats(-align4K(0), -1)\n\t\tm.updateUserGroupStat(ctx, n.Uid, n.Gid, -align4K(0), -1)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *dbMeta) getNodesForUpdate(s *xorm.Session, nodes ...*node) error {\n\t// sort them to avoid deadlock\n\tsort.Slice(nodes, func(i, j int) bool { return nodes[i].Inode < nodes[j].Inode })\n\tfor i := range nodes {\n\t\tok, err := s.ForUpdate().Get(nodes[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) getNodes(s *xorm.Session, nodes ...*node) error {\n\tfor i := range nodes {\n\t\tok, err := s.Get(nodes[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode, tInode *Ino, attr, tAttr *Attr) syscall.Errno {\n\tvar trash Ino\n\tif st := m.checkTrash(parentDst, &trash); st != 0 {\n\t\treturn st\n\t}\n\texchange := flags == RenameExchange\n\tvar opened bool\n\tvar dino Ino\n\tvar dn node\n\tvar newSpace, newInode int64\n\tparentLocks := []Ino{parentDst}\n\tif !parentSrc.IsTrash() { // there should be no conflict if parentSrc is in trash, relax lock to accelerate `restore` subcommand\n\t\tparentLocks = append(parentLocks, parentSrc)\n\t}\n\terr := m.txn(func(s *xorm.Session) error {\n\t\topened = false\n\t\tdino = 0\n\t\tnewSpace, newInode = 0, 0\n\t\tvar spn = node{Inode: parentSrc}\n\t\tvar dpn = node{Inode: parentDst}\n\t\terr := m.getNodes(s, &spn, &dpn)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif spn.Type != TypeDirectory || dpn.Type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif (spn.Flags&FlagAppend) != 0 || (spn.Flags&FlagImmutable) != 0 || (dpn.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar spattr, dpattr Attr\n\t\tm.parseAttr(&spn, &spattr)\n\t\tm.parseAttr(&dpn, &dpattr)\n\t\tif flags&RenameRestore == 0 && dpattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parentSrc, MODE_MASK_W|MODE_MASK_X, &spattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif st := m.Access(ctx, parentDst, MODE_MASK_W|MODE_MASK_X, &dpattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tvar se = edge{Parent: parentSrc, Name: []byte(nameSrc)}\n\t\tok, err := s.Get(&se)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parentSrc, nameSrc); e != nil {\n\t\t\t\tif string(e.Name) != nameSrc || parentSrc != parentDst {\n\t\t\t\t\tok = true\n\t\t\t\t\tse.Inode = e.Inode\n\t\t\t\t\tse.Type = e.Attr.Typ\n\t\t\t\t\tse.Name = e.Name\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif parentSrc == parentDst && string(se.Name) == nameDst {\n\t\t\tif inode != nil {\n\t\t\t\t*inode = se.Inode\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\t// TODO: check parentDst is a subdir of source node\n\t\tif se.Inode == parentDst || se.Inode == dpattr.Parent {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar sn = node{Inode: se.Inode}\n\t\tok, err = s.Get(&sn)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar sattr Attr\n\t\tm.parseAttr(&sn, &sattr)\n\t\tif parentSrc != parentDst && spattr.Mode&0o1000 != 0 && ctx.Uid() != 0 &&\n\t\t\tctx.Uid() != sattr.Uid && (ctx.Uid() != spattr.Uid || sattr.Typ == TypeDirectory) {\n\t\t\treturn syscall.EACCES\n\t\t}\n\t\tif (sn.Flags&FlagAppend) != 0 || (sn.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tif st := m.Access(ctx, parentDst, MODE_MASK_W|MODE_MASK_X, &dpattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tvar de = edge{Parent: parentDst, Name: []byte(nameDst)}\n\t\tok, err = s.Get(&de)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parentDst, nameDst); e != nil {\n\t\t\t\tif string(e.Name) != nameSrc || parentSrc != parentDst {\n\t\t\t\t\tok = true\n\t\t\t\t\tde.Inode = e.Inode\n\t\t\t\t\tde.Type = e.Attr.Typ\n\t\t\t\t\tde.Name = e.Name\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tvar supdate, dupdate bool\n\t\tvar srcnlink, dstnlink int32\n\t\tnow := time.Now().UnixNano()\n\t\tdn = node{Inode: de.Inode}\n\t\tif ok {\n\t\t\tif flags&RenameNoReplace != 0 {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\tdino = de.Inode\n\t\t\tok, err := s.ForUpdate().Get(&dn)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif !ok { // corrupt entry\n\t\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", dino, parentDst, de.Name)\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tif (dn.Flags&FlagAppend) != 0 || (dn.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif (dn.Flags & FlagSkipTrash) != 0 {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tdn.setCtime(now)\n\t\t\tif exchange {\n\t\t\t\tif parentSrc != parentDst {\n\t\t\t\t\tif de.Type == TypeDirectory {\n\t\t\t\t\t\tdn.Parent = parentSrc\n\t\t\t\t\t\tdpn.Nlink--\n\t\t\t\t\t\tdstnlink--\n\t\t\t\t\t\tspn.Nlink++\n\t\t\t\t\t\tsrcnlink++\n\t\t\t\t\t\tsupdate, dupdate = true, true\n\t\t\t\t\t} else if dn.Parent > 0 {\n\t\t\t\t\t\tdn.Parent = parentSrc\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if de.Inode == se.Inode {\n\t\t\t\treturn nil\n\t\t\t} else if se.Type == TypeDirectory && de.Type != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t} else if de.Type == TypeDirectory {\n\t\t\t\tif se.Type != TypeDirectory {\n\t\t\t\t\treturn syscall.EISDIR\n\t\t\t\t}\n\t\t\t\texist, err := s.Exist(&edge{Parent: de.Inode})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif exist {\n\t\t\t\t\treturn syscall.ENOTEMPTY\n\t\t\t\t}\n\t\t\t\tdpn.Nlink--\n\t\t\t\tdstnlink--\n\t\t\t\tdupdate = true\n\t\t\t\tif trash > 0 {\n\t\t\t\t\tdn.Parent = trash\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif trash == 0 {\n\t\t\t\t\tdn.Nlink--\n\t\t\t\t\tif de.Type == TypeFile && dn.Nlink == 0 && m.sid > 0 {\n\t\t\t\t\t\topened = m.of.IsOpen(dn.Inode)\n\t\t\t\t\t}\n\t\t\t\t\tdefer func() { m.of.InvalidateChunk(dino, invalidateAttrOnly) }()\n\t\t\t\t} else if dn.Parent > 0 {\n\t\t\t\t\tdn.Parent = trash\n\t\t\t\t}\n\t\t\t}\n\t\t\tif ctx.Uid() != 0 && dpn.Mode&01000 != 0 && ctx.Uid() != dpn.Uid && ctx.Uid() != dn.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t} else {\n\t\t\tif exchange {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t}\n\t\tif ctx.Uid() != 0 && spn.Mode&01000 != 0 && ctx.Uid() != spn.Uid && ctx.Uid() != sn.Uid {\n\t\t\treturn syscall.EACCES\n\t\t}\n\n\t\tif parentSrc != parentDst {\n\t\t\tif se.Type == TypeDirectory {\n\t\t\t\tsn.Parent = parentDst\n\t\t\t\tspn.Nlink--\n\t\t\t\tsrcnlink--\n\t\t\t\tdpn.Nlink++\n\t\t\t\tdstnlink++\n\t\t\t\tsupdate, dupdate = true, true\n\t\t\t} else if sn.Parent > 0 {\n\t\t\t\tsn.Parent = parentDst\n\t\t\t}\n\t\t}\n\t\tif supdate || time.Duration(now-spn.getMtime()) >= m.conf.SkipDirMtime {\n\t\t\tspn.setMtime(now)\n\t\t\tspn.setCtime(now)\n\t\t\tsupdate = true\n\t\t}\n\t\tif dupdate || time.Duration(now-dpn.getMtime()) >= m.conf.SkipDirMtime {\n\t\t\tdpn.setMtime(now)\n\t\t\tdpn.setCtime(now)\n\t\t\tdupdate = true\n\t\t}\n\t\tsn.setCtime(now)\n\t\tif inode != nil {\n\t\t\t*inode = sn.Inode\n\t\t}\n\t\tm.parseAttr(&sn, attr)\n\t\tif dino > 0 {\n\t\t\t*tInode = dino\n\t\t\tm.parseAttr(&dn, tAttr)\n\t\t}\n\n\t\tif exchange {\n\t\t\tif _, err := s.Cols(\"inode\", \"type\").Update(&de, &edge{Parent: parentSrc, Name: se.Name}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif _, err := s.Cols(\"inode\", \"type\").Update(&se, &edge{Parent: parentDst, Name: de.Name}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif _, err := s.Cols(\"ctime\", \"ctimensec\", \"parent\").Update(dn, &node{Inode: dino}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tif n, err := s.Delete(&edge{Parent: parentSrc, Name: se.Name}); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if n != 1 {\n\t\t\t\treturn fmt.Errorf(\"delete src failed\")\n\t\t\t}\n\t\t\tif dino > 0 {\n\t\t\t\tif trash > 0 {\n\t\t\t\t\tif _, err := s.Cols(\"ctime\", \"ctimensec\", \"parent\").Update(dn, &node{Inode: dino}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tname := m.trashEntry(parentDst, dino, string(de.Name))\n\t\t\t\t\tif err = mustInsert(s, &edge{Parent: trash, Name: []byte(name), Inode: dino, Type: de.Type}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t} else if de.Type != TypeDirectory && dn.Nlink > 0 {\n\t\t\t\t\tif _, err := s.Cols(\"ctime\", \"ctimensec\", \"nlink\", \"parent\").Update(dn, &node{Inode: dino}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif de.Type == TypeFile {\n\t\t\t\t\t\tif opened {\n\t\t\t\t\t\t\tif _, err := s.Cols(\"nlink\", \"ctime\", \"ctimensec\").Update(&dn, &node{Inode: dino}); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif err = mustInsert(s, sustained{Sid: m.sid, Inode: dino}); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tif err = mustInsert(s, delfile{dino, dn.Length, time.Now().Unix()}); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif _, err := s.Delete(&node{Inode: dino}); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tnewSpace, newInode = -align4K(dn.Length), -1\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif de.Type == TypeSymlink {\n\t\t\t\t\t\t\tif _, err := s.Delete(&symlink{Inode: dino}); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif _, err := s.Delete(&node{Inode: dino}); err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tnewSpace, newInode = -align4K(0), -1\n\t\t\t\t\t}\n\t\t\t\t\tif _, err := s.Delete(&xattr{Inode: dino}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, err := s.Delete(&edge{Parent: parentDst, Name: de.Name}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif de.Type == TypeDirectory {\n\t\t\t\t\tif _, err = s.Delete(&dirQuota{Inode: dino}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err = mustInsert(s, &edge{Parent: parentDst, Name: de.Name, Inode: se.Inode, Type: se.Type}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif _, err := s.Cols(\"ctime\", \"ctimensec\", \"parent\").Update(&sn, &node{Inode: sn.Inode}); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif parentDst != parentSrc && !parentSrc.IsTrash() && supdate {\n\t\t\tif dupdate && dpn.Inode < spn.Inode {\n\t\t\t\tif _n, err := s.SetExpr(\"nlink\", fmt.Sprintf(\"nlink + (%d)\", dstnlink)).Cols(\"nlink\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&dpn, &node{Inode: parentDst}); err != nil || _n == 0 {\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tlogger.Infof(\"Update parent node affected rows = %d should be 1 for inode = %d .\", _n, dpn.Inode)\n\t\t\t\t\t\tif m.Name() == \"mysql\" {\n\t\t\t\t\t\t\terr = syscall.EBUSY\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tdupdate = false\n\t\t\t}\n\n\t\t\tif _n, err := s.SetExpr(\"nlink\", fmt.Sprintf(\"nlink + (%d)\", srcnlink)).Cols(\"nlink\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&spn, &node{Inode: parentSrc}); err != nil || _n == 0 {\n\t\t\t\tif err == nil {\n\t\t\t\t\tlogger.Infof(\"Update parent node affected rows = %d should be 1 for inode = %d .\", _n, spn.Inode)\n\t\t\t\t\tif m.Name() == \"mysql\" {\n\t\t\t\t\t\terr = syscall.EBUSY\n\t\t\t\t\t} else {\n\t\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif dupdate {\n\t\t\tif _n, err := s.SetExpr(\"nlink\", fmt.Sprintf(\"nlink + (%d)\", dstnlink)).Cols(\"nlink\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&dpn, &node{Inode: parentDst}); err != nil || _n == 0 {\n\t\t\t\tif err == nil {\n\t\t\t\t\tlogger.Infof(\"Update parent node affected rows = %d should be 1 for inode = %d .\", _n, dpn.Inode)\n\t\t\t\t\tif m.Name() == \"mysql\" {\n\t\t\t\t\t\terr = syscall.EBUSY\n\t\t\t\t\t} else {\n\t\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}, parentLocks...)\n\tif err == nil && !exchange && trash == 0 {\n\t\tif dino > 0 && dn.Type == TypeFile && dn.Nlink == 0 {\n\t\t\tm.fileDeleted(opened, false, dino, dn.Length)\n\t\t}\n\t\tm.updateStats(newSpace, newInode)\n\t\tm.updateUserGroupStat(ctx, dn.Uid, dn.Gid, newSpace, newInode)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *dbMeta) doLink(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tvar pn = node{Inode: parent}\n\t\tok, err := s.Get(&pn)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif pn.Type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pn.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr(&pn, &pattr)\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif pn.Flags&FlagImmutable != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tvar e = edge{Parent: parent, Name: []byte(name)}\n\t\tok, err = s.Get(&e)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif ok || !ok && m.conf.CaseInsensi && m.resolveCase(ctx, parent, name) != nil {\n\t\t\treturn syscall.EEXIST\n\t\t}\n\n\t\tvar n = node{Inode: inode}\n\t\tok, err = s.ForUpdate().Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif n.Type == TypeDirectory {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif (n.Flags&FlagAppend) != 0 || (n.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tvar updateParent bool\n\t\tnow := time.Now().UnixNano()\n\t\tif time.Duration(now-pn.getMtime()) >= m.conf.SkipDirMtime {\n\t\t\tpn.setMtime(now)\n\t\t\tpn.setCtime(now)\n\t\t\tupdateParent = true\n\t\t}\n\t\tn.Parent = 0\n\t\tn.Nlink++\n\t\tn.setCtime(now)\n\n\t\tif err = mustInsert(s, &edge{Parent: parent, Name: []byte(name), Inode: inode, Type: n.Type}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := s.Cols(\"nlink\", \"ctime\", \"ctimensec\", \"parent\").Update(&n, node{Inode: inode}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif updateParent {\n\t\t\tif _n, err := s.Cols(\"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&pn, &node{Inode: parent}); err != nil || _n == 0 {\n\t\t\t\tif err == nil {\n\t\t\t\t\tlogger.Infof(\"Update parent node affected rows = %d should be 1 for inode = %d .\", _n, pn.Inode)\n\t\t\t\t\tif m.Name() == \"mysql\" {\n\t\t\t\t\t\terr = syscall.EBUSY\n\t\t\t\t\t} else {\n\t\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tm.parseAttr(&n, attr)\n\t\treturn err\n\t}, inode))\n}\n\nfunc (m *dbMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry, limit int) syscall.Errno {\n\n\treturn errno(m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\ts = s.Table(&edge{})\n\t\tif plus != 0 {\n\t\t\ts = s.Join(\"INNER\", &node{}, m.sqlConv(\"edge.inode=node.inode\"))\n\t\t}\n\t\tif limit > 0 {\n\t\t\ts = s.Limit(limit, 0)\n\t\t}\n\t\tvar nodes []namedNode\n\t\tif err := s.Find(&nodes, &edge{Parent: inode}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, n := range nodes {\n\t\t\tif len(n.Name) == 0 {\n\t\t\t\tlogger.Errorf(\"Corrupt entry with empty name: inode %d parent %d\", n.Inode, inode)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tentry := &Entry{\n\t\t\t\tInode: n.Inode,\n\t\t\t\tName: n.Name,\n\t\t\t\tAttr: &Attr{},\n\t\t\t}\n\t\t\tif plus != 0 {\n\t\t\t\tm.parseAttr(&n.node, entry.Attr)\n\t\t\t\tm.of.Update(entry.Inode, entry.Attr)\n\t\t\t} else {\n\t\t\t\tentry.Attr.Typ = n.Type\n\t\t\t}\n\t\t\t*entries = append(*entries, entry)\n\t\t}\n\t\treturn nil\n\t}))\n}\n\nfunc (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []*Entry, delta *dirStat, skipCheckTrash ...bool) syscall.Errno {\n\tif len(entries) == 0 {\n\t\treturn 0\n\t}\n\n\tvar trash Ino\n\tif len(skipCheckTrash) == 0 || !skipCheckTrash[0] {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\n\ttype entryInfo struct {\n\t\te *edge\n\t\ttrash Ino\n\t\tn *node // n edges : 1 inode\n\t\ttrashName string // cached trash entry name when hard links go to trash\n\t}\n\tvar entryInfos []*entryInfo\n\ttype dNode struct {\n\t\topened bool\n\t\tlength uint64\n\t}\n\tdelNodes := make(map[Ino]*dNode)\n\n\tbatchSize := m.getTxnBatchNum()\n\tfor len(entries) > 0 {\n\t\tif batchSize > len(entries) {\n\t\t\tbatchSize = len(entries)\n\t\t}\n\t\tbatch := entries[:batchSize]\n\t\tentries = entries[batchSize:]\n\t\tvar batchFsSpace, batchFsInodes int64\n\t\tvar batchDirLength, batchDirSpace, batchDirInodes int64\n\t\tvar deltas ugQuotaDeltas\n\t\terr := m.txn(func(s *xorm.Session) error {\n\t\t\tbatchDirLength, batchDirSpace, batchDirInodes = 0, 0, 0\n\t\t\tbatchFsSpace, batchFsInodes = 0, 0\n\t\t\tdeltas = make(ugQuotaDeltas)\n\t\t\tpn := node{Inode: parent}\n\t\t\tok, err := s.Get(&pn)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif !ok {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t\tif pn.Type != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t\tvar pattr Attr\n\t\t\tm.parseAttr(&pn, &pattr)\n\t\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t\tif (pn.Flags&FlagAppend != 0) || (pn.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tnow := time.Now().UnixNano()\n\t\t\tentryInfos = make([]*entryInfo, 0, len(batch))\n\t\t\tnames := make([][]byte, 0, len(batch))\n\t\t\tfor _, entry := range batch {\n\t\t\t\tnames = append(names, entry.Name)\n\t\t\t}\n\t\t\tvar foundEdges []edge\n\t\t\tif err := s.Where(\"parent=?\", parent).In(\"name\", names).Find(&foundEdges); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tentryMap := make(map[string]*edge)\n\t\t\tfor i := range foundEdges {\n\t\t\t\tentryMap[string(foundEdges[i].Name)] = &foundEdges[i]\n\t\t\t}\n\n\t\t\tinodes := make([]Ino, 0, len(batch))\n\t\t\tinodeM := make(map[Ino]struct{}) // filter hardlinks\n\t\t\tfor _, entry := range batch {\n\t\t\t\te, ok := entryMap[string(entry.Name)]\n\t\t\t\tif !ok {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif e.Inode != entry.Inode || e.Type == TypeDirectory || (entry.Attr != nil && e.Type != entry.Attr.Typ) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tentryInfos = append(entryInfos, &entryInfo{e: e, trash: trash})\n\t\t\t\tif _, exists := inodeM[entry.Inode]; !exists {\n\t\t\t\t\tinodeM[entry.Inode] = struct{}{}\n\t\t\t\t\tinodes = append(inodes, entry.Inode)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif len(inodes) > 0 {\n\t\t\t\tvar nodes []node\n\t\t\t\tif err := s.ForUpdate().In(\"inode\", inodes).Find(&nodes); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tnodeMap := make(map[Ino]*node, len(nodes))\n\t\t\t\t// build quick lookup map from inode to *node\n\t\t\t\tfor i := range nodes {\n\t\t\t\t\tnodeMap[nodes[i].Inode] = &nodes[i]\n\t\t\t\t}\n\n\t\t\t\t// iterate all target entries, apply basic checks and build info for each edge\n\t\t\t\tdumpNode := &node{}\n\t\t\t\tfor _, info := range entryInfos {\n\t\t\t\t\tn, ok := nodeMap[info.e.Inode]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t\tinfo.n = dumpNode\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif ctx.Uid() != 0 && pn.Mode&01000 != 0 && ctx.Uid() != pn.Uid && ctx.Uid() != n.Uid {\n\t\t\t\t\t\treturn syscall.EACCES\n\t\t\t\t\t}\n\t\t\t\t\tif (n.Flags&FlagAppend) != 0 || (n.Flags&FlagImmutable) != 0 {\n\t\t\t\t\t\treturn syscall.EPERM\n\t\t\t\t\t}\n\t\t\t\t\tif (n.Flags & FlagSkipTrash) != 0 {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t}\n\t\t\t\t\tinfo.n = n\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif info.trash > 0 && info.n.Nlink > 1 {\n\t\t\t\t\tinfo.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name))\n\t\t\t\t\tte := edge{\n\t\t\t\t\t\tParent: info.trash,\n\t\t\t\t\t\tName: []byte(info.trashName),\n\t\t\t\t\t\tInode: info.n.Inode,\n\t\t\t\t\t\tType: info.n.Type,\n\t\t\t\t\t}\n\t\t\t\t\tif ok, err := s.Get(&te); err == nil && ok {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tinfo.n.setCtime(now)\n\t\t\t\tif info.trash > 0 && info.n.Parent > 0 {\n\t\t\t\t\tinfo.n.Parent = info.trash\n\t\t\t\t}\n\t\t\t\tif info.trash == 0 && info.n.Nlink > 0 {\n\t\t\t\t\tinfo.n.Nlink--\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// check opened status for all inodes with Nlink == 0 after all decrements\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif info.n != nil && info.trash == 0 && info.n.Nlink == 0 && info.n.Type == TypeFile {\n\t\t\t\t\topened := false\n\t\t\t\t\tif m.sid > 0 {\n\t\t\t\t\t\topened = m.of.IsOpen(info.n.Inode)\n\t\t\t\t\t}\n\t\t\t\t\tdelNodes[info.n.Inode] = &dNode{opened, info.n.Length}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar updateParent bool\n\t\t\tif !parent.IsTrash() && time.Duration(now-pn.getMtime()) >= m.conf.SkipDirMtime {\n\t\t\t\tpn.setMtime(now)\n\t\t\t\tpn.setCtime(now)\n\t\t\t\tupdateParent = true\n\t\t\t}\n\n\t\t\tnowUnix := time.Now().Unix()\n\t\t\tvisited := make(map[Ino]bool)\n\t\t\tvisited[0] = true // skip dummyNode\n\n\t\t\t// buffers for batched operations\n\t\t\tedgesDel := make([]edge, 0)\n\t\t\tsustainedIns := make([]interface{}, 0)\n\t\t\tdelfilesIns := make([]interface{}, 0)\n\t\t\tnodesDel := make([]Ino, 0)\n\t\t\tsymlinksDel := make([]Ino, 0)\n\t\t\txattrsDel := make([]Ino, 0)\n\t\t\tedgesIns := make([]interface{}, 0)\n\t\t\t// walk each edge to decide whether to move to trash, decrement nlink or delete inode & xattrs\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tedgesDel = append(edgesDel, edge{Parent: parent, Name: info.e.Name})\n\t\t\t\tif info.n.Inode != 0 {\n\t\t\t\t\tif info.n.Type == TypeFile {\n\t\t\t\t\t\tbatchDirLength -= int64(info.n.Length)\n\t\t\t\t\t\tbatchDirSpace -= align4K(info.n.Length)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tbatchDirSpace -= align4K(0)\n\t\t\t\t\t}\n\t\t\t\t\tbatchDirInodes--\n\t\t\t\t}\n\t\t\t\tif !visited[info.n.Inode] {\n\t\t\t\t\tif info.n.Nlink > 0 {\n\t\t\t\t\t\t// inode still referenced somewhere: only update metadata\n\t\t\t\t\t\tif _, err := s.Cols(\"nlink\", \"ctime\", \"ctimensec\", \"parent\").Update(info.n, &node{Inode: info.n.Inode}); err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// last link removed: prepare to delete inode and related rows\n\t\t\t\t\t\tvar entrySpace int64\n\t\t\t\t\t\tswitch info.n.Type {\n\t\t\t\t\t\tcase TypeFile:\n\t\t\t\t\t\t\tentrySpace = align4K(info.n.Length)\n\t\t\t\t\t\t\tif dnode, ok := delNodes[info.n.Inode]; ok && dnode.opened {\n\t\t\t\t\t\t\t\tsustainedIns = append(sustainedIns, &sustained{Sid: m.sid, Inode: info.e.Inode})\n\t\t\t\t\t\t\t\tif _, err := s.Cols(\"nlink\", \"ctime\", \"ctimensec\").Update(info.n, &node{Inode: info.n.Inode}); err != nil {\n\t\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t// regular, un-opened file: add to delfile and delete inode later\n\t\t\t\t\t\t\t\tdelfilesIns = append(delfilesIns, &delfile{info.e.Inode, info.n.Length, nowUnix})\n\t\t\t\t\t\t\t\tnodesDel = append(nodesDel, info.e.Inode)\n\t\t\t\t\t\t\t\tbatchFsSpace -= entrySpace\n\t\t\t\t\t\t\t\tbatchFsInodes--\n\t\t\t\t\t\t\t\tdeltas.add(&ugQuotaDelta{\n\t\t\t\t\t\t\t\t\tUid: info.n.Uid,\n\t\t\t\t\t\t\t\t\tGid: info.n.Gid,\n\t\t\t\t\t\t\t\t\tSpace: -entrySpace,\n\t\t\t\t\t\t\t\t\tInodes: -1,\n\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase TypeSymlink:\n\t\t\t\t\t\t\t// symlink: record for batched delete from symlink table\n\t\t\t\t\t\t\tsymlinksDel = append(symlinksDel, info.e.Inode)\n\t\t\t\t\t\t\tfallthrough\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t// other non-file types: record for direct inode deletion\n\t\t\t\t\t\t\tnodesDel = append(nodesDel, info.e.Inode)\n\t\t\t\t\t\t\tif info.n.Type != TypeFile {\n\t\t\t\t\t\t\t\tentrySpace = align4K(0)\n\t\t\t\t\t\t\t\tbatchFsSpace -= entrySpace\n\t\t\t\t\t\t\t\tbatchFsInodes--\n\t\t\t\t\t\t\t\tdeltas.add(&ugQuotaDelta{\n\t\t\t\t\t\t\t\t\tUid: info.n.Uid,\n\t\t\t\t\t\t\t\t\tGid: info.n.Gid,\n\t\t\t\t\t\t\t\t\tSpace: -entrySpace,\n\t\t\t\t\t\t\t\t\tInodes: -1,\n\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\txattrsDel = append(xattrsDel, info.e.Inode)\n\t\t\t\t\t}\n\t\t\t\t\tm.of.InvalidateChunk(info.e.Inode, invalidateAttrOnly)\n\t\t\t\t}\n\t\t\t\tif info.n.Nlink > 0 && info.trash > 0 {\n\t\t\t\t\t// still has links and should be moved to trash; create new trash edge\n\t\t\t\t\tif info.trashName == \"\" {\n\t\t\t\t\t\tinfo.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name))\n\t\t\t\t\t}\n\t\t\t\t\tedgesIns = append(edgesIns, &edge{\n\t\t\t\t\t\tParent: info.trash,\n\t\t\t\t\t\tName: []byte(info.trashName),\n\t\t\t\t\t\tInode: info.n.Inode,\n\t\t\t\t\t\tType: info.n.Type})\n\t\t\t\t}\n\t\t\t\tvisited[info.n.Inode] = true\n\t\t\t}\n\n\t\t\tif len(edgesDel) > 0 {\n\t\t\t\tquery := s.Table(&edge{})\n\t\t\t\tfor j, e := range edgesDel {\n\t\t\t\t\tif j == 0 {\n\t\t\t\t\t\tquery = query.Where(\"parent = ? AND name = ?\", e.Parent, e.Name)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tquery = query.Or(\"parent = ? AND name = ?\", e.Parent, e.Name)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, err := query.Delete(&edge{}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// execute SQL statements in batches\n\t\t\tif len(sustainedIns) > 0 {\n\t\t\t\tif err := mustInsert(s, sustainedIns...); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(delfilesIns) > 0 {\n\t\t\t\tif err := mustInsert(s, delfilesIns...); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(nodesDel) > 0 {\n\t\t\t\tif _, err := s.In(\"inode\", nodesDel).Delete(&node{}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(symlinksDel) > 0 {\n\t\t\t\tif _, err := s.In(\"inode\", symlinksDel).Delete(&symlink{}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(xattrsDel) > 0 {\n\t\t\t\tif _, err := s.In(\"inode\", xattrsDel).Delete(&xattr{}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(edgesIns) > 0 {\n\t\t\t\tif err := mustInsert(s, edgesIns...); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// optionally update parent directory timestamps\n\t\t\tif updateParent {\n\t\t\t\tvar _n int64\n\t\t\t\tif _n, err = s.Cols(\"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&pn, &node{Inode: pn.Inode}); err != nil || _n == 0 {\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tlogger.Infof(\"Update parent node affected rows = %d should be 1 for inode = %d .\", _n, pn.Inode)\n\t\t\t\t\t\tif m.Name() == \"mysql\" {\n\t\t\t\t\t\t\terr = syscall.EBUSY\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn nil\n\t\t})\n\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\n\t\tdelta.length += batchDirLength\n\t\tdelta.space += batchDirSpace\n\t\tdelta.inodes += batchDirInodes\n\t\tm.updateStats(batchFsSpace, batchFsInodes)\n\t\tfor _, q := range deltas {\n\t\t\tm.updateUserGroupStat(ctx, q.Uid, q.Gid, q.Space, q.Inodes)\n\t\t}\n\t}\n\n\t// outside of transaction: trigger data deletion callbacks\n\tfor inode, info := range delNodes {\n\t\tm.fileDeleted(info.opened, parent.IsTrash(), inode, info.length)\n\t}\n\treturn 0\n}\n\nfunc (m *dbMeta) doCleanStaleSession(sid uint64) error {\n\tvar fail bool\n\t// release locks\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tif _, err := s.Delete(flock{Sid: sid}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := s.Delete(plock{Sid: sid}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"Delete flock/plock with sid %d: %s\", sid, err)\n\t\tfail = true\n\t}\n\n\tvar sus []sustained\n\terr = m.simpleTxn(Background(), func(ses *xorm.Session) error {\n\t\tsus = nil\n\t\treturn ses.Find(&sus, &sustained{Sid: sid})\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"Scan sustained with sid %d: %s\", sid, err)\n\t\tfail = true\n\t} else {\n\t\tfor _, su := range sus {\n\t\t\tif err = m.doDeleteSustainedInode(sid, su.Inode); err != nil {\n\t\t\t\tlogger.Warnf(\"Delete sustained inode %d of sid %d: %s\", su.Inode, sid, err)\n\t\t\t\tfail = true\n\t\t\t}\n\t\t}\n\t}\n\n\tif fail {\n\t\treturn fmt.Errorf(\"failed to clean up sid %d\", sid)\n\t} else {\n\t\treturn m.txn(func(s *xorm.Session) error {\n\t\t\tif n, err := s.Delete(&session2{Sid: sid}); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if n == 1 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tok, err := s.IsTableExist(&session{})\n\t\t\tif err == nil && ok {\n\t\t\t\t_, err = s.Delete(&session{Sid: sid})\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t}\n}\n\nfunc (m *dbMeta) doFindStaleSessions(limit int) ([]uint64, error) {\n\tvar sids []uint64\n\t_ = m.simpleTxn(Background(), func(ses *xorm.Session) error {\n\t\tvar ss []session2\n\t\terr := ses.Where(\"Expire < ?\", time.Now().Unix()).Limit(limit, 0).Find(&ss)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, s := range ss {\n\t\t\tsids = append(sids, s.Sid)\n\t\t}\n\t\treturn nil\n\t})\n\n\tlimit -= len(sids)\n\tif limit <= 0 {\n\t\treturn sids, nil\n\t}\n\n\terr := m.simpleTxn(Background(), func(ses *xorm.Session) error {\n\t\tif ok, err := ses.IsTableExist(&session{}); err != nil {\n\t\t\treturn err\n\t\t} else if ok {\n\t\t\tvar ls []session\n\t\t\terr := ses.Where(\"Heartbeat < ?\", time.Now().Add(time.Minute*-5).Unix()).Limit(limit, 0).Find(&ls)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, l := range ls {\n\t\t\t\tsids = append(sids, l.Sid)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlogger.Errorf(\"Check legacy session table: %s\", err)\n\t}\n\n\treturn sids, nil\n}\n\nfunc (m *dbMeta) doRefreshSession() error {\n\treturn m.txn(func(ses *xorm.Session) error {\n\t\tn, err := ses.Cols(\"Expire\").Update(&session2{Expire: m.expireTime()}, &session2{Sid: m.sid})\n\t\tif err == nil && n == 0 {\n\t\t\tlogger.Warnf(\"Session %d was stale and cleaned up, but now it comes back again\", m.sid)\n\t\t\terr = mustInsert(ses, &session2{m.sid, m.expireTime(), m.newSessionInfo()})\n\t\t}\n\t\treturn err\n\t})\n}\n\nfunc (m *dbMeta) doDeleteSustainedInode(sid uint64, inode Ino) error {\n\tvar n = node{Inode: inode}\n\tvar newSpace int64\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tnewSpace = 0\n\t\tn = node{Inode: inode}\n\t\tok, err := s.ForUpdate().Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn nil\n\t\t}\n\t\tnewSpace = -align4K(n.Length)\n\t\tif err = mustInsert(s, &delfile{inode, n.Length, time.Now().Unix()}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err = s.Delete(&sustained{Sid: sid, Inode: inode})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err = s.Delete(&node{Inode: inode})\n\t\treturn err\n\t}, inode)\n\tif err == nil && newSpace < 0 {\n\t\tm.updateStats(newSpace, -1)\n\t\tm.tryDeleteFileData(inode, n.Length, false)\n\t\tm.updateUserGroupStat(Background(), n.Uid, n.Gid, newSpace, 0)\n\t}\n\treturn err\n}\n\nfunc (m *dbMeta) doRead(ctx Context, inode Ino, indx uint32) ([]*slice, syscall.Errno) {\n\tvar c = chunk{Inode: inode, Indx: indx}\n\tif err := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\t_, err := s.MustCols(\"indx\").Get(&c)\n\t\treturn err\n\t}); err != nil {\n\t\treturn nil, errno(err)\n\t}\n\treturn readSliceBuf(c.Slices), 0\n}\n\nfunc (m *dbMeta) doList(ctx Context, inode Ino) ([]*slice, syscall.Errno) {\n\tvar chunks []chunk\n\tif err := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Cols(\"slices\").Find(&chunks, &chunk{Inode: inode})\n\t}); err != nil {\n\t\treturn nil, errno(err)\n\t}\n\tvar slices []*slice\n\tfor _, c := range chunks {\n\t\tss := readSliceBuf(c.Slices)\n\t\tif ss == nil {\n\t\t\tcontinue\n\t\t}\n\t\tslices = append(slices, ss...)\n\t}\n\treturn slices, 0\n}\n\nfunc (m *dbMeta) doWrite(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time, numSlices *int, delta *dirStat, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\t*delta = dirStat{}\n\t\tnodeAttr := node{Inode: inode}\n\t\tok, err := s.ForUpdate().Get(&nodeAttr)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif nodeAttr.Type != TypeFile {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tnewleng := uint64(indx)*ChunkSize + uint64(off) + uint64(slice.Len)\n\t\tif newleng > nodeAttr.Length {\n\t\t\tdelta.length = int64(newleng - nodeAttr.Length)\n\t\t\tdelta.space = align4K(newleng) - align4K(nodeAttr.Length)\n\t\t\tnodeAttr.Length = newleng\n\t\t}\n\t\tif err := m.checkQuota(ctx, delta.space, 0, nodeAttr.Uid, nodeAttr.Gid, m.getParents(s, inode, nodeAttr.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tnodeAttr.setMtime(mtime.UnixNano())\n\t\tnodeAttr.setCtime(time.Now().UnixNano())\n\t\tm.parseAttr(&nodeAttr, attr)\n\n\t\tbuf := marshalSlice(off, slice.Id, slice.Size, slice.Off, slice.Len)\n\t\tvar insert bool // no compaction check for the first slice\n\t\tif err = m.upsertSlice(s, inode, indx, buf, &insert); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err = mustInsert(s, sliceRef{slice.Id, slice.Size, 1}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err = s.Cols(\"length\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&nodeAttr, &node{Inode: inode})\n\t\tif err == nil && !insert {\n\t\t\tck := chunk{Inode: inode, Indx: indx}\n\t\t\t_, _ = s.MustCols(\"indx\").Get(&ck)\n\t\t\t*numSlices = len(ck.Slices) / sliceBytes\n\t\t}\n\t\treturn err\n\t}, inode))\n}\n\nfunc (m *dbMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno {\n\tdefer m.timeit(\"CopyFileRange\", time.Now())\n\tf := m.of.find(fout)\n\tif f != nil {\n\t\tf.Lock()\n\t\tdefer f.Unlock()\n\t}\n\tvar newLength, newSpace int64\n\tvar nin, nout node\n\tdefer func() { m.of.InvalidateChunk(fout, invalidateAllChunks) }()\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tnewLength, newSpace = 0, 0\n\t\tnin = node{Inode: fin}\n\t\tnout = node{Inode: fout}\n\t\terr := m.getNodesForUpdate(s, &nin, &nout)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif nin.Type != TypeFile {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif offIn >= nin.Length {\n\t\t\tif copied != nil {\n\t\t\t\t*copied = 0\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tsize := size\n\t\tif offIn+size > nin.Length {\n\t\t\tsize = nin.Length - offIn\n\t\t}\n\t\tif nout.Type != TypeFile {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif (nout.Flags&FlagImmutable) != 0 || (nout.Flags&FlagAppend) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tnewleng := offOut + size\n\t\tif newleng > nout.Length {\n\t\t\tnewLength = int64(newleng - nout.Length)\n\t\t\tnewSpace = align4K(newleng) - align4K(nout.Length)\n\t\t\tnout.Length = newleng\n\t\t}\n\t\tif err := m.checkQuota(ctx, newSpace, 0, nout.Uid, nout.Gid, m.getParents(s, fout, nout.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tnow := time.Now().UnixNano()\n\t\tnout.setMtime(now)\n\t\tnout.setCtime(now)\n\t\tif outLength != nil {\n\t\t\t*outLength = nout.Length\n\t\t}\n\n\t\tvar cs []chunk\n\t\terr = s.Where(\"inode = ? AND indx >= ? AND indx <= ?\", fin, offIn/ChunkSize, (offIn+size)/ChunkSize).ForUpdate().Find(&cs)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tchunks := make(map[uint32][]*slice)\n\t\tfor _, c := range cs {\n\t\t\tchunks[c.Indx] = readSliceBuf(c.Slices)\n\t\t\tif chunks[c.Indx] == nil {\n\t\t\t\treturn syscall.EIO\n\t\t\t}\n\t\t}\n\n\t\tses := s\n\t\tupdateSlices := func(indx uint32, buf []byte, id uint64, size uint32) error {\n\t\t\tif err := m.appendSlice(ses, fout, indx, buf); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif id > 0 {\n\t\t\t\tif _, err := ses.Exec(m.sqlConv(\"update chunk_ref set refs=refs+1 where chunkid = ? AND size = ?\"), id, size); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tcoff := offIn / ChunkSize * ChunkSize\n\t\tfor coff < offIn+size {\n\t\t\tif coff%ChunkSize != 0 {\n\t\t\t\tpanic(\"coff\")\n\t\t\t}\n\t\t\t// Add a zero chunk for hole\n\t\t\tss := append([]*slice{{len: ChunkSize}}, chunks[uint32(coff/ChunkSize)]...)\n\t\t\tcs := buildSlice(ss)\n\t\t\tfor _, s := range cs {\n\t\t\t\tpos := coff\n\t\t\t\tcoff += uint64(s.Len)\n\t\t\t\tif pos < offIn+size && pos+uint64(s.Len) > offIn {\n\t\t\t\t\tif pos < offIn {\n\t\t\t\t\t\tdec := offIn - pos\n\t\t\t\t\t\ts.Off += uint32(dec)\n\t\t\t\t\t\tpos += dec\n\t\t\t\t\t\ts.Len -= uint32(dec)\n\t\t\t\t\t}\n\t\t\t\t\tif pos+uint64(s.Len) > offIn+size {\n\t\t\t\t\t\tdec := pos + uint64(s.Len) - (offIn + size)\n\t\t\t\t\t\ts.Len -= uint32(dec)\n\t\t\t\t\t}\n\t\t\t\t\tdoff := pos - offIn + offOut\n\t\t\t\t\tindx := uint32(doff / ChunkSize)\n\t\t\t\t\tdpos := uint32(doff % ChunkSize)\n\t\t\t\t\tif dpos+s.Len > ChunkSize {\n\t\t\t\t\t\tif err := updateSlices(indx, marshalSlice(dpos, s.Id, s.Size, s.Off, ChunkSize-dpos), s.Id, s.Size); err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tskip := ChunkSize - dpos\n\t\t\t\t\t\tif err := updateSlices(indx+1, marshalSlice(0, s.Id, s.Size, s.Off+skip, s.Len-skip), s.Id, s.Size); err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif err := updateSlices(indx, marshalSlice(dpos, s.Id, s.Size, s.Off, s.Len), s.Id, s.Size); err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif _, err := s.Cols(\"length\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&nout, &node{Inode: fout}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif copied != nil {\n\t\t\t*copied = size\n\t\t}\n\t\treturn nil\n\t}, fout)\n\tif err == nil {\n\t\tm.updateParentStat(ctx, fout, nout.Parent, newLength, newSpace)\n\t\tm.updateUserGroupStat(ctx, nout.Uid, nout.Gid, newSpace, 0)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *dbMeta) getParents(s *xorm.Session, inode, parent Ino) []Ino {\n\tif parent > 0 {\n\t\treturn []Ino{parent}\n\t}\n\tvar rows []edge\n\tif err := s.Find(&rows, &edge{Inode: inode}); err != nil {\n\t\tlogger.Warnf(\"Scan edge key of inode %d: %s\", inode, err)\n\t\treturn nil\n\t}\n\tps := make(map[Ino]struct{})\n\tfor _, row := range rows {\n\t\tps[row.Parent] = struct{}{}\n\t}\n\tpss := make([]Ino, 0, len(ps))\n\tfor p := range ps {\n\t\tpss = append(pss, p)\n\t}\n\treturn pss\n}\n\nfunc (m *dbMeta) doGetParents(ctx Context, inode Ino) map[Ino]int {\n\tvar rows []edge\n\tif err := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\trows = nil\n\t\treturn s.Find(&rows, &edge{Inode: inode})\n\t}); err != nil {\n\t\tlogger.Warnf(\"Scan edge key of inode %d: %s\", inode, err)\n\t\treturn nil\n\t}\n\tps := make(map[Ino]int)\n\tfor _, row := range rows {\n\t\tps[row.Parent]++\n\t}\n\treturn ps\n}\n\nfunc (m *dbMeta) doUpdateDirStat(ctx Context, batch map[Ino]dirStat) error {\n\ttable := m.db.GetTableMapper().Obj2Table(\"dirStats\")\n\tfileLengthColumn := m.db.GetColumnMapper().Obj2Table(\"DataLength\")\n\tusedSpaceColumn := m.db.GetColumnMapper().Obj2Table(\"UsedSpace\")\n\tusedInodeColumn := m.db.GetColumnMapper().Obj2Table(\"UsedInodes\")\n\tsql := fmt.Sprintf(\n\t\t\"update `%s` set `%s` = `%s` + ?, `%s` = `%s` + ?, `%s` = `%s` + ? where `inode` = ?\",\n\t\ttable,\n\t\tfileLengthColumn, fileLengthColumn,\n\t\tusedSpaceColumn, usedSpaceColumn,\n\t\tusedInodeColumn, usedInodeColumn,\n\t)\n\n\tnonexist := make(map[Ino]bool, 0)\n\n\tfor _, group := range m.groupBatch(batch, 1000) {\n\t\terr := m.txn(func(s *xorm.Session) error {\n\t\t\tfor _, ino := range group {\n\t\t\t\tstat := batch[ino]\n\t\t\t\tret, err := s.Exec(sql, stat.length, stat.space, stat.inodes, ino)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\taffected, err := ret.RowsAffected()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif affected == 0 {\n\t\t\t\t\tnonexist[ino] = true\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif len(nonexist) > 0 {\n\t\tm.parallelSyncDirStat(ctx, nonexist).Wait()\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) doSyncDirStat(ctx Context, ino Ino) (*dirStat, syscall.Errno) {\n\tif m.conf.ReadOnly {\n\t\treturn nil, syscall.EROFS\n\t}\n\tstat, st := m.calcDirStat(ctx, ino)\n\tif st != 0 {\n\t\treturn nil, st\n\t}\n\terr := m.txn(func(s *xorm.Session) error {\n\t\texist, err := s.Exist(&node{Inode: ino})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !exist {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\trecord := &dirStats{ino, stat.length, stat.space, stat.inodes}\n\t\t_, err = s.Insert(record)\n\t\tif err != nil && isDuplicateEntryErr(err) {\n\t\t\t_, err = s.Cols(\"data_length\", \"used_space\", \"used_inodes\").Update(record, &dirStats{Inode: ino})\n\t\t}\n\t\treturn err\n\t})\n\treturn stat, errno(err)\n}\n\nfunc (m *dbMeta) doGetDirStat(ctx Context, ino Ino, trySync bool) (*dirStat, syscall.Errno) {\n\tst := dirStats{Inode: ino}\n\tvar exist bool\n\tvar err error\n\tif err = m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\texist, err = s.Get(&st)\n\t\treturn err\n\t}); err != nil {\n\t\treturn nil, errno(err)\n\t}\n\tif !exist {\n\t\tif trySync {\n\t\t\treturn m.doSyncDirStat(ctx, ino)\n\t\t}\n\t\treturn nil, 0\n\t}\n\n\tif trySync && (st.UsedSpace < 0 || st.UsedInodes < 0) {\n\t\tlogger.Warnf(\n\t\t\t\"dir usage of inode %d is invalid: space %d, inodes %d, try to fix\",\n\t\t\tino, st.UsedSpace, st.UsedInodes,\n\t\t)\n\t\tstat, eno := m.calcDirStat(ctx, ino)\n\t\tif eno != 0 {\n\t\t\treturn nil, eno\n\t\t}\n\t\tst.DataLength, st.UsedSpace, st.UsedInodes = stat.length, stat.space, stat.inodes\n\t\te := m.txn(func(s *xorm.Session) error {\n\t\t\tn, err := s.Cols(\"data_length\", \"used_space\", \"used_inodes\").Update(&st, &dirStats{Inode: ino})\n\t\t\tif err == nil && n != 1 {\n\t\t\t\terr = errors.Errorf(\"update dir usage of inode %d: %d rows affected\", ino, n)\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t\tif e != nil {\n\t\t\tlogger.Warn(e)\n\t\t}\n\t}\n\treturn &dirStat{st.DataLength, st.UsedSpace, st.UsedInodes}, 0\n}\n\nfunc (m *dbMeta) doFindDeletedFiles(ts int64, limit int) (map[Ino]uint64, error) {\n\tfiles := make(map[Ino]uint64)\n\terr := m.simpleTxn(Background(), func(s *xorm.Session) error {\n\t\tvar ds []delfile\n\t\terr := s.Where(\"expire < ?\", ts).Limit(limit, 0).Find(&ds)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, d := range ds {\n\t\t\tfiles[d.Inode] = d.Length\n\t\t}\n\t\treturn nil\n\t})\n\treturn files, err\n}\n\nfunc (m *dbMeta) doCleanupSlices(ctx Context, count *uint64) error {\n\tvar cks []sliceRef\n\tif err := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tcks = nil\n\t\treturn s.Where(\"refs <= 0\").Find(&cks)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tfor _, ck := range cks {\n\t\tm.deleteSlice(ck.Id, ck.Size)\n\t\tif count != nil {\n\t\t\t*count++\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\treturn ctx.Err()\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) deleteChunk(inode Ino, indx uint32) error {\n\tvar ss []*slice\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tss = ss[:0]\n\t\tvar c = chunk{Inode: inode, Indx: indx}\n\t\tok, err := s.ForUpdate().MustCols(\"indx\").Get(&c)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn nil\n\t\t}\n\t\tss = readSliceBuf(c.Slices)\n\t\tif ss == nil {\n\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d, use `gc` to clean up leaked slices\", inode, indx)\n\t\t}\n\t\tfor _, sc := range ss {\n\t\t\tif sc.id == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_, err = s.Exec(m.sqlConv(\"update chunk_ref set refs=refs-1 where chunkid=? AND size=?\"), sc.id, sc.size)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tc.Slices = nil\n\t\tn, err := s.Where(\"inode = ? AND indx = ?\", inode, indx).Delete(&c)\n\t\tif err == nil && n == 0 {\n\t\t\terr = fmt.Errorf(\"chunk %d:%d changed, try restarting transaction\", inode, indx)\n\t\t}\n\t\treturn err\n\t})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"delete slice from chunk %s fail: %s, retry later\", inode, err)\n\t}\n\tfor _, s := range ss {\n\t\tif s.id == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tvar ref = sliceRef{Id: s.id}\n\t\terr := m.simpleTxn(Background(), func(s *xorm.Session) error {\n\t\t\tok, err := s.Get(&ref)\n\t\t\tif err == nil && !ok {\n\t\t\t\terr = errors.New(\"not found\")\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t\tif err == nil && ref.Refs <= 0 {\n\t\t\tm.deleteSlice(s.id, s.size)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) doDeleteFileData(inode Ino, length uint64) {\n\tvar indexes []chunk\n\t_ = m.simpleTxn(Background(), func(s *xorm.Session) error {\n\t\tindexes = nil\n\t\treturn s.Cols(\"indx\").Find(&indexes, &chunk{Inode: inode})\n\t})\n\tfor _, c := range indexes {\n\t\terr := m.deleteChunk(inode, c.Indx)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"deleteChunk inode %d index %d error: %s\", inode, c.Indx, err)\n\t\t\treturn\n\t\t}\n\t}\n\t_ = m.txn(func(s *xorm.Session) error {\n\t\t_, err := s.Delete(delfile{Inode: inode})\n\t\treturn err\n\t})\n}\n\nfunc (m *dbMeta) doCleanupDelayedSlices(ctx Context, edge int64) (int, error) {\n\tvar count int\n\tvar ss []Slice\n\tvar result []delslices\n\tvar batch int = 1e6\n\tfor {\n\t\t_ = m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\t\tresult = result[:0]\n\t\t\treturn s.Where(\"deleted < ?\", edge).Limit(batch, 0).Find(&result)\n\t\t})\n\n\t\tfor _, ds := range result {\n\t\t\tif err := m.txn(func(ses *xorm.Session) error {\n\t\t\t\tss = ss[:0]\n\t\t\t\tds := delslices{Id: ds.Id}\n\t\t\t\tif ok, e := ses.ForUpdate().Get(&ds); e != nil {\n\t\t\t\t\treturn e\n\t\t\t\t} else if !ok {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tm.decodeDelayedSlices(ds.Slices, &ss)\n\t\t\t\tif len(ss) == 0 {\n\t\t\t\t\treturn fmt.Errorf(\"invalid value for delayed slices %d: %v\", ds.Id, ds.Slices)\n\t\t\t\t}\n\t\t\t\tfor _, s := range ss {\n\t\t\t\t\tif _, e := ses.Exec(m.sqlConv(\"update chunk_ref set refs=refs-1 where chunkid=? AND size=?\"), s.Id, s.Size); e != nil {\n\t\t\t\t\t\treturn e\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t_, e := ses.Delete(&delslices{Id: ds.Id})\n\t\t\t\treturn e\n\t\t\t}); err != nil {\n\t\t\t\tlogger.Warnf(\"Cleanup delayed slices %d: %s\", ds.Id, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, s := range ss {\n\t\t\t\tvar ref = sliceRef{Id: s.Id}\n\t\t\t\terr := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\t\t\t\tok, err := s.Get(&ref)\n\t\t\t\t\tif err == nil && !ok {\n\t\t\t\t\t\terr = errors.New(\"not found\")\n\t\t\t\t\t}\n\t\t\t\t\treturn err\n\t\t\t\t})\n\t\t\t\tif err == nil && ref.Refs <= 0 {\n\t\t\t\t\tm.deleteSlice(s.Id, s.Size)\n\t\t\t\t\tcount++\n\t\t\t\t}\n\t\t\t\tif ctx.Canceled() {\n\t\t\t\t\treturn count, ctx.Err()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(result) < batch {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn count, nil\n}\n\nfunc (m *dbMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno {\n\tst := errno(m.txn(func(s *xorm.Session) error {\n\t\tvar c2 = chunk{Inode: inode, Indx: indx}\n\t\t_, err := s.ForUpdate().MustCols(\"indx\").Get(&c2)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(c2.Slices) < len(origin) || !bytes.Equal(origin, c2.Slices[:len(origin)]) {\n\t\t\tlogger.Infof(\"chunk %d:%d was changed %d -> %d\", inode, indx, len(origin), len(c2.Slices))\n\t\t\treturn syscall.EINVAL\n\t\t}\n\n\t\tc2.Slices = append(append(c2.Slices[:skipped*sliceBytes], marshalSlice(pos, id, size, 0, size)...), c2.Slices[len(origin):]...)\n\t\tif _, err := s.Cols(\"slices\").Where(\"Inode = ? AND indx = ?\", inode, indx).Update(c2); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// create the key to tracking it\n\t\tif err = mustInsert(s, sliceRef{id, size, 1}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif delayed != nil {\n\t\t\tif len(delayed) > 0 {\n\t\t\t\tif err = mustInsert(s, &delslices{id, time.Now().Unix(), delayed}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor _, s_ := range ss {\n\t\t\t\tif s_.id == 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif _, err := s.Exec(m.sqlConv(\"update chunk_ref set refs=refs-1 where chunkid=? AND size=?\"), s_.id, s_.size); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}, inode))\n\t// there could be false-negative that the compaction is successful, double-check\n\tif st != 0 && st != syscall.EINVAL {\n\t\tvar ok bool\n\t\tif err := m.simpleTxn(Background(), func(s *xorm.Session) error {\n\t\t\tvar e error\n\t\t\tok, e = s.Get(&sliceRef{Id: id})\n\t\t\treturn e\n\t\t}); err == nil {\n\t\t\tif ok {\n\t\t\t\tst = 0\n\t\t\t} else {\n\t\t\t\tlogger.Infof(\"compacted chunk %d was not used\", id)\n\t\t\t\tst = syscall.EINVAL\n\t\t\t}\n\t\t}\n\t}\n\n\tif st == syscall.EINVAL {\n\t\t_ = m.txn(func(s *xorm.Session) error {\n\t\t\treturn mustInsert(s, &sliceRef{id, size, 0})\n\t\t})\n\t} else if st == 0 && delayed == nil {\n\t\tfor _, s := range ss {\n\t\t\tif s.id == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvar ref = sliceRef{Id: s.id}\n\t\t\tvar ok bool\n\t\t\terr := m.simpleTxn(Background(), func(s *xorm.Session) error {\n\t\t\t\tvar e error\n\t\t\t\tok, e = s.Get(&ref)\n\t\t\t\treturn e\n\t\t\t})\n\t\t\tif err == nil && ok && ref.Refs <= 0 {\n\t\t\t\tm.deleteSlice(s.id, s.size)\n\t\t\t}\n\t\t}\n\t}\n\treturn st\n}\n\nfunc dup(b []byte) []byte {\n\tr := make([]byte, len(b))\n\tcopy(r, b)\n\treturn r\n}\n\nfunc (m *dbMeta) scanAllChunks(ctx Context, ch chan<- cchunk, bar *utils.Bar) error {\n\treturn m.roTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Table(&chunk{}).Iterate(new(chunk), func(idx int, bean interface{}) error {\n\t\t\tc := bean.(*chunk)\n\t\t\tif len(c.Slices) > sliceBytes {\n\t\t\t\tbar.IncrTotal(1)\n\t\t\t\tch <- cchunk{c.Inode, c.Indx, len(c.Slices) / sliceBytes}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n}\n\nfunc (m *dbMeta) ListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno {\n\tif delete {\n\t\t_ = m.doCleanupSlices(ctx, nil)\n\t}\n\terr := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tvar cs []chunk\n\t\terr := s.Find(&cs)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, c := range cs {\n\t\t\tss := readSliceBuf(c.Slices)\n\t\t\tif ss == nil {\n\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d\", c.Inode, c.Indx)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, s := range ss {\n\t\t\t\tif s.id > 0 {\n\t\t\t\t\tslices[c.Inode] = append(slices[c.Inode], Slice{Id: s.id, Size: s.size})\n\t\t\t\t\tif showProgress != nil {\n\t\t\t\t\t\tshowProgress()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\n\tif scanPending {\n\t\t_ = m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\t\tvar cks []sliceRef\n\t\t\terr := s.Where(\"refs <= 0\").Find(&cks)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, ck := range cks {\n\t\t\t\tslices[0] = append(slices[0], Slice{Id: ck.Id, Size: ck.Size})\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tif m.getFormat().TrashDays == 0 {\n\t\treturn 0\n\t}\n\treturn errno(m.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) {\n\t\tslices[1] = append(slices[1], ss...)\n\t\tif showProgress != nil {\n\t\t\tfor range ss {\n\t\t\t\tshowProgress()\n\t\t\t}\n\t\t}\n\t\treturn false, nil\n\t}))\n}\n\nfunc (m *dbMeta) scanTrashSlices(ctx Context, scan trashSliceScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\tvar dss []delslices\n\n\terr := m.simpleTxn(ctx, func(tx *xorm.Session) error {\n\t\tif ok, err := tx.IsTableExist(&delslices{}); err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn nil\n\t\t}\n\t\treturn tx.Find(&dss)\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar ss []Slice\n\tfor _, ds := range dss {\n\t\tvar clean bool\n\t\terr = m.txn(func(tx *xorm.Session) error {\n\t\t\tss = ss[:0]\n\t\t\tdel := delslices{Id: ds.Id}\n\t\t\tfound, err := tx.Get(&del)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrapf(err, \"get delslices %d\", ds.Id)\n\t\t\t}\n\t\t\tif !found {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tm.decodeDelayedSlices(del.Slices, &ss)\n\t\t\tclean, err = scan(ss, del.Deleted)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif clean {\n\t\t\t\tfor _, s := range ss {\n\t\t\t\t\tif _, e := tx.Exec(m.sqlConv(\"update chunk_ref set refs=refs-1 where chunkid=? AND size=?\"), s.Id, s.Size); e != nil {\n\t\t\t\t\t\treturn e\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t_, err = tx.Delete(del)\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif clean {\n\t\t\tfor _, s := range ss {\n\t\t\t\tvar ref = sliceRef{Id: s.Id}\n\t\t\t\terr := m.simpleTxn(ctx, func(tx *xorm.Session) error {\n\t\t\t\t\tok, err := tx.Get(&ref)\n\t\t\t\t\tif err == nil && !ok {\n\t\t\t\t\t\terr = errors.New(\"not found\")\n\t\t\t\t\t}\n\t\t\t\t\treturn err\n\t\t\t\t})\n\t\t\t\tif err == nil && ref.Refs <= 0 {\n\t\t\t\t\tm.deleteSlice(s.Id, s.Size)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) scanPendingSlices(ctx Context, scan pendingSliceScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\tvar refs []sliceRef\n\terr := m.simpleTxn(ctx, func(tx *xorm.Session) error {\n\t\tif ok, err := tx.IsTableExist(&sliceRef{}); err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn nil\n\t\t}\n\t\treturn tx.Where(\"refs <= 0\").Find(&refs)\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"scan slice refs\")\n\t}\n\tfor _, ref := range refs {\n\t\tclean, err := scan(ref.Id, ref.Size)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"scan slice\")\n\t\t}\n\t\tif clean {\n\t\t\t// TODO: m.deleteSlice(ref.Id, ref.Size)\n\t\t\t// avoid lint warning\n\t\t\t_ = clean\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) scanPendingFiles(ctx Context, scan pendingFileScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\n\tvar dfs []delfile\n\tif err := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tif ok, err := s.IsTableExist(&delfile{}); err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn nil\n\t\t}\n\t\treturn s.Find(&dfs)\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tfor _, ds := range dfs {\n\t\tif _, err := scan(ds.Inode, ds.Length, ds.Expire); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (m *dbMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno {\n\tn := &node{\n\t\tInode: inode,\n\t\tType: attr.Typ,\n\t\tMode: attr.Mode,\n\t\tUid: attr.Uid,\n\t\tGid: attr.Gid,\n\t\tLength: attr.Length,\n\t\tParent: attr.Parent,\n\t\tNlink: attr.Nlink,\n\t}\n\tn.setAtime(attr.Atime*1e9 + int64(attr.Atimensec))\n\tn.setMtime(attr.Mtime*1e9 + int64(attr.Mtimensec))\n\tn.setCtime(attr.Ctime*1e9 + int64(attr.Ctimensec))\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tn.Nlink = 2\n\t\tvar rows []edge\n\t\tif err := s.Find(&rows, &edge{Parent: inode}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, row := range rows {\n\t\t\tif row.Type == TypeDirectory {\n\t\t\t\tn.Nlink++\n\t\t\t}\n\t\t}\n\t\tok, err := s.ForUpdate().Get(&node{Inode: inode})\n\t\tif err == nil {\n\t\t\tif ok {\n\t\t\t\tupdateColumns := []string{\n\t\t\t\t\t\"type\", \"mode\",\n\t\t\t\t\t\"uid\", \"gid\",\n\t\t\t\t\t\"length\", \"parent\", \"nlink\",\n\t\t\t\t\t\"atime\", \"mtime\", \"ctime\",\n\t\t\t\t\t\"atimensec\", \"mtimensec\", \"ctimensec\",\n\t\t\t\t}\n\t\t\t\t_, err = s.Cols(updateColumns...).Update(n, &node{Inode: inode})\n\t\t\t} else {\n\t\t\t\terr = mustInsert(s, n)\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}, inode))\n}\n\nfunc (m *dbMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno {\n\tdefer m.timeit(\"GetXattr\", time.Now())\n\tinode = m.checkRoot(inode)\n\treturn errno(m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tvar x = xattr{Inode: inode, Name: name}\n\t\tok, err := s.Get(&x)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn ENOATTR\n\t\t}\n\t\t*vbuff = x.Value\n\t\treturn nil\n\t}))\n}\n\nfunc (m *dbMeta) ListXattr(ctx Context, inode Ino, names *[]byte) syscall.Errno {\n\tdefer m.timeit(\"ListXattr\", time.Now())\n\tinode = m.checkRoot(inode)\n\treturn errno(m.roTxn(ctx, func(s *xorm.Session) error {\n\t\tvar xs []xattr\n\t\terr := s.Where(\"inode = ?\", inode).Find(&xs, &xattr{Inode: inode})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t*names = nil\n\t\tfor _, x := range xs {\n\t\t\t*names = append(*names, []byte(x.Name)...)\n\t\t\t*names = append(*names, 0)\n\t\t}\n\n\t\tvar n = node{Inode: inode}\n\t\tok, err := s.Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tattr := &Attr{}\n\t\tm.parseAttr(&n, attr)\n\t\tsetXAttrACL(names, attr.AccessACL, attr.DefaultACL)\n\t\treturn nil\n\t}))\n}\n\nfunc (m *dbMeta) doSetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tvar k = &xattr{Inode: inode, Name: name}\n\t\tvar x = xattr{Inode: inode, Name: name, Value: value}\n\t\tok, err := s.ForUpdate().Get(k)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\texisting := k.Value\n\t\tk.Value = nil\n\t\tswitch flags {\n\t\tcase XattrCreate:\n\t\t\tif ok {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\terr = mustInsert(s, &x)\n\t\tcase XattrReplace:\n\t\t\tif !ok {\n\t\t\t\treturn ENOATTR\n\t\t\t}\n\t\t\t_, err = s.Cols(\"value\").Update(&x, k)\n\t\tdefault:\n\t\t\tif !ok {\n\t\t\t\terr = mustInsert(s, &x)\n\t\t\t} else if !bytes.Equal(existing, value) {\n\t\t\t\t_, err = s.Cols(\"value\").Update(&x, k)\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}))\n}\n\nfunc (m *dbMeta) doRemoveXattr(ctx Context, inode Ino, name string) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tn, err := s.Delete(&xattr{Inode: inode, Name: name})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t} else if n == 0 {\n\t\t\treturn ENOATTR\n\t\t} else {\n\t\t\treturn nil\n\t\t}\n\t}))\n}\n\nfunc (m *dbMeta) doGetQuota(ctx Context, qtype uint32, key uint64) (*Quota, error) {\n\tif qtype != DirQuotaType && qtype != UserQuotaType && qtype != GroupQuotaType {\n\t\treturn nil, errors.Errorf(\"invalid quota type %d\", qtype)\n\t}\n\n\tvar quota *Quota\n\terr := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tif qtype == DirQuotaType {\n\t\t\tq := &dirQuota{Inode: Ino(key)}\n\t\t\tok, e := s.Get(q)\n\t\t\tif e == nil && ok {\n\t\t\t\tquota = &Quota{\n\t\t\t\t\tMaxSpace: q.MaxSpace,\n\t\t\t\t\tMaxInodes: q.MaxInodes,\n\t\t\t\t\tUsedSpace: q.UsedSpace,\n\t\t\t\t\tUsedInodes: q.UsedInodes}\n\t\t\t}\n\t\t\treturn e\n\t\t} else {\n\t\t\tq := &userGroupQuota{Qtype: qtype, Qkey: key}\n\t\t\tok, e := s.Get(q)\n\t\t\tif e == nil && ok {\n\t\t\t\tquota = &Quota{\n\t\t\t\t\tMaxSpace: q.MaxSpace,\n\t\t\t\t\tMaxInodes: q.MaxInodes,\n\t\t\t\t\tUsedSpace: q.UsedSpace,\n\t\t\t\t\tUsedInodes: q.UsedInodes}\n\t\t\t}\n\t\t\treturn e\n\t\t}\n\t})\n\treturn quota, err\n}\n\nfunc updateQuotaFields(quota *Quota, exist bool, maxSpace, maxInodes *int64, usedSpace, usedInodes *int64) []string {\n\tupdateColumns := make([]string, 0, 4)\n\tif quota.MaxSpace >= 0 {\n\t\t*maxSpace = quota.MaxSpace\n\t\tupdateColumns = append(updateColumns, \"max_space\")\n\t}\n\tif quota.MaxInodes >= 0 {\n\t\t*maxInodes = quota.MaxInodes\n\t\tupdateColumns = append(updateColumns, \"max_inodes\")\n\t}\n\tif quota.UsedSpace >= 0 {\n\t\t*usedSpace = quota.UsedSpace\n\t\tupdateColumns = append(updateColumns, \"used_space\")\n\t} else if !exist {\n\t\t*usedSpace = 0\n\t\tupdateColumns = append(updateColumns, \"used_space\")\n\t}\n\tif quota.UsedInodes >= 0 {\n\t\t*usedInodes = quota.UsedInodes\n\t\tupdateColumns = append(updateColumns, \"used_inodes\")\n\t} else if !exist {\n\t\t*usedInodes = 0\n\t\tupdateColumns = append(updateColumns, \"used_inodes\")\n\t}\n\n\treturn updateColumns\n}\n\nfunc (m *dbMeta) doSetQuota(ctx Context, qtype uint32, key uint64, quota *Quota) (bool, error) {\n\tvar created bool\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tif qtype == DirQuotaType {\n\t\t\torigin := &dirQuota{Inode: Ino(key)}\n\t\t\texist, e := s.ForUpdate().Get(origin)\n\t\t\tif e != nil {\n\t\t\t\treturn e\n\t\t\t}\n\t\t\tcreated = !exist\n\t\t\tupdateColumns := updateQuotaFields(quota, exist, &origin.MaxSpace, &origin.MaxInodes, &origin.UsedSpace, &origin.UsedInodes)\n\t\t\tif exist {\n\t\t\t\t_, e = s.Cols(updateColumns...).Update(origin, &dirQuota{Inode: Ino(key)})\n\t\t\t} else {\n\t\t\t\te = mustInsert(s, origin)\n\t\t\t}\n\t\t\treturn e\n\t\t} else if qtype == UserQuotaType || qtype == GroupQuotaType {\n\t\t\torigin := &userGroupQuota{Qtype: qtype, Qkey: key}\n\t\t\texist, e := s.ForUpdate().Get(origin)\n\t\t\tif e != nil {\n\t\t\t\treturn e\n\t\t\t}\n\t\t\tcreated = !exist\n\t\t\tupdateColumns := updateQuotaFields(quota, exist, &origin.MaxSpace, &origin.MaxInodes, &origin.UsedSpace, &origin.UsedInodes)\n\t\t\tif exist {\n\t\t\t\t_, e = s.Cols(updateColumns...).Update(origin, &userGroupQuota{Qtype: qtype, Qkey: key})\n\t\t\t} else {\n\t\t\t\te = mustInsert(s, origin)\n\t\t\t}\n\t\t\treturn e\n\t\t} else {\n\t\t\treturn errors.Errorf(\"invalid quota type %d\", qtype)\n\t\t}\n\t})\n\n\treturn created, err\n}\n\nfunc (m *dbMeta) doDelQuota(ctx Context, qtype uint32, key uint64) error {\n\tif qtype != DirQuotaType && qtype != UserQuotaType && qtype != GroupQuotaType {\n\t\treturn errors.Errorf(\"invalid quota type %d\", qtype)\n\t}\n\n\treturn m.txn(func(s *xorm.Session) error {\n\t\tif qtype == DirQuotaType {\n\t\t\t_, e := s.Delete(&dirQuota{Inode: Ino(key)})\n\t\t\treturn e\n\t\t} else {\n\t\t\t_, e := s.Cols(\"max_space\", \"max_inodes\").\n\t\t\t\tUpdate(&userGroupQuota{MaxSpace: -1, MaxInodes: -1},\n\t\t\t\t\t&userGroupQuota{Qtype: qtype, Qkey: key})\n\t\t\treturn e\n\t\t}\n\t})\n}\n\nfunc (m *dbMeta) doLoadQuotas(ctx Context) (map[uint64]*Quota, map[uint64]*Quota, map[uint64]*Quota, error) {\n\tvar dirQuotasList []dirQuota\n\tvar userGroupQuotasList []userGroupQuota\n\n\terr := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tif e := s.Find(&dirQuotasList); e != nil {\n\t\t\treturn e\n\t\t}\n\t\tif e := s.Find(&userGroupQuotasList); e != nil {\n\t\t\treturn e\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\treturn nil, nil, nil, err\n\t}\n\n\tdirQuotas := make(map[uint64]*Quota)\n\tuserQuotas := make(map[uint64]*Quota)\n\tgroupQuotas := make(map[uint64]*Quota)\n\n\t// Load directory quotas\n\tfor _, q := range dirQuotasList {\n\t\tquota := &Quota{\n\t\t\tMaxSpace: q.MaxSpace,\n\t\t\tMaxInodes: q.MaxInodes,\n\t\t\tUsedSpace: q.UsedSpace,\n\t\t\tUsedInodes: q.UsedInodes,\n\t\t}\n\t\tdirQuotas[uint64(q.Inode)] = quota\n\t}\n\n\t// Load user and group quotas\n\tfor _, q := range userGroupQuotasList {\n\t\tquota := &Quota{\n\t\t\tMaxSpace: q.MaxSpace,\n\t\t\tMaxInodes: q.MaxInodes,\n\t\t\tUsedSpace: q.UsedSpace,\n\t\t\tUsedInodes: q.UsedInodes,\n\t\t}\n\n\t\tswitch q.Qtype {\n\t\tcase UserQuotaType:\n\t\t\tuserQuotas[q.Qkey] = quota\n\t\tcase GroupQuotaType:\n\t\t\tgroupQuotas[q.Qkey] = quota\n\t\t}\n\t}\n\n\treturn dirQuotas, userQuotas, groupQuotas, nil\n}\n\nfunc (m *dbMeta) doFlushQuotas(ctx Context, quotas []*iQuota) error {\n\tsort.Slice(quotas, func(i, j int) bool { return quotas[i].qkey < quotas[j].qkey })\n\treturn m.txn(func(s *xorm.Session) error {\n\t\tfor _, q := range quotas {\n\t\t\tif q.qtype == DirQuotaType {\n\t\t\t\t_, err := s.Exec(m.sqlConv(\"update dir_quota set used_space=used_space+?, used_inodes=used_inodes+? where inode=?\"),\n\t\t\t\t\tq.quota.newSpace, q.quota.newInodes, q.qkey)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tret, err := s.Exec(m.sqlConv(\"update user_group_quota set used_space=used_space+?, used_inodes=used_inodes+? where qtype=? and qkey=?\"),\n\t\t\t\t\tq.quota.newSpace, q.quota.newInodes, q.qtype, q.qkey)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\taffected, err := ret.RowsAffected()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif affected == 0 {\n\t\t\t\t\tquota := &userGroupQuota{\n\t\t\t\t\t\tQtype: q.qtype,\n\t\t\t\t\t\tQkey: q.qkey,\n\t\t\t\t\t\tMaxSpace: -1,\n\t\t\t\t\t\tMaxInodes: -1,\n\t\t\t\t\t\tUsedSpace: q.quota.newSpace,\n\t\t\t\t\t\tUsedInodes: q.quota.newInodes,\n\t\t\t\t\t}\n\t\t\t\t\tif err := mustInsert(s, quota); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *dbMeta) dumpEntry(s *xorm.Session, inode Ino, typ uint8, e *DumpedEntry, showProgress func(totalIncr, currentIncr int64)) error {\n\tn := &node{Inode: inode}\n\tok, err := s.Get(n)\n\tif err != nil {\n\t\treturn err\n\t}\n\tattr := &Attr{Typ: typ, Nlink: 1}\n\tif !ok {\n\t\tlogger.Warnf(\"The entry of the inode was not found. inode: %d\", inode)\n\t\tif attr.Typ == TypeDirectory {\n\t\t\tattr.Nlink = 2\n\t\t}\n\t} else {\n\t\tm.parseAttr(n, attr)\n\t}\n\tdumpAttr(attr, e.Attr)\n\te.Attr.Inode = inode\n\n\tvar rows []xattr\n\tif err = s.Find(&rows, &xattr{Inode: inode}); err != nil {\n\t\treturn err\n\t}\n\tif len(rows) > 0 {\n\t\txattrs := make([]*DumpedXattr, 0, len(rows))\n\t\tfor _, x := range rows {\n\t\t\txattrs = append(xattrs, &DumpedXattr{x.Name, string(x.Value)})\n\t\t}\n\t\tsort.Slice(xattrs, func(i, j int) bool { return xattrs[i].Name < xattrs[j].Name })\n\t\te.Xattrs = xattrs\n\t}\n\n\taccessACl, err := m.getACL(s, attr.AccessACL)\n\tif err != nil {\n\t\treturn err\n\t}\n\te.AccessACL = dumpACL(accessACl)\n\tdefaultACL, err := m.getACL(s, attr.DefaultACL)\n\tif err != nil {\n\t\treturn err\n\t}\n\te.DefaultACL = dumpACL(defaultACL)\n\n\tif attr.Typ == TypeFile {\n\t\tfor indx := uint32(0); uint64(indx)*ChunkSize < attr.Length; indx++ {\n\t\t\tc := &chunk{Inode: inode, Indx: indx}\n\t\t\tif ok, err = s.MustCols(\"indx\").Get(c); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tss := readSliceBuf(c.Slices)\n\t\t\tif ss == nil {\n\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d\", inode, indx)\n\t\t\t}\n\t\t\tslices := make([]*DumpedSlice, 0, len(ss))\n\t\t\tfor _, s := range ss {\n\t\t\t\tslices = append(slices, &DumpedSlice{Id: s.id, Pos: s.pos, Size: s.size, Off: s.off, Len: s.len})\n\t\t\t}\n\t\t\te.Chunks = append(e.Chunks, &DumpedChunk{indx, slices})\n\t\t}\n\t} else if attr.Typ == TypeSymlink {\n\t\tl := &symlink{Inode: inode}\n\t\tok, err = s.Get(l)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\tlogger.Warnf(\"no link target for inode %d\", inode)\n\t\t}\n\t\te.Symlink = string(l.Target)\n\t} else if attr.Typ == TypeDirectory {\n\t\tvar edges []*edge\n\t\terr := s.Limit(1000, 0).Find(&edges, &edge{Parent: inode})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(int64(len(edges)), 0)\n\t\t}\n\t\tif len(edges) < 1000 {\n\t\t\te.Entries = make(map[string]*DumpedEntry, len(edges))\n\t\t\tfor _, edge := range edges {\n\t\t\t\tname := string(edge.Name)\n\t\t\t\tce := entryPool.Get()\n\t\t\t\tce.Name = name\n\t\t\t\tce.Attr.Inode = edge.Inode\n\t\t\t\tce.Attr.Type = typeToString(edge.Type)\n\t\t\t\te.Entries[name] = ce\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) dumpEntryFast(inode Ino, typ uint8) *DumpedEntry {\n\te := &DumpedEntry{}\n\tn, ok := m.snap.node[inode]\n\tif !ok && inode != TrashInode {\n\t\tlogger.Warnf(\"Corrupt inode: %d, missing attribute\", inode)\n\t}\n\n\tattr := &Attr{Typ: typ, Nlink: 1}\n\tif !ok {\n\t\tlogger.Warnf(\"The entry of the inode was not found. inode: %d\", inode)\n\t\tif attr.Typ == TypeDirectory {\n\t\t\tattr.Nlink = 2\n\t\t}\n\t} else {\n\t\tm.parseAttr(n, attr)\n\t}\n\te.Attr = &DumpedAttr{}\n\tdumpAttr(attr, e.Attr)\n\te.Attr.Inode = inode\n\n\trows, ok := m.snap.xattr[inode]\n\tif ok && len(rows) > 0 {\n\t\txattrs := make([]*DumpedXattr, 0, len(rows))\n\t\tfor _, x := range rows {\n\t\t\txattrs = append(xattrs, &DumpedXattr{x.Name, string(x.Value)})\n\t\t}\n\t\tsort.Slice(xattrs, func(i, j int) bool { return xattrs[i].Name < xattrs[j].Name })\n\t\te.Xattrs = xattrs\n\t}\n\n\tif attr.AccessACL != aclAPI.None {\n\t\te.AccessACL = dumpACL(m.aclCache.Get(attr.AccessACL))\n\t}\n\tif attr.DefaultACL != aclAPI.None {\n\t\te.DefaultACL = dumpACL(m.aclCache.Get(attr.DefaultACL))\n\t}\n\n\tif attr.Typ == TypeFile {\n\t\tfor indx := uint32(0); uint64(indx)*ChunkSize < attr.Length; indx++ {\n\t\t\tc, ok := m.snap.chunk[fmt.Sprintf(\"%d-%d\", inode, indx)]\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tss := readSliceBuf(c.Slices)\n\t\t\tif ss == nil {\n\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d\", inode, indx)\n\t\t\t}\n\t\t\tslices := make([]*DumpedSlice, 0, len(ss))\n\t\t\tfor _, s := range ss {\n\t\t\t\tslices = append(slices, &DumpedSlice{Id: s.id, Pos: s.pos, Size: s.size, Off: s.off, Len: s.len})\n\t\t\t}\n\t\t\te.Chunks = append(e.Chunks, &DumpedChunk{indx, slices})\n\t\t}\n\t} else if attr.Typ == TypeSymlink {\n\t\tl, ok := m.snap.symlink[inode]\n\t\tif !ok {\n\t\t\tlogger.Warnf(\"no link target for inode %d\", inode)\n\t\t\tl = &symlink{}\n\t\t}\n\t\te.Symlink = string(l.Target)\n\t}\n\treturn e\n}\n\nfunc (m *dbMeta) dumpDir(s *xorm.Session, inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth, threads int, showProgress func(totalIncr, currentIncr int64)) error {\n\tbwWrite := func(s string) {\n\t\tif _, err := bw.WriteString(s); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\tif tree.Entries == nil {\n\t\t// retry for large directory\n\t\tvar edges []*edge\n\t\terr := s.Find(&edges, &edge{Parent: inode})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttree.Entries = make(map[string]*DumpedEntry, len(edges))\n\t\tfor _, edge := range edges {\n\t\t\tname := string(edge.Name)\n\t\t\tce := entryPool.Get()\n\t\t\tce.Name = name\n\t\t\tce.Attr.Inode = edge.Inode\n\t\t\tce.Attr.Type = typeToString(edge.Type)\n\t\t\ttree.Entries[name] = ce\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(int64(len(edges))-1000, 0)\n\t\t}\n\t}\n\tvar entries []*DumpedEntry\n\tfor _, e := range tree.Entries {\n\t\tentries = append(entries, e)\n\t}\n\tsort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })\n\t_ = tree.writeJsonWithOutEntry(bw, depth)\n\n\tms := make([]sync.Mutex, threads)\n\tconds := make([]*sync.Cond, threads)\n\tready := make([]bool, threads)\n\tvar err error\n\tfor c := 0; c < threads; c++ {\n\t\tconds[c] = sync.NewCond(&ms[c])\n\t\tif c < len(entries) {\n\t\t\tgo func(c int) {\n\t\t\t\tfor i := c; i < len(entries) && err == nil; i += threads {\n\t\t\t\t\te := entries[i]\n\t\t\t\t\ter := m.roTxn(Background(), func(s *xorm.Session) error {\n\t\t\t\t\t\treturn m.dumpEntry(s, e.Attr.Inode, 0, e, showProgress)\n\t\t\t\t\t})\n\t\t\t\t\tms[c].Lock()\n\t\t\t\t\tready[c] = true\n\t\t\t\t\tif er != nil {\n\t\t\t\t\t\terr = er\n\t\t\t\t\t}\n\t\t\t\t\tconds[c].Signal()\n\t\t\t\t\tfor ready[c] && err == nil {\n\t\t\t\t\t\tconds[c].Wait()\n\t\t\t\t\t}\n\t\t\t\t\tms[c].Unlock()\n\t\t\t\t}\n\t\t\t}(c)\n\t\t}\n\t}\n\n\tfor i, e := range entries {\n\t\tc := i % threads\n\t\tms[c].Lock()\n\t\tfor !ready[c] && err == nil {\n\t\t\tconds[c].Wait()\n\t\t}\n\t\tready[c] = false\n\t\tconds[c].Signal()\n\t\tms[c].Unlock()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif e.Attr.Type == \"directory\" {\n\t\t\terr = m.dumpDir(s, e.Attr.Inode, e, bw, depth+2, threads, showProgress)\n\t\t} else {\n\t\t\terr = e.writeJSON(bw, depth+2)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tentries[i] = nil\n\t\tentryPool.Put(e)\n\t\tif i != len(entries)-1 {\n\t\t\tbwWrite(\",\")\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(0, 1)\n\t\t}\n\t}\n\tbwWrite(fmt.Sprintf(\"\\n%s}\\n%s}\", strings.Repeat(jsonIndent, depth+1), strings.Repeat(jsonIndent, depth)))\n\treturn nil\n}\n\nfunc (m *dbMeta) dumpDirFast(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth int, showProgress func(totalIncr, currentIncr int64)) error {\n\tbwWrite := func(s string) {\n\t\tif _, err := bw.WriteString(s); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\tedges := m.snap.edges[inode]\n\t_ = tree.writeJsonWithOutEntry(bw, depth)\n\tsort.Slice(edges, func(i, j int) bool { return bytes.Compare(edges[i].Name, edges[j].Name) == -1 })\n\n\tfor i, e := range edges {\n\t\tentry := m.dumpEntryFast(e.Inode, e.Type)\n\t\tif entry == nil {\n\t\t\tlogger.Warnf(\"ignore broken entry %s (inode: %d) in %s\", string(e.Name), e.Inode, inode)\n\t\t\tcontinue\n\t\t}\n\n\t\tentry.Name = string(e.Name)\n\t\tif e.Type == TypeDirectory {\n\t\t\t_ = m.dumpDirFast(e.Inode, entry, bw, depth+2, showProgress)\n\t\t} else {\n\t\t\t_ = entry.writeJSON(bw, depth+2)\n\t\t}\n\t\tif i != len(edges)-1 {\n\t\t\tbwWrite(\",\")\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(0, 1)\n\t\t}\n\t}\n\tbwWrite(fmt.Sprintf(\"\\n%s}\\n%s}\", strings.Repeat(jsonIndent, depth+1), strings.Repeat(jsonIndent, depth)))\n\treturn nil\n}\n\nfunc (m *dbMeta) makeSnap(ses *xorm.Session, bar *utils.Bar) error {\n\tsnap := &dbSnap{\n\t\tnode: make(map[Ino]*node),\n\t\tsymlink: make(map[Ino]*symlink),\n\t\txattr: make(map[Ino][]*xattr),\n\t\tedges: make(map[Ino][]*edge),\n\t\tchunk: make(map[string]*chunk),\n\t}\n\n\tfor _, s := range []interface{}{new(node), new(symlink), new(edge), new(xattr), new(chunk), new(acl)} {\n\t\tif count, err := ses.Count(s); err == nil {\n\t\t\tbar.IncrTotal(count)\n\t\t} else {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err := ses.Table(&node{}).Iterate(new(node), func(idx int, bean interface{}) error {\n\t\tn := bean.(*node)\n\t\tsnap.node[n.Inode] = n\n\t\tbar.Increment()\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tif err := ses.Table(&symlink{}).Iterate(new(symlink), func(idx int, bean interface{}) error {\n\t\ts := bean.(*symlink)\n\t\tsnap.symlink[s.Inode] = s\n\t\tbar.Increment()\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\tif err := ses.Table(&edge{}).Iterate(new(edge), func(idx int, bean interface{}) error {\n\t\te := bean.(*edge)\n\t\tsnap.edges[e.Parent] = append(snap.edges[e.Parent], e)\n\t\tbar.Increment()\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tif err := ses.Table(&xattr{}).Iterate(new(xattr), func(idx int, bean interface{}) error {\n\t\tx := bean.(*xattr)\n\t\tsnap.xattr[x.Inode] = append(snap.xattr[x.Inode], x)\n\t\tbar.Increment()\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tif err := ses.Table(&chunk{}).Iterate(new(chunk), func(idx int, bean interface{}) error {\n\t\tc := bean.(*chunk)\n\t\tsnap.chunk[fmt.Sprintf(\"%d-%d\", c.Inode, c.Indx)] = c\n\t\tbar.Increment()\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tif err := ses.Table(&acl{}).Iterate(new(acl), func(idx int, bean interface{}) error {\n\t\ta := bean.(*acl)\n\t\tm.aclCache.Put(a.Id, a.toRule())\n\t\tbar.Increment()\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tm.snap = snap\n\treturn nil\n}\n\nfunc (m *dbMeta) DumpMeta(w io.Writer, root Ino, threads int, keepSecret, fast, skipTrash bool) (err error) {\n\tdefer func() {\n\t\tif p := recover(); p != nil {\n\t\t\tdebug.PrintStack()\n\t\t\tif e, ok := p.(error); ok {\n\t\t\t\terr = e\n\t\t\t} else {\n\t\t\t\terr = fmt.Errorf(\"DumpMeta error: %v\", p)\n\t\t\t}\n\t\t}\n\t}()\n\n\tprogress := utils.NewProgress(false)\n\tvar tree, trash *DumpedEntry\n\troot = m.checkRoot(root)\n\treturn m.roTxn(Background(), func(s *xorm.Session) error {\n\t\tif root == RootInode && fast {\n\t\t\tdefer func() { m.snap = nil }()\n\t\t\tbar := progress.AddCountBar(\"Snapshot keys\", 0)\n\t\t\tif err = m.makeSnap(s, bar); err != nil {\n\t\t\t\treturn fmt.Errorf(\"Fetch all metadata from DB: %s\", err)\n\t\t\t}\n\t\t\tbar.Done()\n\t\t\ttree = m.dumpEntryFast(root, TypeDirectory)\n\t\t\tif !skipTrash {\n\t\t\t\ttrash = m.dumpEntryFast(TrashInode, TypeDirectory)\n\t\t\t}\n\t\t} else {\n\t\t\ttree = &DumpedEntry{\n\t\t\t\tName: \"FSTree\",\n\t\t\t\tAttr: &DumpedAttr{\n\t\t\t\t\tInode: root,\n\t\t\t\t\tType: typeToString(TypeDirectory),\n\t\t\t\t},\n\t\t\t}\n\t\t\tif err = m.dumpEntry(s, root, TypeDirectory, tree, nil); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif root == RootInode && !skipTrash {\n\t\t\t\ttrash = &DumpedEntry{\n\t\t\t\t\tName: \"Trash\",\n\t\t\t\t\tAttr: &DumpedAttr{\n\t\t\t\t\t\tInode: TrashInode,\n\t\t\t\t\t\tType: typeToString(TypeDirectory),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\tif err = m.dumpEntry(s, TrashInode, TypeDirectory, trash, nil); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif tree == nil {\n\t\t\treturn errors.New(\"The entry of the root inode was not found\")\n\t\t}\n\t\ttree.Name = \"FSTree\"\n\n\t\tvar drows []delfile\n\t\t// the statement remembers the table of last Iterator\n\t\tif err := s.Table(&delfile{}).Find(&drows); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdels := make([]*DumpedDelFile, 0, len(drows))\n\t\tfor _, row := range drows {\n\t\t\tdels = append(dels, &DumpedDelFile{row.Inode, row.Length, row.Expire})\n\t\t}\n\t\tvar crows []counter\n\t\tif err = s.Find(&crows); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tcounters := &DumpedCounters{}\n\t\tfor _, row := range crows {\n\t\t\tswitch row.Name {\n\t\t\tcase \"usedSpace\":\n\t\t\t\tcounters.UsedSpace = row.Value\n\t\t\tcase \"totalInodes\":\n\t\t\t\tcounters.UsedInodes = row.Value\n\t\t\tcase \"nextInode\":\n\t\t\t\tcounters.NextInode = row.Value\n\t\t\tcase \"nextChunk\":\n\t\t\t\tcounters.NextChunk = row.Value\n\t\t\tcase \"nextSession\":\n\t\t\t\tcounters.NextSession = row.Value\n\t\t\tcase \"nextTrash\":\n\t\t\t\tcounters.NextTrash = row.Value\n\t\t\t}\n\t\t}\n\n\t\tvar srows []sustained\n\t\tif err := s.Find(&srows); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tss := make(map[uint64][]Ino)\n\t\tfor _, row := range srows {\n\t\t\tss[row.Sid] = append(ss[row.Sid], row.Inode)\n\t\t}\n\t\tsessions := make([]*DumpedSustained, 0, len(ss))\n\t\tfor k, v := range ss {\n\t\t\tsessions = append(sessions, &DumpedSustained{k, v})\n\t\t}\n\n\t\tvar qs []dirQuota\n\t\tif err := s.Find(&qs); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// todo Add user/group quota\n\t\tdumpedQuotas := make(map[Ino]*DumpedQuota, len(qs))\n\t\tfor _, q := range qs {\n\t\t\tdumpedQuotas[Ino(q.Inode)] = &DumpedQuota{q.MaxSpace, q.MaxInodes, 0, 0}\n\t\t}\n\n\t\tdm := DumpedMeta{\n\t\t\tSetting: *m.getFormat(),\n\t\t\tCounters: counters,\n\t\t\tSustained: sessions,\n\t\t\tDelFiles: dels,\n\t\t\tQuotas: dumpedQuotas,\n\t\t}\n\t\tif !keepSecret && dm.Setting.SecretKey != \"\" {\n\t\t\tdm.Setting.SecretKey = \"removed\"\n\t\t\tlogger.Warnf(\"Secret key is removed for the sake of safety\")\n\t\t}\n\t\tif !keepSecret && dm.Setting.SessionToken != \"\" {\n\t\t\tdm.Setting.SessionToken = \"removed\"\n\t\t\tlogger.Warnf(\"Session token is removed for the sake of safety\")\n\t\t}\n\t\tbw, err := dm.writeJsonWithOutTree(w)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tuseTotal := root == RootInode && !skipTrash\n\t\tbar := progress.AddCountBar(\"Dumped entries\", 1) // with root\n\t\tif useTotal {\n\t\t\ttotalBean := &counter{Name: \"totalInodes\"}\n\t\t\tif _, err := s.Get(totalBean); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tbar.SetTotal(totalBean.Value)\n\t\t}\n\t\tbar.Increment()\n\t\tif trash != nil {\n\t\t\ttrash.Name = \"Trash\"\n\t\t\tbar.IncrTotal(1)\n\t\t\tbar.Increment()\n\t\t}\n\t\tshowProgress := func(totalIncr, currentIncr int64) {\n\t\t\tif !useTotal {\n\t\t\t\tbar.IncrTotal(totalIncr)\n\t\t\t}\n\t\t\tbar.IncrInt64(currentIncr)\n\t\t}\n\t\tif m.snap != nil {\n\t\t\t_ = m.dumpDirFast(root, tree, bw, 1, showProgress)\n\t\t} else {\n\t\t\tshowProgress(int64(len(tree.Entries)), 0)\n\t\t\tif err = m.dumpDir(s, root, tree, bw, 1, threads, showProgress); err != nil {\n\t\t\t\tlogger.Errorf(\"dump dir %d failed: %s\", root, err)\n\t\t\t\treturn fmt.Errorf(\"dump dir %d failed\", root) // don't retry\n\t\t\t}\n\t\t}\n\t\tif trash != nil {\n\t\t\tif _, err = bw.WriteString(\",\"); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif m.snap != nil {\n\t\t\t\t_ = m.dumpDirFast(TrashInode, trash, bw, 1, showProgress)\n\t\t\t} else {\n\t\t\t\tshowProgress(int64(len(trash.Entries)), 0)\n\t\t\t\tif err = m.dumpDir(s, TrashInode, trash, bw, 1, threads, showProgress); err != nil {\n\t\t\t\t\tlogger.Errorf(\"dump trash failed: %s\", err)\n\t\t\t\t\treturn fmt.Errorf(\"dump trash failed\") // don't retry\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif _, err = bw.WriteString(\"\\n}\\n\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tprogress.Done()\n\t\treturn bw.Flush()\n\t})\n}\n\nfunc (m *dbMeta) loadEntry(e *DumpedEntry, chs []chan interface{}, aclMaxId *uint32) {\n\tinode := e.Attr.Inode\n\tattr := e.Attr\n\tn := &node{\n\t\tInode: inode,\n\t\tFlags: attr.Flags,\n\t\tType: typeFromString(attr.Type),\n\t\tMode: attr.Mode,\n\t\tUid: attr.Uid,\n\t\tGid: attr.Gid,\n\t\tNlink: attr.Nlink,\n\t\tRdev: attr.Rdev,\n\t\tParent: e.Parents[0],\n\t} // Length not set\n\tn.setAtime(attr.Atime*1e9 + int64(attr.Atimensec))\n\tn.setMtime(attr.Mtime*1e9 + int64(attr.Mtimensec))\n\tn.setCtime(attr.Ctime*1e9 + int64(attr.Ctimensec))\n\n\t// chs: node, edge, chunk, chunkRef, xattr, others\n\tif n.Type == TypeFile {\n\t\tn.Length = attr.Length\n\t\tfor _, c := range e.Chunks {\n\t\t\tif len(c.Slices) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tslices := make([]byte, 0, sliceBytes*len(c.Slices))\n\t\t\tfor _, s := range c.Slices {\n\t\t\t\tslices = append(slices, marshalSlice(s.Pos, s.Id, s.Size, s.Off, s.Len)...)\n\t\t\t}\n\t\t\tchs[2] <- &chunk{Inode: inode, Indx: c.Index, Slices: slices}\n\t\t}\n\t} else if n.Type == TypeDirectory {\n\t\tn.Length = 4 << 10\n\t\tstat := &dirStats{Inode: inode}\n\t\tfor name, c := range e.Entries {\n\t\t\tlength := uint64(0)\n\t\t\tif typeFromString(c.Attr.Type) == TypeFile {\n\t\t\t\tlength = c.Attr.Length\n\t\t\t}\n\t\t\tstat.DataLength += int64(length)\n\t\t\tstat.UsedSpace += align4K(length)\n\t\t\tstat.UsedInodes++\n\n\t\t\tchs[1] <- &edge{\n\t\t\t\tParent: inode,\n\t\t\t\tName: unescape(name),\n\t\t\t\tInode: c.Attr.Inode,\n\t\t\t\tType: typeFromString(c.Attr.Type),\n\t\t\t}\n\t\t}\n\t\tchs[5] <- stat\n\t} else if n.Type == TypeSymlink {\n\t\tsymL := unescape(e.Symlink)\n\t\tn.Length = uint64(len(symL))\n\t\tchs[5] <- &symlink{inode, symL}\n\t}\n\tfor _, x := range e.Xattrs {\n\t\tchs[4] <- &xattr{Inode: inode, Name: x.Name, Value: unescape(x.Value)}\n\t}\n\n\tn.AccessACLId = m.saveACL(loadACL(e.AccessACL), aclMaxId)\n\tn.DefaultACLId = m.saveACL(loadACL(e.DefaultACL), aclMaxId)\n\tchs[0] <- n\n}\n\nfunc (m *dbMeta) getTxnBatchNum() int {\n\tswitch m.Name() {\n\tcase \"sqlite3\":\n\t\treturn 999 / MaxFieldsCountOfTable\n\tcase \"mysql\":\n\t\treturn 65535 / MaxFieldsCountOfTable\n\tcase \"postgres\":\n\t\treturn 1000\n\tdefault:\n\t\treturn 1000\n\t}\n}\n\nfunc (m *dbMeta) checkAddr() error {\n\ttables, err := m.db.DBMetas()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(tables) > 0 {\n\t\taddr := m.addr\n\t\tif !strings.Contains(addr, \"://\") {\n\t\t\taddr = fmt.Sprintf(\"%s://%s\", m.Name(), addr)\n\t\t}\n\t\treturn fmt.Errorf(\"database %s is not empty\", addr)\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) LoadMeta(r io.Reader) error {\n\tif err := m.checkAddr(); err != nil {\n\t\treturn err\n\t}\n\tif err := m.syncAllTables(); err != nil {\n\t\treturn err\n\t}\n\n\tbatch := m.getTxnBatchNum()\n\tchs := make([]chan interface{}, 6) // node, edge, chunk, chunkRef, xattr, others\n\tinsert := func(index int, beans []interface{}) error {\n\t\treturn m.txn(func(s *xorm.Session) error {\n\t\t\tvar n int64\n\t\t\tvar err error\n\t\t\tif index == len(chs)-1 { // multiple tables\n\t\t\t\tn, err = s.Insert(beans...)\n\t\t\t} else { // one table only\n\t\t\t\tn, err = s.Insert(beans)\n\t\t\t}\n\t\t\tif err == nil && int(n) != len(beans) {\n\t\t\t\terr = fmt.Errorf(\"only %d records inserted\", n)\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t}\n\tvar wg sync.WaitGroup\n\tfor i := range chs {\n\t\tchs[i] = make(chan interface{}, batch*2)\n\t\twg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdefer wg.Done()\n\t\t\tbuffer := make([]interface{}, 0, batch)\n\t\t\tfor bean := range chs[i] {\n\t\t\t\tbuffer = append(buffer, bean)\n\t\t\t\tif len(buffer) >= batch {\n\t\t\t\t\tif err := insert(i, buffer); err != nil {\n\t\t\t\t\t\tlogger.Fatalf(\"Write %d beans in channel %d: %s\", len(buffer), i, err)\n\t\t\t\t\t}\n\t\t\t\t\tbuffer = buffer[:0]\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(buffer) > 0 {\n\t\t\t\tif err := insert(i, buffer); err != nil {\n\t\t\t\t\tlogger.Fatalf(\"Write %d beans in channel %d: %s\", len(buffer), i, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}(i)\n\t}\n\n\tvar aclMaxId uint32 = 0\n\tdm, counters, parents, refs, err := loadEntries(r,\n\t\tfunc(e *DumpedEntry) { m.loadEntry(e, chs, &aclMaxId) },\n\t\tfunc(ck *chunkKey) { chs[3] <- &sliceRef{ck.id, ck.size, 1} })\n\tif err != nil {\n\t\treturn err\n\t}\n\tm.loadDumpedQuotas(Background(), dm.Quotas)\n\tif err = m.loadDumpedACLs(Background()); err != nil {\n\t\treturn err\n\t}\n\n\tformat, _ := json.MarshalIndent(dm.Setting, \"\", \"\")\n\tchs[5] <- &setting{\"format\", string(format)}\n\tchs[5] <- &counter{usedSpace, counters.UsedSpace}\n\tchs[5] <- &counter{totalInodes, counters.UsedInodes}\n\tchs[5] <- &counter{\"nextInode\", counters.NextInode}\n\tchs[5] <- &counter{\"nextChunk\", counters.NextChunk}\n\tchs[5] <- &counter{\"nextSession\", counters.NextSession}\n\tchs[5] <- &counter{\"nextTrash\", counters.NextTrash}\n\tfor _, d := range dm.DelFiles {\n\t\tchs[5] <- &delfile{d.Inode, d.Length, d.Expire}\n\t}\n\tfor _, c := range chs {\n\t\tclose(c)\n\t}\n\twg.Wait()\n\n\t// update chunkRefs\n\tif err = m.txn(func(s *xorm.Session) error {\n\t\tfor k, v := range refs {\n\t\t\tif v > 1 {\n\t\t\t\tif _, e := s.Cols(\"refs\").Update(&sliceRef{Refs: int(v)}, &sliceRef{Id: k.id}); e != nil {\n\t\t\t\t\treturn e\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\t// update nlinks and parents for hardlinks\n\treturn m.txn(func(s *xorm.Session) error {\n\t\tfor i, ps := range parents {\n\t\t\tif len(ps) > 1 {\n\t\t\t\t_, err := s.Cols(\"nlink\", \"parent\").Update(&node{Nlink: uint32(len(ps))}, &node{Inode: i})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\ntype checkDupError func(error) bool\n\nvar dupErrorCheckers []checkDupError\n\nfunc isDuplicateEntryErr(err error) bool {\n\tfor _, check := range dupErrorCheckers {\n\t\tif check(err) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (m *dbMeta) validateCloneTarget(ctx Context, s xorm.Interface, ino Ino) (node, error) {\n\tpn := node{Inode: ino}\n\tok, err := s.Get(&pn)\n\tif err != nil {\n\t\treturn pn, err\n\t}\n\tif !ok {\n\t\treturn pn, syscall.ENOENT\n\t}\n\tif pn.Type != TypeDirectory {\n\t\treturn pn, syscall.ENOTDIR\n\t}\n\tif (pn.Flags & FlagImmutable) != 0 {\n\t\treturn pn, syscall.EPERM\n\t}\n\tvar pattr Attr\n\tm.parseAttr(&pn, &pattr)\n\tif st := m.Access(ctx, ino, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\treturn pn, st\n\t}\n\treturn pn, nil\n}\n\nfunc (m *dbMeta) doCloneEntry(ctx Context, srcIno Ino, parent Ino, name string, ino Ino, attr *Attr, cmode uint8, cumask uint16, top bool) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tn := node{Inode: srcIno}\n\t\tok, err := s.ForUpdate().Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tn.Inode = ino\n\t\tn.Parent = parent\n\t\tnow := time.Now()\n\n\t\tm.parseAttr(&n, attr)\n\t\tif eno := m.Access(ctx, srcIno, MODE_MASK_R, attr); eno != 0 {\n\t\t\treturn eno\n\t\t}\n\n\t\tif cmode&CLONE_MODE_PRESERVE_ATTR == 0 {\n\t\t\tn.Uid = ctx.Uid()\n\t\t\tn.Gid = ctx.Gid()\n\t\t\tn.Mode &= ^cumask\n\t\t\tns := now.UnixNano()\n\t\t\tn.setAtime(ns)\n\t\t\tn.setMtime(ns)\n\t\t\tn.setCtime(ns)\n\t\t}\n\t\t// TODO: preserve hardlink\n\t\tif n.Type == TypeFile && n.Nlink > 1 {\n\t\t\tn.Nlink = 1\n\t\t}\n\n\t\tif top {\n\t\t\tpn, err := m.validateCloneTarget(ctx, s, parent)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n.Type != TypeDirectory {\n\t\t\t\tnow := time.Now().UnixNano()\n\t\t\t\tpn.setMtime(now)\n\t\t\t\tpn.setCtime(now)\n\t\t\t\tif _, err = s.Cols(\"nlink\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&pn, &node{Inode: parent}); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif top && n.Type == TypeDirectory {\n\t\t\terr = mustInsert(s, &n, &detachedNode{Inode: ino, Added: time.Now().Unix()})\n\t\t} else {\n\t\t\terr = mustInsert(s, &n, &edge{Parent: parent, Name: []byte(name), Inode: ino, Type: n.Type})\n\t\t\tif isDuplicateEntryErr(err) {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvar xs []xattr\n\t\tif err = s.Where(\"inode = ?\", srcIno).Find(&xs, &xattr{Inode: srcIno}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(xs) > 0 {\n\t\t\tfor i := range xs {\n\t\t\t\txs[i].Id = 0\n\t\t\t\txs[i].Inode = ino\n\t\t\t}\n\t\t\tif err := mustInsert(s, &xs); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tswitch n.Type {\n\t\tcase TypeDirectory:\n\t\t\tvar st = dirStats{Inode: srcIno}\n\t\t\tif exist, err := s.Get(&st); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if exist {\n\t\t\t\tst.Inode = ino\n\t\t\t\tif err := mustInsert(s, &st); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\tcase TypeFile:\n\t\t\t// copy chunks\n\t\t\tif n.Length != 0 {\n\t\t\t\tvar cs []chunk\n\t\t\t\tif err = s.Where(\"inode = ?\", srcIno).ForUpdate().Find(&cs); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tfor i := range cs {\n\t\t\t\t\tcs[i].Id = 0\n\t\t\t\t\tcs[i].Inode = ino\n\t\t\t\t}\n\t\t\t\tif len(cs) != 0 {\n\t\t\t\t\tif err := mustInsert(s, cs); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// TODO: batch?\n\t\t\t\tfor _, c := range cs {\n\t\t\t\t\tfor _, sli := range readSliceBuf(c.Slices) {\n\t\t\t\t\t\tif sli.id > 0 {\n\t\t\t\t\t\t\tif _, err := s.Exec(m.sqlConv(\"update chunk_ref set refs=refs+1 where chunkid = ? AND size = ?\"), sli.id, sli.size); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase TypeSymlink:\n\t\t\tsym := symlink{Inode: srcIno}\n\t\t\tif exists, err := s.Get(&sym); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if !exists {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t\tsym.Inode = ino\n\t\t\treturn mustInsert(s, &sym)\n\t\t}\n\t\treturn nil\n\t}, srcIno))\n}\n\nfunc (m *dbMeta) doBatchClone(ctx Context, srcParent Ino, dstParent Ino, entries []*Entry, cmode uint8, cumask uint16, result *batchCloneResult) syscall.Errno {\n\tif len(entries) == 0 {\n\t\treturn 0\n\t}\n\tif _, err := m.validateCloneTarget(ctx, m.db, dstParent); err != nil {\n\t\treturn errno(err)\n\t}\n\n\ttype cloneInfo struct {\n\t\tsrcIno Ino\n\t\tdstIno Ino\n\t\tname []byte\n\t\tdstNode node\n\t}\n\n\tcloneInfos := make([]*cloneInfo, len(entries))\n\tsrcInodes := make([]Ino, 0, len(entries))\n\tsrcInodeSet := make(map[Ino]struct{}, len(entries))\n\tfor i, e := range entries {\n\t\tdstIno, err := m.nextInode()\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\t\tcloneInfos[i] = &cloneInfo{srcIno: e.Inode, dstIno: dstIno, name: e.Name}\n\t\tif _, exists := srcInodeSet[e.Inode]; !exists {\n\t\t\tsrcInodeSet[e.Inode] = struct{}{}\n\t\t\tsrcInodes = append(srcInodes, e.Inode)\n\t\t}\n\t}\n\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tnowNano := time.Now().UnixNano()\n\t\t*result = batchCloneResult{deltas: make(ugQuotaDeltas)}\n\n\t\tpn, err := m.validateCloneTarget(ctx, s, dstParent)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tvar srcNodes []node\n\t\tif err := s.In(\"inode\", srcInodes).ForUpdate().Find(&srcNodes); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tsrcNodeMap := make(map[Ino]*node, len(srcNodes))\n\t\tfor i := range srcNodes {\n\t\t\tsrcNodeMap[srcNodes[i].Inode] = &srcNodes[i]\n\t\t}\n\n\t\tnodesIns := make([]interface{}, 0, len(entries))\n\t\tedgesIns := make([]interface{}, 0, len(entries))\n\t\tfileInodes := make([]Ino, 0)\n\t\tsymlinkInodes := make([]Ino, 0)\n\t\tsymlinkClones := make([]*cloneInfo, 0)\n\t\tfor _, info := range cloneInfos {\n\t\t\tsn, ok := srcNodeMap[info.srcIno]\n\t\t\tif !ok {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t\tif sn.Type == TypeDirectory {\n\t\t\t\treturn syscall.EINVAL\n\t\t\t}\n\t\t\tvar attr Attr\n\t\t\tm.parseAttr(sn, &attr)\n\t\t\tif st := m.Access(ctx, info.srcIno, MODE_MASK_R, &attr); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\n\t\t\tinfo.dstNode = *sn\n\t\t\tinfo.dstNode.Inode = info.dstIno\n\t\t\tinfo.dstNode.Parent = dstParent\n\t\t\tif cmode&CLONE_MODE_PRESERVE_ATTR == 0 {\n\t\t\t\tinfo.dstNode.Uid = ctx.Uid()\n\t\t\t\tinfo.dstNode.Gid = ctx.Gid()\n\t\t\t\tinfo.dstNode.Mode &= ^cumask\n\t\t\t\tinfo.dstNode.setAtime(nowNano)\n\t\t\t\tinfo.dstNode.setMtime(nowNano)\n\t\t\t\tinfo.dstNode.setCtime(nowNano)\n\t\t\t}\n\t\t\tif sn.Type == TypeFile && sn.Nlink > 1 {\n\t\t\t\tinfo.dstNode.Nlink = 1\n\t\t\t}\n\n\t\t\tnodesIns = append(nodesIns, &info.dstNode)\n\t\t\tedgesIns = append(edgesIns, &edge{\n\t\t\t\tParent: dstParent,\n\t\t\t\tName: info.name,\n\t\t\t\tInode: info.dstIno,\n\t\t\t\tType: sn.Type,\n\t\t\t})\n\n\t\t\tswitch sn.Type {\n\t\t\tcase TypeFile:\n\t\t\t\tif sn.Length > 0 {\n\t\t\t\t\tfileInodes = append(fileInodes, info.srcIno)\n\t\t\t\t}\n\t\t\tcase TypeSymlink:\n\t\t\t\tsymlinkInodes = append(symlinkInodes, info.srcIno)\n\t\t\t\tsymlinkClones = append(symlinkClones, info)\n\t\t\t}\n\n\t\t\tentrySpace := align4K(sn.Length)\n\t\t\tresult.length += int64(sn.Length)\n\t\t\tresult.space += entrySpace\n\t\t\tresult.inodes++\n\t\t\tresult.deltas.add(&ugQuotaDelta{\n\t\t\t\tUid: info.dstNode.Uid,\n\t\t\t\tGid: info.dstNode.Gid,\n\t\t\t\tSpace: entrySpace,\n\t\t\t\tInodes: 1,\n\t\t\t})\n\t\t}\n\n\t\tif err := mustInsert(s, nodesIns...); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := mustInsert(s, edgesIns...); err != nil {\n\t\t\tif isDuplicateEntryErr(err) {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tchunkRefCounts := make(map[uint64]int)\n\t\tif len(fileInodes) > 0 {\n\t\t\tvar srcChunks []chunk\n\t\t\tif err := s.In(\"inode\", fileInodes).ForUpdate().Find(&srcChunks); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tchunksByInode := make(map[Ino][]chunk, len(fileInodes))\n\t\t\tfor _, c := range srcChunks {\n\t\t\t\tchunksByInode[c.Inode] = append(chunksByInode[c.Inode], c)\n\t\t\t}\n\t\t\tchunksIns := make([]interface{}, 0, len(srcChunks))\n\t\t\tfor i := range cloneInfos {\n\t\t\t\tfor _, c := range chunksByInode[cloneInfos[i].srcIno] {\n\t\t\t\t\tchunksIns = append(chunksIns, &chunk{\n\t\t\t\t\t\tInode: cloneInfos[i].dstIno, Indx: c.Indx, Slices: c.Slices,\n\t\t\t\t\t})\n\t\t\t\t\tfor _, sli := range readSliceBuf(c.Slices) {\n\t\t\t\t\t\tif sli.id > 0 {\n\t\t\t\t\t\t\tchunkRefCounts[sli.id]++\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err := mustInsert(s, chunksIns...); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif len(symlinkInodes) > 0 {\n\t\t\tvar srcSymlinks []symlink\n\t\t\tif err := s.In(\"inode\", symlinkInodes).Find(&srcSymlinks); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsymlinkMap := make(map[Ino][]byte, len(srcSymlinks))\n\t\t\tfor _, sl := range srcSymlinks {\n\t\t\t\tsymlinkMap[sl.Inode] = sl.Target\n\t\t\t}\n\t\t\tsymlinksIns := make([]interface{}, 0, len(symlinkClones))\n\t\t\tfor i := range symlinkClones {\n\t\t\t\tif target, ok := symlinkMap[symlinkClones[i].srcIno]; ok {\n\t\t\t\t\tsymlinksIns = append(symlinksIns, &symlink{Inode: symlinkClones[i].dstIno, Target: target})\n\t\t\t\t} else {\n\t\t\t\t\treturn syscall.ENOENT\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err := mustInsert(s, symlinksIns...); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tvar srcXattrs []xattr\n\t\tif err := s.In(\"inode\", srcInodes).Find(&srcXattrs); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(srcXattrs) > 0 {\n\t\t\txattrsByInode := make(map[Ino][]xattr)\n\t\t\tfor _, x := range srcXattrs {\n\t\t\t\txattrsByInode[x.Inode] = append(xattrsByInode[x.Inode], x)\n\t\t\t}\n\t\t\txattrsIns := make([]interface{}, 0, len(srcXattrs))\n\t\t\tfor i := range cloneInfos {\n\t\t\t\tfor _, x := range xattrsByInode[cloneInfos[i].srcIno] {\n\t\t\t\t\txattrsIns = append(xattrsIns, &xattr{Inode: cloneInfos[i].dstIno, Name: x.Name, Value: x.Value})\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err := mustInsert(s, xattrsIns...); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif err := func() error {\n\t\t\tif len(chunkRefCounts) == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tchunkIds := make([]uint64, 0, len(chunkRefCounts))\n\t\t\tfor id := range chunkRefCounts {\n\t\t\t\tchunkIds = append(chunkIds, id)\n\t\t\t}\n\t\t\tslices.Sort(chunkIds)\n\n\t\t\tbatchSize := m.getTxnBatchNum()\n\t\t\tfor start := 0; start < len(chunkIds); start += batchSize {\n\t\t\t\tend := min(start+batchSize, len(chunkIds))\n\t\t\t\tbatch := chunkIds[start:end]\n\t\t\t\tvar sb strings.Builder\n\t\t\t\targs := make([]interface{}, 0, len(batch)*3)\n\t\t\t\tfmt.Fprintf(&sb, \"UPDATE %schunk_ref SET refs = refs + CASE \", m.tablePrefix)\n\t\t\t\tfor _, id := range batch {\n\t\t\t\t\tsb.WriteString(\"WHEN chunkid = ? THEN ? \")\n\t\t\t\t\targs = append(args, id, chunkRefCounts[id])\n\t\t\t\t}\n\t\t\t\tsb.WriteString(\"ELSE 0 END WHERE chunkid IN (\")\n\t\t\t\tfor i, id := range batch {\n\t\t\t\t\tif i > 0 {\n\t\t\t\t\t\tsb.WriteString(\",\")\n\t\t\t\t\t}\n\t\t\t\t\tsb.WriteString(\"?\")\n\t\t\t\t\targs = append(args, id)\n\t\t\t\t}\n\t\t\t\tsb.WriteString(\")\")\n\t\t\t\tif _, err := s.Exec(append([]interface{}{sb.String()}, args...)...); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}(); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif cmode&CLONE_MODE_PRESERVE_ATTR == 0 {\n\t\t\tpn.setMtime(nowNano)\n\t\t\tpn.setCtime(nowNano)\n\t\t\tif _, err := s.Cols(\"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").\n\t\t\t\tUpdate(&pn, &node{Inode: dstParent}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\treturn 0\n}\n\nfunc (m *dbMeta) doFindDetachedNodes(t time.Time) []Ino {\n\tvar inodes []Ino\n\terr := m.roTxn(Background(), func(s *xorm.Session) error {\n\t\tvar nodes []detachedNode\n\t\terr := s.Where(\"added < ?\", t.Unix()).Find(&nodes)\n\t\tfor _, n := range nodes {\n\t\t\tinodes = append(inodes, n.Inode)\n\t\t}\n\t\treturn err\n\t})\n\tif err != nil {\n\t\tlogger.Errorf(\"Scan detached nodes error: %s\", err)\n\t}\n\treturn inodes\n}\n\nfunc (m *dbMeta) doCleanupDetachedNode(ctx Context, ino Ino) syscall.Errno {\n\texist, err := m.db.Exist(&node{Inode: ino})\n\tif err != nil || !exist {\n\t\treturn errno(err)\n\t}\n\trmConcurrent := make(chan int, 10)\n\tif eno := m.emptyDir(ctx, ino, true, nil, rmConcurrent); eno != 0 {\n\t\treturn eno\n\t}\n\tm.updateStats(-align4K(0), -1)\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tif _, err := s.Delete(&node{Inode: ino}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := s.Delete(&dirStats{Inode: ino}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err = s.Delete(&xattr{Inode: ino}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err = s.Delete(&detachedNode{Inode: ino})\n\t\treturn err\n\t}, ino))\n}\n\nfunc (m *dbMeta) doAttachDirNode(ctx Context, parent Ino, inode Ino, name string) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\t// must lock parent node first to avoid deadlock\n\t\tvar n = node{Inode: parent}\n\t\tok, err := s.ForUpdate().Get(&n)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif n.Type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif n.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif (n.Flags & FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tn.Nlink++\n\t\tnow := time.Now().UnixNano()\n\t\tn.setMtime(now)\n\t\tn.setCtime(now)\n\t\tif _, err = s.Cols(\"nlink\", \"mtime\", \"ctime\", \"mtimensec\", \"ctimensec\").Update(&n, &node{Inode: parent}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := mustInsert(s, &edge{Parent: parent, Name: []byte(name), Inode: inode, Type: TypeDirectory}); err != nil {\n\t\t\tif isDuplicateEntryErr(err) {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\t_, err = s.Delete(&detachedNode{Inode: inode})\n\t\treturn err\n\t}, parent))\n}\n\nfunc (m *dbMeta) doTouchAtime(ctx Context, inode Ino, attr *Attr, now time.Time) (bool, error) {\n\tvar updated bool\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tcurNode := node{Inode: inode}\n\t\tok, err := s.ForUpdate().Get(&curNode)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(&curNode, attr)\n\t\tif !m.atimeNeedsUpdate(attr, now) {\n\t\t\treturn nil\n\t\t}\n\t\tcurNode.setAtime(now.UnixNano())\n\t\tattr.Atime = curNode.Atime / 1e6\n\t\tattr.Atimensec = uint32(curNode.Atime%1e6*1000) + uint32(curNode.Atimensec)\n\t\tif _, err = s.Cols(\"atime\", \"atimensec\").Update(&curNode, &node{Inode: inode}); err == nil {\n\t\t\tupdated = true\n\t\t}\n\t\treturn err\n\t}, inode)\n\treturn updated, err\n}\n\nfunc (m *dbMeta) insertACL(s *xorm.Session, rule *aclAPI.Rule) (uint32, error) {\n\tif rule == nil {\n\t\treturn aclAPI.None, nil\n\t}\n\tif err := m.tryLoadMissACLs(s); err != nil {\n\t\tlogger.Warnf(\"Mknode: load miss acls error: %s\", err)\n\t}\n\tvar aclId uint32\n\tif aclId = m.aclCache.GetId(rule); aclId == aclAPI.None {\n\t\t// TODO conflicts from multiple clients are rare and result in only minor duplicates, thus not addressed for now.\n\t\tval := newSQLAcl(rule)\n\t\tif _, err := s.Insert(val); err != nil {\n\t\t\treturn aclAPI.None, err\n\t\t}\n\t\taclId = val.Id\n\t\tm.aclCache.Put(aclId, rule)\n\t}\n\treturn aclId, nil\n}\n\nfunc (m *dbMeta) tryLoadMissACLs(s *xorm.Session) error {\n\tmissIds := m.aclCache.GetMissIds()\n\tif len(missIds) > 0 {\n\t\tvar acls []acl\n\t\tif err := s.In(\"id\", missIds).Find(&acls); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tgot := make(map[uint32]struct{}, len(acls))\n\t\tfor _, data := range acls {\n\t\t\tgot[data.Id] = struct{}{}\n\t\t\tm.aclCache.Put(data.Id, data.toRule())\n\t\t}\n\t\tif len(acls) < len(missIds) {\n\t\t\tfor _, id := range missIds {\n\t\t\t\tif _, ok := got[id]; !ok {\n\t\t\t\t\tm.aclCache.Put(id, aclAPI.EmptyRule())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) getACL(s *xorm.Session, id uint32) (*aclAPI.Rule, error) {\n\tif id == aclAPI.None {\n\t\treturn nil, nil\n\t}\n\tif cRule := m.aclCache.Get(id); cRule != nil {\n\t\treturn cRule, nil\n\t}\n\n\tvar aclVal = &acl{Id: id}\n\tif ok, err := s.Get(aclVal); err != nil {\n\t\treturn nil, err\n\t} else if !ok {\n\t\treturn nil, syscall.EIO\n\t}\n\n\tr := aclVal.toRule()\n\tm.aclCache.Put(id, r)\n\treturn r, nil\n}\n\nfunc (m *dbMeta) doSetFacl(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\tattr := &Attr{}\n\t\tn := &node{Inode: ino}\n\t\tif ok, err := s.ForUpdate().Get(n); err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(n, attr)\n\n\t\tif ctx.Uid() != 0 && ctx.Uid() != attr.Uid {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tif attr.Flags&FlagImmutable != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\toriACL, oriMode := getAttrACLId(attr, aclType), attr.Mode\n\n\t\t// https://github.com/torvalds/linux/blob/480e035fc4c714fb5536e64ab9db04fedc89e910/fs/fuse/acl.c#L143-L151\n\t\t// TODO: check linux capabilities\n\t\tif ctx.Uid() != 0 && !inGroup(ctx, attr.Gid) {\n\t\t\t// clear sgid\n\t\t\tattr.Mode &= 05777\n\t\t}\n\n\t\tif rule.IsEmpty() {\n\t\t\t// remove acl\n\t\t\tsetAttrACLId(attr, aclType, aclAPI.None)\n\t\t} else if rule.IsMinimal() && aclType == aclAPI.TypeAccess {\n\t\t\t// remove acl\n\t\t\tsetAttrACLId(attr, aclType, aclAPI.None)\n\t\t\t// set mode\n\t\t\tattr.Mode &= 07000\n\t\t\tattr.Mode |= ((rule.Owner & 7) << 6) | ((rule.Group & 7) << 3) | (rule.Other & 7)\n\t\t} else {\n\t\t\t// set acl\n\t\t\trule.InheritPerms(attr.Mode)\n\t\t\taclId, err := m.insertACL(s, rule)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsetAttrACLId(attr, aclType, aclId)\n\n\t\t\t// set mode\n\t\t\tif aclType == aclAPI.TypeAccess {\n\t\t\t\tattr.Mode &= 07000\n\t\t\t\tattr.Mode |= ((rule.Owner & 7) << 6) | ((rule.Mask & 7) << 3) | (rule.Other & 7)\n\t\t\t}\n\t\t}\n\n\t\t// update attr\n\t\tvar updateCols []string\n\t\tif oriACL != getAttrACLId(attr, aclType) {\n\t\t\tupdateCols = append(updateCols, getACLIdColName(aclType))\n\t\t}\n\t\tif oriMode != attr.Mode {\n\t\t\tupdateCols = append(updateCols, \"mode\")\n\t\t}\n\t\tif len(updateCols) > 0 {\n\t\t\tupdateCols = append(updateCols, \"ctime\", \"ctimensec\")\n\n\t\t\tvar dirtyNode node\n\t\t\tm.parseNode(attr, &dirtyNode)\n\t\t\tdirtyNode.setCtime(time.Now().UnixNano())\n\t\t\t_, err := s.Cols(updateCols...).Update(&dirtyNode, &node{Inode: ino})\n\t\t\treturn err\n\t\t}\n\n\t\treturn nil\n\t}, ino))\n}\n\nfunc (m *dbMeta) doGetFacl(ctx Context, ino Ino, aclType uint8, aclId uint32, rule *aclAPI.Rule) syscall.Errno {\n\treturn errno(m.roTxn(ctx, func(s *xorm.Session) error {\n\t\tif aclId == aclAPI.None {\n\t\t\tattr := &Attr{}\n\t\t\tn := &node{Inode: ino}\n\t\t\tif ok, err := s.Get(n); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if !ok {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t\tm.parseAttr(n, attr)\n\t\t\tm.of.Update(ino, attr)\n\t\t\taclId = getAttrACLId(attr, aclType)\n\t\t}\n\n\t\ta, err := m.getACL(s, aclId)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif a == nil {\n\t\t\treturn ENOATTR\n\t\t}\n\t\t*rule = *a\n\t\treturn nil\n\t}))\n}\n\nfunc (m *dbMeta) loadDumpedACLs(ctx Context) error {\n\tid2Rule := m.aclCache.GetAll()\n\tif len(id2Rule) == 0 {\n\t\treturn nil\n\t}\n\n\tacls := make([]*acl, 0, len(id2Rule))\n\tfor id, rule := range id2Rule {\n\t\taclV := newSQLAcl(rule)\n\t\taclV.Id = id\n\t\tacls = append(acls, aclV)\n\t}\n\n\treturn m.txn(func(s *xorm.Session) error {\n\t\tn, err := s.Insert(acls)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif int(n) != len(acls) {\n\t\t\treturn fmt.Errorf(\"only %d acls inserted, expected %d\", n, len(acls))\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *dbMeta) doStoreToken(ctx Context, token []byte) (id uint32, st syscall.Errno) {\n\terr := m.txn(func(s *xorm.Session) error {\n\t\tt := &delegationToken{Token: token}\n\t\t_, err := s.Insert(t)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tid = t.Id\n\t\treturn nil\n\t})\n\treturn id, errno(err)\n}\n\nfunc (m *dbMeta) doUpdateToken(ctx Context, id uint32, token []byte) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\t_, err := s.Cols(\"token\").Update(&delegationToken{Id: id, Token: token}, &delegationToken{Id: id})\n\t\treturn err\n\t}))\n}\n\nfunc (m *dbMeta) doLoadToken(ctx Context, id uint32) (token []byte, st syscall.Errno) {\n\terr := m.simpleTxn(ctx, func(s *xorm.Session) error {\n\t\tt := &delegationToken{Id: id}\n\t\tok, err := s.Get(t)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\ttoken = t.Token\n\t\treturn nil\n\t})\n\treturn token, errno(err)\n}\n\nfunc (m *dbMeta) doDeleteTokens(ctx Context, ids []uint32) syscall.Errno {\n\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\t_, err := s.In(\"id\", ids).Delete(&delegationToken{})\n\t\treturn err\n\t}))\n}\n\nfunc (m *dbMeta) doListTokens(ctx Context) (tokens map[uint32][]byte, st syscall.Errno) {\n\terr := m.roTxn(ctx, func(s *xorm.Session) error {\n\t\tvar ts []delegationToken\n\t\terr := s.Find(&ts)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttokens = make(map[uint32][]byte, len(ts))\n\t\tfor _, t := range ts {\n\t\t\ttokens[t.Id] = t.Token\n\t\t}\n\t\treturn nil\n\t})\n\treturn tokens, errno(err)\n}\n\ntype dbDirHandler struct {\n\tdirHandler\n}\n\nfunc (m *dbMeta) newDirHandler(inode Ino, plus bool, entries []*Entry) DirHandler {\n\th := &dbDirHandler{\n\t\tdirHandler: dirHandler{\n\t\t\tinode: inode,\n\t\t\tplus: plus,\n\t\t\tinitEntries: entries,\n\t\t\tfetcher: m.getDirFetcher(),\n\t\t\tbatchNum: DirBatchNum[\"db\"],\n\t\t},\n\t}\n\th.batch, _ = h.fetch(Background(), 0)\n\treturn h\n}\n\nfunc (m *dbMeta) getDirFetcher() dirFetcher {\n\treturn func(ctx Context, inode Ino, cursor interface{}, offset, limit int, plus bool) (interface{}, []*Entry, error) {\n\t\tentries := make([]*Entry, 0, limit)\n\t\terr := m.roTxn(Background(), func(s *xorm.Session) error {\n\t\t\tvar name []byte\n\t\t\tif cursor != nil {\n\t\t\t\tname = cursor.([]byte)\n\t\t\t} else {\n\t\t\t\tif offset > 0 {\n\t\t\t\t\tvar edges []edge\n\t\t\t\t\tif err := s.Table(&edge{}).Where(\"parent = ?\", inode).OrderBy(\"name\").Limit(1, offset-1).Find(&edges); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif len(edges) < 1 {\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\tname = edges[0].Name\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar ids []int64\n\t\t\tvar err error\n\t\t\t// sorted by (parent, name) index\n\t\t\tif name == nil {\n\t\t\t\terr = s.Table(&edge{}).Cols(\"id\").Where(\"parent = ?\", inode).OrderBy(\"name\").Limit(limit).Find(&ids)\n\t\t\t} else {\n\t\t\t\terr = s.Table(&edge{}).Cols(\"id\").Where(\"parent = ? and name > ?\", inode, name).OrderBy(\"name\").Limit(limit).Find(&ids)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\ts = s.Table(&edge{}).In(m.sqlConv(\"edge.id\"), ids).OrderBy(m.sqlConv(\"edge.name\")) // need to sorted by name, otherwise the cursor will be invalid\n\t\t\tif plus {\n\t\t\t\ts = s.Join(\"INNER\", &node{}, m.sqlConv(\"edge.inode=node.inode\")).Cols(m.sqlConv(\"edge.name\"), m.sqlConv(\"node.*\"))\n\t\t\t} else {\n\t\t\t\ts = s.Cols(m.sqlConv(\"edge.id\"), m.sqlConv(\"edge.name\"), m.sqlConv(\"edge.type\"))\n\t\t\t}\n\t\t\tvar nodes []namedNode\n\t\t\tif err := s.Find(&nodes); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tfor _, n := range nodes {\n\t\t\t\tif len(n.Name) == 0 {\n\t\t\t\t\tlogger.Errorf(\"Corrupt entry with empty name: inode %d parent %d\", n.Inode, inode)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tentry := &Entry{\n\t\t\t\t\tInode: n.Inode,\n\t\t\t\t\tName: n.Name,\n\t\t\t\t\tAttr: &Attr{},\n\t\t\t\t}\n\t\t\t\tif plus {\n\t\t\t\t\tm.parseAttr(&n.node, entry.Attr)\n\t\t\t\t\tm.of.Update(n.Inode, entry.Attr)\n\t\t\t\t} else {\n\t\t\t\t\tentry.Attr.Typ = n.Type\n\t\t\t\t}\n\t\t\t\tentries = append(entries, entry)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tif len(entries) == 0 {\n\t\t\treturn nil, nil, nil\n\t\t}\n\t\treturn entries[len(entries)-1].Name, entries, nil\n\t}\n}\n" }, { "path": "pkg/meta/sql_bak.go", "content": "//go:build !nosqlite || !nomysql || !nopg\n// +build !nosqlite !nomysql !nopg\n\n/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"database/sql\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/meta/pb\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/sync/errgroup\"\n\t\"google.golang.org/protobuf/proto\"\n\t\"xorm.io/xorm\"\n)\n\nvar (\n\tsqlDumpBatchSize = 100000\n)\n\nfunc (m *dbMeta) dump(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar dumps = []func(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error{\n\t\tm.dumpFormat,\n\t\tm.dumpCounters,\n\t\tm.dumpNodes,\n\t\tm.dumpChunks,\n\t\tm.dumpEdges,\n\t\tm.dumpSymlinks,\n\t\tm.dumpSustained,\n\t\tm.dumpDelFiles,\n\t\tm.dumpSliceRef,\n\t\tm.dumpACL,\n\t\tm.dumpXattr,\n\t\tm.dumpQuota,\n\t\tm.dumpDirStat,\n\t}\n\n\tctx = ctx.WithValue(txMaxRetryKey{}, 3)\n\tif opt.Threads == 1 {\n\t\t// use same txn for all dumps\n\t\tsess := m.db.NewSession()\n\t\tdefer sess.Close()\n\n\t\topt := sql.TxOptions{\n\t\t\tIsolation: sql.LevelRepeatableRead,\n\t\t\tReadOnly: true,\n\t\t}\n\t\terr := sess.BeginTx(&opt)\n\t\tif err != nil && (strings.Contains(err.Error(), \"READ\") || strings.Contains(err.Error(), \"driver does not support read-only transactions\")) {\n\t\t\tlogger.Warnf(\"the database does not support read-only transaction\")\n\t\t\topt = sql.TxOptions{} // use default level\n\t\t\tif err = sess.BeginTx(&opt); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tdefer sess.Rollback() //nolint:errcheck\n\t\tctx = ctx.WithValue(txSessionKey{}, sess)\n\t} else {\n\t\tlogger.Warnf(\"dump database with %d threads, please make sure that it's readonly, \"+\n\t\t\t\"otherwise the dumped metadata will be inconsistent\", opt.Threads)\n\t}\n\tfor _, f := range dumps {\n\t\terr := f(ctx, opt, ch)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) execTxn(ctx context.Context, f func(s *xorm.Session) error) error {\n\tif val := ctx.Value(txSessionKey{}); val != nil {\n\t\treturn f(val.(*xorm.Session))\n\t}\n\treturn m.roTxn(ctx, f)\n}\n\nfunc sqlQueryBatch(ctx Context, opt *DumpOption, maxId uint64, query func(ctx context.Context, start, end uint64) (int, error)) error {\n\teg, egCtx := errgroup.WithContext(ctx)\n\teg.SetLimit(opt.Threads)\n\n\tsum := int64(0)\n\tbatch := uint64(sqlDumpBatchSize)\n\tfor id := uint64(0); id <= maxId; id += batch {\n\t\tstartId := id\n\t\teg.Go(func() error {\n\t\t\tn, err := query(egCtx, startId, startId+batch)\n\t\t\tatomic.AddInt64(&sum, int64(n))\n\t\t\treturn err\n\t\t})\n\t}\n\tlogger.Debugf(\"dump %d rows\", sum)\n\treturn eg.Wait()\n}\n\nfunc (m *dbMeta) dumpNodes(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tpool := sync.Pool{New: func() interface{} { return &pb.Node{} }}\n\trelease := func(p proto.Message) {\n\t\tfor _, s := range p.(*pb.Batch).Nodes {\n\t\t\tpool.Put(s)\n\t\t}\n\t}\n\n\tvar rows []node\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Where(\"inode >= ?\", TrashInode).Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tnodes := make([]*pb.Node, 0, len(rows))\n\tvar attr Attr\n\tfor _, n := range rows {\n\t\tpn := pool.Get().(*pb.Node)\n\t\tpn.Inode = uint64(n.Inode)\n\t\tm.parseAttr(&n, &attr)\n\t\tpn.Data = m.marshal(&attr)\n\t\tnodes = append(nodes, pn)\n\t}\n\tif err := dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Nodes: nodes}, release}); err != nil {\n\t\treturn errors.Wrap(err, \"dump trash nodes\")\n\t}\n\n\tvar maxInode uint64\n\terr := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\tvar row node\n\t\tok, err := s.Select(\"max(inode) as inode\").Where(\"inode < ?\", TrashInode).Get(&row)\n\t\tif ok {\n\t\t\tmaxInode = uint64(row.Inode)\n\t\t}\n\t\treturn err\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"max inode\")\n\t}\n\n\treturn sqlQueryBatch(ctx, opt, maxInode, func(ctx context.Context, start, end uint64) (int, error) {\n\t\tvar rows []node\n\t\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\t\treturn s.Where(\"inode >= ? AND inode < ?\", start, end).Find(&rows)\n\t\t}); err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\tnodes := make([]*pb.Node, 0, len(rows))\n\t\tvar attr Attr\n\t\tfor _, n := range rows {\n\t\t\tpn := pool.Get().(*pb.Node)\n\t\t\tpn.Inode = uint64(n.Inode)\n\t\t\tm.parseAttr(&n, &attr)\n\t\t\tpn.Data = m.marshal(&attr)\n\t\t\tnodes = append(nodes, pn)\n\t\t}\n\t\treturn len(rows), dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Nodes: nodes}, release})\n\t})\n}\n\nfunc (m *dbMeta) dumpChunks(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tpool := sync.Pool{New: func() interface{} { return &pb.Chunk{} }}\n\trelease := func(p proto.Message) {\n\t\tfor _, s := range p.(*pb.Batch).Chunks {\n\t\t\tpool.Put(s)\n\t\t}\n\t}\n\n\tvar maxId uint64\n\terr := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\tvar row chunk\n\t\tok, err := s.Select(\"MAX(id) as id\").Get(&row)\n\t\tif ok {\n\t\t\tmaxId = uint64(row.Id)\n\t\t}\n\t\treturn err\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn sqlQueryBatch(ctx, opt, maxId, func(ctx context.Context, start, end uint64) (int, error) {\n\t\tvar rows []chunk\n\t\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\t\treturn s.Where(\"id >= ? AND id < ?\", start, end).Find(&rows)\n\t\t}); err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\tchunks := make([]*pb.Chunk, 0, len(rows))\n\t\tfor _, c := range rows {\n\t\t\tpc := pool.Get().(*pb.Chunk)\n\t\t\tpc.Inode = uint64(c.Inode)\n\t\t\tpc.Index = c.Indx\n\t\t\tpc.Slices = c.Slices\n\t\t\tchunks = append(chunks, pc)\n\t\t}\n\t\treturn len(rows), dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Chunks: chunks}, release})\n\t})\n}\n\nfunc (m *dbMeta) dumpEdges(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tpool := sync.Pool{New: func() interface{} { return &pb.Edge{} }}\n\trelease := func(p proto.Message) {\n\t\tfor _, s := range p.(*pb.Batch).Edges {\n\t\t\tpool.Put(s)\n\t\t}\n\t}\n\n\tvar maxId uint64\n\terr := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\tvar row edge\n\t\tok, err := s.Select(\"MAX(id) as id\").Get(&row)\n\t\tif ok {\n\t\t\tmaxId = uint64(row.Id)\n\t\t}\n\t\treturn err\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar mu sync.Mutex\n\tdumpParents := make(map[uint64][]uint64)\n\terr = sqlQueryBatch(ctx, opt, maxId, func(ctx context.Context, start, end uint64) (int, error) {\n\t\tvar rows []edge\n\t\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\t\treturn s.Where(\"id >= ? AND id < ?\", start, end).Find(&rows)\n\t\t}); err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\tedges := make([]*pb.Edge, 0, len(rows))\n\t\tfor _, e := range rows {\n\t\t\tpe := pool.Get().(*pb.Edge)\n\t\t\tpe.Parent = uint64(e.Parent)\n\t\t\tpe.Inode = uint64(e.Inode)\n\t\t\tpe.Name = e.Name\n\t\t\tpe.Type = uint32(e.Type)\n\t\t\tedges = append(edges, pe)\n\t\t\tmu.Lock()\n\t\t\tdumpParents[uint64(e.Inode)] = append(dumpParents[uint64(e.Inode)], uint64(e.Parent))\n\t\t\tmu.Unlock()\n\t\t}\n\t\treturn len(rows), dumpResult(ctx, ch, &dumpedResult{&pb.Batch{Edges: edges}, release})\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tparents := make([]*pb.Parent, 0, sqlDumpBatchSize)\n\tst := make(map[uint64]int64)\n\tfor inode, ps := range dumpParents {\n\t\tif len(ps) > 1 {\n\t\t\tfor k := range st {\n\t\t\t\tdelete(st, k)\n\t\t\t}\n\t\t\tfor _, p := range ps {\n\t\t\t\tst[p] = st[p] + 1\n\t\t\t}\n\t\t\tfor parent, cnt := range st {\n\t\t\t\tparents = append(parents, &pb.Parent{Inode: inode, Parent: parent, Cnt: cnt})\n\t\t\t}\n\t\t}\n\t\tif len(parents) >= sqlDumpBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Parents: parents}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tparents = make([]*pb.Parent, 0, sqlDumpBatchSize)\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Parents: parents}})\n}\n\nfunc (m *dbMeta) dumpSymlinks(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []symlink\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tsymlinks := make([]*pb.Symlink, 0, min(len(rows), sqlDumpBatchSize))\n\tfor i, r := range rows {\n\t\tsymlinks = append(symlinks, &pb.Symlink{Inode: uint64(r.Inode), Target: r.Target})\n\t\tif len(symlinks) >= sqlDumpBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Symlinks: symlinks}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsymlinks = make([]*pb.Symlink, 0, min(len(rows)-i-1, sqlDumpBatchSize))\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Symlinks: symlinks}})\n}\n\nfunc (m *dbMeta) dumpCounters(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []counter\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tvar counters = make([]*pb.Counter, 0, len(rows))\n\tfor _, row := range rows {\n\t\tcounters = append(counters, &pb.Counter{Key: row.Name, Value: row.Value})\n\t}\n\tlogger.Debugf(\"dump counters %+v\", counters)\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Counters: counters}})\n}\n\nfunc (m *dbMeta) dumpSustained(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []sustained\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tss := make(map[uint64][]uint64)\n\tfor _, row := range rows {\n\t\tss[row.Sid] = append(ss[row.Sid], uint64(row.Inode))\n\t}\n\tsustained := make([]*pb.Sustained, 0, len(rows))\n\tfor k, v := range ss {\n\t\tsustained = append(sustained, &pb.Sustained{Sid: k, Inodes: v})\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Sustained: sustained}})\n}\n\nfunc (m *dbMeta) dumpDelFiles(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []delfile\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tdelFiles := make([]*pb.DelFile, 0, min(sqlDumpBatchSize, len(rows)))\n\tfor i, row := range rows {\n\t\tdelFiles = append(delFiles, &pb.DelFile{Inode: uint64(row.Inode), Length: row.Length, Expire: row.Expire})\n\t\tif len(delFiles) >= sqlDumpBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Delfiles: delFiles}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdelFiles = make([]*pb.DelFile, 0, min(sqlDumpBatchSize, len(rows)-i-1))\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Delfiles: delFiles}})\n}\n\nfunc (m *dbMeta) dumpSliceRef(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []sliceRef\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Where(\"refs != 1\").Find(&rows) // skip default refs\n\t}); err != nil {\n\t\treturn err\n\t}\n\tsliceRefs := make([]*pb.SliceRef, 0, min(sqlDumpBatchSize, len(rows)))\n\tfor i, sr := range rows {\n\t\tsliceRefs = append(sliceRefs, &pb.SliceRef{Id: sr.Id, Size: sr.Size, Refs: int64(sr.Refs)})\n\t\tif len(sliceRefs) >= sqlDumpBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{SliceRefs: sliceRefs}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsliceRefs = make([]*pb.SliceRef, 0, min(sqlDumpBatchSize, len(rows)-i-1))\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{SliceRefs: sliceRefs}})\n}\n\nfunc (m *dbMeta) dumpACL(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []acl\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tacls := make([]*pb.Acl, 0, len(rows))\n\tfor _, row := range rows {\n\t\tacls = append(acls, &pb.Acl{\n\t\t\tId: row.Id,\n\t\t\tData: row.toRule().Encode(),\n\t\t})\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Acls: acls}})\n}\n\nfunc (m *dbMeta) dumpXattr(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []xattr\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\txattrs := make([]*pb.Xattr, 0, min(sqlDumpBatchSize, len(rows)))\n\tfor i, x := range rows {\n\t\txattrs = append(xattrs, &pb.Xattr{\n\t\t\tInode: uint64(x.Inode),\n\t\t\tName: x.Name,\n\t\t\tValue: x.Value,\n\t\t})\n\t\tif len(xattrs) >= sqlDumpBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Xattrs: xattrs}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\txattrs = make([]*pb.Xattr, 0, min(sqlDumpBatchSize, len(rows)-i-1))\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Xattrs: xattrs}})\n}\n\nfunc (m *dbMeta) dumpQuota(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []dirQuota\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tquotas := make([]*pb.Quota, 0, len(rows))\n\tfor _, q := range rows {\n\t\tquotas = append(quotas, &pb.Quota{\n\t\t\tInode: uint64(q.Inode),\n\t\t\tMaxSpace: q.MaxSpace,\n\t\t\tMaxInodes: q.MaxInodes,\n\t\t\tUsedSpace: q.UsedSpace,\n\t\t\tUsedInodes: q.UsedInodes,\n\t\t})\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Quotas: quotas}})\n}\n\nfunc (m *dbMeta) dumpDirStat(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar rows []dirStats\n\tif err := m.execTxn(ctx, func(s *xorm.Session) error {\n\t\treturn s.Find(&rows)\n\t}); err != nil {\n\t\treturn err\n\t}\n\tdirStats := make([]*pb.Stat, 0, min(sqlDumpBatchSize, len(rows)))\n\tfor i, st := range rows {\n\t\tdirStats = append(dirStats, &pb.Stat{\n\t\t\tInode: uint64(st.Inode),\n\t\t\tDataLength: st.DataLength,\n\t\t\tUsedInodes: st.UsedInodes,\n\t\t\tUsedSpace: st.UsedSpace,\n\t\t})\n\t\tif len(dirStats) >= sqlDumpBatchSize {\n\t\t\tif err := dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Dirstats: dirStats}}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdirStats = make([]*pb.Stat, 0, min(sqlDumpBatchSize, len(rows)-i-1))\n\t\t}\n\t}\n\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Dirstats: dirStats}})\n}\n\nfunc (m *dbMeta) load(ctx Context, typ int, opt *LoadOption, val proto.Message) error {\n\tswitch typ {\n\tcase segTypeFormat:\n\t\treturn m.loadFormat(ctx, val)\n\tcase segTypeCounter:\n\t\treturn m.loadCounters(ctx, val)\n\tcase segTypeNode:\n\t\treturn m.loadNodes(ctx, val)\n\tcase segTypeChunk:\n\t\treturn m.loadChunks(ctx, val)\n\tcase segTypeEdge:\n\t\treturn m.loadEdges(ctx, val)\n\tcase segTypeSymlink:\n\t\treturn m.loadSymlinks(ctx, val)\n\tcase segTypeSustained:\n\t\treturn m.loadSustained(ctx, val)\n\tcase segTypeDelFile:\n\t\treturn m.loadDelFiles(ctx, val)\n\tcase segTypeSliceRef:\n\t\treturn m.loadSliceRefs(ctx, val)\n\tcase segTypeAcl:\n\t\treturn m.loadAcl(ctx, val)\n\tcase segTypeXattr:\n\t\treturn m.loadXattrs(ctx, val)\n\tcase segTypeQuota:\n\t\treturn m.loadQuota(ctx, val)\n\tcase segTypeStat:\n\t\treturn m.loadDirStats(ctx, val)\n\tcase segTypeParent:\n\t\treturn nil // skip\n\tdefault:\n\t\tlogger.Warnf(\"skip segment type %d\", typ)\n\t\treturn nil\n\t}\n}\n\nfunc (m *dbMeta) loadFormat(ctx Context, msg proto.Message) error {\n\treturn m.insertRows([]interface{}{\n\t\t&setting{\n\t\t\tName: \"format\",\n\t\t\tValue: string(msg.(*pb.Format).Data),\n\t\t},\n\t})\n}\n\nfunc (m *dbMeta) loadCounters(ctx Context, msg proto.Message) error {\n\tvar rows []interface{}\n\tfor _, c := range msg.(*pb.Batch).Counters {\n\t\trows = append(rows, counter{Name: c.Key, Value: c.Value})\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) loadNodes(ctx Context, msg proto.Message) error {\n\tnodes := msg.(*pb.Batch).Nodes\n\tb := m.getBase()\n\trows := make([]interface{}, 0, len(nodes))\n\tns := make([]node, len(nodes))\n\tattr := &Attr{}\n\tfor i, n := range nodes {\n\t\tpn := &ns[i]\n\t\tpn.Inode = Ino(n.Inode)\n\t\tattr.reset()\n\t\tb.parseAttr(n.Data, attr)\n\t\tm.parseNode(attr, pn)\n\t\trows = append(rows, pn)\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc genMultiSQL(stmt string, num int) string {\n\tif num <= 0 {\n\t\treturn \"\"\n\t}\n\tif num == 1 {\n\t\treturn stmt\n\t}\n\tpattern := \"(?,?,?)\"\n\tidx := strings.Index(stmt, pattern)\n\tif idx == -1 {\n\t\treturn stmt\n\t}\n\tvalues := strings.Repeat(pattern+\",\", num)\n\tvalues = values[:len(values)-1]\n\treturn stmt[:idx] + values + stmt[idx+len(pattern):]\n}\n\nfunc insertSliceRefs(m *dbMeta, ss []*sliceRef) error {\n\tdriver := m.Name()\n\tvar stmt string\n\tif driver == \"sqlite3\" || driver == \"postgres\" {\n\t\tstmt = m.sqlConv(`INSERT INTO chunk_ref (chunkid, size, refs) VALUES (?,?,?) ON CONFLICT DO NOTHING`)\n\t} else {\n\t\tstmt = m.sqlConv(`INSERT IGNORE INTO chunk_ref (chunkid, size, refs) VALUES (?,?,?)`)\n\t}\n\n\tbatch := m.getTxnBatchNum()\n\tfor len(ss) > 0 {\n\t\tbs := min(batch, len(ss))\n\t\terr := m.txn(func(s *xorm.Session) error {\n\t\t\tnStmt := genMultiSQL(stmt, bs)\n\t\t\trows := make([]interface{}, 0, 1+bs*3)\n\t\t\trows = append(rows, nStmt)\n\t\t\tfor i := 0; i < bs; i++ {\n\t\t\t\trows = append(rows, ss[i].Id, ss[i].Size, ss[i].Refs)\n\t\t\t}\n\t\t\t_, err := s.Exec(rows...)\n\t\t\treturn err\n\t\t})\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"write %d slice ref: %s\", bs, err)\n\t\t\treturn err\n\t\t}\n\t\tss = ss[bs:]\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) loadChunks(ctx Context, msg proto.Message) error {\n\tchunks := msg.(*pb.Batch).Chunks\n\tchkRows := make([]interface{}, 0, len(chunks))\n\tsrRows := make([]*sliceRef, 0, len(chunks))\n\tcs := make([]chunk, len(chunks))\n\tfor i, c := range chunks {\n\t\tpc := &cs[i]\n\t\tpc.Inode = Ino(c.Inode)\n\t\tpc.Indx = c.Index\n\t\tpc.Slices = c.Slices\n\t\tchkRows = append(chkRows, pc)\n\n\t\tss := readSliceBuf(c.Slices)\n\t\tfor _, s := range ss {\n\t\t\tsrRows = append(srRows, &sliceRef{Id: s.id, Size: s.size, Refs: 1})\n\t\t}\n\t}\n\tif err := m.insertRows(chkRows); err != nil {\n\t\treturn err\n\t}\n\treturn insertSliceRefs(m, srRows)\n}\n\nfunc (m *dbMeta) loadEdges(ctx Context, msg proto.Message) error {\n\tedges := msg.(*pb.Batch).Edges\n\trows := make([]interface{}, 0, len(edges))\n\tes := make([]edge, len(edges))\n\tfor i, e := range edges {\n\t\tpe := &es[i]\n\t\tpe.Parent = Ino(e.Parent)\n\t\tpe.Inode = Ino(e.Inode)\n\t\tpe.Name = e.Name\n\t\tpe.Type = uint8(e.Type)\n\t\trows = append(rows, pe)\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) loadSymlinks(ctx Context, msg proto.Message) error {\n\tsymlinks := msg.(*pb.Batch).Symlinks\n\trows := make([]interface{}, 0, len(symlinks))\n\tfor _, sl := range symlinks {\n\t\trows = append(rows, &symlink{Ino(sl.Inode), sl.Target})\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) loadSustained(ctx Context, msg proto.Message) error {\n\tsustaineds := msg.(*pb.Batch).Sustained\n\trows := make([]interface{}, 0, len(sustaineds))\n\tfor _, s := range sustaineds {\n\t\tfor _, inode := range s.Inodes {\n\t\t\trows = append(rows, sustained{Sid: s.Sid, Inode: Ino(inode)})\n\t\t}\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) loadDelFiles(ctx Context, msg proto.Message) error {\n\tdelfiles := msg.(*pb.Batch).Delfiles\n\trows := make([]interface{}, 0, len(delfiles))\n\tfor _, f := range delfiles {\n\t\trows = append(rows, &delfile{Inode: Ino(f.Inode), Length: f.Length, Expire: f.Expire})\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) upsertSliceRef(s *xorm.Session, id uint64, size uint32, refs int) error {\n\tvar err error\n\tdriver := m.Name()\n\tif driver == \"sqlite3\" || driver == \"postgres\" {\n\t\tstate := m.sqlConv(`\n\t\t\t INSERT INTO chunk_ref (chunkid, size, refs)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON CONFLICT (chunkid)\n\t\t\t DO UPDATE SET size=?, refs=?`)\n\t\t_, err = s.Exec(state, id, size, refs, size, refs)\n\t} else {\n\t\t_, err = s.Exec(m.sqlConv(`\n\t\t\t INSERT INTO chunk_ref (chunkid, size, refs)\n\t\t\t VALUES (?, ?, ?)\n\t\t\t ON DUPLICATE KEY UPDATE\n\t\t\t size=?, refs=?`), id, size, refs, size, refs)\n\t}\n\treturn err\n}\n\nfunc (m *dbMeta) loadSliceRefs(ctx Context, msg proto.Message) error {\n\tbatch := m.getTxnBatchNum()\n\tsrs := msg.(*pb.Batch).SliceRefs\n\tfor len(srs) > 0 {\n\t\tnum := min(batch, len(srs))\n\t\terr := m.txn(func(s *xorm.Session) error {\n\t\t\tvar err error\n\t\t\tfor i := 0; i < num; i++ {\n\t\t\t\tif err = m.upsertSliceRef(s, srs[i].Id, srs[i].Size, int(srs[i].Refs)); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Write %d beans: %s\", num, err)\n\t\t\treturn err\n\t\t}\n\t\tsrs = srs[num:]\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) loadAcl(ctx Context, msg proto.Message) error {\n\tacls := msg.(*pb.Batch).Acls\n\trows := make([]interface{}, 0, len(acls))\n\tfor _, pa := range acls {\n\t\trule := &aclAPI.Rule{}\n\t\trule.Decode(pa.Data)\n\t\tacl := newSQLAcl(rule)\n\t\tacl.Id = pa.Id\n\t\trows = append(rows, acl)\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) loadXattrs(ctx Context, msg proto.Message) error {\n\txattrs := msg.(*pb.Batch).Xattrs\n\trows := make([]interface{}, 0, len(xattrs))\n\tfor _, x := range xattrs {\n\t\trows = append(rows, &xattr{Inode: Ino(x.Inode), Name: x.Name, Value: x.Value})\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) loadQuota(ctx Context, msg proto.Message) error {\n\tquotas := msg.(*pb.Batch).Quotas\n\trows := make([]interface{}, 0, len(quotas))\n\tfor _, q := range quotas {\n\t\trows = append(rows, &dirQuota{\n\t\t\tInode: Ino(q.Inode),\n\t\t\tMaxSpace: q.MaxSpace,\n\t\t\tMaxInodes: q.MaxInodes,\n\t\t\tUsedSpace: q.UsedSpace,\n\t\t\tUsedInodes: q.UsedInodes,\n\t\t})\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) loadDirStats(ctx Context, msg proto.Message) error {\n\tstats := msg.(*pb.Batch).Dirstats\n\trows := make([]interface{}, 0, len(stats))\n\tfor _, st := range stats {\n\t\trows = append(rows, &dirStats{\n\t\t\tInode: Ino(st.Inode),\n\t\t\tDataLength: st.DataLength,\n\t\t\tUsedInodes: st.UsedInodes,\n\t\t\tUsedSpace: st.UsedSpace,\n\t\t})\n\t}\n\treturn m.insertRows(rows)\n}\n\nfunc (m *dbMeta) insertRows(beans []interface{}) error {\n\tbatch := m.getTxnBatchNum()\n\tfor len(beans) > 0 {\n\t\tbs := min(batch, len(beans))\n\t\terr := m.txn(func(s *xorm.Session) error {\n\t\t\tn, err := s.Insert(beans[:bs])\n\t\t\tif err == nil && int(n) != bs {\n\t\t\t\terr = fmt.Errorf(\"only %d records inserted\", n)\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Write %d beans: %s\", bs, err)\n\t\t\treturn err\n\t\t}\n\t\tbeans = beans[bs:]\n\t}\n\treturn nil\n}\n\nfunc (m *dbMeta) prepareLoad(ctx Context, opt *LoadOption) error {\n\topt.check()\n\tif err := m.checkAddr(); err != nil {\n\t\treturn err\n\t}\n\tif err := m.syncAllTables(); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/meta/sql_lock.go", "content": "//go:build !nosqlite || !nomysql || !nopg\n// +build !nosqlite !nomysql !nopg\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"syscall\"\n\t\"time\"\n\t\"xorm.io/xorm\"\n)\n\nfunc (m *dbMeta) Flock(ctx Context, inode Ino, owner_ uint64, ltype uint32, block bool) syscall.Errno {\n\towner := int64(owner_)\n\tif ltype == F_UNLCK {\n\t\treturn errno(m.txn(func(s *xorm.Session) error {\n\t\t\t_, err := s.MustCols(\"inode\", \"owner\", \"sid\").Delete(&flock{Inode: inode, Owner: owner, Sid: m.sid})\n\t\t\treturn err\n\t\t}, inode))\n\t}\n\tvar err syscall.Errno\n\tfor {\n\t\terr = errno(m.txn(func(s *xorm.Session) error {\n\t\t\tif exists, err := s.ForUpdate().Get(&node{Inode: inode}); err != nil || !exists {\n\t\t\t\tif err == nil && !exists {\n\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvar fs []flock\n\t\t\terr := s.ForUpdate().Find(&fs, &flock{Inode: inode})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ttype key struct {\n\t\t\t\tsid uint64\n\t\t\t\to int64\n\t\t\t}\n\t\t\tvar locks = make(map[key]flock)\n\t\t\tfor _, l := range fs {\n\t\t\t\tlocks[key{l.Sid, l.Owner}] = l\n\t\t\t}\n\n\t\t\tme := key{m.sid, owner}\n\t\t\tflk, ok := locks[me]\n\t\t\tdelete(locks, me)\n\t\t\tvar typec byte = 'W'\n\t\t\tif ltype == F_RDLCK {\n\t\t\t\tfor _, l := range locks {\n\t\t\t\t\tif l.Ltype == 'W' {\n\t\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ttypec = 'R'\n\t\t\t} else if len(locks) > 0 {\n\t\t\t\treturn syscall.EAGAIN\n\t\t\t}\n\t\t\tvar n int64\n\t\t\tif ok {\n\t\t\t\tif flk.Ltype != typec {\n\t\t\t\t\tn, err = s.MustCols(\"inode\", \"owner\", \"sid\").Cols(\"Ltype\").Update(&flock{Ltype: typec}, &flock{Inode: inode, Owner: owner, Sid: m.sid})\n\t\t\t\t} else {\n\t\t\t\t\tn = 1\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tn, err = s.InsertOne(&flock{Inode: inode, Owner: owner, Ltype: typec, Sid: m.sid})\n\t\t\t}\n\t\t\tif err == nil && n == 0 {\n\t\t\t\terr = fmt.Errorf(\"insert/update failed\")\n\t\t\t}\n\t\t\treturn err\n\t\t}, inode))\n\n\t\tif !block || err != syscall.EAGAIN {\n\t\t\tbreak\n\t\t}\n\t\tif ltype == F_WRLCK {\n\t\t\ttime.Sleep(time.Millisecond * 1)\n\t\t} else {\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *dbMeta) Getlk(ctx Context, inode Ino, owner_ uint64, ltype *uint32, start, end *uint64, pid *uint32) syscall.Errno {\n\tif *ltype == F_UNLCK {\n\t\t*start = 0\n\t\t*end = 0\n\t\t*pid = 0\n\t\treturn 0\n\t}\n\n\towner := int64(owner_)\n\trows, err := m.db.Rows(&plock{Inode: inode})\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\ttype key struct {\n\t\tsid uint64\n\t\to int64\n\t}\n\tvar locks = make(map[key][]byte)\n\tvar l plock\n\tfor rows.Next() {\n\t\tl.Records = nil\n\t\tif rows.Scan(&l) == nil && !(l.Sid == m.sid && l.Owner == owner) {\n\t\t\tlocks[key{l.Sid, l.Owner}] = dup(l.Records)\n\t\t}\n\t}\n\t_ = rows.Close()\n\n\tfor k, d := range locks {\n\t\tls := loadLocks(d)\n\t\tfor _, l := range ls {\n\t\t\t// find conflicted locks\n\t\t\tif (*ltype == F_WRLCK || l.Type == F_WRLCK) && *end >= l.Start && *start <= l.End {\n\t\t\t\t*ltype = l.Type\n\t\t\t\t*start = l.Start\n\t\t\t\t*end = l.End\n\t\t\t\tif k.sid == m.sid {\n\t\t\t\t\t*pid = l.Pid\n\t\t\t\t} else {\n\t\t\t\t\t*pid = 0\n\t\t\t\t}\n\t\t\t\treturn 0\n\t\t\t}\n\t\t}\n\t}\n\t*ltype = F_UNLCK\n\t*start = 0\n\t*end = 0\n\t*pid = 0\n\treturn 0\n}\n\nfunc (m *dbMeta) Setlk(ctx Context, inode Ino, owner_ uint64, block bool, ltype uint32, start, end uint64, pid uint32) syscall.Errno {\n\tvar err syscall.Errno\n\tlock := plockRecord{ltype, pid, start, end}\n\towner := int64(owner_)\n\tfor {\n\t\terr = errno(m.txn(func(s *xorm.Session) error {\n\t\t\tif exists, err := s.ForUpdate().Get(&node{Inode: inode}); err != nil || !exists {\n\t\t\t\tif err == nil && !exists {\n\t\t\t\t\terr = syscall.ENOENT\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif ltype == F_UNLCK {\n\t\t\t\tvar l = plock{Inode: inode, Owner: owner, Sid: m.sid}\n\t\t\t\tok, err := s.ForUpdate().MustCols(\"inode\", \"owner\", \"sid\").Get(&l)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tls := loadLocks(l.Records)\n\t\t\t\tif len(ls) == 0 {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tls = updateLocks(ls, lock)\n\t\t\t\tif len(ls) == 0 {\n\t\t\t\t\t_, err = s.MustCols(\"inode\", \"owner\", \"sid\").Delete(&plock{Inode: inode, Owner: owner, Sid: m.sid})\n\t\t\t\t} else {\n\t\t\t\t\t_, err = s.MustCols(\"inode\", \"owner\", \"sid\").Cols(\"records\").Update(plock{Records: dumpLocks(ls)}, l)\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvar ps []plock\n\t\t\terr := s.ForUpdate().Find(&ps, &plock{Inode: inode})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ttype key struct {\n\t\t\t\tsid uint64\n\t\t\t\towner int64\n\t\t\t}\n\t\t\tvar locks = make(map[key][]byte)\n\t\t\tfor _, l := range ps {\n\t\t\t\tlocks[key{l.Sid, l.Owner}] = l.Records\n\t\t\t}\n\t\t\tlkey := key{m.sid, owner}\n\t\t\tfor k, d := range locks {\n\t\t\t\tif k == lkey {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tls := loadLocks(d)\n\t\t\t\tfor _, l := range ls {\n\t\t\t\t\t// find conflicted locks\n\t\t\t\t\tif (ltype == F_WRLCK || l.Type == F_WRLCK) && end >= l.Start && start <= l.End {\n\t\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tls := updateLocks(loadLocks(locks[lkey]), lock)\n\t\t\tvar n int64\n\t\t\trecords := dumpLocks(ls)\n\t\t\tif len(locks[lkey]) > 0 {\n\t\t\t\tif !bytes.Equal(locks[lkey], records) {\n\t\t\t\t\tn, err = s.MustCols(\"inode\", \"owner\", \"sid\").Cols(\"records\").Update(plock{Records: records},\n\t\t\t\t\t\t&plock{Inode: inode, Sid: m.sid, Owner: owner})\n\t\t\t\t} else {\n\t\t\t\t\tn = 1\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tn, err = s.InsertOne(&plock{Inode: inode, Sid: m.sid, Owner: owner, Records: records})\n\t\t\t}\n\t\t\tif err == nil && n == 0 {\n\t\t\t\terr = fmt.Errorf(\"insert/update failed\")\n\t\t\t}\n\t\t\treturn err\n\t\t}, inode))\n\n\t\tif !block || err != syscall.EAGAIN {\n\t\t\tbreak\n\t\t}\n\t\tif ltype == F_WRLCK {\n\t\t\ttime.Sleep(time.Millisecond * 1)\n\t\t} else {\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (r *dbMeta) ListLocks(ctx context.Context, inode Ino) ([]PLockItem, []FLockItem, error) {\n\tvar fs []flock\n\tif err := r.db.Find(&fs, &flock{Inode: inode}); err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tflocks := make([]FLockItem, 0, len(fs))\n\tfor _, f := range fs {\n\t\tflocks = append(flocks, FLockItem{ownerKey{f.Sid, uint64(f.Owner)}, string(f.Ltype)})\n\t}\n\n\tvar ps []plock\n\tif err := r.db.Find(&ps, &plock{Inode: inode}); err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tplocks := make([]PLockItem, 0)\n\tfor _, p := range ps {\n\t\tls := loadLocks(p.Records)\n\t\tfor _, l := range ls {\n\t\t\tplocks = append(plocks, PLockItem{ownerKey{p.Sid, uint64(p.Owner)}, l})\n\t\t}\n\t}\n\treturn plocks, flocks, nil\n}\n" }, { "path": "pkg/meta/sql_mysql.go", "content": "//go:build !nomysql\n// +build !nomysql\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"github.com/go-sql-driver/mysql\"\n)\n\nfunc isMySQLDuplicateEntryErr(err error) bool {\n\tif e, ok := err.(*mysql.MySQLError); ok {\n\t\treturn e.Number == 1062\n\t}\n\treturn false\n}\n\nfunc setMySQLTransactionIsolation(dns string) (string, error) {\n\tcfg, err := mysql.ParseDSN(dns)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif cfg.Params == nil {\n\t\tcfg.Params = make(map[string]string)\n\t}\n\tcfg.Params[\"transaction_isolation\"] = \"'repeatable-read'\"\n\treturn cfg.FormatDSN(), nil\n}\n\nfunc init() {\n\tdupErrorCheckers = append(dupErrorCheckers, isMySQLDuplicateEntryErr)\n\tsetTransactionIsolation = setMySQLTransactionIsolation\n\tRegister(\"mysql\", newSQLMeta)\n}\n" }, { "path": "pkg/meta/sql_pg.go", "content": "//go:build !nopg\n// +build !nopg\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"github.com/jackc/pgx/v5/pgconn\"\n\t_ \"github.com/jackc/pgx/v5/stdlib\"\n)\n\nfunc isPGDuplicateEntryErr(err error) bool {\n\tif e, ok := err.(*pgconn.PgError); ok {\n\t\treturn e.Code == \"23505\"\n\t}\n\treturn false\n}\n\nfunc init() {\n\tdupErrorCheckers = append(dupErrorCheckers, isPGDuplicateEntryErr)\n\tRegister(\"postgres\", newSQLMeta)\n}\n" }, { "path": "pkg/meta/sql_sqlite.go", "content": "//go:build !nosqlite\n// +build !nosqlite\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"github.com/mattn/go-sqlite3\"\n)\n\nfunc isSQLiteDuplicateEntryErr(err error) bool {\n\tif e, ok := err.(sqlite3.Error); ok {\n\t\treturn e.Code == sqlite3.ErrConstraint\n\t}\n\treturn false\n}\n\nfunc init() {\n\terrBusy = sqlite3.ErrBusy\n\tdupErrorCheckers = append(dupErrorCheckers, isSQLiteDuplicateEntryErr)\n\tRegister(\"sqlite3\", newSQLMeta)\n}\n" }, { "path": "pkg/meta/sql_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n//nolint:errcheck\npackage meta\n\nimport (\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestSQLiteClient(t *testing.T) {\n\tm, err := newSQLMeta(\"sqlite3\", path.Join(t.TempDir(), \"jfs-unit-test.db\"), testConfig())\n\tif err != nil || m.Name() != \"sqlite3\" {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestMySQLClient(t *testing.T) { //skip mutate\n\tm, err := newSQLMeta(\"mysql\", \"root:@/dev\", testConfig())\n\tif err != nil || m.Name() != \"mysql\" {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestPostgreSQLClient(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SKIP_NON_CORE\") == \"true\" {\n\t\tt.Skipf(\"skip non-core test\")\n\t}\n\tm, err := newSQLMeta(\"postgres\", \"localhost:5432/test?sslmode=disable\", testConfig())\n\tif err != nil || m.Name() != \"postgres\" {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestPostgreSQLClientWithSearchPath(t *testing.T) { //skip mutate\n\t_, err := newSQLMeta(\"postgres\", \"localhost:5432/test?sslmode=disable&search_path=juicefs,public\", testConfig())\n\tif !strings.Contains(err.Error(), \"currently, only one schema is supported in search_path\") {\n\t\tt.Fatalf(\"TestPostgreSQLClientWithSearchPath error: %s\", err)\n\t}\n}\n\nfunc TestRecoveryMysqlPwd(t *testing.T) { //skip mutate\n\ttestCase := []struct {\n\t\taddr string\n\t\texpect string\n\t}{\n\t\t// no password\n\t\t{\"root@(localhost:3306)/db1\",\n\t\t\t\"root@(localhost:3306)/db1\",\n\t\t},\n\t\t// no password\n\t\t{\"root:@(localhost:3306)/db1\",\n\t\t\t\"root:@(localhost:3306)/db1\",\n\t\t},\n\n\t\t{\"root::@@(localhost:3306)/db1\",\n\t\t\t\"root::@@(localhost:3306)/db1\",\n\t\t},\n\n\t\t{\"root:@:@(localhost:3306)/db1\",\n\t\t\t\"root:@:@(localhost:3306)/db1\",\n\t\t},\n\n\t\t// no special char\n\t\t{\"root:password@(localhost:3306)/db1\",\n\t\t\t\"root:password@(localhost:3306)/db1\",\n\t\t},\n\n\t\t// set from env @\n\t\t{\"root:pass%40word@(localhost:3306)/db1\",\n\t\t\t\"root:pass@word@(localhost:3306)/db1\",\n\t\t},\n\n\t\t// direct pass special char @\n\t\t{\"root:pass@word@(localhost:3306)/db1\",\n\t\t\t\"root:pass@word@(localhost:3306)/db1\",\n\t\t},\n\n\t\t// set from env |\n\t\t{\"root:pass%7Cword@(localhost:3306)/db1\",\n\t\t\t\"root:pass|word@(localhost:3306)/db1\",\n\t\t},\n\n\t\t// direct pass special char |\n\t\t{\"root:pass|word@(localhost:3306)/db1\",\n\t\t\t\"root:pass|word@(localhost:3306)/db1\",\n\t\t},\n\n\t\t// set from env :\n\t\t{\"root:pass%3Aword@(localhost:3306)/db1\",\n\t\t\t\"root:pass:word@(localhost:3306)/db1\",\n\t\t},\n\n\t\t// direct pass special char :\n\t\t{\"root:pass:word@(localhost:3306)/db1\",\n\t\t\t\"root:pass:word@(localhost:3306)/db1\",\n\t\t},\n\t}\n\tfor _, tc := range testCase {\n\t\tif got := recoveryMysqlPwd(tc.addr); got != tc.expect {\n\t\t\tt.Fatalf(\"recoveryMysqlPwd error: expect %s but got %s\", tc.expect, got)\n\t\t}\n\t}\n}\n\nfunc TestGetCustomConfig(t *testing.T) {\n\tu := \"mysql://root:password@tcp(localhost:3306)/db1?max_open_conns=100¬Define=str\"\n\t_, after, _ := strings.Cut(u, \"?\")\n\tquery, err := url.ParseQuery(after)\n\tif err != nil {\n\t\tt.Fatalf(\"url parse query error: %s\", err)\n\t}\n\tmaxOpenConns, err := extractCustomConfig(&query, \"max_open_conns\", 1)\n\tif err != nil {\n\t\tt.Fatalf(\"getCustomConfig error: %s\", err)\n\t}\n\tif maxOpenConns != 100 {\n\t\tt.Fatalf(\"getCustomConfig error: expect 100 but got %d\", maxOpenConns)\n\t}\n\tif query.Has(\"max_open_conns\") {\n\t\tt.Fatalf(\"getCustomConfig error: expect not found but found\")\n\t}\n\n\tnot, err := extractCustomConfig(&query, \"notSetKey\", \"default\")\n\tif err != nil {\n\t\tt.Fatalf(\"getCustomConfig error: %s\", err)\n\t}\n\tif not != \"default\" {\n\t\tt.Fatalf(\"getCustomConfig error: expect default but got %s\", not)\n\t}\n\tif !query.Has(\"notDefine\") {\n\t\tt.Fatalf(\"getCustomConfig error: expect found but not\")\n\t}\n\n}\n" }, { "path": "pkg/meta/status.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\n// Statistic contains the statistics of the filesystem\ntype Statistic struct {\n\tUsedSpace uint64\n\tAvailableSpace uint64\n\tUsedInodes uint64\n\tAvailableInodes uint64\n\tTrashFileCount int64 `json:\",omitempty\"`\n\tTrashFileSize int64 `json:\",omitempty\"`\n\tPendingDeletedFileCount int64 `json:\",omitempty\"`\n\tPendingDeletedFileSize int64 `json:\",omitempty\"`\n\tTrashSliceCount int64 `json:\",omitempty\"`\n\tTrashSliceSize int64 `json:\",omitempty\"`\n\tPendingDeletedSliceCount int64 `json:\",omitempty\"`\n\tPendingDeletedSliceSize int64 `json:\",omitempty\"`\n}\n\ntype Sections struct {\n\tSetting *Format\n\tSessions []*Session\n\tStat *Statistic\n}\n\n// Status retrieves the status of the filesystem\nfunc Status(ctx context.Context, m Meta, trash bool, sections *Sections) error {\n\tformat, err := m.Load(true)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"load setting: %v\", err)\n\t}\n\tformat.RemoveSecret()\n\n\tsessions, err := m.ListSessions()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"list sessions: %v\", err)\n\t}\n\n\tstat := &Statistic{}\n\tvar totalSpace uint64\n\tif err = m.StatFS(Background(), RootInode, &totalSpace, &stat.AvailableSpace, &stat.UsedInodes, &stat.AvailableInodes); err != syscall.Errno(0) {\n\t\treturn fmt.Errorf(\"stat fs: %v\", err)\n\t}\n\tstat.UsedSpace = totalSpace - stat.AvailableSpace\n\n\tif trash {\n\t\tprogress := utils.NewProgress(false)\n\t\ttrashFileSpinner := progress.AddDoubleSpinner(\"Trash Files\")\n\t\tpendingDeletedFileSpinner := progress.AddDoubleSpinner(\"Pending Deleted Files\")\n\t\ttrashSlicesSpinner := progress.AddDoubleSpinner(\"Trash Slices\")\n\t\tpendingDeletedSlicesSpinner := progress.AddDoubleSpinner(\"Pending Deleted Slices\")\n\t\terr = m.ScanDeletedObject(\n\t\t\tWrapContext(ctx),\n\t\t\tfunc(ss []Slice, _ int64) (bool, error) {\n\t\t\t\tfor _, s := range ss {\n\t\t\t\t\ttrashSlicesSpinner.IncrInt64(int64(s.Size))\n\t\t\t\t}\n\t\t\t\treturn false, nil\n\t\t\t},\n\t\t\tfunc(_ uint64, size uint32) (bool, error) {\n\t\t\t\tpendingDeletedSlicesSpinner.IncrInt64(int64(size))\n\t\t\t\treturn false, nil\n\t\t\t},\n\t\t\tfunc(_ Ino, size uint64, _ time.Time) (bool, error) {\n\t\t\t\ttrashFileSpinner.IncrInt64(int64(size))\n\t\t\t\treturn false, nil\n\t\t\t},\n\t\t\tfunc(_ Ino, size uint64, _ int64) (bool, error) {\n\t\t\t\tpendingDeletedFileSpinner.IncrInt64(int64(size))\n\t\t\t\treturn false, nil\n\t\t\t},\n\t\t)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"statistic: %v\", err)\n\t\t}\n\n\t\ttrashSlicesSpinner.Done()\n\t\tpendingDeletedSlicesSpinner.Done()\n\t\ttrashFileSpinner.Done()\n\t\tpendingDeletedFileSpinner.Done()\n\t\tprogress.Done()\n\t\tstat.TrashSliceCount, stat.TrashSliceSize = trashSlicesSpinner.Current()\n\t\tstat.PendingDeletedSliceCount, stat.PendingDeletedSliceSize = pendingDeletedSlicesSpinner.Current()\n\t\tstat.TrashFileCount, stat.TrashFileSize = trashFileSpinner.Current()\n\t\tstat.PendingDeletedFileCount, stat.PendingDeletedFileSize = pendingDeletedFileSpinner.Current()\n\t}\n\n\tif sections != nil {\n\t\tsections.Setting = format\n\t\tsections.Sessions = sessions\n\t\tsections.Stat = stat\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/meta/tkv.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"math/rand\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\taclAPI \"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\ntype kvtxn interface {\n\tget(key []byte) []byte\n\tgets(keys ...[]byte) [][]byte\n\t// scan stops when handler returns false; begin and end must not be nil\n\tscan(begin, end []byte, keysOnly bool, handler func(k, v []byte) bool)\n\texist(prefix []byte) bool\n\tset(key, value []byte)\n\tappend(key []byte, value []byte)\n\tincrBy(key []byte, value int64) int64\n\tdelete(key []byte)\n}\n\ntype tkvClient interface {\n\tname() string\n\tsimpleTxn(ctx context.Context, f func(*kvTxn) error, retry int) error // should only be used for point get scenarios\n\ttxn(ctx context.Context, f func(*kvTxn) error, retry int) error\n\tscan(prefix []byte, handler func(key, value []byte) bool) error\n\treset(prefix []byte) error\n\tclose() error\n\tshouldRetry(err error) bool\n\tgc()\n\tconfig(key string) interface{}\n}\n\ntype kvTxn struct {\n\tkvtxn\n\tretry int\n}\n\nfunc (tx *kvTxn) deleteKeys(prefix []byte) {\n\ttx.scan(prefix, nextKey(prefix), true, func(k, v []byte) bool {\n\t\ttx.delete(k)\n\t\treturn true\n\t})\n}\n\ntype kvMeta struct {\n\t*baseMeta\n\tclient tkvClient\n\tsnap map[Ino]*DumpedEntry\n}\n\nvar _ Meta = (*kvMeta)(nil)\nvar _ engine = (*kvMeta)(nil)\n\nvar drivers = make(map[string]func(string) (tkvClient, error))\n\nfunc newTkvClient(driver, addr string) (tkvClient, error) {\n\tfn, ok := drivers[driver]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"unsupported driver %s\", driver)\n\t}\n\treturn fn(addr)\n}\n\nfunc newKVMeta(driver, addr string, conf *Config) (Meta, error) {\n\tclient, err := newTkvClient(driver, addr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"connect to addr %s: %s\", addr, err)\n\t}\n\t// TODO: ping server and check latency > Millisecond\n\t// logger.Warnf(\"The latency to database is too high: %s\", time.Since(start))\n\tm := &kvMeta{\n\t\tbaseMeta: newBaseMeta(addr, conf),\n\t\tclient: client,\n\t}\n\tm.en = m\n\treturn m, nil\n}\n\nfunc (m *kvMeta) Shutdown() error {\n\treturn m.client.close()\n}\n\nfunc (m *kvMeta) Name() string {\n\treturn m.client.name()\n}\n\nfunc (m *kvMeta) doDeleteSlice(id uint64, size uint32) error {\n\treturn m.deleteKeys(m.sliceKey(id, size))\n}\n\nfunc (m *kvMeta) keyLen(args ...interface{}) int {\n\tvar c int\n\tfor _, a := range args {\n\t\tswitch a := a.(type) {\n\t\tcase byte:\n\t\t\tc++\n\t\tcase uint32:\n\t\t\tc += 4\n\t\tcase uint64:\n\t\t\tc += 8\n\t\tcase Ino:\n\t\t\tc += 8\n\t\tcase string:\n\t\t\tc += len(a)\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"invalid type %T, value %v\", a, a))\n\t\t}\n\t}\n\treturn c\n}\n\nfunc (m *kvMeta) fmtKey(args ...interface{}) []byte {\n\tb := utils.NewBuffer(uint32(m.keyLen(args...)))\n\tfor _, a := range args {\n\t\tswitch a := a.(type) {\n\t\tcase byte:\n\t\t\tb.Put8(a)\n\t\tcase uint32:\n\t\t\tb.Put32(a)\n\t\tcase uint64:\n\t\t\tb.Put64(a)\n\t\tcase Ino:\n\t\t\tm.encodeInode(a, b.Get(8))\n\t\tcase string:\n\t\t\tb.Put([]byte(a))\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"invalid type %T, value %v\", a, a))\n\t\t}\n\t}\n\treturn b.Bytes()\n}\n\n/**\n Ino iiiiiiii\n Length llllllll\n Indx nnnn\n name ...\n sliceId cccccccc\n session ssssssss\n aclId aaaa\n\nAll keys:\n setting format\n C... counter\n AiiiiiiiiI inode attribute\n AiiiiiiiiD... dentry\n AiiiiiiiiPiiiiiiii parents // for hard links\n AiiiiiiiiCnnnn file chunks\n AiiiiiiiiS symlink target\n AiiiiiiiiX... extented attribute\n Diiiiiiiillllllll delete inodes\n Fiiiiiiii Flocks\n Piiiiiiii POSIX locks\n Kccccccccnnnn slice refs\n Lttttttttcccccccc delayed slices\n SEssssssss session expire time\n SHssssssss session heartbeat // for legacy client\n SIssssssss session info\n SSssssssssiiiiiiii sustained inode\n Uiiiiiiii data length, space and inodes usage in directory\n Niiiiiiii detached inde\n QDiiiiiiii directory quota\n Raaaa\t\t\t POSIX acl\n KDaaaa\t\t\t delegation token\n*/\n\nfunc (m *kvMeta) inodeKey(inode Ino) []byte {\n\treturn m.fmtKey(\"A\", inode, \"I\")\n}\n\nfunc (m *kvMeta) entryKey(parent Ino, name string) []byte {\n\treturn m.fmtKey(\"A\", parent, \"D\", name)\n}\n\nfunc (m *kvMeta) parentKey(inode, parent Ino) []byte {\n\treturn m.fmtKey(\"A\", inode, \"P\", parent)\n}\n\nfunc (m *kvMeta) chunkKey(inode Ino, indx uint32) []byte {\n\treturn m.fmtKey(\"A\", inode, \"C\", indx)\n}\n\nfunc (m *kvMeta) sliceKey(id uint64, size uint32) []byte {\n\treturn m.fmtKey(\"K\", id, size)\n}\n\nfunc (m *kvMeta) delSliceKey(ts int64, id uint64) []byte {\n\treturn m.fmtKey(\"L\", uint64(ts), id)\n}\n\nfunc (m *kvMeta) symKey(inode Ino) []byte {\n\treturn m.fmtKey(\"A\", inode, \"S\")\n}\n\nfunc (m *kvMeta) xattrKey(inode Ino, name string) []byte {\n\treturn m.fmtKey(\"A\", inode, \"X\", name)\n}\n\nfunc (m *kvMeta) flockKey(inode Ino) []byte {\n\treturn m.fmtKey(\"F\", inode)\n}\n\nfunc (m *kvMeta) plockKey(inode Ino) []byte {\n\treturn m.fmtKey(\"P\", inode)\n}\n\nfunc (m *kvMeta) sessionKey(sid uint64) []byte {\n\treturn m.fmtKey(\"SE\", sid)\n}\n\nfunc (m *kvMeta) legacySessionKey(sid uint64) []byte {\n\treturn m.fmtKey(\"SH\", sid)\n}\n\nfunc (m *kvMeta) dirStatKey(inode Ino) []byte {\n\treturn m.fmtKey(\"U\", inode)\n}\n\nfunc (m *kvMeta) detachedKey(inode Ino) []byte {\n\treturn m.fmtKey(\"N\", inode)\n}\n\nfunc (m *kvMeta) dirQuotaKey(inode Ino) []byte {\n\treturn m.fmtKey(\"QD\", inode)\n}\n\nfunc (m *kvMeta) userQuotaKey(uid uint64) []byte {\n\treturn m.fmtKey(\"QU\", uid)\n}\n\nfunc (m *kvMeta) groupQuotaKey(gid uint64) []byte {\n\treturn m.fmtKey(\"QG\", gid)\n}\n\nfunc (m *kvMeta) aclKey(id uint32) []byte {\n\treturn m.fmtKey(\"R\", id)\n}\n\nfunc (m *kvMeta) krbTokenKey(id uint32) []byte {\n\treturn m.fmtKey(\"KD\", id)\n}\n\nfunc (m *kvMeta) parseACLId(key string) uint32 {\n\t// trim \"R\"\n\trb := utils.ReadBuffer([]byte(key[1:]))\n\treturn rb.Get32()\n}\n\nfunc (m *kvMeta) parseSid(key string) uint64 {\n\tbuf := []byte(key[2:]) // \"SE\" or \"SH\"\n\tif len(buf) != 8 {\n\t\tpanic(\"invalid sid value\")\n\t}\n\treturn binary.BigEndian.Uint64(buf)\n}\n\nfunc (m *kvMeta) sessionInfoKey(sid uint64) []byte {\n\treturn m.fmtKey(\"SI\", sid)\n}\n\nfunc (m *kvMeta) sustainedKey(sid uint64, inode Ino) []byte {\n\treturn m.fmtKey(\"SS\", sid, inode)\n}\n\nfunc (m *kvMeta) encodeInode(ino Ino, buf []byte) {\n\tbinary.LittleEndian.PutUint64(buf, uint64(ino))\n}\n\nfunc (m *kvMeta) decodeInode(buf []byte) Ino {\n\treturn Ino(binary.LittleEndian.Uint64(buf))\n}\n\nfunc (m *kvMeta) delfileKey(inode Ino, length uint64) []byte {\n\treturn m.fmtKey(\"D\", inode, length)\n}\n\nfunc (m *kvMeta) counterKey(key string) []byte {\n\treturn m.fmtKey(\"C\", key)\n}\n\n// Used for values that are modified by directly set; mostly timestamps\nfunc (m *kvMeta) packInt64(value int64) []byte {\n\tb := make([]byte, 8)\n\tbinary.BigEndian.PutUint64(b, uint64(value))\n\treturn b\n}\n\nfunc (m *kvMeta) parseInt64(buf []byte) int64 {\n\tif len(buf) == 0 {\n\t\treturn 0\n\t}\n\tif len(buf) != 8 {\n\t\tpanic(\"invalid value\")\n\t}\n\treturn int64(binary.BigEndian.Uint64(buf))\n}\n\n// Used for most counter values that are modified by incrBy\nfunc packCounter(value int64) []byte {\n\tb := make([]byte, 8)\n\tbinary.LittleEndian.PutUint64(b, uint64(value))\n\treturn b\n}\n\nfunc parseCounter(buf []byte) int64 {\n\tif len(buf) == 0 {\n\t\treturn 0\n\t}\n\tif len(buf) != 8 {\n\t\tpanic(\"invalid counter value\")\n\t}\n\treturn int64(binary.LittleEndian.Uint64(buf))\n}\n\nfunc (m *kvMeta) packEntry(_type uint8, inode Ino) []byte {\n\tb := utils.NewBuffer(9)\n\tb.Put8(_type)\n\tb.Put64(uint64(inode))\n\treturn b.Bytes()\n}\n\nfunc (m *kvMeta) parseEntry(buf []byte) (uint8, Ino) {\n\tb := utils.FromBuffer(buf)\n\treturn b.Get8(), Ino(b.Get64())\n}\n\nfunc (m *kvMeta) packDirStat(st *dirStat) []byte {\n\tb := utils.NewBuffer(24)\n\tb.Put64(uint64(st.length))\n\tb.Put64(uint64(st.space))\n\tb.Put64(uint64(st.inodes))\n\treturn b.Bytes()\n}\n\nfunc (m *kvMeta) parseDirStat(buf []byte) *dirStat {\n\tb := utils.FromBuffer(buf)\n\treturn &dirStat{int64(b.Get64()), int64(b.Get64()), int64(b.Get64())}\n}\n\nfunc (m *kvMeta) packQuota(q *Quota) []byte {\n\tb := utils.NewBuffer(32)\n\tb.Put64(uint64(q.MaxSpace))\n\tb.Put64(uint64(q.MaxInodes))\n\tb.Put64(uint64(q.UsedSpace))\n\tb.Put64(uint64(q.UsedInodes))\n\treturn b.Bytes()\n}\n\nfunc (m *kvMeta) parseQuota(buf []byte) *Quota {\n\tb := utils.FromBuffer(buf)\n\treturn &Quota{\n\t\tMaxSpace: int64(b.Get64()),\n\t\tMaxInodes: int64(b.Get64()),\n\t\tUsedSpace: int64(b.Get64()),\n\t\tUsedInodes: int64(b.Get64()),\n\t}\n}\n\nfunc (m *kvMeta) get(key []byte) ([]byte, error) {\n\tvar value []byte\n\terr := m.client.simpleTxn(Background(), func(tx *kvTxn) error {\n\t\tvalue = tx.get(key)\n\t\treturn nil\n\t}, 0)\n\treturn value, err\n}\n\nfunc (m *kvMeta) scanKeys(ctx context.Context, prefix []byte) ([][]byte, error) {\n\tvar keys [][]byte\n\terr := m.client.txn(ctx, func(tx *kvTxn) error {\n\t\ttx.scan(prefix, nextKey(prefix), true, func(k, v []byte) bool {\n\t\t\tkeys = append(keys, k)\n\t\t\treturn true\n\t\t})\n\t\treturn nil\n\t}, 0)\n\treturn keys, err\n}\n\nfunc (m *kvMeta) scanValues(ctx context.Context, prefix []byte, limit int, filter func(k, v []byte) bool) (map[string][]byte, error) {\n\tif limit == 0 {\n\t\treturn nil, nil\n\t}\n\tvalues := make(map[string][]byte)\n\terr := m.client.txn(ctx, func(tx *kvTxn) error {\n\t\tvar c int\n\t\ttx.scan(prefix, nextKey(prefix), false, func(k, v []byte) bool {\n\t\t\tif filter == nil || filter(k, v) {\n\t\t\t\tvalues[string(k)] = v\n\t\t\t\tc++\n\t\t\t}\n\t\t\treturn limit < 0 || c < limit\n\t\t})\n\t\treturn nil\n\t}, 0)\n\treturn values, err\n}\n\nfunc (m *kvMeta) scan(startKey, endKey []byte, limit int, filter func(k, v []byte) bool) ([][]byte, [][]byte, error) {\n\tif limit == 0 {\n\t\treturn nil, nil, nil\n\t}\n\tvar keys, vals [][]byte\n\terr := m.client.txn(Background(), func(tx *kvTxn) error {\n\t\tvar c int\n\t\ttx.scan(startKey, endKey, false, func(k, v []byte) bool {\n\t\t\tif filter == nil || filter(k, v) {\n\t\t\t\tkeys = append(keys, k)\n\t\t\t\tvals = append(vals, v)\n\t\t\t\tc++\n\t\t\t}\n\t\t\treturn limit < 0 || c < limit\n\t\t})\n\t\treturn nil\n\t}, 0)\n\treturn keys, vals, err\n}\n\nfunc (m *kvMeta) doInit(format *Format, force bool) error {\n\tbody, err := m.get(m.fmtKey(\"setting\"))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif body != nil {\n\t\tvar old Format\n\t\terr = json.Unmarshal(body, &old)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"json: %s\", err)\n\t\t}\n\t\tif !old.DirStats && format.DirStats {\n\t\t\t// remove dir stats as they are outdated\n\t\t\tvar keys [][]byte\n\t\t\tprefix := m.fmtKey(\"U\")\n\t\t\terr := m.client.txn(Background(), func(tx *kvTxn) error {\n\t\t\t\ttx.scan(prefix, nextKey(prefix), true, func(k, v []byte) bool {\n\t\t\t\t\tif len(k) == 9 {\n\t\t\t\t\t\tkeys = append(keys, k)\n\t\t\t\t\t}\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t\t\treturn nil\n\t\t\t}, 0)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"scan dir stats\")\n\t\t\t}\n\t\t\terr = m.deleteKeys(keys...)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"delete dir stats\")\n\t\t\t}\n\t\t}\n\t\tif !old.UserGroupQuota && format.UserGroupQuota {\n\t\t\t// remove user group quota as they are outdated\n\t\t\tuserPrefix := m.fmtKey(\"QU\")\n\t\t\tgroupPrefix := m.fmtKey(\"QG\")\n\t\t\terr := m.client.txn(Background(), func(tx *kvTxn) error {\n\t\t\t\ttx.deleteKeys(userPrefix)\n\t\t\t\ttx.deleteKeys(groupPrefix)\n\t\t\t\treturn nil\n\t\t\t}, 0)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"delete user group quota\")\n\t\t\t}\n\t\t}\n\t\tif err = format.update(&old, force); err != nil {\n\t\t\treturn errors.Wrap(err, \"update format\")\n\t\t}\n\t}\n\n\tdata, err := json.MarshalIndent(format, \"\", \"\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"json: %s\", err)\n\t}\n\n\tm.fmt = format\n\tts := time.Now().Unix()\n\tattr := &Attr{\n\t\tTyp: TypeDirectory,\n\t\tAtime: ts,\n\t\tMtime: ts,\n\t\tCtime: ts,\n\t\tNlink: 2,\n\t\tLength: 4 << 10,\n\t\tParent: RootInode,\n\t}\n\treturn m.txn(Background(), func(tx *kvTxn) error {\n\t\tif format.TrashDays > 0 {\n\t\t\tbuf := tx.get(m.inodeKey(TrashInode))\n\t\t\tif buf == nil {\n\t\t\t\tattr.Mode = 0555\n\t\t\t\ttx.set(m.inodeKey(TrashInode), m.marshal(attr))\n\t\t\t}\n\t\t}\n\t\ttx.set(m.fmtKey(\"setting\"), data)\n\t\tif body == nil || m.client.name() == \"memkv\" {\n\t\t\tattr.Mode = 0777\n\t\t\ttx.set(m.inodeKey(RootInode), m.marshal(attr))\n\t\t\ttx.incrBy(m.counterKey(\"nextInode\"), 2)\n\t\t\ttx.incrBy(m.counterKey(\"nextChunk\"), 1)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) cacheACLs(ctx Context) error {\n\tif !m.getFormat().EnableACL {\n\t\treturn nil\n\t}\n\n\tacls, err := m.scanValues(ctx, m.fmtKey(\"R\"), -1, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor key, val := range acls {\n\t\ttmpRule := &aclAPI.Rule{}\n\t\ttmpRule.Decode(val)\n\t\tm.aclCache.Put(m.parseACLId(key), tmpRule)\n\t}\n\treturn nil\n}\n\nfunc (m *kvMeta) Reset() error {\n\treturn m.client.reset(nil)\n}\n\nfunc (m *kvMeta) doLoad() ([]byte, error) {\n\treturn m.get(m.fmtKey(\"setting\"))\n}\n\nfunc (m *kvMeta) updateStats(space int64, inodes int64) {\n\tatomic.AddInt64(&m.newSpace, space)\n\tatomic.AddInt64(&m.newInodes, inodes)\n}\n\nfunc (m *kvMeta) doFlushStats() {\n\tif space := atomic.LoadInt64(&m.newSpace); space != 0 {\n\t\tif v, err := m.incrCounter(usedSpace, space); err == nil {\n\t\t\tatomic.AddInt64(&m.newSpace, -space)\n\t\t\tatomic.StoreInt64(&m.usedSpace, v)\n\t\t} else {\n\t\t\tlogger.Warnf(\"Update space stats: %s\", err)\n\t\t}\n\t}\n\tif inodes := atomic.LoadInt64(&m.newInodes); inodes != 0 {\n\t\tif v, err := m.incrCounter(totalInodes, inodes); err == nil {\n\t\t\tatomic.AddInt64(&m.newInodes, -inodes)\n\t\t\tatomic.StoreInt64(&m.usedInodes, v)\n\t\t} else {\n\t\t\tlogger.Warnf(\"Update inodes stats: %s\", err)\n\t\t}\n\t}\n}\n\nfunc (m *kvMeta) doNewSession(sinfo []byte, update bool) error {\n\tif err := m.setValue(m.sessionKey(m.sid), m.packInt64(m.expireTime())); err != nil {\n\t\treturn fmt.Errorf(\"set session ID %d: %s\", m.sid, err)\n\t}\n\tif err := m.setValue(m.sessionInfoKey(m.sid), sinfo); err != nil {\n\t\treturn fmt.Errorf(\"set session info: %s\", err)\n\t}\n\treturn nil\n}\n\nfunc (m *kvMeta) doRefreshSession() error {\n\treturn m.txn(Background(), func(tx *kvTxn) error {\n\t\tbuf := tx.get(m.sessionKey(m.sid))\n\t\tif buf == nil {\n\t\t\tlogger.Warnf(\"Session %d was stale and cleaned up, but now it comes back again\", m.sid)\n\t\t\ttx.set(m.sessionInfoKey(m.sid), m.newSessionInfo())\n\t\t}\n\t\ttx.set(m.sessionKey(m.sid), m.packInt64(m.expireTime()))\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) doCleanStaleSession(sid uint64) error {\n\tvar fail bool\n\t// release locks\n\tctx := Background()\n\tif flocks, err := m.scanValues(ctx, m.fmtKey(\"F\"), -1, nil); err == nil {\n\t\tfor k, v := range flocks {\n\t\t\tls := unmarshalFlock(v)\n\t\t\tfor o := range ls {\n\t\t\t\tif o.sid == sid {\n\t\t\t\t\tif err = m.txn(ctx, func(tx *kvTxn) error {\n\t\t\t\t\t\tv := tx.get([]byte(k))\n\t\t\t\t\t\tls := unmarshalFlock(v)\n\t\t\t\t\t\tdelete(ls, o)\n\t\t\t\t\t\tif len(ls) > 0 {\n\t\t\t\t\t\t\ttx.set([]byte(k), marshalFlock(ls))\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\ttx.delete([]byte(k))\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlogger.Warnf(\"Delete flock with sid %d: %s\", sid, err)\n\t\t\t\t\t\tfail = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlogger.Warnf(\"Scan flock with sid %d: %s\", sid, err)\n\t\tfail = true\n\t}\n\n\tif plocks, err := m.scanValues(ctx, m.fmtKey(\"P\"), -1, nil); err == nil {\n\t\tfor k, v := range plocks {\n\t\t\tls := unmarshalPlock(v)\n\t\t\tfor o := range ls {\n\t\t\t\tif o.sid == sid {\n\t\t\t\t\tif err = m.txn(ctx, func(tx *kvTxn) error {\n\t\t\t\t\t\tv := tx.get([]byte(k))\n\t\t\t\t\t\tls := unmarshalPlock(v)\n\t\t\t\t\t\tdelete(ls, o)\n\t\t\t\t\t\tif len(ls) > 0 {\n\t\t\t\t\t\t\ttx.set([]byte(k), marshalPlock(ls))\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\ttx.delete([]byte(k))\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlogger.Warnf(\"Delete plock with sid %d: %s\", sid, err)\n\t\t\t\t\t\tfail = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlogger.Warnf(\"Scan plock with sid %d: %s\", sid, err)\n\t\tfail = true\n\t}\n\n\tif keys, err := m.scanKeys(ctx, m.fmtKey(\"SS\", sid)); err == nil {\n\t\tfor _, key := range keys {\n\t\t\tinode := m.decodeInode(key[10:]) // \"SS\" + sid\n\t\t\tif err = m.doDeleteSustainedInode(sid, inode); err != nil {\n\t\t\t\tlogger.Warnf(\"Delete sustained inode %d of sid %d: %s\", inode, sid, err)\n\t\t\t\tfail = true\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlogger.Warnf(\"Scan sustained with sid %d: %s\", sid, err)\n\t\tfail = true\n\t}\n\n\tif fail {\n\t\treturn fmt.Errorf(\"failed to clean up sid %d\", sid)\n\t} else {\n\t\treturn m.deleteKeys(m.sessionKey(sid), m.legacySessionKey(sid), m.sessionInfoKey(sid))\n\t}\n}\n\nfunc (m *kvMeta) doFindStaleSessions(limit int) ([]uint64, error) {\n\tctx := Background()\n\tvals, err := m.scanValues(ctx, m.fmtKey(\"SE\"), limit, func(k, v []byte) bool {\n\t\treturn m.parseInt64(v) < time.Now().Unix()\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsids := make([]uint64, 0, len(vals))\n\tfor k := range vals {\n\t\tsids = append(sids, m.parseSid(k))\n\t}\n\tlimit -= len(sids)\n\tif limit <= 0 {\n\t\treturn sids, nil\n\t}\n\n\t// check clients with version before 1.0-beta3 as well\n\tvals, err = m.scanValues(ctx, m.fmtKey(\"SH\"), limit, func(k, v []byte) bool {\n\t\treturn m.parseInt64(v) < time.Now().Add(time.Minute*-5).Unix()\n\t})\n\tif err != nil {\n\t\tlogger.Errorf(\"Scan stale legacy sessions: %s\", err)\n\t\treturn sids, nil\n\t}\n\tfor k := range vals {\n\t\tsids = append(sids, m.parseSid(k))\n\t}\n\treturn sids, nil\n}\n\nfunc (m *kvMeta) getSession(sid uint64, detail bool) (*Session, error) {\n\tinfo, err := m.get(m.sessionInfoKey(sid))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif info == nil {\n\t\tinfo = []byte(\"{}\")\n\t}\n\tvar s Session\n\tif err = json.Unmarshal(info, &s); err != nil {\n\t\treturn nil, fmt.Errorf(\"corrupted session info; json error: %s\", err)\n\t}\n\ts.Sid = sid\n\tif detail {\n\t\tctx := Background()\n\t\tinodes, err := m.scanKeys(ctx, m.fmtKey(\"SS\", sid))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ts.Sustained = make([]Ino, 0, len(inodes))\n\t\tfor _, sinode := range inodes {\n\t\t\tinode := m.decodeInode(sinode[10:]) // \"SS\" + sid\n\t\t\ts.Sustained = append(s.Sustained, inode)\n\t\t}\n\t\tflocks, err := m.scanValues(ctx, m.fmtKey(\"F\"), -1, nil)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfor k, v := range flocks {\n\t\t\tinode := m.decodeInode([]byte(k[1:])) // \"F\"\n\t\t\tls := unmarshalFlock(v)\n\t\t\tfor o, l := range ls {\n\t\t\t\tif o.sid == sid {\n\t\t\t\t\ts.Flocks = append(s.Flocks, Flock{inode, o.sid, string(l)})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tplocks, err := m.scanValues(ctx, m.fmtKey(\"P\"), -1, nil)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfor k, v := range plocks {\n\t\t\tinode := m.decodeInode([]byte(k[1:])) // \"P\"\n\t\t\tls := unmarshalPlock(v)\n\t\t\tfor o, l := range ls {\n\t\t\t\tif o.sid == sid {\n\t\t\t\t\ts.Plocks = append(s.Plocks, Plock{inode, o.sid, loadLocks(l)})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn &s, nil\n}\n\nfunc (m *kvMeta) GetSession(sid uint64, detail bool) (*Session, error) {\n\tvar legacy bool\n\tvalue, err := m.get(m.sessionKey(sid))\n\tif err == nil && value == nil {\n\t\tlegacy = true\n\t\tvalue, err = m.get(m.legacySessionKey(sid))\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif value == nil {\n\t\treturn nil, fmt.Errorf(\"session not found: %d\", sid)\n\t}\n\ts, err := m.getSession(sid, detail)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ts.Expire = time.Unix(m.parseInt64(value), 0)\n\tif legacy {\n\t\ts.Expire = s.Expire.Add(time.Minute * 5)\n\t}\n\treturn s, nil\n}\n\nfunc (m *kvMeta) ListSessions() ([]*Session, error) {\n\tctx := Background()\n\tvals, err := m.scanValues(ctx, m.fmtKey(\"SE\"), -1, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsessions := make([]*Session, 0, len(vals))\n\tfor k, v := range vals {\n\t\ts, err := m.getSession(m.parseSid(k), false)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"get session: %s\", err)\n\t\t\tcontinue\n\t\t}\n\t\ts.Expire = time.Unix(m.parseInt64(v), 0)\n\t\tsessions = append(sessions, s)\n\t}\n\n\t// add clients with version before 1.0-beta3 as well\n\tvals, err = m.scanValues(ctx, m.fmtKey(\"SH\"), -1, nil)\n\tif err != nil {\n\t\tlogger.Errorf(\"Scan legacy sessions: %s\", err)\n\t\treturn sessions, nil\n\t}\n\tfor k, v := range vals {\n\t\ts, err := m.getSession(m.parseSid(k), false)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Get legacy session: %s\", err)\n\t\t\tcontinue\n\t\t}\n\t\ts.Expire = time.Unix(m.parseInt64(v), 0).Add(time.Minute * 5)\n\t\tsessions = append(sessions, s)\n\t}\n\treturn sessions, nil\n}\n\nfunc (m *kvMeta) shouldRetry(err error) bool {\n\treturn m.client.shouldRetry(err)\n}\n\nfunc (m *kvMeta) txn(ctx Context, f func(tx *kvTxn) error, inodes ...Ino) error {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tstart := time.Now()\n\tdefer func() { m.txDist.Observe(time.Since(start).Seconds()) }()\n\tdefer m.txBatchLock(inodes...)()\n\tvar (\n\t\tlastErr error\n\t\tmethod txMethod\n\t)\n\n\tfor i := 0; i < 50; i++ {\n\t\tif ctx.Canceled() {\n\t\t\tlogger.Warnf(\"Transaction %s interrupted after %s, tried %d, inodes: %v\", method.name(ctx), time.Since(start), i+1, inodes)\n\t\t\treturn syscall.EINTR\n\t\t}\n\t\terr := m.client.txn(ctx, f, i)\n\t\tif eno, ok := err.(syscall.Errno); ok && eno == 0 {\n\t\t\terr = nil\n\t\t}\n\t\tif err != nil && m.shouldRetry(err) {\n\t\t\tm.txRestart.WithLabelValues(method.name(ctx)).Add(1)\n\t\t\tlogger.Debugf(\"Transaction failed, restart it (tried %d): %s\", i+1, err)\n\t\t\tlastErr = err\n\t\t\ttime.Sleep(time.Millisecond * time.Duration(rand.Int()%((i+1)*(i+1))))\n\t\t\tcontinue\n\t\t} else if err == nil && i > 1 {\n\t\t\tlogger.Warnf(\"Transaction succeeded after %d tries (%s), inodes: %v, method: %s, error: %s\", i+1, time.Since(start), inodes, method.name(ctx), lastErr)\n\t\t}\n\t\treturn err\n\t}\n\tlogger.Warnf(\"Already tried 50 times, returning: %s\", lastErr)\n\treturn lastErr\n}\n\nfunc (m *kvMeta) setValue(key, value []byte) error {\n\treturn m.txn(Background(), func(tx *kvTxn) error {\n\t\ttx.set(key, value)\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) getCounter(name string) (int64, error) {\n\tbuf, err := m.get(m.counterKey(name))\n\treturn parseCounter(buf), err\n}\n\nfunc (m *kvMeta) incrCounter(name string, value int64) (int64, error) {\n\tvar new int64\n\tkey := m.counterKey(name)\n\terr := m.txn(Background().WithValue(txMethodKey{}, \"incrCounter:\"+name), func(tx *kvTxn) error {\n\t\tnew = tx.incrBy(key, value)\n\t\treturn nil\n\t})\n\treturn new, err\n}\n\nfunc (m *kvMeta) setIfSmall(name string, value, diff int64) (bool, error) {\n\tvar changed bool\n\tkey := m.counterKey(name)\n\terr := m.txn(Background().WithValue(txMethodKey{}, \"setIfSmall:\"+name), func(tx *kvTxn) error {\n\t\tchanged = false\n\t\tif m.parseInt64(tx.get(key)) > value-diff {\n\t\t\treturn nil\n\t\t} else {\n\t\t\tchanged = true\n\t\t\ttx.set(key, m.packInt64(value))\n\t\t\treturn nil\n\t\t}\n\t})\n\n\treturn changed, err\n}\n\nfunc (m *kvMeta) deleteKeys(keys ...[]byte) error {\n\tif len(keys) == 0 {\n\t\treturn nil\n\t}\n\treturn m.txn(Background(), func(tx *kvTxn) error {\n\t\tfor _, key := range keys {\n\t\t\ttx.delete(key)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) doLookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr) syscall.Errno {\n\tbuf, err := m.get(m.entryKey(parent, name))\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tif buf == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tfoundType, foundIno := m.parseEntry(buf)\n\ta, err := m.get(m.inodeKey(foundIno))\n\tif a != nil {\n\t\tm.parseAttr(a, attr)\n\t\tm.of.Update(foundIno, attr)\n\t} else if err == nil {\n\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", foundIno, parent, name)\n\t\t*attr = Attr{Typ: foundType}\n\t}\n\t*inode = foundIno\n\treturn errno(err)\n}\n\nfunc (m *kvMeta) doGetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno {\n\treturn errno(m.client.simpleTxn(ctx, func(tx *kvTxn) error {\n\t\tval := tx.get(m.inodeKey(inode))\n\t\tif val == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(val, attr)\n\t\treturn nil\n\t}, 0))\n}\n\nfunc (m *kvMeta) doSetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr, oldAttr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tvar cur Attr\n\t\ta := tx.get(m.inodeKey(inode))\n\t\tif a == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(a, &cur)\n\t\tif oldAttr != nil {\n\t\t\t*oldAttr = cur\n\t\t}\n\t\tif cur.Parent > TrashInode {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tnow := time.Now()\n\n\t\trule, err := m.getACL(tx, cur.AccessACL)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\trule = rule.Dup()\n\t\tdirtyAttr, st := m.mergeAttr(ctx, inode, set, &cur, attr, now, rule)\n\t\tif st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif dirtyAttr == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\tdirtyAttr.AccessACL, err = m.insertACL(tx, rule)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tdirtyAttr.Ctime = now.Unix()\n\t\tdirtyAttr.Ctimensec = uint32(now.Nanosecond())\n\t\ttx.set(m.inodeKey(inode), m.marshal(dirtyAttr))\n\t\t*attr = *dirtyAttr\n\t\treturn nil\n\t}, inode))\n}\n\nfunc (m *kvMeta) doTruncate(ctx Context, inode Ino, flags uint8, length uint64, delta *dirStat, attr *Attr, skipPermCheck bool) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\t*delta = dirStat{}\n\t\ta := tx.get(m.inodeKey(inode))\n\t\tif a == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tt := Attr{}\n\t\tm.parseAttr(a, &t)\n\t\tif t.Typ != TypeFile || t.Flags&(FlagImmutable|t.Flags&FlagAppend) != 0 || (flags == 0 && t.Parent > TrashInode) {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif !skipPermCheck {\n\t\t\tif st := m.Access(ctx, inode, MODE_MASK_W, &t); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t}\n\t\tif length == t.Length {\n\t\t\t*attr = t\n\t\t\treturn nil\n\t\t}\n\t\tdelta.length = int64(length) - int64(t.Length)\n\t\tdelta.space = align4K(length) - align4K(t.Length)\n\t\tif err := m.checkQuota(ctx, delta.space, 0, t.Uid, t.Gid, m.getParents(tx, inode, t.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tvar left, right = t.Length, length\n\t\tif left > right {\n\t\t\tright, left = left, right\n\t\t}\n\t\tif right/ChunkSize-left/ChunkSize > 1 {\n\t\t\tbuf := marshalSlice(0, 0, 0, 0, ChunkSize)\n\t\t\ttx.scan(m.chunkKey(inode, uint32(left/ChunkSize)+1), m.chunkKey(inode, uint32(right/ChunkSize)),\n\t\t\t\tfalse, func(k, v []byte) bool {\n\t\t\t\t\ttx.set(k, append(v, buf...))\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t}\n\t\tl := uint32(right - left)\n\t\tif right > (left/ChunkSize+1)*ChunkSize {\n\t\t\tl = ChunkSize - uint32(left%ChunkSize)\n\t\t}\n\t\ttx.append(m.chunkKey(inode, uint32(left/ChunkSize)), marshalSlice(uint32(left%ChunkSize), 0, 0, 0, l))\n\t\tif right > (left/ChunkSize+1)*ChunkSize && right%ChunkSize > 0 {\n\t\t\ttx.append(m.chunkKey(inode, uint32(right/ChunkSize)), marshalSlice(0, 0, 0, 0, uint32(right%ChunkSize)))\n\t\t}\n\t\tt.Length = length\n\t\tnow := time.Now()\n\t\tt.Mtime = now.Unix()\n\t\tt.Mtimensec = uint32(now.Nanosecond())\n\t\tt.Ctime = now.Unix()\n\t\tt.Ctimensec = uint32(now.Nanosecond())\n\t\ttx.set(m.inodeKey(inode), m.marshal(&t))\n\t\t*attr = t\n\t\treturn nil\n\t}, inode))\n}\n\nfunc (m *kvMeta) doFallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, delta *dirStat, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\t*delta = dirStat{}\n\t\ta := tx.get(m.inodeKey(inode))\n\t\tif a == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tt := Attr{}\n\t\tm.parseAttr(a, &t)\n\t\tif t.Typ == TypeFIFO {\n\t\t\treturn syscall.EPIPE\n\t\t}\n\t\tif t.Typ != TypeFile || (t.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W, &t); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (t.Flags&FlagAppend) != 0 && (mode&^fallocKeepSize) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tlength := t.Length\n\t\tif off+size > t.Length {\n\t\t\tif mode&fallocKeepSize == 0 {\n\t\t\t\tlength = off + size\n\t\t\t}\n\t\t}\n\n\t\told := t.Length\n\t\tdelta.length = int64(length) - int64(t.Length)\n\t\tdelta.space = align4K(length) - align4K(t.Length)\n\t\tif err := m.checkQuota(ctx, delta.space, 0, t.Uid, t.Gid, m.getParents(tx, inode, t.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tt.Length = length\n\t\tnow := time.Now()\n\t\tt.Mtime = now.Unix()\n\t\tt.Mtimensec = uint32(now.Nanosecond())\n\t\tt.Ctime = now.Unix()\n\t\tt.Ctimensec = uint32(now.Nanosecond())\n\t\ttx.set(m.inodeKey(inode), m.marshal(&t))\n\t\tif mode&(fallocZeroRange|fallocPunchHole) != 0 && off < old {\n\t\t\toff, size := off, size\n\t\t\tif off+size > old {\n\t\t\t\tsize = old - off\n\t\t\t}\n\t\t\tfor size > 0 {\n\t\t\t\tindx := uint32(off / ChunkSize)\n\t\t\t\tcoff := off % ChunkSize\n\t\t\t\tl := size\n\t\t\t\tif coff+size > ChunkSize {\n\t\t\t\t\tl = ChunkSize - coff\n\t\t\t\t}\n\t\t\t\ttx.append(m.chunkKey(inode, indx), marshalSlice(uint32(coff), 0, 0, 0, uint32(l)))\n\t\t\t\toff += l\n\t\t\t\tsize -= l\n\t\t\t}\n\t\t}\n\t\t*attr = t\n\t\treturn nil\n\t}, inode))\n}\n\nfunc (m *kvMeta) doReadlink(ctx Context, inode Ino, noatime bool) (atime int64, target []byte, err error) {\n\tif noatime {\n\t\ttarget, err = m.get(m.symKey(inode))\n\t\treturn\n\t}\n\n\tattr := &Attr{}\n\tnow := time.Now()\n\terr = m.txn(ctx, func(tx *kvTxn) error {\n\t\trs := tx.gets(m.inodeKey(inode), m.symKey(inode))\n\t\tif rs[0] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(rs[0], attr)\n\t\tif attr.Typ != TypeSymlink {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif rs[1] == nil {\n\t\t\treturn syscall.EIO\n\t\t}\n\t\ttarget = rs[1]\n\t\tif !m.atimeNeedsUpdate(attr, now) {\n\t\t\tatime = attr.Atime*int64(time.Second) + int64(attr.Atimensec)\n\t\t\treturn nil\n\t\t}\n\t\tattr.Atime = now.Unix()\n\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\tatime = now.UnixNano()\n\t\ttx.set(m.inodeKey(inode), m.marshal(attr))\n\t\treturn nil\n\t}, inode)\n\treturn\n}\n\nfunc (m *kvMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode, cumask uint16, path string, inode *Ino, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tvar pattr Attr\n\t\trs := tx.gets(m.inodeKey(parent), m.entryKey(parent, name))\n\t\tif rs[0] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(rs[0], &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pattr.Flags & FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif (pattr.Flags & FlagSkipTrash) != 0 {\n\t\t\tattr.Flags |= FlagSkipTrash\n\t\t}\n\n\t\tbuf := rs[1]\n\t\tvar foundIno Ino\n\t\tvar foundType uint8\n\t\tif buf != nil {\n\t\t\tfoundType, foundIno = m.parseEntry(buf)\n\t\t} else if m.conf.CaseInsensi {\n\t\t\tif entry := m.resolveCase(ctx, parent, name); entry != nil {\n\t\t\t\tfoundType, foundIno = entry.Attr.Typ, entry.Inode\n\t\t\t}\n\t\t}\n\t\tif foundIno != 0 {\n\t\t\tif _type == TypeFile || _type == TypeDirectory {\n\t\t\t\ta := tx.get(m.inodeKey(foundIno))\n\t\t\t\tif a != nil {\n\t\t\t\t\tm.parseAttr(a, attr)\n\t\t\t\t} else {\n\t\t\t\t\t*attr = Attr{Typ: foundType, Parent: parent} // corrupt entry\n\t\t\t\t}\n\t\t\t\t*inode = foundIno\n\t\t\t}\n\t\t\treturn syscall.EEXIST\n\t\t} else if parent == TrashInode { // user's inode is allocated by prefetch, trash inode is allocated on demand\n\t\t\tkey := m.counterKey(\"nextTrash\")\n\t\t\tnext := tx.incrBy(key, 1)\n\t\t\t*inode = TrashInode + Ino(next)\n\t\t}\n\n\t\tmode &= 07777\n\t\tif pattr.DefaultACL != aclAPI.None && _type != TypeSymlink {\n\t\t\t// inherit default acl\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tattr.DefaultACL = pattr.DefaultACL\n\t\t\t}\n\n\t\t\t// set access acl by parent's default acl\n\t\t\trule, err := m.getACL(tx, pattr.DefaultACL)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif rule.IsMinimal() {\n\t\t\t\t// simple acl as default\n\t\t\t\tattr.Mode = mode & (0xFE00 | rule.GetMode())\n\t\t\t} else {\n\t\t\t\tcRule := rule.ChildAccessACL(mode)\n\t\t\t\tid, err := m.insertACL(tx, cRule)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tattr.AccessACL = id\n\t\t\t\tattr.Mode = (mode & 0xFE00) | cRule.GetMode()\n\t\t\t}\n\t\t} else {\n\t\t\tattr.Mode = mode & ^cumask\n\t\t}\n\n\t\tvar updateParent bool\n\t\tnow := time.Now()\n\t\tif parent != TrashInode {\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tpattr.Nlink++\n\t\t\t\tif m.conf.SkipDirNlink <= 0 || tx.retry < m.conf.SkipDirNlink {\n\t\t\t\t\tupdateParent = true\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Warnf(\"Skip updating nlink of directory %d to reduce conflict\", parent)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif updateParent || now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {\n\t\t\t\tpattr.Mtime = now.Unix()\n\t\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\t\tpattr.Ctime = now.Unix()\n\t\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tupdateParent = true\n\t\t\t}\n\t\t}\n\t\tattr.Atime = now.Unix()\n\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\tattr.Mtime = now.Unix()\n\t\tattr.Mtimensec = uint32(now.Nanosecond())\n\t\tattr.Ctime = now.Unix()\n\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\tif ctx.Value(CtxKey(\"behavior\")) == \"Hadoop\" || runtime.GOOS == \"darwin\" {\n\t\t\tattr.Gid = pattr.Gid\n\t\t} else if runtime.GOOS == \"linux\" && pattr.Mode&02000 != 0 {\n\t\t\tattr.Gid = pattr.Gid\n\t\t\tif _type == TypeDirectory {\n\t\t\t\tattr.Mode |= 02000\n\t\t\t} else if attr.Mode&02010 == 02010 && ctx.Uid() != 0 {\n\t\t\t\tvar found bool\n\t\t\t\tfor _, gid := range ctx.Gids() {\n\t\t\t\t\tif gid == pattr.Gid {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tattr.Mode &= ^uint16(02000)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\ttx.set(m.entryKey(parent, name), m.packEntry(_type, *inode))\n\t\tif updateParent {\n\t\t\ttx.set(m.inodeKey(parent), m.marshal(&pattr))\n\t\t}\n\t\ttx.set(m.inodeKey(*inode), m.marshal(attr))\n\t\tif _type == TypeSymlink {\n\t\t\ttx.set(m.symKey(*inode), []byte(path))\n\t\t}\n\t\tif _type == TypeDirectory {\n\t\t\ttx.set(m.dirStatKey(*inode), m.packDirStat(&dirStat{}))\n\t\t}\n\t\treturn nil\n\t}, parent))\n}\n\nfunc (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno {\n\tvar trash Ino\n\tif !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\n\tif attr == nil {\n\t\tattr = &Attr{}\n\t}\n\tvar _type uint8\n\tvar inode Ino\n\tvar opened bool\n\tvar newSpace, newInode int64\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\topened = false\n\t\t*attr = Attr{}\n\t\tnewSpace, newInode = 0, 0\n\t\tbuf := tx.get(m.entryKey(parent, name))\n\t\tif buf == nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parent, name); e != nil {\n\t\t\t\tname = string(e.Name)\n\t\t\t\tbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t}\n\t\t}\n\t\tif buf == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\t_type, inode = m.parseEntry(buf)\n\t\tif _type == TypeDirectory {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tkeys := [][]byte{m.inodeKey(parent), m.inodeKey(inode)}\n\t\tif trash > 0 {\n\t\t\tkeys = append(keys, m.entryKey(trash, m.trashEntry(parent, inode, name)))\n\t\t}\n\t\trs := tx.gets(keys...)\n\t\tif rs[0] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr(rs[0], &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\topened = false\n\t\tnow := time.Now()\n\t\tif rs[1] != nil {\n\t\t\tm.parseAttr(rs[1], attr)\n\t\t\tif ctx.Uid() != 0 && pattr.Mode&01000 != 0 && ctx.Uid() != pattr.Uid && ctx.Uid() != attr.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t\tif (attr.Flags&FlagAppend) != 0 || (attr.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif (attr.Flags & FlagSkipTrash) != 0 {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tif trash > 0 && attr.Nlink > 1 && rs[2] != nil {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tattr.Ctime = now.Unix()\n\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tif trash == 0 {\n\t\t\t\tattr.Nlink--\n\t\t\t\tif _type == TypeFile && attr.Nlink == 0 && m.sid > 0 {\n\t\t\t\t\topened = m.of.IsOpen(inode)\n\t\t\t\t}\n\t\t\t} else if attr.Parent > 0 {\n\t\t\t\tattr.Parent = trash\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", inode, parent, name)\n\t\t\ttrash = 0\n\t\t}\n\n\t\tdefer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }()\n\t\tvar updateParent bool\n\t\tif !parent.IsTrash() && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {\n\t\t\tpattr.Mtime = now.Unix()\n\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tpattr.Ctime = now.Unix()\n\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tupdateParent = true\n\t\t}\n\n\t\ttx.delete(m.entryKey(parent, name))\n\t\tif updateParent {\n\t\t\ttx.set(m.inodeKey(parent), m.marshal(&pattr))\n\t\t}\n\t\tif attr.Nlink > 0 {\n\t\t\ttx.set(m.inodeKey(inode), m.marshal(attr))\n\t\t\tif trash > 0 {\n\t\t\t\ttx.set(m.entryKey(trash, m.trashEntry(parent, inode, name)), buf)\n\t\t\t\tif attr.Parent == 0 {\n\t\t\t\t\ttx.incrBy(m.parentKey(inode, trash), 1)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif attr.Parent == 0 {\n\t\t\t\ttx.incrBy(m.parentKey(inode, parent), -1)\n\t\t\t}\n\t\t} else {\n\t\t\tswitch _type {\n\t\t\tcase TypeFile:\n\t\t\t\tif opened {\n\t\t\t\t\ttx.set(m.inodeKey(inode), m.marshal(attr))\n\t\t\t\t\ttx.set(m.sustainedKey(m.sid, inode), []byte{1})\n\t\t\t\t} else {\n\t\t\t\t\ttx.set(m.delfileKey(inode, attr.Length), m.packInt64(now.Unix()))\n\t\t\t\t\ttx.delete(m.inodeKey(inode))\n\t\t\t\t\tnewSpace, newInode = -align4K(attr.Length), -1\n\t\t\t\t}\n\t\t\tcase TypeSymlink:\n\t\t\t\ttx.delete(m.symKey(inode))\n\t\t\t\tfallthrough\n\t\t\tdefault:\n\t\t\t\ttx.delete(m.inodeKey(inode))\n\t\t\t\tnewSpace, newInode = -align4K(0), -1\n\t\t\t}\n\t\t\ttx.deleteKeys(m.xattrKey(inode, \"\"))\n\t\t\tif attr.Parent == 0 {\n\t\t\t\ttx.deleteKeys(m.fmtKey(\"A\", inode, \"P\"))\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}, parent)\n\tif err == nil && trash == 0 {\n\t\tif _type == TypeFile && attr.Nlink == 0 {\n\t\t\tm.fileDeleted(opened, parent.IsTrash(), inode, attr.Length)\n\t\t}\n\t\tm.updateStats(newSpace, newInode)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, newSpace, newInode)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *kvMeta) doBatchUnlink(ctx Context, parent Ino, entries []*Entry, delta *dirStat, skipCheckTrash ...bool) syscall.Errno {\n\tif len(entries) == 0 {\n\t\treturn 0\n\t}\n\n\t// Each entry averages ~6 tx operations, so batch size should be 10000/6\n\tmaxOps := 10000\n\tif m.Name() == \"etcd\" {\n\t\tmaxOps = 128\n\t}\n\tbatchNum := maxOps / 6\n\n\ttype entryInfo struct {\n\t\tname string\n\t\tinode Ino\n\t\ttyp uint8\n\t\ttrash Ino\n\t\tattr *Attr\n\t\ttrashName string\n\t\tbuf []byte\n\t}\n\ttype dNode struct {\n\t\topened bool\n\t\tlength uint64\n\t}\n\n\tfor len(entries) > 0 {\n\t\tbatchSize := batchNum\n\t\tif batchSize > len(entries) {\n\t\t\tbatchSize = len(entries)\n\t\t}\n\t\tbatch := entries[:batchSize]\n\t\tentries = entries[batchSize:]\n\n\t\tvar trash Ino\n\t\tif len(skipCheckTrash) == 0 || !skipCheckTrash[0] {\n\t\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t}\n\n\t\tvar entryInfos []*entryInfo\n\t\tvar batchDirLength, batchDirSpace, batchDirInodes int64\n\t\tvar batchFsSpace, batchFsInodes int64\n\t\tvar deltas ugQuotaDeltas\n\t\tvar delNodes map[Ino]*dNode\n\n\t\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\t\tbatchDirLength, batchDirSpace, batchDirInodes = 0, 0, 0\n\t\t\tbatchFsSpace, batchFsInodes = 0, 0\n\t\t\tdeltas = make(ugQuotaDeltas)\n\t\t\tdelNodes = make(map[Ino]*dNode)\n\t\t\tpbuf := tx.get(m.inodeKey(parent))\n\t\t\tif pbuf == nil {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t\tvar pattr Attr\n\t\t\tm.parseAttr(pbuf, &pattr)\n\t\t\tif pattr.Typ != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\t\treturn st\n\t\t\t}\n\t\t\tif (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\n\t\t\tentryInfos = make([]*entryInfo, 0, len(batch))\n\t\t\tnow := time.Now()\n\t\t\tkeys := make([][]byte, 0, len(batch))\n\t\t\tfor _, entry := range batch {\n\t\t\t\tkeys = append(keys, m.entryKey(parent, string(entry.Name)))\n\t\t\t}\n\t\t\tvals := tx.gets(keys...)\n\t\t\tfor idx, entry := range batch {\n\t\t\t\tif vals[idx] == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\ttyp, ino := m.parseEntry(vals[idx])\n\t\t\t\tif ino != entry.Inode || typ == TypeDirectory || (entry.Attr != nil && typ != entry.Attr.Typ) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tinfo := entryInfo{\n\t\t\t\t\tname: string(entry.Name),\n\t\t\t\t\tinode: ino,\n\t\t\t\t\ttyp: typ,\n\t\t\t\t\ttrash: trash,\n\t\t\t\t\tbuf: vals[idx],\n\t\t\t\t}\n\t\t\t\tentryInfos = append(entryInfos, &info)\n\t\t\t}\n\n\t\t\t// Collect unique inodes\n\t\t\tinodesSet := make(map[Ino]struct{}, len(entryInfos))\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif _, ok := inodesSet[info.inode]; !ok {\n\t\t\t\t\tinodesSet[info.inode] = struct{}{}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Load inode attrs for all distinct inodes\n\t\t\tif len(inodesSet) > 0 {\n\t\t\t\tinodesList := make([]Ino, 0, len(inodesSet))\n\t\t\t\tkeys := make([][]byte, 0, len(inodesSet))\n\t\t\t\tfor ino := range inodesSet {\n\t\t\t\t\tinodesList = append(inodesList, ino)\n\t\t\t\t\tkeys = append(keys, m.inodeKey(ino))\n\t\t\t\t}\n\t\t\t\trs := tx.gets(keys...)\n\t\t\t\tnodeMap := make(map[Ino]*Attr, len(inodesList))\n\t\t\t\tfor i, v := range rs {\n\t\t\t\t\tif v == nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tvar a Attr\n\t\t\t\t\tm.parseAttr(v, &a)\n\t\t\t\t\tnodeMap[inodesList[i]] = &a\n\t\t\t\t}\n\n\t\t\t\t// Iterate all target entries, apply basic checks and build info\n\t\t\t\tfor _, info := range entryInfos {\n\t\t\t\t\tattr, ok := nodeMap[info.inode]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t\tinfo.attr = nil\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif ctx.Uid() != 0 && pattr.Mode&01000 != 0 && ctx.Uid() != pattr.Uid && ctx.Uid() != attr.Uid {\n\t\t\t\t\t\treturn syscall.EACCES\n\t\t\t\t\t}\n\t\t\t\t\tif (attr.Flags&FlagAppend) != 0 || (attr.Flags&FlagImmutable) != 0 {\n\t\t\t\t\t\treturn syscall.EPERM\n\t\t\t\t\t}\n\t\t\t\t\tif (attr.Flags & FlagSkipTrash) != 0 {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t}\n\t\t\t\t\tinfo.attr = attr\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Check trash entries for hard links\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif info.attr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif info.trash > 0 && info.attr.Nlink > 1 {\n\t\t\t\t\tinfo.trashName = m.trashEntry(parent, info.inode, info.name)\n\t\t\t\t\ttrashEntryKey := m.entryKey(info.trash, info.trashName)\n\t\t\t\t\tif tx.get(trashEntryKey) != nil {\n\t\t\t\t\t\tinfo.trash = 0\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Update ctime\n\t\t\t\tinfo.attr.Ctime = now.Unix()\n\t\t\t\tinfo.attr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tif info.trash > 0 && info.attr.Parent > 0 {\n\t\t\t\t\tinfo.attr.Parent = info.trash\n\t\t\t\t}\n\t\t\t\tif info.trash == 0 && info.attr.Nlink > 0 {\n\t\t\t\t\tinfo.attr.Nlink--\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Check opened status for all inodes with Nlink == 0 after all decrements\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\tif info.attr != nil && info.trash == 0 && info.attr.Nlink == 0 && info.typ == TypeFile {\n\t\t\t\t\topened := false\n\t\t\t\t\tif m.sid > 0 {\n\t\t\t\t\t\topened = m.of.IsOpen(info.inode)\n\t\t\t\t\t}\n\t\t\t\t\tdelNodes[info.inode] = &dNode{opened, info.attr.Length}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar updateParent bool\n\t\t\tif !parent.IsTrash() && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {\n\t\t\t\tpattr.Mtime = now.Unix()\n\t\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\t\tpattr.Ctime = now.Unix()\n\t\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tupdateParent = true\n\t\t\t}\n\n\t\t\tnowUnix := now.Unix()\n\t\t\tvisited := make(map[Ino]bool)\n\t\t\tvisited[0] = true // skip dummyNode\n\n\t\t\tfor _, info := range entryInfos {\n\t\t\t\ttx.delete(m.entryKey(parent, info.name))\n\t\t\t\tif info.attr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif info.typ == TypeFile {\n\t\t\t\t\tbatchDirLength -= int64(info.attr.Length)\n\t\t\t\t\tbatchDirSpace -= align4K(info.attr.Length)\n\t\t\t\t} else {\n\t\t\t\t\tbatchDirSpace -= align4K(0)\n\t\t\t\t}\n\t\t\t\tbatchDirInodes--\n\t\t\t\tif !visited[info.inode] {\n\t\t\t\t\tif info.attr.Nlink > 0 {\n\t\t\t\t\t\ttx.set(m.inodeKey(info.inode), m.marshal(info.attr))\n\t\t\t\t\t} else {\n\t\t\t\t\t\tswitch info.typ {\n\t\t\t\t\t\tcase TypeFile:\n\t\t\t\t\t\t\tif dnode, ok := delNodes[info.inode]; ok && dnode.opened {\n\t\t\t\t\t\t\t\ttx.set(m.inodeKey(info.inode), m.marshal(info.attr))\n\t\t\t\t\t\t\t\ttx.set(m.sustainedKey(m.sid, info.inode), []byte{1})\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\ttx.set(m.delfileKey(info.inode, info.attr.Length), m.packInt64(nowUnix))\n\t\t\t\t\t\t\t\ttx.delete(m.inodeKey(info.inode))\n\t\t\t\t\t\t\t\tbatchFsSpace -= align4K(info.attr.Length)\n\t\t\t\t\t\t\t\tbatchFsInodes--\n\t\t\t\t\t\t\t\tdeltas.add(&ugQuotaDelta{\n\t\t\t\t\t\t\t\t\tUid: info.attr.Uid,\n\t\t\t\t\t\t\t\t\tGid: info.attr.Gid,\n\t\t\t\t\t\t\t\t\tSpace: -align4K(info.attr.Length),\n\t\t\t\t\t\t\t\t\tInodes: -1,\n\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase TypeSymlink:\n\t\t\t\t\t\t\ttx.delete(m.symKey(info.inode))\n\t\t\t\t\t\t\tfallthrough\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\ttx.delete(m.inodeKey(info.inode))\n\t\t\t\t\t\t\tbatchFsSpace -= align4K(0)\n\t\t\t\t\t\t\tbatchFsInodes--\n\t\t\t\t\t\t\tdeltas.add(&ugQuotaDelta{\n\t\t\t\t\t\t\t\tUid: info.attr.Uid,\n\t\t\t\t\t\t\t\tGid: info.attr.Gid,\n\t\t\t\t\t\t\t\tSpace: -align4K(0),\n\t\t\t\t\t\t\t\tInodes: -1,\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Delete xattrs and parent keys\n\t\t\t\t\t\ttx.deleteKeys(m.xattrKey(info.inode, \"\"))\n\t\t\t\t\t\tif info.attr.Parent == 0 {\n\t\t\t\t\t\t\ttx.deleteKeys(m.fmtKey(\"A\", info.inode, \"P\"))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tm.of.InvalidateChunk(info.inode, invalidateAttrOnly)\n\t\t\t\t}\n\t\t\t\tvisited[info.inode] = true\n\n\t\t\t\tif info.trash > 0 {\n\t\t\t\t\tif info.trashName == \"\" {\n\t\t\t\t\t\tinfo.trashName = m.trashEntry(parent, info.inode, info.name)\n\t\t\t\t\t}\n\t\t\t\t\ttx.set(m.entryKey(info.trash, info.trashName), info.buf)\n\t\t\t\t\tif info.attr.Parent == 0 {\n\t\t\t\t\t\ttx.incrBy(m.parentKey(info.inode, info.trash), 1)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif info.attr.Parent == 0 && info.attr.Nlink > 0 {\n\t\t\t\t\ttx.incrBy(m.parentKey(info.inode, parent), -1)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Update parent directory if needed\n\t\t\tif updateParent {\n\t\t\t\ttx.set(m.inodeKey(parent), m.marshal(&pattr))\n\t\t\t}\n\n\t\t\treturn nil\n\t\t}, parent)\n\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\n\t\t// Outside of transaction: trigger data deletion callbacks\n\t\tfor inode, info := range delNodes {\n\t\t\tm.fileDeleted(info.opened, parent.IsTrash(), inode, info.length)\n\t\t}\n\n\t\tdelta.length += batchDirLength\n\t\tdelta.space += batchDirSpace\n\t\tdelta.inodes += batchDirInodes\n\t\tm.updateStats(batchFsSpace, batchFsInodes)\n\t\tfor _, q := range deltas {\n\t\t\tm.updateUserGroupStat(ctx, q.Uid, q.Gid, q.Space, q.Inodes)\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *kvMeta) doRmdir(ctx Context, parent Ino, name string, pinode *Ino, oldAttr *Attr, skipCheckTrash ...bool) syscall.Errno {\n\tvar trash Ino\n\tvar attr Attr\n\tif !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {\n\t\tif st := m.checkTrash(parent, &trash); st != 0 {\n\t\t\treturn st\n\t\t}\n\t}\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\tbuf := tx.get(m.entryKey(parent, name))\n\t\tif buf == nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parent, name); e != nil {\n\t\t\t\tname = string(e.Name)\n\t\t\t\tbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t}\n\t\t}\n\t\tif buf == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\t_type, inode := m.parseEntry(buf)\n\t\tif _type != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pinode != nil {\n\t\t\t*pinode = inode\n\t\t}\n\t\trs := tx.gets(m.inodeKey(parent), m.inodeKey(inode))\n\t\tif rs[0] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr(rs[0], &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif tx.exist(m.entryKey(inode, \"\")) {\n\t\t\treturn syscall.ENOTEMPTY\n\t\t}\n\n\t\tnow := time.Now()\n\t\tif rs[1] != nil {\n\t\t\tm.parseAttr(rs[1], &attr)\n\t\t\tif oldAttr != nil {\n\t\t\t\t*oldAttr = attr\n\t\t\t}\n\t\t\tif ctx.Uid() != 0 && pattr.Mode&01000 != 0 && ctx.Uid() != pattr.Uid && ctx.Uid() != attr.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t\tif (attr.Flags & FlagSkipTrash) != 0 {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tif trash > 0 {\n\t\t\t\tattr.Ctime = now.Unix()\n\t\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\tattr.Parent = trash\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", inode, parent, name)\n\t\t\ttrash = 0\n\t\t}\n\t\tpattr.Nlink--\n\t\tvar updateParent bool\n\t\tif m.conf.SkipDirNlink <= 0 || tx.retry < m.conf.SkipDirNlink {\n\t\t\tupdateParent = true\n\t\t} else {\n\t\t\tlogger.Warnf(\"Skip updating nlink of directory %d to reduce conflict\", parent)\n\t\t}\n\t\tif updateParent || now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {\n\t\t\tpattr.Mtime = now.Unix()\n\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tpattr.Ctime = now.Unix()\n\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tupdateParent = true\n\t\t}\n\n\t\tif !parent.IsTrash() && updateParent {\n\t\t\ttx.set(m.inodeKey(parent), m.marshal(&pattr))\n\t\t}\n\t\ttx.delete(m.entryKey(parent, name))\n\t\ttx.delete(m.dirStatKey(inode))\n\t\ttx.delete(m.dirQuotaKey(inode))\n\t\tif trash > 0 {\n\t\t\ttx.set(m.inodeKey(inode), m.marshal(&attr))\n\t\t\ttx.set(m.entryKey(trash, m.trashEntry(parent, inode, name)), buf)\n\t\t} else {\n\t\t\ttx.delete(m.inodeKey(inode))\n\t\t\ttx.deleteKeys(m.xattrKey(inode, \"\"))\n\t\t}\n\t\treturn nil\n\t}, parent)\n\tif err == nil && trash == 0 {\n\t\tm.updateStats(-align4K(0), -1)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, -align4K(0), -1)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *kvMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode, tInode *Ino, attr, tAttr *Attr) syscall.Errno {\n\tvar trash Ino\n\tif st := m.checkTrash(parentDst, &trash); st != 0 {\n\t\treturn st\n\t}\n\texchange := flags == RenameExchange\n\tvar opened bool\n\tvar dino Ino\n\tvar dtyp uint8\n\tvar tattr Attr\n\tvar newSpace, newInode int64\n\tparentLocks := []Ino{parentDst}\n\tif !parentSrc.IsTrash() { // there should be no conflict if parentSrc is in trash, relax lock to accelerate `restore` subcommand\n\t\tparentLocks = append(parentLocks, parentSrc)\n\t}\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\topened = false\n\t\tdino, dtyp = 0, 0\n\t\ttattr = Attr{}\n\t\tnewSpace, newInode = 0, 0\n\t\tbuf := tx.get(m.entryKey(parentSrc, nameSrc))\n\t\tif buf == nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parentSrc, nameSrc); e != nil {\n\t\t\t\tnameSrc = string(e.Name)\n\t\t\t\tbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t}\n\t\t}\n\t\tif buf == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\ttyp, ino := m.parseEntry(buf)\n\t\tif parentSrc == parentDst && nameSrc == nameDst {\n\t\t\tif inode != nil {\n\t\t\t\t*inode = ino\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\trs := tx.gets(m.inodeKey(parentSrc), m.inodeKey(parentDst), m.inodeKey(ino))\n\t\tif rs[0] == nil || rs[1] == nil || rs[2] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar sattr, dattr, iattr Attr\n\t\tm.parseAttr(rs[0], &sattr)\n\t\tif sattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif st := m.Access(ctx, parentSrc, MODE_MASK_W|MODE_MASK_X, &sattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tm.parseAttr(rs[1], &dattr)\n\t\tif dattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif flags&RenameRestore == 0 && dattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parentDst, MODE_MASK_W|MODE_MASK_X, &dattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\t// TODO: check parentDst is a subdir of source node\n\t\tif ino == parentDst || ino == dattr.Parent {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tm.parseAttr(rs[2], &iattr)\n\t\tif (sattr.Flags&FlagAppend) != 0 || (sattr.Flags&FlagImmutable) != 0 || (dattr.Flags&FlagImmutable) != 0 || (iattr.Flags&FlagAppend) != 0 || (iattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif parentSrc != parentDst && sattr.Mode&0o1000 != 0 && ctx.Uid() != 0 &&\n\t\t\tctx.Uid() != iattr.Uid && (ctx.Uid() != sattr.Uid || iattr.Typ == TypeDirectory) {\n\t\t\treturn syscall.EACCES\n\t\t}\n\n\t\tdbuf := tx.get(m.entryKey(parentDst, nameDst))\n\t\tif dbuf == nil && m.conf.CaseInsensi {\n\t\t\tif e := m.resolveCase(ctx, parentDst, nameDst); e != nil {\n\t\t\t\tif string(e.Name) != nameSrc || parentDst != parentSrc {\n\t\t\t\t\tnameDst = string(e.Name)\n\t\t\t\t\tdbuf = m.packEntry(e.Attr.Typ, e.Inode)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tvar supdate, dupdate bool\n\t\tnow := time.Now()\n\t\tif dbuf != nil {\n\t\t\tif flags&RenameNoReplace != 0 {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\tdtyp, dino = m.parseEntry(dbuf)\n\t\t\ta := tx.get(m.inodeKey(dino))\n\t\t\tif a == nil { // corrupt entry\n\t\t\t\tlogger.Warnf(\"no attribute for inode %d (%d, %s)\", dino, parentDst, nameDst)\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\tm.parseAttr(a, &tattr)\n\t\t\tif (tattr.Flags&FlagAppend) != 0 || (tattr.Flags&FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif (tattr.Flags & FlagSkipTrash) != 0 {\n\t\t\t\ttrash = 0\n\t\t\t}\n\t\t\ttattr.Ctime = now.Unix()\n\t\t\ttattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tif exchange {\n\t\t\t\tif parentSrc != parentDst {\n\t\t\t\t\tif dtyp == TypeDirectory {\n\t\t\t\t\t\ttattr.Parent = parentSrc\n\t\t\t\t\t\tdattr.Nlink--\n\t\t\t\t\t\tsattr.Nlink++\n\t\t\t\t\t\tif m.conf.SkipDirNlink <= 0 || tx.retry < m.conf.SkipDirNlink {\n\t\t\t\t\t\t\tsupdate, dupdate = true, true\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tlogger.Warnf(\"Skip updating nlink of directory %d,%d to reduce conflict\", parentSrc, parentDst)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if tattr.Parent > 0 {\n\t\t\t\t\t\ttattr.Parent = parentSrc\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if dino == ino {\n\t\t\t\treturn nil\n\t\t\t} else if typ == TypeDirectory && dtyp != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t} else if typ != TypeDirectory && dtyp == TypeDirectory {\n\t\t\t\treturn syscall.EISDIR\n\t\t\t} else {\n\t\t\t\tif dtyp == TypeDirectory {\n\t\t\t\t\tif tx.exist(m.entryKey(dino, \"\")) {\n\t\t\t\t\t\treturn syscall.ENOTEMPTY\n\t\t\t\t\t}\n\t\t\t\t\tdattr.Nlink--\n\t\t\t\t\tdupdate = true\n\t\t\t\t\tif trash > 0 {\n\t\t\t\t\t\ttattr.Parent = trash\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif trash == 0 {\n\t\t\t\t\t\ttattr.Nlink--\n\t\t\t\t\t\tif dtyp == TypeFile && tattr.Nlink == 0 && m.sid > 0 {\n\t\t\t\t\t\t\topened = m.of.IsOpen(dino)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tdefer func() { m.of.InvalidateChunk(dino, invalidateAttrOnly) }()\n\t\t\t\t\t} else if tattr.Parent > 0 {\n\t\t\t\t\t\ttattr.Parent = trash\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif ctx.Uid() != 0 && dattr.Mode&01000 != 0 && ctx.Uid() != dattr.Uid && ctx.Uid() != tattr.Uid {\n\t\t\t\treturn syscall.EACCES\n\t\t\t}\n\t\t} else {\n\t\t\tif exchange {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t}\n\t\tif ctx.Uid() != 0 && sattr.Mode&01000 != 0 && ctx.Uid() != sattr.Uid && ctx.Uid() != iattr.Uid {\n\t\t\treturn syscall.EACCES\n\t\t}\n\n\t\tif parentSrc != parentDst {\n\t\t\tif typ == TypeDirectory {\n\t\t\t\tiattr.Parent = parentDst\n\t\t\t\tsattr.Nlink--\n\t\t\t\tdattr.Nlink++\n\t\t\t\tif m.conf.SkipDirNlink <= 0 || tx.retry < m.conf.SkipDirNlink {\n\t\t\t\t\tsupdate, dupdate = true, true\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Warnf(\"Skip updating nlink of directory %d,%d to reduce conflict\", parentSrc, parentDst)\n\t\t\t\t}\n\t\t\t} else if iattr.Parent > 0 {\n\t\t\t\tiattr.Parent = parentDst\n\t\t\t}\n\t\t}\n\t\tif supdate || now.Sub(time.Unix(sattr.Mtime, int64(sattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {\n\t\t\tsattr.Mtime = now.Unix()\n\t\t\tsattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tsattr.Ctime = now.Unix()\n\t\t\tsattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tsupdate = true\n\t\t}\n\t\tif dupdate || now.Sub(time.Unix(dattr.Mtime, int64(dattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {\n\t\t\tdattr.Mtime = now.Unix()\n\t\t\tdattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tdattr.Ctime = now.Unix()\n\t\t\tdattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tdupdate = true\n\t\t}\n\t\tiattr.Ctime = now.Unix()\n\t\tiattr.Ctimensec = uint32(now.Nanosecond())\n\t\tif inode != nil {\n\t\t\t*inode = ino\n\t\t}\n\t\tif attr != nil {\n\t\t\t*attr = iattr\n\t\t}\n\t\tif dino > 0 {\n\t\t\t*tInode = dino\n\t\t\t*tAttr = tattr\n\t\t}\n\n\t\tif exchange { // dino > 0\n\t\t\ttx.set(m.entryKey(parentSrc, nameSrc), dbuf)\n\t\t\ttx.set(m.inodeKey(dino), m.marshal(&tattr))\n\t\t\tif parentSrc != parentDst && tattr.Parent == 0 {\n\t\t\t\ttx.incrBy(m.parentKey(dino, parentSrc), 1)\n\t\t\t\ttx.incrBy(m.parentKey(dino, parentDst), -1)\n\t\t\t}\n\t\t} else {\n\t\t\ttx.delete(m.entryKey(parentSrc, nameSrc))\n\t\t\tif dino > 0 {\n\t\t\t\tif trash > 0 {\n\t\t\t\t\ttx.set(m.inodeKey(dino), m.marshal(&tattr))\n\t\t\t\t\ttx.set(m.entryKey(trash, m.trashEntry(parentDst, dino, nameDst)), dbuf)\n\t\t\t\t\tif tattr.Parent == 0 {\n\t\t\t\t\t\ttx.incrBy(m.parentKey(dino, trash), 1)\n\t\t\t\t\t\ttx.incrBy(m.parentKey(dino, parentDst), -1)\n\t\t\t\t\t}\n\t\t\t\t} else if dtyp != TypeDirectory && tattr.Nlink > 0 {\n\t\t\t\t\ttx.set(m.inodeKey(dino), m.marshal(&tattr))\n\t\t\t\t\tif tattr.Parent == 0 {\n\t\t\t\t\t\ttx.incrBy(m.parentKey(dino, parentDst), -1)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif dtyp == TypeFile {\n\t\t\t\t\t\tif opened {\n\t\t\t\t\t\t\ttx.set(m.inodeKey(dino), m.marshal(&tattr))\n\t\t\t\t\t\t\ttx.set(m.sustainedKey(m.sid, dino), []byte{1})\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\ttx.set(m.delfileKey(dino, tattr.Length), m.packInt64(now.Unix()))\n\t\t\t\t\t\t\ttx.delete(m.inodeKey(dino))\n\t\t\t\t\t\t\tnewSpace, newInode = -align4K(tattr.Length), -1\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif dtyp == TypeSymlink {\n\t\t\t\t\t\t\ttx.delete(m.symKey(dino))\n\t\t\t\t\t\t}\n\t\t\t\t\t\ttx.delete(m.inodeKey(dino))\n\t\t\t\t\t\tnewSpace, newInode = -align4K(0), -1\n\t\t\t\t\t}\n\t\t\t\t\ttx.deleteKeys(m.xattrKey(dino, \"\"))\n\t\t\t\t\tif tattr.Parent == 0 {\n\t\t\t\t\t\ttx.deleteKeys(m.fmtKey(\"A\", dino, \"P\"))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif dtyp == TypeDirectory {\n\t\t\t\t\ttx.delete(m.dirQuotaKey(dino))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif parentDst != parentSrc {\n\t\t\tif !parentSrc.IsTrash() && supdate {\n\t\t\t\ttx.set(m.inodeKey(parentSrc), m.marshal(&sattr))\n\t\t\t}\n\t\t\tif iattr.Parent == 0 {\n\t\t\t\ttx.incrBy(m.parentKey(ino, parentDst), 1)\n\t\t\t\ttx.incrBy(m.parentKey(ino, parentSrc), -1)\n\t\t\t}\n\t\t}\n\t\ttx.set(m.inodeKey(ino), m.marshal(&iattr))\n\t\ttx.set(m.entryKey(parentDst, nameDst), buf)\n\t\tif dupdate {\n\t\t\ttx.set(m.inodeKey(parentDst), m.marshal(&dattr))\n\t\t}\n\t\treturn nil\n\t}, parentLocks...)\n\tif err == nil && !exchange && trash == 0 {\n\t\tif dino > 0 && dtyp == TypeFile && tattr.Nlink == 0 {\n\t\t\tm.fileDeleted(opened, false, dino, tattr.Length)\n\t\t}\n\t\tm.updateStats(newSpace, newInode)\n\t\tm.updateUserGroupStat(ctx, tattr.Uid, tattr.Gid, newSpace, newInode)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *kvMeta) doLink(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\trs := tx.gets(m.inodeKey(parent), m.inodeKey(inode))\n\t\tif rs[0] == nil || rs[1] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar pattr, iattr Attr\n\t\tm.parseAttr(rs[0], &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tif pattr.Flags&FlagImmutable != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tm.parseAttr(rs[1], &iattr)\n\t\tif iattr.Typ == TypeDirectory {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif (iattr.Flags&FlagAppend) != 0 || (iattr.Flags&FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tbuf := tx.get(m.entryKey(parent, name))\n\t\tif buf != nil || m.conf.CaseInsensi && m.resolveCase(ctx, parent, name) != nil {\n\t\t\treturn syscall.EEXIST\n\t\t}\n\n\t\tvar updateParent bool\n\t\tnow := time.Now()\n\t\tif now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {\n\t\t\tpattr.Mtime = now.Unix()\n\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tpattr.Ctime = now.Unix()\n\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\tupdateParent = true\n\t\t}\n\t\toldParent := iattr.Parent\n\t\tiattr.Parent = 0\n\t\tiattr.Ctime = now.Unix()\n\t\tiattr.Ctimensec = uint32(now.Nanosecond())\n\t\tiattr.Nlink++\n\t\ttx.set(m.entryKey(parent, name), m.packEntry(iattr.Typ, inode))\n\t\tif updateParent {\n\t\t\ttx.set(m.inodeKey(parent), m.marshal(&pattr))\n\t\t}\n\t\ttx.set(m.inodeKey(inode), m.marshal(&iattr))\n\t\tif oldParent > 0 {\n\t\t\ttx.incrBy(m.parentKey(inode, oldParent), 1)\n\t\t}\n\t\ttx.incrBy(m.parentKey(inode, parent), 1)\n\t\tif attr != nil {\n\t\t\t*attr = iattr\n\t\t}\n\t\treturn nil\n\t}, parent))\n}\n\nfunc (m *kvMeta) fillAttr(entries []*Entry) (err error) {\n\tif len(entries) == 0 {\n\t\treturn nil\n\t}\n\tvar keys = make([][]byte, len(entries))\n\tfor i, e := range entries {\n\t\tkeys[i] = m.inodeKey(e.Inode)\n\t}\n\tvar rs [][]byte\n\terr = m.client.simpleTxn(Background(), func(tx *kvTxn) error {\n\t\trs = tx.gets(keys...)\n\t\treturn nil\n\t}, 0)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor j, re := range rs {\n\t\tif re != nil {\n\t\t\tm.parseAttr(re, entries[j].Attr)\n\t\t\t// If `readdirplus` returns complete attributes, kernel may not invoke `GetAttr`. Therefore, we must also validate chunk cache here to prevent stale cache, which may lead to data corruption.\n\t\t\tm.of.Update(entries[j].Inode, entries[j].Attr)\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (m *kvMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry, limit int) syscall.Errno {\n\t// TODO: handle big directory\n\tvals, err := m.scanValues(ctx, m.entryKey(inode, \"\"), limit, nil)\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tprefix := len(m.entryKey(inode, \"\"))\n\tfor name, buf := range vals {\n\t\ttyp, ino := m.parseEntry(buf)\n\t\tif len(name) == prefix {\n\t\t\tlogger.Errorf(\"Corrupt entry with empty name: inode %d parent %d\", ino, inode)\n\t\t\tcontinue\n\t\t}\n\t\t*entries = append(*entries, &Entry{\n\t\t\tInode: ino,\n\t\t\tName: []byte(name)[prefix:],\n\t\t\tAttr: &Attr{Typ: typ},\n\t\t})\n\t}\n\n\tif plus != 0 && len(*entries) != 0 {\n\t\tif ctx.Canceled() {\n\t\t\treturn errno(ctx.Err())\n\t\t}\n\t\tbatchSize := 4096\n\t\tnEntries := len(*entries)\n\t\tif nEntries <= batchSize {\n\t\t\terr = m.fillAttr(*entries)\n\t\t} else {\n\t\t\tindexCh := make(chan []*Entry, 10)\n\t\t\tvar wg sync.WaitGroup\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tfor es := range indexCh {\n\t\t\t\t\t\tif e := m.fillAttr(es); e != nil {\n\t\t\t\t\t\t\terr = e\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\t\t\tfor i := 0; i < nEntries; i += batchSize {\n\t\t\t\tif i+batchSize > nEntries {\n\t\t\t\t\tindexCh <- (*entries)[i:]\n\t\t\t\t} else {\n\t\t\t\t\tindexCh <- (*entries)[i : i+batchSize]\n\t\t\t\t}\n\t\t\t}\n\t\t\tclose(indexCh)\n\t\t\twg.Wait()\n\t\t}\n\t\tif err != nil {\n\t\t\treturn errno(err)\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (m *kvMeta) doDeleteSustainedInode(sid uint64, inode Ino) error {\n\tvar attr Attr\n\tvar newSpace int64\n\terr := m.txn(Background(), func(tx *kvTxn) error {\n\t\tnewSpace = 0\n\t\ta := tx.get(m.inodeKey(inode))\n\t\tif a == nil {\n\t\t\treturn nil\n\t\t}\n\t\tm.parseAttr(a, &attr)\n\t\tnewSpace = -align4K(attr.Length)\n\t\ttx.set(m.delfileKey(inode, attr.Length), m.packInt64(time.Now().Unix()))\n\t\ttx.delete(m.inodeKey(inode))\n\t\ttx.delete(m.sustainedKey(sid, inode))\n\t\treturn nil\n\t}, inode)\n\tif err == nil && newSpace < 0 {\n\t\tm.updateStats(newSpace, -1)\n\t\tm.tryDeleteFileData(inode, attr.Length, false)\n\t\tm.updateUserGroupStat(Background(), attr.Uid, attr.Gid, newSpace, 0)\n\t}\n\treturn err\n}\n\nfunc (m *kvMeta) doRead(ctx Context, inode Ino, indx uint32) ([]*slice, syscall.Errno) {\n\tval, err := m.get(m.chunkKey(inode, indx))\n\tif err != nil {\n\t\treturn nil, errno(err)\n\t}\n\treturn readSliceBuf(val), 0\n}\n\nfunc (m *kvMeta) doList(ctx Context, inode Ino) ([]*slice, syscall.Errno) {\n\tvals, err := m.scanValues(ctx, m.fmtKey(\"A\", inode, \"C\"), -1, nil)\n\tif err != nil {\n\t\tlogger.Warnf(\"list of inode %d: %s\", inode, err)\n\t\treturn nil, errno(err)\n\t}\n\tvar slices []*slice\n\tfor _, v := range vals {\n\t\tss := readSliceBuf(v)\n\t\tif ss == nil {\n\t\t\tcontinue\n\t\t}\n\t\tslices = append(slices, ss...)\n\t}\n\n\treturn slices, 0\n}\n\nfunc (m *kvMeta) doWrite(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time, numSlices *int, delta *dirStat, attr *Attr) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\t*delta = dirStat{}\n\t\t*attr = Attr{}\n\t\trs := tx.gets(m.inodeKey(inode), m.chunkKey(inode, indx))\n\t\tif rs[0] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(rs[0], attr)\n\t\tif attr.Typ != TypeFile {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif len(rs[1])%sliceBytes != 0 {\n\t\t\tlogger.Errorf(\"Invalid chunk value for inode %d indx %d: %d\", inode, indx, len(rs[1]))\n\t\t\treturn syscall.EIO\n\t\t}\n\t\tnewleng := uint64(indx)*ChunkSize + uint64(off) + uint64(slice.Len)\n\t\tif newleng > attr.Length {\n\t\t\tdelta.length = int64(newleng - attr.Length)\n\t\t\tdelta.space = align4K(newleng) - align4K(attr.Length)\n\t\t\tattr.Length = newleng\n\t\t}\n\t\tif err := m.checkQuota(ctx, delta.space, 0, attr.Uid, attr.Gid, m.getParents(tx, inode, attr.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tnow := time.Now()\n\t\tattr.Mtime = mtime.Unix()\n\t\tattr.Mtimensec = uint32(mtime.Nanosecond())\n\t\tattr.Ctime = now.Unix()\n\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\tval := marshalSlice(off, slice.Id, slice.Size, slice.Off, slice.Len)\n\t\tfor i := 0; i < len(rs[1]); i += sliceBytes {\n\t\t\tif bytes.Equal(rs[1][i:i+sliceBytes], val) {\n\t\t\t\tlogger.Warnf(\"Write same slice for inode %d indx %d sliceId %d\", inode, indx, slice.Id)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tval = append(rs[1], val...)\n\t\ttx.set(m.inodeKey(inode), m.marshal(attr))\n\t\ttx.set(m.chunkKey(inode, indx), val)\n\t\t*numSlices = len(val) / sliceBytes\n\t\treturn nil\n\t}, inode))\n}\n\nfunc (m *kvMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno {\n\tdefer m.timeit(\"CopyFileRange\", time.Now())\n\tvar newLength, newSpace int64\n\tf := m.of.find(fout)\n\tif f != nil {\n\t\tf.Lock()\n\t\tdefer f.Unlock()\n\t}\n\tdefer func() { m.of.InvalidateChunk(fout, invalidateAllChunks) }()\n\tvar sattr, attr Attr\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\tnewLength, newSpace = 0, 0\n\t\trs := tx.gets(m.inodeKey(fin), m.inodeKey(fout))\n\t\tif rs[0] == nil || rs[1] == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tsattr = Attr{}\n\t\tm.parseAttr(rs[0], &sattr)\n\t\tif sattr.Typ != TypeFile {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif offIn >= sattr.Length {\n\t\t\tif copied != nil {\n\t\t\t\t*copied = 0\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tsize := size\n\t\tif offIn+size > sattr.Length {\n\t\t\tsize = sattr.Length - offIn\n\t\t}\n\t\tattr = Attr{}\n\t\tm.parseAttr(rs[1], &attr)\n\t\tif attr.Typ != TypeFile {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif (attr.Flags&FlagImmutable) != 0 || (attr.Flags&FlagAppend) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tnewleng := offOut + size\n\t\tif newleng > attr.Length {\n\t\t\tnewLength = int64(newleng - attr.Length)\n\t\t\tnewSpace = align4K(newleng) - align4K(attr.Length)\n\t\t\tattr.Length = newleng\n\t\t}\n\t\tif err := m.checkQuota(ctx, newSpace, 0, attr.Uid, attr.Gid, m.getParents(tx, fout, attr.Parent)...); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tnow := time.Now()\n\t\tattr.Mtime = now.Unix()\n\t\tattr.Mtimensec = uint32(now.Nanosecond())\n\t\tattr.Ctime = now.Unix()\n\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\tif outLength != nil {\n\t\t\t*outLength = attr.Length\n\t\t}\n\n\t\tvals := make(map[string][]byte)\n\t\ttx.scan(m.chunkKey(fin, uint32(offIn/ChunkSize)), m.chunkKey(fin, uint32((offIn+size)/ChunkSize)+1),\n\t\t\tfalse, func(k, v []byte) bool {\n\t\t\t\tvals[string(k)] = v\n\t\t\t\treturn true\n\t\t\t})\n\t\tchunks := make(map[uint32][]*slice)\n\t\tfor indx := uint32(offIn / ChunkSize); indx <= uint32((offIn+size)/ChunkSize); indx++ {\n\t\t\tif v, ok := vals[string(m.chunkKey(fin, indx))]; ok {\n\t\t\t\tchunks[indx] = readSliceBuf(v)\n\t\t\t\tif chunks[indx] == nil {\n\t\t\t\t\treturn syscall.EIO\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcoff := offIn / ChunkSize * ChunkSize\n\t\tchunksMap := make(map[string][]byte)\n\t\tfor coff < offIn+size {\n\t\t\tif coff%ChunkSize != 0 {\n\t\t\t\tpanic(\"coff\")\n\t\t\t}\n\t\t\t// Add a zero chunk for hole\n\t\t\tss := append([]*slice{{len: ChunkSize}}, chunks[uint32(coff/ChunkSize)]...)\n\t\t\tcs := buildSlice(ss)\n\t\t\tfor _, s := range cs {\n\t\t\t\tpos := coff\n\t\t\t\tcoff += uint64(s.Len)\n\t\t\t\tif pos < offIn+size && pos+uint64(s.Len) > offIn {\n\t\t\t\t\tif pos < offIn {\n\t\t\t\t\t\tdec := offIn - pos\n\t\t\t\t\t\ts.Off += uint32(dec)\n\t\t\t\t\t\tpos += dec\n\t\t\t\t\t\ts.Len -= uint32(dec)\n\t\t\t\t\t}\n\t\t\t\t\tif pos+uint64(s.Len) > offIn+size {\n\t\t\t\t\t\tdec := pos + uint64(s.Len) - (offIn + size)\n\t\t\t\t\t\ts.Len -= uint32(dec)\n\t\t\t\t\t}\n\t\t\t\t\tdoff := pos - offIn + offOut\n\t\t\t\t\tindx := uint32(doff / ChunkSize)\n\t\t\t\t\tdpos := uint32(doff % ChunkSize)\n\t\t\t\t\tif dpos+s.Len > ChunkSize {\n\t\t\t\t\t\tchunksMap[string(m.chunkKey(fout, indx))] = append(chunksMap[string(m.chunkKey(fout, indx))], marshalSlice(dpos, s.Id, s.Size, s.Off, ChunkSize-dpos)...)\n\t\t\t\t\t\tif s.Id > 0 {\n\t\t\t\t\t\t\ttx.incrBy(m.sliceKey(s.Id, s.Size), 1)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tskip := ChunkSize - dpos\n\t\t\t\t\t\tchunksMap[string(m.chunkKey(fout, indx+1))] = append(chunksMap[string(m.chunkKey(fout, indx+1))], marshalSlice(0, s.Id, s.Size, s.Off+skip, s.Len-skip)...)\n\t\t\t\t\t\tif s.Id > 0 {\n\t\t\t\t\t\t\ttx.incrBy(m.sliceKey(s.Id, s.Size), 1)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tchunksMap[string(m.chunkKey(fout, indx))] = append(chunksMap[string(m.chunkKey(fout, indx))], marshalSlice(dpos, s.Id, s.Size, s.Off, s.Len)...)\n\t\t\t\t\t\tif s.Id > 0 {\n\t\t\t\t\t\t\ttx.incrBy(m.sliceKey(s.Id, s.Size), 1)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor k, v := range chunksMap {\n\t\t\ttx.append([]byte(k), v)\n\t\t}\n\t\ttx.set(m.inodeKey(fout), m.marshal(&attr))\n\t\tif copied != nil {\n\t\t\t*copied = size\n\t\t}\n\t\treturn nil\n\t}, fout)\n\tif err == nil {\n\t\tm.updateParentStat(ctx, fout, attr.Parent, newLength, newSpace)\n\t\tm.updateUserGroupStat(ctx, attr.Uid, attr.Gid, newSpace, 0)\n\t}\n\treturn errno(err)\n}\n\nfunc (m *kvMeta) getParents(tx *kvTxn, inode, parent Ino) []Ino {\n\tif parent > 0 {\n\t\treturn []Ino{parent}\n\t}\n\tvar ps []Ino\n\tprefix := m.fmtKey(\"A\", inode, \"P\")\n\ttx.scan(prefix, nextKey(prefix), false, func(k, v []byte) bool {\n\t\tif len(k) == 1+8+1+8 && parseCounter(v) > 0 {\n\t\t\tps = append(ps, m.decodeInode([]byte(k[10:])))\n\t\t}\n\t\treturn true\n\t})\n\treturn ps\n}\n\nfunc (m *kvMeta) doGetParents(ctx Context, inode Ino) map[Ino]int {\n\tvals, err := m.scanValues(ctx, m.fmtKey(\"A\", inode, \"P\"), -1, func(k, v []byte) bool {\n\t\t// parents: AiiiiiiiiPiiiiiiii\n\t\treturn len(k) == 1+8+1+8 && parseCounter(v) > 0\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"Scan parent key of inode %d: %s\", inode, err)\n\t\treturn nil\n\t}\n\tps := make(map[Ino]int)\n\tfor k, v := range vals {\n\t\tps[m.decodeInode([]byte(k[10:]))] = int(parseCounter(v))\n\t}\n\treturn ps\n}\n\nfunc (m *kvMeta) doSyncDirStat(ctx Context, ino Ino) (*dirStat, syscall.Errno) {\n\tif m.conf.ReadOnly {\n\t\treturn nil, syscall.EROFS\n\t}\n\tstat, st := m.calcDirStat(ctx, ino)\n\tif st != 0 {\n\t\treturn nil, st\n\t}\n\terr := m.client.txn(ctx, func(tx *kvTxn) error {\n\t\tif tx.get(m.inodeKey(ino)) == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\ttx.set(m.dirStatKey(ino), m.packDirStat(stat))\n\t\treturn nil\n\t}, 0)\n\tif err != nil && m.shouldRetry(err) {\n\t\t// other clients have synced\n\t\terr = nil\n\t}\n\treturn stat, errno(err)\n}\n\nfunc (m *kvMeta) doUpdateDirStat(ctx Context, batch map[Ino]dirStat) error {\n\tsyncMap := make(map[Ino]bool, 0)\n\tfor _, group := range m.groupBatch(batch, 20) {\n\t\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\t\tkeys := make([][]byte, 0, len(group))\n\t\t\tfor _, ino := range group {\n\t\t\t\tkeys = append(keys, m.dirStatKey(ino))\n\t\t\t}\n\t\t\tfor i, rawStat := range tx.gets(keys...) {\n\t\t\t\tino := group[i]\n\t\t\t\tif rawStat == nil {\n\t\t\t\t\tsyncMap[ino] = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tst := m.parseDirStat(rawStat)\n\t\t\t\tstat := batch[ino]\n\t\t\t\tst.length += stat.length\n\t\t\t\tst.space += stat.space\n\t\t\t\tst.inodes += stat.inodes\n\t\t\t\tif st.length < 0 || st.space < 0 || st.inodes < 0 {\n\t\t\t\t\tlogger.Warnf(\"dir stat of inode %d is invalid: %+v, try to sync\", ino, st)\n\t\t\t\t\tsyncMap[ino] = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\ttx.set(keys[i], m.packDirStat(st))\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif len(syncMap) > 0 {\n\t\tm.parallelSyncDirStat(ctx, syncMap).Wait()\n\t}\n\treturn nil\n}\n\nfunc (m *kvMeta) doGetDirStat(ctx Context, ino Ino, trySync bool) (*dirStat, syscall.Errno) {\n\trawStat, err := m.get(m.dirStatKey(ino))\n\tif err != nil {\n\t\treturn nil, errno(err)\n\t}\n\tif rawStat != nil {\n\t\treturn m.parseDirStat(rawStat), 0\n\t}\n\tif trySync {\n\t\treturn m.doSyncDirStat(ctx, ino)\n\t}\n\treturn nil, 0\n}\n\nfunc (m *kvMeta) doFindDeletedFiles(ts int64, limit int) (map[Ino]uint64, error) {\n\tif limit == 0 {\n\t\treturn nil, nil\n\t}\n\tklen := 1 + 8 + 8\n\tfiles := make(map[Ino]uint64)\n\tvar count int\n\terr := m.client.scan(m.fmtKey(\"D\"), func(k, v []byte) bool {\n\t\tif len(k) == klen && len(v) == 8 && m.parseInt64(v) < ts {\n\t\t\trb := utils.FromBuffer([]byte(k)[1:])\n\t\t\tfiles[m.decodeInode(rb.Get(8))] = rb.Get64()\n\t\t\tcount++\n\t\t}\n\t\treturn limit < 0 || count < limit\n\t})\n\treturn files, err\n}\n\nfunc (m *kvMeta) doCleanupSlices(ctx Context, count *uint64) error {\n\tif m.Name() == \"tikv\" {\n\t\tm.client.gc()\n\t}\n\tklen := 1 + 8 + 4\n\tvar sErr, cErr error\n\tif sErr = m.client.scan(m.fmtKey(\"K\"), func(k, v []byte) bool {\n\t\tif len(k) == klen && len(v) == 8 && parseCounter(v) <= 0 {\n\t\t\trb := utils.FromBuffer(k[1:])\n\t\t\tid := rb.Get64()\n\t\t\tsize := rb.Get32()\n\t\t\trefs := parseCounter(v)\n\t\t\tif refs < 0 {\n\t\t\t\tm.deleteSlice(id, size)\n\t\t\t\tif count != nil {\n\t\t\t\t\t*count++\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tm.cleanupZeroRef(id, size)\n\t\t\t}\n\t\t\tif ctx.Canceled() {\n\t\t\t\tcErr = ctx.Err()\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\t}); sErr != nil {\n\t\treturn sErr\n\t}\n\treturn cErr\n}\n\nfunc (m *kvMeta) deleteChunk(inode Ino, indx uint32) error {\n\tkey := m.chunkKey(inode, indx)\n\tvar todel []*slice\n\terr := m.txn(Background(), func(tx *kvTxn) error {\n\t\ttodel = todel[:0]\n\t\tbuf := tx.get(key)\n\t\tslices := readSliceBuf(buf)\n\t\tif slices == nil {\n\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d, use `gc` to clean up leaked slices\", inode, indx)\n\t\t}\n\t\ttx.delete(key)\n\t\tfor _, s := range slices {\n\t\t\tif s.id > 0 && tx.incrBy(m.sliceKey(s.id, s.size), -1) < 0 {\n\t\t\t\ttodel = append(todel, s)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}, inode)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, s := range todel {\n\t\tm.deleteSlice(s.id, s.size)\n\t}\n\treturn nil\n}\n\nfunc (m *kvMeta) cleanupZeroRef(id uint64, size uint32) {\n\t_ = m.txn(Background(), func(tx *kvTxn) error {\n\t\tv := tx.incrBy(m.sliceKey(id, size), 0)\n\t\tif v != 0 {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\ttx.delete(m.sliceKey(id, size))\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) doDeleteFileData(inode Ino, length uint64) {\n\tkeys, err := m.scanKeys(Background(), m.fmtKey(\"A\", inode, \"C\"))\n\tif err != nil {\n\t\tlogger.Warnf(\"delete chunks of inode %d: %s\", inode, err)\n\t\treturn\n\t}\n\tfor i := range keys {\n\t\tidx := binary.BigEndian.Uint32(keys[i][10:])\n\t\terr := m.deleteChunk(inode, idx)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"delete chunk %d:%d: %s\", inode, idx, err)\n\t\t\treturn\n\t\t}\n\t}\n\t_ = m.deleteKeys(m.delfileKey(inode, length))\n}\n\nfunc (m *kvMeta) doCleanupDelayedSlices(ctx Context, edge int64) (int, error) {\n\tvar count int\n\tvar ss []Slice\n\tvar rs []int64\n\tvar keys [][]byte\n\tvar batch int = 1e5\n\tfor {\n\t\tif err := m.client.txn(ctx, func(tx *kvTxn) error {\n\t\t\tkeys = keys[:0]\n\t\t\tvar c int\n\t\t\ttx.scan(m.delSliceKey(0, 0), m.delSliceKey(edge, 0),\n\t\t\t\ttrue, func(k, v []byte) bool {\n\t\t\t\t\tif len(k) == 1+8+8 { // delayed slices: Lttttttttcccccccc\n\t\t\t\t\t\tkeys = append(keys, k)\n\t\t\t\t\t\tc++\n\t\t\t\t\t}\n\t\t\t\t\treturn c < batch\n\t\t\t\t})\n\t\t\treturn nil\n\t\t}, 0); err != nil {\n\t\t\tlogger.Warnf(\"Scan delayed slices: %s\", err)\n\t\t\treturn count, err\n\t\t}\n\n\t\tfor _, key := range keys {\n\t\t\tif ctx.Canceled() {\n\t\t\t\treturn count, ctx.Err()\n\t\t\t}\n\t\t\tif err := m.txn(ctx, func(tx *kvTxn) error {\n\t\t\t\tss, rs = ss[:0], rs[:0]\n\t\t\t\tbuf := tx.get(key)\n\t\t\t\tif len(buf) == 0 {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tm.decodeDelayedSlices(buf, &ss)\n\t\t\t\tif len(ss) == 0 {\n\t\t\t\t\treturn fmt.Errorf(\"invalid value for delayed slices %q: %v\", key, buf)\n\t\t\t\t}\n\t\t\t\tfor _, s := range ss {\n\t\t\t\t\trs = append(rs, tx.incrBy(m.sliceKey(s.Id, s.Size), -1))\n\t\t\t\t}\n\t\t\t\ttx.delete(key)\n\t\t\t\treturn nil\n\t\t\t}); err != nil {\n\t\t\t\tlogger.Warnf(\"Cleanup delayed slices %q: %s\", key, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor i, s := range ss {\n\t\t\t\tif rs[i] < 0 {\n\t\t\t\t\tm.deleteSlice(s.Id, s.Size)\n\t\t\t\t\tcount++\n\t\t\t\t}\n\t\t\t\tif ctx.Canceled() {\n\t\t\t\t\treturn count, ctx.Err()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(keys) < batch {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn count, nil\n}\n\nfunc (m *kvMeta) doCompactChunk(inode Ino, indx uint32, buf []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno {\n\tst := errno(m.txn(Background(), func(tx *kvTxn) error {\n\t\tbuf2 := tx.get(m.chunkKey(inode, indx))\n\t\tif len(buf2) < len(buf) || !bytes.Equal(buf, buf2[:len(buf)]) {\n\t\t\tlogger.Infof(\"chunk %d:%d was changed %d -> %d\", inode, indx, len(buf), len(buf2))\n\t\t\treturn syscall.EINVAL\n\t\t}\n\n\t\tbuf2 = append(append(buf2[:skipped*sliceBytes], marshalSlice(pos, id, size, 0, size)...), buf2[len(buf):]...)\n\t\ttx.set(m.chunkKey(inode, indx), buf2)\n\t\t// create the key to tracking it\n\t\ttx.set(m.sliceKey(id, size), make([]byte, 8))\n\t\tif delayed != nil {\n\t\t\tif len(delayed) > 0 {\n\t\t\t\ttx.set(m.delSliceKey(time.Now().Unix(), id), delayed)\n\t\t\t}\n\t\t} else {\n\t\t\tfor _, s := range ss {\n\t\t\t\tif s.id > 0 {\n\t\t\t\t\ttx.incrBy(m.sliceKey(s.id, s.size), -1)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}, inode)) // less conflicts with `write`\n\t// there could be false-negative that the compaction is successful, double-check\n\tif st != 0 && st != syscall.EINVAL {\n\t\trefs, e := m.get(m.sliceKey(id, size))\n\t\tif e == nil {\n\t\t\tif len(refs) > 0 {\n\t\t\t\tst = 0\n\t\t\t} else {\n\t\t\t\tlogger.Infof(\"compacted chunk %d was not used\", id)\n\t\t\t\tst = syscall.EINVAL\n\t\t\t}\n\t\t}\n\t}\n\n\tif st == syscall.EINVAL {\n\t\t_ = m.txn(Background(), func(tx *kvTxn) error {\n\t\t\ttx.incrBy(m.sliceKey(id, size), -1)\n\t\t\treturn nil\n\t\t})\n\t} else if st == 0 {\n\t\tm.cleanupZeroRef(id, size)\n\t\tif delayed == nil {\n\t\t\tvar refs int64\n\t\t\tfor _, s := range ss {\n\t\t\t\tif s.id > 0 && m.client.txn(Background(), func(tx *kvTxn) error {\n\t\t\t\t\trefs = tx.incrBy(m.sliceKey(s.id, s.size), 0)\n\t\t\t\t\treturn nil\n\t\t\t\t}, 0) == nil && refs < 0 {\n\t\t\t\t\tm.deleteSlice(s.id, s.size)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn st\n}\n\nfunc (m *kvMeta) scanAllChunks(ctx Context, ch chan<- cchunk, bar *utils.Bar) error {\n\t// AiiiiiiiiCnnnn file chunks\n\tklen := 1 + 8 + 1 + 4\n\treturn m.client.scan(m.fmtKey(\"A\"), func(k, v []byte) bool {\n\t\tif len(k) == klen && k[1+8] == 'C' && len(v) > sliceBytes {\n\t\t\tbar.IncrTotal(1)\n\t\t\tch <- cchunk{\n\t\t\t\tinode: m.decodeInode(k[1:9]),\n\t\t\t\tindx: binary.BigEndian.Uint32(k[10:]),\n\t\t\t\tslices: len(v) / sliceBytes,\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n}\n\nfunc (m *kvMeta) ListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno {\n\tif delete {\n\t\t_ = m.doCleanupSlices(ctx, nil)\n\t}\n\t// AiiiiiiiiCnnnn file chunks\n\tklen := 1 + 8 + 1 + 4\n\tif err := m.client.scan(m.fmtKey(\"A\"), func(key, value []byte) bool {\n\t\tif len(key) != klen || key[1+8] != 'C' {\n\t\t\treturn true\n\t\t}\n\t\tinode := m.decodeInode([]byte(key)[1:9])\n\t\tss := readSliceBuf(value)\n\t\tif ss == nil {\n\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk key %s\", inode, key)\n\t\t\treturn true\n\t\t}\n\t\tfor _, s := range ss {\n\t\t\tif s.id > 0 {\n\t\t\t\tslices[inode] = append(slices[inode], Slice{Id: s.id, Size: s.size})\n\t\t\t\tif showProgress != nil {\n\t\t\t\t\tshowProgress()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn true\n\t}); err != nil {\n\t\treturn errno(err)\n\t}\n\n\tif scanPending {\n\t\t// slice refs: Kccccccccnnnn\n\t\tklen = 1 + 8 + 4\n\t\t_ = m.client.scan(m.fmtKey(\"K\"), func(k, v []byte) bool {\n\t\t\tif len(k) == klen && len(v) == 8 && parseCounter(v) < 0 {\n\t\t\t\trb := utils.FromBuffer([]byte(k)[1:])\n\t\t\t\tslices[0] = append(slices[0], Slice{Id: rb.Get64(), Size: rb.Get32()})\n\t\t\t}\n\t\t\treturn true\n\n\t\t})\n\t}\n\n\tif m.getFormat().TrashDays == 0 {\n\t\treturn 0\n\t}\n\treturn errno(m.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) {\n\t\tslices[1] = append(slices[1], ss...)\n\t\tif showProgress != nil {\n\t\t\tfor range ss {\n\t\t\t\tshowProgress()\n\t\t\t}\n\t\t}\n\t\treturn false, nil\n\t}))\n}\n\nfunc (m *kvMeta) scanTrashSlices(ctx Context, scan trashSliceScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\n\t// delayed slices: Lttttttttcccccccc\n\tklen := 1 + 8 + 8\n\tvar ss []Slice\n\tvar rs []int64\n\treturn m.client.scan(m.fmtKey(\"L\"), func(key, value []byte) bool {\n\t\tif len(key) != klen || len(value) == 0 {\n\t\t\treturn true\n\t\t}\n\t\tvar clean bool\n\t\tvar err error\n\t\terr = m.txn(ctx, func(tx *kvTxn) error {\n\t\t\tss, rs = ss[:0], rs[:0]\n\t\t\tv := tx.get(key)\n\t\t\tif len(v) == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tb := utils.ReadBuffer(key[1:])\n\t\t\tts := b.Get64()\n\t\t\tm.decodeDelayedSlices(v, &ss)\n\t\t\tclean, err = scan(ss, int64(ts))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif clean {\n\t\t\t\tfor _, s := range ss {\n\t\t\t\t\trs = append(rs, tx.incrBy(m.sliceKey(s.Id, s.Size), -1))\n\t\t\t\t}\n\t\t\t\ttx.delete(key)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"scan trash slices %s: %s\", key, err)\n\t\t\treturn true\n\t\t}\n\t\tif clean && len(rs) == len(ss) {\n\t\t\tfor i, s := range ss {\n\t\t\t\tif rs[i] < 0 {\n\t\t\t\t\tm.deleteSlice(s.Id, s.Size)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n}\n\nfunc (m *kvMeta) scanPendingSlices(ctx Context, scan pendingSliceScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\n\t// slice refs: Kiiiiiiiissss\n\tklen := 1 + 8 + 4\n\treturn m.client.scan(m.fmtKey(\"K\"), func(key, v []byte) bool {\n\t\trefs := parseCounter(v)\n\t\tif len(key) == klen && refs < 0 {\n\t\t\tb := utils.ReadBuffer([]byte(key)[1:])\n\t\t\tid := b.Get64()\n\t\t\tsize := b.Get32()\n\t\t\tclean, err := scan(id, size)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"scan pending deleted slices %d %d: %s\", id, size, err)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif clean {\n\t\t\t\t// TODO: m.deleteSlice(id, size)\n\t\t\t\t// avoid lint warning\n\t\t\t\t_ = clean\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n}\n\nfunc (m *kvMeta) scanPendingFiles(ctx Context, scan pendingFileScan) error {\n\tif scan == nil {\n\t\treturn nil\n\t}\n\t// deleted files: Diiiiiiiissssssss\n\tklen := 1 + 8 + 8\n\n\tvar scanErr error\n\tif err := m.client.scan(m.fmtKey(\"D\"), func(key, val []byte) bool {\n\t\tif scanErr != nil {\n\t\t\treturn true\n\t\t}\n\t\tif len(key) != klen {\n\t\t\tscanErr = fmt.Errorf(\"invalid key %x\", key)\n\t\t\treturn true\n\t\t}\n\t\tino := m.decodeInode(key[1:9])\n\t\tsize := binary.BigEndian.Uint64(key[9:])\n\t\tts := m.parseInt64(val)\n\t\t_, scanErr = scan(ino, size, ts)\n\t\treturn true\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\treturn scanErr\n}\n\nfunc (m *kvMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno {\n\tprefix := m.entryKey(inode, \"\")\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tattr.Nlink = 2\n\t\ttx.scan(prefix, nextKey(prefix), false, func(k, v []byte) bool {\n\t\t\ttyp, _ := m.parseEntry(v)\n\t\t\tif typ == TypeDirectory {\n\t\t\t\tattr.Nlink++\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\ttx.set(m.inodeKey(inode), m.marshal(attr))\n\t\treturn nil\n\t}, inode))\n}\n\nfunc (m *kvMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno {\n\tdefer m.timeit(\"GetXattr\", time.Now())\n\tinode = m.checkRoot(inode)\n\tbuf, err := m.get(m.xattrKey(inode, name))\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tif buf == nil {\n\t\treturn ENOATTR\n\t}\n\t*vbuff = buf\n\treturn 0\n}\n\nfunc (m *kvMeta) ListXattr(ctx Context, inode Ino, names *[]byte) syscall.Errno {\n\tdefer m.timeit(\"ListXattr\", time.Now())\n\tinode = m.checkRoot(inode)\n\tkeys, err := m.scanKeys(ctx, m.xattrKey(inode, \"\"))\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\t*names = nil\n\tprefix := len(m.xattrKey(inode, \"\"))\n\tfor _, name := range keys {\n\t\t*names = append(*names, name[prefix:]...)\n\t\t*names = append(*names, 0)\n\t}\n\n\tval, err := m.get(m.inodeKey(inode))\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\tif val == nil {\n\t\treturn syscall.ENOENT\n\t}\n\tattr := &Attr{}\n\tm.parseAttr(val, attr)\n\tsetXAttrACL(names, attr.AccessACL, attr.DefaultACL)\n\treturn 0\n}\n\nfunc (m *kvMeta) doSetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno {\n\tif len(value) == 0 && m.Name() == \"tikv\" {\n\t\treturn syscall.EINVAL\n\t}\n\tkey := m.xattrKey(inode, name)\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tv := tx.get(key)\n\t\tswitch flags {\n\t\tcase XattrCreate:\n\t\t\tif v != nil {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\tcase XattrReplace:\n\t\t\tif v == nil {\n\t\t\t\treturn ENOATTR\n\t\t\t}\n\t\t}\n\t\tif v == nil || !bytes.Equal(v, value) {\n\t\t\ttx.set(key, value)\n\t\t}\n\t\treturn nil\n\t}))\n}\n\nfunc (m *kvMeta) doRemoveXattr(ctx Context, inode Ino, name string) syscall.Errno {\n\tkey := m.xattrKey(inode, name)\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tvalue := tx.get(key)\n\t\tif value == nil {\n\t\t\treturn ENOATTR\n\t\t}\n\t\ttx.delete(key)\n\t\treturn nil\n\t}))\n}\n\nfunc (m *kvMeta) getQuotaKey(qtype uint32, key uint64) ([]byte, error) {\n\tswitch qtype {\n\tcase DirQuotaType:\n\t\treturn m.dirQuotaKey(Ino(key)), nil\n\tcase UserQuotaType:\n\t\treturn m.userQuotaKey(key), nil\n\tcase GroupQuotaType:\n\t\treturn m.groupQuotaKey(key), nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"invalid quota type: %d\", qtype)\n\t}\n}\n\nfunc (m *kvMeta) doGetQuota(ctx Context, qtype uint32, key uint64) (*Quota, error) {\n\tquotaKey, err := m.getQuotaKey(qtype, key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbuf, err := m.get(quotaKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif buf == nil {\n\t\treturn nil, nil\n\t}\n\tif len(buf) != 32 {\n\t\treturn nil, fmt.Errorf(\"invalid quota value: %v\", buf)\n\t}\n\n\treturn m.parseQuota(buf), nil\n}\n\nfunc (m *kvMeta) doSetQuota(ctx Context, qtype uint32, key uint64, quota *Quota) (bool, error) {\n\tquotaKey, err := m.getQuotaKey(qtype, key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tvar created bool\n\terr = m.txn(ctx, func(tx *kvTxn) error {\n\t\tbuf := tx.get(quotaKey)\n\t\tvar origin *Quota\n\t\tvar exists bool\n\t\tif len(buf) == 32 {\n\t\t\torigin = m.parseQuota(buf)\n\t\t\texists = true\n\t\t} else if len(buf) != 0 {\n\t\t\treturn fmt.Errorf(\"invalid quota value: %v\", buf)\n\t\t}\n\n\t\tif !exists {\n\t\t\tcreated = true\n\t\t\torigin = new(Quota)\n\t\t\torigin.MaxInodes, origin.MaxSpace = -1, -1\n\t\t} else {\n\t\t\tcreated = false\n\t\t}\n\n\t\tif quota.MaxSpace >= 0 {\n\t\t\torigin.MaxSpace = quota.MaxSpace\n\t\t}\n\t\tif quota.MaxInodes >= 0 {\n\t\t\torigin.MaxInodes = quota.MaxInodes\n\t\t}\n\t\tif quota.UsedSpace >= 0 {\n\t\t\torigin.UsedSpace = quota.UsedSpace\n\t\t}\n\t\tif quota.UsedInodes >= 0 {\n\t\t\torigin.UsedInodes = quota.UsedInodes\n\t\t}\n\t\ttx.set(quotaKey, m.packQuota(origin))\n\t\treturn nil\n\t})\n\treturn created, err\n}\n\nfunc (m *kvMeta) doDelQuota(ctx Context, qtype uint32, key uint64) error {\n\tquotaKey, err := m.getQuotaKey(qtype, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif qtype == UserQuotaType || qtype == GroupQuotaType {\n\t\tquota := &Quota{}\n\t\tval, err := m.get(quotaKey)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(val) > 0 {\n\t\t\tquota = m.parseQuota(val)\n\t\t}\n\t\tquota.MaxSpace = -1\n\t\tquota.MaxInodes = -1\n\t\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\t\ttx.set(quotaKey, m.packQuota(quota))\n\t\t\treturn nil\n\t\t})\n\t} else {\n\t\t// For dir quotas, remove all data\n\t\treturn m.deleteKeys(quotaKey)\n\t}\n}\n\nfunc (m *kvMeta) doLoadQuotas(ctx Context) (map[uint64]*Quota, map[uint64]*Quota, map[uint64]*Quota, error) {\n\tquotaTypes := []struct {\n\t\tprefix string\n\t\tname string\n\t}{\n\t\t{\"QD\", \"dir\"},\n\t\t{\"QU\", \"user\"},\n\t\t{\"QG\", \"group\"},\n\t}\n\n\tquotaMaps := make([]map[uint64]*Quota, 3)\n\tfor i, qt := range quotaTypes {\n\t\tpairs, err := m.scanValues(ctx, m.fmtKey(qt.prefix), -1, nil)\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, fmt.Errorf(\"failed to load %s quotas: %w\", qt.name, err)\n\t\t}\n\t\tvar quotas map[uint64]*Quota\n\t\tif len(pairs) == 0 {\n\t\t\tquotas = make(map[uint64]*Quota)\n\t\t} else {\n\t\t\tquotas = make(map[uint64]*Quota, len(pairs))\n\t\t\tfor k, v := range pairs {\n\t\t\t\tvar id uint64\n\t\t\t\tif qt.prefix == \"QD\" {\n\t\t\t\t\tid = uint64(m.decodeInode([]byte(k[2:]))) // skip prefix\n\t\t\t\t} else {\n\t\t\t\t\tid = binary.BigEndian.Uint64([]byte(k[2:])) // skip prefix\n\t\t\t\t}\n\t\t\t\tquotas[id] = m.parseQuota(v)\n\t\t\t}\n\t\t}\n\t\tquotaMaps[i] = quotas\n\t}\n\n\treturn quotaMaps[0], quotaMaps[1], quotaMaps[2], nil\n}\n\nfunc (m *kvMeta) doSyncVolumeStat(ctx Context) error {\n\tif m.conf.ReadOnly {\n\t\treturn syscall.EROFS\n\t}\n\tvar used, inodes int64\n\tif err := m.client.txn(ctx, func(tx *kvTxn) error {\n\t\tprefix := m.fmtKey(\"U\")\n\t\ttx.scan(prefix, nextKey(prefix), false, func(k, v []byte) bool {\n\t\t\tstat := m.parseDirStat(v)\n\t\t\tused += stat.space\n\t\t\tinodes += stat.inodes\n\t\t\treturn true\n\t\t})\n\t\treturn nil\n\t}, 0); err != nil {\n\t\treturn err\n\t}\n\t// need add sustained file size\n\tvals, err := m.scanKeys(ctx, m.fmtKey(\"SS\"))\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar attr Attr\n\tfor _, k := range vals {\n\t\tb := utils.FromBuffer(k[2:])\n\t\tif b.Len() != 16 {\n\t\t\tlogger.Warnf(\"Invalid sustainedKey: %v\", k)\n\t\t\tcontinue\n\t\t}\n\t\t_ = b.Get64()\n\t\tinode := m.decodeInode(b.Get(8))\n\t\tif eno := m.doGetAttr(ctx, inode, &attr); eno != 0 {\n\t\t\tlogger.Warnf(\"Get attr of inode %d: %s\", inode, eno)\n\t\t\tcontinue\n\t\t}\n\t\tused += align4K(attr.Length)\n\t\tinodes += 1\n\t}\n\n\tif err := m.scanTrashEntry(ctx, func(_ Ino, length uint64) {\n\t\tused += align4K(length)\n\t\tinodes += 1\n\t}); err != nil {\n\t\treturn err\n\t}\n\tlogger.Debugf(\"Used space: %s, inodes: %d\", humanize.IBytes(uint64(used)), inodes)\n\terr = m.setValue(m.counterKey(totalInodes), packCounter(inodes))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"set total inodes: %w\", err)\n\t}\n\treturn m.setValue(m.counterKey(usedSpace), packCounter(used))\n}\n\nfunc (m *kvMeta) doFlushQuotas(ctx Context, quotas []*iQuota) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tkeys := make([][]byte, 0, len(quotas))\n\t\tqs := make([]*iQuota, 0, len(quotas))\n\t\tfor _, q := range quotas {\n\t\t\tkey, err := m.getQuotaKey(q.qtype, q.qkey)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tkeys = append(keys, key)\n\t\t\tqs = append(qs, q)\n\t\t}\n\t\tfor i, v := range tx.gets(keys...) {\n\t\t\tif len(v) == 0 {\n\t\t\t\tif qs[i].qtype == UserQuotaType || qs[i].qtype == GroupQuotaType {\n\t\t\t\t\tquota := &Quota{\n\t\t\t\t\t\tMaxSpace: -1,\n\t\t\t\t\t\tMaxInodes: -1,\n\t\t\t\t\t\tUsedSpace: qs[i].quota.newSpace,\n\t\t\t\t\t\tUsedInodes: qs[i].quota.newInodes,\n\t\t\t\t\t}\n\t\t\t\t\ttx.set(keys[i], m.packQuota(quota))\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(v) != 32 {\n\t\t\t\tlogger.Errorf(\"Invalid quota value: %v\", v)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tq := m.parseQuota(v)\n\t\t\tq.UsedSpace += qs[i].quota.newSpace\n\t\t\tq.UsedInodes += qs[i].quota.newInodes\n\t\t\ttx.set(keys[i], m.packQuota(q))\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) dumpEntry(inode Ino, e *DumpedEntry, showProgress func(totalIncr, currentIncr int64)) error {\n\tctx := Background()\n\treturn m.client.txn(ctx, func(tx *kvTxn) error {\n\t\ta := tx.get(m.inodeKey(inode))\n\t\tif a == nil {\n\t\t\tlogger.Warnf(\"inode %d not found\", inode)\n\t\t}\n\n\t\tattr := &Attr{Nlink: 1}\n\t\tm.parseAttr(a, attr)\n\t\tif a == nil && e.Attr != nil {\n\t\t\tattr.Typ = typeFromString(e.Attr.Type)\n\t\t\tif attr.Typ == TypeDirectory {\n\t\t\t\tattr.Nlink = 2\n\t\t\t}\n\t\t}\n\t\tdumpAttr(attr, e.Attr)\n\t\te.Attr.Inode = inode\n\n\t\tvar xattrs []*DumpedXattr\n\t\ttx.scan(m.xattrKey(inode, \"\"), nextKey(m.xattrKey(inode, \"\")), false, func(k, v []byte) bool {\n\t\t\txattrs = append(xattrs, &DumpedXattr{string(k[10:]), string(v)}) // \"A\" + inode + \"X\"\n\t\t\treturn true\n\t\t})\n\t\tif len(xattrs) > 0 {\n\t\t\tsort.Slice(xattrs, func(i, j int) bool { return xattrs[i].Name < xattrs[j].Name })\n\t\t\te.Xattrs = xattrs\n\t\t}\n\n\t\taccessACl, err := m.getACL(tx, attr.AccessACL)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\te.AccessACL = dumpACL(accessACl)\n\t\tdefaultACL, err := m.getACL(tx, attr.DefaultACL)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\te.DefaultACL = dumpACL(defaultACL)\n\n\t\tif attr.Typ == TypeFile {\n\t\t\te.Chunks = e.Chunks[:0]\n\t\t\tvals := make(map[string][]byte)\n\t\t\ttx.scan(m.chunkKey(inode, 0), m.chunkKey(inode, uint32(attr.Length/ChunkSize)+1),\n\t\t\t\tfalse, func(k, v []byte) bool {\n\t\t\t\t\tvals[string(k)] = v\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t\tfor indx := uint32(0); uint64(indx)*ChunkSize < attr.Length; indx++ {\n\t\t\t\tv, ok := vals[string(m.chunkKey(inode, indx))]\n\t\t\t\tif !ok {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tss := readSliceBuf(v)\n\t\t\t\tif ss == nil {\n\t\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d\", inode, indx)\n\t\t\t\t}\n\t\t\t\tslices := make([]*DumpedSlice, 0, len(ss))\n\t\t\t\tfor _, s := range ss {\n\t\t\t\t\tslices = append(slices, &DumpedSlice{Id: s.id, Pos: s.pos, Size: s.size, Off: s.off, Len: s.len})\n\t\t\t\t}\n\t\t\t\te.Chunks = append(e.Chunks, &DumpedChunk{indx, slices})\n\t\t\t}\n\t\t} else if attr.Typ == TypeSymlink {\n\t\t\tl := tx.get(m.symKey(inode))\n\t\t\tif l == nil {\n\t\t\t\tlogger.Warnf(\"no link target for inode %d\", inode)\n\t\t\t}\n\t\t\te.Symlink = string(l)\n\t\t} else if attr.Typ == TypeDirectory {\n\t\t\tvals, err := m.scanValues(ctx, m.entryKey(inode, \"\"), 10000, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif showProgress != nil {\n\t\t\t\tshowProgress(int64(len(e.Entries)), 0)\n\t\t\t}\n\t\t\tif len(vals) < 10000 {\n\t\t\t\te.Entries = make(map[string]*DumpedEntry, len(vals))\n\t\t\t\tfor k, value := range vals {\n\t\t\t\t\tname := k[10:]\n\t\t\t\t\tce := entryPool.Get()\n\t\t\t\t\tce.Name = name\n\t\t\t\t\ttyp, inode := m.parseEntry(value)\n\t\t\t\t\tce.Attr.Inode = inode\n\t\t\t\t\tce.Attr.Type = typeToString(typ)\n\t\t\t\t\te.Entries[name] = ce\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}, 0)\n}\n\nfunc (m *kvMeta) dumpDir(ctx Context, inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth, threads int, showProgress func(totalIncr, currentIncr int64)) error {\n\tbwWrite := func(s string) {\n\t\tif _, err := bw.WriteString(s); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\tif tree.Entries == nil {\n\t\t// retry for large directory\n\t\tvals, err := m.scanValues(ctx, m.entryKey(inode, \"\"), -1, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttree.Entries = make(map[string]*DumpedEntry, len(vals))\n\t\tfor k, value := range vals {\n\t\t\tname := k[10:]\n\t\t\tce := entryPool.Get()\n\t\t\tce.Name = name\n\t\t\ttyp, inode := m.parseEntry(value)\n\t\t\tce.Attr.Inode = inode\n\t\t\tce.Attr.Type = typeToString(typ)\n\t\t\ttree.Entries[name] = ce\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(int64(len(tree.Entries))-10000, 0)\n\t\t}\n\t}\n\tvar entries []*DumpedEntry\n\tfor _, e := range tree.Entries {\n\t\tentries = append(entries, e)\n\t}\n\tsort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })\n\t_ = tree.writeJsonWithOutEntry(bw, depth)\n\n\tms := make([]sync.Mutex, threads)\n\tconds := make([]*sync.Cond, threads)\n\tready := make([]bool, threads)\n\tvar err error\n\tfor c := 0; c < threads; c++ {\n\t\tconds[c] = sync.NewCond(&ms[c])\n\t\tif c < len(entries) {\n\t\t\tgo func(c int) {\n\t\t\t\tfor i := c; i < len(entries) && err == nil; i += threads {\n\t\t\t\t\te := entries[i]\n\t\t\t\t\ter := m.dumpEntry(e.Attr.Inode, e, showProgress)\n\t\t\t\t\tms[c].Lock()\n\t\t\t\t\tready[c] = true\n\t\t\t\t\tif er != nil {\n\t\t\t\t\t\terr = er\n\t\t\t\t\t}\n\t\t\t\t\tconds[c].Signal()\n\t\t\t\t\tfor ready[c] && err == nil {\n\t\t\t\t\t\tconds[c].Wait()\n\t\t\t\t\t}\n\t\t\t\t\tms[c].Unlock()\n\t\t\t\t}\n\t\t\t}(c)\n\t\t}\n\t}\n\n\tfor i, e := range entries {\n\t\tc := i % threads\n\t\tms[c].Lock()\n\t\tfor !ready[c] && err == nil {\n\t\t\tconds[c].Wait()\n\t\t}\n\t\tready[c] = false\n\t\tconds[c].Signal()\n\t\tms[c].Unlock()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif e.Attr.Type == \"directory\" {\n\t\t\terr = m.dumpDir(ctx, e.Attr.Inode, e, bw, depth+2, threads, showProgress)\n\t\t} else {\n\t\t\terr = e.writeJSON(bw, depth+2)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tentries[i] = nil\n\t\tentryPool.Put(e)\n\t\tif i != len(entries)-1 {\n\t\t\tbwWrite(\",\")\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(0, 1)\n\t\t}\n\t}\n\tbwWrite(fmt.Sprintf(\"\\n%s}\\n%s}\", strings.Repeat(jsonIndent, depth+1), strings.Repeat(jsonIndent, depth)))\n\treturn nil\n}\n\nfunc (m *kvMeta) dumpDirFast(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth int, showProgress func(totalIncr, currentIncr int64)) error {\n\tbwWrite := func(s string) {\n\t\tif _, err := bw.WriteString(s); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\tvar names []string\n\tentries := tree.Entries\n\tfor n, de := range entries {\n\t\tif !de.Attr.full && de.Attr.Inode != TrashInode {\n\t\t\tlogger.Warnf(\"Corrupt inode: %d, missing attribute\", inode)\n\t\t}\n\t\tnames = append(names, n)\n\t}\n\tsort.Slice(names, func(i, j int) bool { return names[i] < names[j] })\n\t_ = tree.writeJsonWithOutEntry(bw, depth)\n\tfor i, name := range names {\n\t\te := entries[name]\n\t\te.Name = name\n\t\tinode := e.Attr.Inode\n\t\tif e.Attr.Type == \"directory\" {\n\t\t\t_ = m.dumpDirFast(inode, e, bw, depth+2, showProgress)\n\t\t} else {\n\t\t\t_ = e.writeJSON(bw, depth+2)\n\t\t}\n\t\tif i != len(entries)-1 {\n\t\t\tbwWrite(\",\")\n\t\t}\n\t\tif showProgress != nil {\n\t\t\tshowProgress(0, 1)\n\t\t}\n\t}\n\tbwWrite(fmt.Sprintf(\"\\n%s}\\n%s}\", strings.Repeat(jsonIndent, depth+1), strings.Repeat(jsonIndent, depth)))\n\treturn nil\n}\n\nfunc (m *kvMeta) DumpMeta(w io.Writer, root Ino, threads int, keepSecret, fast, skipTrash bool) (err error) {\n\tdefer func() {\n\t\tif p := recover(); p != nil {\n\t\t\tdebug.PrintStack()\n\t\t\tif e, ok := p.(error); ok {\n\t\t\t\terr = e\n\t\t\t} else {\n\t\t\t\terr = errors.Errorf(\"DumpMeta error: %v\", p)\n\t\t\t}\n\t\t}\n\t}()\n\tctx := Background()\n\tvals, err := m.scanValues(ctx, m.fmtKey(\"D\"), -1, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdels := make([]*DumpedDelFile, 0, len(vals))\n\tfor k, v := range vals {\n\t\tb := utils.FromBuffer([]byte(k[1:])) // \"D\"\n\t\tif b.Len() != 16 {\n\t\t\tlogger.Warnf(\"invalid delfileKey: %s\", k)\n\t\t\tcontinue\n\t\t}\n\t\tinode := m.decodeInode(b.Get(8))\n\t\tdels = append(dels, &DumpedDelFile{inode, b.Get64(), m.parseInt64(v)})\n\t}\n\n\tprogress := utils.NewProgress(false)\n\tvar tree, trash *DumpedEntry\n\troot = m.checkRoot(root)\n\n\tbInodes, _ := m.get(m.counterKey(totalInodes))\n\tinodeTotal := parseCounter(bInodes)\n\tif root == RootInode && fast { // make snap\n\t\tm.snap = make(map[Ino]*DumpedEntry)\n\t\tdefer func() {\n\t\t\tm.snap = nil\n\t\t}()\n\t\tbar := progress.AddCountBar(\"Scan keys\", 0)\n\t\tbUsed, _ := m.get(m.counterKey(usedSpace))\n\t\tused := parseCounter(bUsed)\n\t\tvar guessKeyTotal int64 = 3 // setting, nextInode, nextChunk\n\t\tif inodeTotal > 0 {\n\t\t\tguessKeyTotal += int64(math.Ceil((float64(used/inodeTotal/(64*1024*1024)) + float64(3)) * float64(inodeTotal)))\n\t\t}\n\t\tbar.SetCurrent(0) // Reset\n\t\tbar.SetTotal(guessKeyTotal)\n\t\tthreshold := 0.1\n\t\tvar cnt int\n\n\t\tif err = m.cacheACLs(Background()); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\terr := m.client.scan(nil, func(key, value []byte) bool {\n\t\t\tif len(key) > 9 && key[0] == 'A' {\n\t\t\t\tino := m.decodeInode(key[1:9])\n\t\t\t\te := m.snap[ino]\n\t\t\t\tif e == nil {\n\t\t\t\t\te = &DumpedEntry{Attr: &DumpedAttr{Inode: ino}}\n\t\t\t\t\tm.snap[ino] = e\n\t\t\t\t}\n\t\t\t\tswitch key[9] {\n\t\t\t\tcase 'I':\n\t\t\t\t\tattr := &Attr{Nlink: 1}\n\t\t\t\t\tm.parseAttr(value, attr)\n\t\t\t\t\tdumpAttr(attr, e.Attr)\n\t\t\t\t\te.Attr.Inode = ino\n\t\t\t\t\te.AccessACL = dumpACL(m.aclCache.Get(attr.AccessACL))\n\t\t\t\t\te.DefaultACL = dumpACL(m.aclCache.Get(attr.DefaultACL))\n\t\t\t\tcase 'C':\n\t\t\t\t\tindx := binary.BigEndian.Uint32(key[10:])\n\t\t\t\t\tss := readSliceBuf(value)\n\t\t\t\t\tif ss == nil {\n\t\t\t\t\t\tlogger.Errorf(\"Corrupt value for inode %d chunk index %d\", ino, indx)\n\t\t\t\t\t}\n\t\t\t\t\tslices := make([]*DumpedSlice, 0, len(ss))\n\t\t\t\t\tfor _, s := range ss {\n\t\t\t\t\t\tslices = append(slices, &DumpedSlice{Id: s.id, Pos: s.pos, Size: s.size, Off: s.off, Len: s.len})\n\t\t\t\t\t}\n\t\t\t\t\te.Chunks = append(e.Chunks, &DumpedChunk{indx, slices})\n\t\t\t\tcase 'D':\n\t\t\t\t\tname := string(key[10:])\n\t\t\t\t\ttyp, inode := m.parseEntry(value)\n\t\t\t\t\tchild := m.snap[inode]\n\t\t\t\t\tif child == nil {\n\t\t\t\t\t\tchild = &DumpedEntry{Attr: &DumpedAttr{Inode: inode, Type: typeToString(typ)}}\n\t\t\t\t\t\tm.snap[inode] = child\n\t\t\t\t\t} else if child.Attr.Type == \"\" {\n\t\t\t\t\t\tchild.Attr.Type = typeToString(typ)\n\t\t\t\t\t}\n\t\t\t\t\tif e.Entries == nil {\n\t\t\t\t\t\te.Entries = map[string]*DumpedEntry{}\n\t\t\t\t\t}\n\t\t\t\t\te.Entries[name] = child\n\t\t\t\tcase 'X':\n\t\t\t\t\te.Xattrs = append(e.Xattrs, &DumpedXattr{string(key[10:]), string(value)})\n\t\t\t\tcase 'S':\n\t\t\t\t\te.Symlink = string(value)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcnt++\n\t\t\tif cnt%100 == 0 && bar.Current() > int64(math.Ceil(float64(guessKeyTotal)*(1-threshold))) {\n\t\t\t\tguessKeyTotal += int64(math.Ceil(float64(guessKeyTotal) * threshold))\n\t\t\t\tbar.SetTotal(guessKeyTotal)\n\t\t\t}\n\t\t\tbar.Increment()\n\t\t\treturn true\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tbar.Done()\n\t\ttree = m.snap[root]\n\t\tif !skipTrash {\n\t\t\ttrash = m.snap[TrashInode]\n\t\t\tif trash == nil {\n\t\t\t\ttrash = &DumpedEntry{\n\t\t\t\t\tAttr: &DumpedAttr{\n\t\t\t\t\t\tInode: TrashInode,\n\t\t\t\t\t\tType: \"directory\",\n\t\t\t\t\t\tNlink: 2,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\tm.snap[TrashInode] = trash\n\t\t\t}\n\t\t}\n\t} else {\n\t\ttree = &DumpedEntry{\n\t\t\tAttr: &DumpedAttr{\n\t\t\t\tInode: root,\n\t\t\t\tType: \"directory\",\n\t\t\t},\n\t\t}\n\t\tif err = m.dumpEntry(root, tree, nil); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif root == RootInode && !skipTrash {\n\t\t\ttrash = &DumpedEntry{\n\t\t\t\tAttr: &DumpedAttr{\n\t\t\t\t\tInode: TrashInode,\n\t\t\t\t\tType: \"directory\",\n\t\t\t\t},\n\t\t\t}\n\t\t\tif err = m.dumpEntry(TrashInode, trash, nil); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\tif tree == nil || tree.Attr == nil {\n\t\treturn errors.New(\"The entry of the root inode was not found\")\n\t}\n\ttree.Name = \"FSTree\"\n\n\tvar rs [][]byte\n\terr = m.txn(Background(), func(tx *kvTxn) error {\n\t\trs = tx.gets(m.counterKey(usedSpace),\n\t\t\tm.counterKey(totalInodes),\n\t\t\tm.counterKey(\"nextInode\"),\n\t\t\tm.counterKey(\"nextChunk\"),\n\t\t\tm.counterKey(\"nextSession\"),\n\t\t\tm.counterKey(\"nextTrash\"))\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tcs := make([]int64, len(rs))\n\tfor i, r := range rs {\n\t\tif r != nil {\n\t\t\tcs[i] = parseCounter(r)\n\t\t}\n\t}\n\n\tvals, err = m.scanValues(ctx, m.fmtKey(\"SS\"), -1, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tss := make(map[uint64][]Ino)\n\tfor k := range vals {\n\t\tb := utils.FromBuffer([]byte(k[2:])) // \"SS\"\n\t\tif b.Len() != 16 {\n\t\t\treturn fmt.Errorf(\"invalid sustainedKey: %s\", k)\n\t\t}\n\t\tsid := b.Get64()\n\t\tinode := m.decodeInode(b.Get(8))\n\t\tss[sid] = append(ss[sid], inode)\n\t}\n\tsessions := make([]*DumpedSustained, 0, len(ss))\n\tfor k, v := range ss {\n\t\tsessions = append(sessions, &DumpedSustained{k, v})\n\t}\n\n\tpairs, err := m.scanValues(ctx, m.fmtKey(\"QD\"), -1, func(k, v []byte) bool {\n\t\treturn len(k) == 10 && len(v) == 32\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tquotas := make(map[Ino]*DumpedQuota, len(pairs))\n\tfor k, v := range pairs {\n\t\tinode := m.decodeInode([]byte(k[2:]))\n\t\tquota := m.parseQuota(v)\n\t\tquotas[inode] = &DumpedQuota{quota.MaxSpace, quota.MaxInodes, 0, 0}\n\t}\n\n\tdm := DumpedMeta{\n\t\tSetting: *m.getFormat(),\n\t\tCounters: &DumpedCounters{\n\t\t\tUsedSpace: cs[0],\n\t\t\tUsedInodes: cs[1],\n\t\t\tNextInode: cs[2],\n\t\t\tNextChunk: cs[3],\n\t\t\tNextSession: cs[4],\n\t\t\tNextTrash: cs[5],\n\t\t},\n\t\tSustained: sessions,\n\t\tDelFiles: dels,\n\t\tQuotas: quotas,\n\t}\n\tif !keepSecret && dm.Setting.SecretKey != \"\" {\n\t\tdm.Setting.SecretKey = \"removed\"\n\t\tlogger.Warnf(\"Secret key is removed for the sake of safety\")\n\t}\n\tif !keepSecret && dm.Setting.SessionToken != \"\" {\n\t\tdm.Setting.SessionToken = \"removed\"\n\t\tlogger.Warnf(\"Session token is removed for the sake of safety\")\n\t}\n\tbw, err := dm.writeJsonWithOutTree(w)\n\tif err != nil {\n\t\treturn err\n\t}\n\tuseTotal := root == RootInode && !skipTrash\n\tbar := progress.AddCountBar(\"Dumped entries\", 1) // with root\n\tif useTotal {\n\t\tbar.SetTotal(inodeTotal)\n\t}\n\tbar.Increment()\n\tif trash != nil {\n\t\ttrash.Name = \"Trash\"\n\t\tbar.IncrTotal(1)\n\t\tbar.Increment()\n\t}\n\tshowProgress := func(totalIncr, currentIncr int64) {\n\t\tif !useTotal {\n\t\t\tbar.IncrTotal(totalIncr)\n\t\t}\n\t\tbar.IncrInt64(currentIncr)\n\t}\n\tif m.snap != nil {\n\t\tif err = m.dumpDirFast(root, tree, bw, 1, showProgress); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tshowProgress(int64(len(tree.Entries)), 0)\n\t\tif err = m.dumpDir(ctx, root, tree, bw, 1, threads, showProgress); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif trash != nil {\n\t\tif _, err = bw.WriteString(\",\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif m.snap != nil {\n\t\t\tif err = m.dumpDirFast(TrashInode, trash, bw, 1, showProgress); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tshowProgress(int64(len(tree.Entries)), 0)\n\t\t\tif err = m.dumpDir(ctx, TrashInode, trash, bw, 1, threads, showProgress); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\tif _, err = bw.WriteString(\"\\n}\\n\"); err != nil {\n\t\treturn err\n\t}\n\tprogress.Done()\n\n\treturn bw.Flush()\n}\n\ntype pair struct {\n\tkey []byte\n\tvalue []byte\n}\n\nfunc (m *kvMeta) loadEntry(e *DumpedEntry, kv chan *pair, aclMaxId *uint32) {\n\tinode := e.Attr.Inode\n\tattr := loadAttr(e.Attr)\n\tattr.Parent = e.Parents[0]\n\tif attr.Typ == TypeFile {\n\t\tattr.Length = e.Attr.Length\n\t\tfor _, c := range e.Chunks {\n\t\t\tif len(c.Slices) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tslices := make([]byte, 0, sliceBytes*len(c.Slices))\n\t\t\tfor _, s := range c.Slices {\n\t\t\t\tslices = append(slices, marshalSlice(s.Pos, s.Id, s.Size, s.Off, s.Len)...)\n\t\t\t}\n\t\t\tkv <- &pair{m.chunkKey(inode, c.Index), slices}\n\t\t}\n\t} else if attr.Typ == TypeDirectory {\n\t\tattr.Length = 4 << 10\n\t\tvar stat dirStat\n\t\tfor name, c := range e.Entries {\n\t\t\tlength := uint64(0)\n\t\t\tif typeFromString(c.Attr.Type) == TypeFile {\n\t\t\t\tlength = c.Attr.Length\n\t\t\t}\n\t\t\tstat.length += int64(length)\n\t\t\tstat.space += align4K(length)\n\t\t\tstat.inodes++\n\n\t\t\tkv <- &pair{m.entryKey(inode, string(unescape(name))), m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode)}\n\t\t}\n\t\tkv <- &pair{m.dirStatKey(inode), m.packDirStat(&stat)}\n\t} else if attr.Typ == TypeSymlink {\n\t\tsymL := unescape(e.Symlink)\n\t\tattr.Length = uint64(len(symL))\n\t\tkv <- &pair{m.symKey(inode), []byte(symL)}\n\t}\n\tfor _, x := range e.Xattrs {\n\t\tkv <- &pair{m.xattrKey(inode, x.Name), []byte(unescape(x.Value))}\n\t}\n\n\tattr.AccessACL = m.saveACL(loadACL(e.AccessACL), aclMaxId)\n\tattr.DefaultACL = m.saveACL(loadACL(e.DefaultACL), aclMaxId)\n\tkv <- &pair{m.inodeKey(inode), m.marshal(attr)}\n}\n\nfunc (m *kvMeta) LoadMeta(r io.Reader) error {\n\tvar exist bool\n\terr := m.txn(Background(), func(tx *kvTxn) error {\n\t\texist = tx.exist(m.fmtKey())\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tif exist {\n\t\treturn fmt.Errorf(\"Database %s://%s is not empty\", m.Name(), m.addr)\n\t}\n\n\tkv := make(chan *pair, 10000)\n\tbatch := 10000\n\tif m.Name() == \"etcd\" {\n\t\tbatch = 128\n\t}\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tvar buffer []*pair\n\t\t\tvar total int\n\t\t\tfor p := range kv {\n\t\t\t\tbuffer = append(buffer, p)\n\t\t\t\ttotal += len(p.key) + len(p.value)\n\t\t\t\tif len(buffer) >= batch || total > 5<<20 {\n\t\t\t\t\terr := m.txn(Background(), func(tx *kvTxn) error {\n\t\t\t\t\t\tfor _, p := range buffer {\n\t\t\t\t\t\t\ttx.set(p.key, p.value)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlogger.Fatalf(\"write %d pairs: %s\", len(buffer), err)\n\t\t\t\t\t}\n\t\t\t\t\tbuffer = buffer[:0]\n\t\t\t\t\ttotal = 0\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(buffer) > 0 {\n\t\t\t\terr := m.txn(Background(), func(tx *kvTxn) error {\n\t\t\t\t\tfor _, p := range buffer {\n\t\t\t\t\t\ttx.set(p.key, p.value)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Fatalf(\"write %d pairs: %s\", len(buffer), err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tvar aclMaxId uint32\n\tdm, counters, parents, refs, err := loadEntries(r, func(e *DumpedEntry) { m.loadEntry(e, kv, &aclMaxId) }, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif err = m.loadDumpedACLs(Background()); err != nil {\n\t\treturn err\n\t}\n\n\tformat, _ := json.MarshalIndent(dm.Setting, \"\", \"\")\n\tkv <- &pair{m.fmtKey(\"setting\"), format}\n\tkv <- &pair{m.counterKey(usedSpace), packCounter(counters.UsedSpace)}\n\tkv <- &pair{m.counterKey(totalInodes), packCounter(counters.UsedInodes)}\n\tkv <- &pair{m.counterKey(\"nextInode\"), packCounter(counters.NextInode)}\n\tkv <- &pair{m.counterKey(\"nextChunk\"), packCounter(counters.NextChunk)}\n\tkv <- &pair{m.counterKey(\"nextSession\"), packCounter(counters.NextSession)}\n\tkv <- &pair{m.counterKey(\"nextTrash\"), packCounter(counters.NextTrash)}\n\tfor _, d := range dm.DelFiles {\n\t\tkv <- &pair{m.delfileKey(d.Inode, d.Length), m.packInt64(d.Expire)}\n\t}\n\tfor k, v := range refs {\n\t\tif v > 1 {\n\t\t\tkv <- &pair{m.sliceKey(k.id, k.size), packCounter(v - 1)}\n\t\t}\n\t}\n\tclose(kv)\n\twg.Wait()\n\n\t// update nlinks and parents for hardlinks\n\tst := make(map[Ino]int64)\n\tdefer m.loadDumpedQuotas(Background(), dm.Quotas)\n\treturn m.txn(Background(), func(tx *kvTxn) error {\n\t\tfor i, ps := range parents {\n\t\t\tif len(ps) > 1 {\n\t\t\t\ta := tx.get(m.inodeKey(i))\n\t\t\t\t// reset nlink and parent\n\t\t\t\tbinary.BigEndian.PutUint32(a[47:51], uint32(len(ps))) // nlink\n\t\t\t\tbinary.BigEndian.PutUint64(a[63:71], 0)\n\t\t\t\ttx.set(m.inodeKey(i), a)\n\t\t\t\tfor k := range st {\n\t\t\t\t\tdelete(st, k)\n\t\t\t\t}\n\t\t\t\tfor _, p := range ps {\n\t\t\t\t\tst[p] = st[p] + 1\n\t\t\t\t}\n\t\t\t\tfor p, c := range st {\n\t\t\t\t\ttx.set(m.parentKey(i, p), packCounter(c))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) doCloneEntry(ctx Context, srcIno Ino, parent Ino, name string, ino Ino, originAttr *Attr, cmode uint8, cumask uint16, top bool) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\ta := tx.get(m.inodeKey(srcIno))\n\t\tif a == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(a, originAttr)\n\t\tattr := *originAttr\n\t\tif eno := m.Access(ctx, srcIno, MODE_MASK_R, &attr); eno != 0 {\n\t\t\treturn eno\n\t\t}\n\t\tattr.Parent = parent\n\t\tnow := time.Now()\n\t\tif cmode&CLONE_MODE_PRESERVE_ATTR == 0 {\n\t\t\tattr.Uid = ctx.Uid()\n\t\t\tattr.Gid = ctx.Gid()\n\t\t\tattr.Mode &= ^cumask\n\t\t\tattr.Atime = now.Unix()\n\t\t\tattr.Mtime = now.Unix()\n\t\t\tattr.Ctime = now.Unix()\n\t\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\t\tattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t}\n\t\t// TODO: preserve hardlink\n\t\tif attr.Typ == TypeFile && attr.Nlink > 1 {\n\t\t\tattr.Nlink = 1\n\t\t}\n\n\t\tif top {\n\t\t\tvar pattr Attr\n\t\t\ta = tx.get(m.inodeKey(parent))\n\t\t\tif a == nil {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t\tm.parseAttr(a, &pattr)\n\t\t\tif pattr.Typ != TypeDirectory {\n\t\t\t\treturn syscall.ENOTDIR\n\t\t\t}\n\t\t\tif (pattr.Flags & FlagImmutable) != 0 {\n\t\t\t\treturn syscall.EPERM\n\t\t\t}\n\t\t\tif tx.get(m.entryKey(parent, name)) != nil {\n\t\t\t\treturn syscall.EEXIST\n\t\t\t}\n\t\t\tif eno := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); eno != 0 {\n\t\t\t\treturn eno\n\t\t\t}\n\t\t\tif attr.Typ != TypeDirectory {\n\t\t\t\tnow := time.Now()\n\t\t\t\tpattr.Mtime = now.Unix()\n\t\t\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\t\t\tpattr.Ctime = now.Unix()\n\t\t\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\t\ttx.set(m.inodeKey(parent), m.marshal(&pattr))\n\t\t\t}\n\t\t}\n\n\t\ttx.set(m.inodeKey(ino), m.marshal(&attr))\n\t\tprefix := m.xattrKey(srcIno, \"\")\n\t\ttx.scan(prefix, nextKey(prefix), false, func(k, v []byte) bool {\n\t\t\ttx.set(m.xattrKey(ino, string(k[len(prefix):])), v)\n\t\t\treturn true\n\t\t})\n\t\tif top && attr.Typ == TypeDirectory {\n\t\t\ttx.set(m.detachedKey(ino), m.packInt64(time.Now().Unix()))\n\t\t} else {\n\t\t\ttx.set(m.entryKey(parent, name), m.packEntry(attr.Typ, ino))\n\t\t}\n\n\t\tswitch attr.Typ {\n\t\tcase TypeDirectory:\n\t\t\ttx.set(m.dirStatKey(ino), tx.get(m.dirStatKey(srcIno)))\n\t\tcase TypeFile:\n\t\t\tif attr.Length != 0 {\n\t\t\t\tvals := make(map[string][]byte)\n\t\t\t\ttx.scan(m.chunkKey(srcIno, 0), m.chunkKey(srcIno, uint32(attr.Length/ChunkSize)+1),\n\t\t\t\t\tfalse, func(k, v []byte) bool {\n\t\t\t\t\t\tvals[string(k)] = v\n\t\t\t\t\t\treturn true\n\t\t\t\t\t})\n\n\t\t\t\trefKeys := make([][]byte, 0, len(vals))\n\t\t\t\tfor indx := uint32(0); indx <= uint32(attr.Length/ChunkSize); indx++ {\n\t\t\t\t\tif v, ok := vals[string(m.chunkKey(srcIno, indx))]; ok {\n\t\t\t\t\t\ttx.set(m.chunkKey(ino, indx), v)\n\t\t\t\t\t\tss := readSliceBuf(v)\n\t\t\t\t\t\tfor _, s := range ss {\n\t\t\t\t\t\t\tif s.id > 0 {\n\t\t\t\t\t\t\t\trefKeys = append(refKeys, m.sliceKey(s.id, s.size))\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\trefs := tx.gets(refKeys...)\n\t\t\t\tfor i := range refKeys {\n\t\t\t\t\ttx.set(refKeys[i], packCounter(parseCounter(refs[i])+1))\n\t\t\t\t}\n\t\t\t}\n\t\tcase TypeSymlink:\n\t\t\ttx.set(m.symKey(ino), tx.get(m.symKey(srcIno)))\n\t\t}\n\t\treturn nil\n\t}, srcIno))\n}\n\nfunc (m *kvMeta) doFindDetachedNodes(t time.Time) []Ino {\n\tvals, err := m.scanValues(Background(), m.fmtKey(\"N\"), -1, func(k, v []byte) bool {\n\t\treturn len(k) == 9 && m.parseInt64(v) < t.Unix()\n\t})\n\tif err != nil {\n\t\tlogger.Errorf(\"Scan detached nodes error: %s\", err)\n\t\treturn nil\n\t}\n\tvar inodes []Ino\n\tfor k := range vals {\n\t\tinodes = append(inodes, m.decodeInode([]byte(k)[1:]))\n\t}\n\treturn inodes\n}\n\nfunc (m *kvMeta) doCleanupDetachedNode(ctx Context, ino Ino) syscall.Errno {\n\tbuf, err := m.get(m.inodeKey(ino))\n\tif err != nil || buf == nil {\n\t\treturn errno(err)\n\t}\n\trmConcurrent := make(chan int, 10)\n\tif eno := m.emptyDir(ctx, ino, true, nil, rmConcurrent); eno != 0 {\n\t\treturn eno\n\t}\n\tm.updateStats(-align4K(0), -1)\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\ttx.delete(m.inodeKey(ino))\n\t\ttx.deleteKeys(m.xattrKey(ino, \"\"))\n\t\ttx.delete(m.dirStatKey(ino))\n\t\ttx.delete(m.detachedKey(ino))\n\t\treturn nil\n\t}, ino))\n}\n\nfunc (m *kvMeta) doBatchClone(ctx Context, srcParent Ino, dstParent Ino, entries []*Entry, cmode uint8, cumask uint16, result *batchCloneResult) syscall.Errno {\n\t// TODO: Implement batch clone for TKV backend\n\treturn syscall.ENOTSUP\n}\n\nfunc (m *kvMeta) doAttachDirNode(ctx Context, parent Ino, inode Ino, name string) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\ta := tx.get(m.inodeKey(parent))\n\t\tif a == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tvar pattr Attr\n\t\tm.parseAttr(a, &pattr)\n\t\tif pattr.Typ != TypeDirectory {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t\tif pattr.Parent > TrashInode {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tif (pattr.Flags & FlagImmutable) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif tx.get(m.entryKey(parent, name)) != nil {\n\t\t\treturn syscall.EEXIST\n\t\t}\n\n\t\tpattr.Nlink++\n\t\tnow := time.Now()\n\t\tpattr.Mtime = now.Unix()\n\t\tpattr.Mtimensec = uint32(now.Nanosecond())\n\t\tpattr.Ctime = now.Unix()\n\t\tpattr.Ctimensec = uint32(now.Nanosecond())\n\t\ttx.set(m.inodeKey(parent), m.marshal(&pattr))\n\t\ttx.set(m.entryKey(parent, name), m.packEntry(TypeDirectory, inode))\n\t\ttx.delete(m.detachedKey(inode))\n\t\treturn nil\n\t}, parent))\n}\n\nfunc (m *kvMeta) doTouchAtime(ctx Context, inode Ino, attr *Attr, now time.Time) (bool, error) {\n\tvar updated bool\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\ta := tx.get(m.inodeKey(inode))\n\t\tif a == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tm.parseAttr(a, attr)\n\t\tif !m.atimeNeedsUpdate(attr, now) {\n\t\t\treturn nil\n\t\t}\n\t\tattr.Atime = now.Unix()\n\t\tattr.Atimensec = uint32(now.Nanosecond())\n\t\ttx.set(m.inodeKey(inode), m.marshal(attr))\n\t\tupdated = true\n\t\treturn nil\n\t}, inode)\n\treturn updated, err\n}\n\nfunc (m *kvMeta) doSetFacl(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tval := tx.get(m.inodeKey(ino))\n\t\tif val == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\tattr := &Attr{}\n\t\tm.parseAttr(val, attr)\n\n\t\tif ctx.Uid() != 0 && ctx.Uid() != attr.Uid {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\tif attr.Flags&FlagImmutable != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\n\t\toriACL, oriMode := getAttrACLId(attr, aclType), attr.Mode\n\n\t\t// https://github.com/torvalds/linux/blob/480e035fc4c714fb5536e64ab9db04fedc89e910/fs/fuse/acl.c#L143-L151\n\t\t// TODO: check linux capabilities\n\t\tif ctx.Uid() != 0 && !inGroup(ctx, attr.Gid) {\n\t\t\t// clear sgid\n\t\t\tattr.Mode &= 05777\n\t\t}\n\n\t\tif rule.IsEmpty() {\n\t\t\t// remove acl\n\t\t\tsetAttrACLId(attr, aclType, aclAPI.None)\n\t\t} else if rule.IsMinimal() && aclType == aclAPI.TypeAccess {\n\t\t\t// remove acl\n\t\t\tsetAttrACLId(attr, aclType, aclAPI.None)\n\t\t\t// set mode\n\t\t\tattr.Mode &= 07000\n\t\t\tattr.Mode |= ((rule.Owner & 7) << 6) | ((rule.Group & 7) << 3) | (rule.Other & 7)\n\t\t} else {\n\t\t\t// set acl\n\t\t\trule.InheritPerms(attr.Mode)\n\t\t\taclId, err := m.insertACL(tx, rule)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsetAttrACLId(attr, aclType, aclId)\n\n\t\t\t// set mode\n\t\t\tif aclType == aclAPI.TypeAccess {\n\t\t\t\tattr.Mode &= 07000\n\t\t\t\tattr.Mode |= ((rule.Owner & 7) << 6) | ((rule.Mask & 7) << 3) | (rule.Other & 7)\n\t\t\t}\n\t\t}\n\n\t\t// update attr\n\t\tif oriACL != getAttrACLId(attr, aclType) || oriMode != attr.Mode {\n\t\t\tnow := time.Now()\n\t\t\tattr.Ctime = now.Unix()\n\t\t\tattr.Ctimensec = uint32(now.Nanosecond())\n\t\t\ttx.set(m.inodeKey(ino), m.marshal(attr))\n\t\t}\n\t\treturn nil\n\t}, ino))\n}\n\nfunc (m *kvMeta) doGetFacl(ctx Context, ino Ino, aclType uint8, aclId uint32, rule *aclAPI.Rule) syscall.Errno {\n\treturn errno(m.client.txn(ctx, func(tx *kvTxn) error {\n\t\tif aclId == aclAPI.None {\n\t\t\tval := tx.get(m.inodeKey(ino))\n\t\t\tif val == nil {\n\t\t\t\treturn syscall.ENOENT\n\t\t\t}\n\t\t\tattr := &Attr{}\n\t\t\tm.parseAttr(val, attr)\n\t\t\tm.of.Update(ino, attr)\n\n\t\t\taclId = getAttrACLId(attr, aclType)\n\t\t}\n\n\t\ta, err := m.getACL(tx, aclId)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif a == nil {\n\t\t\treturn ENOATTR\n\t\t}\n\t\t*rule = *a\n\t\treturn nil\n\t}, 0))\n}\n\nfunc (m *kvMeta) insertACL(tx *kvTxn, rule *aclAPI.Rule) (uint32, error) {\n\tif rule == nil || rule.IsEmpty() {\n\t\treturn aclAPI.None, nil\n\t}\n\n\tif err := m.tryLoadMissACLs(tx); err != nil {\n\t\tlogger.Warnf(\"load miss acls error: %s\", err)\n\t}\n\n\tvar aclId uint32\n\tif aclId = m.aclCache.GetId(rule); aclId == aclAPI.None {\n\t\tnewId, err := m.incrCounter(aclCounter, 1)\n\t\tif err != nil {\n\t\t\treturn aclAPI.None, err\n\t\t}\n\t\taclId = uint32(newId)\n\n\t\ttx.set(m.aclKey(aclId), rule.Encode())\n\t\tm.aclCache.Put(aclId, rule)\n\t}\n\treturn aclId, nil\n}\n\nfunc (m *kvMeta) tryLoadMissACLs(tx *kvTxn) error {\n\tmissIds := m.aclCache.GetMissIds()\n\tif len(missIds) > 0 {\n\t\tmissKeys := make([][]byte, len(missIds))\n\t\tfor i, id := range missIds {\n\t\t\tmissKeys[i] = m.aclKey(id)\n\t\t}\n\n\t\tacls := tx.gets(missKeys...)\n\t\tfor i, data := range acls {\n\t\t\tvar rule aclAPI.Rule\n\t\t\tif len(data) > 0 {\n\t\t\t\trule.Decode(data)\n\t\t\t}\n\t\t\tm.aclCache.Put(missIds[i], &rule)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *kvMeta) getACL(tx *kvTxn, id uint32) (*aclAPI.Rule, error) {\n\tif id == aclAPI.None {\n\t\treturn nil, nil\n\t}\n\tif cRule := m.aclCache.Get(id); cRule != nil {\n\t\treturn cRule, nil\n\t}\n\n\tval := tx.get(m.aclKey(id))\n\tif val == nil {\n\t\treturn nil, syscall.EIO\n\t}\n\n\trule := &aclAPI.Rule{}\n\trule.Decode(val)\n\tm.aclCache.Put(id, rule)\n\treturn rule, nil\n}\n\nfunc (m *kvMeta) loadDumpedACLs(ctx Context) error {\n\tid2Rule := m.aclCache.GetAll()\n\tif len(id2Rule) == 0 {\n\t\treturn nil\n\t}\n\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tmaxId := uint32(0)\n\t\tfor id, rule := range id2Rule {\n\t\t\tif id > maxId {\n\t\t\t\tmaxId = id\n\t\t\t}\n\t\t\ttx.set(m.aclKey(id), rule.Encode())\n\t\t}\n\t\ttx.set(m.counterKey(aclCounter), packCounter(int64(maxId)))\n\t\treturn nil\n\t})\n}\n\nfunc (m *kvMeta) doStoreToken(ctx Context, token []byte) (id uint32, st syscall.Errno) {\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\tnewId, err := m.incrCounter(krbTokenCounter, 1)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttx.set(m.krbTokenKey(uint32(newId)), token)\n\t\tid = uint32(newId)\n\t\treturn nil\n\t})\n\treturn id, errno(err)\n}\n\nfunc (m *kvMeta) doUpdateToken(ctx Context, id uint32, token []byte) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tif tx.get(m.krbTokenKey(id)) == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\ttx.set(m.krbTokenKey(id), token)\n\t\treturn nil\n\t}))\n}\n\nfunc (m *kvMeta) doLoadToken(ctx Context, id uint32) (token []byte, st syscall.Errno) {\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\ttoken = tx.get(m.krbTokenKey(id))\n\t\tif token == nil {\n\t\t\treturn syscall.ENOENT\n\t\t}\n\t\treturn nil\n\t})\n\treturn token, errno(err)\n}\n\nfunc (m *kvMeta) doDeleteTokens(ctx Context, ids []uint32) syscall.Errno {\n\treturn errno(m.txn(ctx, func(tx *kvTxn) error {\n\t\tfor _, id := range ids {\n\t\t\ttx.delete(m.krbTokenKey(id))\n\t\t}\n\t\treturn nil\n\t}))\n}\n\nfunc (m *kvMeta) doListTokens(ctx Context) (tokens map[uint32][]byte, st syscall.Errno) {\n\ttokens = make(map[uint32][]byte)\n\terr := m.client.scan(m.fmtKey(\"KD\"), func(k, v []byte) bool {\n\t\trb := utils.FromBuffer(k[2:])\n\t\tid := rb.Get32()\n\t\ttokens[id] = v\n\t\treturn true\n\t})\n\treturn tokens, errno(err)\n}\n\ntype kvDirHandler struct {\n\tdirHandler\n}\n\nfunc (m *kvMeta) newDirHandler(inode Ino, plus bool, entries []*Entry) DirHandler {\n\ts := &kvDirHandler{\n\t\tdirHandler: dirHandler{\n\t\t\tinode: inode,\n\t\t\tplus: plus,\n\t\t\tinitEntries: entries,\n\t\t\tfetcher: m.getDirFetcher(),\n\t\t\tbatchNum: DirBatchNum[\"kv\"],\n\t\t},\n\t}\n\ts.batch, _ = s.fetch(Background(), 0)\n\treturn s\n}\n\nfunc (m *kvMeta) getDirFetcher() dirFetcher {\n\treturn func(ctx Context, inode Ino, cursor interface{}, offset, limit int, plus bool) (interface{}, []*Entry, error) {\n\t\tvar startKey []byte\n\t\tsCursor := \"\"\n\t\tvar total int\n\t\tif cursor == nil {\n\t\t\tif offset > 0 {\n\t\t\t\ttotal += offset\n\t\t\t}\n\t\t} else {\n\t\t\tlimit += 1 // skip the cursor\n\t\t\tsCursor = string(cursor.([]byte))\n\t\t}\n\t\ttotal += limit\n\t\tstartKey = m.entryKey(inode, sCursor)\n\t\tendKey := nextKey(m.entryKey(inode, \"\"))\n\n\t\tkeys, vals, err := m.scan(startKey, endKey, total, nil)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\tif cursor != nil {\n\t\t\tkeys, vals = keys[1:], vals[1:]\n\t\t}\n\n\t\tif total > limit && offset <= len(keys) {\n\t\t\tkeys, vals = keys[offset:], vals[offset:]\n\t\t}\n\n\t\tprefix := len(m.entryKey(inode, \"\"))\n\t\tentries := make([]*Entry, 0, len(keys))\n\t\tvar name []byte\n\t\tvar typ uint8\n\t\tvar ino Ino\n\t\tfor i, buf := range vals {\n\t\t\tname = keys[i]\n\t\t\ttyp, ino = m.parseEntry(buf)\n\t\t\tif len(name) == prefix {\n\t\t\t\tlogger.Errorf(\"Corrupt entry with empty name: inode %d parent %d\", ino, inode)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tentries = append(entries, &Entry{\n\t\t\t\tInode: ino,\n\t\t\t\tName: []byte(name)[prefix:],\n\t\t\t\tAttr: &Attr{Typ: typ},\n\t\t\t})\n\t\t}\n\n\t\tif plus {\n\t\t\tif err = m.fillAttr(entries); err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\n\t\tif len(entries) == 0 {\n\t\t\treturn nil, nil, nil\n\t\t}\n\t\treturn entries[len(entries)-1].Name, entries, nil\n\t}\n}\n" }, { "path": "pkg/meta/tkv_badger.go", "content": "//go:build !nobadger\n// +build !nobadger\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/dgraph-io/badger/v4\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\ntype badgerTxn struct {\n\tt *badger.Txn\n\tc *badger.DB\n}\n\nfunc (tx *badgerTxn) get(key []byte) []byte {\n\titem, err := tx.t.Get(key)\n\tif err == badger.ErrKeyNotFound {\n\t\treturn nil\n\t}\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tvalue, err := item.ValueCopy(nil)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn value\n}\n\nfunc (tx *badgerTxn) gets(keys ...[]byte) [][]byte {\n\tvalues := make([][]byte, len(keys))\n\tfor i, key := range keys {\n\t\tvalues[i] = tx.get(key)\n\t}\n\treturn values\n}\n\nfunc (tx *badgerTxn) scan(begin, end []byte, keysOnly bool, handler func(k, v []byte) bool) {\n\tvar prefix bool\n\toptions := badger.IteratorOptions{}\n\tif keysOnly {\n\t\toptions.PrefetchValues = false\n\t\toptions.PrefetchSize = 0\n\t}\n\tif bytes.Equal(nextKey(begin), end) {\n\t\tprefix = true\n\t\toptions.Prefix = begin\n\t}\n\tit := tx.t.NewIterator(options)\n\tif prefix {\n\t\tit.Rewind()\n\t} else {\n\t\tit.Seek(begin)\n\t}\n\tdefer it.Close()\n\tfor ; it.Valid(); it.Next() {\n\t\titem := it.Item()\n\t\tif !prefix && bytes.Compare(item.Key(), end) >= 0 {\n\t\t\tbreak\n\t\t}\n\t\tvar value []byte\n\t\tif !keysOnly {\n\t\t\tvar err error\n\t\t\tvalue, err = item.ValueCopy(nil)\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t}\n\t\tif !handler(item.KeyCopy(nil), value) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (tx *badgerTxn) exist(prefix []byte) bool {\n\tit := tx.t.NewIterator(badger.IteratorOptions{\n\t\tPrefix: prefix,\n\t\tPrefetchSize: 1,\n\t})\n\tdefer it.Close()\n\tit.Rewind()\n\treturn it.Valid()\n}\n\nfunc (tx *badgerTxn) set(key, value []byte) {\n\tif err := tx.t.Set(key, value); err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc (tx *badgerTxn) append(key []byte, value []byte) {\n\tlist := append(tx.get(key), value...)\n\ttx.set(key, list)\n}\n\nfunc (tx *badgerTxn) incrBy(key []byte, value int64) int64 {\n\tbuf := tx.get(key)\n\tnewCounter := parseCounter(buf)\n\tif value != 0 {\n\t\tnewCounter += value\n\t\ttx.set(key, packCounter(newCounter))\n\t}\n\treturn newCounter\n}\n\nfunc (tx *badgerTxn) delete(key []byte) {\n\tif err := tx.t.Delete(key); err != nil {\n\t\tpanic(err)\n\t}\n}\n\ntype badgerClient struct {\n\tclient *badger.DB\n\tticker *time.Ticker\n\tdone chan struct{}\n}\n\nfunc (c *badgerClient) name() string {\n\treturn \"badger\"\n}\n\nfunc (c *badgerClient) shouldRetry(err error) bool {\n\treturn err == badger.ErrConflict\n}\n\nfunc (c *badgerClient) config(key string) interface{} {\n\treturn nil\n}\n\nfunc (c *badgerClient) simpleTxn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\treturn c.txn(ctx, f, retry)\n}\n\nfunc (c *badgerClient) txn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\ttx := &badgerTxn{c.client.NewTransaction(true), c.client}\n\tdefer func() { tx.t.Discard() }()\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tfe, ok := r.(error)\n\t\t\tif ok {\n\t\t\t\terr = fe\n\t\t\t} else {\n\t\t\t\tpanic(r)\n\t\t\t}\n\t\t}\n\t}()\n\terr = f(&kvTxn{tx, retry})\n\tif err != nil {\n\t\treturn err\n\t}\n\t// tx.t may differ from the original\n\treturn tx.t.Commit()\n}\n\nfunc (c *badgerClient) scan(prefix []byte, handler func(key []byte, value []byte) bool) error {\n\ttx := c.client.NewTransaction(false)\n\tdefer tx.Discard()\n\tit := tx.NewIterator(badger.IteratorOptions{\n\t\tPrefix: prefix,\n\t\tPrefetchValues: true,\n\t\tPrefetchSize: 10240,\n\t})\n\tdefer it.Close()\n\tfor it.Rewind(); it.Valid(); it.Next() {\n\t\titem := it.Item()\n\t\tvalue, err := item.ValueCopy(nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !handler(item.KeyCopy(nil), value) {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *badgerClient) reset(prefix []byte) error {\n\tif prefix == nil {\n\t\treturn c.client.DropAll()\n\t}\n\treturn c.client.DropPrefix(prefix)\n}\n\nfunc (c *badgerClient) close() error {\n\tclose(c.done)\n\tc.ticker.Stop()\n\treturn c.client.Close()\n}\n\nfunc (c *badgerClient) gc() {}\n\nfunc newBadgerClient(addr string) (tkvClient, error) {\n\topt := badger.DefaultOptions(addr)\n\topt.Logger = utils.GetLogger(\"badger\")\n\topt.MetricsEnabled = false\n\tclient, err := badger.Open(opt)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tticker := time.NewTicker(time.Hour)\n\tdone := make(chan struct{})\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ticker.C:\n\t\t\t\tfor client.RunValueLogGC(0.7) == nil {\n\t\t\t\t}\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\treturn &badgerClient{client, ticker, done}, nil\n}\n\nfunc init() {\n\tRegister(\"badger\", newKVMeta)\n\tdrivers[\"badger\"] = newBadgerClient\n}\n" }, { "path": "pkg/meta/tkv_bak.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"sort\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta/pb\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/sync/errgroup\"\n\t\"google.golang.org/protobuf/proto\"\n)\n\nvar (\n\tkvDumpBatchSize = 10000\n)\n\nfunc (m *kvMeta) dump(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tvar dumps = []func(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error{\n\t\tm.dumpFormat,\n\t\tm.dumpCounters,\n\t\tm.dumpMix, // node, edge, chunk, symlink, xattr, parent\n\t\tm.dumpSustained,\n\t\tm.dumpDelFiles,\n\t\tm.dumpSliceRef,\n\t\tm.dumpACL,\n\t\tm.dumpQuota,\n\t\tm.dumpDirStat,\n\t}\n\tts := m.client.config(\"startTS\")\n\tif ts == nil && m.Name() == \"tikv\" {\n\t\treturn errors.New(\"failed to get startTS, which is required for TiKV to ensure consistency\")\n\t}\n\tif ts != nil {\n\t\tlogger.Infof(\"dump kv with startTS: %d\", ts.(uint64))\n\t\tctx = ctx.WithValue(txSessionKey{}, ts)\n\t}\n\n\tfor _, f := range dumps {\n\t\terr := f(ctx, opt, ch)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *kvMeta) load(ctx Context, typ int, opt *LoadOption, val proto.Message) error {\n\treturn errors.New(\"not implemented, use kvMeta.LoadMetaV2 instead\")\n}\n\nfunc (m *kvMeta) prepareLoad(ctx Context, opt *LoadOption) error {\n\topt.check()\n\n\tvar exist bool\n\terr := m.txn(ctx, func(tx *kvTxn) error {\n\t\texist = tx.exist(m.fmtKey())\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tif exist {\n\t\treturn fmt.Errorf(\"database %s://%s is not empty\", m.Name(), m.addr)\n\t}\n\treturn nil\n}\n\nfunc printSums(sums map[int]*atomic.Uint64) string {\n\tvar p string\n\tfor typ, sum := range sums {\n\t\tp += fmt.Sprintf(\"%d num: %d\\n\", typ, sum.Load())\n\t}\n\treturn p\n}\n\nfunc (m *kvMeta) dumpCounters(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tcounters := make([]*pb.Counter, 0, len(counterNames))\n\t\tfor _, name := range counterNames {\n\t\t\tval := tx.get(m.counterKey(name))\n\t\t\tcounters = append(counters, &pb.Counter{Key: name, Value: parseCounter(val)})\n\t\t}\n\t\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Counters: counters}})\n\t})\n}\n\nfunc splitInodeRange(n byte) [][2]byte {\n\tif n == 0 {\n\t\treturn nil\n\t}\n\n\tstep := 0xFF / n\n\tintervals := make([][2]byte, 0, n)\n\n\tfor i := byte(0); i < n; i++ {\n\t\tstart, end := i*step, (i+1)*step\n\t\tif i == n-1 {\n\t\t\tend = 0xFF\n\t\t}\n\t\tintervals = append(intervals, [2]byte{start, end})\n\t}\n\treturn intervals\n}\n\nfunc (m *kvMeta) dumpMix(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\tpools := map[int]*sync.Pool{\n\t\tsegTypeNode: {New: func() interface{} { return &pb.Node{} }},\n\t\tsegTypeEdge: {New: func() interface{} { return &pb.Edge{} }},\n\t\tsegTypeChunk: {New: func() interface{} { return &pb.Chunk{} }},\n\t\tsegTypeSymlink: {New: func() interface{} { return &pb.Symlink{} }},\n\t\tsegTypeXattr: {New: func() interface{} { return &pb.Xattr{} }},\n\t\tsegTypeParent: {New: func() interface{} { return &pb.Parent{} }},\n\t}\n\trelease := func(msg proto.Message) {\n\t\tbatch := msg.(*pb.Batch)\n\t\tfor _, node := range batch.Nodes {\n\t\t\tpools[segTypeNode].Put(node)\n\t\t}\n\t\tfor _, edge := range batch.Edges {\n\t\t\tpools[segTypeEdge].Put(edge)\n\t\t}\n\t\tfor _, chunk := range batch.Chunks {\n\t\t\tpools[segTypeChunk].Put(chunk)\n\t\t}\n\t\tfor _, symlink := range batch.Symlinks {\n\t\t\tpools[segTypeSymlink].Put(symlink)\n\t\t}\n\t\tfor _, xattr := range batch.Xattrs {\n\t\t\tpools[segTypeXattr].Put(xattr)\n\t\t}\n\t\tfor _, parent := range batch.Parents {\n\t\t\tpools[segTypeParent].Put(parent)\n\t\t}\n\t}\n\n\tvar sums = map[int]*atomic.Uint64{\n\t\tsegTypeNode: {},\n\t\tsegTypeEdge: {},\n\t\tsegTypeChunk: {},\n\t\tsegTypeSymlink: {},\n\t\tsegTypeXattr: {},\n\t\tsegTypeParent: {},\n\t}\n\tcreateMsg := func(typ int) *pb.Batch {\n\t\tswitch typ {\n\t\tcase segTypeNode:\n\t\t\treturn &pb.Batch{Nodes: make([]*pb.Node, 0, kvDumpBatchSize)}\n\t\tcase segTypeEdge:\n\t\t\treturn &pb.Batch{Edges: make([]*pb.Edge, 0, kvDumpBatchSize)}\n\t\tcase segTypeChunk:\n\t\t\treturn &pb.Batch{Chunks: make([]*pb.Chunk, 0, kvDumpBatchSize)}\n\t\tcase segTypeSymlink:\n\t\t\treturn &pb.Batch{Symlinks: make([]*pb.Symlink, 0, kvDumpBatchSize)}\n\t\tcase segTypeXattr:\n\t\t\treturn &pb.Batch{Xattrs: make([]*pb.Xattr, 0, kvDumpBatchSize)}\n\t\tcase segTypeParent:\n\t\t\treturn &pb.Batch{Parents: make([]*pb.Parent, 0, kvDumpBatchSize)}\n\t\tdefault:\n\t\t\treturn nil\n\t\t}\n\t}\n\tvar lists = make(map[int]*pb.Batch)\n\tfor typ := range sums {\n\t\tlists[typ] = createMsg(typ)\n\t}\n\n\tvar err error // final error\n\teg, egCtx := errgroup.WithContext(ctx)\n\teg.SetLimit(opt.Threads)\n\n\ttype entry struct {\n\t\tk []byte\n\t\tv []byte\n\t}\n\tentryPool := &sync.Pool{\n\t\tNew: func() interface{} {\n\t\t\treturn &entry{}\n\t\t},\n\t}\n\tentryCh := make(chan *entry, kvDumpBatchSize)\n\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tvar e *entry\n\t\tvar typ int\n\t\tvar n int\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase e = <-entryCh:\n\t\t\t}\n\t\t\tif e == nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tino := m.decodeInode(e.k[1:9])\n\t\t\tswitch e.k[9] {\n\t\t\tcase 'I':\n\t\t\t\ttyp = segTypeNode\n\t\t\t\tnode := pools[typ].Get().(*pb.Node)\n\t\t\t\tnode.Inode = uint64(ino)\n\t\t\t\tnode.Data = e.v\n\t\t\t\tlists[typ].Nodes = append(lists[typ].Nodes, node)\n\t\t\t\tn = len(lists[typ].Nodes)\n\t\t\tcase 'D':\n\t\t\t\ttyp = segTypeEdge\n\t\t\t\tedge := pools[typ].Get().(*pb.Edge)\n\t\t\t\tedge.Parent = uint64(ino)\n\t\t\t\tedge.Name = e.k[10:]\n\t\t\t\tnTyp, inode := m.parseEntry(e.v)\n\t\t\t\tedge.Type, edge.Inode = uint32(nTyp), uint64(inode)\n\t\t\t\tlists[typ].Edges = append(lists[typ].Edges, edge)\n\t\t\t\tn = len(lists[typ].Edges)\n\t\t\tcase 'C':\n\t\t\t\ttyp = segTypeChunk\n\t\t\t\tchk := pools[typ].Get().(*pb.Chunk)\n\t\t\t\tchk.Inode = uint64(ino)\n\t\t\t\tchk.Index = binary.BigEndian.Uint32(e.k[10:])\n\t\t\t\tchk.Slices = e.v\n\t\t\t\tlists[typ].Chunks = append(lists[typ].Chunks, chk)\n\t\t\t\tn = len(lists[typ].Chunks)\n\t\t\tcase 'S':\n\t\t\t\ttyp = segTypeSymlink\n\t\t\t\tsym := pools[typ].Get().(*pb.Symlink)\n\t\t\t\tsym.Inode = uint64(ino)\n\t\t\t\tsym.Target = unescape(string(e.v))\n\t\t\t\tlists[typ].Symlinks = append(lists[typ].Symlinks, sym)\n\t\t\t\tn = len(lists[typ].Symlinks)\n\t\t\tcase 'X':\n\t\t\t\ttyp = segTypeXattr\n\t\t\t\txattr := pools[typ].Get().(*pb.Xattr)\n\t\t\t\txattr.Inode = uint64(ino)\n\t\t\t\txattr.Name = string(e.k[10:])\n\t\t\t\txattr.Value = e.v\n\t\t\t\tlists[typ].Xattrs = append(lists[typ].Xattrs, xattr)\n\t\t\t\tn = len(lists[typ].Xattrs)\n\t\t\tcase 'P':\n\t\t\t\ttyp = segTypeParent\n\t\t\t\tparent := pools[typ].Get().(*pb.Parent)\n\t\t\t\tparent.Inode = uint64(ino)\n\t\t\t\tparent.Parent = uint64(m.decodeInode(e.k[10:]))\n\t\t\t\tparent.Cnt = parseCounter(e.v)\n\t\t\t\tlists[typ].Parents = append(lists[typ].Parents, parent)\n\t\t\t\tn = len(lists[typ].Parents)\n\t\t\tdefault:\n\t\t\t\ttyp = segTypeUnknown\n\t\t\t}\n\t\t\tentryPool.Put(e)\n\t\t\tif typ != segTypeUnknown {\n\t\t\t\tsums[typ].Add(1)\n\t\t\t\tif n >= kvDumpBatchSize {\n\t\t\t\t\tif err = dumpResult(ctx, ch, &dumpedResult{lists[typ], release}); err != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tlists[typ] = createMsg(typ)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor _, list := range lists {\n\t\t\t_ = dumpResult(ctx, ch, &dumpedResult{list, release})\n\t\t}\n\t}()\n\n\tif opt.Threads > 0xFF {\n\t\topt.Threads = 0xFF\n\t}\n\trs := splitInodeRange(byte(opt.Threads))\n\tfor i, r := range rs {\n\t\tstart, end := []byte{'A', r[0]}, []byte{'A', r[1]}\n\t\tif i == len(rs)-1 {\n\t\t\tend = []byte{'B'}\n\t\t}\n\t\tlogger.Debugf(\"range: %v-%v\", start, end)\n\t\teg.Go(func() error {\n\t\t\treturn m.txn(WrapContext(egCtx), func(tx *kvTxn) error {\n\t\t\t\tvar ent *entry\n\t\t\t\ttx.scan(start, end, false, func(k, v []byte) bool {\n\t\t\t\t\tif egCtx.Err() != nil {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t\tif len(k) <= 9 || k[0] != 'A' {\n\t\t\t\t\t\treturn true\n\t\t\t\t\t}\n\t\t\t\t\tent = entryPool.Get().(*entry)\n\t\t\t\t\tent.k, ent.v = k, v\n\t\t\t\t\tentryCh <- ent\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n\n\tif iErr := eg.Wait(); iErr != nil {\n\t\tctx.Cancel()\n\t\twg.Wait()\n\t\treturn iErr\n\t}\n\n\tclose(entryCh)\n\twg.Wait()\n\treturn err\n}\n\nfunc (m *kvMeta) dumpSustained(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tsids := make(map[uint64][]uint64)\n\t\tcnt := 0\n\t\ttx.scan(m.fmtKey(\"SS\"), nextKey(m.fmtKey(\"SS\")), true, func(k, v []byte) bool {\n\t\t\tb := utils.FromBuffer([]byte(k[2:])) // \"SS\"\n\t\t\tif b.Len() != 16 {\n\t\t\t\tlogger.Warnf(\"invalid sustainedKey: %s\", k)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tsid := b.Get64()\n\t\t\tinode := uint64(m.decodeInode(b.Get(8)))\n\t\t\tsids[sid] = append(sids[sid], inode)\n\t\t\tcnt++\n\t\t\treturn true\n\t\t})\n\n\t\tsustained := make([]*pb.Sustained, 0, cnt)\n\t\tfor sid, inodes := range sids {\n\t\t\tsustained = append(sustained, &pb.Sustained{\n\t\t\t\tSid: sid,\n\t\t\t\tInodes: inodes,\n\t\t\t})\n\t\t}\n\t\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Sustained: sustained}})\n\t})\n}\n\nfunc (m *kvMeta) dumpDelFiles(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tdelFiles := make([]*pb.DelFile, 0, kvDumpBatchSize)\n\t\ttx.scan(m.fmtKey(\"D\"), nextKey(m.fmtKey(\"D\")), false, func(k, v []byte) bool {\n\t\t\tb := utils.FromBuffer([]byte(k[1:])) // \"D\"\n\t\t\tif b.Len() != 16 {\n\t\t\t\tlogger.Warnf(\"invalid delfileKey: %s\", k)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tinode := m.decodeInode(b.Get(8))\n\t\t\tdelFiles = append(delFiles, &pb.DelFile{Inode: uint64(inode), Length: b.Get64(), Expire: m.parseInt64(v)})\n\t\t\treturn true\n\t\t})\n\t\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Delfiles: delFiles}})\n\t})\n}\n\nfunc (m *kvMeta) dumpSliceRef(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tsliceRefs := make([]*pb.SliceRef, 0, kvDumpBatchSize)\n\t\ttx.scan(m.fmtKey(\"K\"), nextKey(m.fmtKey(\"K\")), false, func(k, v []byte) bool {\n\t\t\tb := utils.FromBuffer([]byte(k[1:])) // \"K\"\n\t\t\tif b.Len() != 12 {\n\t\t\t\tlogger.Warnf(\"invalid sliceRefKey: %s\", k)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tid := b.Get64()\n\t\t\tsize := b.Get32()\n\t\t\tsliceRefs = append(sliceRefs, &pb.SliceRef{Id: id, Size: size, Refs: parseCounter(v) + 1})\n\t\t\treturn true\n\t\t})\n\t\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{SliceRefs: sliceRefs}})\n\t})\n}\n\nfunc (m *kvMeta) dumpACL(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tacls := make([]*pb.Acl, 0, 128)\n\t\ttx.scan(m.fmtKey(\"R\"), nextKey(m.fmtKey(\"R\")), false, func(k, v []byte) bool {\n\t\t\tb := utils.FromBuffer([]byte(k[1:])) // \"R\"\n\t\t\tif b.Len() != 4 {\n\t\t\t\tlogger.Warnf(\"invalid aclKey: %s\", k)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tacls = append(acls, &pb.Acl{Id: b.Get32(), Data: v})\n\t\t\treturn true\n\t\t})\n\t\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Acls: acls}})\n\t})\n}\n\nfunc (m *kvMeta) dumpQuota(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tquotas := make([]*pb.Quota, 0, 128)\n\t\ttx.scan(m.fmtKey(\"QD\"), nextKey(m.fmtKey(\"QD\")), false, func(k, v []byte) bool {\n\t\t\tq := &pb.Quota{}\n\t\t\tq.Inode = uint64(m.decodeInode([]byte(k)[2:]))\n\t\t\tb := utils.FromBuffer(v)\n\t\t\tq.MaxSpace = int64(b.Get64())\n\t\t\tq.MaxInodes = int64(b.Get64())\n\t\t\tq.UsedSpace = int64(b.Get64())\n\t\t\tq.UsedInodes = int64(b.Get64())\n\t\t\tquotas = append(quotas, q)\n\t\t\treturn true\n\t\t})\n\t\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Quotas: quotas}})\n\t})\n}\n\nfunc (m *kvMeta) dumpDirStat(ctx Context, opt *DumpOption, ch chan<- *dumpedResult) error {\n\treturn m.txn(ctx, func(tx *kvTxn) error {\n\t\tstats := make([]*pb.Stat, 0, kvDumpBatchSize)\n\t\ttx.scan(m.fmtKey(\"U\"), nextKey(m.fmtKey(\"U\")), false, func(k, v []byte) bool {\n\t\t\ts := &pb.Stat{}\n\t\t\ts.Inode = uint64(m.decodeInode([]byte(k)[1:]))\n\t\t\tb := utils.FromBuffer(v)\n\t\t\ts.DataLength = int64(b.Get64())\n\t\t\ts.UsedSpace = int64(b.Get64())\n\t\t\ts.UsedInodes = int64(b.Get64())\n\t\t\tstats = append(stats, s)\n\t\t\treturn true\n\t\t})\n\t\treturn dumpResult(ctx, ch, &dumpedResult{msg: &pb.Batch{Dirstats: stats}})\n\t})\n}\n\nfunc (m *kvMeta) insertKVs(ctx Context, pairs []*pair, threads int) error {\n\tif len(pairs) == 0 {\n\t\treturn nil\n\t}\n\n\tsort.Slice(pairs, func(i, j int) bool {\n\t\treturn bytes.Compare(pairs[i].key, pairs[j].key) < 0\n\t})\n\n\tmaxSize, maxNum := 5<<20, m.maxTxnBatchNum()\n\tn := len(pairs)\n\tlast, num, size := 0, 0, 0\n\n\teg, egCtx := errgroup.WithContext(ctx)\n\teg.SetLimit(threads)\n\n\tfor i, pair := range pairs {\n\t\tnum++\n\t\tsize += len(pair.key) + len(pair.value)\n\t\tif num >= maxNum || size >= maxSize || i >= n-1 {\n\t\t\tePairs := pairs[last : i+1]\n\t\t\tnum, size, last = 0, 0, i+1\n\t\t\teg.Go(func() error {\n\t\t\t\treturn m.txn(WrapContext(egCtx), func(tx *kvTxn) error {\n\t\t\t\t\tfor _, ep := range ePairs {\n\t\t\t\t\t\ttx.set(ep.key, ep.value)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t})\n\t\t}\n\t}\n\treturn eg.Wait()\n}\n\nfunc (m *kvMeta) loadFormat(ctx Context, msg proto.Message, pairs *[]*pair) {\n\t*pairs = append(*pairs, &pair{m.fmtKey(\"setting\"), msg.(*pb.Format).Data})\n}\n\nfunc (m *kvMeta) loadCounters(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tfor _, counter := range msg.(*pb.Batch).Counters {\n\t\t*pairs = append(*pairs, &pair{m.counterKey(counter.Key), packCounter(counter.Value)})\n\t}\n}\n\nfunc (m *kvMeta) loadNodes(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, pn := range batch.Nodes {\n\t\t*pairs = append(*pairs, &pair{m.inodeKey(Ino(pn.Inode)), pn.Data})\n\t}\n}\n\nfunc (m *kvMeta) loadChunks(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, chk := range batch.Chunks {\n\t\t*pairs = append(*pairs, &pair{m.chunkKey(Ino(chk.Inode), chk.Index), chk.Slices})\n\t}\n}\n\nfunc (m *kvMeta) loadEdges(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, edge := range batch.Edges {\n\t\tbuff := utils.NewBuffer(9)\n\t\tbuff.Put8(uint8(edge.Type))\n\t\tbuff.Put64(edge.Inode)\n\t\t*pairs = append(*pairs, &pair{m.entryKey(Ino(edge.Parent), string(edge.Name)), buff.Bytes()})\n\t}\n}\n\nfunc (m *kvMeta) loadSymlinks(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, symlink := range batch.Symlinks {\n\t\t*pairs = append(*pairs, &pair{m.symKey(Ino(symlink.Inode)), []byte(symlink.Target)})\n\t}\n}\n\nfunc (m *kvMeta) loadSustained(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, sustained := range batch.Sustained {\n\t\tfor _, inode := range sustained.Inodes {\n\t\t\t*pairs = append(*pairs, &pair{m.sustainedKey(sustained.Sid, Ino(inode)), []byte{1}})\n\t\t}\n\t}\n}\n\nfunc (m *kvMeta) loadDelFiles(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, f := range batch.Delfiles {\n\t\t*pairs = append(*pairs, &pair{m.delfileKey(Ino(f.Inode), f.Length), m.packInt64(f.Expire)})\n\t}\n}\n\nfunc (m *kvMeta) loadSliceRefs(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, r := range batch.SliceRefs {\n\t\t*pairs = append(*pairs, &pair{m.sliceKey(r.Id, r.Size), packCounter(r.Refs - 1)})\n\t}\n}\n\nfunc (m *kvMeta) loadAcl(ctx Context, msg proto.Message, pairs *[]*pair, maxAclId *uint32) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, acl := range batch.Acls {\n\t\tif acl.Id > *maxAclId {\n\t\t\t*maxAclId = acl.Id\n\t\t}\n\t\t*pairs = append(*pairs, &pair{m.aclKey(acl.Id), acl.Data})\n\t}\n}\n\nfunc (m *kvMeta) loadXattrs(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, xattr := range batch.Xattrs {\n\t\t*pairs = append(*pairs, &pair{m.xattrKey(Ino(xattr.Inode), xattr.Name), xattr.Value})\n\t}\n}\n\nfunc (m *kvMeta) loadQuota(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, q := range batch.Quotas {\n\t\tb := utils.NewBuffer(32)\n\t\tb.Put64(uint64(q.MaxSpace))\n\t\tb.Put64(uint64(q.MaxInodes))\n\t\tb.Put64(uint64(q.UsedSpace))\n\t\tb.Put64(uint64(q.UsedInodes))\n\t\t*pairs = append(*pairs, &pair{m.dirQuotaKey(Ino(q.Inode)), b.Bytes()})\n\t}\n}\n\nfunc (m *kvMeta) loadDirStats(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, s := range batch.Dirstats {\n\t\tb := utils.NewBuffer(24)\n\t\tb.Put64(uint64(s.DataLength))\n\t\tb.Put64(uint64(s.UsedSpace))\n\t\tb.Put64(uint64(s.UsedInodes))\n\t\t*pairs = append(*pairs, &pair{m.dirStatKey(Ino(s.Inode)), b.Bytes()})\n\t}\n}\n\nfunc (m *kvMeta) loadParents(ctx Context, msg proto.Message, pairs *[]*pair) {\n\tbatch := msg.(*pb.Batch)\n\tfor _, parent := range batch.Parents {\n\t\t*pairs = append(*pairs, &pair{m.parentKey(Ino(parent.Inode), Ino(parent.Parent)), packCounter(parent.Cnt)})\n\t}\n}\n\nfunc (m *kvMeta) maxTxnBatchNum() int {\n\tif m.Name() == \"etcd\" {\n\t\treturn 128\n\t}\n\treturn 10240\n}\n\nfunc (m *kvMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error {\n\tif opt == nil {\n\t\topt = &LoadOption{}\n\t}\n\tif err := m.en.prepareLoad(ctx, opt); err != nil {\n\t\treturn err\n\t}\n\n\ttype task struct {\n\t\ttyp int\n\t\tmsg proto.Message\n\t}\n\ttaskCh := make(chan *task, 100)\n\n\tvar (\n\t\twg sync.WaitGroup\n\t\tmaxAclId uint32\n\t)\n\tworkerFunc := func(ctx Context, taskCh <-chan *task) {\n\t\tdefer wg.Done()\n\t\tvar task *task\n\t\tmaxNum := m.maxTxnBatchNum() * opt.Threads\n\t\tpairs := make([]*pair, 0, maxNum)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase task = <-taskCh:\n\t\t\t}\n\t\t\tif task == nil {\n\t\t\t\tif err := m.insertKVs(ctx, pairs, opt.Threads); err != nil {\n\t\t\t\t\tlogger.Errorf(\"insert kvs failed: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tif maxAclId != 0 {\n\t\t\t\t\tif err := m.txn(ctx, func(tx *kvTxn) error {\n\t\t\t\t\t\ttx.set(m.counterKey(aclCounter), packCounter(int64(maxAclId)))\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlogger.Errorf(\"update maxAclId failed: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tswitch task.typ {\n\t\t\tcase segTypeFormat:\n\t\t\t\tm.loadFormat(ctx, task.msg, &pairs)\n\t\t\tcase segTypeCounter:\n\t\t\t\tm.loadCounters(ctx, task.msg, &pairs)\n\t\t\tcase segTypeNode:\n\t\t\t\tm.loadNodes(ctx, task.msg, &pairs)\n\t\t\tcase segTypeEdge:\n\t\t\t\tm.loadEdges(ctx, task.msg, &pairs)\n\t\t\tcase segTypeChunk:\n\t\t\t\tm.loadChunks(ctx, task.msg, &pairs)\n\t\t\tcase segTypeSymlink:\n\t\t\t\tm.loadSymlinks(ctx, task.msg, &pairs)\n\t\t\tcase segTypeXattr:\n\t\t\t\tm.loadXattrs(ctx, task.msg, &pairs)\n\t\t\tcase segTypeParent:\n\t\t\t\tm.loadParents(ctx, task.msg, &pairs)\n\t\t\tcase segTypeSustained:\n\t\t\t\tm.loadSustained(ctx, task.msg, &pairs)\n\t\t\tcase segTypeDelFile:\n\t\t\t\tm.loadDelFiles(ctx, task.msg, &pairs)\n\t\t\tcase segTypeSliceRef:\n\t\t\t\tm.loadSliceRefs(ctx, task.msg, &pairs)\n\t\t\tcase segTypeAcl:\n\t\t\t\tm.loadAcl(ctx, task.msg, &pairs, &maxAclId)\n\t\t\tcase segTypeQuota:\n\t\t\t\tm.loadQuota(ctx, task.msg, &pairs)\n\t\t\tcase segTypeStat:\n\t\t\t\tm.loadDirStats(ctx, task.msg, &pairs)\n\t\t\t}\n\t\t\tif len(pairs) >= maxNum {\n\t\t\t\tif err := m.insertKVs(ctx, pairs, opt.Threads); err != nil {\n\t\t\t\t\tlogger.Errorf(\"insert kvs failed: %v\", err)\n\t\t\t\t\tctx.Cancel()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tpairs = make([]*pair, 0, maxNum)\n\t\t\t}\n\t\t}\n\t}\n\n\twg.Add(1)\n\tgo workerFunc(ctx, taskCh)\n\n\tbak := &BakFormat{}\n\tfor {\n\t\tseg, err := bak.ReadSegment(r)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, errBakEOF) {\n\t\t\t\tclose(taskCh)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tctx.Cancel()\n\t\t\twg.Wait()\n\t\t\treturn err\n\t\t}\n\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\twg.Wait()\n\t\t\treturn ctx.Err()\n\t\tcase taskCh <- &task{int(seg.typ), seg.val}:\n\t\t\tif opt.Progress != nil {\n\t\t\t\topt.Progress(seg.Name(), int(seg.num()))\n\t\t\t}\n\t\t}\n\t}\n\twg.Wait()\n\treturn nil\n}\n" }, { "path": "pkg/meta/tkv_etcd.go", "content": "//go:build !noetcd\n// +build !noetcd\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\tetcd \"go.etcd.io/etcd/client/v3\"\n\t\"go.etcd.io/etcd/pkg/transport\"\n)\n\ntype etcdTxn struct {\n\tctx context.Context\n\tkv etcd.KV\n\tobserved map[string]int64\n\tbuffer map[string][]byte\n}\n\nfunc (tx *etcdTxn) get(key []byte) []byte {\n\tk := string(key)\n\tif v, ok := tx.buffer[k]; ok {\n\t\treturn v\n\t}\n\tresp, err := tx.kv.Get(tx.ctx, k, etcd.WithLimit(1))\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"get %v: %s\", k, err))\n\t}\n\tif resp.Count == 0 {\n\t\ttx.observed[k] = 0\n\t\treturn nil\n\t}\n\tif resp.Count > 1 {\n\t\tpanic(fmt.Errorf(\"expect 1 keys but got %d\", resp.Count))\n\t}\n\tfor _, pair := range resp.Kvs {\n\t\tif bytes.Equal(pair.Key, key) {\n\t\t\ttx.observed[k] = pair.ModRevision\n\t\t\treturn pair.Value\n\t\t} else {\n\t\t\tpanic(fmt.Errorf(\"expect key %v, but got %v\", k, string(pair.Key)))\n\t\t}\n\t}\n\tpanic(\"unreachable\")\n}\n\nfunc (tx *etcdTxn) gets(keys ...[]byte) [][]byte {\n\tif len(keys) > 128 {\n\t\tvar rs = make([][]byte, 0, len(keys))\n\t\tfor i := 0; i < len(keys); i += 128 {\n\t\t\trs = append(rs, tx.gets(keys[i:min(i+128, len(keys))]...)...)\n\t\t}\n\t\treturn rs\n\t}\n\tops := make([]etcd.Op, len(keys))\n\tfor i, key := range keys {\n\t\tops[i] = etcd.OpGet(string(key))\n\t}\n\tr, err := tx.kv.Do(tx.ctx, etcd.OpTxn(nil, ops, nil))\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"batch get with %d keys: %s\", len(keys), err))\n\t}\n\trs := make(map[string][]byte)\n\tfor _, res := range r.Txn().Responses {\n\t\tfor _, p := range res.GetResponseRange().Kvs {\n\t\t\tk := string(p.Key)\n\t\t\ttx.observed[k] = p.ModRevision\n\t\t\trs[k] = p.Value\n\t\t}\n\t}\n\tvalues := make([][]byte, len(keys))\n\tfor i, key := range keys {\n\t\tk := string(key)\n\t\tif v, ok := tx.buffer[k]; ok {\n\t\t\tvalues[i] = v\n\t\t\tcontinue\n\t\t}\n\t\tvalues[i] = rs[k]\n\t\tif len(values[i]) == 0 {\n\t\t\ttx.observed[k] = 0\n\t\t}\n\t}\n\treturn values\n}\n\nfunc (tx *etcdTxn) scan(begin, end []byte, keysOnly bool, handler func(k, v []byte) bool) {\n\topts := []etcd.OpOption{etcd.WithRange(string(end))}\n\tif keysOnly {\n\t\topts = append(opts, etcd.WithKeysOnly())\n\t}\n\tresp, err := tx.kv.Get(tx.ctx, string(begin), opts...)\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"get range [%v-%v): %s\", string(begin), string(end), err))\n\t}\n\tfor _, kv := range resp.Kvs {\n\t\ttx.observed[string(kv.Key)] = kv.ModRevision\n\t\tif !handler(kv.Key, kv.Value) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (tx *etcdTxn) exist(prefix []byte) bool {\n\tresp, err := tx.kv.Get(tx.ctx, string(prefix), etcd.WithPrefix(), etcd.WithCountOnly())\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"get prefix %v with count only: %s\", string(prefix), err))\n\t}\n\treturn resp.Count > 0\n}\n\nfunc (tx *etcdTxn) set(key, value []byte) {\n\ttx.buffer[string(key)] = value\n\tif len(tx.buffer) >= 128 {\n\t\terr := tx.commmit()\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\ttx.observed = make(map[string]int64)\n\t\ttx.buffer = make(map[string][]byte)\n\t}\n}\n\nfunc (tx *etcdTxn) append(key []byte, value []byte) {\n\tnew := append(tx.get(key), value...)\n\ttx.set(key, new)\n}\n\nfunc (tx *etcdTxn) incrBy(key []byte, value int64) int64 {\n\tbuf := tx.get(key)\n\tnew := parseCounter(buf)\n\tif value != 0 {\n\t\tnew += value\n\t\ttx.set(key, packCounter(new))\n\t}\n\treturn new\n}\n\nfunc (tx *etcdTxn) delete(key []byte) {\n\ttx.buffer[string(key)] = nil\n}\n\nfunc (tx *etcdTxn) commmit() error {\n\tstart := time.Now()\n\tvar conds []etcd.Cmp\n\tvar ops []etcd.Op\n\tfor k, v := range tx.observed {\n\t\tconds = append(conds, etcd.Compare(etcd.ModRevision(k), \"=\", v))\n\t}\n\tfor k, v := range tx.buffer {\n\t\tvar op etcd.Op\n\t\tif v == nil {\n\t\t\top = etcd.OpDelete(string(k))\n\t\t} else {\n\t\t\top = etcd.OpPut(string(k), string(v))\n\t\t}\n\t\tops = append(ops, op)\n\t}\n\tresp, err := tx.kv.Txn(tx.ctx).If(conds...).Then(ops...).Commit()\n\tif time.Since(start) > time.Millisecond*10 {\n\t\tlogger.Debugf(\"txn with %d conds and %d ops took %s\", len(conds), len(ops), time.Since(start))\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.Succeeded {\n\t\treturn nil\n\t}\n\treturn conflicted\n}\n\ntype etcdClient struct {\n\tclient *etcd.Client\n\tkv etcd.KV\n}\n\nfunc (c *etcdClient) name() string {\n\treturn \"etcd\"\n}\n\nfunc (c *etcdClient) shouldRetry(err error) bool {\n\treturn errors.Is(err, conflicted)\n}\n\nfunc (c *etcdClient) config(key string) interface{} {\n\treturn nil\n}\n\nfunc (c *etcdClient) simpleTxn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\treturn c.txn(ctx, f, retry)\n}\n\nfunc (c *etcdClient) txn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\ttx := &etcdTxn{\n\t\tctx,\n\t\tc.kv,\n\t\tmake(map[string]int64),\n\t\tmake(map[string][]byte),\n\t}\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tfe, ok := r.(error)\n\t\t\tif ok {\n\t\t\t\terr = fe\n\t\t\t} else {\n\t\t\t\tpanic(r)\n\t\t\t}\n\t\t}\n\t}()\n\terr = f(&kvTxn{tx, retry})\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(tx.buffer) == 0 {\n\t\treturn nil // read only\n\t}\n\treturn tx.commmit()\n}\n\nvar conflicted = errors.New(\"conflicted transaction\")\n\nfunc (c *etcdClient) scan(prefix []byte, handler func(key []byte, value []byte) bool) error {\n\tvar start = prefix\n\tvar end = string(nextKey(prefix))\n\tresp, err := c.client.Get(context.Background(), \"anything\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tcurrentRev := resp.Header.Revision\n\tvar following bool\n\tfor {\n\t\tresp, err := c.client.Get(context.Background(),\n\t\t\tstring(start),\n\t\t\tetcd.WithRange(end),\n\t\t\tetcd.WithLimit(1024),\n\t\t\tetcd.WithMaxModRev(currentRev),\n\t\t\tetcd.WithSerializable())\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"get start %v: %s\", string(start), err)\n\t\t}\n\t\tif following && len(resp.Kvs) > 0 {\n\t\t\tresp.Kvs = resp.Kvs[1:]\n\t\t}\n\t\tif len(resp.Kvs) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tfor _, kv := range resp.Kvs {\n\t\t\tif !handler(kv.Key, kv.Value) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tstart = resp.Kvs[len(resp.Kvs)-1].Key\n\t\tfollowing = true\n\t}\n\treturn nil\n}\n\nfunc (c *etcdClient) reset(prefix []byte) error {\n\t_, err := c.kv.Delete(context.Background(), string(prefix), etcd.WithPrefix())\n\treturn err\n}\n\nfunc (c *etcdClient) close() error {\n\treturn c.client.Close()\n}\n\nfunc (c *etcdClient) gc() {}\n\nfunc buildTlsConfig(u *url.URL) (*tls.Config, error) {\n\tvar tsinfo transport.TLSInfo\n\tq := u.Query()\n\ttsinfo.CAFile = q.Get(\"cacert\")\n\ttsinfo.CertFile = q.Get(\"cert\")\n\ttsinfo.KeyFile = q.Get(\"key\")\n\ttsinfo.ServerName = q.Get(\"server-name\")\n\ttsinfo.InsecureSkipVerify = q.Get(\"insecure-skip-verify\") != \"\"\n\tif tsinfo.CAFile != \"\" || tsinfo.CertFile != \"\" || tsinfo.KeyFile != \"\" || tsinfo.ServerName != \"\" {\n\t\treturn tsinfo.ClientConfig()\n\t}\n\treturn nil, nil\n}\n\nfunc newEtcdClient(addr string) (tkvClient, error) {\n\tif !strings.Contains(addr, \"://\") {\n\t\taddr = \"http://\" + addr\n\t}\n\tu, err := url.Parse(addr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"parse %s: %s\", addr, err)\n\t}\n\tpasswd, _ := u.User.Password()\n\thosts := strings.Split(u.Host, \",\")\n\tfor i, h := range hosts {\n\t\th, _, err := net.SplitHostPort(h)\n\t\tif err != nil {\n\t\t\thosts[i] = net.JoinHostPort(h, \"2379\")\n\t\t}\n\t}\n\tconf := etcd.Config{\n\t\tEndpoints: hosts,\n\t\tUsername: u.User.Username(),\n\t\tPassword: passwd,\n\t\tAutoSyncInterval: time.Minute,\n\t}\n\tconf.TLS, err = buildTlsConfig(u)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"build tls config from %s: %s\", u.RawQuery, err)\n\t}\n\tc, err := etcd.New(conf)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tmaxCompactSlices = 100\n\tvar prefix string = u.Path + \"\\xFD\"\n\treturn withPrefix(&etcdClient{c, etcd.NewKV(c)}, []byte(prefix)), nil\n}\n\nfunc init() {\n\tRegister(\"etcd\", newKVMeta)\n\tdrivers[\"etcd\"] = newEtcdClient\n}\n" }, { "path": "pkg/meta/tkv_fdb.go", "content": "//go:build fdb\n// +build fdb\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/url\"\n\n\t\"github.com/apple/foundationdb/bindings/go/src/fdb\"\n)\n\nfunc init() {\n\tRegister(\"fdb\", newKVMeta)\n\tdrivers[\"fdb\"] = newFdbClient\n}\n\ntype fdbTxn struct {\n\tfdb.Transaction\n}\n\ntype fdbClient struct {\n\tclient fdb.Database\n}\n\nfunc newFdbClient(addr string) (tkvClient, error) {\n\terr := fdb.APIVersion(630)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"set API version: %s\", err)\n\t}\n\tu, err := url.Parse(\"fdb://\" + addr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdb, err := fdb.OpenDatabase(u.Path)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"open database: %s\", err)\n\t}\n\t// TODO: database options\n\treturn withPrefix(&fdbClient{db}, append([]byte(u.Query().Get(\"prefix\")), 0xFD)), nil\n}\n\nfunc (c *fdbClient) name() string {\n\treturn \"fdb\"\n}\n\nfunc (c *fdbClient) config(key string) interface{} {\n\treturn nil\n}\n\nfunc (c *fdbClient) simpleTxn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\treturn c.txn(ctx, f, retry)\n}\n\nfunc (c *fdbClient) txn(ctx context.Context, f func(*kvTxn) error, retry int) error {\n\t_, err := c.client.Transact(func(t fdb.Transaction) (interface{}, error) {\n\t\te := f(&kvTxn{&fdbTxn{t}, retry})\n\t\treturn nil, e\n\t})\n\treturn err\n}\n\nfunc (c *fdbClient) scan(prefix []byte, handler func(key, value []byte) bool) error {\n\tbegin := fdb.Key(prefix)\n\tend := fdb.Key(nextKey(prefix))\n\tlimit := 102400\n\tvar done bool\n\tfor {\n\t\tif _, err := c.client.ReadTransact(func(t fdb.ReadTransaction) (interface{}, error) {\n\t\t\t// TODO: t.Options().SetPriorityBatch()\n\t\t\tsnapshot := t.Snapshot()\n\t\t\titer := snapshot.GetRange(\n\t\t\t\tfdb.KeyRange{Begin: begin, End: end},\n\t\t\t\tfdb.RangeOptions{Limit: limit, Mode: fdb.StreamingModeWantAll},\n\t\t\t).Iterator()\n\t\t\tvar r fdb.KeyValue\n\t\t\tvar count int\n\t\t\tfor iter.Advance() {\n\t\t\t\tr = iter.MustGet()\n\t\t\t\tif !handler(r.Key, r.Value) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tif count < limit {\n\t\t\t\tdone = true\n\t\t\t} else {\n\t\t\t\tbegin = append(r.Key, 0)\n\t\t\t}\n\t\t\treturn nil, nil\n\t\t}); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif done {\n\t\t\treturn nil\n\t\t}\n\t}\n}\n\nfunc (c *fdbClient) reset(prefix []byte) error {\n\t_, err := c.client.Transact(func(t fdb.Transaction) (interface{}, error) {\n\t\tt.ClearRange(fdb.KeyRange{\n\t\t\tBegin: fdb.Key(prefix),\n\t\t\tEnd: fdb.Key(nextKey(prefix)),\n\t\t})\n\t\treturn nil, nil\n\t})\n\treturn err\n}\n\nfunc (c *fdbClient) close() error {\n\treturn nil\n}\n\nfunc (c *fdbClient) shouldRetry(err error) bool {\n\treturn false\n}\n\nfunc (c *fdbClient) gc() {}\n\nfunc (tx *fdbTxn) get(key []byte) []byte {\n\treturn tx.Get(fdb.Key(key)).MustGet()\n}\n\nfunc (tx *fdbTxn) gets(keys ...[]byte) [][]byte {\n\tfut := make([]fdb.FutureByteSlice, len(keys))\n\tfor i, key := range keys {\n\t\tfut[i] = tx.Get(fdb.Key(key))\n\t}\n\tret := make([][]byte, len(keys))\n\tfor i, f := range fut {\n\t\tret[i] = f.MustGet()\n\t}\n\treturn ret\n}\n\nfunc (tx *fdbTxn) scan(begin, end []byte, keysOnly bool, handler func(k, v []byte) bool) {\n\tit := tx.GetRange(fdb.KeyRange{Begin: fdb.Key(begin), End: fdb.Key(end)},\n\t\tfdb.RangeOptions{Mode: fdb.StreamingModeWantAll}).Iterator()\n\tfor it.Advance() {\n\t\tkv := it.MustGet()\n\t\tif !handler(kv.Key, kv.Value) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (tx *fdbTxn) exist(prefix []byte) bool {\n\treturn tx.GetRange(\n\t\tfdb.KeyRange{Begin: fdb.Key(prefix), End: fdb.Key(nextKey(prefix))},\n\t\tfdb.RangeOptions{Mode: fdb.StreamingModeWantAll},\n\t).Iterator().Advance()\n}\n\nfunc (tx *fdbTxn) set(key, value []byte) {\n\ttx.Set(fdb.Key(key), value)\n}\n\nfunc (tx *fdbTxn) append(key []byte, value []byte) {\n\ttx.AppendIfFits(fdb.Key(key), fdb.Key(value))\n}\n\nfunc (tx *fdbTxn) incrBy(key []byte, value int64) int64 {\n\ttx.Add(fdb.Key(key), packCounter(value))\n\t// TODO: don't return new value if not needed\n\treturn parseCounter(tx.Get(fdb.Key(key)).MustGet())\n}\n\nfunc (tx *fdbTxn) delete(key []byte) {\n\ttx.Clear(fdb.Key(key))\n}\n" }, { "path": "pkg/meta/tkv_fdb_test.go", "content": "//go:build fdb\n// +build fdb\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n//nolint:errcheck\npackage meta\n\nimport (\n\t\"testing\"\n)\n\nfunc TestFdbClient(t *testing.T) { //skip mutate\n\tm, err := newKVMeta(\"fdb\", \"/etc/foundationdb/fdb.cluster?prefix=test2\", testConfig())\n\tif err != nil {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestFdb(t *testing.T) { //skip mutate\n\tc, err := newFdbClient(\"/etc/foundationdb/fdb.cluster?prefix=test1\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestTKV(t, c)\n}\n" }, { "path": "pkg/meta/tkv_lock.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"strconv\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\ntype lockOwner struct {\n\tsid uint64\n\towner uint64\n}\n\nfunc marshalFlock(ls map[lockOwner]byte) []byte {\n\tb := utils.NewBuffer(uint32(len(ls)) * 17)\n\tfor o, l := range ls {\n\t\tb.Put64(o.sid)\n\t\tb.Put64(o.owner)\n\t\tb.Put8(l)\n\t}\n\treturn b.Bytes()\n}\n\nfunc unmarshalFlock(buf []byte) map[lockOwner]byte {\n\tb := utils.FromBuffer(buf)\n\tvar ls = make(map[lockOwner]byte)\n\tfor b.HasMore() {\n\t\tsid := b.Get64()\n\t\towner := b.Get64()\n\t\tltype := b.Get8()\n\t\tls[lockOwner{sid, owner}] = ltype\n\t}\n\treturn ls\n}\n\nfunc (m *kvMeta) Flock(ctx Context, inode Ino, owner uint64, ltype uint32, block bool) syscall.Errno {\n\tikey := m.flockKey(inode)\n\tctx = ctx.WithValue(txMethodKey{}, \"Flock\"+strconv.Itoa(int(ltype)))\n\tvar err error\n\tlkey := lockOwner{m.sid, owner}\n\tfor {\n\t\terr = m.txn(ctx, func(tx *kvTxn) error {\n\t\t\tv := tx.get(ikey)\n\t\t\tls := unmarshalFlock(v)\n\t\t\tswitch ltype {\n\t\t\tcase F_UNLCK:\n\t\t\t\tdelete(ls, lkey)\n\t\t\tcase F_RDLCK:\n\t\t\t\tfor o, l := range ls {\n\t\t\t\t\tif l == 'W' && o != lkey {\n\t\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tls[lkey] = 'R'\n\t\t\tcase F_WRLCK:\n\t\t\t\tdelete(ls, lkey)\n\t\t\t\tif len(ls) > 0 {\n\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t}\n\t\t\t\tls[lkey] = 'W'\n\t\t\tdefault:\n\t\t\t\treturn syscall.EINVAL\n\t\t\t}\n\t\t\tif len(ls) == 0 {\n\t\t\t\ttx.delete(ikey)\n\t\t\t} else {\n\t\t\t\ttx.set(ikey, marshalFlock(ls))\n\t\t\t}\n\t\t\treturn nil\n\t\t}, inode)\n\n\t\tif !block || err != syscall.EAGAIN {\n\t\t\tbreak\n\t\t}\n\t\tif ltype == F_WRLCK {\n\t\t\ttime.Sleep(time.Millisecond * 1)\n\t\t} else {\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t}\n\treturn errno(err)\n}\n\nfunc marshalPlock(ls map[lockOwner][]byte) []byte {\n\tvar size uint32\n\tfor _, l := range ls {\n\t\tsize += 8 + 8 + 4 + uint32(len(l))\n\t}\n\tb := utils.NewBuffer(size)\n\tfor k, records := range ls {\n\t\tb.Put64(k.sid)\n\t\tb.Put64(k.owner)\n\t\tb.Put32(uint32(len(records)))\n\t\tb.Put(records)\n\t}\n\treturn b.Bytes()\n}\n\nfunc unmarshalPlock(buf []byte) map[lockOwner][]byte {\n\tb := utils.FromBuffer(buf)\n\tvar ls = make(map[lockOwner][]byte)\n\tfor b.HasMore() {\n\t\tsid := b.Get64()\n\t\towner := b.Get64()\n\t\trecords := b.Get(int(b.Get32()))\n\t\tls[lockOwner{sid, owner}] = records\n\t}\n\treturn ls\n}\n\nfunc (m *kvMeta) Getlk(ctx Context, inode Ino, owner uint64, ltype *uint32, start, end *uint64, pid *uint32) syscall.Errno {\n\tif *ltype == F_UNLCK {\n\t\t*start = 0\n\t\t*end = 0\n\t\t*pid = 0\n\t\treturn 0\n\t}\n\tv, err := m.get(m.plockKey(inode))\n\tif err != nil {\n\t\treturn errno(err)\n\t}\n\towners := unmarshalPlock(v)\n\tdelete(owners, lockOwner{m.sid, owner})\n\tfor o, records := range owners {\n\t\tls := loadLocks(records)\n\t\tfor _, l := range ls {\n\t\t\t// find conflicted locks\n\t\t\tif (*ltype == F_WRLCK || l.Type == F_WRLCK) && *end >= l.Start && *start <= l.End {\n\t\t\t\t*ltype = l.Type\n\t\t\t\t*start = l.Start\n\t\t\t\t*end = l.End\n\t\t\t\tif o.sid == m.sid {\n\t\t\t\t\t*pid = l.Pid\n\t\t\t\t} else {\n\t\t\t\t\t*pid = 0\n\t\t\t\t}\n\t\t\t\treturn 0\n\t\t\t}\n\t\t}\n\t}\n\t*ltype = F_UNLCK\n\t*start = 0\n\t*end = 0\n\t*pid = 0\n\treturn 0\n}\n\nfunc (m *kvMeta) Setlk(ctx Context, inode Ino, owner uint64, block bool, ltype uint32, start, end uint64, pid uint32) syscall.Errno {\n\tikey := m.plockKey(inode)\n\tctx = ctx.WithValue(txMethodKey{}, \"Setlk\"+strconv.Itoa(int(ltype)))\n\tvar err error\n\tlock := plockRecord{ltype, pid, start, end}\n\tlkey := lockOwner{m.sid, owner}\n\tfor {\n\t\terr = m.txn(ctx, func(tx *kvTxn) error {\n\t\t\towners := unmarshalPlock(tx.get(ikey))\n\t\t\tif ltype == F_UNLCK {\n\t\t\t\trecords := owners[lkey]\n\t\t\t\tls := loadLocks(records)\n\t\t\t\tif len(ls) == 0 {\n\t\t\t\t\treturn nil // change nothing\n\t\t\t\t}\n\t\t\t\tls = updateLocks(ls, lock)\n\t\t\t\tif len(ls) == 0 {\n\t\t\t\t\tdelete(owners, lkey)\n\t\t\t\t} else {\n\t\t\t\t\towners[lkey] = dumpLocks(ls)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tls := loadLocks(owners[lkey])\n\t\t\t\tdelete(owners, lkey)\n\t\t\t\tfor _, d := range owners {\n\t\t\t\t\tls := loadLocks(d)\n\t\t\t\t\tfor _, l := range ls {\n\t\t\t\t\t\t// find conflicted locks\n\t\t\t\t\t\tif (ltype == F_WRLCK || l.Type == F_WRLCK) && end >= l.Start && start <= l.End {\n\t\t\t\t\t\t\treturn syscall.EAGAIN\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tls = updateLocks(ls, lock)\n\t\t\t\towners[lkey] = dumpLocks(ls)\n\t\t\t}\n\t\t\tif len(owners) == 0 {\n\t\t\t\ttx.delete(ikey)\n\t\t\t} else {\n\t\t\t\ttx.set(ikey, marshalPlock(owners))\n\t\t\t}\n\t\t\treturn nil\n\t\t}, inode)\n\n\t\tif !block || err != syscall.EAGAIN {\n\t\t\tbreak\n\t\t}\n\t\tif ltype == F_WRLCK {\n\t\t\ttime.Sleep(time.Millisecond * 1)\n\t\t} else {\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t}\n\treturn errno(err)\n}\n\nfunc (m *kvMeta) ListLocks(ctx context.Context, inode Ino) ([]PLockItem, []FLockItem, error) {\n\tfKey := m.flockKey(inode)\n\tpKey := m.plockKey(inode)\n\n\tvar flocks []FLockItem\n\tvar plocks []PLockItem\n\tfv, err := m.get(fKey)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tfs := unmarshalFlock(fv)\n\tfor k, t := range fs {\n\t\tflocks = append(flocks, FLockItem{ownerKey{k.sid, k.owner}, string(t)})\n\t}\n\n\tpv, err := m.get(pKey)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\towners := unmarshalPlock(pv)\n\tfor k, records := range owners {\n\t\tls := loadLocks(records)\n\t\tfor _, l := range ls {\n\t\t\tplocks = append(plocks, PLockItem{ownerKey{k.sid, k.owner}, l})\n\t\t}\n\t}\n\treturn plocks, flocks, nil\n}\n" }, { "path": "pkg/meta/tkv_mem.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/google/btree\"\n)\n\nfunc init() {\n\tRegister(\"memkv\", newKVMeta)\n\tdrivers[\"memkv\"] = newMockClient\n}\n\nconst settingPath = \"/tmp/juicefs.memkv.setting.json\"\n\nfunc newMockClient(addr string) (tkvClient, error) {\n\tclient := &memKV{items: btree.New(2), temp: &kvItem{}}\n\tif d, err := os.ReadFile(settingPath); err == nil {\n\t\tvar buffer map[string][]byte\n\t\tif err = json.Unmarshal(d, &buffer); err == nil {\n\t\t\tfor k, v := range buffer {\n\t\t\t\tclient.set(k, v) // not locked\n\t\t\t}\n\t\t}\n\t}\n\treturn client, nil\n}\n\ntype memTxn struct {\n\tstore *memKV\n\tobserved map[string]int\n\tbuffer map[string][]byte\n}\n\nfunc (tx *memTxn) get(key []byte) []byte {\n\tk := string(key)\n\tif v, ok := tx.buffer[k]; ok {\n\t\treturn v\n\t}\n\ttx.store.Lock()\n\tdefer tx.store.Unlock()\n\tit := tx.store.get(k)\n\tif it != nil {\n\t\ttx.observed[k] = it.ver\n\t\treturn it.value\n\t} else {\n\t\ttx.observed[k] = 0\n\t\treturn nil\n\t}\n}\n\nfunc (tx *memTxn) gets(keys ...[]byte) [][]byte {\n\tvalues := make([][]byte, len(keys))\n\tfor i, key := range keys {\n\t\tvalues[i] = tx.get(key)\n\t}\n\treturn values\n}\n\nfunc (tx *memTxn) scan(begin, end []byte, keysOnly bool, handler func(k, v []byte) bool) {\n\ttx.store.Lock()\n\tdefer tx.store.Unlock()\n\ttx.store.items.AscendGreaterOrEqual(&kvItem{key: string(begin)}, func(i btree.Item) bool {\n\t\tit := i.(*kvItem)\n\t\tkey := []byte(it.key)\n\t\tif bytes.Compare(key, end) >= 0 {\n\t\t\treturn false\n\t\t}\n\t\ttx.observed[it.key] = it.ver\n\t\treturn handler(key, it.value)\n\t})\n}\n\nfunc nextKey(key []byte) []byte {\n\tif len(key) == 0 {\n\t\treturn nil\n\t}\n\tnext := make([]byte, len(key))\n\tcopy(next, key)\n\tp := len(next) - 1\n\tfor {\n\t\tnext[p]++\n\t\tif next[p] != 0 {\n\t\t\tbreak\n\t\t}\n\t\tp--\n\t\tif p < 0 {\n\t\t\tpanic(\"can't scan keys for 0xFF\")\n\t\t}\n\t}\n\treturn next\n}\n\nfunc (tx *memTxn) exist(prefix []byte) bool {\n\tvar ret bool\n\ttx.store.Lock()\n\tdefer tx.store.Unlock()\n\ttx.store.items.AscendGreaterOrEqual(&kvItem{key: string(prefix)}, func(i btree.Item) bool {\n\t\tit := i.(*kvItem)\n\t\tif strings.HasPrefix(it.key, string(prefix)) {\n\t\t\ttx.observed[it.key] = it.ver\n\t\t\tret = true\n\t\t}\n\t\treturn false\n\t})\n\treturn ret\n}\n\nfunc (tx *memTxn) set(key, value []byte) {\n\ttx.buffer[string(key)] = value\n}\n\nfunc (tx *memTxn) append(key []byte, value []byte) {\n\tnew := append(tx.get(key), value...)\n\ttx.set(key, new)\n}\n\nfunc (tx *memTxn) incrBy(key []byte, value int64) int64 {\n\tbuf := tx.get(key)\n\tnew := parseCounter(buf)\n\tif value != 0 {\n\t\tnew += value\n\t\ttx.set(key, packCounter(new))\n\t}\n\treturn new\n}\n\nfunc (tx *memTxn) delete(key []byte) {\n\ttx.buffer[string(key)] = nil\n}\n\ntype kvItem struct {\n\tkey string\n\tver int\n\tvalue []byte\n}\n\nfunc (it *kvItem) Less(o btree.Item) bool {\n\treturn it.key < o.(*kvItem).key\n}\n\ntype memKV struct {\n\tsync.Mutex\n\titems *btree.BTree\n\ttemp *kvItem\n}\n\nfunc (c *memKV) name() string {\n\treturn \"memkv\"\n}\n\nfunc (c *memKV) shouldRetry(err error) bool {\n\treturn strings.Contains(err.Error(), \"write conflict\")\n}\n\nfunc (c *memKV) config(key string) interface{} {\n\treturn nil\n}\n\nfunc (c *memKV) get(key string) *kvItem {\n\tc.temp.key = key\n\tit := c.items.Get(c.temp)\n\tif it != nil {\n\t\treturn it.(*kvItem)\n\t}\n\treturn nil\n}\n\nfunc (c *memKV) set(key string, value []byte) {\n\tc.temp.key = key\n\tif value == nil {\n\t\tc.items.Delete(c.temp)\n\t\treturn\n\t}\n\tit := c.items.Get(c.temp)\n\tif it != nil {\n\t\tit.(*kvItem).ver++\n\t\tit.(*kvItem).value = value\n\t} else {\n\t\tc.items.ReplaceOrInsert(&kvItem{key: key, ver: 1, value: value})\n\t}\n}\n\nfunc (c *memKV) simpleTxn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\treturn c.txn(ctx, f, retry)\n}\n\nfunc (c *memKV) txn(ctx context.Context, f func(*kvTxn) error, retry int) error {\n\ttx := &memTxn{\n\t\tstore: c,\n\t\tobserved: make(map[string]int),\n\t\tbuffer: make(map[string][]byte),\n\t}\n\tif err := f(&kvTxn{tx, retry}); err != nil {\n\t\treturn err\n\t}\n\n\tif len(tx.buffer) == 0 {\n\t\treturn nil\n\t}\n\tc.Lock()\n\tdefer c.Unlock()\n\tfor k, ver := range tx.observed {\n\t\tit := c.get(k)\n\t\tif it == nil && ver != 0 {\n\t\t\treturn fmt.Errorf(\"write conflict: %s was version %d, now deleted\", k, ver)\n\t\t} else if it != nil && it.ver > ver {\n\t\t\treturn fmt.Errorf(\"write conflict: %s %d > %d\", k, it.ver, ver)\n\t\t}\n\t}\n\tif _, ok := tx.buffer[\"setting\"]; ok {\n\t\td, _ := json.Marshal(tx.buffer)\n\t\tif err := os.WriteFile(settingPath, d, 0644); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor k, value := range tx.buffer {\n\t\tc.set(k, value)\n\t}\n\treturn nil\n}\n\nfunc (c *memKV) scan(prefix []byte, handler func(key []byte, value []byte) bool) error {\n\tc.Lock()\n\tsnap := c.items.Clone()\n\tc.Unlock()\n\tbegin := string(prefix)\n\tend := string(nextKey(prefix))\n\tsnap.AscendGreaterOrEqual(&kvItem{key: begin}, func(i btree.Item) bool {\n\t\tit := i.(*kvItem)\n\t\tif end != \"\" && it.key >= end {\n\t\t\treturn false\n\t\t}\n\t\treturn handler([]byte(it.key), it.value)\n\t})\n\treturn nil\n}\n\nfunc (c *memKV) reset(prefix []byte) error {\n\tif len(prefix) == 0 {\n\t\tc.Lock()\n\t\tc.items = btree.New(2)\n\t\tc.temp = &kvItem{}\n\t\tc.Unlock()\n\t\treturn nil\n\t}\n\treturn c.txn(Background(), func(kt *kvTxn) error {\n\t\treturn c.scan(prefix, func(key, value []byte) bool {\n\t\t\tkt.delete(key)\n\t\t\treturn true\n\t\t})\n\t}, 0)\n}\n\nfunc (c *memKV) close() error {\n\treturn nil\n}\n\nfunc (c *memKV) gc() {}\n" }, { "path": "pkg/meta/tkv_prefix.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"fmt\"\n)\n\ntype prefixTxn struct {\n\t*kvTxn\n\tprefix []byte\n}\n\nfunc (tx *prefixTxn) realKey(key []byte) []byte {\n\tk := make([]byte, len(tx.prefix)+len(key))\n\tcopy(k, tx.prefix)\n\tcopy(k[len(tx.prefix):], key)\n\treturn k\n}\n\nfunc (tx *prefixTxn) origKey(key []byte) []byte {\n\treturn key[len(tx.prefix):]\n}\n\nfunc (tx *prefixTxn) get(key []byte) []byte {\n\treturn tx.kvTxn.get(tx.realKey(key))\n}\n\nfunc (tx *prefixTxn) gets(keys ...[]byte) [][]byte {\n\trealKeys := make([][]byte, len(keys))\n\tfor i, key := range keys {\n\t\trealKeys[i] = tx.realKey(key)\n\t}\n\treturn tx.kvTxn.gets(realKeys...)\n}\n\nfunc (tx *prefixTxn) scan(begin, end []byte, keysOnly bool, handler func(k, v []byte) bool) {\n\ttx.kvTxn.scan(tx.realKey(begin), tx.realKey(end), keysOnly, func(k, v []byte) bool {\n\t\treturn handler(tx.origKey(k), v)\n\t})\n}\n\nfunc (tx *prefixTxn) exist(prefix []byte) bool {\n\treturn tx.kvTxn.exist(tx.realKey(prefix))\n}\n\nfunc (tx *prefixTxn) set(key, value []byte) {\n\ttx.kvTxn.set(tx.realKey(key), value)\n}\n\nfunc (tx *prefixTxn) append(key []byte, value []byte) {\n\ttx.kvTxn.append(tx.realKey(key), value)\n}\n\nfunc (tx *prefixTxn) incrBy(key []byte, value int64) int64 {\n\treturn tx.kvTxn.incrBy(tx.realKey(key), value)\n}\n\nfunc (tx *prefixTxn) delete(key []byte) {\n\ttx.kvTxn.delete(tx.realKey(key))\n}\n\ntype prefixClient struct {\n\ttkvClient\n\tprefix []byte\n}\n\nfunc (c *prefixClient) simpleTxn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\treturn c.tkvClient.simpleTxn(ctx, func(tx *kvTxn) error {\n\t\treturn f(&kvTxn{&prefixTxn{tx, c.prefix}, retry})\n\t}, retry)\n}\n\nfunc (c *prefixClient) txn(ctx context.Context, f func(*kvTxn) error, retry int) error {\n\treturn c.tkvClient.txn(ctx, func(tx *kvTxn) error {\n\t\treturn f(&kvTxn{&prefixTxn{tx, c.prefix}, retry})\n\t}, retry)\n}\n\nfunc (c *prefixClient) scan(prefix []byte, handler func(key, value []byte) bool) error {\n\tk := make([]byte, len(c.prefix)+len(prefix))\n\tcopy(k, c.prefix)\n\tcopy(k[len(c.prefix):], prefix)\n\treturn c.tkvClient.scan(k, func(key, value []byte) bool {\n\t\treturn handler(key[len(c.prefix):], value)\n\t})\n}\n\nfunc (c *prefixClient) reset(prefix []byte) error {\n\tif prefix != nil {\n\t\treturn fmt.Errorf(\"prefix must be nil, but got %v\", prefix)\n\t}\n\treturn c.tkvClient.reset(c.prefix)\n}\n\nfunc withPrefix(client tkvClient, prefix []byte) tkvClient {\n\treturn &prefixClient{client, prefix}\n}\n" }, { "path": "pkg/meta/tkv_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n//mutate:disable\n//nolint:errcheck\npackage meta\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"sort\"\n\t\"testing\"\n\n\t\"github.com/dgraph-io/badger/v4\"\n)\n\nfunc TestMemKVClient(t *testing.T) {\n\t_ = os.Remove(settingPath)\n\tm, err := newKVMeta(\"memkv\", \"jfs-unit-test\", testConfig())\n\tif err != nil || m.Name() != \"memkv\" {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestTiKVClient(t *testing.T) { //skip mutate\n\tm, err := newKVMeta(\"tikv\", \"127.0.0.1:2379/jfs-unit-test\", testConfig())\n\tif err != nil || m.Name() != \"tikv\" {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestBadgerClient(t *testing.T) {\n\tm, err := newKVMeta(\"badger\", \"badger\", testConfig())\n\tif err != nil || m.Name() != \"badger\" {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc TestEtcdClient(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SKIP_NON_CORE\") == \"true\" {\n\t\tt.Skipf(\"skip non-core test\")\n\t}\n\tm, err := newKVMeta(\"etcd\", os.Getenv(\"ETCD_ADDR\"), testConfig())\n\tif err != nil {\n\t\tt.Fatalf(\"create meta: %s\", err)\n\t}\n\ttestMeta(t, m)\n}\n\nfunc testTKV(t *testing.T, c tkvClient) {\n\ttxn := func(f func(kt *kvTxn)) {\n\t\tif err := c.txn(Background(), func(kt *kvTxn) error {\n\t\t\tf(kt)\n\t\t\treturn nil\n\t\t}, 0); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\t// basic\n\terr := c.reset(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"reset: %s\", err)\n\t}\n\tvar hasKey bool\n\ttxn(func(kt *kvTxn) { hasKey = kt.exist(nil) })\n\tif hasKey {\n\t\tt.Fatalf(\"has key after reset\")\n\t}\n\tk := []byte(\"k\")\n\tv := []byte(\"value\")\n\n\ttxn(func(kt *kvTxn) {\n\t\tkt.set(k, v)\n\t\tkt.append(k, v)\n\t})\n\tvar r []byte\n\ttxn(func(kt *kvTxn) { r = kt.get(k) })\n\tif !bytes.Equal(r, []byte(\"valuevalue\")) {\n\t\tt.Fatalf(\"expect 'valuevalue', but got %v\", string(r))\n\t}\n\ttxn(func(kt *kvTxn) {\n\t\tkt.set([]byte(\"k2\"), v)\n\t\tkt.set([]byte(\"v\"), k)\n\t})\n\tvar ks [][]byte\n\ttxn(func(kt *kvTxn) { ks = kt.gets([]byte(\"k1\"), []byte(\"k2\")) })\n\tif ks[0] != nil || string(ks[1]) != \"value\" {\n\t\tt.Fatalf(\"gets k1,k2: %+v != %+v\", ks, [][]byte{nil, []byte(\"value\")})\n\t}\n\n\tvar keys [][]byte\n\tc.scan([]byte(\"k\"), func(key, value []byte) bool {\n\t\tkeys = append(keys, key)\n\t\treturn true\n\t})\n\tif len(keys) != 2 || string(keys[0]) != \"k\" || string(keys[1]) != \"k2\" {\n\t\tt.Fatalf(\"keys: %+v\", keys)\n\t}\n\tkeys = keys[:0]\n\ttxn(func(kt *kvTxn) {\n\t\tkt.scan([]byte(\"a\"), []byte(\"z\"), true, func(k, v []byte) bool {\n\t\t\tif len(k) == 1 {\n\t\t\t\tkeys = append(keys, k)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t})\n\tif len(keys) != 2 || string(keys[0]) != \"k\" || string(keys[1]) != \"v\" {\n\t\tt.Fatalf(\"keys: %+v\", keys)\n\t}\n\tkeys = keys[:0]\n\ttxn(func(kt *kvTxn) {\n\t\tkt.scan([]byte(\"k\"), []byte(\"l\"), true, func(k, v []byte) bool {\n\t\t\tkeys = append(keys, k)\n\t\t\treturn true\n\t\t})\n\t})\n\tif len(keys) != 2 || string(keys[0]) != \"k\" || string(keys[1]) != \"k2\" {\n\t\tt.Fatalf(\"keys: %+v\", keys)\n\t}\n\tkeys = keys[:0]\n\ttxn(func(kt *kvTxn) {\n\t\tkt.scan([]byte(\"a\"), []byte(\"z\"), true, func(k, v []byte) bool {\n\t\t\tkeys = append(keys, k)\n\t\t\treturn true\n\t\t})\n\t})\n\tif len(keys) != 3 || string(keys[0]) != \"k\" || string(keys[1]) != \"k2\" || string(keys[2]) != \"v\" {\n\t\tt.Fatalf(\"keys: %+v\", keys)\n\t}\n\tvalues := make(map[string][]byte)\n\ttxn(func(kt *kvTxn) {\n\t\tkt.scan([]byte(\"k\"), nextKey([]byte(\"k\")), false, func(k, v []byte) bool {\n\t\t\tif len(v) == 5 {\n\t\t\t\tvalues[string(k)] = v\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t})\n\tif len(values) != 1 || string(values[\"k2\"]) != \"value\" {\n\t\tt.Fatalf(\"scan values: %+v\", values)\n\t}\n\tvalues = make(map[string][]byte)\n\ttxn(func(kt *kvTxn) {\n\t\tkt.scan([]byte(\"k2\"), []byte(\"v\"),\n\t\t\tfalse, func(k, v []byte) bool {\n\t\t\t\tvalues[string(k)] = v\n\t\t\t\treturn true\n\t\t\t})\n\t})\n\tif len(values) != 1 || string(values[\"k2\"]) != \"value\" {\n\t\tt.Fatalf(\"scanRange: %+v\", values)\n\t}\n\n\t// exists\n\ttxn(func(kt *kvTxn) { hasKey = kt.exist([]byte(\"k\")) })\n\tif !hasKey {\n\t\tt.Fatalf(\"has key k*\")\n\t}\n\ttxn(func(kt *kvTxn) {\n\t\tfor _, key := range keys {\n\t\t\tkt.delete(key)\n\t\t}\n\t})\n\ttxn(func(kt *kvTxn) { r = kt.get(k) })\n\tif r != nil {\n\t\tt.Fatalf(\"expect nil, but got %v\", string(r))\n\t}\n\tkeys = keys[:0]\n\ttxn(func(kt *kvTxn) {\n\t\tkt.scan([]byte(\"a\"), []byte(\"z\"), true, func(k, v []byte) bool {\n\t\t\tkeys = append(keys, k)\n\t\t\treturn true\n\t\t})\n\t})\n\tif len(keys) != 0 {\n\t\tt.Fatalf(\"no keys: %+v\", keys)\n\t}\n\ttxn(func(kt *kvTxn) { hasKey = kt.exist(nil) })\n\tif hasKey {\n\t\tt.Fatalf(\"has not keys\")\n\t}\n\n\t// counters\n\tvar count int64\n\tc.txn(Background(), func(tx *kvTxn) error {\n\t\tcount = tx.incrBy([]byte(\"counter\"), -1)\n\t\treturn nil\n\t}, 0)\n\tif count != -1 {\n\t\tt.Fatalf(\"counter should be -1, but got %d\", count)\n\t}\n\tc.txn(Background(), func(tx *kvTxn) error {\n\t\tcount = tx.incrBy([]byte(\"counter\"), 0)\n\t\treturn nil\n\t}, 0)\n\tif count != -1 {\n\t\tt.Fatalf(\"counter should be -1, but got %d\", count)\n\t}\n\tc.txn(Background(), func(tx *kvTxn) error {\n\t\tcount = tx.incrBy([]byte(\"counter\"), 2)\n\t\treturn nil\n\t}, 0)\n\tif count != 1 {\n\t\tt.Fatalf(\"counter should be 1, but got %d\", count)\n\t}\n\n\t// key with zeros\n\tk = []byte(\"k\\x001\")\n\ttxn(func(kt *kvTxn) {\n\t\tkt.set(k, v)\n\t})\n\tvar v2 []byte\n\ttxn(func(kt *kvTxn) {\n\t\tv2 = kt.get(k)\n\t})\n\tif !bytes.Equal(v2, v) {\n\t\tt.Fatalf(\"expect %v but got %v\", v, v2)\n\t}\n\n\t// scan many key-value pairs\n\tkeys = make([][]byte, 0, 100000)\n\tfor i := 0; i < 1000; i++ {\n\t\ttxn(func(kt *kvTxn) {\n\t\t\tfor j := 0; j < 100; j++ {\n\t\t\t\tk := []byte(fmt.Sprintf(\"Key_%d_%d\", i, j))\n\t\t\t\tv := []byte(fmt.Sprintf(\"Value_%d_%d\", i, j))\n\t\t\t\tkt.set(k, v)\n\t\t\t\tkeys = append(keys, k)\n\t\t\t}\n\t\t})\n\t}\n\tkvs := make([][]byte, 0, 200000)\n\ttxn(func(kt *kvTxn) {\n\t\tkt.scan([]byte(\"A\"), []byte(\"Z\"), false, func(k, v []byte) bool {\n\t\t\tkvs = append(kvs, k, v)\n\t\t\treturn true\n\t\t})\n\t})\n\tsort.Slice(keys, func(i, j int) bool { return bytes.Compare(keys[i], keys[j]) < 0 })\n\tfor i, k := range keys {\n\t\tif !bytes.Equal(k, kvs[i*2]) || !bytes.Equal([]byte(fmt.Sprintf(\"Value%s\", k[3:])), kvs[i*2+1]) {\n\t\t\tt.Fatalf(\"expect %s but got %s, %s\", k, keys[i*2], keys[i*2+1])\n\t\t}\n\t}\n}\n\nfunc TestBadgerKV(t *testing.T) {\n\tc, err := newBadgerClient(\"test_badger\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestTKV(t, c)\n}\n\nfunc TestEtcd(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SKIP_NON_CORE\") == \"true\" {\n\t\tt.Skipf(\"skip non-core test\")\n\t}\n\tc, err := newEtcdClient(fmt.Sprintf(\"%s/jfs\", os.Getenv(\"ETCD_ADDR\")))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestTKV(t, c)\n}\n\nfunc TestMemKV(t *testing.T) {\n\tc, _ := newTkvClient(\"memkv\", \"\")\n\tc = withPrefix(c, []byte(\"jfs\"))\n\ttestTKV(t, c)\n}\n\nfunc TestBadgerScanKeysOnlyNilValues(t *testing.T) {\n\tc, err := newBadgerClient(t.TempDir())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer c.close()\n\n\tif err := c.txn(Background(), func(kt *kvTxn) error {\n\t\tkt.set([]byte(\"key1\"), []byte(\"value1\"))\n\t\tkt.set([]byte(\"key2\"), []byte(\"value2\"))\n\t\treturn nil\n\t}, 0); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar scanned int\n\tif err := c.txn(Background(), func(kt *kvTxn) error {\n\t\tkt.scan([]byte(\"key\"), nextKey([]byte(\"key\")), true, func(k, v []byte) bool {\n\t\t\tif v != nil {\n\t\t\t\tt.Errorf(\"keysOnly=true: expected nil value for key %q, got %q\", k, v)\n\t\t\t}\n\t\t\tscanned++\n\t\t\treturn true\n\t\t})\n\t\treturn nil\n\t}, 0); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif scanned != 2 {\n\t\tt.Fatalf(\"expected 2 keys scanned, got %d\", scanned)\n\t}\n}\n\nfunc TestBadgerDeleteTxnTooBig(t *testing.T) {\n\tdir := t.TempDir()\n\n\topt := badger.DefaultOptions(dir)\n\topt.Logger = nil\n\topt.MetricsEnabled = false\n\topt.MemTableSize = 1 << 20\n\topt.ValueThreshold = 1 << 10\n\tdb, err := badger.Open(opt)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer db.Close()\n\n\tconst numKeys = 5000\n\twb := db.NewWriteBatch()\n\tfor i := 0; i < numKeys; i++ {\n\t\tif err := wb.Set([]byte(fmt.Sprintf(\"txbig_%05d\", i)), []byte(\"v\")); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif err := wb.Flush(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar keys [][]byte\n\trtx := db.NewTransaction(false)\n\tit := rtx.NewIterator(badger.IteratorOptions{Prefix: []byte(\"txbig_\"), PrefetchValues: false})\n\tfor it.Rewind(); it.Valid(); it.Next() {\n\t\tkeys = append(keys, it.Item().KeyCopy(nil))\n\t}\n\tit.Close()\n\trtx.Discard()\n\n\tclient := &badgerClient{client: db, done: make(chan struct{})}\n\n\terr = client.txn(Background(), func(kt *kvTxn) error {\n\t\tfor _, key := range keys {\n\t\t\tkt.delete(key)\n\t\t}\n\t\treturn nil\n\t}, 0)\n\n\tif err != badger.ErrTxnTooBig {\n\t\tt.Fatalf(\"expected ErrTxnTooBig, got %v\", err)\n\t}\n}\n" }, { "path": "pkg/meta/tkv_tikv.go", "content": "//go:build !notikv\n// +build !notikv\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"math\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\tplog \"github.com/pingcap/log\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/tikv/client-go/v2/config\"\n\ttikverr \"github.com/tikv/client-go/v2/error\"\n\t\"github.com/tikv/client-go/v2/oracle\"\n\t\"github.com/tikv/client-go/v2/tikv\"\n\t\"github.com/tikv/client-go/v2/txnkv\"\n\t\"github.com/tikv/client-go/v2/txnkv/txnutil\"\n\tpd \"github.com/tikv/pd/client\"\n\t\"go.uber.org/zap\"\n)\n\nfunc init() {\n\tRegister(\"tikv\", newKVMeta)\n\tdrivers[\"tikv\"] = newTikvClient\n\n}\n\nfunc newTikvClient(addr string) (tkvClient, error) {\n\tvar plvl string // TiKV (PingCap) uses uber-zap logging, make it less verbose\n\tswitch logger.Level {\n\tcase logrus.TraceLevel:\n\t\tplvl = \"debug\"\n\tcase logrus.DebugLevel:\n\t\tplvl = \"info\"\n\tcase logrus.InfoLevel, logrus.WarnLevel:\n\t\tplvl = \"warn\"\n\tcase logrus.ErrorLevel:\n\t\tplvl = \"error\"\n\tdefault:\n\t\tplvl = \"dpanic\"\n\t}\n\tl, prop, _ := plog.InitLogger(&plog.Config{Level: plvl}, zap.Fields(zap.String(\"component\", \"tikv\"), zap.Int(\"pid\", os.Getpid())))\n\tplog.ReplaceGlobals(l, prop)\n\n\ttUrl, err := url.Parse(\"tikv://\" + addr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tquery := tUrl.Query()\n\tconfig.UpdateGlobal(func(conf *config.Config) {\n\t\tconf.Security = config.NewSecurity(\n\t\t\tquery.Get(\"ca\"),\n\t\t\tquery.Get(\"cert\"),\n\t\t\tquery.Get(\"key\"),\n\t\t\tstrings.Split(query.Get(\"verify-cn\"), \",\"))\n\t})\n\tinterval := time.Hour * 3\n\tif dur, err := time.ParseDuration(query.Get(\"gc-interval\")); err == nil {\n\t\tif dur != 0 && dur < time.Hour {\n\t\t\tlogger.Warnf(\"TiKV gc-interval (%s) is too short, and is reset to 1h\", dur)\n\t\t\tdur = time.Hour\n\t\t}\n\t\tinterval = dur\n\t}\n\tlogger.Infof(\"TiKV gc interval is set to %s\", interval)\n\n\tclient, err := txnkv.NewClient(strings.Split(tUrl.Host, \",\"))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif strings.ToLower(query.Get(\"open-tso-follower-proxy\")) == \"true\" {\n\t\tif err := client.KVStore.GetPDClient().UpdateOption(pd.EnableTSOFollowerProxy, true); err != nil {\n\t\t\tlogger.Warnf(\"Failed to enable TSO Follower Proxy: %v\", err)\n\t\t} else {\n\t\t\tlogger.Infof(\"Enabling TSO Follower Proxy\")\n\t\t}\n\t}\n\n\tif waitStr := query.Get(\"max-tso-batch-wait-interval\"); waitStr != \"\" {\n\t\tif waitDur, err := time.ParseDuration(waitStr); err == nil {\n\t\t\tif err := client.KVStore.GetPDClient().UpdateOption(pd.MaxTSOBatchWaitInterval, waitDur); err != nil {\n\t\t\t\tlogger.Warnf(\"Failed to set MaxTSOBatchWaitInterval: %v\", err)\n\t\t\t} else {\n\t\t\t\tlogger.Infof(\"Set MaxTSOBatchWaitInterval to %s\", waitDur)\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"Failed to parse max-tso-batch-wait-interval (%s): %v\", waitStr, err)\n\t\t}\n\t}\n\n\tprefix := strings.TrimLeft(tUrl.Path, \"/\")\n\treturn withPrefix(&tikvClient{client.KVStore, interval}, append([]byte(prefix), 0xFD)), nil\n}\n\ntype tikvTxn struct {\n\t*tikv.KVTxn\n}\n\nfunc (tx *tikvTxn) get(key []byte) []byte {\n\tvalue, err := tx.Get(context.TODO(), key)\n\tif tikverr.IsErrNotFound(err) {\n\t\treturn nil\n\t}\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn value\n}\n\nfunc (tx *tikvTxn) gets(keys ...[]byte) [][]byte {\n\tret, err := tx.BatchGet(context.TODO(), keys)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tvalues := make([][]byte, len(keys))\n\tfor i, key := range keys {\n\t\tvalues[i] = ret[string(key)]\n\t}\n\treturn values\n}\n\nfunc (tx *tikvTxn) scan(begin, end []byte, keysOnly bool, handler func(k, v []byte) bool) {\n\tsnap := tx.GetSnapshot()\n\tsnap.SetScanBatchSize(10240)\n\tsnap.SetNotFillCache(true)\n\tsnap.SetKeyOnly(keysOnly)\n\tit, err := tx.Iter(begin, end)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer it.Close()\n\tfor it.Valid() && handler(it.Key(), it.Value()) {\n\t\tif err = it.Next(); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n}\n\nfunc (tx *tikvTxn) exist(prefix []byte) bool {\n\tit, err := tx.Iter(prefix, nextKey(prefix))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer it.Close()\n\treturn it.Valid()\n}\n\nfunc (tx *tikvTxn) set(key, value []byte) {\n\tif err := tx.Set(key, value); err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc (tx *tikvTxn) append(key []byte, value []byte) {\n\tnew := append(tx.get(key), value...)\n\ttx.set(key, new)\n}\n\nfunc (tx *tikvTxn) incrBy(key []byte, value int64) int64 {\n\tbuf := tx.get(key)\n\tnew := parseCounter(buf)\n\tif value != 0 {\n\t\tnew += value\n\t\ttx.set(key, packCounter(new))\n\t}\n\treturn new\n}\n\nfunc (tx *tikvTxn) delete(key []byte) {\n\tif err := tx.Delete(key); err != nil {\n\t\tpanic(err)\n\t}\n}\n\ntype tikvClient struct {\n\tclient *tikv.KVStore\n\tgcInterval time.Duration\n}\n\nfunc (c *tikvClient) name() string {\n\treturn \"tikv\"\n}\n\nfunc (c *tikvClient) shouldRetry(err error) bool {\n\treturn strings.Contains(err.Error(), \"write conflict\") || strings.Contains(err.Error(), \"TxnLockNotFound\")\n}\n\nfunc (c *tikvClient) config(key string) interface{} {\n\tif key == \"startTS\" {\n\t\tts, err := c.client.CurrentTimestamp(oracle.GlobalTxnScope)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"TiKV get startTS: %s\", err)\n\t\t\treturn nil\n\t\t}\n\t\treturn ts\n\t}\n\treturn nil\n}\n\nfunc (c *tikvClient) simpleTxn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\ttx, err := c.client.Begin(tikv.WithStartTS(math.MaxUint64)) // math.MaxUint64 means to point get the latest committed data without PD access\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to begin transaction\")\n\t}\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tif e, ok := r.(error); ok {\n\t\t\t\terr = e\n\t\t\t} else {\n\t\t\t\terr = errors.Errorf(\"panic in point get transaction: %v\", r)\n\t\t\t}\n\t\t}\n\t}()\n\tif err = f(&kvTxn{&tikvTxn{tx}, retry}); err != nil {\n\t\treturn err\n\t}\n\tif !tx.IsReadOnly() {\n\t\treturn syscall.EINVAL\n\t}\n\treturn nil\n}\n\nfunc (c *tikvClient) txn(ctx context.Context, f func(*kvTxn) error, retry int) (err error) {\n\tvar opts []tikv.TxnOption\n\tif val := ctx.Value(txSessionKey{}); val != nil {\n\t\topts = append(opts, tikv.WithStartTS(val.(uint64)))\n\t}\n\n\ttx, err := c.client.Begin(opts...)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tfe, ok := r.(error)\n\t\t\tif ok {\n\t\t\t\terr = fe\n\t\t\t} else {\n\t\t\t\terr = errors.Errorf(\"tikv client txn func error: %v\", r)\n\t\t\t}\n\t\t}\n\t}()\n\tif err = f(&kvTxn{&tikvTxn{tx}, retry}); err != nil {\n\t\treturn err\n\t}\n\tif !tx.IsReadOnly() {\n\t\ttx.SetEnable1PC(true)\n\t\ttx.SetEnableAsyncCommit(true)\n\t\terr = tx.Commit(ctx)\n\t}\n\treturn err\n}\n\nfunc (c *tikvClient) scan(prefix []byte, handler func(key, value []byte) bool) error {\n\tend := nextKey(prefix)\n\tstart := prefix\nOUT:\n\tfor {\n\t\tts, err := c.client.CurrentTimestamp(oracle.GlobalTxnScope)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tsnap := c.client.GetSnapshot(ts)\n\t\tsnap.SetScanBatchSize(10240)\n\t\tsnap.SetNotFillCache(true)\n\t\tsnap.SetPriority(txnutil.PriorityLow)\n\t\tit, err := snap.Iter(start, end)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvar lastKey []byte\n\t\tfor it.Valid() && handler(it.Key(), it.Value()) {\n\t\t\tlastKey = it.Key()\n\t\t\tif err = it.Next(); err != nil {\n\t\t\t\tit.Close()\n\t\t\t\tif _, ok := err.(*tikverr.ErrGCTooEarly); !ok {\n\t\t\t\t\tlogger.Warnf(\"scan next key: %s\", err)\n\t\t\t\t\treturn err\n\t\t\t\t} else { // restart scan\n\t\t\t\t\tstart = nextKey(lastKey)\n\t\t\t\t\tcontinue OUT\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tit.Close()\n\t\treturn nil\n\t}\n}\n\nfunc (c *tikvClient) reset(prefix []byte) error {\n\t_, err := c.client.DeleteRange(context.Background(), prefix, nextKey(prefix), 1)\n\treturn err\n}\n\nfunc (c *tikvClient) close() error {\n\treturn c.client.Close()\n}\n\nfunc (c *tikvClient) gc() {\n\tif c.gcInterval == 0 {\n\t\treturn\n\t}\n\n\tcurrentTs, err := c.client.CurrentTimestamp(oracle.GlobalTxnScope)\n\tif err != nil {\n\t\tlogger.Warnf(\"TiKV GC was skipped due to failure in obtaining the current timestamp.\")\n\t\treturn\n\t}\n\n\tsafePoint, err := c.client.GC(context.Background(), oracle.GoTimeToTS(oracle.GetTimeFromTS(currentTs).Add(-c.gcInterval)))\n\tif err == nil {\n\t\tlogger.Debugf(\"TiKV GC returns new safe point: %d (%s)\", safePoint, oracle.GetTimeFromTS(safePoint))\n\t} else {\n\t\tlogger.Warnf(\"TiKV GC: %s\", err)\n\t}\n}\n" }, { "path": "pkg/meta/utils.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"path\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\nconst (\n\taclCounter = \"aclMaxId\"\n\tusedSpace = \"usedSpace\"\n\ttotalInodes = \"totalInodes\"\n\tlegacySessions = \"sessions\"\n\tkrbTokenCounter = \"krbTokenMaxId\"\n)\n\nvar counterNames = []string{usedSpace, totalInodes, \"nextInode\", \"nextChunk\", \"nextSession\", \"nextTrash\"}\n\nconst (\n\t// fallocate\n\tfallocKeepSize = 0x01\n\tfallocPunchHole = 0x02\n\t// RESERVED: fallocNoHideStale = 0x04\n\tfallocCollapesRange = 0x08\n\tfallocZeroRange = 0x10\n\tfallocInsertRange = 0x20\n)\nconst (\n\t// clone mode\n\tCLONE_MODE_CAN_OVERWRITE = 0x01\n\tCLONE_MODE_PRESERVE_ATTR = 0x02\n\tCLONE_MODE_PRESERVE_HARDLINKS = 0x08\n\n\t// clone concurrency\n\tCLONE_DEFAULT_CONCURRENCY = 4\n\n\t// atime mode\n\tNoAtime = \"noatime\"\n\tRelAtime = \"relatime\"\n\tStrictAtime = \"strictatime\"\n)\n\nconst (\n\tMODE_MASK_R = 0b100\n\tMODE_MASK_W = 0b010\n\tMODE_MASK_X = 0b001\n)\n\ntype msgCallbacks struct {\n\tsync.Mutex\n\tcallbacks map[uint32]MsgCallback\n}\n\ntype freeID struct {\n\tnext uint64\n\tmaxid uint64\n}\n\nvar logger = utils.GetLogger(\"juicefs\")\n\ntype queryMap struct {\n\t*url.Values\n}\n\nfunc (qm *queryMap) duration(key, originalKey string, d time.Duration) time.Duration {\n\tval := qm.Get(key)\n\tif val == \"\" {\n\t\toVal := qm.Get(originalKey)\n\t\tif oVal == \"\" {\n\t\t\treturn d\n\t\t}\n\t\tval = oVal\n\t}\n\n\tqm.Del(key)\n\tif dur, err := time.ParseDuration(val); err == nil {\n\t\treturn dur\n\t} else {\n\t\tlogger.Warnf(\"Parse duration %s for key %s: %s\", val, key, err)\n\t\treturn d\n\t}\n}\n\nfunc (qm *queryMap) getInt(key, originalKey string, defaultValue int) int {\n\tval := qm.Get(key)\n\tif val == \"\" {\n\t\toVal := qm.Get(originalKey)\n\t\tif oVal == \"\" {\n\t\t\treturn defaultValue\n\t\t}\n\t\tval = oVal\n\t}\n\n\tqm.Del(key)\n\tif i, err := strconv.ParseInt(val, 10, 32); err == nil {\n\t\treturn int(i)\n\t} else {\n\t\tlogger.Warnf(\"Parse int %s for key %s: %s\", val, key, err)\n\t\treturn defaultValue\n\t}\n}\n\nfunc (qm *queryMap) pop(key string) string {\n\tdefer qm.Del(key)\n\treturn qm.Get(key)\n}\n\nfunc errno(err error) syscall.Errno {\n\tif err == nil {\n\t\treturn 0\n\t}\n\tif err == context.Canceled {\n\t\treturn syscall.EINTR\n\t}\n\tif errors.Is(err, context.DeadlineExceeded) {\n\t\treturn syscall.ETIMEDOUT\n\t}\n\tif eno, ok := err.(syscall.Errno); ok {\n\t\treturn eno\n\t}\n\tif err == redis.Nil {\n\t\treturn syscall.ENOENT\n\t}\n\tif strings.HasPrefix(err.Error(), \"OOM\") {\n\t\treturn syscall.ENOSPC\n\t}\n\tlogger.Errorf(\"error: %s\\n%s\", err, debug.Stack())\n\treturn syscall.EIO\n}\n\nfunc accessMode(attr *Attr, uid uint32, gids []uint32) uint8 {\n\tif uid == 0 {\n\t\treturn 0x7\n\t}\n\tmode := attr.Mode\n\tif uid == attr.Uid {\n\t\treturn uint8(mode>>6) & 7\n\t}\n\tfor _, gid := range gids {\n\t\tif gid == attr.Gid {\n\t\t\treturn uint8(mode>>3) & 7\n\t\t}\n\t}\n\treturn uint8(mode & 7)\n}\n\nfunc align4K(length uint64) int64 {\n\tif length == 0 {\n\t\treturn 1 << 12\n\t}\n\treturn int64((((length - 1) >> 12) + 1) << 12)\n}\n\ntype plockRecord struct {\n\tType uint32\n\tPid uint32\n\tStart uint64\n\tEnd uint64\n}\n\ntype ownerKey struct {\n\tSid uint64\n\tOwner uint64\n}\n\ntype PLockItem struct {\n\townerKey\n\tplockRecord\n}\n\ntype FLockItem struct {\n\townerKey\n\tType string\n}\n\nfunc parseOwnerKey(key string) (*ownerKey, error) {\n\tpair := strings.Split(key, \"_\")\n\tif len(pair) != 2 {\n\t\treturn nil, fmt.Errorf(\"invalid owner key: %s\", key)\n\t}\n\tsid, err := strconv.ParseUint(pair[0], 10, 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\towner, err := strconv.ParseUint(pair[1], 16, 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &ownerKey{sid, owner}, nil\n}\n\nfunc loadLocks(d []byte) []plockRecord {\n\tvar ls []plockRecord\n\trb := utils.FromBuffer(d)\n\tfor rb.HasMore() {\n\t\tls = append(ls, plockRecord{rb.Get32(), rb.Get32(), rb.Get64(), rb.Get64()})\n\t}\n\treturn ls\n}\n\nfunc dumpLocks(ls []plockRecord) []byte {\n\twb := utils.NewBuffer(uint32(len(ls)) * 24)\n\tfor _, l := range ls {\n\t\twb.Put32(l.Type)\n\t\twb.Put32(l.Pid)\n\t\twb.Put64(l.Start)\n\t\twb.Put64(l.End)\n\t}\n\treturn wb.Bytes()\n}\n\nfunc updateLocks(ls []plockRecord, nl plockRecord) []plockRecord {\n\t// ls is ordered by l.start without overlap\n\tsize := len(ls)\n\tfor i := 0; i < size && nl.Start <= nl.End; i++ {\n\t\tl := ls[i]\n\t\tif nl.Start < l.Start && nl.End >= l.Start {\n\t\t\t// split nl\n\t\t\tls = append(ls, nl)\n\t\t\tls[len(ls)-1].End = l.Start - 1\n\t\t\tnl.Start = l.Start\n\t\t}\n\t\tif nl.Start > l.Start && nl.Start <= l.End {\n\t\t\t// split l\n\t\t\tl.End = nl.Start - 1\n\t\t\tls = append(ls, l)\n\t\t\tls[i].Start = nl.Start\n\t\t\tl = ls[i]\n\t\t}\n\t\tif nl.Start == l.Start {\n\t\t\tls[i].Type = nl.Type // update l\n\t\t\tls[i].Pid = nl.Pid\n\t\t\tif l.End > nl.End {\n\t\t\t\t// split l\n\t\t\t\tls[i].End = nl.End\n\t\t\t\tl.Start = nl.End + 1\n\t\t\t\tls = append(ls, l)\n\t\t\t}\n\t\t\tnl.Start = ls[i].End + 1\n\t\t}\n\t}\n\tif nl.Start <= nl.End {\n\t\tls = append(ls, nl)\n\t}\n\tsort.Slice(ls, func(i, j int) bool { return ls[i].Start < ls[j].Start })\n\tfor i := 0; i < len(ls); {\n\t\tif ls[i].Type == F_UNLCK || ls[i].Start > ls[i].End {\n\t\t\t// remove empty one\n\t\t\tcopy(ls[i:], ls[i+1:])\n\t\t\tls = ls[:len(ls)-1]\n\t\t} else {\n\t\t\tif i+1 < len(ls) && ls[i].Type == ls[i+1].Type && ls[i].Pid == ls[i+1].Pid && ls[i].End+1 == ls[i+1].Start {\n\t\t\t\t// combine continuous range\n\t\t\t\tls[i].End = ls[i+1].End\n\t\t\t\tls[i+1].Start = ls[i+1].End + 1\n\t\t\t}\n\t\t\ti++\n\t\t}\n\t}\n\treturn ls\n}\n\nfunc (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count *uint64, concurrent chan int) syscall.Errno {\n\tfor {\n\t\tvar entries []*Entry\n\t\tif st := m.en.doReaddir(ctx, inode, 0, &entries, 10000); st != 0 && st != syscall.ENOENT {\n\t\t\treturn st\n\t\t}\n\t\tif len(entries) == 0 {\n\t\t\treturn 0\n\t\t}\n\t\tif st := m.Access(ctx, inode, MODE_MASK_W|MODE_MASK_X, nil); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tvar wg sync.WaitGroup\n\t\tvar status syscall.Errno\n\t\tvar nonDirEntries []*Entry\n\t\tfor i, e := range entries {\n\t\t\tif e.Attr.Typ == TypeDirectory {\n\t\t\t\tselect {\n\t\t\t\tcase concurrent <- 1:\n\t\t\t\t\twg.Add(1)\n\t\t\t\t\tgo func(child Ino, name string) {\n\t\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\t\tst := m.emptyEntry(ctx, inode, name, child, skipCheckTrash, count, concurrent)\n\t\t\t\t\t\tif st != 0 && st != syscall.ENOENT {\n\t\t\t\t\t\t\tstatus = st\n\t\t\t\t\t\t}\n\t\t\t\t\t\t<-concurrent\n\t\t\t\t\t}(e.Inode, string(e.Name))\n\t\t\t\tdefault:\n\t\t\t\t\tif st := m.emptyEntry(ctx, inode, string(e.Name), e.Inode, skipCheckTrash, count, concurrent); st != 0 && st != syscall.ENOENT {\n\t\t\t\t\t\tctx.Cancel()\n\t\t\t\t\t\treturn st\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tnonDirEntries = append(nonDirEntries, e)\n\t\t\t}\n\t\t\tif ctx.Canceled() {\n\t\t\t\treturn syscall.EINTR\n\t\t\t}\n\t\t\tentries[i] = nil // release memory\n\t\t}\n\t\twg.Wait()\n\n\t\tif status == 0 {\n\t\t\tstatus = m.BatchUnlink(ctx, inode, nonDirEntries, count, skipCheckTrash)\n\t\t}\n\n\t\tif status != 0 || inode == TrashInode { // try only once for .trash\n\t\t\treturn status\n\t\t}\n\t}\n}\n\nfunc (m *baseMeta) emptyEntry(ctx Context, parent Ino, name string, inode Ino, skipCheckTrash bool, count *uint64, concurrent chan int) syscall.Errno {\n\tif ctx.Canceled() {\n\t\treturn syscall.EINTR\n\t}\n\tst := m.emptyDir(ctx, inode, skipCheckTrash, count, concurrent)\n\tif st == 0 && !inode.IsTrash() {\n\t\tst = m.Rmdir(ctx, parent, name, skipCheckTrash)\n\t\tif st == syscall.ENOTEMPTY {\n\t\t\t// redo when concurrent conflict may happen\n\t\t\tst = m.emptyEntry(ctx, parent, name, inode, skipCheckTrash, count, concurrent)\n\t\t} else if count != nil {\n\t\t\tatomic.AddUint64(count, 1)\n\t\t}\n\t}\n\treturn st\n}\n\nfunc (m *baseMeta) Remove(ctx Context, parent Ino, name string, skipTrash bool, numThreads int, count *uint64) syscall.Errno {\n\tparent = m.checkRoot(parent)\n\tif st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, nil); st != 0 {\n\t\treturn st\n\t}\n\tvar inode Ino\n\tvar attr Attr\n\tif st := m.Lookup(ctx, parent, name, &inode, &attr, false); st != 0 {\n\t\treturn st\n\t}\n\tif attr.Typ != TypeDirectory {\n\t\tif count != nil {\n\t\t\tatomic.AddUint64(count, 1)\n\t\t}\n\t\treturn m.Unlink(ctx, parent, name, skipTrash)\n\t}\n\tif numThreads <= 0 {\n\t\tlogger.Infof(\"invalid threads number %d , auto adjust to %d\", numThreads, RmrDefaultThreads)\n\t\tnumThreads = RmrDefaultThreads\n\t} else if numThreads > 255 {\n\t\tlogger.Infof(\"threads number %d too large, auto adjust to 255 .\", numThreads)\n\t\tnumThreads = 255\n\t}\n\tlogger.Debugf(\"Start emptyEntry with %d concurrent threads .\", numThreads)\n\tconcurrent := make(chan int, numThreads)\n\treturn m.emptyEntry(ctx, parent, name, inode, skipTrash, count, concurrent)\n}\n\nfunc (m *baseMeta) GetSummary(ctx Context, inode Ino, summary *Summary, recursive bool, strict bool) syscall.Errno {\n\tvar attr Attr\n\tif st := m.GetAttr(ctx, inode, &attr); st != 0 {\n\t\treturn st\n\t}\n\tif attr.Typ != TypeDirectory {\n\t\tsummary.Files++\n\t\tsummary.Size += uint64(align4K(attr.Length))\n\t\tif attr.Typ == TypeFile {\n\t\t\tsummary.Length += attr.Length\n\t\t}\n\t\treturn 0\n\t}\n\tsummary.Dirs++\n\tsummary.Size += uint64(align4K(0))\n\tconcurrent := make(chan struct{}, 50)\n\tinode = m.checkRoot(inode)\n\treturn m.getDirSummary(ctx, inode, summary, recursive, strict, concurrent, nil)\n}\n\nfunc (m *baseMeta) getDirSummary(ctx Context, inode Ino, summary *Summary, recursive bool, strict bool, concurrent chan struct{}, updateProgress func(count uint64, bytes uint64)) syscall.Errno {\n\tvar entries []*Entry\n\tvar err syscall.Errno\n\tformat := m.getFormat()\n\tif strict || !format.DirStats {\n\t\terr = m.en.doReaddir(ctx, inode, 1, &entries, -1)\n\t} else {\n\t\tvar st *dirStat\n\t\tst, err = m.GetDirStat(ctx, inode)\n\t\tif err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tatomic.AddUint64(&summary.Size, uint64(st.space))\n\t\tatomic.AddUint64(&summary.Length, uint64(st.length))\n\t\tif updateProgress != nil {\n\t\t\tupdateProgress(uint64(st.inodes), uint64(st.space))\n\t\t}\n\t\tvar attr Attr\n\t\terr = m.en.doGetAttr(ctx, inode, &attr)\n\t\tif err == 0 {\n\t\t\tif attr.Nlink > 2 {\n\t\t\t\terr = m.en.doReaddir(ctx, inode, 0, &entries, -1)\n\t\t\t} else {\n\t\t\t\tatomic.AddUint64(&summary.Files, uint64(st.inodes))\n\t\t\t}\n\t\t}\n\t}\n\tif err != 0 {\n\t\treturn err\n\t}\n\n\tvar wg sync.WaitGroup\n\tvar errCh = make(chan syscall.Errno, 1)\n\tfor _, e := range entries {\n\t\tif e.Attr.Typ == TypeDirectory {\n\t\t\tatomic.AddUint64(&summary.Dirs, 1)\n\t\t} else {\n\t\t\tatomic.AddUint64(&summary.Files, 1)\n\t\t}\n\t\tif strict || !format.DirStats {\n\t\t\tatomic.AddUint64(&summary.Size, uint64(align4K(e.Attr.Length)))\n\t\t\tif e.Attr.Typ == TypeFile {\n\t\t\t\tatomic.AddUint64(&summary.Length, e.Attr.Length)\n\t\t\t}\n\t\t\tif updateProgress != nil {\n\t\t\t\tupdateProgress(1, uint64(align4K(e.Attr.Length)))\n\t\t\t}\n\t\t}\n\t\tif e.Attr.Typ != TypeDirectory || !recursive {\n\t\t\tcontinue\n\t\t}\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn syscall.EINTR\n\t\tcase err := <-errCh:\n\t\t\t// TODO: cancel others\n\t\t\treturn err\n\t\tcase concurrent <- struct{}{}:\n\t\t\twg.Add(1)\n\t\t\tgo func(e *Entry) {\n\t\t\t\tdefer wg.Done()\n\t\t\t\terr := m.getDirSummary(ctx, e.Inode, summary, recursive, strict, concurrent, updateProgress)\n\t\t\t\t<-concurrent\n\t\t\t\tif err != 0 && err != syscall.ENOENT {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase errCh <- err:\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}(e)\n\t\tdefault:\n\t\t\tif err := m.getDirSummary(ctx, e.Inode, summary, recursive, strict, concurrent, updateProgress); err != 0 && err != syscall.ENOENT {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\twg.Wait()\n\tselect {\n\tcase err = <-errCh:\n\tdefault:\n\t}\n\treturn err\n}\n\nfunc (m *baseMeta) GetTreeSummary(ctx Context, root *TreeSummary, depth, topN uint8, strict bool,\n\tupdateProgress func(count uint64, bytes uint64)) syscall.Errno {\n\tvar attr Attr\n\tif st := m.GetAttr(ctx, root.Inode, &attr); st != 0 {\n\t\treturn st\n\t}\n\tif updateProgress != nil {\n\t\tupdateProgress(1, uint64(align4K(0)))\n\t}\n\tif attr.Typ != TypeDirectory {\n\t\troot.Files++\n\t\troot.Size += uint64(align4K(attr.Length))\n\t\treturn 0\n\t}\n\troot.Dirs++\n\troot.Size += uint64(align4K(0))\n\tconcurrent := make(chan struct{}, 50)\n\troot.Inode = m.checkRoot(root.Inode)\n\treturn m.getTreeSummary(ctx, root, depth, topN, strict, concurrent, updateProgress)\n}\n\nfunc (m *baseMeta) getTreeSummary(ctx Context, tree *TreeSummary, depth, topN uint8, strict bool, concurrent chan struct{},\n\tupdateProgress func(count uint64, bytes uint64)) syscall.Errno {\n\tif depth <= 0 {\n\t\tvar summary Summary\n\t\terr := m.getDirSummary(ctx, tree.Inode, &summary, true, strict, concurrent, updateProgress)\n\t\tif err == 0 {\n\t\t\ttree.Dirs += summary.Dirs\n\t\t\ttree.Files += summary.Files\n\t\t\ttree.Size += summary.Size\n\t\t}\n\t\treturn err\n\t}\n\n\tvar entries []*Entry\n\tif err := m.en.doReaddir(ctx, tree.Inode, 1, &entries, -1); err != 0 {\n\t\treturn err\n\t}\n\tvar wg sync.WaitGroup\n\ttree.Children = make([]*TreeSummary, len(entries))\n\terrCh := make(chan syscall.Errno, 1)\n\tvar err syscall.Errno\n\tfor i, e := range entries {\n\t\tchild := &TreeSummary{\n\t\t\tInode: e.Inode,\n\t\t\tPath: path.Join(tree.Path, string(e.Name)),\n\t\t\tType: e.Attr.Typ,\n\t\t\tSize: uint64(align4K(e.Attr.Length)),\n\t\t}\n\t\ttree.Children[i] = child\n\t\tif updateProgress != nil {\n\t\t\tupdateProgress(1, uint64(align4K(e.Attr.Length)))\n\t\t}\n\t\tif e.Attr.Typ != TypeDirectory {\n\t\t\tchild.Files++\n\t\t\tcontinue\n\t\t}\n\t\tchild.Dirs++\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn syscall.EINTR\n\t\tcase err = <-errCh:\n\t\t\treturn err\n\t\tcase concurrent <- struct{}{}:\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\terr := m.getTreeSummary(ctx, child, depth-1, topN, strict, concurrent, updateProgress)\n\t\t\t\t<-concurrent\n\t\t\t\tif err != 0 && err != syscall.ENOENT {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase errCh <- err:\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\tdefault:\n\t\t\tif err = m.getTreeSummary(ctx, child, depth-1, topN, strict, concurrent, updateProgress); err != 0 && err != syscall.ENOENT {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\twg.Wait()\n\tselect {\n\tcase err = <-errCh:\n\t\treturn err\n\tdefault:\n\t}\n\n\t// pick top N\n\tfor _, c := range tree.Children {\n\t\ttree.Dirs += c.Dirs\n\t\ttree.Files += c.Files\n\t\ttree.Size += c.Size\n\t}\n\tsort.Slice(tree.Children, func(i, j int) bool {\n\t\treturn tree.Children[i].Size > tree.Children[j].Size\n\t})\n\tif len(tree.Children) > int(topN) {\n\t\tomitChild := &TreeSummary{\n\t\t\tPath: path.Join(tree.Path, \"...\"),\n\t\t\tType: TypeFile,\n\t\t}\n\t\tfor _, child := range tree.Children[topN:] {\n\t\t\tomitChild.Size += child.Size\n\t\t\tomitChild.Files += child.Files\n\t\t\tomitChild.Dirs += child.Dirs\n\t\t}\n\t\ttree.Children = append(tree.Children[:topN], omitChild)\n\t}\n\treturn 0\n}\n\nfunc (m *baseMeta) atimeNeedsUpdate(attr *Attr, now time.Time) bool {\n\treturn m.conf.AtimeMode != NoAtime && relatimeNeedUpdate(attr, now) ||\n\t\t// update atime only for > 1 second accesses\n\t\tm.conf.AtimeMode == StrictAtime && now.Sub(time.Unix(attr.Atime, int64(attr.Atimensec))) > time.Second\n}\n\n// With relative atime, only update atime if the previous atime is earlier than either the ctime or\n// mtime or if at least a day has passed since the last atime update.\nfunc relatimeNeedUpdate(attr *Attr, now time.Time) bool {\n\tatime := time.Unix(attr.Atime, int64(attr.Atimensec))\n\tmtime := time.Unix(attr.Mtime, int64(attr.Mtimensec))\n\tctime := time.Unix(attr.Ctime, int64(attr.Ctimensec))\n\treturn mtime.After(atime) || ctime.After(atime) || now.Sub(atime) > 24*time.Hour\n}\n\ntype txMethodKey struct{}\n\ntype txMethod string\n\nfunc (m *txMethod) name(ctx context.Context) string {\n\tif *m == \"\" {\n\t\t*m = txMethod(callerName(ctx)) // lazy evaluation\n\t}\n\treturn string(*m)\n}\n\nfunc callerName(ctx context.Context) string {\n\tif method, ok := ctx.Value(txMethodKey{}).(string); ok {\n\t\treturn method // Fast path, prefer explicitly provided method name\n\t}\n\tconst minSkip = 3\n\tfor i := minSkip; i < 20; i++ { // Slow path, find the real caller\n\t\tpc, _, _, ok := runtime.Caller(i)\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tfn := runtime.FuncForPC(pc)\n\t\tif fn == nil {\n\t\t\tcontinue\n\t\t}\n\t\tname := fn.Name()\n\t\t// Skip frames containing anonymous functions (indicated by dot+number)\n\t\tif !strings.Contains(name, \".func\") {\n\t\t\treturn utils.MethodName(name)\n\t\t}\n\t}\n\treturn \"unknown\"\n}\n" }, { "path": "pkg/meta/utils_darwin.go", "content": "package meta\n\nimport (\n\t\"syscall\"\n\n\tsys \"golang.org/x/sys/unix\"\n)\n\nconst ENOATTR = syscall.ENOATTR\nconst (\n\tF_UNLCK = syscall.F_UNLCK\n\tF_RDLCK = syscall.F_RDLCK\n\tF_WRLCK = syscall.F_WRLCK\n)\n\nconst (\n\tXattrCreateOrReplace = 0\n\tXattrCreate = sys.XATTR_CREATE\n\tXattrReplace = sys.XATTR_REPLACE\n)\n" }, { "path": "pkg/meta/utils_linux.go", "content": "package meta\n\nimport (\n\t\"syscall\"\n\n\tsys \"golang.org/x/sys/unix\"\n)\n\nconst ENOATTR = syscall.ENODATA\nconst (\n\tF_UNLCK = syscall.F_UNLCK\n\tF_RDLCK = syscall.F_RDLCK\n\tF_WRLCK = syscall.F_WRLCK\n)\n\nconst (\n\tXattrCreateOrReplace = 0\n\tXattrCreate = sys.XATTR_CREATE\n\tXattrReplace = sys.XATTR_REPLACE\n)\n" }, { "path": "pkg/meta/utils_test.go", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestRelatimeNeedUpdate(t *testing.T) {\n\tattr := &Attr{\n\t\tAtime: 1000,\n\t}\n\tif !relatimeNeedUpdate(attr, time.Now()) {\n\t\tt.Fatal(\"atime not updated for 24 hours\")\n\t}\n\n\tnow := time.Now()\n\tattr.Atime = now.Unix()\n\tattr.Ctime = now.Unix() + 10\n\tif !relatimeNeedUpdate(attr, time.Now()) {\n\t\tt.Fatal(\"atime not updated for ctime\")\n\t}\n\n\tnow = time.Now()\n\tattr.Atime = now.Unix()\n\tattr.Mtime = now.Unix() + 10\n\tif !relatimeNeedUpdate(attr, time.Now()) {\n\t\tt.Fatal(\"atime not updated for mtime\")\n\t}\n\n\tnow = time.Now()\n\tattr.Atime = now.Unix()\n\tattr.Mtime = now.Unix()\n\tattr.Ctime = now.Unix()\n\tif relatimeNeedUpdate(attr, now) {\n\t\tt.Fatal(\"atime should not be updated\")\n\t}\n}\n\nfunc TestAtimeNeedsUpdate(t *testing.T) {\n\tm := &baseMeta{\n\t\tconf: &Config{\n\t\t\tAtimeMode: NoAtime,\n\t\t},\n\t}\n\tattr := &Attr{\n\t\tAtime: 1000,\n\t}\n\tnow := time.Now()\n\tif m.atimeNeedsUpdate(attr, now) {\n\t\tt.Fatal(\"atime updated for noatime\")\n\t}\n\n\tm.conf.AtimeMode = RelAtime\n\tif !m.atimeNeedsUpdate(attr, now) {\n\t\tt.Fatal(\"atime not updated for relatime\")\n\t}\n\tattr.Atime = now.Unix()\n\tif m.atimeNeedsUpdate(attr, now) {\n\t\tt.Fatal(\"atime updated for relatime\")\n\t}\n\n\tm.conf.AtimeMode = StrictAtime\n\tattr.Atime = now.Unix() - 2\n\tif !m.atimeNeedsUpdate(attr, now) {\n\t\tt.Fatal(\"atime not updated for strictatime\")\n\t}\n\n\tattr.Atime = now.Unix() - 1\n\tattr.Atimensec = uint32(now.Nanosecond())\n\tif m.atimeNeedsUpdate(attr, now) {\n\t\tt.Fatal(\"atime updated for strictatime when < 1s\")\n\t}\n}\n\nfunc Test_getCallerName(t *testing.T) {\n\tctx := context.WithValue(context.Background(), txMethodKey{}, \"test\")\n\tvar method txMethod\n\tif method.name(ctx) != \"test\" {\n\t\tt.Fatalf(\"expected %q, got %q\", \"test\", method)\n\t}\n\tfunc() {\n\t\tvar method txMethod\n\t\tif method.name(context.Background()) != \"Test_getCallerName\" {\n\t\t\tt.Fatalf(\"expected %q, got %q\", \"Test_getCallerName\", method)\n\t\t}\n\t}()\n}\n" }, { "path": "pkg/meta/utils_windows.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage meta\n\nimport \"syscall\"\n\nconst ENOATTR = syscall.ENODATA\n\nconst (\n\tF_UNLCK = 1\n\tF_RDLCK = 2\n\tF_WRLCK = 3\n)\n\nconst (\n\tXattrCreateOrReplace = 0\n\tXattrCreate = 1\n\tXattrReplace = 2\n)\n" }, { "path": "pkg/metric/metrics.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage metric\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"strconv\"\n\t\"time\"\n\n\tconsulapi \"github.com/hashicorp/consul/api\"\n\t\"github.com/hashicorp/go-hclog\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nvar logger = utils.GetLogger(\"juicefs\")\n\nvar (\n\tstart = time.Now()\n\tcpu = prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\tName: \"cpu_usage\",\n\t\tHelp: \"Accumulated CPU usage in seconds.\",\n\t}, func() float64 {\n\t\tru := utils.GetRusage()\n\t\treturn ru.GetStime() + ru.GetUtime()\n\t})\n\tmemory = prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\tName: \"memory\",\n\t\tHelp: \"Used memory in bytes.\",\n\t}, func() float64 {\n\t\t_, rss := utils.MemoryUsage()\n\t\treturn float64(rss)\n\t})\n\tuptime = prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\tName: \"uptime\",\n\t\tHelp: \"Total running time in seconds.\",\n\t}, func() float64 {\n\t\treturn time.Since(start).Seconds()\n\t})\n)\n\nfunc UpdateMetrics(registerer prometheus.Registerer) {\n\tif registerer == nil {\n\t\treturn\n\t}\n\tregisterer.MustRegister(cpu)\n\tregisterer.MustRegister(memory)\n\tregisterer.MustRegister(uptime)\n}\n\nfunc RegisterToConsul(consulAddr, metricsAddr string, metadata map[string]string) {\n\tif metricsAddr == \"\" {\n\t\tlogger.Errorf(\"Metrics server start err,so can't register to consul\")\n\t\treturn\n\t}\n\tlocalIp, portStr, err := net.SplitHostPort(metricsAddr)\n\tif err != nil {\n\t\tlogger.Errorf(\"Metrics url format err:%s\", err)\n\t\treturn\n\t}\n\n\t// Don't register 0.0.0.0 to consul\n\tif localIp == \"0.0.0.0\" || localIp == \"::\" {\n\t\tlocalIp, err = utils.GetLocalIp(consulAddr)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Get local ip failed: %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\tport, err := strconv.Atoi(portStr)\n\tif err != nil {\n\t\tlogger.Errorf(\"Metrics port set err:%s\", err)\n\t\treturn\n\t}\n\tconfig := consulapi.DefaultConfigWithLogger(hclog.New(&hclog.LoggerOptions{ //nolint:typecheck\n\t\tName: \"consul-api\",\n\t\tOutput: logger.Out,\n\t}))\n\tconfig.Address = consulAddr\n\tclient, err := consulapi.NewClient(config)\n\tif err != nil {\n\t\tlogger.Errorf(\"Creat consul client failed:%s\", err)\n\t\treturn\n\t}\n\n\thostname, err := os.Hostname()\n\tif err != nil {\n\t\tlogger.Errorf(\"Get hostname failed:%s\", err)\n\t\treturn\n\t}\n\tmetadata[\"hostName\"] = hostname\n\tvar id, name string\n\tif mp, ok := metadata[\"mountPoint\"]; ok {\n\t\tid = fmt.Sprintf(\"%s:%s\", localIp, mp)\n\t\tname = \"juicefs\"\n\t} else {\n\t\t// for sync metrics, id format: 127.0.0.1;src->dst;pid=6666\n\t\tid = fmt.Sprintf(\"%s;%s->%s;pid=%s\", localIp, metadata[\"src\"], metadata[\"dst\"], metadata[\"pid\"])\n\t\tdelete(metadata, \"src\")\n\t\tdelete(metadata, \"dst\")\n\t\tname = \"juicefs-sync\"\n\t}\n\n\tcheck := &consulapi.AgentServiceCheck{\n\t\tHTTP: fmt.Sprintf(\"http://%s:%d/metrics\", localIp, port),\n\t\tTimeout: \"5s\",\n\t\tInterval: \"5s\",\n\t\tDeregisterCriticalServiceAfter: \"30s\",\n\t}\n\n\tregistration := consulapi.AgentServiceRegistration{\n\t\tID: id,\n\t\tName: name,\n\t\tPort: port,\n\t\tAddress: localIp,\n\t\tMeta: metadata,\n\t\tCheck: check,\n\t}\n\tif err = client.Agent().ServiceRegister(®istration); err != nil {\n\t\tlogger.Errorf(\"Service register failed: %s\", err)\n\t} else {\n\t\tlogger.Infof(\"Juicefs register to consul success, id: %q, port: %d\", id, port)\n\t}\n}\n" }, { "path": "pkg/object/azure.go", "content": "//go:build !noazure\n// +build !noazure\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\n\t\"github.com/Azure/azure-sdk-for-go/sdk/azcore\"\n\t\"github.com/Azure/azure-sdk-for-go/sdk/azidentity\"\n\t\"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob\"\n\tblob2 \"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob\"\n\t\"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror\"\n\t\"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container\"\n\t\"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas\"\n)\n\ntype wasb struct {\n\tDefaultObjectStorage\n\tcontainer *container.Client\n\tazblobCli *azblob.Client\n\tsc string\n\tcName string\n\tuseTokenAuth bool // true when using managed identity/token-based auth, false for shared key/connection string\n}\n\nfunc (b *wasb) String() string {\n\treturn fmt.Sprintf(\"wasb://%s/\", b.cName)\n}\n\nfunc (b *wasb) Create(ctx context.Context) error {\n\t_, err := b.container.Create(ctx, nil)\n\tif err != nil {\n\t\tif e, ok := err.(*azcore.ResponseError); ok && e.ErrorCode == string(bloberror.ContainerAlreadyExists) {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (b *wasb) Head(ctx context.Context, key string) (Object, error) {\n\tproperties, err := b.container.NewBlobClient(key).GetProperties(ctx, nil)\n\tif err != nil {\n\t\tif e, ok := err.(*azcore.ResponseError); ok && e.ErrorCode == string(bloberror.BlobNotFound) {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\n\treturn &obj{\n\t\tkey,\n\t\t*properties.ContentLength,\n\t\t*properties.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t*properties.AccessTier,\n\t}, nil\n}\n\nfunc (b *wasb) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tdownload, err := b.container.NewBlobClient(key).DownloadStream(ctx, &azblob.DownloadStreamOptions{Range: blob2.HTTPRange{Offset: off, Count: limit}})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tattrs := ApplyGetters(getters...)\n\t// TODO fire another property request to get the actual storage class\n\tattrs.SetRequestID(aws.ToString(download.RequestID)).SetStorageClass(b.sc)\n\treturn download.Body, err\n}\n\nfunc str2Tier(tier string) *blob2.AccessTier {\n\tfor _, v := range blob2.PossibleAccessTierValues() {\n\t\tif string(v) == tier {\n\t\t\treturn &v\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (b *wasb) Put(ctx context.Context, key string, data io.Reader, getters ...AttrGetter) error {\n\toptions := azblob.UploadStreamOptions{}\n\tif b.sc != \"\" {\n\t\toptions.AccessTier = str2Tier(b.sc)\n\t}\n\tresp, err := b.azblobCli.UploadStream(ctx, b.cName, key, data, &options)\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(aws.ToString(resp.RequestID)).SetStorageClass(b.sc)\n\treturn err\n}\n\nfunc (b *wasb) Copy(ctx context.Context, dst, src string) error {\n\tdstCli := b.container.NewBlobClient(dst)\n\tsrcCli := b.container.NewBlobClient(src)\n\toptions := &blob2.CopyFromURLOptions{}\n\tif b.sc != \"\" {\n\t\toptions.Tier = str2Tier(b.sc)\n\t}\n\n\tvar srcURL string\n\tvar err error\n\n\tif b.useTokenAuth {\n\t\t// Token-based authentication: use direct blob URL\n\t\t// Azure will authenticate using the OAuth token from the credential chain\n\t\tsrcURL = srcCli.URL()\n\t\tlogger.Debugf(\"Using token-based authentication for Copy operation (direct URL without SAS)\")\n\t} else {\n\t\t// Shared key authentication: generate SAS token for source blob\n\t\tsrcURL, err = srcCli.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(10*time.Second), nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tlogger.Debugf(\"Using shared key authentication for Copy operation (SAS URL)\")\n\t}\n\n\t_, err = dstCli.CopyFromURL(ctx, srcURL, options)\n\treturn err\n}\n\nfunc (b *wasb) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tresp, err := b.container.NewBlobClient(key).Delete(ctx, nil)\n\tif err != nil {\n\t\tif e, ok := err.(*azcore.ResponseError); ok && e.ErrorCode == string(bloberror.BlobNotFound) {\n\t\t\terr = nil\n\t\t}\n\t}\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(aws.ToString(resp.RequestID))\n\treturn err\n}\n\nfunc (b *wasb) List(ctx context.Context, prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\n\tlimit32 := int32(limit)\n\tpager := b.azblobCli.NewListBlobsFlatPager(b.cName, &azblob.ListBlobsFlatOptions{Prefix: &prefix, Marker: &token, MaxResults: &limit32})\n\tpage, err := pager.NextPage(ctx)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tvar n int\n\tif page.Segment != nil {\n\t\tn = len(page.Segment.BlobItems)\n\t}\n\tobjs := make([]Object, 0, n)\n\tfor i := 0; i < n; i++ {\n\t\tblob := page.Segment.BlobItems[i]\n\t\tif *blob.Name <= startAfter {\n\t\t\tcontinue\n\t\t}\n\t\tmtime := blob.Properties.LastModified\n\t\tobjs = append(objs, &obj{\n\t\t\t*blob.Name,\n\t\t\t*blob.Properties.ContentLength,\n\t\t\t*mtime,\n\t\t\tstrings.HasSuffix(*blob.Name, \"/\"),\n\t\t\tstring(*blob.Properties.AccessTier),\n\t\t})\n\t}\n\n\tvar nextMarker string\n\tif pager.More() {\n\t\tnextMarker = *page.NextMarker\n\t}\n\treturn objs, pager.More(), nextMarker, nil\n}\n\nfunc (b *wasb) SetStorageClass(sc string) error {\n\tb.sc = sc\n\treturn nil\n}\n\n// createAzureCredential creates a credential for Azure authentication.\n// Uses DefaultAzureCredential which attempts authentication via:\n// - Environment variables (service principal)\n// - Workload Identity (Kubernetes)\n// - Managed Identity (system-assigned and user-assigned)\n// - Azure CLI\n// - Azure Developer CLI\nfunc createAzureCredential() (azcore.TokenCredential, error) {\n\tlogger.Debugf(\"Creating DefaultAzureCredential for token-based authentication\")\n\tcred, err := azidentity.NewDefaultAzureCredential(nil)\n\tif err != nil {\n\t\tlogger.Debugf(\"Failed to create DefaultAzureCredential: %v\", err)\n\t\treturn nil, err\n\t}\n\treturn cred, nil\n}\n\nfunc autoWasbEndpoint(containerName, accountName, scheme string, credential *azblob.SharedKeyCredential) (string, error) {\n\tbaseURLs := []string{\"blob.core.windows.net\", \"blob.core.chinacloudapi.cn\"}\n\tendpoint := \"\"\n\tfor _, baseURL := range baseURLs {\n\t\tif _, err := net.LookupIP(fmt.Sprintf(\"%s.%s\", accountName, baseURL)); err != nil {\n\t\t\tlogger.Debugf(\"Attempt to resolve domain name %s failed: %s\", baseURL, err)\n\t\t\tcontinue\n\t\t}\n\t\tclient, err := azblob.NewClientWithSharedKeyCredential(fmt.Sprintf(\"%s://%s.%s\", scheme, accountName, baseURL), credential, nil)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tif _, err = client.ServiceClient().GetProperties(ctx, nil); err != nil {\n\t\t\tlogger.Debugf(\"Try to get containers properties at %s failed: %s\", baseURL, err)\n\t\t\tcontinue\n\t\t}\n\t\tendpoint = baseURL\n\t\tbreak\n\t}\n\n\tif endpoint == \"\" {\n\t\treturn \"\", fmt.Errorf(\"fail to get endpoint for container %s\", containerName)\n\t}\n\treturn endpoint, nil\n}\n\nfunc autoWasbEndpointWithToken(containerName, accountName, scheme string, credential azcore.TokenCredential) (string, error) {\n\tbaseURLs := []string{\"blob.core.windows.net\", \"blob.core.chinacloudapi.cn\"}\n\tendpoint := \"\"\n\tfor _, baseURL := range baseURLs {\n\t\tif _, err := net.LookupIP(fmt.Sprintf(\"%s.%s\", accountName, baseURL)); err != nil {\n\t\t\tlogger.Debugf(\"Attempt to resolve domain name %s failed: %s\", baseURL, err)\n\t\t\tcontinue\n\t\t}\n\t\tclient, err := azblob.NewClient(fmt.Sprintf(\"%s://%s.%s\", scheme, accountName, baseURL), credential, nil)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tif _, err = client.ServiceClient().GetProperties(ctx, nil); err != nil {\n\t\t\tlogger.Debugf(\"Try to get service properties at %s failed: %s\", baseURL, err)\n\t\t\tcontinue\n\t\t}\n\t\tendpoint = baseURL\n\t\tbreak\n\t}\n\n\tif endpoint == \"\" {\n\t\treturn \"\", fmt.Errorf(\"fail to get endpoint for container %s\", containerName)\n\t}\n\treturn endpoint, nil\n}\n\nfunc newWasb(endpoint, accountName, accountKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\tcontainerName := hostParts[0]\n\n\t// Priority 1: Connection string support\n\t// DefaultEndpointsProtocol=[http|https];AccountName=***;AccountKey=***;EndpointSuffix=[core.windows.net|core.chinacloudapi.cn]\n\tif connString := os.Getenv(\"AZURE_STORAGE_CONNECTION_STRING\"); connString != \"\" {\n\t\tlogger.Debugf(\"Using Azure connection string authentication\")\n\t\tvar client *azblob.Client\n\t\tif client, err = azblob.NewClientFromConnectionString(connString, nil); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &wasb{container: client.ServiceClient().NewContainerClient(containerName), azblobCli: client, cName: containerName, useTokenAuth: false}, nil\n\t}\n\n\t// Priority 2: Try managed identity / token-based authentication if no account key provided\n\tif accountKey == \"\" {\n\t\tlogger.Debugf(\"No account key provided, attempting token-based authentication (managed identity, Azure CLI, etc.)\")\n\t\ttokenCred, err := createAzureCredential()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Failed to create Azure credential (managed identity/Azure CLI): %v\", err)\n\t\t}\n\n\t\tvar domain string\n\t\tif len(hostParts) > 1 {\n\t\t\tdomain = hostParts[1]\n\t\t\tif !strings.HasPrefix(hostParts[1], \"blob\") {\n\t\t\t\tdomain = fmt.Sprintf(\"blob.%s\", hostParts[1])\n\t\t\t}\n\t\t} else if domain, err = autoWasbEndpointWithToken(containerName, accountName, uri.Scheme, tokenCred); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Unable to get endpoint of container %s: %s\", containerName, err)\n\t\t}\n\n\t\tserviceURL := fmt.Sprintf(\"%s://%s.%s\", uri.Scheme, accountName, domain)\n\t\tclient, err := azblob.NewClient(serviceURL, tokenCred, nil)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Failed to create Azure blob client with token credential: %v\", err)\n\t\t}\n\t\tlogger.Debugf(\"Successfully authenticated using token-based credential\")\n\t\treturn &wasb{container: client.ServiceClient().NewContainerClient(containerName), azblobCli: client, cName: containerName, useTokenAuth: true}, nil\n\t}\n\n\t// Priority 3: Shared key authentication (existing behavior)\n\tlogger.Debugf(\"Using Azure shared key authentication\")\n\tcredential, err := azblob.NewSharedKeyCredential(accountName, accountKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar domain string\n\tif len(hostParts) > 1 {\n\t\tdomain = hostParts[1]\n\t\tif !strings.HasPrefix(hostParts[1], \"blob\") {\n\t\t\tdomain = fmt.Sprintf(\"blob.%s\", hostParts[1])\n\t\t}\n\t} else if domain, err = autoWasbEndpoint(containerName, accountName, uri.Scheme, credential); err != nil {\n\t\treturn nil, fmt.Errorf(\"Unable to get endpoint of container %s: %s\", containerName, err)\n\t}\n\n\tclient, err := azblob.NewClientWithSharedKeyCredential(fmt.Sprintf(\"%s://%s.%s\", uri.Scheme, accountName, domain), credential, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &wasb{container: client.ServiceClient().NewContainerClient(containerName), azblobCli: client, cName: containerName, useTokenAuth: false}, nil\n}\n\nfunc init() {\n\tRegister(\"wasb\", newWasb)\n}\n" }, { "path": "pkg/object/b2.go", "content": "//go:build !nob2\n// +build !nob2\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"gopkg.in/kothar/go-backblaze.v0\"\n)\n\ntype b2client struct {\n\tDefaultObjectStorage\n\tbucket *backblaze.Bucket\n}\n\nfunc (c *b2client) String() string {\n\treturn fmt.Sprintf(\"b2://%s/\", c.bucket.Name)\n}\n\nfunc (c *b2client) Create(ctx context.Context) error {\n\treturn nil\n}\n\nfunc (c *b2client) getFileInfo(key string) (*backblaze.File, error) {\n\tvar f *backblaze.File\n\tvar r io.ReadCloser\n\tvar err error\n\tf, r, err = c.bucket.DownloadFileRangeByName(key, &backblaze.FileRange{Start: 0, End: 1})\n\tif err != nil {\n\t\t//\tget empty file info\n\t\tif e, ok := err.(*backblaze.B2Error); ok && e.Status == http.StatusRequestedRangeNotSatisfiable {\n\t\t\tf, r, err = c.bucket.DownloadFileRangeByName(key, nil)\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar buf [2]byte\n\t_, _ = r.Read(buf[:])\n\t_ = r.Close()\n\treturn f, nil\n}\n\nfunc (c *b2client) Head(ctx context.Context, key string) (Object, error) {\n\tf, err := c.getFileInfo(key)\n\tif err != nil {\n\t\tif e, ok := err.(*backblaze.B2Error); ok && e.Status == http.StatusNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tf.Name,\n\t\tf.ContentLength,\n\t\ttime.Unix(f.UploadTimestamp/1000, 0),\n\t\tstrings.HasSuffix(f.Name, \"/\"),\n\t\t\"\",\n\t}, nil\n}\n\nfunc (c *b2client) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tif off == 0 && limit == -1 {\n\t\t_, r, err := c.bucket.DownloadFileByName(key)\n\t\treturn r, err\n\t}\n\tif limit == -1 {\n\t\tlimit = 1 << 50\n\t}\n\trang := &backblaze.FileRange{Start: off, End: off + limit - 1}\n\t_, r, err := c.bucket.DownloadFileRangeByName(key, rang)\n\treturn r, err\n}\n\nfunc (c *b2client) Put(ctx context.Context, key string, data io.Reader, getters ...AttrGetter) error {\n\t_, err := c.bucket.UploadFile(key, nil, data)\n\treturn err\n}\n\nfunc (c *b2client) Copy(ctx context.Context, dst, src string) error {\n\tf, err := c.getFileInfo(src)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// destinationBucketId must be set,otherwise it will return 400 Bad destinationBucketId\n\t_, err = c.bucket.CopyFile(f.ID, dst, c.bucket.ID, backblaze.FileMetaDirectiveCopy)\n\treturn err\n}\n\nfunc (c *b2client) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tf, err := c.getFileInfo(key)\n\tif err != nil {\n\t\tif strings.HasPrefix(err.Error(), \"not_found\") {\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\t_, err = c.bucket.DeleteFileVersion(key, f.ID)\n\treturn err\n}\n\nfunc (c *b2client) List(ctx context.Context, prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\n\tresp, err := c.bucket.ListFileNamesWithPrefix(startAfter, int(limit), prefix, delimiter)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\n\tn := len(resp.Files)\n\tobjs := make([]Object, 0, n)\n\tfor i := 0; i < n; i++ {\n\t\tif resp.Files[i].Name <= startAfter {\n\t\t\tcontinue\n\t\t}\n\t\tf := resp.Files[i]\n\t\tobjs = append(objs, &obj{\n\t\t\tf.Name,\n\t\t\tf.ContentLength,\n\t\t\ttime.Unix(f.UploadTimestamp/1000, 0),\n\t\t\tstrings.HasSuffix(f.Name, \"/\"),\n\t\t\t\"\",\n\t\t})\n\t}\n\treturn objs, resp.NextFileName != \"\", resp.NextFileName, nil\n}\n\n// TODO: support multipart upload using S3 client\n\nfunc newB2(endpoint, keyID, applicationKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\thostParts := strings.Split(uri.Host, \".\")\n\tname := hostParts[0]\n\tclient, err := backblaze.NewB2(backblaze.Credentials{\n\t\tKeyID: keyID,\n\t\tApplicationKey: applicationKey,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"create B2 client: %s\", err)\n\t}\n\tclient.MaxIdleUploads = 20\n\tbucket, err := client.Bucket(name)\n\tif err != nil {\n\t\tlogger.Warnf(\"access bucket %s: %s\", name, err)\n\t}\n\tif err == nil && bucket == nil {\n\t\tbucket, err = client.CreateBucket(name, \"allPrivate\")\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"create bucket %s: %s\", name, err)\n\t\t}\n\t}\n\tif bucket == nil {\n\t\treturn nil, fmt.Errorf(\"can't find bucket %s with provided Key ID\", name)\n\t}\n\treturn &b2client{bucket: bucket}, nil\n}\n\nfunc init() {\n\tRegister(\"b2\", newB2)\n}\n" }, { "path": "pkg/object/bos.go", "content": "//go:build !nobos\n// +build !nobos\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/baidubce/bce-sdk-go/bce\"\n\t\"github.com/baidubce/bce-sdk-go/services/bos\"\n\t\"github.com/baidubce/bce-sdk-go/services/bos/api\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\ntype bosclient struct {\n\tDefaultObjectStorage\n\tbucket string\n\tsc string\n\tc *bos.Client\n}\n\nfunc (q *bosclient) String() string {\n\treturn fmt.Sprintf(\"bos://%s/\", q.bucket)\n}\n\nfunc (q *bosclient) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 100 << 10,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (q *bosclient) SetStorageClass(sc string) error {\n\tq.sc = sc\n\treturn nil\n}\n\nfunc (q *bosclient) Create(ctx context.Context) error {\n\t_, err := q.c.PutBucket(q.bucket)\n\tif err == nil && q.sc != \"\" {\n\t\tif err := q.c.PutBucketStorageclass(q.bucket, q.sc); err != nil {\n\t\t\tlogger.Warnf(\"failed to set storage class: %v\", err)\n\t\t}\n\t}\n\tif err != nil && isExists(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (q *bosclient) Head(ctx context.Context, key string) (Object, error) {\n\tr, err := q.c.GetObjectMeta(q.bucket, key)\n\tif err != nil {\n\t\tif e, ok := err.(*bce.BceServiceError); ok && e.StatusCode == http.StatusNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\tmtime, _ := time.Parse(time.RFC1123, r.LastModified)\n\treturn &obj{\n\t\tkey,\n\t\tr.ContentLength,\n\t\tmtime,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\tr.StorageClass,\n\t}, nil\n}\n\nfunc (q *bosclient) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (resp io.ReadCloser, err error) {\n\tvar r *api.GetObjectResult\n\tvar needCheck bool\n\tif limit > 0 {\n\t\tr, err = q.c.GetObject(q.bucket, key, nil, off, off+limit-1)\n\t} else if off > 0 {\n\t\tr, err = q.c.GetObject(q.bucket, key, nil, off)\n\t} else {\n\t\tr, err = q.c.GetObject(q.bucket, key, nil)\n\t\tneedCheck = true\n\t}\n\tif err != nil {\n\t\treturn\n\t}\n\tif needCheck {\n\t\tif r.UserMeta[checksumAlgr] != \"\" {\n\t\t\tresp = verifyChecksum(r.Body, r.UserMeta[checksumAlgr], r.ContentLength)\n\t\t} else {\n\t\t\tresp = verifyChecksum0(r.Body, r.ContentCrc32, r.ContentLength, crc32.IEEETable)\n\t\t}\n\t} else {\n\t\tresp = r.Body\n\t}\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetStorageClass(r.StorageClass)\n\treturn\n}\n\nfunc (q *bosclient) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tb, vlen, err := findLen(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar data []byte\n\tif bf, ok := b.(*bytes.Buffer); ok {\n\t\tdata = bf.Bytes()\n\t} else {\n\t\tdata = utils.Alloc0(int(vlen))\n\t\tdefer utils.Free0(data)\n\t\t_, err = io.ReadFull(b, data)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tbody, err := bce.NewBodyFromBytes(data)\n\tif err != nil {\n\t\treturn err\n\t}\n\targs := new(api.PutObjectArgs)\n\tif q.sc != \"\" {\n\t\targs.StorageClass = q.sc\n\t}\n\targs.UserMeta = make(map[string]string)\n\targs.UserMeta[checksumAlgr] = strconv.Itoa(int(crc32.Update(0, crc32c, data)))\n\t_, err = q.c.PutObject(q.bucket, key, body, args)\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetStorageClass(q.sc)\n\treturn err\n}\n\nfunc (q *bosclient) Copy(ctx context.Context, dst, src string) error {\n\tvar args *api.CopyObjectArgs\n\tif q.sc != \"\" {\n\t\targs = &api.CopyObjectArgs{ObjectMeta: api.ObjectMeta{StorageClass: q.sc}}\n\t}\n\t_, err := q.c.CopyObject(q.bucket, dst, q.bucket, src, args)\n\treturn err\n}\n\nfunc (q *bosclient) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\terr := q.c.DeleteObject(q.bucket, key)\n\tif err != nil && strings.Contains(err.Error(), \"NoSuchKey\") {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (q *bosclient) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\tlimit_ := int(limit)\n\tout, err := q.c.SimpleListObjects(q.bucket, prefix, limit_, start, delimiter)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(out.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\tk := out.Contents[i]\n\t\tmod, _ := time.Parse(\"2006-01-02T15:04:05Z\", k.LastModified)\n\t\tobjs[i] = &obj{k.Key, int64(k.Size), mod, strings.HasSuffix(k.Key, \"/\"), k.StorageClass}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range out.CommonPrefixes {\n\t\t\tobjs = append(objs, &obj{p.Prefix, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, out.IsTruncated, out.NextMarker, nil\n}\n\nfunc (q *bosclient) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\targs := new(api.InitiateMultipartUploadArgs)\n\tif q.sc != \"\" {\n\t\targs.StorageClass = q.sc\n\t}\n\tr, err := q.c.InitiateMultipartUpload(q.bucket, key, \"\", args)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: r.UploadId, MinPartSize: 4 << 20, MaxCount: 10000}, nil\n}\n\nfunc (q *bosclient) UploadPart(ctx context.Context, key string, uploadID string, num int, data []byte) (*Part, error) {\n\tbody, _ := bce.NewBodyFromBytes(data)\n\tetag, err := q.c.BasicUploadPart(q.bucket, key, uploadID, num, body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, Size: len(data), ETag: etag}, nil\n}\n\nfunc (q *bosclient) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tresult, err := q.c.UploadPartCopy(q.bucket, key, q.bucket, srcKey, uploadID, num,\n\t\t&api.UploadPartCopyArgs{SourceRange: fmt.Sprintf(\"bytes=%d-%d\", off, off+size-1)})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, Size: int(size), ETag: result.ETag}, nil\n}\n\nfunc (q *bosclient) AbortUpload(ctx context.Context, key string, uploadID string) {\n\t_ = q.c.AbortMultipartUpload(q.bucket, key, uploadID)\n}\n\nfunc (q *bosclient) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\toparts := make([]api.UploadInfoType, len(parts))\n\tfor i := range parts {\n\t\toparts[i] = api.UploadInfoType{\n\t\t\tPartNumber: parts[i].Num,\n\t\t\tETag: parts[i].ETag,\n\t\t}\n\t}\n\tps := api.CompleteMultipartUploadArgs{Parts: oparts}\n\t_, err := q.c.CompleteMultipartUploadFromStruct(q.bucket, key, uploadID, &ps)\n\treturn err\n}\n\nfunc (q *bosclient) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tresult, err := q.c.ListMultipartUploads(q.bucket, &api.ListMultipartUploadsArgs{\n\t\tMaxUploads: 1000,\n\t\tKeyMarker: marker,\n\t})\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tparts[i] = &PendingPart{u.Key, u.UploadId, time.Time{}}\n\t}\n\treturn parts, result.NextKeyMarker, nil\n}\n\nfunc autoBOSEndpoint(bucketName, accessKey, secretKey string) (string, error) {\n\tregion := bce.DEFAULT_REGION\n\tif r := os.Getenv(\"BDCLOUD_DEFAULT_REGION\"); r != \"\" {\n\t\tregion = r\n\t}\n\n\tendpoint := fmt.Sprintf(\"https://%s.bcebos.com\", region)\n\tbosCli, err := bos.NewClient(accessKey, secretKey, endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif location, err := bosCli.GetBucketLocation(bucketName); err != nil {\n\t\treturn \"\", err\n\t} else {\n\t\treturn fmt.Sprintf(\"%s.bcebos.com\", location), nil\n\t}\n}\n\nfunc newBOS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\tif len(hostParts) != 2 {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint: %v\", endpoint)\n\t}\n\tbucketName := hostParts[0]\n\tif accessKey == \"\" {\n\t\taccessKey = os.Getenv(\"BDCLOUD_ACCESS_KEY\")\n\t\tsecretKey = os.Getenv(\"BDCLOUD_SECRET_KEY\")\n\t}\n\tendpoint = hostParts[1]\n\tif hostParts[1] == \"bcebos.com\" {\n\t\tif endpoint, err = autoBOSEndpoint(bucketName, accessKey, secretKey); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Fail to get location of bucket %q: %s\", bucketName, err)\n\t\t}\n\t}\n\tendpoint = fmt.Sprintf(\"%s://%s\", uri.Scheme, endpoint)\n\tlogger.Debugf(\"Use endpoint: %s\", endpoint)\n\t// endpoint format like https://bj.bcebos.com\n\tbosClient, err := bos.NewClient(accessKey, secretKey, endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbosClient.Config.Retry = bce.NewNoRetryPolicy()\n\tbosClient.Config.UserAgent = UserAgent\n\treturn &bosclient{bucket: bucketName, c: bosClient}, nil\n}\n\nfunc init() {\n\tRegister(\"bos\", newBOS)\n}\n" }, { "path": "pkg/object/bunny.go", "content": "//go:build bunny\n// +build bunny\n\n/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"strings\"\n\t\"time\"\n\n\tbunny \"github.com/l0wl3vel/bunny-storage-go-sdk\"\n)\n\ntype bunnyClient struct {\n\tDefaultObjectStorage\n\tclient *bunny.Client\n\tendpoint string\n}\n\n// Description of the object storage.\nfunc (b *bunnyClient) String() string {\n\treturn fmt.Sprintf(\"bunny://%v\", b.endpoint)\n}\n\n// Get the data for the given object specified by key.\nfunc (b *bunnyClient) Get(ctx context.Context, key string, off int64, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tvar end int64\n\tif limit == -1 {\n\t\tend = math.MaxInt64\n\t} else {\n\t\tend = off + limit - 1\n\t}\n\tbody, err := b.client.DownloadPartial(key, off, end)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn io.NopCloser(bytes.NewReader(body)), nil\n}\n\n// Put data read from a reader to an object specified by key.\nfunc (b *bunnyClient) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tcontent, readErr := io.ReadAll(in)\n\tif readErr != nil {\n\t\treturn readErr\n\t}\n\treturn b.client.Upload(key, content, true)\n}\n\n// Delete a object.\nfunc (b *bunnyClient) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\terr := b.client.Delete(key, false)\n\tif err != nil && err.Error() == \"Not Found\" {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (b *bunnyClient) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tvar output []Object\n\tvar dir = prefix\n\tif !strings.HasSuffix(dir, dirSuffix) { // If no Directory list in parent directory\n\t\tdir = path.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t}\n\n\tlistedObjects, err := b.client.List(dir)\n\tif err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\terr = nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\n\tfor _, o := range listedObjects {\n\t\tnormalizedPath := normalizedObjectNameWithinZone(o)\n\t\tif !strings.HasPrefix(normalizedPath, prefix) || (marker != \"\" && normalizedPath <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\toutput = append(output, parseObjectMetadata(o))\n\t\tif len(output) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn generateListResult(output, limit)\n}\n\n// The Object Path returned by the Bunny API contains the Storage Zone Name, which this function removes\nfunc normalizedObjectNameWithinZone(o bunny.Object) string {\n\tnormalizedPath := path.Join(o.Path, o.ObjectName)\n\tif o.IsDirectory {\n\t\tnormalizedPath = normalizedPath + \"/\" // Append a trailing slash to allow deletion of directories\n\t}\n\treturn strings.TrimPrefix(normalizedPath, \"/\"+o.StorageZoneName+\"/\")\n}\n\nfunc parseObjectMetadata(object bunny.Object) Object {\n\tlastChanged, _ := time.Parse(\"2006-01-02T15:04:05\", object.LastChanged)\n\n\tkey := normalizedObjectNameWithinZone(object)\n\tif object.IsDirectory && !strings.HasSuffix(key, \"/\") {\n\t\tkey = key + \"/\"\n\t}\n\treturn &obj{\n\t\tkey,\n\t\tint64(object.Length),\n\t\tlastChanged,\n\t\tobject.IsDirectory,\n\t\t\"\",\n\t}\n}\n\nfunc (b *bunnyClient) Head(ctx context.Context, key string) (Object, error) {\n\tobject, err := b.client.Describe(key)\n\tif err != nil {\n\t\tif err.Error() == \"Not Found\" {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn parseObjectMetadata(object), nil\n}\n\nfunc newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\tendpoint_url, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tclient := bunny.NewClient(*endpoint_url, password)\n\treturn &bunnyClient{client: &client, endpoint: endpoint}, nil\n}\n\nfunc init() {\n\tRegister(\"bunny\", newBunny)\n}\n" }, { "path": "pkg/object/ceph.go", "content": "//go:build ceph\n// +build ceph\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/ceph/go-ceph/rados\"\n)\n\ntype ceph struct {\n\tDefaultObjectStorage\n\tname string\n\tconn *rados.Conn\n\tfree chan *rados.IOContext\n}\n\nfunc (c *ceph) String() string {\n\treturn fmt.Sprintf(\"ceph://%s/\", c.name)\n}\n\nfunc (c *ceph) Shutdown() {\n\tc.conn.Shutdown()\n}\n\nfunc (c *ceph) Create(_ context.Context) error {\n\tnames, err := c.conn.ListPools()\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, name := range names {\n\t\tif name == c.name {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn c.conn.MakePool(c.name)\n}\n\nfunc (c *ceph) newContext() (*rados.IOContext, error) {\n\tselect {\n\tcase ctx := <-c.free:\n\t\treturn ctx, nil\n\tdefault:\n\t\tctx, err := c.conn.OpenIOContext(c.name)\n\t\tif err == nil {\n\t\t\t_ = ctx.SetPoolFullTry()\n\t\t}\n\t\treturn ctx, err\n\t}\n}\n\nfunc (c *ceph) release(ctx *rados.IOContext) {\n\tselect {\n\tcase c.free <- ctx:\n\tdefault:\n\t\tctx.Destroy()\n\t}\n}\n\nfunc (c *ceph) do(f func(ctx *rados.IOContext) error) (err error) {\n\tctx, err := c.newContext()\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = f(ctx)\n\tif err != nil {\n\t\tctx.Destroy()\n\t} else {\n\t\tc.release(ctx)\n\t}\n\treturn\n}\n\ntype cephReader struct {\n\tc *ceph\n\tctx *rados.IOContext\n\tkey string\n\toff int64\n\tlimit int64\n}\n\nfunc (r *cephReader) Read(buf []byte) (n int, err error) {\n\tif r.limit == 0 {\n\t\treturn 0, io.EOF\n\t}\n\tif r.limit > 0 && int64(len(buf)) > r.limit {\n\t\tbuf = buf[:r.limit]\n\t}\n\tn, err = r.ctx.Read(r.key, buf, uint64(r.off))\n\tr.off += int64(n)\n\tif r.limit > 0 {\n\t\tr.limit -= int64(n)\n\t}\n\tif err == nil && n < len(buf) {\n\t\terr = io.EOF\n\t}\n\treturn\n}\n\nfunc (r *cephReader) Close() error {\n\tif r.ctx != nil {\n\t\tr.c.release(r.ctx)\n\t\tr.ctx = nil\n\t}\n\treturn nil\n}\n\nfunc (c *ceph) Get(_ context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tif _, err := c.Head(context.TODO(), key); err != nil {\n\t\treturn nil, err\n\t}\n\tctx, err := c.newContext()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &cephReader{c, ctx, key, off, limit}, nil\n}\n\nvar cephPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn make([]byte, 1<<20)\n\t},\n}\n\nfunc (c *ceph) Put(_ context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\t// ceph default osd_max_object_size = 128M\n\treturn c.do(func(ctx *rados.IOContext) error {\n\t\tif b, ok := in.(*bytes.Reader); ok {\n\t\t\tv := reflect.ValueOf(b)\n\t\t\tdata := v.Elem().Field(0).Bytes()\n\t\t\tif len(data) == 0 {\n\t\t\t\treturn notSupported\n\t\t\t}\n\t\t\t// If the data exceeds 90M, ceph will report an error: 'rados: ret=-90, Message too long'\n\t\t\tif len(data) < 85<<20 {\n\t\t\t\treturn ctx.WriteFull(key, data)\n\t\t\t}\n\t\t}\n\t\tbuf := cephPool.Get().([]byte)\n\t\tdefer cephPool.Put(buf)\n\t\tvar off uint64\n\t\tfor {\n\t\t\tn, err := in.Read(buf)\n\t\t\tif n > 0 {\n\t\t\t\tif err = ctx.Write(key, buf[:n], off); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\toff += uint64(n)\n\t\t\t} else {\n\t\t\t\tif err == io.EOF {\n\t\t\t\t\tif off == 0 {\n\t\t\t\t\t\treturn errors.New(\"ceph: can't put empty file\")\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc (c *ceph) Delete(_ context.Context, key string, getters ...AttrGetter) error {\n\terr := c.do(func(ctx *rados.IOContext) error {\n\t\treturn ctx.Delete(key)\n\t})\n\tif err == rados.ErrNotFound {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (c *ceph) Head(_ context.Context, key string) (Object, error) {\n\tvar o *obj\n\terr := c.do(func(ctx *rados.IOContext) error {\n\t\tstat, err := ctx.Stat(key)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\to = &obj{key, int64(stat.Size), stat.ModTime, strings.HasSuffix(key, \"/\"), \"\"}\n\t\treturn nil\n\t})\n\tif err == rados.ErrNotFound {\n\t\terr = os.ErrNotExist\n\t}\n\treturn o, err\n}\n\nfunc (c *ceph) ListAll(_ context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\tctx, err := c.newContext()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\titer, err := ctx.Iter()\n\tif err != nil {\n\t\tctx.Destroy()\n\t\treturn nil, err\n\t}\n\tdefer iter.Close()\n\n\t// FIXME: this will be really slow for many objects\n\tkeys := make([]string, 0, 1000)\n\tfor iter.Next() {\n\t\tkey := iter.Value()\n\t\tif key <= marker || !strings.HasPrefix(key, prefix) {\n\t\t\tcontinue\n\t\t}\n\t\tkeys = append(keys, key)\n\t}\n\t// the keys are not ordered, sort them first\n\tsort.Strings(keys)\n\tc.release(ctx)\n\n\tvar objs = make(chan Object, 1000)\n\tvar concurrent = 20\n\tms := make([]sync.Mutex, concurrent)\n\tconds := make([]*sync.Cond, concurrent)\n\tready := make([]bool, concurrent)\n\tresults := make([]Object, concurrent)\n\terrs := make([]error, concurrent)\n\tfor j := 0; j < concurrent; j++ {\n\t\tconds[j] = sync.NewCond(&ms[j])\n\t\tif j < len(keys) {\n\t\t\tgo func(j int) {\n\t\t\t\tctx, err := c.newContext()\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"new context: %s\", err)\n\t\t\t\t\terrs[j] = err\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tdefer ctx.Destroy()\n\t\t\t\tfor i := j; i < len(keys); i += concurrent {\n\t\t\t\t\tkey := keys[i]\n\t\t\t\t\tst, err := ctx.Stat(key)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tif errors.Is(err, rados.ErrNotFound) {\n\t\t\t\t\t\t\tlogger.Debugf(\"Skip non-existent key: %s\", key)\n\t\t\t\t\t\t\tresults[j] = nil\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tlogger.Errorf(\"Stat key %s: %s\", key, err)\n\t\t\t\t\t\t\terrs[j] = err\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tresults[j] = &obj{key, int64(st.Size), st.ModTime, strings.HasSuffix(key, \"/\"), \"\"}\n\t\t\t\t\t}\n\n\t\t\t\t\tms[j].Lock()\n\t\t\t\t\tready[j] = true\n\t\t\t\t\tconds[j].Signal()\n\t\t\t\t\tif errs[j] != nil {\n\t\t\t\t\t\tms[j].Unlock()\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tfor ready[j] {\n\t\t\t\t\t\tconds[j].Wait()\n\t\t\t\t\t}\n\t\t\t\t\tms[j].Unlock()\n\t\t\t\t}\n\t\t\t}(j)\n\t\t}\n\t}\n\tgo func() {\n\t\tdefer close(objs)\n\t\tfor i := range keys {\n\t\t\tj := i % concurrent\n\t\t\tms[j].Lock()\n\t\t\tfor !ready[j] {\n\t\t\t\tconds[j].Wait()\n\t\t\t}\n\t\t\tif errs[j] != nil {\n\t\t\t\tobjs <- nil\n\t\t\t\tms[j].Unlock()\n\t\t\t\t// some goroutines will be leaked, but it's ok\n\t\t\t\t// since we won't call ListAll() many times in a process\n\t\t\t\tbreak\n\t\t\t} else if results[j] != nil {\n\t\t\t\tobjs <- results[j]\n\t\t\t}\n\t\t\tready[j] = false\n\t\t\tconds[j].Signal()\n\t\t\tms[j].Unlock()\n\t\t}\n\t}()\n\treturn objs, nil\n}\n\nfunc newCeph(endpoint, cluster, user, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"ceph://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tname := uri.Host\n\tconn, err := rados.NewConnWithClusterAndUser(cluster, user)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Can't create connection to cluster %s for user %s: %s\", cluster, user, err)\n\t}\n\tif opt := os.Getenv(\"CEPH_ADMIN_SOCKET\"); opt != \"none\" {\n\t\tif opt == \"\" {\n\t\t\topt = \"$run_dir/jfs-$cluster-$name-$pid.asok\"\n\t\t}\n\t\tif err = conn.SetConfigOption(\"admin_socket\", opt); err != nil {\n\t\t\tlogger.Warnf(\"Failed to set admin_socket to %s: %s\", opt, err)\n\t\t}\n\t}\n\tif opt := os.Getenv(\"CEPH_LOG_FILE\"); opt != \"none\" {\n\t\tif opt == \"\" {\n\t\t\topt = \"/var/log/ceph/jfs-$cluster-$name.log\"\n\t\t}\n\t\tif err = conn.SetConfigOption(\"log_file\", opt); err != nil {\n\t\t\tlogger.Warnf(\"Failed to set log_file to %s: %s\", opt, err)\n\t\t}\n\t}\n\tif os.Getenv(\"JFS_NO_CHECK_OBJECT_STORAGE\") == \"\" {\n\t\tif err := conn.ReadDefaultConfigFile(); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Can't read default config file: %s\", err)\n\t\t}\n\t\tif err := conn.Connect(); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Can't connect to cluster %s: %s\", cluster, err)\n\t\t}\n\t}\n\treturn &ceph{\n\t\tname: name,\n\t\tconn: conn,\n\t\tfree: make(chan *rados.IOContext, 50),\n\t}, nil\n}\n\nfunc init() {\n\tRegister(\"ceph\", newCeph)\n}\n" }, { "path": "pkg/object/checksum.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"reflect\"\n\t\"strconv\"\n)\n\nconst checksumAlgr = \"Crc32c\"\n\nvar crc32c = crc32.MakeTable(crc32.Castagnoli)\n\nfunc generateChecksum(in io.ReadSeeker) string {\n\tif b, ok := in.(*bytes.Reader); ok {\n\t\tv := reflect.ValueOf(b)\n\t\tdata := v.Elem().Field(0).Bytes()\n\t\treturn strconv.Itoa(int(crc32.Update(0, crc32c, data)))\n\t}\n\tvar hash uint32\n\tcrcBuffer := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(crcBuffer)\n\tdefer func() { _, _ = in.Seek(0, io.SeekStart) }()\n\tfor {\n\t\tn, err := in.Read(*crcBuffer)\n\t\thash = crc32.Update(hash, crc32c, (*crcBuffer)[:n])\n\t\tif err != nil {\n\t\t\tif err != io.EOF {\n\t\t\t\treturn \"\"\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\treturn strconv.Itoa(int(hash))\n}\n\ntype checksumReader struct {\n\tio.ReadCloser\n\texpected uint32\n\tchecksum uint32\n\tremainingLength int64\n\ttable *crc32.Table\n}\n\nfunc (c *checksumReader) Read(buf []byte) (n int, err error) {\n\tn, err = c.ReadCloser.Read(buf)\n\tc.checksum = crc32.Update(c.checksum, c.table, buf[:n])\n\tc.remainingLength -= int64(n)\n\tif (err == io.EOF || c.remainingLength == 0) && c.checksum != c.expected {\n\t\treturn 0, fmt.Errorf(\"verify checksum failed: %d != %d\", c.checksum, c.expected)\n\t}\n\treturn\n}\nfunc verifyChecksum(in io.ReadCloser, checksum string, contentLength int64) io.ReadCloser {\n\treturn verifyChecksum0(in, checksum, contentLength, crc32c)\n}\nfunc verifyChecksum0(in io.ReadCloser, checksum string, contentLength int64, table *crc32.Table) io.ReadCloser {\n\tif checksum == \"\" {\n\t\treturn in\n\t}\n\texpected, err := strconv.Atoi(checksum)\n\tif err != nil {\n\t\tlogger.Errorf(\"invalid crc32c: %s\", checksum)\n\t\treturn in\n\t}\n\treturn &checksumReader{in, uint32(expected), 0, contentLength, table}\n}\n" }, { "path": "pkg/object/checksum_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nfunc TestChecksum(t *testing.T) {\n\tb := []byte(\"hello\")\n\texpected := crc32.Update(0, crc32c, b)\n\tactual := generateChecksum(bytes.NewReader(b))\n\tif actual != strconv.Itoa(int(expected)) {\n\t\tt.Errorf(\"expect %d but got %s\", expected, actual)\n\t\tt.FailNow()\n\t}\n\n\tactual = generateChecksum(bytes.NewReader(b))\n\tif actual != strconv.Itoa(int(expected)) {\n\t\tt.Errorf(\"expect %d but got %s\", expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestChecksumRead(t *testing.T) {\n\tlength := 10240\n\tcontent := make([]byte, length)\n\tutils.RandRead(content)\n\tactual := generateChecksum(bytes.NewReader(content))\n\n\t// content length equal buff length case\n\tlens := []int64{-1, int64(length)}\n\tfor _, contentLength := range lens {\n\t\treader := verifyChecksum(io.NopCloser(bytes.NewReader(content)), actual, contentLength)\n\t\tn, err := reader.Read(make([]byte, length))\n\t\tif n != length || (err != nil && err != io.EOF) {\n\t\t\tt.Fatalf(\"verify checksum should success\")\n\t\t}\n\t}\n\n\t// verify success case\n\tfor _, contentLength := range lens {\n\t\treader := verifyChecksum(io.NopCloser(bytes.NewReader(content)), actual, contentLength)\n\t\tn, err := reader.Read(make([]byte, length+100))\n\t\tif n != length || (err != nil && err != io.EOF) {\n\t\t\tt.Fatalf(\"verify checksum should success\")\n\t\t}\n\t}\n\n\t// verify failed case\n\tfor _, contentLength := range lens {\n\t\tcontent[0] = 'a'\n\t\treader := verifyChecksum(io.NopCloser(bytes.NewReader(content)), actual, contentLength)\n\t\tn, err := reader.Read(make([]byte, length))\n\t\tif contentLength == -1 && (err != nil && err != io.EOF || n != length) {\n\t\t\tt.Fatalf(\"dont verify checksum when content length is -1\")\n\t\t}\n\t\tif contentLength != -1 && (err == nil || err == io.EOF || !strings.HasPrefix(err.Error(), \"verify checksum failed\")) {\n\t\t\tt.Fatalf(\"verify checksum should failed\")\n\t\t}\n\t}\n\n\t// verify read length less than content length case\n\tfor _, contentLength := range lens {\n\t\treader := verifyChecksum(io.NopCloser(bytes.NewReader(content)), actual, contentLength)\n\t\tn, err := reader.Read(make([]byte, length-100))\n\t\tif err != nil || n != length-100 {\n\t\t\tt.Fatalf(\"error should be nil and read length should be %d\", length-100)\n\t\t}\n\t}\n}\n" }, { "path": "pkg/object/cifs.go", "content": "//go:build !nocifs\n// +build !nocifs\n\n/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cloudsoda/go-smb2\"\n)\n\ntype cifsConn struct {\n\tsession *smb2.Session\n\tshare *smb2.Share\n\tlastUsed time.Time\n}\n\nvar _ ObjectStorage = (*cifsStore)(nil)\nvar _ FileSystem = (*cifsStore)(nil)\n\ntype cifsStore struct {\n\tDefaultObjectStorage\n\thost string\n\tport string\n\tshare string\n\tuser string\n\tpassword string\n\tpool chan *cifsConn\n\tconnIdleTimeout time.Duration\n}\n\n// Chmod changes the mode of the file to mode.\n//\n// Note: SAMBA protocol has limited support for Unix file permissions.\n// it controls the FILE_ATTRIBUTE_READONLY attribute. All other permission bits are ignored.\n//\n// Examples:\n// - chmod(0644), chmod(0666), chmod(0755) -> file becomes writable(666)\n// - chmod(0444), chmod(0400), chmod(0555) -> file becomes read-only(444)\n//\n// The returned mode from Stat() will always be either 0666 (writable) or 0444 (read-only)\n// regardless of the specific mode bits passed to this function.\nfunc (c *cifsStore) Chmod(path string, mode os.FileMode) error {\n\treturn c.withConn(context.Background(), func(share *smb2.Share) error {\n\t\treturn share.Chmod(path, mode)\n\t})\n}\n\n// Chown implements FileSystem.\nfunc (c *cifsStore) Chown(path string, owner string, group string) error {\n\treturn notSupported\n}\n\n// Chtimes implements MtimeChanger.\nfunc (c *cifsStore) Chtimes(path string, mtime time.Time) error {\n\treturn c.withConn(context.Background(), func(share *smb2.Share) error {\n\t\treturn share.Chtimes(path, time.Time{}, mtime)\n\t})\n}\n\nfunc (c *cifsStore) String() string {\n\treturn fmt.Sprintf(\"cifs://%s@%s:%s/%s/\", c.user, c.host, c.port, c.share)\n}\n\n// getConnection returns a CIFS connection from the pool or creates a new one\nfunc (c *cifsStore) getConnection(ctx context.Context) (*cifsConn, error) {\n\tnow := time.Now()\n\tfor {\n\t\tselect {\n\t\tcase conn := <-c.pool:\n\t\t\tif conn.session == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif now.Sub(conn.lastUsed) > c.connIdleTimeout {\n\t\t\t\tc.closeConnectionAsync(conn)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tconn.lastUsed = now\n\t\t\treturn conn, nil\n\t\tcase <-ctx.Done():\n\t\t\treturn nil, ctx.Err()\n\t\tdefault:\n\t\t\tgoto CREATE\n\t\t}\n\t}\n\nCREATE:\n\t// Create new connection\n\t// FIXME: may create a large number of connection in a short period, exceeding the limit.\n\tconn := &cifsConn{}\n\tconn.lastUsed = now\n\n\t// Establish SMB connection\n\taddress := net.JoinHostPort(c.host, c.port)\n\td := &smb2.Dialer{\n\t\tInitiator: &smb2.NTLMInitiator{\n\t\t\tUser: c.user,\n\t\t\tPassword: c.password,\n\t\t},\n\t}\n\n\tvar err error\n\tconn.session, err = d.Dial(ctx, address)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"SMB authentication failed: %v\", err)\n\t}\n\n\tconn.share, err = conn.session.WithContext(ctx).Mount(c.share)\n\tif err != nil {\n\t\tc.closeConnection(conn)\n\t\treturn nil, fmt.Errorf(\"failed to mount SMB share %s: %v\", c.share, err)\n\t}\n\n\treturn conn, nil\n}\n\nfunc (c *cifsStore) closeConnection(conn *cifsConn) {\n\tif conn == nil || conn.session == nil {\n\t\treturn\n\t}\n\n\tsession := conn.session\n\tconn.session = nil\n\tconn.share = nil\n\n\t_ = session.WithContext(context.Background()).Logoff()\n}\n\nfunc (c *cifsStore) closeConnectionAsync(conn *cifsConn) {\n\tgo c.closeConnection(conn)\n}\n\n// releaseConnection returns a connection to the pool or closes it if there's an error\nfunc (c *cifsStore) releaseConnection(conn *cifsConn, err error) {\n\tif conn == nil {\n\t\treturn\n\t}\n\n\tif err == nil {\n\t\tselect {\n\t\tcase c.pool <- conn:\n\t\t\treturn\n\t\tdefault:\n\t\t}\n\t}\n\n\t// close connection if there's an error or if the pool is full\n\tif conn.session != nil {\n\t\t_ = conn.session.Logoff()\n\t}\n}\n\nfunc (c *cifsStore) withConn(ctx context.Context, f func(*smb2.Share) error) (err error) {\n\tconn, err := c.getConnection(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tc.releaseConnection(conn, err)\n\t}()\n\treturn f(conn.share.WithContext(ctx))\n}\n\nfunc (c *cifsStore) Head(ctx context.Context, key string) (oj Object, err error) {\n\terr = c.withConn(ctx, func(share *smb2.Share) error {\n\t\tfi, err := share.Lstat(key)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tisSymlink := fi.Mode()&os.ModeSymlink != 0\n\t\tif isSymlink {\n\t\t\t// SMB doesn't fully support symlinks like POSIX, but we'll try our best\n\t\t\tfi, err = share.Stat(key)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\toj = c.fileInfo(key, fi, isSymlink)\n\t\treturn nil\n\t})\n\treturn oj, err\n}\n\n// cifsReadCloser wraps a file reader and releases the connection when closed\ntype cifsReadCloser struct {\n\tio.ReadCloser\n\tstore *cifsStore\n\tconn *cifsConn\n}\n\nfunc (r *cifsReadCloser) Close() error {\n\terr := r.ReadCloser.Close()\n\tr.store.releaseConnection(r.conn, err)\n\treturn err\n}\n\nfunc (c *cifsStore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tif off < 0 {\n\t\toff = 0\n\t}\n\n\tconn, err := c.getConnection(ctx)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tshare := conn.share.WithContext(ctx)\n\tf, err := share.Open(key)\n\tif err != nil {\n\t\tc.releaseConnection(conn, err)\n\t\treturn nil, err\n\t}\n\n\tfinfo, err := f.Stat()\n\tif err != nil {\n\t\t_ = f.Close()\n\t\tc.releaseConnection(conn, err)\n\t\treturn nil, err\n\t}\n\n\tif finfo.IsDir() || off >= finfo.Size() {\n\t\t_ = f.Close()\n\t\tc.releaseConnection(conn, nil)\n\t\treturn io.NopCloser(bytes.NewBuffer([]byte{})), nil\n\t}\n\n\tvar readCloser io.ReadCloser\n\tif limit > 0 {\n\t\treadCloser = &SectionReaderCloser{\n\t\t\tSectionReader: io.NewSectionReader(f, off, limit),\n\t\t\tCloser: f,\n\t\t}\n\t} else {\n\t\t// When limit <= 0, read from off to end of file\n\t\tif off > 0 {\n\t\t\treadCloser = &SectionReaderCloser{\n\t\t\t\tSectionReader: io.NewSectionReader(f, off, finfo.Size()-off),\n\t\t\t\tCloser: f,\n\t\t\t}\n\t\t} else {\n\t\t\treadCloser = f\n\t\t}\n\t}\n\n\treturn &cifsReadCloser{\n\t\tReadCloser: readCloser,\n\t\tstore: c,\n\t\tconn: conn,\n\t}, nil\n}\n\nfunc (c *cifsStore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) (err error) {\n\treturn c.withConn(ctx, func(share *smb2.Share) error {\n\t\tp := key\n\t\tif strings.HasSuffix(key, dirSuffix) {\n\t\t\t// perm will not take effect, is not used\n\t\t\t// ref: https://github.com/cloudsoda/go-smb2/blob/c8e61c7a5fa7bcd1143359f071f9425a9f4dda3f/client.go#L341-L370\n\t\t\treturn share.MkdirAll(p, 0755)\n\t\t}\n\n\t\tvar tmp string\n\t\tif PutInplace {\n\t\t\ttmp = p\n\t\t} else {\n\t\t\tname := path.Base(p)\n\t\t\tif len(name) > 200 {\n\t\t\t\tname = name[:200]\n\t\t\t}\n\t\t\ttmp = TmpFilePath(p, name)\n\t\t\tdefer func() {\n\t\t\t\tif err != nil {\n\t\t\t\t\t_ = share.Remove(tmp)\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\n\t\tf, err := share.Create(tmp)\n\t\tif err != nil && os.IsNotExist(err) {\n\t\t\tdirPath := path.Dir(p)\n\t\t\tif dirPath != \"/\" {\n\t\t\t\terr = share.MkdirAll(dirPath, 0755)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tf, err = share.Create(tmp)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tbuf := bufPool.Get().(*[]byte)\n\t\tdefer bufPool.Put(buf)\n\t\t_, err = io.CopyBuffer(f, in, *buf)\n\t\tif err != nil {\n\t\t\t_ = f.Close()\n\t\t\treturn err\n\t\t}\n\n\t\terr = f.Close()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif !PutInplace {\n\t\t\terr = share.Rename(tmp, p)\n\t\t}\n\t\treturn err\n\t})\n}\n\nfunc (c *cifsStore) Delete(ctx context.Context, key string, getters ...AttrGetter) (err error) {\n\treturn c.withConn(ctx, func(share *smb2.Share) error {\n\t\tp := strings.TrimRight(key, dirSuffix)\n\t\terr = share.Remove(p)\n\t\tif err != nil && os.IsNotExist(err) {\n\t\t\terr = nil\n\t\t}\n\t\treturn err\n\t})\n}\n\nfunc (c *cifsStore) fileInfo(key string, fi os.FileInfo, isSymlink bool) Object {\n\towner, group := \"nobody\", \"nobody\"\n\tff := &file{\n\t\tobj{key, fi.Size(), fi.ModTime(), fi.IsDir(), \"\"},\n\t\towner,\n\t\tgroup,\n\t\tfi.Mode(),\n\t\tisSymlink,\n\t}\n\tif fi.IsDir() {\n\t\tif key != \"\" && !strings.HasSuffix(key, \"/\") {\n\t\t\tff.key += \"/\"\n\t\t}\n\t}\n\treturn ff\n}\n\nfunc (c *cifsStore) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\n\tdir := prefix\n\tvar objs []Object\n\tif !strings.HasSuffix(dir, \"/\") {\n\t\tdir = path.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t} else if marker == \"\" {\n\t\tobj, err := c.Head(ctx, prefix)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil, false, \"\", nil\n\t\t\t}\n\t\t\treturn nil, false, \"\", err\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\tvar mEntries []*mEntry\n\terr := c.withConn(ctx, func(share *smb2.Share) error {\n\t\t// Ensure directory exists before listing\n\t\t_, err := share.Stat(dir)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Read directory entries\n\t\tentries, err := share.ReadDir(dir)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Process entries\n\t\tmEntries = make([]*mEntry, 0, len(entries))\n\t\tfor _, e := range entries {\n\t\t\tisSymlink := e.Mode()&os.ModeSymlink != 0\n\t\t\tif e.IsDir() {\n\t\t\t\tmEntries = append(mEntries, &mEntry{e, e.Name() + dirSuffix, nil, false})\n\t\t\t} else if isSymlink && followLink {\n\t\t\t\t// SMB doesn't fully support symlinks like POSIX, but we'll try our best\n\t\t\t\tfi, err := share.Stat(path.Join(dir, e.Name()))\n\t\t\t\tif err != nil {\n\t\t\t\t\tmEntries = append(mEntries, &mEntry{e, e.Name(), nil, true})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tname := e.Name()\n\t\t\t\tif fi.IsDir() {\n\t\t\t\t\tname = e.Name() + dirSuffix\n\t\t\t\t}\n\t\t\t\tmEntries = append(mEntries, &mEntry{e, name, fi, false})\n\t\t\t} else {\n\t\t\t\tmEntries = append(mEntries, &mEntry{e, e.Name(), nil, isSymlink})\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif os.IsNotExist(err) || os.IsPermission(err) {\n\t\tlogger.Warnf(\"skip %s: %s\", dir, err)\n\t\treturn nil, false, \"\", nil\n\t}\n\n\t// Sort entries by name\n\tsort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })\n\n\t// Generate object list\n\tfor _, e := range mEntries {\n\t\tp := path.Join(dir, e.Name())\n\t\tif e.IsDir() && !strings.HasSuffix(p, \"/\") {\n\t\t\tp = p + \"/\"\n\t\t}\n\t\tkey := p\n\t\tif !strings.HasPrefix(key, prefix) || (marker != \"\" && key <= marker) {\n\t\t\tcontinue\n\t\t}\n\n\t\tinfo := e.Info()\n\t\tf := c.fileInfo(key, info, e.isSymlink)\n\t\tobjs = append(objs, f)\n\t\tif len(objs) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn generateListResult(objs, limit)\n}\n\nfunc (c *cifsStore) Copy(ctx context.Context, dst, src string) error {\n\tr, err := c.Get(ctx, src, 0, -1)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer r.Close()\n\treturn c.Put(ctx, dst, r)\n}\n\nfunc parseEndpoint(endpoint string) (host, port, share string, err error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = \"cifs://\" + endpoint\n\t}\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn\n\t}\n\tif u.Scheme != \"\" && (u.Scheme != \"cifs\" && u.Scheme != \"smb\") {\n\t\terr = fmt.Errorf(\"invalid scheme %s, should be cifs:// or smb://\", u.Scheme)\n\t\treturn\n\t}\n\n\thost = u.Hostname()\n\tport = u.Port()\n\tif port == \"\" {\n\t\tport = \"445\" // Default SMB port\n\t}\n\tparts := strings.Split(u.Path, \"/\")\n\tif len(parts) < 2 || parts[1] == \"\" {\n\t\terr = fmt.Errorf(\"endpoint should be a valid share name (%s)\", \"\\\\\\\\\\\\\")\n\t\treturn\n\t}\n\tif len(parts) > 2 && parts[2] != \"\" {\n\t\terr = fmt.Errorf(\"endpoint should be a valid share name (%s)\", \"\\\\\\\\\\\\\")\n\t\treturn\n\t}\n\tshare = parts[1]\n\treturn\n}\n\nfunc newCifs(endpoint, username, password, _ string) (ObjectStorage, error) {\n\thost, port, share, err := parseEndpoint(endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif username == \"\" {\n\t\treturn nil, fmt.Errorf(\"CIFS username/ak is required\")\n\t}\n\n\tif password == \"\" {\n\t\treturn nil, fmt.Errorf(\"CIFS password/sk is required\")\n\t}\n\n\tmaxPool := 8\n\tif v := os.Getenv(\"JFS_CIFS_MAX_POOL\"); v != \"\" {\n\t\tif n, err := strconv.Atoi(v); err == nil {\n\t\t\tmaxPool = n\n\t\t}\n\t}\n\n\tstore := &cifsStore{\n\t\thost: host,\n\t\tport: port,\n\t\tshare: share,\n\t\tuser: username,\n\t\tpassword: password,\n\t\tconnIdleTimeout: 5 * time.Minute,\n\t\tpool: make(chan *cifsConn, maxPool),\n\t}\n\n\t// Test connection\n\tconn, err := store.getConnection(context.Background())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstore.releaseConnection(conn, nil)\n\n\treturn store, nil\n}\n\nfunc init() {\n\t// Allow both cifs:// and smb:// schemes\n\tRegister(\"cifs\", newCifs)\n\tRegister(\"smb\", newCifs)\n}\n" }, { "path": "pkg/object/cos.go", "content": "//go:build !nocos\n// +build !nocos\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/tencentyun/cos-go-sdk-v5\"\n)\n\nconst (\n\tcosChecksumKey = \"x-cos-meta-\" + checksumAlgr\n\tcosRequestIDKey = \"X-Cos-Request-Id\"\n\tcosStorageClassHeader = \"X-Cos-Storage-Class\"\n)\n\ntype COS struct {\n\tc *cos.Client\n\tendpoint string\n\tsc string\n}\n\nfunc (c *COS) String() string {\n\treturn fmt.Sprintf(\"cos://%s/\", strings.Split(c.endpoint, \".\")[0])\n}\n\nfunc (c *COS) Create(ctx context.Context) error {\n\t_, err := c.c.Bucket.Put(ctx, nil)\n\tif err != nil && isExists(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (c *COS) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 1 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (c *COS) Head(ctx context.Context, key string) (Object, error) {\n\tresp, err := c.c.Object.Head(ctx, key, nil)\n\tif err != nil {\n\t\tif exist, err := c.c.Object.IsExist(ctx, key); err == nil && !exist {\n\t\t\treturn nil, os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\theader := resp.Header\n\tvar size int64\n\tif val, ok := header[\"Content-Length\"]; ok {\n\t\tif length, err := strconv.ParseInt(val[0], 10, 64); err == nil {\n\t\t\tsize = length\n\t\t}\n\t}\n\tvar mtime time.Time\n\tif val, ok := header[\"Last-Modified\"]; ok {\n\t\tmtime, _ = time.Parse(time.RFC1123, val[0])\n\t}\n\tvar sc string\n\tif val := header.Get(cosStorageClassHeader); val != \"\" {\n\t\tsc = val\n\t} else {\n\t\t// https://cloud.tencent.com/document/product/436/7745\n\t\t// This header is returned only if the object is not STANDARD storage class.\n\t\tsc = \"STANDARD\"\n\t}\n\treturn &obj{key, size, mtime, strings.HasSuffix(key, \"/\"), sc}, nil\n}\n\nfunc (c *COS) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tparams := &cos.ObjectGetOptions{Range: getRange(off, limit)}\n\tresp, err := c.c.Object.Get(ctx, key, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err = checkGetStatus(resp.StatusCode, params.Range != \"\"); err != nil {\n\t\t_ = resp.Body.Close()\n\t\treturn nil, err\n\t}\n\tif off == 0 && limit == -1 {\n\t\tlength, err := strconv.ParseInt(resp.Header.Get(\"Content-Length\"), 10, 64)\n\t\tif err != nil {\n\t\t\tlength = -1\n\t\t\tlogger.Warnf(\"failed to parse content-length %s: %s\", resp.Header.Get(\"Content-Length\"), err)\n\t\t}\n\t\tresp.Body = verifyChecksum(resp.Body, resp.Header.Get(cosChecksumKey), length)\n\t}\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.Header.Get(cosRequestIDKey)).SetStorageClass(resp.Header.Get(cosStorageClassHeader))\n\t}\n\treturn resp.Body, nil\n}\n\nfunc (c *COS) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tvar options cos.ObjectPutOptions\n\tif ins, ok := in.(io.ReadSeeker); ok {\n\t\theader := http.Header(map[string][]string{\n\t\t\tcosChecksumKey: {generateChecksum(ins)},\n\t\t})\n\t\toptions.ObjectPutHeaderOptions = &cos.ObjectPutHeaderOptions{XCosMetaXXX: &header}\n\t}\n\tif c.sc != \"\" {\n\t\tif options.ObjectPutHeaderOptions == nil {\n\t\t\toptions.ObjectPutHeaderOptions = &cos.ObjectPutHeaderOptions{}\n\t\t}\n\t\toptions.ObjectPutHeaderOptions.XCosStorageClass = c.sc\n\t}\n\tresp, err := c.c.Object.Put(ctx, key, in, &options)\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.Header.Get(cosRequestIDKey)).SetStorageClass(c.sc)\n\t}\n\treturn err\n}\n\nfunc (c *COS) Copy(ctx context.Context, dst, src string) error {\n\tvar opt cos.ObjectCopyOptions\n\tif c.sc != \"\" {\n\t\topt.ObjectCopyHeaderOptions = &cos.ObjectCopyHeaderOptions{XCosStorageClass: c.sc}\n\t}\n\tsource := fmt.Sprintf(\"%s/%s\", c.endpoint, src)\n\t_, _, err := c.c.Object.Copy(ctx, dst, source, &opt)\n\treturn err\n}\n\nfunc (c *COS) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tresp, err := c.c.Object.Delete(ctx, key)\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.Header.Get(cosRequestIDKey))\n\t}\n\treturn err\n}\n\nfunc (c *COS) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tparam := cos.BucketGetOptions{\n\t\tPrefix: prefix,\n\t\tMarker: start,\n\t\tMaxKeys: int(limit),\n\t\tDelimiter: delimiter,\n\t\tEncodingType: \"url\",\n\t}\n\tresp, _, err := c.c.Bucket.Get(ctx, ¶m)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(resp.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\to := resp.Contents[i]\n\t\tt, _ := time.Parse(time.RFC3339, o.LastModified)\n\t\tkey, err := cos.DecodeURIComponent(o.Key)\n\t\tif err != nil {\n\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode key %s\", o.Key)\n\t\t}\n\t\tobjs[i] = &obj{key, int64(o.Size), t, strings.HasSuffix(key, \"/\"), o.StorageClass}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range resp.CommonPrefixes {\n\t\t\tkey, err := cos.DecodeURIComponent(p)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode commonPrefixes %s\", p)\n\t\t\t}\n\t\t\tobjs = append(objs, &obj{key, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, resp.IsTruncated, resp.NextMarker, nil\n}\n\nfunc (c *COS) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (c *COS) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tvar options cos.InitiateMultipartUploadOptions\n\tif c.sc != \"\" {\n\t\toptions.ObjectPutHeaderOptions = &cos.ObjectPutHeaderOptions{XCosStorageClass: c.sc}\n\t}\n\tresp, _, err := c.c.Object.InitiateMultipartUpload(ctx, key, &options)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: resp.UploadID, MinPartSize: 5 << 20, MaxCount: 10000}, nil\n}\n\nfunc (c *COS) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\tresp, err := c.c.Object.UploadPart(ctx, key, uploadID, num, bytes.NewReader(body), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: resp.Header.Get(\"Etag\")}, nil\n}\n\nfunc (c *COS) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tresult, _, err := c.c.Object.CopyPart(ctx, key, uploadID, num, c.endpoint+\"/\"+srcKey, &cos.ObjectCopyPartOptions{\n\t\tXCosCopySourceRange: fmt.Sprintf(\"bytes=%d-%d\", off, off+size-1),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: result.ETag}, nil\n}\n\nfunc (c *COS) AbortUpload(ctx context.Context, key string, uploadID string) {\n\t_, _ = c.c.Object.AbortMultipartUpload(ctx, key, uploadID)\n}\n\nfunc (c *COS) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\tvar cosParts []cos.Object\n\tfor i := range parts {\n\t\tcosParts = append(cosParts, cos.Object{ETag: parts[i].ETag, PartNumber: parts[i].Num})\n\t}\n\t_, _, err := c.c.Object.CompleteMultipartUpload(ctx, key, uploadID, &cos.CompleteMultipartUploadOptions{Parts: cosParts})\n\treturn err\n}\n\nfunc (c *COS) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tinput := &cos.ListMultipartUploadsOptions{\n\t\tKeyMarker: marker,\n\t}\n\tresult, _, err := c.c.Bucket.ListMultipartUploads(ctx, input)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tt, _ := time.Parse(time.RFC3339, u.Initiated)\n\t\tparts[i] = &PendingPart{u.Key, u.UploadID, t}\n\t}\n\treturn parts, result.NextKeyMarker, nil\n}\n\nfunc (c *COS) SetStorageClass(sc string) error {\n\tc.sc = sc\n\treturn nil\n}\n\nfunc autoCOSEndpoint(bucketName, accessKey, secretKey, token string) (string, error) {\n\tclient := cos.NewClient(nil, &http.Client{\n\t\tTransport: &cos.AuthorizationTransport{\n\t\t\tSecretID: accessKey,\n\t\t\tSecretKey: secretKey,\n\t\t\tSessionToken: token,\n\t\t},\n\t})\n\tclient.UserAgent = UserAgent\n\ts, _, err := client.Service.Get(ctx)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tfor _, b := range s.Buckets {\n\t\t// fmt.Printf(\"%#v\\n\", b)\n\t\tif b.Name == bucketName {\n\t\t\treturn fmt.Sprintf(\"https://%s.cos.%s.myqcloud.com\", b.Name, b.Region), nil\n\t\t}\n\t}\n\n\treturn \"\", fmt.Errorf(\"bucket %q doesn't exist\", bucketName)\n}\n\nfunc newCOS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\n\tif accessKey == \"\" {\n\t\taccessKey = os.Getenv(\"COS_SECRETID\")\n\t\tsecretKey = os.Getenv(\"COS_SECRETKEY\")\n\t}\n\n\tif len(hostParts) == 1 {\n\t\tif endpoint, err = autoCOSEndpoint(hostParts[0], accessKey, secretKey, token); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Unable to get endpoint of bucket %s: %s\", hostParts[0], err)\n\t\t}\n\t\tif uri, err = url.ParseRequestURI(endpoint); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t\t}\n\t\tlogger.Debugf(\"Use endpoint %q\", endpoint)\n\t}\n\n\tb := &cos.BaseURL{BucketURL: uri}\n\tclient := cos.NewClient(b, &http.Client{\n\t\tTransport: &cos.AuthorizationTransport{\n\t\t\tSecretID: accessKey,\n\t\t\tSecretKey: secretKey,\n\t\t\tSessionToken: token,\n\t\t\tTransport: httpClient.Transport,\n\t\t},\n\t})\n\tclient.UserAgent = UserAgent\n\tdisableChecksum := strings.EqualFold(uri.Query().Get(\"disable-checksum\"), \"true\")\n\tif disableChecksum {\n\t\tlogger.Infof(\"default CRC checksum is disabled\")\n\t}\n\tclient.Conf.EnableCRC = !disableChecksum\n\treturn &COS{c: client, endpoint: uri.Host}, nil\n}\n\nfunc init() {\n\tRegister(\"cos\", newCOS)\n}\n" }, { "path": "pkg/object/dragonfly.go", "content": "//go:build !nodragonfly\n// +build !nodragonfly\n\n/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"mime/multipart\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-http-utils/headers\"\n)\n\nconst (\n\t// AsyncWriteBack writes the object asynchronously to the backend.\n\tAsyncWriteBack = iota\n\n\t// WriteBack writes the object synchronously to the backend.\n\tWriteBack\n\n\t// Ephemeral only writes the object to the dfdaemon.\n\t// It is only provided for creating temporary objects between peers,\n\t// and users are not allowed to use this mode.\n\tEphemeral\n)\n\nconst (\n\t// HeaderDragonflyObjectMetaLastModifiedTime is used for last modified time of object storage.\n\tHeaderDragonflyObjectMetaLastModifiedTime = \"X-Dragonfly-Object-Meta-Last-Modified-Time\"\n\n\t// HeaderDragonflyObjectMetaStorageClass is used for storage class of object storage.\n\tHeaderDragonflyObjectMetaStorageClass = \"X-Dragonfly-Object-Meta-Storage-Class\"\n\n\t// HeaderDragonflyObjectOperation is used for object storage operation.\n\tHeaderDragonflyObjectOperation = \"X-Dragonfly-Object-Operation\"\n)\n\nconst (\n\t// Upper limit of maxGetObjectMetadatas.\n\tMaxGetObjectMetadatasLimit = 1000\n\n\t// DefaultMaxReplicas is the default value of maxReplicas.\n\tDefaultMaxReplicas = 0\n\n\t// Upper limit of maxReplicas.\n\tMaxReplicasLimit = 100\n)\n\nconst (\n\t// CopyOperation is the operation of copying object.\n\tCopyOperation = \"copy\"\n)\n\nconst (\n\t// FilterOSS is the filter of oss url for generating task id.\n\tFilterOSS = \"Expires&Signature\"\n\n\t// FilterS3 is the filter of s3 url for generating task id.\n\tFilterS3 = \"X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature\"\n\n\t// FilterOBS is the filter of obs url for generating task id.\n\tFilterOBS = \"X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Obs-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature\"\n)\n\n// ObjectMetadatas is the object metadata list.\ntype ObjectMetadatas struct {\n\t// CommonPrefixes are similar prefixes in object storage.\n\tCommonPrefixes []string `json:\"CommonPrefixes\"`\n\n\t// Metadatas are object metadata.\n\tMetadatas []*ObjectMetadata `json:\"Metadatas\"`\n}\n\n// ObjectMetadata is the object metadata.\ntype ObjectMetadata struct {\n\t// Key is object key.\n\tKey string\n\n\t// ContentDisposition is Content-Disposition header.\n\tContentDisposition string\n\n\t// ContentEncoding is Content-Encoding header.\n\tContentEncoding string\n\n\t// ContentLanguage is Content-Language header.\n\tContentLanguage string\n\n\t// ContentLength is Content-Length header.\n\tContentLength int64\n\n\t// ContentType is Content-Type header.\n\tContentType string\n\n\t// ETag is ETag header.\n\tETag string\n\n\t// Digest is object digest.\n\tDigest string\n\n\t// LastModifiedTime is last modified time.\n\tLastModifiedTime time.Time\n\n\t// StorageClass is object storage class.\n\tStorageClass string\n}\n\n// ObjectStorageMetadata is the object storage metadata.\ntype ObjectStorageMetadata struct {\n\t// Name is object storage name of type, it can be s3, oss or obs.\n\tName string\n\n\t// Region is storage region.\n\tRegion string\n\n\t// Endpoint is datacenter endpoint.\n\tEndpoint string\n}\n\n// dragonfly is the dragonfly object storage.\ntype dragonfly struct {\n\t// DefaultObjectStorage is the default object storage.\n\tDefaultObjectStorage\n\n\t// Address of the object storage service.\n\tendpoint string\n\n\t// Filter is used to generate a unique Task ID by\n\t// filtering unnecessary query params in the URL,\n\t// it is separated by & character.\n\tfilter string\n\n\t// Mode is the mode in which the backend is written,\n\t// including WriteBack and AsyncWriteBack.\n\tmode int\n\n\t// MaxReplicas is the maximum number of\n\t// replicas of an object cache in seed peers.\n\tmaxReplicas int\n\n\t// ObjectStorage bucket name.\n\tbucket string\n\n\t// http client.\n\tclient *http.Client\n}\n\n// String returns the string representation of the dragonfly.\nfunc (d *dragonfly) String() string {\n\treturn fmt.Sprintf(\"dragonfly://%s/\", d.bucket)\n}\n\n// Create creates the object if it does not exist.\nfunc (d *dragonfly) Create(ctx context.Context) error {\n\tif _, _, _, err := d.List(ctx, \"\", \"\", \"\", \"\", 1, false); err == nil {\n\t\treturn nil\n\t}\n\n\tu, err := url.Parse(d.endpoint)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tu.Path = path.Join(\"buckets\", d.bucket)\n\tquery := u.Query()\n\tu.RawQuery = query.Encode()\n\treq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), nil)\n\tif err != nil && !isExists(err) {\n\t\treturn err\n\t}\n\n\tresp, err := d.client.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn fmt.Errorf(\"bad response status %s\", resp.Status)\n\t}\n\n\treturn nil\n}\n\n// Head returns the object metadata if it exists.\nfunc (d *dragonfly) Head(ctx context.Context, key string) (Object, error) {\n\t// get get object metadata request.\n\tu, err := url.Parse(d.endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tu.Path = path.Join(\"buckets\", d.bucket, \"objects\", key)\n\tif strings.HasSuffix(key, \"/\") {\n\t\tu.Path += \"/\"\n\t}\n\n\treq, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Head object.\n\tresp, err := d.client.Do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\tif resp.StatusCode == http.StatusNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\n\t\treturn nil, err\n\t}\n\n\tcontentLength, err := strconv.ParseInt(resp.Header.Get(headers.ContentLength), 10, 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tlastModifiedTime, err := time.Parse(http.TimeFormat, resp.Header.Get(HeaderDragonflyObjectMetaLastModifiedTime))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &obj{\n\t\tkey,\n\t\tint64(contentLength),\n\t\tlastModifiedTime,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\tresp.Header.Get(HeaderDragonflyObjectMetaStorageClass),\n\t}, nil\n}\n\n// Get returns the object if it exists.\nfunc (d *dragonfly) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tu, err := url.Parse(d.endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tu.Path = path.Join(\"buckets\", d.bucket, \"objects\", key)\n\tif strings.HasSuffix(key, \"/\") {\n\t\tu.Path += \"/\"\n\t}\n\n\tquery := u.Query()\n\tif d.filter != \"\" {\n\t\tquery.Add(\"filter\", d.filter)\n\t}\n\n\tu.RawQuery = query.Encode()\n\treq, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treq.Header.Set(headers.Range, getRange(off, limit))\n\tresp, err := d.client.Do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn nil, fmt.Errorf(\"bad response status %s\", resp.Status)\n\t}\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetStorageClass(resp.Header.Get(HeaderDragonflyObjectMetaStorageClass))\n\n\treturn resp.Body, nil\n}\n\n// Put creates or replaces the object.\nfunc (d *dragonfly) Put(ctx context.Context, key string, data io.Reader, getters ...AttrGetter) error {\n\tbody := &bytes.Buffer{}\n\twriter := multipart.NewWriter(body)\n\n\t// AsyncWriteBack mode is used by default.\n\tif err := writer.WriteField(\"mode\", fmt.Sprint(d.mode)); err != nil {\n\t\treturn err\n\t}\n\n\tif d.filter != \"\" {\n\t\tif err := writer.WriteField(\"filter\", d.filter); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif d.maxReplicas > 0 {\n\t\tif err := writer.WriteField(\"maxReplicas\", fmt.Sprint(d.maxReplicas)); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tpart, err := writer.CreateFormFile(\"file\", path.Base(key))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif _, err := io.Copy(part, data); err != nil {\n\t\treturn err\n\t}\n\n\tif err := writer.Close(); err != nil {\n\t\treturn err\n\t}\n\n\tu, err := url.Parse(d.endpoint)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tu.Path = path.Join(\"buckets\", d.bucket, \"objects\", key)\n\tif strings.HasSuffix(key, \"/\") {\n\t\tu.Path += \"/\"\n\t}\n\n\treq, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body)\n\tif err != nil {\n\t\treturn err\n\t}\n\treq.Header.Add(headers.ContentType, writer.FormDataContentType())\n\n\t// Put object.\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn fmt.Errorf(\"bad response status %s\", resp.Status)\n\t}\n\n\treturn nil\n}\n\n// Copy copies the object if it exists.\nfunc (d *dragonfly) Copy(ctx context.Context, dst, src string) error {\n\tbody := &bytes.Buffer{}\n\twriter := multipart.NewWriter(body)\n\n\tif err := writer.WriteField(\"source_object_key\", src); err != nil {\n\t\treturn err\n\t}\n\n\tif err := writer.Close(); err != nil {\n\t\treturn err\n\t}\n\n\tu, err := url.Parse(d.endpoint)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tu.Path = path.Join(\"buckets\", d.bucket, \"objects\", dst)\n\tquery := u.Query()\n\tu.RawQuery = query.Encode()\n\treq, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(headers.ContentType, writer.FormDataContentType())\n\treq.Header.Add(HeaderDragonflyObjectOperation, fmt.Sprint(CopyOperation))\n\n\t// copy object.\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn fmt.Errorf(\"bad response status %s\", resp.Status)\n\t}\n\n\treturn nil\n}\n\n// Delete deletes the object if it exists.\nfunc (d *dragonfly) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\t// get delete object request.\n\tu, err := url.Parse(d.endpoint)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tu.Path = path.Join(\"buckets\", d.bucket, \"objects\", key)\n\tif strings.HasSuffix(key, \"/\") {\n\t\tu.Path += \"/\"\n\t}\n\n\treq, err := http.NewRequestWithContext(ctx, http.MethodDelete, u.String(), nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Delete object.\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn fmt.Errorf(\"bad response status %s\", resp.Status)\n\t}\n\n\treturn nil\n}\n\n// List lists the objects with the given prefix.\nfunc (d *dragonfly) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > MaxGetObjectMetadatasLimit {\n\t\tlimit = MaxGetObjectMetadatasLimit\n\t}\n\n\tu, err := url.Parse(d.endpoint)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\n\tu.Path = path.Join(\"buckets\", d.bucket, \"metadatas\")\n\tquery := u.Query()\n\tif prefix != \"\" {\n\t\tquery.Set(\"prefix\", prefix)\n\t}\n\n\tif marker != \"\" {\n\t\tquery.Set(\"marker\", marker)\n\t}\n\n\tif delimiter != \"\" {\n\t\tquery.Set(\"delimiter\", delimiter)\n\t}\n\n\tif limit != 0 {\n\t\tquery.Set(\"limit\", fmt.Sprint(limit))\n\t}\n\n\tu.RawQuery = query.Encode()\n\treq, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\n\t// List object.\n\tresp, err := d.client.Do(req)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn nil, false, \"\", fmt.Errorf(\"bad response status %s\", resp.Status)\n\t}\n\n\tvar objectMetadatas ObjectMetadatas\n\tif err := json.NewDecoder(resp.Body).Decode(&objectMetadatas); err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\n\tobjs := make([]Object, 0, len(objectMetadatas.Metadatas))\n\tfor _, meta := range objectMetadatas.Metadatas {\n\t\tobjs = append(objs, &obj{\n\t\t\tmeta.Key,\n\t\t\tmeta.ContentLength,\n\t\t\tmeta.LastModifiedTime,\n\t\t\tstrings.HasSuffix(meta.Key, \"/\"),\n\t\t\tmeta.StorageClass,\n\t\t})\n\t}\n\n\tif delimiter != \"\" {\n\t\tfor _, o := range objectMetadatas.CommonPrefixes {\n\t\t\tobjs = append(objs, &obj{o, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn generateListResult(objs, limit)\n}\n\n// newDragonfly creates a new dragonfly object storage.\nfunc newDragonfly(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"http://%s\", endpoint)\n\t}\n\n\t// Parse the endpoint.\n\turi, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tendpoint = uri.Scheme + \"://\" + uri.Host\n\tbucket := uri.Path\n\tif bucket == \"\" {\n\t\treturn nil, fmt.Errorf(\"bucket name required\")\n\t}\n\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"http://%s\", endpoint)\n\t}\n\n\tmode := WriteBack\n\tif value := uri.Query().Get(\"mode\"); value != \"\" {\n\t\tmode, err = strconv.Atoi(value)\n\t\tif err != nil || (mode != WriteBack && mode != AsyncWriteBack) {\n\t\t\treturn nil, fmt.Errorf(\"unexpected dragonfly mode: %s\", value)\n\t\t}\n\t}\n\n\tmaxReplicas := DefaultMaxReplicas\n\tif value := uri.Query().Get(\"maxReplicas\"); value != \"\" {\n\t\tmaxReplicas, err = strconv.Atoi(value)\n\t\tif err != nil || maxReplicas > MaxReplicasLimit || maxReplicas < 0 {\n\t\t\treturn nil, fmt.Errorf(\"unexpected dragonfly max replicas: %s\", value)\n\t\t}\n\t}\n\n\tmetadata, err := getObjectStorageMetadata(endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar filter string\n\tswitch metadata.Name {\n\tcase \"s3\":\n\t\tfilter = FilterS3\n\tcase \"oss\":\n\t\tfilter = FilterOSS\n\tcase \"obs\":\n\t\tfilter = FilterOBS\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unexpected dragonfly object storage name: %s\", metadata.Name)\n\t}\n\n\treturn &dragonfly{\n\t\tendpoint: endpoint,\n\t\tfilter: filter,\n\t\tmode: mode,\n\t\tmaxReplicas: maxReplicas,\n\t\tbucket: bucket,\n\t\tclient: httpClient,\n\t}, nil\n}\n\n// getObjectStorageMetadata returns the object storage metadata.\nfunc getObjectStorageMetadata(endpoint string) (*ObjectStorageMetadata, error) {\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn nil, nil\n\t}\n\n\tu.Path = path.Join(\"metadata\")\n\treq, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Get object storage Metadata.\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn nil, fmt.Errorf(\"bad response status %s\", resp.Status)\n\t}\n\n\tvar objectStorageMetadata ObjectStorageMetadata\n\tif err := json.NewDecoder(resp.Body).Decode(&objectStorageMetadata); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &objectStorageMetadata, nil\n}\n\n// init registers the dragonfly object storage.\nfunc init() {\n\tRegister(\"dragonfly\", newDragonfly)\n}\n" }, { "path": "pkg/object/encrypt.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"crypto/rand\"\n\t\"crypto/rsa\"\n\t\"crypto/sha256\"\n\t\"crypto/x509\"\n\t\"encoding/pem\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/emmansun/gmsm/pkcs8\"\n\t\"github.com/emmansun/gmsm/sm2\"\n\t\"github.com/emmansun/gmsm/sm4\"\n\t\"golang.org/x/crypto/chacha20poly1305\"\n)\n\ntype Encryptor interface {\n\tEncrypt(plaintext []byte) ([]byte, error)\n\tDecrypt(ciphertext []byte) ([]byte, error)\n}\n\nfunc ExportRsaPrivateKeyToPem(key *rsa.PrivateKey, passphrase string) string {\n\tbuf := x509.MarshalPKCS1PrivateKey(key)\n\tblock := &pem.Block{\n\t\tType: \"RSA PRIVATE KEY\",\n\t\tBytes: buf,\n\t}\n\tif passphrase != \"\" {\n\t\tvar err error\n\t\t// nolint:staticcheck\n\t\tblock, _ = x509.EncryptPEMBlock(rand.Reader, block.Type, buf, []byte(passphrase), x509.PEMCipherAES256)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\tprivPEM := pem.EncodeToMemory(block)\n\treturn string(privPEM)\n}\n\nvar ErrKeyNeedPasswd = errors.New(\"passphrase is required to private key\")\n\nfunc ParsePrivateKeyFromPem(enc []byte, passphrase []byte) (any, error) {\n\tblock, _ := pem.Decode(enc)\n\tif block == nil {\n\t\treturn nil, errors.New(\"failed to parse PEM block containing the key\")\n\t}\n\n\tbuf := block.Bytes\n\tif len(passphrase) == 0 {\n\t\t// nolint:staticcheck\n\t\tif strings.Contains(block.Headers[\"Proc-Type\"], \"ENCRYPTED\") && x509.IsEncryptedPEMBlock(block) {\n\t\t\treturn nil, ErrKeyNeedPasswd\n\t\t}\n\t\tif strings.Contains(block.Type, \"ENCRYPTED\") {\n\t\t\treturn nil, ErrKeyNeedPasswd\n\t\t}\n\t} else {\n\t\tvar err error\n\t\t// nolint:staticcheck\n\t\tbuf, err = x509.DecryptPEMBlock(block, passphrase)\n\t\tif err != nil {\n\t\t\tif err == x509.IncorrectPasswordError {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tkey, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, passphrase)\n\t\t\tif err == nil {\n\t\t\t\treturn key, nil\n\t\t\t}\n\t\t\tkey, err = pkcs8.ParsePKCS8PrivateKey(block.Bytes)\n\t\t\tif err == nil {\n\t\t\t\treturn key, nil\n\t\t\t}\n\t\t\tif !strings.Contains(err.Error(), \"ParsePKCS1PrivateKey\") {\n\t\t\t\treturn nil, fmt.Errorf(\"cannot decode encrypted private keys: %v\", err)\n\t\t\t}\n\t\t\tbuf = block.Bytes\n\t\t}\n\t}\n\n\trsaKey, err := x509.ParsePKCS1PrivateKey(buf)\n\tif err == nil {\n\t\treturn rsaKey, nil\n\t}\n\tkey, err := pkcs8.ParsePKCS8PrivateKey(buf)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn key, nil\n}\n\nfunc ParseRsaPrivateKeyFromPath(path, passphrase string) (any, error) {\n\tb, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn ParsePrivateKeyFromPem(b, []byte(passphrase))\n}\n\ntype rsaEncryptor struct {\n\tprivKey *rsa.PrivateKey\n\tlabel []byte\n}\n\nfunc NewRSAEncryptor(privKey *rsa.PrivateKey) Encryptor {\n\treturn &rsaEncryptor{privKey, []byte(\"keys\")}\n}\n\nfunc (e *rsaEncryptor) Encrypt(plaintext []byte) ([]byte, error) {\n\treturn rsa.EncryptOAEP(sha256.New(), rand.Reader, &e.privKey.PublicKey, plaintext, e.label)\n}\n\nfunc (e *rsaEncryptor) Decrypt(ciphertext []byte) ([]byte, error) {\n\treturn rsa.DecryptOAEP(sha256.New(), rand.Reader, e.privKey, ciphertext, e.label)\n}\n\ntype sm2Encryptor struct {\n\tprivKey *sm2.PrivateKey\n}\n\nfunc NewSM2Encryptor(privKey *sm2.PrivateKey) Encryptor {\n\treturn &sm2Encryptor{privKey}\n}\n\nfunc (e *sm2Encryptor) Encrypt(plaintext []byte) ([]byte, error) {\n\treturn sm2.EncryptASN1(rand.Reader, &e.privKey.PublicKey, plaintext)\n}\n\nfunc (e *sm2Encryptor) Decrypt(ciphertext []byte) ([]byte, error) {\n\treturn sm2.Decrypt(e.privKey, ciphertext)\n}\n\nfunc NewKeyEncryptor(privKey any) Encryptor {\n\tswitch k := privKey.(type) {\n\tcase *rsa.PrivateKey:\n\t\treturn NewRSAEncryptor(k)\n\tcase *sm2.PrivateKey:\n\t\treturn NewSM2Encryptor(k)\n\t}\n\tpanic(fmt.Sprintf(\"unsupported key type %T\", privKey)) // should not happen\n}\n\ntype dataEncryptor struct {\n\tkeyEncryptor Encryptor\n\tkeyLen int\n\taead func(key []byte) (cipher.AEAD, error)\n}\n\nconst (\n\tAES256GCM_RSA = \"aes256gcm-rsa\"\n\tCHACHA20_RSA = \"chacha20-rsa\"\n\tSM4GCM = \"sm4gcm\"\n)\n\nfunc NewDataEncryptor(keyEncryptor Encryptor, algo string) (Encryptor, error) {\n\tswitch algo {\n\tcase \"\", AES256GCM_RSA:\n\t\taead := func(key []byte) (cipher.AEAD, error) {\n\t\t\tblock, err := aes.NewCipher(key)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn cipher.NewGCM(block)\n\t\t}\n\t\treturn &dataEncryptor{keyEncryptor, 32, aead}, nil\n\tcase CHACHA20_RSA:\n\t\treturn &dataEncryptor{keyEncryptor, chacha20poly1305.KeySize, chacha20poly1305.New}, nil\n\tcase SM4GCM:\n\t\t// TODO: support other modes?\n\t\t// GCM not in [GB/T 17964-2021](http://c.gb688.cn/bzgk/gb/showGb?type=online&hcno=4F89D833626340B1F71068D25EAC737D)\n\t\taead := func(key []byte) (cipher.AEAD, error) {\n\t\t\tblock, err := sm4.NewCipher(key)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn cipher.NewGCM(block)\n\t\t}\n\t\treturn &dataEncryptor{keyEncryptor, 16, aead}, nil\n\t}\n\treturn nil, fmt.Errorf(\"unsupport cipher: %s\", algo)\n}\n\nfunc (e *dataEncryptor) Encrypt(plaintext []byte) ([]byte, error) {\n\tkey := make([]byte, e.keyLen)\n\tif _, err := io.ReadFull(rand.Reader, key); err != nil {\n\t\treturn nil, err\n\t}\n\tcipherkey, err := e.keyEncryptor.Encrypt(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\taead, err := e.aead(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnonce := make([]byte, aead.NonceSize())\n\tif _, err := io.ReadFull(rand.Reader, nonce); err != nil {\n\t\treturn nil, err\n\t}\n\n\theaderSize := 3 + len(cipherkey) + len(nonce)\n\tbuf := make([]byte, headerSize+len(plaintext)+aead.Overhead())\n\tbuf[0] = byte(len(cipherkey) >> 8)\n\tbuf[1] = byte(len(cipherkey) & 0xFF)\n\tbuf[2] = byte(len(nonce))\n\tp := buf[3:]\n\tcopy(p, cipherkey)\n\tp = p[len(cipherkey):]\n\tcopy(p, nonce)\n\tp = p[len(nonce):]\n\tciphertext := aead.Seal(p[:0], nonce, plaintext, nil)\n\treturn buf[:headerSize+len(ciphertext)], nil\n}\n\nfunc (e *dataEncryptor) Decrypt(ciphertext []byte) ([]byte, error) {\n\tif len(ciphertext) < 3 {\n\t\treturn nil, fmt.Errorf(\"received encrypted text length is less than 3, the object is corrupted\")\n\t}\n\tkeyLen := int(ciphertext[0])<<8 + int(ciphertext[1])\n\tnonceLen := int(ciphertext[2])\n\tif 3+keyLen+nonceLen >= len(ciphertext) {\n\t\treturn nil, fmt.Errorf(\"malformed ciphertext: %d %d\", keyLen, nonceLen)\n\t}\n\tciphertext = ciphertext[3:]\n\tcipherkey := ciphertext[:keyLen]\n\tnonce := ciphertext[keyLen : keyLen+nonceLen]\n\tciphertext = ciphertext[keyLen+nonceLen:]\n\n\tkey, err := e.keyEncryptor.Decrypt(cipherkey)\n\tif err != nil {\n\t\treturn nil, errors.New(\"decryt key: \" + err.Error())\n\t}\n\taead, err := e.aead(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn aead.Open(ciphertext[:0], nonce, ciphertext, nil)\n}\n\ntype encrypted struct {\n\tObjectStorage\n\tenc Encryptor\n}\n\n// NewEncrypted returns a encrypted object storage\nfunc NewEncrypted(o ObjectStorage, enc Encryptor) ObjectStorage {\n\treturn &encrypted{o, enc}\n}\n\nfunc (e *encrypted) String() string {\n\treturn fmt.Sprintf(\"%s(encrypted)\", e.ObjectStorage)\n}\n\nfunc (e *encrypted) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tr, err := e.ObjectStorage.Get(ctx, key, 0, -1, getters...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer r.Close()\n\tciphertext, err := io.ReadAll(r)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tplain, err := e.enc.Decrypt(ciphertext)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Decrypt: %s\", err)\n\t}\n\tl := int64(len(plain))\n\tif off > l {\n\t\toff = l\n\t}\n\tif limit == -1 || off+limit > l {\n\t\tlimit = l - off\n\t}\n\tdata := plain[off : off+limit]\n\treturn io.NopCloser(bytes.NewBuffer(data)), nil\n}\n\nfunc (e *encrypted) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tplain, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\tciphertext, err := e.enc.Encrypt(plain)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn e.ObjectStorage.Put(ctx, key, bytes.NewReader(ciphertext), getters...)\n}\n\nvar _ ObjectStorage = (*encrypted)(nil)\n" }, { "path": "pkg/object/encrypt_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/rand\"\n\t\"crypto/rsa\"\n\t\"crypto/x509\"\n\t\"encoding/pem\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/emmansun/gmsm/sm2\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nvar rsaInPKCS8 = `-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIICzzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIEEEvSFbVLkUCAggA\nMB0GCWCGSAFlAwQBKgQQhuaBA77wcAHA7bl4dkbFsQSCAoBi5hgqWhK2ic3HBSUX\nJBFFh7omdhU4uK7mQzVVx/RvnUCbw5T7ghfboJhP5bHkj+UnnFiKD6vFZfSgH/Q3\n5KUjPIveLa0urly1bC1SMequXggjEgSPUe8XBjWJJcwkbELFiQzD76GSnveCMokZ\nX7WvoZeY0AaSAnQwe5r1evAdilWXdb2fUmRA23gco8pgSrkdVPyz9lb+FbDjrd8j\n7qiMDcoZ4qFrQ4v8IQJv+ED5f7fLen7UGpG5uOZT9Ez153f7Zw+eEAmp5qwE5SCP\nJbVLsR++HXkntJg1q2Yw4rIOi7qing408jwroed/W6AzS8A49RvrI2/Ac5dHfEnB\nLkC23Ep47/e9B8cZQCmIZXEnUpcjSwWKe5U4nCXyeskuhRhTtA3EpYFXx+/P3yNE\nYISywz6brtAxDwfk8LNAGkZRQ5c+nIFh43M+m5LLHAOSug/TbIvVvgottdc0VRHl\nQ72zeXu8X7PF8dhnoxVSBdKfRYCHQWg+PBw8IYn1KA1SfvwakeVnYcU8P4BMOXo5\n36Q4CVDIW9zWCrW49Cq/dxi0yqYyoA5hw8lIqMzmewdiUH0BwlsaOBz0utz0GhOi\nmBsK7O5819orKnuGmWzuvEETiRJ+HZTgkWAC0Wu1r7gjbMKow8grkygQz0iqMrSE\nkY7gfcnT2mpR7ow0DbWqiidb4PrxYsk0X1hOswsAek62xL/sdqlA9C3eZuqPNfqa\nyatWjKjQY4ukKUm7QplPOgOuP01GN0XF7zMEqXtl1GxPp9uDnKFzDopQau+3OrID\nljSQG+zYqxPFeLZ06zh3bYqS5E5RjlguF6055m5NaudQ9b+/7NjPDHdpWth6cQFx\nOIGw\n-----END ENCRYPTED PRIVATE KEY-----\n`\n\nvar rsaInPKCS1 = `-----BEGIN RSA PRIVATE KEY-----\nMIICXAIBAAKBgQCaPxhJMEfX0CaYIziQxwjlVlh75xw1xWlF14GGdpZYaM70BzMu\nXdB22X7PnkK38PHk4saXKz0blhaf/qllJP7mcdqFEcs4sWsVhU1KoLdRNH/1AJQK\n0/Oehr3vov1CjsR+51RRuDFcVOBd7lpglK5s0+TGRYyImFc+JhZ23RVFNwIDAQAB\nAoGAGtobDzqxdxeMcHXJNiMAIHScqM098vpv7jGrIc5pM/Di/kZ2mX7JeLc6RUiG\n0uDGK5NzAQQM+k1xmN7LfIkpOo2pSlL0gC/M6q0TAJqRLXBKjMVqlHLUytqKTtEg\n4PeF93GnxJZt9NSqo5HH87OwkjXeG1brqhZTfKtL/tbRpRECQQDLJAIFri3pGzni\nXq4s2NogxUnC8cg9I7jEv4gTH/KuFQTsh/5i2+1tsGyFdXKzFd0A8DcZx+MzBm7q\nqwF46vw5AkEAwmIN0s9EcUVeVyorgdphl81QV+x9TR5wZbksigPQcNqU2NJVZKtd\n1f0o2H3E2XHV2DLjeLWctlmx+i0k3Sos7wJAJR/Sgsk/OK+yF22oNSf4TS7g+RCI\nwKurk8FRE/WtuyS6PqPn2JdKv9YTLxy0tofTWN2NpFeEbQnK8XYJEdkX+QJAR/GC\nrEOKUWIbSKeS8ryg4k5bLi+ZMLHTZ9LhaTOAMkS0UouGj3vdfxXzyCzEbrZzL1Gm\nX0bYeaU4+h87RaAWgQJBALhNFDDGXnEd0Lj2pUhBcdaRXGqrg8PZWekr0GLDPEvO\ns+yhHoqRlGKUwQtwwB3HCIEWxe7siOa0YTy9MJ5QySY=\n-----END RSA PRIVATE KEY-----`\n\nvar sm2InPKCS8Plain = `-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqBHM9VAYItBG0wawIBAQQgUWhfo4lpH0j/Toc7\nESiTd+1FsWJgIR9MlBVeQ0lYi62hRANCAASAuZzZAg6zj+ZXclqBx0UfZVNeN9+R\nL5MzLV1dmrLZQqbt+j8oDAN3QU3VPXziKzGttdTvgItUrLavxaCMXOL+\n-----END PRIVATE KEY-----`\n\nvar sm2InPKCS8WithSM4 = `-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIHzMF4GCSqGSIb3DQEFDTBRMDEGCSqGSIb3DQEFDDAkBBAm2QGHlzGOPNqAyOCZ\nzWOrAgIIADAMBggqhkiG9w0CCQUAMBwGCCqBHM9VAWgCBBC0yGrxfFu2t51rF8RX\nN/4+BIGQkIa24K2nOv+fkmohJHaya9b+LJUs6VR50K+2n3QuJokRvxlGB9TknxDs\ne3ZJfNKRoksL7V4Ttd82pgF6a68jBB0//iOSysc6d/ovx5oKrJ8kx+t/U5NbxWRV\n8UrHPN50rzxS4l6niklnwUM2q36Lf6R+xYduTVmTfWDAAPFSRIlKUDmhgPlT8MHB\njxqPfZVO\n-----END ENCRYPTED PRIVATE KEY-----`\n\nvar sm2InPKCS8WithAES = `-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIHzMF4GCSqGSIb3DQEFDTBRMDAGCSqGSIb3DQEFDDAjBBA19eEcvLDwQqrQx0Yo\n4vKAAgFkMAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBCniW2M8JL78D06Hqxk\nhQtcBIGQd7zfctW4ry2MqfNpnsx5L2kT6Sv11ecehBJt8e9C/d33YLjBuAA9GTLO\nAoz7Z9lb9ivf/TZL0EXBI7llNQitxV+NEx32jCpwO3rEoFUqoGZZh2jcRmLsufS2\npwq8iHhypwUx6EDLJXTXOFlMsqgHYC1ZV9LqnmdLAKyqXQeHtGN9QZgDQwy221yi\nxI3CLucj\n-----END ENCRYPTED PRIVATE KEY-----`\n\nfunc TestParsePrivateKey(t *testing.T) {\n\tvar cases = []struct {\n\t\tname string\n\t\tkey string\n\t\tpass []byte\n\t\texpect bool\n\t}{\n\t\t{\"rsa key in pkcs#1, parse without passphrase\", rsaInPKCS1, nil, true},\n\t\t{\"rsa key in pkcs#1, parse with passphrase\", rsaInPKCS1, []byte(\"123\"), true},\n\t\t{\"rsa key in pkcs#8, parse with correct passphrase\", rsaInPKCS8, []byte(\"12345678\"), true},\n\t\t{\"rsa key in pkcs#8, parse with incorrect passphrase\", rsaInPKCS8, []byte(\"1234567\"), false},\n\t\t{\"rsa key in pkcs#8, parse without passphrase\", rsaInPKCS8, nil, false},\n\t\t{\"sm2 key in pkcs#8 plain, parse without passphrase\", sm2InPKCS8Plain, nil, true},\n\t\t{\"sm2 key in pkcs#8 plain, parse with passphrase\", sm2InPKCS8Plain, []byte(\"any\"), true},\n\t\t{\"sm2 key in pkcs#8 with sm4, parse with correct passphrase\", sm2InPKCS8WithSM4, []byte(\"12345678\"), true},\n\t\t{\"sm2 key in pkcs#8 with sm4, parse with incorrect passphrase\", sm2InPKCS8WithSM4, []byte(\"1234567\"), false},\n\t\t{\"sm2 key in pkcs#8 with sm4, parse without passphrase\", sm2InPKCS8WithSM4, nil, false},\n\t\t{\"sm2 key in pkcs#8 with aes, parse with correct passphrase\", sm2InPKCS8WithAES, []byte(\"12345678\"), true},\n\t\t{\"sm2 key in pkcs#8 with aes, parse with incorrect passphrase\", sm2InPKCS8WithAES, []byte(\"1234567\"), false},\n\t\t{\"sm2 key in pkcs#8 with aes, parse without passphrase\", sm2InPKCS8WithAES, nil, false},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\t_, err := ParsePrivateKeyFromPem([]byte(c.key), c.pass)\n\t\t\trequire.Equal(t, c.expect, err == nil, \"unexpected result: %v\", err)\n\t\t})\n\t}\n}\n\nfunc genPrivateKey(typ string) any {\n\tswitch typ {\n\tcase \"rsa\":\n\t\tkey, err := rsa.GenerateKey(rand.Reader, 2048)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\treturn key\n\tcase \"sm2\":\n\t\tkey, err := sm2.GenerateKey(rand.Reader)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\treturn key\n\tdefault:\n\t\tpanic(fmt.Errorf(\"unknown key type: %s\", typ))\n\t}\n}\n\nvar rsaKey = genPrivateKey(\"rsa\").(*rsa.PrivateKey)\n\nfunc TestSM2(t *testing.T) {\n\tsm2Key := genPrivateKey(\"sm2\").(*sm2.PrivateKey)\n\tsm2 := NewSM2Encryptor(sm2Key)\n\tcipherText, err := sm2.Encrypt([]byte(\"hello\"))\n\trequire.NoError(t, err)\n\tplainText, err := sm2.Decrypt(cipherText)\n\trequire.NoError(t, err)\n\trequire.Equal(t, []byte(\"hello\"), plainText)\n}\n\nfunc TestRSA(t *testing.T) {\n\tc1 := NewRSAEncryptor(rsaKey)\n\tciphertext, _ := c1.Encrypt([]byte(\"hello\"))\n\n\tprivPEM := ExportRsaPrivateKeyToPem(rsaKey, \"abc\")\n\n\tkey2, _ := ParsePrivateKeyFromPem([]byte(privPEM), []byte(\"abc\"))\n\tc2 := NewKeyEncryptor(key2)\n\tplaintext, _ := c2.Decrypt(ciphertext)\n\tif string(plaintext) != \"hello\" {\n\t\tt.Fail()\n\t}\n\n\t_, err := ParsePrivateKeyFromPem([]byte(privPEM), nil)\n\tif err == nil {\n\t\tt.Errorf(\"parse without passphrase should fail\")\n\t\tt.Fail()\n\t}\n\t_, err = ParsePrivateKeyFromPem([]byte(privPEM), []byte(\"ab\"))\n\tif err == nil {\n\t\tt.Errorf(\"parse with incorrect passphrase should return fail\")\n\t\tt.Fail()\n\t}\n\n\tdir := t.TempDir()\n\n\tif err := genrsa(filepath.Join(dir, \"private.pem\"), \"\"); err != nil {\n\t\tt.Error(err)\n\t\tt.Fail()\n\t}\n\tif _, err = ParseRsaPrivateKeyFromPath(filepath.Join(dir, \"private.pem\"), \"\"); err != nil {\n\t\tt.Error(err)\n\t\tt.Fail()\n\t}\n\n\tif err := genrsa(filepath.Join(dir, \"private.pem\"), \"abcd\"); err != nil {\n\t\tt.Error(err)\n\t\tt.Fail()\n\t}\n\tif _, err = ParseRsaPrivateKeyFromPath(filepath.Join(dir, \"private.pem\"), \"abcd\"); err != nil {\n\t\tt.Error(err)\n\t\tt.Fail()\n\t}\n}\n\nfunc genrsa(path string, password string) error {\n\tkey, err := rsa.GenerateKey(rand.Reader, 2048)\n\tif err != nil {\n\t\treturn err\n\t}\n\tblock := &pem.Block{\n\t\tType: \"RSA PRIVATE KEY\",\n\t\tBytes: x509.MarshalPKCS1PrivateKey(key),\n\t}\n\tif password != \"\" {\n\t\t// nolint:staticcheck\n\t\tblock, err = x509.EncryptPEMBlock(rand.Reader, block.Type, block.Bytes, []byte(password), x509.PEMCipherAES256)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err := os.WriteFile(path, pem.EncodeToMemory(block), 0755); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc BenchmarkKeyEncryptionKey(b *testing.B) {\n\tsecret := make([]byte, 32)\n\tkeyTypes := []string{\"rsa\", \"sm2\"}\n\n\tfor _, typ := range keyTypes {\n\t\tke := NewKeyEncryptor(genPrivateKey(typ))\n\t\tcipherText, _ := ke.Encrypt(secret)\n\t\tb.ResetTimer()\n\t\tb.Run(typ+\"_encrypt\", func(b *testing.B) {\n\t\t\tfor n := 0; n < b.N; n++ {\n\t\t\t\t_, _ = ke.Encrypt(secret)\n\t\t\t}\n\t\t})\n\t\tb.Run(typ+\"_decrypt\", func(b *testing.B) {\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t_, _ = ke.Decrypt(cipherText)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestDataEncryptor(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tkek string\n\t\talgo string\n\t}{\n\t\t{\"rsa_aesgcm\", \"rsa\", AES256GCM_RSA},\n\t\t{\"rsa_chacha20\", \"rsa\", CHACHA20_RSA},\n\t\t{\"sm2_sm4gcm\", \"sm2\", SM4GCM},\n\t}\n\tdata := []byte(\"hello\")\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\tke := NewKeyEncryptor(genPrivateKey(c.kek))\n\t\t\tde, err := NewDataEncryptor(ke, c.algo)\n\t\t\trequire.NoError(t, err, \"failed to create data encryptor\")\n\t\t\tcipherText, err := de.Encrypt(data)\n\t\t\trequire.NoError(t, err, \"failed to encrypt data\")\n\t\t\tplainText, err := de.Decrypt(cipherText)\n\t\t\trequire.NoError(t, err, \"failed to decrypt data\")\n\t\t\trequire.Equal(t, data, plainText, \"decrypted data not equal to original\")\n\t\t})\n\t}\n}\n\nfunc BenchmarkDataEncryptor(b *testing.B) {\n\tcases := []struct {\n\t\tname string\n\t\tkek string\n\t\talgo string\n\t}{\n\t\t{\"rsa_aesgcm\", \"rsa\", AES256GCM_RSA},\n\t\t{\"rsa_chacha20\", \"rsa\", CHACHA20_RSA},\n\t\t{\"sm2_sm4gcm\", \"sm2\", SM4GCM},\n\t}\n\tdata := make([]byte, 4<<20)\n\tif _, err := rand.Read(data); err != nil {\n\t\tb.Fatalf(\"failed to generate random data: %v\", err)\n\t}\n\tfor _, c := range cases {\n\t\tke := NewKeyEncryptor(genPrivateKey(c.kek))\n\t\tde, err := NewDataEncryptor(ke, c.algo)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"failed to create data encryptor: %v\", err)\n\t\t}\n\t\tcipherText, err := de.Encrypt(data)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"failed to encrypt data: %v\", err)\n\t\t}\n\t\tb.Run(c.name+\"_encrypt\", func(b *testing.B) {\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t_, _ = de.Encrypt(data)\n\t\t\t}\n\t\t})\n\t\tb.Run(c.name+\"_decrypt\", func(b *testing.B) {\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t_, _ = de.Decrypt(cipherText)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestEncryptedStore(t *testing.T) {\n\tctx := context.Background()\n\ts, _ := CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tkc := NewRSAEncryptor(rsaKey)\n\tdc, _ := NewDataEncryptor(kc, AES256GCM_RSA)\n\tes := NewEncrypted(s, dc)\n\t_ = es.Put(ctx, \"a\", bytes.NewReader([]byte(\"hello\")))\n\tr, err := es.Get(ctx, \"a\", 1, 2)\n\tif err != nil {\n\t\tt.Errorf(\"Get a: %s\", err)\n\t\tt.Fail()\n\t}\n\td, _ := io.ReadAll(r)\n\tif string(d) != \"el\" {\n\t\tt.Fail()\n\t}\n\n\tr, _ = es.Get(ctx, \"a\", 0, -1)\n\td, _ = io.ReadAll(r)\n\tif string(d) != \"hello\" {\n\t\tt.Fail()\n\t}\n\t_ = s.Put(ctx, \"emptyfile\", bytes.NewReader([]byte(\"\")))\n\t_, err = es.Get(ctx, \"emptyfile\", 0, -1)\n\tif err == nil || !strings.Contains(err.Error(), \"the object is corrupted\") {\n\t\tt.Fail()\n\t}\n}\n" }, { "path": "pkg/object/eos.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n)\n\ntype eos struct {\n\ts3client\n}\n\nfunc (s *eos) String() string {\n\treturn fmt.Sprintf(\"eos://%s/\", s.s3client.bucket)\n}\n\nfunc (s *eos) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 4 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc newEos(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\thostParts := strings.Split(uri.Host, \".\")\n\tbucket := hostParts[0]\n\tendpoint = uri.Scheme + \"://\" + uri.Host[len(bucket)+1:]\n\tregion := \"us-east-1\"\n\n\tif accessKey == \"\" {\n\t\taccessKey = os.Getenv(\"EOS_ACCESS_KEY\")\n\t}\n\tif secretKey == \"\" {\n\t\tsecretKey = os.Getenv(\"EOS_SECRET_KEY\")\n\t}\n\tif token == \"\" {\n\t\ttoken = os.Getenv(\"EOS_TOKEN\")\n\t}\n\tcfg, err := config.LoadDefaultConfig(ctx,\n\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to load config: %s\", err)\n\t}\n\tclient := s3.NewFromConfig(cfg, func(options *s3.Options) {\n\t\toptions.BaseEndpoint = aws.String(endpoint)\n\t\toptions.Region = region\n\t\toptions.EndpointOptions.DisableHTTPS = !ssl\n\t\toptions.UsePathStyle = defaultPathStyle()\n\t\toptions.HTTPClient = httpClient\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\n\treturn &eos{s3client{bucket: bucket, s3: client, region: region}}, nil\n}\n\nfunc init() {\n\tRegister(\"eos\", newEos)\n}\n" }, { "path": "pkg/object/etcd.go", "content": "//go:build !noetcd\n// +build !noetcd\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\tetcd \"go.etcd.io/etcd/client/v3\"\n\t\"go.etcd.io/etcd/pkg/transport\"\n)\n\ntype etcdClient struct {\n\tDefaultObjectStorage\n\tclient *etcd.Client\n\tkv etcd.KV\n\taddr string\n}\n\nfunc (c *etcdClient) String() string {\n\treturn fmt.Sprintf(\"etcd://%s/\", c.addr)\n}\n\nfunc (c *etcdClient) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tresp, err := c.kv.Get(ctx, key, etcd.WithLimit(1))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, pair := range resp.Kvs {\n\t\tif string(pair.Key) == key {\n\t\t\tif off > int64(len(pair.Value)) {\n\t\t\t\toff = int64(len(pair.Value))\n\t\t\t}\n\t\t\tdata := pair.Value[off:]\n\t\t\tif limit > 0 && limit < int64(len(data)) {\n\t\t\t\tdata = data[:limit]\n\t\t\t}\n\t\t\treturn io.NopCloser(bytes.NewBuffer(data)), nil\n\t\t}\n\t}\n\treturn nil, os.ErrNotExist\n}\n\nfunc (c *etcdClient) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\td, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = c.kv.Put(ctx, key, string(d))\n\treturn err\n}\n\nfunc (c *etcdClient) Head(ctx context.Context, key string) (Object, error) {\n\tresp, err := c.kv.Get(ctx, key, etcd.WithLimit(1))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, p := range resp.Kvs {\n\t\tif string(p.Key) == key {\n\t\t\treturn &obj{\n\t\t\t\tkey,\n\t\t\t\tint64(len(p.Value)),\n\t\t\t\ttime.Now(),\n\t\t\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\t\t\"\",\n\t\t\t}, nil\n\t\t}\n\t}\n\treturn nil, os.ErrNotExist\n}\n\nfunc (c *etcdClient) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\t_, err := c.kv.Delete(ctx, key)\n\treturn err\n}\n\nfunc genNextKey(key string) string {\n\tnext := make([]byte, len(key))\n\tcopy(next, key)\n\tp := len(next) - 1\n\tnext[p]++\n\tfor next[p] == 0 {\n\t\tp--\n\t\tnext[p]++\n\t}\n\treturn string(next)\n}\n\nfunc (c *etcdClient) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tif start == \"\" {\n\t\tstart = prefix\n\t}\n\tvar opts = []etcd.OpOption{etcd.WithLimit(limit), etcd.WithSort(etcd.SortByKey, etcd.SortAscend)}\n\tif len(prefix) > 0 && prefix[0] != 0xFF {\n\t\topts = append(opts, etcd.WithRange(genNextKey(prefix)))\n\t} else {\n\t\topts = append(opts, etcd.WithFromKey())\n\t}\n\tresp, err := c.client.Get(ctx, start, opts...)\n\tif err != nil {\n\t\treturn nil, false, \"\", fmt.Errorf(\"get start %v: %s\", start, err)\n\t}\n\tvar objs []Object\n\tfor _, kv := range resp.Kvs {\n\t\tk := string(kv.Key)\n\t\tif !strings.HasPrefix(k, prefix) {\n\t\t\tbreak\n\t\t}\n\t\tobjs = append(objs, &obj{\n\t\t\tk,\n\t\t\tint64(len(kv.Value)),\n\t\t\ttime.Now(),\n\t\t\tstrings.HasSuffix(k, \"/\"),\n\t\t\t\"\",\n\t\t})\n\t}\n\tvar nextMarker string\n\tif resp.More && len(objs) > 0 {\n\t\tnextMarker = objs[len(objs)-1].Key()\n\t}\n\treturn objs, resp.More, nextMarker, nil\n}\n\nfunc buildTlsConfig(u *url.URL) (*tls.Config, error) {\n\tvar tsinfo transport.TLSInfo\n\tq := u.Query()\n\ttsinfo.CAFile = q.Get(\"cacert\")\n\ttsinfo.CertFile = q.Get(\"cert\")\n\ttsinfo.KeyFile = q.Get(\"key\")\n\ttsinfo.ServerName = q.Get(\"server-name\")\n\ttsinfo.InsecureSkipVerify = q.Get(\"insecure-skip-verify\") != \"\"\n\tif tsinfo.CAFile != \"\" || tsinfo.CertFile != \"\" || tsinfo.KeyFile != \"\" || tsinfo.ServerName != \"\" {\n\t\treturn tsinfo.ClientConfig()\n\t}\n\treturn nil, nil\n}\n\nfunc newEtcd(addr, user, passwd, token string) (ObjectStorage, error) {\n\tif !strings.HasPrefix(addr, \"etcd://\") {\n\t\taddr = \"etcd://\" + addr\n\t}\n\tu, err := url.Parse(addr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"parse %s: %s\", addr, err)\n\t}\n\thosts := strings.Split(u.Host, \",\")\n\tfor i, h := range hosts {\n\t\th, _, err := net.SplitHostPort(h)\n\t\tif err != nil {\n\t\t\thosts[i] = net.JoinHostPort(h, \"2379\")\n\t\t}\n\t}\n\tconf := etcd.Config{\n\t\tEndpoints: hosts,\n\t\tUsername: user,\n\t\tPassword: passwd,\n\t\tAutoSyncInterval: time.Minute,\n\t}\n\tconf.TLS, err = buildTlsConfig(u)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"build tls config from %s: %s\", u.RawQuery, err)\n\t}\n\tc, err := etcd.New(conf)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &etcdClient{DefaultObjectStorage{}, c, c.KV, u.Host}, nil\n}\n\nfunc init() {\n\tRegister(\"etcd\", newEtcd)\n}\n" }, { "path": "pkg/object/file.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nconst (\n\tdirSuffix = \"/\"\n)\n\nvar TryCFR bool // try copy_file_range\nvar PutInplace bool\n\ntype filestore struct {\n\tDefaultObjectStorage\n\troot string\n}\n\nfunc (d *filestore) Symlink(oldName, newName string) error {\n\tp := d.path(newName)\n\tif _, err := os.Stat(filepath.Dir(p)); err != nil && os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(filepath.Dir(p), os.FileMode(0777)); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\treturn os.Symlink(oldName, p)\n}\n\nfunc (d *filestore) Readlink(name string) (string, error) {\n\treturn os.Readlink(d.path(name))\n}\n\nfunc (d *filestore) String() string {\n\tif runtime.GOOS == \"windows\" {\n\t\treturn \"file:///\" + d.root\n\t}\n\treturn \"file://\" + d.root\n}\n\nfunc (d *filestore) path(key string) string {\n\tif strings.HasSuffix(d.root, dirSuffix) {\n\t\treturn filepath.Join(d.root, key)\n\t}\n\treturn filepath.Clean(d.root + key)\n}\n\nfunc (d *filestore) Head(ctx context.Context, key string) (Object, error) {\n\tp := d.path(key)\n\tfi, err := os.Lstat(p)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tisSymlink := fi.Mode()&os.ModeSymlink != 0\n\tif isSymlink {\n\t\tfi, err = os.Stat(p)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn toFile(key, fi, isSymlink, getOwnerGroup), nil\n}\n\nfunc toFile(key string, fi fs.FileInfo, isSymlink bool, ownerGetter func(fs.FileInfo) (string, string)) *file {\n\tsize := fi.Size()\n\tif fi.IsDir() {\n\t\tsize = 0\n\t}\n\towner, group := ownerGetter(fi)\n\treturn &file{\n\t\tobj{\n\t\t\tkey,\n\t\t\tsize,\n\t\t\tfi.ModTime(),\n\t\t\tfi.IsDir(),\n\t\t\t\"\",\n\t\t},\n\t\towner,\n\t\tgroup,\n\t\tfi.Mode(),\n\t\tisSymlink,\n\t}\n}\n\ntype SectionReaderCloser struct {\n\t*io.SectionReader\n\tio.Closer\n}\n\nfunc (d *filestore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tp := d.path(key)\n\n\tf, err := os.Open(p)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfinfo, err := f.Stat()\n\tif err != nil {\n\t\t_ = f.Close()\n\t\treturn nil, err\n\t}\n\tif finfo.IsDir() || off >= finfo.Size() {\n\t\t_ = f.Close()\n\t\treturn io.NopCloser(bytes.NewBuffer([]byte{})), nil\n\t}\n\n\tif limit > 0 {\n\t\treturn &SectionReaderCloser{\n\t\t\tSectionReader: io.NewSectionReader(f, off, limit),\n\t\t\tCloser: f,\n\t\t}, nil\n\t}\n\treturn f, nil\n}\n\nfunc (d *filestore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) (err error) {\n\tp := d.path(key)\n\n\tif strings.HasSuffix(key, dirSuffix) || key == \"\" && strings.HasSuffix(d.root, dirSuffix) {\n\t\treturn os.MkdirAll(p, os.FileMode(0777))\n\t}\n\n\tvar tmp string\n\tif PutInplace {\n\t\ttmp = p\n\t} else {\n\t\tname := filepath.Base(p)\n\t\tif len(name) > 200 {\n\t\t\tname = name[:200]\n\t\t}\n\t\ttmp = TmpFilePath(p, name)\n\t\tdefer func() {\n\t\t\tif err != nil {\n\t\t\t\t_ = os.Remove(tmp)\n\t\t\t}\n\t\t}()\n\t}\n\tf, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)\n\tif err != nil && os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(filepath.Dir(p), os.FileMode(0777)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tf, err = os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif TryCFR {\n\t\t_, err = io.Copy(f, in)\n\t} else {\n\t\tbuf := bufPool.Get().(*[]byte)\n\t\tdefer bufPool.Put(buf)\n\t\t_, err = io.CopyBuffer(onlyWriter{f}, in, *buf)\n\t}\n\tif err != nil {\n\t\t_ = f.Close()\n\t\treturn err\n\t}\n\terr = f.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !PutInplace {\n\t\terr = os.Rename(tmp, p)\n\t}\n\treturn err\n}\n\nfunc (d *filestore) Copy(ctx context.Context, dst, src string) error {\n\tr, err := d.Get(ctx, src, 0, -1)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer r.Close()\n\treturn d.Put(ctx, dst, r)\n}\n\nfunc (d *filestore) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\terr := os.Remove(d.path(key))\n\tif err != nil && os.IsNotExist(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\ntype mEntry struct {\n\tos.FileInfo\n\tname string\n\tfi os.FileInfo\n\tisSymlink bool\n}\n\nfunc (m *mEntry) Name() string {\n\treturn m.name\n}\n\nfunc (m *mEntry) Info() os.FileInfo {\n\tif m.fi != nil {\n\t\treturn m.fi\n\t}\n\treturn m.FileInfo\n}\n\nfunc (m *mEntry) IsDir() bool {\n\tif m.fi != nil {\n\t\treturn m.fi.IsDir()\n\t}\n\treturn m.FileInfo.IsDir()\n}\n\n// readDirSorted reads the directory named by dir and returns\n// a sorted list of directory entries.\nfunc readDirSorted(dir string, followLink bool) ([]*mEntry, error) {\n\tf, err := os.Open(dir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer f.Close()\n\tentries, err := f.Readdir(-1)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tmEntries := make([]*mEntry, len(entries))\n\tfor i, e := range entries {\n\t\tisSymlink := e.Mode()&os.ModeSymlink != 0\n\t\tif e.IsDir() {\n\t\t\tmEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}\n\t\t} else if isSymlink && followLink {\n\t\t\tfi, err := os.Stat(filepath.Join(dir, e.Name()))\n\t\t\tif err != nil {\n\t\t\t\tmEntries[i] = &mEntry{e, e.Name(), nil, true}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tname := e.Name()\n\t\t\tif fi.IsDir() {\n\t\t\t\tname = e.Name() + dirSuffix\n\t\t\t}\n\t\t\tmEntries[i] = &mEntry{e, name, fi, false}\n\t\t} else {\n\t\t\tmEntries[i] = &mEntry{e, e.Name(), nil, isSymlink}\n\t\t}\n\t}\n\tsort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })\n\treturn mEntries, err\n}\n\nfunc (d *filestore) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tvar dir string = d.root + prefix\n\tvar objs []Object\n\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\tdir = path.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t} else if marker == \"\" {\n\t\tobj, err := d.Head(ctx, prefix)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil, false, \"\", nil\n\t\t\t}\n\t\t\treturn nil, false, \"\", err\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\tentries, err := readDirSorted(dir, followLink)\n\tif err != nil {\n\t\tif os.IsPermission(err) {\n\t\t\tlogger.Warnf(\"skip %s: %s\", dir, err)\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\tif os.IsNotExist(err) {\n\t\t\tlogger.Warnf(\"skip %s: %s\", dir, err)\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\tfor _, e := range entries {\n\t\tp := path.Join(dir, e.Name())\n\t\tif e.IsDir() {\n\t\t\tp = p + \"/\"\n\t\t}\n\t\tif !strings.HasPrefix(p, d.root) {\n\t\t\tcontinue\n\t\t}\n\t\tkey := p[len(d.root):]\n\t\tif !strings.HasPrefix(key, prefix) || (marker != \"\" && key <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\tinfo := e.Info()\n\t\tf := toFile(key, info, e.isSymlink, getOwnerGroup)\n\t\tobjs = append(objs, f)\n\t\tif len(objs) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc (d *filestore) Chmod(key string, mode os.FileMode) error {\n\tp := d.path(key)\n\treturn os.Chmod(p, mode)\n}\n\nfunc (d *filestore) Chown(key string, owner, group string) error {\n\tp := d.path(key)\n\tuid := utils.LookupUser(owner)\n\tgid := utils.LookupGroup(group)\n\tif uid == -1 || gid == -1 {\n\t\treturn fmt.Errorf(\"user(%s):group(%s) not found\", owner, group)\n\t}\n\treturn os.Lchown(p, uid, gid)\n}\n\nfunc newDisk(root, accesskey, secretkey, token string) (ObjectStorage, error) {\n\t// For Windows, the path looks like /C:/a/b/c/\n\tif runtime.GOOS == \"windows\" {\n\t\troot = strings.TrimPrefix(root, \"/\")\n\t}\n\treturn &filestore{root: root}, nil\n}\n\nfunc init() {\n\tRegister(\"file\", newDisk)\n}\n" }, { "path": "pkg/object/file_darwin.go", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\n// nolint:unused\nfunc getAtime(fi os.FileInfo) time.Time {\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn time.Unix(sst.Atimespec.Unix())\n\t} else {\n\t\treturn fi.ModTime()\n\t}\n}\n\nfunc lchtimes(name string, atime time.Time, mtime time.Time) error {\n\tvar ts = make([]unix.Timespec, 2)\n\t///Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include//sys/stat.h\n\t// define UTIME_NOW -1\n\t// define UTIME_OMIT -2\n\t// only change mtime\n\tts[0] = unix.Timespec{Sec: -2, Nsec: -2}\n\tts[1] = unix.NsecToTimespec(mtime.UnixNano())\n\tif e := unix.UtimesNanoAt(unix.AT_FDCWD, name, ts, unix.AT_SYMLINK_NOFOLLOW); e != nil {\n\t\treturn &os.PathError{Op: \"lchtimes\", Path: name, Err: e}\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/object/file_linux.go", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\n// nolint:unused\nfunc getAtime(fi os.FileInfo) time.Time {\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn time.Unix(sst.Atim.Unix())\n\t}\n\treturn fi.ModTime()\n}\n\nfunc lchtimes(name string, atime time.Time, mtime time.Time) error {\n\tvar ts = make([]unix.Timespec, 2)\n\t// only change mtime\n\tts[0] = unix.Timespec{Sec: unix.UTIME_OMIT, Nsec: unix.UTIME_OMIT}\n\tts[1] = unix.NsecToTimespec(mtime.UnixNano())\n\n\tif e := unix.UtimesNanoAt(unix.AT_FDCWD, name, ts, unix.AT_SYMLINK_NOFOLLOW); e != nil {\n\t\treturn &os.PathError{Op: \"lchtimes\", Path: name, Err: e}\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/object/file_unix.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/sftp\"\n)\n\nfunc getOwnerGroup(info os.FileInfo) (string, string) {\n\tvar owner, group string\n\tswitch st := info.Sys().(type) {\n\tcase *syscall.Stat_t:\n\t\towner = utils.UserName(int(st.Uid))\n\t\tgroup = utils.GroupName(int(st.Gid))\n\tcase *sftp.FileStat:\n\t\towner = utils.UserName(int(st.UID))\n\t\tgroup = utils.GroupName(int(st.GID))\n\t}\n\treturn owner, group\n}\n\nfunc (d *filestore) Chtimes(key string, mtime time.Time) error {\n\tp := d.path(key)\n\treturn lchtimes(p, time.Time{}, mtime)\n}\n" }, { "path": "pkg/object/file_unix_test.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestLChtimes(t *testing.T) {\n\tfilePath := \"/tmp/LChtimesTestAfile1\"\n\tlinkPath := \"/tmp/LChtimesTestLink1\"\n\tos.Remove(filePath)\n\tos.Remove(linkPath)\n\t_, err := os.Create(filePath)\n\tif err != nil {\n\t\tt.Fatalf(\"create file failed: %s\", err)\n\t}\n\terr = os.Symlink(filePath, linkPath)\n\tif err != nil {\n\t\tt.Fatalf(\"symlink file failed: %s\", err)\n\t}\n\toldStat, err := os.Lstat(linkPath)\n\tif err != nil {\n\t\tt.Fatalf(\"lstat file failed: %s\", err)\n\t}\n\n\toldAtime := getAtime(oldStat)\n\tnewMtime := oldStat.ModTime().Add(-time.Hour)\n\terr = lchtimes(linkPath, time.Time{}, newMtime)\n\tif err != nil {\n\t\tt.Fatalf(\"lchtimes file failed: %s\", err)\n\t}\n\tnewStat, err := os.Lstat(linkPath)\n\tif err != nil {\n\t\tt.Fatalf(\"lstat file failed: %s\", err)\n\t}\n\tif newStat.ModTime() != newMtime {\n\t\tt.Fatalf(\"mtime change failed\")\n\t}\n\tnewAtime := getAtime(newStat)\n\tif newAtime != oldAtime {\n\t\tt.Fatalf(\"atime change failed\")\n\t}\n}\n" }, { "path": "pkg/object/file_windows.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"os\"\n\t\"time\"\n)\n\nfunc getOwnerGroup(info os.FileInfo) (string, string) {\n\treturn \"\", \"\"\n}\n\nfunc lookupUser(name string) int {\n\treturn 0\n}\n\nfunc lookupGroup(name string) int {\n\treturn 0\n}\n\nfunc (d *filestore) Chtimes(key string, mtime time.Time) error {\n\tp := d.path(key)\n\treturn os.Chtimes(p, time.Time{}, mtime)\n}\n" }, { "path": "pkg/object/filesystem_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc testKeysEqual(objs []Object, expectedKeys []string) error {\n\tgottenKeys := make([]string, len(objs))\n\tfor idx, obj := range objs {\n\t\tgottenKeys[idx] = obj.Key()\n\t}\n\tif len(gottenKeys) != len(expectedKeys) {\n\t\treturn fmt.Errorf(\"Expected {%s}, got {%s}\", strings.Join(expectedKeys, \", \"),\n\t\t\tstrings.Join(gottenKeys, \", \"))\n\t}\n\n\tfor idx, key := range gottenKeys {\n\t\tif key != expectedKeys[idx] {\n\t\t\treturn fmt.Errorf(\"Expected {%s}, got {%s}\", strings.Join(expectedKeys, \", \"),\n\t\t\t\tstrings.Join(gottenKeys, \", \"))\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc TestDisk2(t *testing.T) {\n\tdiskPath := \"/tmp/abc/\"\n\t_ = os.RemoveAll(diskPath)\n\ts, _ := newDisk(diskPath, \"\", \"\", \"\")\n\ts = WithPrefix(s, \"prefix/\")\n\ttestFileSystem(t, s)\n}\n\nfunc TestSftp2(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SFTP_HOST\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tsftp, err := newSftp(os.Getenv(\"SFTP_HOST\"), os.Getenv(\"SFTP_USER\"), os.Getenv(\"SFTP_PASS\"), \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"sftp: %s\", err)\n\t}\n\ttestFileSystem(t, sftp)\n}\n\nfunc TestCifs2(t *testing.T) { //skip mutate\n\tif os.Getenv(\"CIFS_ADDR\") == \"\" {\n\t\tfmt.Println(\"skip CIFS test\")\n\t\tt.SkipNow()\n\t}\n\tcifs, err := newCifs(os.Getenv(\"CIFS_ADDR\"), os.Getenv(\"CIFS_USER\"), os.Getenv(\"CIFS_PASSWORD\"), \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create: %s\", err)\n\t}\n\ttestFileSystem(t, cifs)\n}\n\nfunc TestHDFS2(t *testing.T) { //skip mutate\n\tif os.Getenv(\"HDFS_ADDR\") == \"\" {\n\t\tt.Skip()\n\t}\n\tdfs, _ := newHDFS(os.Getenv(\"HDFS_ADDR\"), \"testUser1\", \"\", \"\")\n\ttestFileSystem(t, dfs)\n}\n\nfunc TestNFS2(t *testing.T) { //skip mutate\n\tif os.Getenv(\"NFS_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, err := newNFSStore(os.Getenv(\"NFS_ADDR\"), os.Getenv(\"NFS_ACCESS_KEY\"), os.Getenv(\"NFS_SECRET_KEY\"), \"\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestFileSystem(t, b)\n}\n\nfunc testFileSystem(t *testing.T, s ObjectStorage) {\n\tctx := context.Background()\n\tkeys := []string{\n\t\t\"x/\",\n\t\t\"x/x.txt\",\n\t\t\"xy.txt\",\n\t\t\"xyz/\",\n\t\t\"xyz/xyz.txt\",\n\t}\n\t// initialize directory tree\n\tfor _, key := range keys {\n\t\tif err := s.Put(ctx, key, bytes.NewReader([]byte{'a', 'b'})); err != nil {\n\t\t\tt.Fatalf(\"PUT object `%s` failed: %q\", key, err)\n\t\t}\n\t}\n\tif o, err := s.Head(ctx, \"x/\"); err != nil {\n\t\tt.Fatalf(\"Head x/: %s\", err)\n\t} else if f, ok := o.(File); !ok {\n\t\tt.Fatalf(\"Head should return File\")\n\t} else if !f.IsDir() {\n\t\tt.Fatalf(\"x/ should be a dir\")\n\t}\n\t// cleanup\n\tdefer func() {\n\t\t// delete reversely, directory only can be deleted when it's empty\n\t\tobjs, err := listAll(ctx, s, \"\", \"\", 100, true)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"listall failed: %s\", err)\n\t\t}\n\t\tgottenKeys := make([]string, len(objs))\n\t\tfor idx, obj := range objs {\n\t\t\tgottenKeys[idx] = obj.Key()\n\t\t}\n\t\tidx := len(gottenKeys) - 1\n\t\tfor ; idx >= 0; idx-- {\n\t\t\tif err := s.Delete(ctx, gottenKeys[idx]); err != nil {\n\t\t\t\tt.Fatalf(\"DELETE object `%s` failed: %q\", gottenKeys[idx], err)\n\t\t\t}\n\t\t}\n\t}()\n\tobjs, err := listAll(ctx, s, \"x/\", \"\", 100, true)\n\tif err != nil {\n\t\tt.Fatalf(\"list failed: %s\", err)\n\t}\n\texpectedKeys := []string{\"x/\", \"x/x.txt\"}\n\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t}\n\n\tobjs, err = listAll(ctx, s, \"x\", \"\", 100, true)\n\tif err != nil {\n\t\tt.Fatalf(\"list failed: %s\", err)\n\t}\n\texpectedKeys = []string{\"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t}\n\n\tif ss, ok := s.(FileSystem); ok {\n\t\tfor _, mode := range []uint32{0022, 0122, 0422} {\n\t\t\tt.Logf(\"test mode %o\", os.FileMode(mode))\n\t\t\terr := ss.Chmod(\"x/\", os.FileMode(mode))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"chmod %ofailed: %s\", mode, err)\n\t\t\t}\n\n\t\t\tobjs, err = listAll(ctx, s, \"x\", \"\", 100, true)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"list failed: %s mode %o\", err, mode)\n\t\t\t}\n\t\t\texpectedKeys = []string{\"x/\", \"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\t\t\tif _, ok := ss.(*nfsStore); ok {\n\t\t\t\texpectedKeys = []string{\"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\t\t\t}\n\t\t\tif _, ok := ss.(*cifsStore); ok {\n\t\t\t\texpectedKeys = []string{\"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\t\t\t}\n\t\t\tif mode == 0422 {\n\t\t\t\tif strings.HasPrefix(s.String(), \"gluster://\") {\n\t\t\t\t\texpectedKeys = []string{\"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\t\t\tt.Fatalf(\"testKeysEqual fail: %s mode %o\", err, mode)\n\t\t\t}\n\t\t\terr = ss.Chmod(\"x/\", os.FileMode(0777))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"chmod %o failed: %s\", mode, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tobjs, err = listAll(ctx, s, \"xy\", \"\", 100, true)\n\tif err != nil {\n\t\tt.Fatalf(\"list failed: %s\", err)\n\t}\n\texpectedKeys = []string{\"xy.txt\", \"xyz/\", \"xyz/xyz.txt\"}\n\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t}\n\n\tif ss, ok := s.(SupportSymlink); ok {\n\t\t// a< a- < a/ < a0 < b< b- < b/ < b0\n\t\t_ = s.Put(ctx, \"a-\", bytes.NewReader([]byte{}))\n\t\t_ = s.Put(ctx, \"a0\", bytes.NewReader([]byte{}))\n\t\t_ = s.Put(ctx, \"b-\", bytes.NewReader([]byte{}))\n\t\t_ = s.Put(ctx, \"b0\", bytes.NewReader(make([]byte, 10)))\n\t\t_ = s.Put(ctx, \"xyz/ol1/p.txt\", bytes.NewReader([]byte{}))\n\n\t\terr = ss.Symlink(\"../b0\", \"bb/b1\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"symlink: %s\", err)\n\t\t}\n\t\tif target, err := ss.Readlink(\"bb/b1\"); err != nil {\n\t\t\tt.Fatalf(\"readlink: %s\", err)\n\t\t} else if target != \"../b0\" {\n\t\t\tt.Fatalf(\"target should be ../b0, but got %s\", target)\n\t\t}\n\t\tif fi, err := s.Head(ctx, \"bb/b1\"); err != nil || !fi.IsSymlink() || fi.Size() != 10 {\n\t\t\tt.Fatalf(\"haed of symlink: err=%s, size=%d isSymlink=%v\", err, fi.Size(), fi.IsSymlink())\n\t\t}\n\t\terr = ss.Symlink(\"../notExist\", \"bb/brokenLink\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"symlink: %s\", err)\n\t\t}\n\t\tif _, err := s.Head(ctx, \"bb/brokenLink\"); !errors.Is(err, os.ErrNotExist) {\n\t\t\tt.Fatalf(\"head broken symlink: err=%s, should be os.ErrNotExist\", err)\n\t\t}\n\t\t_ = s.Delete(ctx, \"bb/brokenLink\")\n\t\tif err = ss.Symlink(\"xyz/ol1/\", \"a\"); err != nil {\n\t\t\tt.Fatalf(\"symlink: a: %s\", err)\n\t\t}\n\t\t_ = ss.Symlink(\"xyz/notExist/\", \"b\")\n\n\t\tobjs, err = listAll(ctx, s, \"\", \"\", 100, true)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"listall failed: %s\", err)\n\t\t}\n\t\texpectedKeys = []string{\"\", \"a-\", \"a/\", \"a/p.txt\", \"a0\", \"b\", \"b-\", \"b0\", \"bb/\", \"bb/b1\", \"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/ol1/\", \"xyz/ol1/p.txt\", \"xyz/xyz.txt\"}\n\t\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t\t}\n\t\tif objs[2].Size() != 0 {\n\t\t\tt.Fatalf(\"size of target(dir) should be 0\")\n\t\t}\n\t\tif objs[9].Size() != 10 {\n\t\t\tt.Fatalf(\"size of target(file) should be 10\")\n\t\t}\n\n\t\t// test don't follow symlink\n\t\tif _, ok := s.(*hdfsclient); !ok {\n\t\t\tobjs, err = listAll(ctx, s, \"\", \"\", 100, false)\n\t\t\texpectedKeys = []string{\"\", \"a\", \"a-\", \"a0\", \"b\", \"b-\", \"b0\", \"bb/\", \"bb/b1\", \"x/\", \"x/x.txt\", \"xy.txt\", \"xyz/\", \"xyz/ol1/\", \"xyz/ol1/p.txt\", \"xyz/xyz.txt\"}\n\t\t\tif err = testKeysEqual(objs, expectedKeys); err != nil {\n\t\t\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\t// put a file with very long name\n\tlongName := strings.Repeat(\"a\", 255)\n\tif err := s.Put(ctx, \"dir/\"+longName, bytes.NewReader([]byte{0})); err != nil {\n\t\tt.Fatalf(\"PUT a file with long name `%s` failed: %q\", longName, err)\n\t}\n}\n" }, { "path": "pkg/object/gluster.go", "content": "//go:build gluster\n// +build gluster\n\n/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/juicedata/gogfapi/gfapi\"\n)\n\ntype gluster struct {\n\tDefaultObjectStorage\n\tname string\n\tindx uint64\n\tvols []*gfapi.Volume\n}\n\nfunc (g *gluster) String() string {\n\treturn fmt.Sprintf(\"gluster://%s/\", g.name)\n}\n\nfunc (g *gluster) vol() *gfapi.Volume {\n\tif len(g.vols) == 1 {\n\t\treturn g.vols[0]\n\t}\n\tn := atomic.AddUint64(&g.indx, 1)\n\treturn g.vols[n%uint64(len(g.vols))]\n}\n\nfunc (g *gluster) Head(ctx context.Context, key string) (Object, error) {\n\tfi, err := g.vol().Stat(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn g.toFile(key, fi, false), nil\n}\n\nfunc (g *gluster) toFile(key string, fi fs.FileInfo, isSymlink bool) *file {\n\tsize := fi.Size()\n\tif fi.IsDir() {\n\t\tsize = 0\n\t}\n\towner, group := getOwnerGroup(fi)\n\treturn &file{\n\t\tobj{\n\t\t\tkey,\n\t\t\tsize,\n\t\t\tfi.ModTime(),\n\t\t\tfi.IsDir(),\n\t\t\t\"\",\n\t\t},\n\t\towner,\n\t\tgroup,\n\t\tfi.Mode(),\n\t\tisSymlink,\n\t}\n}\n\nfunc (g *gluster) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tf, err := g.vol().Open(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfinfo, err := f.Stat()\n\tif err != nil {\n\t\t_ = f.Close()\n\t\treturn nil, err\n\t}\n\tif finfo.IsDir() || off > finfo.Size() {\n\t\t_ = f.Close()\n\t\treturn io.NopCloser(bytes.NewBuffer([]byte{})), nil\n\t}\n\n\tif limit > 0 {\n\t\treturn &SectionReaderCloser{\n\t\t\tSectionReader: io.NewSectionReader(f, off, limit),\n\t\t\tCloser: f,\n\t\t}, nil\n\t}\n\treturn f, nil\n}\n\nfunc (g *gluster) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tv := g.vol()\n\tif strings.HasSuffix(key, dirSuffix) {\n\t\treturn v.MkdirAll(key, os.FileMode(0777))\n\t}\n\tf, err := v.OpenFile(key, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)\n\tif err != nil && os.IsNotExist(err) {\n\t\tif err := v.MkdirAll(filepath.Dir(key), os.FileMode(0777)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tf, err = v.OpenFile(key, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tif err != nil {\n\t\t\t_ = v.Unlink(key)\n\t\t}\n\t}()\n\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\t_, err = io.CopyBuffer(f, in, *buf)\n\tif err != nil {\n\t\t_ = f.Close()\n\t\treturn err\n\t}\n\tif err = f.Sync(); err != nil {\n\t\t_ = f.Close()\n\t\treturn err\n\t}\n\terr = f.Close()\n\treturn err\n}\n\nfunc (g *gluster) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tv := g.vol()\n\terr := v.Unlink(key)\n\tif err != nil && strings.Contains(err.Error(), \"is a directory\") {\n\t\terr = v.Rmdir(key)\n\t}\n\tif os.IsNotExist(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\n// readDirSorted reads the directory named by dirname and returns\n// a sorted list of directory entries.\nfunc (g *gluster) readDirSorted(dirname string, followLink bool) ([]*mEntry, error) {\n\tv := g.vol()\n\tf, err := v.OpenDir(dirname)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer f.Close()\n\tentries, err := f.Readdir(0)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tmEntries := make([]*mEntry, 0, len(entries))\n\tfor _, e := range entries {\n\t\tname := e.Name()\n\t\tif name == \".\" || name == \"..\" {\n\t\t\tcontinue\n\t\t}\n\t\tisSymlink := e.Mode()&os.ModeSymlink != 0\n\t\tif e.IsDir() {\n\t\t\tmEntries = append(mEntries, &mEntry{nil, name + dirSuffix, e, false})\n\t\t} else if isSymlink && followLink {\n\t\t\tfi, err := v.Stat(filepath.Join(dirname, name))\n\t\t\tif err != nil {\n\t\t\t\tmEntries = append(mEntries, &mEntry{nil, name, e, true})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif fi.IsDir() {\n\t\t\t\tname += dirSuffix\n\t\t\t}\n\t\t\tmEntries = append(mEntries, &mEntry{nil, name, fi, false})\n\t\t} else {\n\t\t\tmEntries = append(mEntries, &mEntry{nil, name, e, isSymlink})\n\t\t}\n\t}\n\tsort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })\n\treturn mEntries, err\n}\n\nfunc (g *gluster) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tvar dir string = prefix\n\tvar objs []Object\n\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\tdir = path.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t} else if marker == \"\" {\n\t\tobj, err := g.Head(ctx, prefix)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil, false, \"\", nil\n\t\t\t}\n\t\t\treturn nil, false, \"\", err\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\tentries, err := g.readDirSorted(dir, followLink)\n\tif err != nil {\n\t\tif os.IsPermission(err) {\n\t\t\tlogger.Warnf(\"skip %s: %s\", dir, err)\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\tif os.IsNotExist(err) {\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\tfor _, e := range entries {\n\t\tp := filepath.Join(dir, e.Name())\n\t\tif e.IsDir() {\n\t\t\tp = filepath.ToSlash(p + \"/\")\n\t\t}\n\t\tkey := p\n\t\tif !strings.HasPrefix(key, prefix) || (marker != \"\" && key <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\tinfo := e.Info()\n\t\tf := g.toFile(key, info, e.isSymlink)\n\t\tobjs = append(objs, f)\n\t\tif len(objs) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc (g *gluster) Chtimes(path string, mtime time.Time) error {\n\treturn notSupported\n}\n\nfunc (g *gluster) Chmod(path string, mode os.FileMode) error {\n\treturn g.vol().Chmod(path, mode)\n}\n\nfunc (g *gluster) Chown(path string, owner, group string) error {\n\treturn notSupported\n}\n\nfunc newGluster(endpoint, ak, sk, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"gluster://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tps := strings.Split(uri.Path, \"/\")\n\tif len(ps) == 1 {\n\t\treturn nil, fmt.Errorf(\"no volume provided\")\n\t}\n\tname := ps[1]\n\t// multiple clients for possible performance improvement\n\tvar size int\n\tif ssize := os.Getenv(\"JFS_NUM_GLUSTER_CLIENTS\"); ssize != \"\" {\n\t\tsize, _ = strconv.Atoi(ssize)\n\t\tif size > 8 {\n\t\t\tsize = 8\n\t\t}\n\t}\n\tif size < 1 {\n\t\tsize = 1\n\t}\n\t// logging\n\tlevel := gfapi.LogInfo\n\tif slevel := os.Getenv(\"JFS_GLUSTER_LOG_LEVEL\"); slevel != \"\" {\n\t\tswitch strings.ToUpper(slevel) {\n\t\tcase \"ERROR\":\n\t\t\tlevel = gfapi.LogError\n\t\tcase \"WARN\", \"WARNING\":\n\t\t\tlevel = gfapi.LogWarning\n\t\tcase \"INFO\":\n\t\t\tlevel = gfapi.LogInfo\n\t\tcase \"DEBUG\":\n\t\t\tlevel = gfapi.LogDebug\n\t\tcase \"TRACE\":\n\t\t\tlevel = gfapi.LogTrace\n\t\t}\n\t}\n\tlogPath := os.Getenv(\"JFS_GLUSTER_LOG_PATH\")\n\thosts := strings.Split(uri.Host, \",\")\n\tpid := os.Getpid()\n\tostore := gluster{\n\t\tname: name,\n\t\tvols: make([]*gfapi.Volume, size),\n\t}\n\tfor i := range ostore.vols {\n\t\tv := &gfapi.Volume{}\n\t\t// TODO: support port in host\n\t\terr = v.Init(name, hosts...)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"init %s: %s\", name, err)\n\t\t}\n\t\tif logPath == \"\" {\n\t\t\terr = v.SetLogging(fmt.Sprintf(\"/var/log/glusterfs/%s-%s-%d-%d.log\", hosts[0], name, pid, i), level)\n\t\t} else {\n\t\t\terr = v.SetLogging(logPath, level)\n\t\t}\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"Set gluster logging for vol %s: %s\", name, err)\n\t\t}\n\t\terr = v.Mount()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"mount %s: %s\", name, err)\n\t\t}\n\t\tostore.vols[i] = v\n\t}\n\treturn &ostore, nil\n}\n\nfunc init() {\n\tRegister(\"gluster\", newGluster)\n}\n" }, { "path": "pkg/object/gluster_test.go", "content": "//go:build gluster\n// +build gluster\n\n/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestGluster(t *testing.T) {\n\tif os.Getenv(\"GLUSTER_VOLUME\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newGluster(os.Getenv(\"GLUSTER_VOLUME\"), \"\", \"\", \"\")\n\ttestStorage(t, b)\n\n}\n\nfunc TestGluster2(t *testing.T) {\n\tif os.Getenv(\"GLUSTER_VOLUME\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newGluster(os.Getenv(\"GLUSTER_VOLUME\"), \"\", \"\", \"\")\n\ttestFileSystem(t, b)\n}\n" }, { "path": "pkg/object/gs.go", "content": "//go:build !nogs\n// +build !nogs\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"cloud.google.com/go/compute/metadata\"\n\t\"cloud.google.com/go/storage\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/oauth2/google\"\n\t\"google.golang.org/api/iterator\"\n)\n\ntype gs struct {\n\tDefaultObjectStorage\n\tclients []*storage.Client\n\tindex uint64\n\tbucket string\n\tregion string\n\tsc string\n}\n\nfunc (g *gs) String() string {\n\treturn fmt.Sprintf(\"gs://%s/\", g.bucket)\n}\n\nfunc (g *gs) getClient() *storage.Client {\n\tif len(g.clients) == 1 {\n\t\treturn g.clients[0]\n\t}\n\tn := atomic.AddUint64(&g.index, 1)\n\treturn g.clients[n%(uint64(len(g.clients)))]\n}\n\nfunc (g *gs) Create(ctx context.Context) error {\n\t// check if the bucket is already exists\n\tif objs, _, _, err := g.List(ctx, \"\", \"\", \"\", \"\", 1, true); err == nil && len(objs) > 0 {\n\t\treturn nil\n\t}\n\tprojectID := os.Getenv(\"GOOGLE_CLOUD_PROJECT\")\n\tif projectID == \"\" {\n\t\tprojectID, _ = metadata.ProjectIDWithContext(ctx)\n\t}\n\tif projectID == \"\" {\n\t\tcred, err := google.FindDefaultCredentials(ctx)\n\t\tif err == nil {\n\t\t\tprojectID = cred.ProjectID\n\t\t}\n\t}\n\tif projectID == \"\" {\n\t\treturn errors.New(\"GOOGLE_CLOUD_PROJECT environment variable must be set\")\n\t}\n\t// Guess region when region is not provided\n\tif g.region == \"\" {\n\t\tzone, err := metadata.ZoneWithContext(ctx)\n\t\tif err == nil && len(zone) > 2 {\n\t\t\tg.region = zone[:len(zone)-2]\n\t\t}\n\t}\n\n\terr := g.getClient().Bucket(g.bucket).Create(ctx, projectID, &storage.BucketAttrs{\n\t\tName: g.bucket,\n\t\tStorageClass: g.sc,\n\t\tLocation: g.region,\n\t})\n\tif err != nil && strings.Contains(err.Error(), \"You already own this bucket\") {\n\t\treturn nil\n\t}\n\treturn err\n}\n\nfunc (g *gs) Head(ctx context.Context, key string) (Object, error) {\n\tattrs, err := g.getClient().Bucket(g.bucket).Object(key).Attrs(ctx)\n\tif err != nil {\n\t\tif err == storage.ErrObjectNotExist {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\n\treturn &obj{\n\t\tkey,\n\t\tattrs.Size,\n\t\tattrs.Updated,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\tattrs.StorageClass,\n\t}, nil\n}\n\nfunc (g *gs) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\treader, err := g.getClient().Bucket(g.bucket).Object(key).NewRangeReader(ctx, off, limit)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// TODO fire another attr request to get the actual storage class\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetStorageClass(g.sc)\n\treturn reader, nil\n}\n\nfunc (g *gs) Put(ctx context.Context, key string, data io.Reader, getters ...AttrGetter) error {\n\twriter := g.getClient().Bucket(g.bucket).Object(key).NewWriter(ctx)\n\twriter.StorageClass = g.sc\n\n\t// If you upload small objects (< 16MiB), you should set ChunkSize\n\t// to a value slightly larger than the objects' sizes to avoid memory bloat.\n\t// This is especially important if you are uploading many small objects concurrently.\n\twriter.ChunkSize = 5 << 20\n\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\t_, err := io.CopyBuffer(writer, data, *buf)\n\tif err != nil {\n\t\treturn err\n\t}\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetStorageClass(g.sc)\n\treturn writer.Close()\n}\n\nfunc (g *gs) Copy(ctx context.Context, dst, src string) error {\n\tclient := g.getClient()\n\tsrcObj := client.Bucket(g.bucket).Object(src)\n\tdstObj := client.Bucket(g.bucket).Object(dst)\n\tcopier := dstObj.CopierFrom(srcObj)\n\tif g.sc != \"\" {\n\t\tcopier.StorageClass = g.sc\n\t}\n\t_, err := copier.Run(ctx)\n\treturn err\n}\n\nfunc (g *gs) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tif err := g.getClient().Bucket(g.bucket).Object(key).Delete(ctx); err != storage.ErrObjectNotExist {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (g *gs) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tobjectIterator := g.getClient().Bucket(g.bucket).Objects(ctx, &storage.Query{Prefix: prefix, Delimiter: delimiter, StartOffset: start})\n\tpager := iterator.NewPager(objectIterator, int(limit), token)\n\tvar entries []*storage.ObjectAttrs\n\tnextPageToken, err := pager.NextPage(&entries)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(entries)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\titem := entries[i]\n\t\tif delimiter != \"\" && item.Prefix != \"\" {\n\t\t\tobjs[i] = &obj{item.Prefix, 0, time.Unix(0, 0), true, item.StorageClass}\n\t\t} else {\n\t\t\tobjs[i] = &obj{item.Name, item.Size, item.Updated, strings.HasSuffix(item.Name, \"/\"), item.StorageClass}\n\t\t}\n\t}\n\tif delimiter != \"\" {\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, nextPageToken != \"\", nextPageToken, nil\n}\n\nfunc (g *gs) SetStorageClass(sc string) error {\n\tg.sc = sc\n\treturn nil\n}\n\nfunc newGS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"gs://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, errors.Errorf(\"Invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\thostParts := strings.Split(uri.Host, \".\")\n\tbucket := hostParts[0]\n\tvar region string\n\tif len(hostParts) > 1 {\n\t\tregion = hostParts[1]\n\t}\n\n\tvar size int\n\tif ssize := os.Getenv(\"JFS_NUM_GOOGLE_CLIENTS\"); ssize != \"\" {\n\t\tif size, err = strconv.Atoi(ssize); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tif size < 1 {\n\t\tsize = 5\n\t}\n\tclis := make([]*storage.Client, size)\n\tfor i := 0; i < size; i++ {\n\t\tclient, err := storage.NewClient(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tclis[i] = client\n\t}\n\n\treturn &gs{clients: clis, bucket: bucket, region: region}, nil\n}\n\nfunc init() {\n\tRegister(\"gs\", newGS)\n}\n" }, { "path": "pkg/object/hdfs.go", "content": "//go:build !nohdfs\n// +build !nohdfs\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"os\"\n\t\"os/user\"\n\t\"path\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/colinmarc/hdfs/v2\"\n\t\"github.com/colinmarc/hdfs/v2/hadoopconf\"\n)\n\nvar superuser = \"hdfs\"\nvar supergroup = \"supergroup\"\n\ntype hdfsclient struct {\n\tDefaultObjectStorage\n\taddr string\n\tbasePath string\n\tc *hdfs.Client\n\tdfsReplication int\n\tumask os.FileMode\n\tcloseTimeout time.Duration\n\tcloseMaxDelay time.Duration\n}\n\nfunc (h *hdfsclient) String() string {\n\treturn fmt.Sprintf(\"hdfs://%s%s\", h.addr, h.basePath)\n}\n\nfunc (h *hdfsclient) path(key string) string {\n\treturn h.basePath + key\n}\n\nfunc (h *hdfsclient) Head(ctx context.Context, key string) (Object, error) {\n\tinfo, err := h.c.Stat(h.path(key))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn h.toFile(key, info), nil\n}\n\nfunc (h *hdfsclient) toFile(key string, info os.FileInfo) *file {\n\thinfo := info.(*hdfs.FileInfo)\n\tf := &file{\n\t\tobj{\n\t\t\tkey,\n\t\t\tinfo.Size(),\n\t\t\tinfo.ModTime(),\n\t\t\tinfo.IsDir(),\n\t\t\t\"\",\n\t\t},\n\t\thinfo.Owner(),\n\t\thinfo.OwnerGroup(),\n\t\tinfo.Mode(),\n\t\tfalse,\n\t}\n\tif f.owner == superuser {\n\t\tf.owner = \"root\"\n\t}\n\tif f.group == supergroup {\n\t\tf.group = \"root\"\n\t}\n\t// stickybit from HDFS is different than golang\n\tif f.mode&01000 != 0 {\n\t\tf.mode &= ^os.FileMode(01000)\n\t\tf.mode |= os.ModeSticky\n\t}\n\tif info.IsDir() {\n\t\tf.size = 0\n\t\tif !strings.HasSuffix(f.key, \"/\") && f.key != \"\" {\n\t\t\tf.key += \"/\"\n\t\t}\n\t}\n\treturn f\n}\n\nfunc (h *hdfsclient) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tf, err := h.c.Open(h.path(key))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfinfo := f.Stat()\n\tif finfo.IsDir() || off >= finfo.Size() {\n\t\t_ = f.Close()\n\t\treturn io.NopCloser(bytes.NewBuffer([]byte{})), nil\n\t}\n\n\tif limit > 0 {\n\t\treturn &SectionReaderCloser{\n\t\t\tSectionReader: io.NewSectionReader(f, off, limit),\n\t\t\tCloser: f,\n\t\t}, nil\n\t}\n\treturn f, nil\n}\n\nfunc (h *hdfsclient) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) (err error) {\n\tp := h.path(key)\n\tif strings.HasSuffix(p, dirSuffix) {\n\t\treturn h.c.MkdirAll(p, 0777&^h.umask)\n\t}\n\tvar tmp string\n\tif PutInplace {\n\t\ttmp = p\n\t} else {\n\t\tname := path.Base(p)\n\t\tif len(name) > 200 {\n\t\t\tname = name[:200]\n\t\t}\n\t\ttmp = TmpFilePath(p, name)\n\t\tdefer func() {\n\t\t\tif err != nil {\n\t\t\t\t_ = h.c.Remove(tmp)\n\t\t\t}\n\t\t}()\n\t}\n\tf, err := h.c.CreateFile(tmp, h.dfsReplication, 128<<20, 0666&^h.umask)\n\tif err != nil {\n\t\tif pe, ok := err.(*os.PathError); ok && pe.Err == os.ErrNotExist {\n\t\t\t_ = h.c.MkdirAll(path.Dir(p), 0777&^h.umask)\n\t\t\tf, err = h.c.CreateFile(tmp, h.dfsReplication, 128<<20, 0666&^h.umask)\n\t\t}\n\t\tif pe, ok := err.(*os.PathError); ok && errors.Is(pe.Err, os.ErrExist) {\n\t\t\t_ = h.c.Remove(tmp)\n\t\t\tf, err = h.c.CreateFile(tmp, h.dfsReplication, 128<<20, 0666&^h.umask)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\t_, err = io.CopyBuffer(f, in, *buf)\n\tif err != nil {\n\t\t_ = f.Close()\n\t\treturn err\n\t}\n\tstart := time.Now()\n\tsleeptime := 400 * time.Millisecond\n\tfor {\n\t\terr = f.Close()\n\t\tif IsErrReplicating(err) && start.Add(h.closeTimeout).After(time.Now()) {\n\t\t\ttime.Sleep(sleeptime)\n\t\t\tsleeptime = min(2*sleeptime, h.closeMaxDelay)\n\t\t\tcontinue\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !PutInplace {\n\t\terr = h.c.Rename(tmp, p)\n\t}\n\treturn err\n}\n\nfunc min(a, b time.Duration) time.Duration {\n\tif a < b {\n\t\treturn a\n\t}\n\treturn b\n}\n\nfunc IsErrReplicating(err error) bool {\n\tpe, ok := err.(*os.PathError)\n\treturn ok && pe.Err == hdfs.ErrReplicating\n}\n\nfunc (h *hdfsclient) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\terr := h.c.Remove(h.path(key))\n\tif err != nil && os.IsNotExist(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (h *hdfsclient) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tdir := h.path(prefix)\n\tvar objs []Object\n\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\tdir = path.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t} else if marker == \"\" {\n\t\tobj, err := h.Head(ctx, prefix)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil, false, \"\", nil\n\t\t\t}\n\t\t\treturn nil, false, \"\", err\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\n\tfile, err := h.c.Open(dir)\n\tvar entries []os.FileInfo\n\tif file != nil {\n\t\tentries, err = file.Readdir(0)\n\t}\n\tif err != nil {\n\t\tif os.IsPermission(err) {\n\t\t\tlogger.Warnf(\"skip %s: %s\", dir, err)\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\tif os.IsNotExist(err) {\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\n\t// make sure they are ordered in full path\n\tentryMap := make(map[string]fs.FileInfo)\n\tnames := make([]string, len(entries))\n\tfor i, info := range entries {\n\t\tif info.IsDir() {\n\t\t\tnames[i] = info.Name() + \"/\"\n\t\t} else {\n\t\t\tnames[i] = info.Name()\n\t\t}\n\t\tentryMap[names[i]] = info\n\t}\n\tsort.Strings(names)\n\n\tfor _, name := range names {\n\t\tp := dir + name\n\t\tif !strings.HasPrefix(p, h.basePath) {\n\t\t\tcontinue\n\t\t}\n\t\tkey := p[len(h.basePath):]\n\t\tif !strings.HasPrefix(key, prefix) || (marker != \"\" && key <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\tf := h.toFile(key, entryMap[name])\n\t\tobjs = append(objs, f)\n\t\tif len(objs) >= int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc (h *hdfsclient) Chtimes(key string, mtime time.Time) error {\n\t// fixme: need set the atime in hdfs.SetTimesRequestProto to -1 to avoid updating the atime\n\t// ref: https://hadoop.apache.org/docs/stable/api/org/apache/hadoop/fs/FileSystem.html#setTimes-org.apache.hadoop.fs.Path-long-long-\n\treturn h.c.Chtimes(h.path(key), mtime, mtime)\n}\n\nfunc (h *hdfsclient) Chmod(key string, mode os.FileMode) error {\n\treturn h.c.Chmod(h.path(key), mode)\n}\n\nfunc (h *hdfsclient) Chown(key string, owner, group string) error {\n\tif owner == \"root\" {\n\t\towner = superuser\n\t}\n\tif group == \"root\" {\n\t\tgroup = supergroup\n\t}\n\treturn h.c.Chown(h.path(key), owner, group)\n}\n\nfunc newHDFS(addr, username, sk, token string) (ObjectStorage, error) {\n\tconf, err := hadoopconf.LoadFromEnvironment()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Problem loading configuration: %s\", err)\n\t}\n\n\trpcAddr, basePath := parseHDFSAddr(addr, conf)\n\toptions := hdfs.ClientOptionsFromConf(conf)\n\tif addr != \"\" {\n\t\toptions.Addresses = rpcAddr\n\t\tlogger.Infof(\"HDFS Addresses: %s, basePath: %s\", rpcAddr, basePath)\n\t}\n\n\tif options.KerberosClient != nil {\n\t\toptions.KerberosClient, err = getKerberosClient()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Problem with kerberos authentication: %s\", err)\n\t\t}\n\t} else {\n\t\tif username == \"\" {\n\t\t\tusername = os.Getenv(\"HADOOP_USER_NAME\")\n\t\t}\n\t\tif username == \"\" {\n\t\t\tcurrent, err := user.Current()\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"get current user: %s\", err)\n\t\t\t}\n\t\t\tusername = current.Username\n\t\t}\n\t\toptions.User = username\n\t}\n\n\tc, err := hdfs.NewClient(options)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"new HDFS client %s: %s\", rpcAddr, err)\n\t}\n\tif os.Getenv(\"HADOOP_SUPER_USER\") != \"\" {\n\t\tsuperuser = os.Getenv(\"HADOOP_SUPER_USER\")\n\t}\n\tif os.Getenv(\"HADOOP_SUPER_GROUP\") != \"\" {\n\t\tsupergroup = os.Getenv(\"HADOOP_SUPER_GROUP\")\n\t}\n\n\tvar replication = 3\n\tif v, found := conf[\"dfs.replication\"]; found {\n\t\tif x, err := strconv.Atoi(v); err == nil {\n\t\t\treplication = x\n\t\t}\n\t}\n\tvar umask uint16 = 022\n\tif v, found := conf[\"fs.permissions.umask-mode\"]; found {\n\t\tif x, err := strconv.ParseUint(v, 8, 16); err == nil {\n\t\t\tumask = uint16(x)\n\t\t}\n\t}\n\tvar closeTimeout = 120 * time.Second\n\tif v, found := conf[\"ipc.client.rpc-timeout.ms\"]; found {\n\t\tif x, err := strconv.Atoi(v); err == nil {\n\t\t\tcloseTimeout = time.Duration(x) * time.Millisecond\n\t\t}\n\t}\n\tvar closeMaxDelay = 60 * time.Second\n\tif v, found := conf[\"dfs.client.block.write.locateFollowingBlock.max.delay.ms\"]; found {\n\t\tif x, err := strconv.Atoi(v); err == nil {\n\t\t\tcloseMaxDelay = time.Duration(x) * time.Millisecond\n\t\t}\n\t}\n\n\treturn &hdfsclient{\n\t\taddr: strings.Join(rpcAddr, \",\"),\n\t\tbasePath: basePath,\n\t\tc: c,\n\t\tdfsReplication: replication,\n\t\tumask: os.FileMode(umask),\n\t\tcloseTimeout: closeTimeout,\n\t\tcloseMaxDelay: closeMaxDelay,\n\t}, nil\n}\n\n// addr can be hdfs://nameservice e.g. hdfs://example, hdfs://example/user/juicefs\n// convert the nameservice as a comma separated list of host:port by referencing hadoop conf\nfunc parseHDFSAddr(addr string, conf hadoopconf.HadoopConf) (rpcAddresses []string, basePath string) {\n\taddr = strings.TrimPrefix(addr, \"hdfs://\")\n\tsp := strings.SplitN(addr, \"/\", 2)\n\tauthority := sp[0]\n\n\t// check if it is a nameservice\n\tvar nns []string\n\tconfParam := \"dfs.namenode.rpc-address.\" + authority\n\tfor key, value := range conf {\n\t\tif key == confParam || strings.HasPrefix(key, confParam+\".\") {\n\t\t\tnns = append(nns, value)\n\t\t}\n\t}\n\tif len(nns) > 0 {\n\t\trpcAddresses = nns\n\t} else {\n\t\trpcAddresses = strings.Split(authority, \",\")\n\t}\n\tbasePath = \"/\"\n\tif len(sp) > 1 && len(sp[1]) > 0 {\n\t\tbasePath += strings.TrimRight(sp[1], \"/\") + \"/\"\n\t}\n\treturn\n}\n\nfunc init() {\n\tRegister(\"hdfs\", newHDFS)\n}\n" }, { "path": "pkg/object/hdfs_kerberos.go", "content": "//go:build !nohdfs\n// +build !nohdfs\n\n// Copyright 2014 Colin Marc (colinmarc@gmail.com)\n// borrowed from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go\n\npackage object\n\nimport (\n\t\"encoding/base64\"\n\t\"fmt\"\n\t\"github.com/jcmturner/gokrb5/v8/keytab\"\n\t\"os\"\n\t\"os/user\"\n\t\"strings\"\n\n\tkrb \"github.com/jcmturner/gokrb5/v8/client\"\n\t\"github.com/jcmturner/gokrb5/v8/config\"\n\t\"github.com/jcmturner/gokrb5/v8/credentials\"\n)\n\nfunc getKerberosClient() (*krb.Client, error) {\n\tconfigPath := os.Getenv(\"KRB5_CONFIG\")\n\tif configPath == \"\" {\n\t\tconfigPath = \"/etc/krb5.conf\"\n\t}\n\n\tcfg, err := config.Load(configPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Try to authenticate with keytab file first.\n\tkeytabPath := os.Getenv(\"KRB5KEYTAB\")\n\tkeytabBase64 := os.Getenv(\"KRB5KEYTAB_BASE64\")\n\tprincipal := os.Getenv(\"KRB5PRINCIPAL\")\n\n\tvar kt *keytab.Keytab\n\tif keytabBase64 != \"\" {\n\t\tdecodedKeytab, err := base64.StdEncoding.DecodeString(keytabBase64)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding Base64 encoded data %s\", err)\n\t\t}\n\t\tkt = new(keytab.Keytab)\n\t\terr = kt.Unmarshal(decodedKeytab)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if keytabPath != \"\" {\n\t\tkt, err = keytab.Load(keytabPath)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tif kt != nil {\n\t\t// e.g. KRB5PRINCIPAL=\"primary/instance@realm\"\n\t\tsp := strings.Split(principal, \"@\")\n\t\tif len(sp) != 2 {\n\t\t\treturn nil, fmt.Errorf(\"unusable kerberos principal: %s\", principal)\n\t\t}\n\t\tusername, realm := sp[0], sp[1]\n\t\tlogger.Infof(\"username: %s, realm: %s\", username, realm)\n\t\tclient := krb.NewWithKeytab(username, realm, kt, cfg)\n\t\treturn client, nil\n\t}\n\n\t// Determine the ccache location from the environment, falling back to the\n\t// default location.\n\tccachePath := os.Getenv(\"KRB5CCNAME\")\n\tif strings.Contains(ccachePath, \":\") {\n\t\tif strings.HasPrefix(ccachePath, \"FILE:\") {\n\t\t\tccachePath = strings.SplitN(ccachePath, \":\", 2)[1]\n\t\t} else {\n\t\t\treturn nil, fmt.Errorf(\"unusable ccache: %s\", ccachePath)\n\t\t}\n\t} else if ccachePath == \"\" {\n\t\tu, err := user.Current()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tccachePath = fmt.Sprintf(\"/tmp/krb5cc_%s\", u.Uid)\n\t}\n\n\tccache, err := credentials.LoadCCache(ccachePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tclient, err := krb.NewFromCCache(ccache, cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn client, nil\n}\n" }, { "path": "pkg/object/ibmcos.go", "content": "//go:build !noibmcos\n// +build !noibmcos\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/IBM/ibm-cos-sdk-go/aws\"\n\t\"github.com/IBM/ibm-cos-sdk-go/aws/awserr\"\n\t\"github.com/IBM/ibm-cos-sdk-go/aws/credentials/ibmiam\"\n\t\"github.com/IBM/ibm-cos-sdk-go/aws/request\"\n\t\"github.com/IBM/ibm-cos-sdk-go/aws/session\"\n\t\"github.com/IBM/ibm-cos-sdk-go/service/s3\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n)\n\ntype ibmcos struct {\n\tbucket string\n\ts3 *s3.S3\n\tsc string\n}\n\nfunc (s *ibmcos) String() string {\n\treturn fmt.Sprintf(\"ibmcos://%s/\", s.bucket)\n}\n\nfunc (s *ibmcos) Create(ctx context.Context) error {\n\tinput := &s3.CreateBucketInput{Bucket: &s.bucket}\n\t// https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-classes&code=go\n\tif s.sc != \"\" {\n\t\tinput.CreateBucketConfiguration = &s3.CreateBucketConfiguration{\n\t\t\tLocationConstraint: &s.sc,\n\t\t}\n\t}\n\t_, err := s.s3.CreateBucket(input)\n\tif err != nil && isExists(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (s *ibmcos) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tMinPartSize: 5 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (s *ibmcos) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tparams := &s3.GetObjectInput{Bucket: &s.bucket, Key: &key}\n\tif off > 0 || limit > 0 {\n\t\tvar r string\n\t\tif limit > 0 {\n\t\t\tr = fmt.Sprintf(\"bytes=%d-%d\", off, off+limit-1)\n\t\t} else {\n\t\t\tr = fmt.Sprintf(\"bytes=%d-\", off)\n\t\t}\n\t\tparams.Range = &r\n\t}\n\tvar reqID string\n\tresp, err := s.s3.GetObjectWithContext(ctx, params, request.WithGetResponseHeader(s3RequestIDKey, &reqID))\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(reqID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif resp.StorageClass != nil {\n\t\tattrs.SetStorageClass(*resp.StorageClass)\n\t}\n\treturn resp.Body, nil\n}\n\nfunc (s *ibmcos) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tvar body io.ReadSeeker\n\tif b, ok := in.(io.ReadSeeker); ok {\n\t\tbody = b\n\t} else {\n\t\tdata, err := io.ReadAll(in)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tbody = bytes.NewReader(data)\n\t}\n\tmimeType := utils.GuessMimeType(key)\n\tparams := &s3.PutObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tBody: body,\n\t\tContentType: &mimeType,\n\t}\n\tif s.sc != \"\" {\n\t\tparams.SetStorageClass(s.sc)\n\t}\n\tvar reqID string\n\t_, err := s.s3.PutObjectWithContext(ctx, params, request.WithGetResponseHeader(s3RequestIDKey, &reqID))\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(reqID).SetStorageClass(s.sc)\n\treturn err\n}\n\nfunc (s *ibmcos) Copy(ctx context.Context, dst, src string) error {\n\tsrc = s.bucket + \"/\" + src\n\tparams := &s3.CopyObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &dst,\n\t\tCopySource: &src,\n\t}\n\tif s.sc != \"\" {\n\t\tparams.SetStorageClass(s.sc)\n\t}\n\t_, err := s.s3.CopyObjectWithContext(ctx, params)\n\treturn err\n}\n\nfunc (s *ibmcos) Head(ctx context.Context, key string) (Object, error) {\n\tparam := s3.HeadObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\tr, err := s.s3.HeadObjectWithContext(ctx, ¶m)\n\tif err != nil {\n\t\tif e, ok := err.(awserr.RequestFailure); ok && e.StatusCode() == http.StatusNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tkey,\n\t\t*r.ContentLength,\n\t\t*r.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t*r.StorageClass,\n\t}, nil\n}\n\nfunc (s *ibmcos) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tparam := s3.DeleteObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\tvar reqID string\n\t_, err := s.s3.DeleteObjectWithContext(ctx, ¶m, request.WithGetResponseHeader(s3RequestIDKey, &reqID))\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(reqID)\n\treturn err\n}\n\nfunc (s *ibmcos) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tparam := s3.ListObjectsInput{\n\t\tBucket: &s.bucket,\n\t\tPrefix: &prefix,\n\t\tMarker: &start,\n\t\tMaxKeys: &limit,\n\t\tEncodingType: aws.String(\"url\"),\n\t}\n\tif delimiter != \"\" {\n\t\tparam.Delimiter = &delimiter\n\t}\n\tresp, err := s.s3.ListObjectsWithContext(ctx, ¶m)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(resp.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\to := resp.Contents[i]\n\t\toKey, err := decodeKey(*o.Key, resp.EncodingType)\n\t\tif err != nil {\n\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode key %s\", *o.Key)\n\t\t}\n\t\tobjs[i] = &obj{oKey, *o.Size, *o.LastModified, strings.HasSuffix(oKey, \"/\"), *o.StorageClass}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range resp.CommonPrefixes {\n\t\t\tprefix, err := decodeKey(*p.Prefix, resp.EncodingType)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode commonPrefixes %s\", *p.Prefix)\n\t\t\t}\n\t\t\tobjs = append(objs, &obj{prefix, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, *resp.IsTruncated, *resp.NextMarker, nil\n}\n\nfunc (s *ibmcos) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (s *ibmcos) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tparams := &s3.CreateMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\tif s.sc != \"\" {\n\t\tparams.SetStorageClass(s.sc)\n\t}\n\tresp, err := s.s3.CreateMultipartUploadWithContext(ctx, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: *resp.UploadId, MinPartSize: 5 << 20, MaxCount: 10000}, nil\n}\n\nfunc (s *ibmcos) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\tn := int64(num)\n\tparams := &s3.UploadPartInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t\tBody: bytes.NewReader(body),\n\t\tPartNumber: &n,\n\t}\n\tresp, err := s.s3.UploadPartWithContext(ctx, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: *resp.ETag}, nil\n}\n\nfunc (s *ibmcos) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\treturn nil, notSupported\n}\n\nfunc (s *ibmcos) AbortUpload(ctx context.Context, key string, uploadID string) {\n\tparams := &s3.AbortMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t}\n\t_, _ = s.s3.AbortMultipartUploadWithContext(ctx, params)\n}\n\nfunc (s *ibmcos) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\tvar s3Parts []*s3.CompletedPart\n\tfor i := range parts {\n\t\tn := new(int64)\n\t\t*n = int64(parts[i].Num)\n\t\ts3Parts = append(s3Parts, &s3.CompletedPart{ETag: &parts[i].ETag, PartNumber: n})\n\t}\n\tparams := &s3.CompleteMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t\tMultipartUpload: &s3.CompletedMultipartUpload{Parts: s3Parts},\n\t}\n\t_, err := s.s3.CompleteMultipartUploadWithContext(ctx, params)\n\treturn err\n}\n\nfunc (s *ibmcos) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tinput := &s3.ListMultipartUploadsInput{\n\t\tBucket: aws.String(s.bucket),\n\t\tKeyMarker: aws.String(marker),\n\t}\n\t// FIXME: parsing time \"2018-08-23T12:23:26.046+08:00\" as \"2006-01-02T15:04:05Z\"\n\tresult, err := s.s3.ListMultipartUploadsWithContext(ctx, input)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tparts[i] = &PendingPart{*u.Key, *u.UploadId, *u.Initiated}\n\t}\n\tvar nextMarker string\n\tif result.NextKeyMarker != nil {\n\t\tnextMarker = *result.NextKeyMarker\n\t}\n\treturn parts, nextMarker, nil\n}\n\nfunc (s *ibmcos) SetStorageClass(sc string) error {\n\ts.sc = sc\n\treturn nil\n}\n\nfunc newIBMCOS(endpoint, apiKey, serviceInstanceID, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, _ := url.ParseRequestURI(endpoint)\n\thostParts := strings.Split(uri.Host, \".\")\n\tbucket := hostParts[0]\n\tregion := hostParts[2]\n\tauthEndpoint := \"https://iam.cloud.ibm.com/identity/token\"\n\tserviceEndpoint := \"https://\" + strings.SplitN(uri.Host, \".\", 2)[1]\n\tconf := aws.NewConfig().\n\t\tWithRegion(region).\n\t\tWithEndpoint(serviceEndpoint).\n\t\tWithCredentials(ibmiam.NewStaticCredentials(aws.NewConfig(),\n\t\t\tauthEndpoint, apiKey, serviceInstanceID)).\n\t\tWithS3ForcePathStyle(defaultPathStyle())\n\tsess := session.Must(session.NewSession())\n\tclient := s3.New(sess, conf)\n\treturn &ibmcos{bucket: bucket, s3: client}, nil\n}\n\nfunc init() {\n\tRegister(\"ibmcos\", newIBMCOS)\n}\n" }, { "path": "pkg/object/interface.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"io\"\n\t\"time\"\n)\n\ntype Object interface {\n\tKey() string\n\tSize() int64\n\tMtime() time.Time\n\tIsDir() bool\n\tIsSymlink() bool\n\tStorageClass() string\n}\n\ntype obj struct {\n\tkey string\n\tsize int64\n\tmtime time.Time\n\tisDir bool\n\tsc string\n}\n\nfunc (o *obj) Key() string { return o.key }\nfunc (o *obj) Size() int64 { return o.size }\nfunc (o *obj) Mtime() time.Time { return o.mtime }\nfunc (o *obj) IsDir() bool { return o.isDir }\nfunc (o *obj) IsSymlink() bool { return false }\nfunc (o *obj) StorageClass() string { return o.sc }\n\ntype MultipartUpload struct {\n\tMinPartSize int\n\tMaxCount int\n\tUploadID string\n}\n\ntype Part struct {\n\tNum int\n\tSize int\n\tETag string\n}\n\ntype PendingPart struct {\n\tKey string\n\tUploadID string\n\tCreated time.Time\n}\n\ntype Limits struct {\n\tIsSupportMultipartUpload bool\n\tIsSupportUploadPartCopy bool\n\tMinPartSize int\n\tMaxPartSize int64\n\tMaxPartCount int\n}\n\n// ObjectStorage is the interface for object storage.\n// all of these API should be idempotent.\ntype ObjectStorage interface {\n\t// Description of the object storage.\n\tString() string\n\t// Limits of the object storage.\n\tLimits() Limits\n\t// Create the bucket if not existed.\n\tCreate(ctx context.Context) error\n\t// Get the data for the given object specified by key.\n\tGet(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)\n\t// Put data read from a reader to an object specified by key.\n\tPut(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error\n\t// Copy an object from src to dst.\n\tCopy(ctx context.Context, dst, src string) error\n\t// Delete a object.\n\tDelete(ctx context.Context, key string, getters ...AttrGetter) error\n\n\t// Head returns some information about the object or an error if not found.\n\tHead(ctx context.Context, key string) (Object, error)\n\t// List returns a list of objects using ListObjectV2.\n\tList(ctx context.Context, prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error)\n\t// ListAll returns all the objects as a channel.\n\tListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error)\n\n\t// CreateMultipartUpload starts to upload a large object part by part.\n\tCreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)\n\t// UploadPart upload a part of an object.\n\tUploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error)\n\t// UploadPartCopy Uploads a part by copying data from an existing object as data source.\n\tUploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)\n\t// AbortUpload abort a multipart upload.\n\tAbortUpload(ctx context.Context, key string, uploadID string)\n\t// CompleteUpload finish a multipart upload.\n\tCompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error\n\t// ListUploads lists existing multipart uploads.\n\tListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)\n}\n\ntype Shutdownable interface {\n\tShutdown()\n}\n\nfunc Shutdown(o ObjectStorage) {\n\tfn := func(o ObjectStorage) {\n\t\tif s, ok := o.(Shutdownable); ok {\n\t\t\ts.Shutdown()\n\t\t}\n\t}\n\n\tswitch o := o.(type) {\n\tcase *encrypted:\n\t\tfn(o.ObjectStorage)\n\tcase *withPrefix:\n\t\tfn(o.os)\n\tcase *sharded:\n\t\tfor _, s := range o.stores {\n\t\t\tfn(s)\n\t\t}\n\tdefault:\n\t\tfn(o)\n\t}\n}\n" }, { "path": "pkg/object/ks3.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/ks3sdklib/aws-sdk-go/aws\"\n\t\"github.com/ks3sdklib/aws-sdk-go/aws/awserr\"\n\t\"github.com/ks3sdklib/aws-sdk-go/aws/credentials\"\n\t\"github.com/ks3sdklib/aws-sdk-go/service/s3\"\n)\n\nconst s3StorageClassHdr = \"X-Amz-Storage-Class\"\n\ntype ks3 struct {\n\tbucket string\n\ts3 *s3.S3\n\tsc string\n}\n\nfunc (s *ks3) String() string {\n\treturn fmt.Sprintf(\"ks3://%s/\", s.bucket)\n}\n\nfunc (s *ks3) Create(ctx context.Context) error {\n\t_, err := s.s3.CreateBucketWithContext(ctx, &s3.CreateBucketInput{Bucket: &s.bucket})\n\tif err != nil && isExists(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (s *ks3) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 5 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (s *ks3) Head(ctx context.Context, key string) (Object, error) {\n\tparam := s3.HeadObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\n\tr, err := s.s3.HeadObjectWithContext(ctx, ¶m)\n\tif err != nil {\n\t\tif e, ok := err.(awserr.RequestFailure); ok && e.StatusCode() == http.StatusNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\n\tvar sc string\n\tif val, ok := r.Metadata[s3StorageClassHdr]; ok {\n\t\tsc = *val\n\t} else {\n\t\tsc = \"STANDARD\"\n\t}\n\treturn &obj{\n\t\tkey,\n\t\t*r.ContentLength,\n\t\t*r.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\tsc,\n\t}, nil\n}\n\nfunc (s *ks3) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tparams := &s3.GetObjectInput{Bucket: &s.bucket, Key: &key}\n\tif off > 0 || limit > 0 {\n\t\tvar r string\n\t\tif limit > 0 {\n\t\t\tr = fmt.Sprintf(\"bytes=%d-%d\", off, off+limit-1)\n\t\t} else {\n\t\t\tr = fmt.Sprintf(\"bytes=%d-\", off)\n\t\t}\n\t\tparams.Range = &r\n\t}\n\tresp, err := s.s3.GetObjectWithContext(ctx, params)\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(aws.ToString(resp.Metadata[s3RequestIDKey]))\n\t\tattrs.SetStorageClass(aws.ToString(resp.Metadata[s3StorageClassHdr]))\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn resp.Body, nil\n}\n\nfunc (s *ks3) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tvar body io.ReadSeeker\n\tif b, ok := in.(io.ReadSeeker); ok {\n\t\tbody = b\n\t} else {\n\t\tdata, err := io.ReadAll(in)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tbody = bytes.NewReader(data)\n\t}\n\tmimeType := utils.GuessMimeType(key)\n\tparams := &s3.PutObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tBody: body,\n\t\tContentType: &mimeType,\n\t}\n\tif s.sc != \"\" {\n\t\tparams.StorageClass = aws.String(s.sc)\n\t}\n\tresp, err := s.s3.PutObjectWithContext(ctx, params)\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(aws.ToString(resp.Metadata[s3RequestIDKey])).SetStorageClass(s.sc)\n\t}\n\treturn err\n}\nfunc (s *ks3) Copy(ctx context.Context, dst, src string) error {\n\tsrc = s.bucket + \"/\" + src\n\tparams := &s3.CopyObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &dst,\n\t\tCopySource: &src,\n\t}\n\tif s.sc != \"\" {\n\t\tparams.StorageClass = aws.String(s.sc)\n\t}\n\t_, err := s.s3.CopyObjectWithContext(ctx, params)\n\treturn err\n}\n\nfunc (s *ks3) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tparam := s3.DeleteObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\tresp, err := s.s3.DeleteObjectWithContext(ctx, ¶m)\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(aws.ToString(resp.Metadata[s3RequestIDKey]))\n\t}\n\tif e, ok := err.(awserr.RequestFailure); ok && e.StatusCode() == http.StatusNotFound {\n\t\treturn nil\n\t}\n\treturn err\n}\n\nfunc (s *ks3) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tparam := s3.ListObjectsInput{\n\t\tBucket: &s.bucket,\n\t\tPrefix: &prefix,\n\t\tMarker: &start,\n\t\tMaxKeys: &limit,\n\t\tEncodingType: aws.String(\"url\"),\n\t}\n\tif delimiter != \"\" {\n\t\tparam.Delimiter = &delimiter\n\t}\n\tresp, err := s.s3.ListObjectsWithContext(ctx, ¶m)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(resp.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\to := resp.Contents[i]\n\t\toKey, err := decodeKey(*o.Key, resp.EncodingType)\n\t\tif err != nil {\n\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode key %s\", *o.Key)\n\t\t}\n\t\tobjs[i] = &obj{oKey, *o.Size, *o.LastModified, strings.HasSuffix(oKey, \"/\"), *o.StorageClass}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range resp.CommonPrefixes {\n\t\t\tprefix, err := decodeKey(*p.Prefix, resp.EncodingType)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode commonPrefixes %s\", *p.Prefix)\n\t\t\t}\n\t\t\tobjs = append(objs, &obj{prefix, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\tvar nextMarker string\n\tif resp.NextMarker != nil {\n\t\tnextMarker = *resp.NextMarker\n\t}\n\treturn objs, *resp.IsTruncated, nextMarker, nil\n}\n\nfunc (s *ks3) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (s *ks3) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tparams := &s3.CreateMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\tif s.sc != \"\" {\n\t\tparams.StorageClass = aws.String(s.sc)\n\t}\n\tresp, err := s.s3.CreateMultipartUploadWithContext(ctx, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: *resp.UploadID, MinPartSize: 5 << 20, MaxCount: 10000}, nil\n}\n\nfunc (s *ks3) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\tn := int64(num)\n\tparams := &s3.UploadPartInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadID: &uploadID,\n\t\tBody: bytes.NewReader(body),\n\t\tPartNumber: &n,\n\t}\n\tresp, err := s.s3.UploadPartWithContext(ctx, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: *resp.ETag}, nil\n}\n\nfunc (s *ks3) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tresp, err := s.s3.UploadPartCopyWithContext(ctx, &s3.UploadPartCopyInput{\n\t\tBucket: aws.String(s.bucket),\n\t\tCopySource: aws.String(s.bucket + \"/\" + srcKey),\n\t\tCopySourceRange: aws.String(fmt.Sprintf(\"bytes=%d-%d\", off, off+size-1)),\n\t\tKey: aws.String(key),\n\t\tPartNumber: aws.Long(int64(num)),\n\t\tUploadID: aws.String(uploadID),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: *resp.CopyPartResult.ETag}, nil\n}\n\nfunc (s *ks3) AbortUpload(ctx context.Context, key string, uploadID string) {\n\tparams := &s3.AbortMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadID: &uploadID,\n\t}\n\t_, _ = s.s3.AbortMultipartUploadWithContext(ctx, params)\n}\n\nfunc (s *ks3) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\tvar s3Parts []*s3.CompletedPart\n\tfor i := range parts {\n\t\tn := new(int64)\n\t\t*n = int64(parts[i].Num)\n\t\ts3Parts = append(s3Parts, &s3.CompletedPart{ETag: &parts[i].ETag, PartNumber: n})\n\t}\n\tparams := &s3.CompleteMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadID: &uploadID,\n\t\tMultipartUpload: &s3.CompletedMultipartUpload{Parts: s3Parts},\n\t}\n\t_, err := s.s3.CompleteMultipartUploadWithContext(ctx, params)\n\treturn err\n}\n\nfunc (s *ks3) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tinput := &s3.ListMultipartUploadsInput{\n\t\tBucket: aws.String(s.bucket),\n\t\tKeyMarker: aws.String(marker),\n\t}\n\t// FIXME: parsing time \"2018-08-23T12:23:26.046+08:00\" as \"2006-01-02T15:04:05Z\"\n\tresult, err := s.s3.ListMultipartUploadsWithContext(ctx, input)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tparts[i] = &PendingPart{*u.Key, *u.UploadID, *u.Initiated}\n\t}\n\tvar nextMarker string\n\tif result.NextKeyMarker != nil {\n\t\tnextMarker = *result.NextKeyMarker\n\t}\n\treturn parts, nextMarker, nil\n}\n\nfunc (s *ks3) SetStorageClass(sc string) error {\n\ts.sc = sc\n\treturn nil\n}\n\nvar ks3Regions = map[string]string{\n\t\"cn-beijing\": \"BEIJING\",\n\t\"cn-shanghai\": \"SHANGHAI\",\n\t\"cn-guangzhou\": \"GUANGZHOU\",\n\t\"cn-qingdao\": \"QINGDAO\",\n\t\"jr-beijing\": \"JR_BEIJING\",\n\t\"jr-shanghai\": \"JR_SHANGHAI\",\n\t\"\": \"HANGZHOU\",\n\t\"cn-hk-1\": \"HONGKONG\",\n\t\"rus\": \"RUSSIA\",\n\t\"sgp\": \"SINGAPORE\",\n}\n\nfunc newKS3(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, _ := url.ParseRequestURI(endpoint)\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\thostParts := strings.Split(uri.Host, \".\")\n\tif len(hostParts) < 2 {\n\t\treturn nil, fmt.Errorf(\"invalid endpoint: %s\", endpoint)\n\t}\n\tbucket := hostParts[0]\n\tregion := hostParts[1][3:]\n\tregion = strings.TrimLeft(region, \"-\")\n\tvar pathStyle bool = defaultPathStyle()\n\tif strings.HasSuffix(uri.Host, \"ksyun.com\") || strings.HasSuffix(uri.Host, \"ksyuncs.com\") {\n\t\tregion = strings.TrimSuffix(region, \"-internal\")\n\t\tregion = ks3Regions[region]\n\t\tpathStyle = false\n\t} else if envRegion := os.Getenv(\"AWS_REGION\"); envRegion != \"\" {\n\t\tregion = envRegion\n\t}\n\tif region == \"\" {\n\t\tregion = \"us-east-1\"\n\t}\n\n\tvar err error\n\taccessKey, err = url.PathUnescape(accessKey)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unescape access key: %s\", err)\n\t}\n\tsecretKey, err = url.PathUnescape(secretKey)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unescape secret key: %s\", err)\n\t}\n\tawsConfig := &aws.Config{\n\t\tRegion: region,\n\t\tEndpoint: strings.SplitN(uri.Host, \".\", 2)[1],\n\t\tDisableSSL: !ssl,\n\t\tHTTPClient: httpClient,\n\t\tS3ForcePathStyle: pathStyle,\n\t\tCredentials: credentials.NewStaticCredentials(accessKey, secretKey, token),\n\t}\n\n\treturn &ks3{bucket: bucket, s3: s3.New(awsConfig)}, nil\n}\n\nfunc init() {\n\tRegister(\"ks3\", newKS3)\n}\n" }, { "path": "pkg/object/mem.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype mobj struct {\n\tdata []byte\n\tmtime time.Time\n\tmode os.FileMode\n\towner string\n\tgroup string\n}\n\ntype memStore struct {\n\tsync.Mutex\n\tDefaultObjectStorage\n\tname string\n\tobjects map[string]*mobj\n}\n\nfunc (m *memStore) String() string {\n\treturn fmt.Sprintf(\"mem://%s/\", m.name)\n}\n\nfunc (m *memStore) Head(ctx context.Context, key string) (Object, error) {\n\tm.Lock()\n\tdefer m.Unlock()\n\t// Minimum length is 1.\n\tif key == \"\" {\n\t\treturn nil, errors.New(\"object key cannot be empty\")\n\t}\n\to, ok := m.objects[key]\n\tif !ok {\n\t\treturn nil, os.ErrNotExist\n\t}\n\tf := &file{\n\t\tobj{\n\t\t\tkey,\n\t\t\tint64(len(o.data)),\n\t\t\to.mtime,\n\t\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\t\"\",\n\t\t},\n\t\to.owner,\n\t\to.group,\n\t\to.mode,\n\t\tfalse,\n\t}\n\treturn f, nil\n}\n\nfunc (m *memStore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tm.Lock()\n\tdefer m.Unlock()\n\t// Minimum length is 1.\n\tif key == \"\" {\n\t\treturn nil, errors.New(\"object key cannot be empty\")\n\t}\n\td, ok := m.objects[key]\n\tif !ok {\n\t\treturn nil, errors.New(\"not exists\")\n\t}\n\tif off > int64(len(d.data)) {\n\t\toff = int64(len(d.data))\n\t}\n\tdata := d.data[off:]\n\tif limit > 0 && limit < int64(len(data)) {\n\t\tdata = data[:limit]\n\t}\n\treturn io.NopCloser(bytes.NewBuffer(data)), nil\n}\n\nfunc (m *memStore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tm.Lock()\n\tdefer m.Unlock()\n\t// Minimum length is 1.\n\tif key == \"\" {\n\t\treturn errors.New(\"object key cannot be empty\")\n\t}\n\t_, ok := m.objects[key]\n\tif ok {\n\t\tlogger.Debugf(\"overwrite %s\", key)\n\t}\n\tdata, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\tm.objects[key] = &mobj{data: data, mtime: time.Now()}\n\treturn nil\n}\n\nfunc (m *memStore) Copy(ctx context.Context, dst, src string) error {\n\td, err := m.Get(ctx, src, 0, -1)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn m.Put(ctx, dst, d)\n}\n\nfunc (m *memStore) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tm.Lock()\n\tdefer m.Unlock()\n\tdelete(m.objects, key)\n\treturn nil\n}\n\nfunc (m *memStore) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tobjs := make([]Object, 0)\n\tcommonPrefixsMap := make(map[string]bool, 0)\n\tfor k := range m.objects {\n\t\tif strings.HasPrefix(k, prefix) && k > marker {\n\t\t\to := m.objects[k]\n\t\t\tif delimiter != \"\" {\n\t\t\t\tremainString := strings.TrimPrefix(k, prefix)\n\t\t\t\tif pos := strings.Index(remainString, delimiter); pos != -1 {\n\t\t\t\t\tcommonPrefix := remainString[0 : pos+1]\n\t\t\t\t\tif _, ok := commonPrefixsMap[commonPrefix]; ok {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tf := &file{\n\t\t\t\t\t\tobj{\n\t\t\t\t\t\t\tprefix + commonPrefix,\n\t\t\t\t\t\t\t0,\n\t\t\t\t\t\t\ttime.Unix(0, 0),\n\t\t\t\t\t\t\tstrings.HasSuffix(commonPrefix, \"/\"),\n\t\t\t\t\t\t\t\"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\to.owner,\n\t\t\t\t\t\to.group,\n\t\t\t\t\t\to.mode,\n\t\t\t\t\t\tfalse,\n\t\t\t\t\t}\n\t\t\t\t\tobjs = append(objs, f)\n\t\t\t\t\tcommonPrefixsMap[commonPrefix] = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tf := &file{\n\t\t\t\tobj{\n\t\t\t\t\tk,\n\t\t\t\t\tint64(len(o.data)),\n\t\t\t\t\to.mtime,\n\t\t\t\t\tstrings.HasSuffix(k, \"/\"),\n\t\t\t\t\t\"\",\n\t\t\t\t},\n\t\t\t\to.owner,\n\t\t\t\to.group,\n\t\t\t\to.mode,\n\t\t\t\tfalse,\n\t\t\t}\n\t\t\tobjs = append(objs, f)\n\t\t}\n\t}\n\tsort.Slice(objs, func(i, j int) bool {\n\t\treturn objs[i].Key() < objs[j].Key()\n\t})\n\tif int64(len(objs)) > limit {\n\t\tobjs = objs[:limit]\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc newMem(endpoint, accesskey, secretkey, token string) (ObjectStorage, error) {\n\tstore := &memStore{name: endpoint}\n\tstore.objects = make(map[string]*mobj)\n\treturn store, nil\n}\n\nfunc init() {\n\tRegister(\"mem\", newMem)\n}\n" }, { "path": "pkg/object/minio.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n)\n\ntype minio struct {\n\ts3client\n}\n\nfunc (m *minio) String() string {\n\tif m.s3.Options().BaseEndpoint != nil {\n\t\tendpoint := *m.s3.Options().BaseEndpoint\n\t\tif idx := strings.Index(endpoint, \"://\"); idx >= 0 {\n\t\t\tendpoint = endpoint[idx+3:]\n\t\t}\n\t\treturn fmt.Sprintf(\"minio://%s/%s/\", endpoint, m.bucket)\n\t}\n\treturn fmt.Sprintf(\"minio://%s/\", m.bucket)\n}\n\nfunc (m *minio) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 5 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc newMinio(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"http://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\tregion := uri.Query().Get(\"region\")\n\tif region == \"\" {\n\t\tregion = os.Getenv(\"MINIO_REGION\")\n\t}\n\tif region == \"\" {\n\t\tregion = awsDefaultRegion\n\t}\n\tif accessKey == \"\" {\n\t\taccessKey = os.Getenv(\"MINIO_ACCESS_KEY\")\n\t}\n\tif secretKey == \"\" {\n\t\tsecretKey = os.Getenv(\"MINIO_SECRET_KEY\")\n\t}\n\tvar cfg aws.Config\n\tif accessKey != \"\" {\n\t\tcfg, err = config.LoadDefaultConfig(ctx,\n\t\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\t} else {\n\t\tcfg, err = config.LoadDefaultConfig(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to load config: %s\", err)\n\t}\n\tclient := s3.NewFromConfig(cfg, func(options *s3.Options) {\n\t\toptions.Region = region\n\t\toptions.BaseEndpoint = aws.String(uri.Scheme + \"://\" + uri.Host)\n\t\toptions.EndpointOptions.DisableHTTPS = !ssl\n\t\toptions.UsePathStyle = defaultPathStyle()\n\t\toptions.HTTPClient = httpClient\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\tif len(uri.Path) < 2 {\n\t\treturn nil, fmt.Errorf(\"no bucket name provided in %s\", endpoint)\n\t}\n\tbucket := uri.Path[1:]\n\tif strings.Contains(bucket, \"/\") && strings.HasPrefix(bucket, \"minio/\") {\n\t\tbucket = bucket[len(\"minio/\"):]\n\t}\n\tbucket = strings.Split(bucket, \"/\")[0]\n\treturn &minio{s3client{bucket: bucket, s3: client, region: region}}, nil\n}\n\nfunc init() {\n\tRegister(\"minio\", newMinio)\n}\n" }, { "path": "pkg/object/nfs.go", "content": "//go:build !nonfs\n// +build !nonfs\n\n/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/user\"\n\t\"path\"\n\t\"sort\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/vmware/go-nfs-client/nfs\"\n\t\"github.com/vmware/go-nfs-client/nfs/rpc\"\n)\n\nvar _ ObjectStorage = (*nfsStore)(nil)\n\ntype nfsStore struct {\n\tDefaultObjectStorage\n\tusername string\n\thost string\n\troot string\n\tfmode os.FileMode\n\tdmode os.FileMode\n\n\ttarget *nfs.Target\n}\n\ntype nfsEntry struct {\n\t*nfs.EntryPlus\n\tname string\n\tfi os.FileInfo\n\tisSymlink bool\n}\n\nfunc (e *nfsEntry) Name() string {\n\treturn e.name\n}\n\nfunc (e *nfsEntry) Size() int64 {\n\tif e.fi != nil {\n\t\treturn e.fi.Size()\n\t}\n\treturn e.EntryPlus.Size()\n}\n\nfunc (e *nfsEntry) Info() (os.FileInfo, error) {\n\tif e.fi != nil {\n\t\treturn e.fi, nil\n\t}\n\treturn e.EntryPlus, nil\n}\n\nfunc (e *nfsEntry) IsDir() bool {\n\tif e.fi != nil {\n\t\treturn e.fi.IsDir()\n\t}\n\treturn e.EntryPlus.IsDir()\n}\n\nfunc (n *nfsStore) String() string {\n\treturn fmt.Sprintf(\"nfs://%s@%s:%s\", n.username, n.host, n.root)\n}\n\nfunc (n *nfsStore) path(key string) string {\n\tif key == \"\" {\n\t\treturn \"./\"\n\t}\n\treturn key\n}\n\nfunc (n *nfsStore) Head(ctx context.Context, key string) (Object, error) {\n\tp := n.path(key)\n\tfi, _, err := n.target.Lookup(p)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif attr, ok := fi.(*nfs.Fattr); ok && attr.Type == nfs.NF3Lnk {\n\t\tsrc, err := n.Readlink(p)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdir, _ := path.Split(p)\n\t\tff, err := n.Head(ctx, path.Join(dir, src))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif f2, ok := ff.(*file); ok {\n\t\t\tf2.isSymlink = true\n\t\t}\n\t\treturn ff, nil\n\t}\n\treturn n.fileInfo(key, fi), nil\n}\n\nfunc (n *nfsStore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tp := n.path(key)\n\tif strings.HasSuffix(p, \"/\") {\n\t\treturn io.NopCloser(bytes.NewBuffer([]byte{})), nil\n\t}\n\n\tff, err := n.target.Open(p)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"open %s\", p)\n\t}\n\n\tif limit > 0 {\n\t\treturn &SectionReaderCloser{\n\t\t\tSectionReader: io.NewSectionReader(ff, off, limit),\n\t\t\tCloser: ff,\n\t\t}, nil\n\t}\n\treturn ff, err\n}\n\nfunc (n *nfsStore) mkdirAll(p string) error {\n\tp = strings.TrimSuffix(p, \"/\")\n\tfi, _, err := n.target.Lookup(p)\n\tif err == nil {\n\t\tif fi.IsDir() {\n\t\t\tlogger.Tracef(\"nfs mkdir: path %s already exists\", p)\n\t\t\treturn nil\n\t\t} else {\n\t\t\treturn syscall.ENOTDIR\n\t\t}\n\t} else if !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\n\tdir, _ := path.Split(p)\n\tif dir != \".\" {\n\t\tif err = n.mkdirAll(dir); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t_, err = n.target.Mkdir(p, n.dmode)\n\treturn err\n}\n\nfunc (n *nfsStore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) (err error) {\n\tp := n.path(key)\n\tif strings.HasSuffix(p, dirSuffix) {\n\t\treturn n.mkdirAll(p)\n\t}\n\tvar tmp string\n\tif PutInplace {\n\t\ttmp = p\n\t} else {\n\t\tname := path.Base(p)\n\t\tif len(name) > 200 {\n\t\t\tname = name[:200]\n\t\t}\n\t\ttmp = TmpFilePath(p, name)\n\t\tdefer func() {\n\t\t\tif err != nil {\n\t\t\t\t_ = n.target.Remove(tmp)\n\t\t\t}\n\t\t}()\n\t}\n\t_, err = n.target.Create(tmp, n.fmode)\n\tif os.IsNotExist(err) {\n\t\t_ = n.mkdirAll(path.Dir(p))\n\t\t_, err = n.target.Create(tmp, n.fmode)\n\t}\n\tif os.IsExist(err) {\n\t\t_ = n.target.Remove(tmp)\n\t\t_, err = n.target.Create(tmp, n.fmode)\n\t}\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"create %s\", tmp)\n\t}\n\tff, err := n.target.Open(tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\t_, err = io.CopyBuffer(ff, in, *buf)\n\tif err != nil {\n\t\t_ = ff.Close()\n\t\treturn err\n\t}\n\terr = ff.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !PutInplace {\n\t\t// overwrite dst\n\t\terr = n.target.Rename(tmp, p)\n\t}\n\treturn err\n}\n\nfunc (n *nfsStore) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tpath := n.path(key)\n\tif path == \"./\" {\n\t\treturn nil\n\t}\n\tfi, _, err := n.target.Lookup(path)\n\tif err != nil {\n\t\tif nfs.IsNotDirError(err) || os.IsNotExist(err) {\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\tp := strings.TrimSuffix(path, \"/\")\n\tif fi.IsDir() {\n\t\terr = n.target.RmDir(p)\n\t} else {\n\t\terr = n.target.Remove(p)\n\t}\n\tif err != nil && os.IsNotExist(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (n *nfsStore) fileInfo(key string, fi os.FileInfo) Object {\n\towner, group := n.getOwnerGroup(fi)\n\tisSymlink := fi.Mode()&os.ModeSymlink != 0\n\tff := &file{\n\t\tobj{key, fi.Size(), fi.ModTime(), fi.IsDir(), \"\"},\n\t\towner,\n\t\tgroup,\n\t\tfi.Mode(),\n\t\tisSymlink,\n\t}\n\tif fi.IsDir() {\n\t\tif key != \"\" && !strings.HasSuffix(key, \"/\") {\n\t\t\tff.key += \"/\"\n\t\t}\n\t\tff.size = 0\n\t}\n\treturn ff\n}\n\nfunc (n *nfsStore) readDirSorted(ctx context.Context, dir string, followLink bool) ([]*nfsEntry, error) {\n\to, err := n.Head(ctx, strings.TrimSuffix(dir, \"/\"))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdirname := o.Key()\n\tentries, err := n.target.ReadDirPlus(dirname)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"readdir %s\", dirname)\n\t}\n\tnfsEntries := make([]*nfsEntry, len(entries))\n\tfor i, e := range entries {\n\t\tif e.IsDir() {\n\t\t\tnfsEntries[i] = &nfsEntry{e, e.Name() + dirSuffix, nil, false}\n\t\t} else if e.Attr.Attr.Type == nfs.NF3Lnk && followLink {\n\t\t\t// follow symlink\n\t\t\tnfsEntries[i] = &nfsEntry{e, e.Name(), nil, true}\n\t\t\tsrc, err := n.Readlink(path.Join(dirname, e.Name()))\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"readlink %s: %s\", e.Name(), err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsrcPath := path.Clean(path.Join(dirname, src))\n\t\t\tfi, _, err := n.target.Lookup(srcPath)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"follow link `%s`: lookup `%s`: %s\", path.Join(dirname, e.Name()), srcPath, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tname := e.Name()\n\t\t\tif fi.IsDir() {\n\t\t\t\tname = e.Name() + dirSuffix\n\t\t\t}\n\t\t\tnfsEntries[i] = &nfsEntry{e, name, fi, false}\n\t\t} else {\n\t\t\tnfsEntries[i] = &nfsEntry{e, e.Name(), nil, e.Attr.Attr.Type == nfs.NF3Lnk}\n\t\t}\n\t}\n\tsort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].Name() < nfsEntries[j].Name() })\n\treturn nfsEntries, err\n}\n\nfunc (n *nfsStore) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tdir := prefix\n\tvar objs []Object\n\tif dir != \"\" && !strings.HasSuffix(dir, dirSuffix) {\n\t\tdir = path.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t} else if marker == \"\" {\n\t\tobj, err := n.Head(ctx, dir)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil, false, \"\", nil\n\t\t\t}\n\t\t\treturn nil, false, \"\", err\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\tentries, err := n.readDirSorted(ctx, dir, followLink)\n\tif err != nil {\n\t\tif os.IsPermission(err) || errors.Is(err, nfs.NFS3Error(nfs.NFS3ErrAcces)) {\n\t\t\tlogger.Warnf(\"skip %s: %s\", dir, err)\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\tif os.IsNotExist(err) {\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\tfor _, e := range entries {\n\t\tp := path.Join(dir, e.Name())\n\t\tif e.IsDir() && !e.isSymlink {\n\t\t\tp = p + \"/\"\n\t\t}\n\t\tif !strings.HasPrefix(p, prefix) || (marker != \"\" && p <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\tf := toFile(p, e, e.isSymlink, n.getOwnerGroup)\n\t\tobjs = append(objs, f)\n\t\tif len(objs) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc (n *nfsStore) setAttr(path string, attrSet func(attr *nfs.Fattr) nfs.Sattr3) error {\n\tp := n.path(path)\n\tfi, fh, err := n.target.Lookup(p)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfattr := fi.(*nfs.Fattr)\n\t_, err = n.target.SetAttr(fh, attrSet(fattr))\n\treturn err\n}\n\nfunc (n *nfsStore) Chtimes(path string, mtime time.Time) error {\n\treturn n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 {\n\t\treturn nfs.Sattr3{\n\t\t\tMtime: nfs.SetTime{\n\t\t\t\tSetIt: nfs.SetToClientTime,\n\t\t\t\tTime: nfs.NFS3Time{\n\t\t\t\t\tSeconds: uint32(mtime.Unix()),\n\t\t\t\t\tNseconds: uint32(mtime.Nanosecond()),\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t})\n}\n\nfunc (n *nfsStore) Chmod(path string, mode os.FileMode) error {\n\treturn n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 {\n\t\treturn nfs.Sattr3{\n\t\t\tMode: nfs.SetMode{\n\t\t\t\tSetIt: true,\n\t\t\t\tMode: uint32(mode),\n\t\t\t},\n\t\t}\n\t})\n}\n\nfunc (n *nfsStore) Chown(path string, owner, group string) error {\n\tuid := utils.LookupUser(owner)\n\tgid := utils.LookupGroup(group)\n\tif uid == -1 || gid == -1 {\n\t\treturn fmt.Errorf(\"user(%s):group(%s) not found\", owner, group)\n\t}\n\treturn n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 {\n\t\treturn nfs.Sattr3{\n\t\t\tUID: nfs.SetUID{\n\t\t\t\tSetIt: true,\n\t\t\t\tUID: uint32(uid),\n\t\t\t},\n\t\t\tGID: nfs.SetUID{\n\t\t\t\tSetIt: true,\n\t\t\t\tUID: uint32(gid),\n\t\t\t},\n\t\t}\n\t})\n}\n\nfunc (n *nfsStore) Symlink(oldName, newName string) error {\n\tnewName = strings.TrimRight(newName, \"/\")\n\tp := n.path(newName)\n\tdir := path.Dir(p)\n\tif _, _, err := n.target.Lookup(dir); err != nil && os.IsNotExist(err) {\n\t\tif _, err := n.target.Mkdir(dir, n.dmode); err != nil && !os.IsExist(err) {\n\t\t\treturn errors.Wrapf(err, \"mkdir %s\", dir)\n\t\t}\n\t} else if err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\treturn n.target.Symlink(n.path(oldName), n.path(newName))\n}\n\nfunc (n *nfsStore) Readlink(name string) (string, error) {\n\tf, err := n.target.Open(n.path(name))\n\tif err != nil {\n\t\treturn \"\", errors.Wrapf(err, \"open %s\", name)\n\t}\n\treturn f.Readlink()\n}\n\nfunc (n *nfsStore) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (n *nfsStore) findOwnerGroup(attr *nfs.Fattr) (string, string) {\n\treturn utils.UserName(int(attr.UID)), utils.GroupName(int(attr.GID))\n}\n\nfunc (n *nfsStore) getOwnerGroup(info os.FileInfo) (string, string) {\n\tif st, match := info.(*nfs.Fattr); match {\n\t\treturn n.findOwnerGroup(st)\n\t}\n\tif st, match := info.Sys().(*nfs.Fattr); match {\n\t\treturn n.findOwnerGroup(st)\n\t}\n\treturn \"\", \"\"\n}\n\nfunc newNFSStore(addr, username, pass, token string) (ObjectStorage, error) {\n\tif username == \"\" {\n\t\tu, err := user.Current()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"current user: %s\", err)\n\t\t}\n\t\tusername = u.Username\n\t}\n\tb := strings.Split(addr, \":\")\n\tif len(b) != 2 {\n\t\treturn nil, fmt.Errorf(\"invalid NFS address %s\", addr)\n\t}\n\thost := b[0]\n\tpath := b[1]\n\tmount, err := nfs.DialMount(host, time.Second*3)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to dial MOUNT service %s: %v\", addr, err)\n\t}\n\tauth := rpc.NewAuthUnix(username, uint32(utils.GetCurrentUID()), uint32(utils.GetCurrentGID()))\n\ttarget, err := mount.Mount(path, auth.Auth())\n\ttarget.Config.DirCount = 1 << 17\n\t// Readdir returns up to 1M at a time, even if MaxCount is set larger\n\ttarget.Config.MaxCount = 1 << 20\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to mount %s: %v\", addr, err)\n\t}\n\tumask := utils.GetUmask()\n\treturn &nfsStore{\n\t\tusername: username,\n\t\thost: host,\n\t\troot: path,\n\t\tfmode: os.FileMode(0666 &^ umask),\n\t\tdmode: os.FileMode(0777 &^ umask),\n\t\ttarget: target}, nil\n}\n\nfunc init() {\n\tRegister(\"nfs\", newNFSStore)\n}\n" }, { "path": "pkg/object/object_storage.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nvar ctx = context.Background()\nvar logger = utils.GetLogger(\"juicefs\")\n\nvar UserAgent = \"JuiceFS\"\n\ntype MtimeChanger interface {\n\tChtimes(path string, mtime time.Time) error\n}\n\ntype SupportSymlink interface {\n\t// Symlink create a symbolic link\n\tSymlink(oldName, newName string) error\n\t// Readlink read a symbolic link\n\tReadlink(name string) (string, error)\n}\n\ntype File interface {\n\tObject\n\tOwner() string\n\tGroup() string\n\tMode() os.FileMode\n}\n\ntype onlyWriter struct {\n\tio.Writer\n}\n\ntype file struct {\n\tobj\n\towner string\n\tgroup string\n\tmode os.FileMode\n\tisSymlink bool\n}\n\nfunc (f *file) Owner() string { return f.owner }\nfunc (f *file) Group() string { return f.group }\nfunc (f *file) Mode() os.FileMode { return f.mode }\nfunc (f *file) IsSymlink() bool { return f.isSymlink }\n\nfunc MarshalObject(o Object) map[string]interface{} {\n\tm := make(map[string]interface{})\n\tm[\"key\"] = o.Key()\n\tm[\"size\"] = o.Size()\n\tm[\"mtime\"] = strconv.FormatInt(o.Mtime().UnixNano(), 10)\n\tm[\"isdir\"] = o.IsDir()\n\tif f, ok := o.(File); ok {\n\t\tm[\"mode\"] = f.Mode()\n\t\tm[\"owner\"] = f.Owner()\n\t\tm[\"group\"] = f.Group()\n\t\tm[\"isSymlink\"] = f.IsSymlink()\n\t}\n\treturn m\n}\n\nfunc UnmarshalObject(m map[string]interface{}) Object {\n\tmtime_int64, _ := strconv.ParseInt(m[\"mtime\"].(string), 10, 64)\n\tmtime := time.Unix(0, mtime_int64)\n\to := obj{\n\t\tkey: m[\"key\"].(string),\n\t\tsize: int64(m[\"size\"].(float64)),\n\t\tmtime: mtime,\n\t\tisDir: m[\"isdir\"].(bool)}\n\tif _, ok := m[\"mode\"]; ok {\n\t\tf := file{o, m[\"owner\"].(string), m[\"group\"].(string), os.FileMode(m[\"mode\"].(float64)), m[\"isSymlink\"].(bool)}\n\t\treturn &f\n\t}\n\treturn &o\n}\n\ntype FileSystem interface {\n\tMtimeChanger\n\tChmod(path string, mode os.FileMode) error\n\tChown(path string, owner, group string) error\n}\n\nvar notSupported = utils.ENOTSUP\n\ntype DefaultObjectStorage struct{}\n\nfunc (s DefaultObjectStorage) Create(ctx context.Context) error {\n\treturn nil\n}\n\nfunc (s DefaultObjectStorage) Limits() Limits {\n\treturn Limits{IsSupportMultipartUpload: false, IsSupportUploadPartCopy: false}\n}\n\nfunc (s DefaultObjectStorage) Head(key string) (Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (s DefaultObjectStorage) Copy(ctx context.Context, dst, src string) error {\n\treturn notSupported\n}\n\nfunc (s DefaultObjectStorage) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\treturn nil, notSupported\n}\n\nfunc (s DefaultObjectStorage) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\treturn nil, notSupported\n}\n\nfunc (s DefaultObjectStorage) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\treturn nil, notSupported\n}\n\nfunc (s DefaultObjectStorage) AbortUpload(ctx context.Context, key string, uploadID string) {}\n\nfunc (s DefaultObjectStorage) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\treturn notSupported\n}\n\nfunc (s DefaultObjectStorage) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\treturn nil, \"\", nil\n}\n\nfunc (s DefaultObjectStorage) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\treturn nil, false, \"\", notSupported\n}\n\nfunc (s DefaultObjectStorage) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\ntype Creator func(bucket, accessKey, secretKey, token string) (ObjectStorage, error)\n\nvar storages = make(map[string]Creator)\n\nfunc Register(name string, register Creator) {\n\tstorages[name] = register\n}\n\nfunc CreateStorage(name, endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tf, ok := storages[name]\n\tif ok {\n\t\tlogger.Debugf(\"Creating %s storage at endpoint %s\", name, endpoint)\n\t\treturn f(endpoint, accessKey, secretKey, token)\n\t}\n\treturn nil, fmt.Errorf(\"invalid storage: %s\", name)\n}\n\nvar bufPool = sync.Pool{\n\tNew: func() interface{} {\n\t\t// Default io.Copy uses 32KB buffer, here we choose a larger one (1MiB io-size increases throughput by ~20%)\n\t\tbuf := make([]byte, 1<<20)\n\t\treturn &buf\n\t},\n}\n\ntype listThread struct {\n\tsync.Mutex\n\tcond *utils.Cond\n\tready bool\n\terr error\n\tentries []Object\n\tnextToken string\n\thasMore bool\n}\n\nfunc (l *listThread) reset() {\n\tl.err = nil\n\tl.entries = nil\n\tl.nextToken = \"\"\n\tl.hasMore = false\n}\n\nfunc ListAllWithDelimiter(ctx context.Context, store ObjectStorage, prefix, start, end string, followLink bool) (<-chan Object, error) {\n\tentries, _, _, err := store.List(ctx, prefix, start, \"\", \"/\", 1e9, followLink)\n\tif err != nil {\n\t\tlogger.Errorf(\"list %s: %s\", prefix, err)\n\t\treturn nil, err\n\t}\n\n\tlisted := make(chan Object, 10240)\n\tvar walk func(string, []Object) error\n\twalk = func(prefix string, entries []Object) error {\n\t\tvar concurrent = 10\n\t\tvar err error\n\t\tthreads := make([]listThread, concurrent)\n\t\tfor c := 0; c < concurrent; c++ {\n\t\t\tt := &threads[c]\n\t\t\tt.cond = utils.NewCond(t)\n\t\t\tgo func(c int) {\n\t\t\t\tfor i := c; i < len(entries); i += concurrent {\n\t\t\t\t\tkey := entries[i].Key()\n\t\t\t\t\tif end != \"\" && key >= end {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tif key < start && !strings.HasPrefix(start, key) {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif !entries[i].IsDir() || key == prefix {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tt.entries, t.hasMore, t.nextToken, t.err = store.List(ctx, key, \"\\x00\", t.nextToken, \"/\", 1000, followLink) // exclude itself\n\t\t\t\t\tt.Lock()\n\t\t\t\t\tt.ready = true\n\t\t\t\t\tt.cond.Signal()\n\t\t\t\t\tfor t.ready {\n\t\t\t\t\t\tt.cond.WaitWithTimeout(time.Second)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tt.Unlock()\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tt.Unlock()\n\t\t\t\t}\n\t\t\t}(c)\n\t\t}\n\n\t\tfor i, e := range entries {\n\t\t\tkey := e.Key()\n\t\t\tif end != \"\" && key >= end {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif key >= start {\n\t\t\t\tlisted <- e\n\t\t\t} else if !strings.HasPrefix(start, key) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !e.IsDir() || key == prefix {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tt := &threads[i%concurrent]\n\t\t\tt.Lock()\n\t\t\tfor !t.ready {\n\t\t\t\tt.cond.WaitWithTimeout(time.Millisecond * 10)\n\t\t\t}\n\t\t\tif t.err != nil {\n\t\t\t\terr = t.err\n\t\t\t\tt.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor t.hasMore {\n\t\t\t\tvar more []Object\n\t\t\t\tstartAfter := t.entries[len(t.entries)-1].Key()\n\t\t\t\tmore, t.hasMore, t.nextToken, t.err = store.List(ctx, key, startAfter, t.nextToken, \"/\", 1e9, followLink)\n\t\t\t\tif t.err != nil {\n\t\t\t\t\terr = t.err\n\t\t\t\t\tt.Unlock()\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tt.entries = append(t.entries, more...)\n\t\t\t}\n\t\t\tt.ready = false\n\t\t\tt.cond.Signal()\n\t\t\tchildren := t.entries\n\t\t\tt.reset()\n\t\t\tt.Unlock()\n\n\t\t\terr = walk(key, children)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\tgo func() {\n\t\tdefer close(listed)\n\t\terr := walk(prefix, entries)\n\t\tif err != nil {\n\t\t\tlisted <- nil\n\t\t}\n\t}()\n\treturn listed, nil\n}\n\nfunc generateListResult(objs []Object, limit int64) ([]Object, bool, string, error) {\n\tvar nextMarker string\n\tif len(objs) > 0 {\n\t\tnextMarker = objs[len(objs)-1].Key()\n\t}\n\treturn objs, len(objs) == int(limit), nextMarker, nil\n}\n\nfunc decodeKey(value string, typ *string) (string, error) {\n\tif typ != nil && *typ == \"url\" {\n\t\treturn url.QueryUnescape(value)\n\t}\n\treturn value, nil\n}\n\nfunc TmpFilePath(parent, name string) string {\n\treturn filepath.Join(filepath.Dir(parent), \".jfs.\"+name+\".tmp.\"+strconv.Itoa(rand.Int()))\n}\n" }, { "path": "pkg/object/object_storage_test.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/rand\"\n\t\"crypto/rsa\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss\"\n\t\"github.com/baidubce/bce-sdk-go/services/bos/api\"\n\t\"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs\"\n\n\t\"github.com/colinmarc/hdfs/v2/hadoopconf\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\n\tblob2 \"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob\"\n\n\t\"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum\"\n\n\t\"github.com/redis/go-redis/v9\"\n)\n\nfunc get(s ObjectStorage, k string, off, limit int64, getters ...AttrGetter) (string, error) {\n\tr, err := s.Get(context.Background(), k, off, limit, getters...)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer r.Close()\n\tdata, err := io.ReadAll(r)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn string(data), nil\n}\n\nfunc listAll(ctx context.Context, s ObjectStorage, prefix, marker string, limit int64, followLink bool) ([]Object, error) {\n\tch, err := ListAll(ctx, s, prefix, marker, followLink, true)\n\tif err == nil {\n\t\tobjs := make([]Object, 0)\n\t\tfor obj := range ch {\n\t\t\tif len(objs) < int(limit) {\n\t\t\t\tobjs = append(objs, obj)\n\t\t\t}\n\t\t}\n\t\treturn objs, nil\n\t}\n\treturn nil, err\n}\n\nfunc setStorageClass(o ObjectStorage) string {\n\tif osc, ok := o.(SupportStorageClass); ok {\n\t\tvar sc = \"STANDARD_IA\"\n\t\tswitch o.(type) {\n\t\tcase *wasb:\n\t\t\tsc = string(blob2.AccessTierCool)\n\t\tcase *gs:\n\t\t\tsc = \"NEARLINE\"\n\t\tcase *ossClient:\n\t\t\tsc = string(oss.StorageClassIA)\n\t\tcase *tosClient:\n\t\t\tsc = string(enum.StorageClassIa)\n\t\tcase *obsClient:\n\t\t\tsc = string(obs.StorageClassStandard)\n\t\tcase *bosclient:\n\t\t\tsc = api.STORAGE_CLASS_STANDARD\n\t\tcase *minio:\n\t\t\tsc = \"REDUCED_REDUNDANCY\"\n\t\tcase *scw:\n\t\t\tsc = \"ONEZONE_IA\" // STANDARD, ONEZONE_IA, GLACIER\n\t\t}\n\t\terr := osc.SetStorageClass(sc)\n\t\tif err != nil {\n\t\t\tsc = \"\"\n\t\t}\n\t\treturn sc\n\t}\n\treturn \"\"\n}\n\n// nolint:errcheck\nfunc testStorage(t *testing.T, s ObjectStorage) {\n\tctx := context.Background()\n\tsc := setStorageClass(s)\n\tif err := s.Create(ctx); err != nil {\n\t\tt.Fatalf(\"Can't create bucket %s: %s\", s, err)\n\t}\n\tif err := s.Create(ctx); err != nil {\n\t\tt.Fatalf(\"err should be nil when creating a bucket with the same name\")\n\t}\n\tprefix := \"unit-test/\"\n\ts = WithPrefix(s, prefix)\n\tdefer func() {\n\t\tif err := s.Delete(ctx, \"test\"); err != nil {\n\t\t\tt.Fatalf(\"delete failed: %s\", err)\n\t\t}\n\t}()\n\tall, err := listAll(ctx, s, \"\", \"\", 10000, true)\n\tvar dels []string\n\tfor _, object := range all {\n\t\tdels = append(dels, object.Key())\n\t}\n\tfor i := len(dels) - 1; i >= 0; i-- {\n\t\t_ = s.Delete(ctx, dels[i])\n\t}\n\n\tvar scPut string\n\tkey := \"测试编码文件\" + `{\"name\":\"juicefs\"}` + string('\\u001F') + \"%uFF081%uFF09.jpg\"\n\tif err := s.Put(ctx, key, bytes.NewReader(nil), WithStorageClass(&scPut)); err != nil {\n\t\tt.Logf(\"PUT testEncodeFile failed: %s\", err.Error())\n\t} else {\n\t\tif scPut != sc {\n\t\t\tt.Fatalf(\"Storage class should be %q, got %q\", sc, scPut)\n\t\t}\n\t\tif resp, _, _, err := s.List(ctx, \"测试编码文件\", \"\", \"\", \"\", 1, true); err != nil && err != notSupported {\n\t\t\tt.Logf(\"List testEncodeFile Failed: %s\", err)\n\t\t} else if len(resp) == 1 && resp[0].Key() != key {\n\t\t\tt.Logf(\"List testEncodeFile Failed: expect key %s, but got %s\", key, resp[0].Key())\n\t\t}\n\t}\n\t_ = s.Delete(ctx, key)\n\n\t_, err = s.Get(ctx, \"not_exists\", 0, -1)\n\tif err == nil {\n\t\tt.Fatalf(\"Get should failed: %s\", err)\n\t}\n\tif _, err := s.Head(ctx, string(make([]byte, 8<<10))); err == nil {\n\t\tt.Logf(\"Head should failed: %s\", err)\n\t}\n\n\tbr := []byte(\"hello\")\n\tif err := s.Put(ctx, \"test\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\n\tvar scGet string\n\t// get all\n\tif d, e := get(s, \"test\", 0, -1, WithStorageClass(&scGet)); e != nil || d != \"hello\" {\n\t\tt.Fatalf(\"expect hello, but got %v, error: %s\", d, e)\n\t}\n\tif scGet != sc { // Relax me when testing against a storage that doesn't use specified storage class\n\t\tt.Fatalf(\"Storage class should be %q, got %q\", sc, scGet)\n\t}\n\n\tif d, e := get(s, \"test\", 0, 5); e != nil || d != \"hello\" {\n\t\tt.Fatalf(\"expect hello, but got %v, error: %s\", d, e)\n\t}\n\t// get first\n\tif d, e := get(s, \"test\", 0, 1); e != nil || d != \"h\" {\n\t\tt.Fatalf(\"expect h, but got %v, error: %s\", d, e)\n\t}\n\t// get last\n\tif d, e := get(s, \"test\", 4, 1); e != nil || d != \"o\" {\n\t\tt.Fatalf(\"expect o, but got %v, error: %s\", d, e)\n\t}\n\t// get last 3\n\tif d, e := get(s, \"test\", 2, 3); e != nil || d != \"llo\" {\n\t\tt.Fatalf(\"expect llo, but got %v, error: %s\", d, e)\n\t}\n\t// get middle\n\tif d, e := get(s, \"test\", 2, 2); e != nil || d != \"ll\" {\n\t\tt.Fatalf(\"expect ll, but got %v, error: %s\", d, e)\n\t}\n\t// get the end out of range\n\tif d, e := get(s, \"test\", 4, 2); e != nil || d != \"o\" {\n\t\tt.Logf(\"out-of-range get: 'o', but got %v, error: %s\", len(d), e)\n\t}\n\t// get the off out of range\n\tif d, e := get(s, \"test\", 6, 2); e != nil || d != \"\" {\n\t\tt.Logf(\"out-of-range get: '', but got %v, error: %s\", len(d), e)\n\t}\n\tswitch s.(*withPrefix).os.(type) {\n\tcase FileSystem:\n\t\tobjs, err2 := listAll(ctx, s, \"\", \"\", 2, true)\n\t\tif err2 == nil {\n\t\t\tif len(objs) != 2 {\n\t\t\t\tt.Fatalf(\"List should return 2 keys, but got %d\", len(objs))\n\t\t\t}\n\t\t\tif objs[0].Key() != \"\" {\n\t\t\t\tt.Fatalf(\"First key should be empty string, but got %s\", objs[0].Key())\n\t\t\t}\n\t\t\tif objs[0].Size() != 0 {\n\t\t\t\tt.Fatalf(\"First object size should be 0, but got %d\", objs[0].Size())\n\t\t\t}\n\t\t\tif objs[1].Key() != \"test\" {\n\t\t\t\tt.Fatalf(\"Second key should be test, but got %s\", objs[1].Key())\n\t\t\t}\n\t\t\tif !strings.Contains(s.String(), \"encrypted\") && objs[1].Size() != 5 {\n\t\t\t\tt.Fatalf(\"Size of first key shold be 5, but got %v\", objs[1].Size())\n\t\t\t}\n\t\t\tnow := time.Now()\n\t\t\tif objs[1].Mtime().Before(now.Add(-30*time.Second)) || objs[1].Mtime().After(now.Add(time.Second*30)) {\n\t\t\t\tt.Fatalf(\"Mtime of key should be within 30 seconds, but got %s\", objs[1].Mtime().Sub(now))\n\t\t\t}\n\t\t} else {\n\t\t\tt.Fatalf(\"list failed: %s\", err2.Error())\n\t\t}\n\n\t\tobjs, err2 = listAll(ctx, s, \"\", \"test2\", 1, true)\n\t\tif err2 != nil {\n\t\t\tt.Fatalf(\"list3 failed: %s\", err2.Error())\n\t\t} else if len(objs) != 0 {\n\t\t\tt.Fatalf(\"list3 should not return anything, but got %d\", len(objs))\n\t\t}\n\tdefault:\n\t\tobjs, err2 := listAll(ctx, s, \"\", \"\", 1, true)\n\t\tif err2 == nil {\n\t\t\tif len(objs) != 1 {\n\t\t\t\tt.Fatalf(\"List should return 1 keys, but got %d\", len(objs))\n\t\t\t}\n\t\t\tif objs[0].Key() != \"test\" {\n\t\t\t\tt.Fatalf(\"First key should be test, but got %s\", objs[0].Key())\n\t\t\t}\n\t\t\tif !strings.Contains(s.String(), \"encrypted\") && objs[0].Size() != 5 {\n\t\t\t\tt.Fatalf(\"Size of first key shold be 5, but got %v\", objs[0].Size())\n\t\t\t}\n\t\t\tnow := time.Now()\n\t\t\tif objs[0].Mtime().Before(now.Add(-30*time.Second)) || objs[0].Mtime().After(now.Add(time.Second*30)) {\n\t\t\t\tt.Fatalf(\"Mtime of key should be within 30 seconds, but got %s\", objs[0].Mtime().Sub(now))\n\t\t\t}\n\t\t} else {\n\t\t\tt.Fatalf(\"list failed: %s\", err2.Error())\n\t\t}\n\n\t\tobjs, err2 = listAll(ctx, s, \"\", \"test2\", 1, true)\n\t\tif err2 != nil {\n\t\t\tt.Fatalf(\"list3 failed: %s\", err2.Error())\n\t\t} else if len(objs) != 0 {\n\t\t\tt.Fatalf(\"list3 should not return anything, but got %d\", len(objs))\n\t\t}\n\t}\n\n\tdefer s.Delete(ctx, \"a/\")\n\tdefer s.Delete(ctx, \"a/a\")\n\tif err := s.Put(ctx, \"a/a\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\tdefer s.Delete(ctx, \"a/a1\")\n\tif err := s.Put(ctx, \"a/a1\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\tdefer s.Delete(ctx, \"b/\")\n\tdefer s.Delete(ctx, \"b/b\")\n\tif err := s.Put(ctx, \"b/b\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\tdefer s.Delete(ctx, \"b/b1\")\n\tif err := s.Put(ctx, \"b/b1\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\tdefer s.Delete(ctx, \"c/\")\n\t//tikv will appear empty value is not supported\n\tif err1 := s.Put(ctx, \"c/\", bytes.NewReader(nil)); err1 != nil {\n\t\t//minio will appear XMinioObjectExistsAsDirectory: Object name already exists as a directory. status code: 409\n\t\tif err2 := s.Put(ctx, \"c/\", bytes.NewReader(br)); err2 != nil {\n\t\t\tt.Fatalf(\"PUT failed err1: %s, err2: %s\", err1.Error(), err2.Error())\n\t\t}\n\t}\n\tdefer s.Delete(ctx, \"a1\")\n\tif err := s.Put(ctx, \"a1\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\tdefer s.Delete(ctx, \"a/b/c/d/e/f\")\n\tif err := s.Put(ctx, \"a/b/c/d/e/f\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\n\tbr = []byte(\"hello2\")\n\tif err := s.Put(ctx, \"a1\", bytes.NewReader(br)); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t}\n\n\tif obs, more, nextMarker, err := s.List(ctx, \"\", \"\", \"\", \"/\", 4, true); err != nil {\n\t\tif !errors.Is(err, notSupported) {\n\t\t\tt.Fatalf(\"list: %s\", err)\n\t\t} else {\n\t\t\tt.Logf(\"list is not supported\")\n\t\t}\n\t} else {\n\t\tif _, ok := s.(*withPrefix).os.(FileSystem); !ok {\n\t\t\tkeys := []string{\"a/\", \"a1\", \"b/\", \"c/\"}\n\t\t\tif len(obs) != 4 {\n\t\t\t\tt.Fatalf(\"list should return 4 results but got %d\", len(obs))\n\t\t\t}\n\t\t\tfor i, o := range obs {\n\t\t\t\tif o.Key() != keys[i] {\n\t\t\t\t\tt.Fatalf(\"should get key %s but got %s\", keys[i], o.Key())\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !more {\n\t\t\t\tt.Fatalf(\"should have more results\")\n\t\t\t}\n\t\t\tif nextMarker == \"\" {\n\t\t\t\tt.Fatalf(\"next marker should not be empty\")\n\t\t\t}\n\t\t\tobs, more, nextMarker, err = s.List(ctx, \"\", obs[len(obs)-1].Key(), nextMarker, \"/\", 4, true)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"list with marker: %s\", err)\n\t\t\t}\n\t\t\tif len(obs) != 1 {\n\t\t\t\tt.Fatalf(\"list should return 1 results but got %d\", len(obs))\n\t\t\t}\n\t\t\tif obs[0].Key() != \"test\" {\n\t\t\t\tt.Fatalf(\"should get key test but got %s\", obs[0].Key())\n\t\t\t}\n\t\t\t_, more, nextMarker, err = s.List(ctx, \"\", obs[len(obs)-1].Key(), nextMarker, \"/\", 4, true)\n\t\t\tif more {\n\t\t\t\tt.Fatalf(\"should no more results\")\n\t\t\t}\n\t\t\tif nextMarker != \"\" {\n\t\t\t\tt.Fatalf(\"next marker should not be empty\")\n\t\t\t}\n\t\t}\n\t}\n\n\tif obs, _, _, err := s.List(ctx, \"\", \"\", \"\", \"/\", 10, true); err != nil {\n\t\tif !errors.Is(err, notSupported) {\n\t\t\tt.Fatalf(\"list with delimiter: %s\", err)\n\t\t} else {\n\t\t\tt.Logf(\"list with delimiter is not supported\")\n\t\t}\n\t} else {\n\t\tswitch s.(*withPrefix).os.(type) {\n\t\tcase FileSystem:\n\t\t\tif len(obs) == 0 || obs[0].Key() != \"\" {\n\t\t\t\tt.Fatalf(\"list should return itself\")\n\t\t\t} else {\n\t\t\t\tobs = obs[1:] // ignore itself\n\t\t\t}\n\t\t}\n\t\tif len(obs) != 5 {\n\t\t\tt.Fatalf(\"list with delimiter should return five results but got %d\", len(obs))\n\t\t}\n\t\tkeys := []string{\"a/\", \"a1\", \"b/\", \"c/\", \"test\"}\n\t\tfor i, o := range obs {\n\t\t\tif o.Key() != keys[i] {\n\t\t\t\tt.Fatalf(\"should get key %s but got %s\", keys[i], o.Key())\n\t\t\t}\n\t\t}\n\t}\n\n\tif obs, _, _, err := s.List(ctx, \"a\", \"\", \"\", \"/\", 10, true); err != nil {\n\t\tif !errors.Is(err, notSupported) {\n\t\t\tt.Fatalf(\"list with delimiter: %s\", err)\n\t\t}\n\t} else {\n\t\tif len(obs) != 2 {\n\t\t\tt.Fatalf(\"list with delimiter should return two results but got %d\", len(obs))\n\t\t}\n\t\tkeys := []string{\"a/\", \"a1\"}\n\t\tfor i, o := range obs {\n\t\t\tif o.Key() != keys[i] {\n\t\t\t\tt.Fatalf(\"should get key %s but got %s\", keys[i], o.Key())\n\t\t\t}\n\t\t}\n\t}\n\n\tif obs, _, _, err := s.List(ctx, \"a/\", \"\", \"\", \"/\", 10, true); err != nil {\n\t\tif !errors.Is(err, notSupported) {\n\t\t\tt.Fatalf(\"list with delimiter: %s\", err)\n\t\t} else {\n\t\t\tt.Logf(\"list with delimiter is not supported\")\n\t\t}\n\t} else {\n\t\tswitch s.(*withPrefix).os.(type) {\n\t\tcase FileSystem:\n\t\t\tif len(obs) == 0 || obs[0].Key() != \"a/\" {\n\t\t\t\tt.Fatalf(\"list should return itself\")\n\t\t\t} else {\n\t\t\t\tobs = obs[1:] // ignore itself\n\t\t\t}\n\t\t}\n\t\tif len(obs) != 3 {\n\t\t\tt.Fatalf(\"list with delimiter should return three results but got %d\", len(obs))\n\t\t}\n\t\tkeys := []string{\"a/a\", \"a/a1\", \"a/b/\"}\n\t\tfor i, o := range obs {\n\t\t\tif o.Key() != keys[i] {\n\t\t\t\tt.Fatalf(\"should get key %s but got %s\", keys[i], o.Key())\n\t\t\t}\n\t\t}\n\t}\n\n\t// test redis cluster list all api\n\tkeyTotal := 100\n\tvar sortedKeys []string\n\tfor i := 0; i < keyTotal; i++ {\n\t\tk := fmt.Sprintf(\"hashKey%d\", i)\n\t\tsortedKeys = append(sortedKeys, k)\n\t\tif err := s.Put(ctx, k, bytes.NewReader(br)); err != nil {\n\t\t\tt.Fatalf(\"PUT failed: %s\", err.Error())\n\t\t}\n\t}\n\tsort.Strings(sortedKeys)\n\tdefer func() {\n\t\tfor i := 0; i < keyTotal; i++ {\n\t\t\t_ = s.Delete(ctx, fmt.Sprintf(\"hashKey%d\", i))\n\t\t}\n\t}()\n\tobjs, err := listAll(ctx, s, \"hashKey\", \"\", int64(keyTotal), true)\n\tif err != nil {\n\t\tt.Fatalf(\"list4 failed: %s\", err.Error())\n\t} else {\n\t\tfor i := 0; i < keyTotal; i++ {\n\t\t\tif objs[i].Key() != sortedKeys[i] {\n\t\t\t\tt.Fatal(\"The result for list4 is incorrect\")\n\t\t\t}\n\t\t\tif sc != \"\" && objs[i].StorageClass() != sc {\n\t\t\t\tt.Fatal(\"storage class is not correct\")\n\t\t\t}\n\t\t}\n\t}\n\n\tf, _ := os.CreateTemp(\"\", \"test\")\n\tf.Write([]byte(\"this is a file\"))\n\tf.Seek(0, 0)\n\tos.Remove(f.Name())\n\tdefer f.Close()\n\tif err := s.Put(ctx, \"file\", f); err != nil {\n\t\tt.Fatalf(\"failed to put from file\")\n\t} else if _, err := s.Head(ctx, \"file\"); err != nil {\n\t\tt.Fatalf(\"file should exists\")\n\t} else {\n\t\tif err := s.Delete(ctx, \"file\"); err != nil {\n\t\t\tt.Fatalf(\"delete failed %s\", err)\n\t\t}\n\t}\n\n\tif _, err := s.Head(ctx, \"not-exist-file\"); !os.IsNotExist(err) {\n\t\tt.Fatal(\"err should be os.ErrNotExist\")\n\t}\n\n\tif o, err := s.Head(ctx, \"test\"); err != nil {\n\t\tt.Fatalf(\"check exists failed: %s\", err.Error())\n\t} else if sc != \"\" && o.StorageClass() != sc {\n\t\tt.Fatalf(\"storage class should be %s but got %s\", sc, o.StorageClass())\n\t}\n\n\tdstKey := \"test-copy\"\n\tdefer s.Delete(ctx, dstKey)\n\terr = s.Copy(ctx, fmt.Sprintf(\"%s%s\", prefix, dstKey), fmt.Sprintf(\"%stest\", prefix))\n\tif err != nil && err != notSupported {\n\t\tt.Fatalf(\"copy failed: %s\", err.Error())\n\t}\n\tif err == nil {\n\t\tif o, err := s.Head(ctx, dstKey); err != nil {\n\t\t\tt.Fatalf(\"check exists failed: %s\", err.Error())\n\t\t} else if sc != \"\" && o.StorageClass() != sc {\n\t\t\tt.Fatalf(\"storage class should be %s but got %s\", sc, o.StorageClass())\n\t\t}\n\t}\n\n\tif err := s.Delete(ctx, \"test\"); err != nil {\n\t\tt.Fatalf(\"delete failed: %s\", err)\n\t}\n\n\tif err := s.Delete(ctx, \"test\"); err != nil {\n\t\tt.Fatalf(\"delete non exists: %v\", err)\n\t}\n\n\tgetMockData := func(seed []byte, idx int) []byte {\n\t\tsize := len(seed)\n\t\tif size == 0 {\n\t\t\treturn nil\n\t\t}\n\t\tcontent := make([]byte, size)\n\t\tif idx == 0 {\n\t\t\tcontent = seed\n\t\t} else {\n\t\t\ti := idx % size\n\t\t\tcopy(content[:size-i], seed[i:size])\n\t\t\tcopy(content[size-i:size], seed[:i])\n\t\t}\n\t\treturn content\n\t}\n\tk := \"large\"\n\tdefer s.Delete(ctx, k)\n\n\tif upload, err := s.CreateMultipartUpload(ctx, k); err == nil {\n\t\ttotal := 3\n\t\tseed := make([]byte, upload.MinPartSize)\n\t\tutils.RandRead(seed)\n\t\tparts := make([]*Part, total)\n\t\tcontent := make([][]byte, total)\n\t\tfor i := 0; i < total; i++ {\n\t\t\tcontent[i] = getMockData(seed, i)\n\t\t}\n\t\tpool := make(chan struct{}, 4)\n\t\terrCh := make(chan error, total)\n\t\tvar wg sync.WaitGroup\n\t\tfor i := 1; i <= total; i++ {\n\t\t\tpool <- struct{}{}\n\t\t\twg.Add(1)\n\t\t\tnum := i\n\t\t\tgo func() {\n\t\t\t\tdefer func() {\n\t\t\t\t\t<-pool\n\t\t\t\t\twg.Done()\n\t\t\t\t}()\n\t\t\t\tparts[num-1], err = s.UploadPart(ctx, k, upload.UploadID, num, content[num-1])\n\t\t\t\tif err != nil {\n\t\t\t\t\terrCh <- fmt.Errorf(\"multipart upload error: %v\", err)\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t\twg.Wait()\n\t\tclose(errCh)\n\n\t\tfor err := range errCh {\n\t\t\tt.Fatalf(\"Test failed: %v\", err)\n\t\t}\n\n\t\t// overwrite the first part\n\t\tfirstPartContent := append(getMockData(seed, 0), getMockData(seed, 0)...)\n\t\tif len(firstPartContent) < int(s.Limits().MaxPartSize) {\n\t\t\tfirstPartContent = getMockData(seed, 0)\n\t\t\tfirstPartContent[0] = 'a'\n\t\t}\n\t\toldPart := parts[0]\n\t\tif parts[0], err = s.UploadPart(ctx, k, upload.UploadID, 1, firstPartContent); err != nil {\n\t\t\tt.Logf(\"overwrite the first part error: %v\", err)\n\t\t\tparts[0] = oldPart\n\t\t} else {\n\t\t\tcontent[0] = firstPartContent\n\t\t}\n\n\t\t// overwrite the last part\n\t\tlastPartContent := []byte(\"hello\")\n\t\toldPart = parts[total-1]\n\t\tif parts[total-1], err = s.UploadPart(ctx, k, upload.UploadID, total, lastPartContent); err != nil {\n\t\t\tt.Logf(\"overwrite the last part error: %v\", err)\n\t\t\tparts[total-1] = oldPart\n\t\t} else {\n\t\t\tcontent[total-1] = lastPartContent\n\t\t}\n\n\t\tif err = s.CompleteUpload(ctx, k, upload.UploadID, parts); err != nil {\n\t\t\tt.Fatalf(\"failed to complete multipart upload: %v\", err)\n\t\t}\n\t\tif meta, err := s.Head(ctx, k); err != nil {\n\t\t\tt.Fatalf(\"failed to head object: %v\", err)\n\t\t} else if sc != \"\" && meta.StorageClass() != sc {\n\t\t\tt.Fatalf(\"storage class should be %s but got %s\", sc, meta.StorageClass())\n\t\t}\n\t\tcheckContent := func(key string, content []byte) {\n\t\t\tr, err := s.Get(ctx, key, 0, -1)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"failed to get multipart upload file: %v\", err)\n\t\t\t}\n\t\t\tdefer r.Close()\n\t\t\tcnt, err := io.ReadAll(r)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"failed to get multipart upload file: %v\", err)\n\t\t\t}\n\t\t\tif !bytes.Equal(cnt, content) {\n\t\t\t\tt.Fatal(\"the content of the multipart upload file is incorrect\")\n\t\t\t}\n\t\t}\n\t\tcheckContent(k, bytes.Join(content, nil))\n\n\t\tif s.Limits().IsSupportUploadPartCopy {\n\t\t\tvar copyUpload *MultipartUpload\n\t\t\tvar dstKey = \"dstUploadPartCopyKey\"\n\t\t\tdefer s.Delete(ctx, dstKey)\n\t\t\tif copyUpload, err = s.CreateMultipartUpload(ctx, dstKey); err != nil {\n\t\t\t\tt.Fatalf(\"failed to create multipart upload: %v\", err)\n\t\t\t}\n\t\t\tcopyParts := make([]*Part, total)\n\t\t\tvar startIdx = 0\n\t\t\tfor i, c := range content {\n\t\t\t\tcopyParts[i], err = s.UploadPartCopy(ctx, dstKey, copyUpload.UploadID, i+1, k, int64(startIdx), int64(len(c)))\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"failed to upload part copy: %v\", err)\n\t\t\t\t}\n\t\t\t\tstartIdx += len(c)\n\t\t\t}\n\t\t\tif err = s.CompleteUpload(ctx, dstKey, copyUpload.UploadID, copyParts); err != nil {\n\t\t\t\tt.Fatalf(\"failed to complete multipart upload: %v\", err)\n\t\t\t}\n\t\t\tcheckContent(dstKey, bytes.Join(content, nil))\n\t\t}\n\t} else {\n\t\tt.Logf(\"%s does not support multipart upload: %s\", s, err.Error())\n\t}\n\n\t// Copy empty objects\n\tdefer func() {\n\t\tif err := s.Delete(ctx, \"empty\"); err != nil {\n\t\t\tt.Logf(\"delete empty file failed: %s\", err)\n\t\t}\n\t}()\n\n\tif err := s.Put(ctx, \"empty\", bytes.NewReader([]byte{})); err != nil {\n\t\tt.Logf(\"PUT empty object failed: %s\", err.Error())\n\t}\n\n\t// Copy `/` suffixed object\n\tdefer func() {\n\t\tif err := s.Delete(ctx, \"slash/\"); err != nil {\n\t\t\tt.Logf(\"delete slash/ failed %s\", err)\n\t\t}\n\t}()\n\tif err := s.Put(ctx, \"slash/\", bytes.NewReader([]byte{})); err != nil {\n\t\tt.Logf(\"PUT `/` suffixed object failed: %s\", err.Error())\n\t}\n}\n\nfunc TestMem(t *testing.T) {\n\tm, _ := newMem(\"\", \"\", \"\", \"\")\n\ttestStorage(t, m)\n}\n\nfunc TestDisk(t *testing.T) {\n\t_ = os.RemoveAll(\"/tmp/abc/\")\n\ts, _ := newDisk(\"/tmp/abc/\", \"\", \"\", \"\")\n\ttestStorage(t, s)\n}\n\nfunc TestQingStor(t *testing.T) { //skip mutate\n\tif os.Getenv(\"QY_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newQingStor(os.Getenv(\"QY_ENDPOINT\"),\n\t\tos.Getenv(\"QY_ACCESS_KEY\"), os.Getenv(\"QY_SECRET_KEY\"), \"\")\n\ttestStorage(t, s)\n\n\t//private cloud\n\tif os.Getenv(\"PRIVATE_QY_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts2, _ := newQingStor(\"http://test.jn1.is.shanhe.com\",\n\t\tos.Getenv(\"PRIVATE_QY_ACCESS_KEY\"), os.Getenv(\"PRIVATE_QY_SECRET_KEY\"), \"\")\n\ttestStorage(t, s2)\n}\n\nfunc TestS3(t *testing.T) { //skip mutate\n\tif os.Getenv(\"AWS_ACCESS_KEY_ID\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newS3(os.Getenv(\"AWS_ENDPOINT\"),\n\t\tos.Getenv(\"AWS_ACCESS_KEY_ID\"),\n\t\tos.Getenv(\"AWS_SECRET_ACCESS_KEY\"),\n\t\tos.Getenv(\"AWS_SESSION_TOKEN\"))\n\ttestStorage(t, s)\n}\n\nfunc TestOracleCompileRegexp(t *testing.T) {\n\tep := \"axntujn0ebj1.compat.objectstorage.ap-singapore-1.oraclecloud.com\"\n\toracleCompile := regexp.MustCompile(oracleCompileRegexp)\n\tif oracleCompile.MatchString(ep) {\n\t\tif submatch := oracleCompile.FindStringSubmatch(ep); len(submatch) >= 2 {\n\t\t\tif submatch[1] != \"ap-singapore-1\" {\n\t\t\t\tt.Fatalf(\"oracle endpoint parse failed\")\n\t\t\t}\n\t\t} else {\n\t\t\tt.Fatalf(\"oracle endpoint parse failed\")\n\t\t}\n\t} else {\n\t\tt.Fatalf(\"oracle endpoint parse failed\")\n\t}\n}\n\nfunc TestOVHCompileRegexp(t *testing.T) {\n\tfor _, ep := range []string{\"s3.gra.cloud.ovh.net\", \"s3.gra.perf.cloud.ovh.net\", \"s3.gra.io.cloud.ovh.net\"} {\n\t\tovhCompile := regexp.MustCompile(OVHCompileRegexp)\n\t\tif ovhCompile.MatchString(ep) {\n\t\t\tif submatch := ovhCompile.FindStringSubmatch(ep); len(submatch) >= 2 {\n\t\t\t\tif submatch[1] != \"gra\" {\n\t\t\t\t\tt.Fatalf(\"ovh endpoint parse failed\")\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"ovh endpoint parse failed\")\n\t\t\t}\n\t\t} else {\n\t\t\tt.Fatalf(\"ovh endpoint parse failed\")\n\t\t}\n\t}\n}\n\nfunc TestOSS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"ALICLOUD_ACCESS_KEY_ID\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newOSS(os.Getenv(\"ALICLOUD_ENDPOINT\"),\n\t\tos.Getenv(\"ALICLOUD_ACCESS_KEY_ID\"),\n\t\tos.Getenv(\"ALICLOUD_ACCESS_KEY_SECRET\"), \"\")\n\ttestStorage(t, s)\n}\n\nfunc TestUFile(t *testing.T) { //skip mutate\n\tif os.Getenv(\"UCLOUD_PUBLIC_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tufile, _ := newUFile(os.Getenv(\"UCLOUD_ENDPOINT\"),\n\t\tos.Getenv(\"UCLOUD_PUBLIC_KEY\"), os.Getenv(\"UCLOUD_PRIVATE_KEY\"), \"\")\n\ttestStorage(t, ufile)\n}\n\nfunc TestGS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"GOOGLE_APPLICATION_CREDENTIALS\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tgs, _ := newGS(os.Getenv(\"GOOGLE_ENDPOINT\"), \"\", \"\", \"\")\n\ttestStorage(t, gs)\n}\n\nfunc TestQiniu(t *testing.T) { //skip mutate\n\tif os.Getenv(\"QINIU_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tqiniu, _ := newQiniu(os.Getenv(\"QINIU_ENDPOINT\"),\n\t\tos.Getenv(\"QINIU_ACCESS_KEY\"), os.Getenv(\"QINIU_SECRET_KEY\"), \"\")\n\ttestStorage(t, qiniu)\n\t//qiniu, _ = newQiniu(\"https://test.cn-north-1-s3.qiniu.com\",\n\t//\tos.Getenv(\"QINIU_ACCESS_KEY\"), os.Getenv(\"QINIU_SECRET_KEY\"))\n\t//testStorage(t, qiniu)\n}\n\nfunc TestKS3(t *testing.T) { //skip mutate\n\tif os.Getenv(\"KS3_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tks3, _ := newKS3(os.Getenv(\"KS3_ENDPOINT\"),\n\t\tos.Getenv(\"KS3_ACCESS_KEY\"), os.Getenv(\"KS3_SECRET_KEY\"), \"\")\n\ttestStorage(t, ks3)\n}\n\nfunc TestCOS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"COS_SECRETID\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tcos, _ := newCOS(\n\t\tos.Getenv(\"COS_ENDPOINT\"),\n\t\tos.Getenv(\"COS_SECRETID\"), os.Getenv(\"COS_SECRETKEY\"), \"\")\n\ttestStorage(t, cos)\n}\n\nfunc TestAzure(t *testing.T) { //skip mutate\n\tif os.Getenv(\"AZURE_STORAGE_ACCOUNT\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\t//https://containersName.core.windows.net\n\tabs, _ := newWasb(os.Getenv(\"AZURE_ENDPOINT\"),\n\t\tos.Getenv(\"AZURE_STORAGE_ACCOUNT\"), os.Getenv(\"AZURE_STORAGE_KEY\"), \"\")\n\ttestStorage(t, abs)\n}\n\nfunc TestB2(t *testing.T) { //skip mutate\n\tif os.Getenv(\"B2_ACCOUNT_ID\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, err := newB2(os.Getenv(\"B2_ENDPOINT\"), os.Getenv(\"B2_ACCOUNT_ID\"), os.Getenv(\"B2_APP_KEY\"), \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create B2: %s\", err)\n\t}\n\ttestStorage(t, b)\n}\n\nfunc TestSpace(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SPACE_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newSpace(os.Getenv(\"SPACE_ENDPOINT\"), os.Getenv(\"SPACE_ACCESS_KEY\"), os.Getenv(\"SPACE_SECRET_KEY\"), \"\")\n\ttestStorage(t, b)\n}\n\nfunc TestBOS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"BDCLOUD_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newBOS(os.Getenv(\"BDCLOUD_ENDPOINT\"),\n\t\tos.Getenv(\"BDCLOUD_ACCESS_KEY\"), os.Getenv(\"BDCLOUD_SECRET_KEY\"), \"\")\n\ttestStorage(t, b)\n}\n\nfunc TestSftp(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SFTP_HOST\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newSftp(os.Getenv(\"SFTP_HOST\"), os.Getenv(\"SFTP_USER\"), os.Getenv(\"SFTP_PASS\"), \"\")\n\ttestStorage(t, b)\n}\n\nfunc TestOBS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"HWCLOUD_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newOBS(os.Getenv(\"HWCLOUD_ENDPOINT\"),\n\t\tos.Getenv(\"HWCLOUD_ACCESS_KEY\"), os.Getenv(\"HWCLOUD_SECRET_KEY\"), \"\")\n\ttestStorage(t, b)\n}\n\nfunc TestNFS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"NFS_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, err := newNFSStore(os.Getenv(\"NFS_ADDR\"), os.Getenv(\"NFS_ACCESS_KEY\"), os.Getenv(\"NFS_SECRET_KEY\"), \"\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestStorage(t, b)\n}\n\nfunc TestHDFS(t *testing.T) { //skip mutate\n\tconf := make(hadoopconf.HadoopConf)\n\tconf[\"dfs.namenode.rpc-address.ns.namenode1\"] = \"hadoop01:8020\"\n\tconf[\"dfs.namenode.rpc-address.ns.namenode2\"] = \"hadoop02:8020\"\n\n\tcheckAddr := func(addr string, expected []string, base string) {\n\t\taddresses, basePath := parseHDFSAddr(addr, conf)\n\t\tsort.Strings(addresses)\n\t\tif !reflect.DeepEqual(addresses, expected) {\n\t\t\tt.Fatalf(\"expected addrs is %+v but got %+v from %s\", expected, addresses, addr)\n\t\t}\n\t\tif basePath != base {\n\t\t\tt.Fatalf(\"expected path is %s but got %s from %s\", base, basePath, addr)\n\t\t}\n\t}\n\n\tcheckAddr(\"hadoop01:8020\", []string{\"hadoop01:8020\"}, \"/\")\n\tcheckAddr(\"hdfs://hadoop01:8020/\", []string{\"hadoop01:8020\"}, \"/\")\n\tcheckAddr(\"hadoop01:8020/user/juicefs/\", []string{\"hadoop01:8020\"}, \"/user/juicefs/\")\n\tcheckAddr(\"hadoop01:8020/user/juicefs\", []string{\"hadoop01:8020\"}, \"/user/juicefs/\")\n\tcheckAddr(\"hdfs://hadoop01:8020/user/juicefs/\", []string{\"hadoop01:8020\"}, \"/user/juicefs/\")\n\n\t// for HA\n\tcheckAddr(\"hadoop01:8020,hadoop02:8020\", []string{\"hadoop01:8020\", \"hadoop02:8020\"}, \"/\")\n\tcheckAddr(\"hadoop01:8020,hadoop02:8020/user/juicefs/\", []string{\"hadoop01:8020\", \"hadoop02:8020\"}, \"/user/juicefs/\")\n\tcheckAddr(\"hdfs://ns/user/juicefs\", []string{\"hadoop01:8020\", \"hadoop02:8020\"}, \"/user/juicefs/\")\n\tcheckAddr(\"ns/user/juicefs/\", []string{\"hadoop01:8020\", \"hadoop02:8020\"}, \"/user/juicefs/\")\n\n\tif os.Getenv(\"HDFS_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tdfs, _ := newHDFS(os.Getenv(\"HDFS_ADDR\"), \"\", \"\", \"\")\n\ttestStorage(t, dfs)\n}\n\nfunc TestOOS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"OOS_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newOOS(os.Getenv(\"OOS_ENDPOINT\"),\n\t\tos.Getenv(\"OOS_ACCESS_KEY\"), os.Getenv(\"OOS_SECRET_KEY\"), \"\")\n\ttestStorage(t, b)\n}\n\nfunc TestScw(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SCW_ACCESS_KEY\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newScw(os.Getenv(\"SCW_ENDPOINT\"), os.Getenv(\"SCW_ACCESS_KEY\"), os.Getenv(\"SCW_SECRET_KEY\"), \"\")\n\ttestStorage(t, b)\n}\n\nfunc TestMinIO(t *testing.T) {\n\tif os.Getenv(\"MINIO_TEST_BUCKET\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tb, _ := newMinio(os.Getenv(\"MINIO_TEST_BUCKET\"), os.Getenv(\"MINIO_ACCESS_KEY\"), os.Getenv(\"MINIO_SECRET_KEY\"), \"\")\n\ttestStorage(t, b)\n}\n\n// func TestUpYun(t *testing.T) {\n// \ts, _ := newUpyun(\"http://jfstest\", \"test\", \"\")\n// \ttestStorage(t, s)\n// }\n\nfunc TestTiKV(t *testing.T) { //skip mutate\n\tif os.Getenv(\"TIKV_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, err := newTiKV(os.Getenv(\"TIKV_ADDR\"), \"\", \"\", \"\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestStorage(t, s)\n}\n\nfunc TestRedis(t *testing.T) {\n\tif os.Getenv(\"REDIS_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\n\topt, _ := redis.ParseURL(os.Getenv(\"REDIS_ADDR\"))\n\trdb := redis.NewClient(opt)\n\t_ = rdb.FlushDB(context.Background())\n\n\ts, err := newRedis(os.Getenv(\"REDIS_ADDR\"), \"\", \"\", \"\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestStorage(t, s)\n}\n\nfunc TestSwift(t *testing.T) { //skip mutate\n\tif os.Getenv(\"SWIFT_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, err := newSwiftOSS(os.Getenv(\"SWIFT_ADDR\"), \"\", \"\", \"\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttestStorage(t, s)\n}\n\nfunc TestWebDAV(t *testing.T) { //skip mutate\n\tif os.Getenv(\"WEBDAV_TEST_BUCKET\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newWebDAV(os.Getenv(\"WEBDAV_TEST_BUCKET\"), \"\", \"\", \"\")\n\ttestStorage(t, s)\n}\n\nfunc TestEncrypted(t *testing.T) {\n\ts, _ := CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tprivkey, _ := rsa.GenerateKey(rand.Reader, 2048)\n\tkc := NewRSAEncryptor(privkey)\n\tdc, _ := NewDataEncryptor(kc, AES256GCM_RSA)\n\tes := NewEncrypted(s, dc)\n\ttestStorage(t, es)\n}\n\nfunc TestMarsharl(t *testing.T) {\n\tctx := context.Background()\n\tif os.Getenv(\"HDFS_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newHDFS(os.Getenv(\"HDFS_ADDR\"), \"\", \"\", \"\")\n\tif err := s.Put(ctx, \"hello\", bytes.NewReader([]byte(\"world\"))); err != nil {\n\t\tt.Fatalf(\"PUT failed: %s\", err)\n\t}\n\tfs := s.(FileSystem)\n\t_ = fs.Chown(\"hello\", \"user\", \"group\")\n\t_ = fs.Chmod(\"hello\", 0764)\n\to, err := s.Head(ctx, \"hello\")\n\tif err != nil {\n\t\tt.Fatalf(\"HEAD failed: %s\", err)\n\t}\n\n\tm := MarshalObject(o)\n\td, _ := json.Marshal(m)\n\tvar m2 map[string]interface{}\n\tif err := json.Unmarshal(d, &m2); err != nil {\n\t\tt.Fatalf(\"unmarshal: %s\", err)\n\t}\n\to2 := UnmarshalObject(m2)\n\tif math.Abs(float64(o2.Mtime().UnixNano()-o.Mtime().UnixNano())) > 1000 {\n\t\tt.Fatalf(\"mtime %s != %s\", o2.Mtime(), o.Mtime())\n\t}\n\to2.(*file).mtime = o.Mtime()\n\tif !reflect.DeepEqual(o, o2) {\n\t\tt.Fatalf(\"%+v != %+v\", o2, o)\n\t}\n}\n\nfunc TestSharding(t *testing.T) {\n\ts, _ := NewSharded(\"mem\", \"%d\", \"\", \"\", \"\", 10)\n\ttestStorage(t, s)\n}\n\nfunc TestSQLite(t *testing.T) {\n\ts, err := newSQLStore(\"sqlite3\", \"/tmp/teststore.db\", \"\", \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create: %s\", err)\n\t}\n\ttestStorage(t, s)\n}\n\nfunc TestPG(t *testing.T) { //skip mutate\n\tif os.Getenv(\"PG_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, err := newSQLStore(\"postgres\", os.Getenv(\"PG_ADDR\"), os.Getenv(\"PG_USER\"), os.Getenv(\"PG_PASSWORD\"))\n\tif err != nil {\n\t\tt.Fatalf(\"create: %s\", err)\n\t}\n\ttestStorage(t, s)\n\n}\nfunc TestPGWithSearchPath(t *testing.T) { //skip mutate\n\t_, err := newSQLStore(\"postgres\", \"127.0.0.1:5432/test?sslmode=disable&search_path=juicefs,public\", \"\", \"\")\n\tif !strings.Contains(err.Error(), \"currently, only one schema is supported in search_path\") {\n\t\tt.Fatalf(\"TestPGWithSearchPath error: %s\", err)\n\t}\n}\n\nfunc TestMySQL(t *testing.T) { //skip mutate\n\tif os.Getenv(\"MYSQL_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, err := newSQLStore(\"mysql\", os.Getenv(\"MYSQL_ADDR\"), os.Getenv(\"MYSQL_USER\"), os.Getenv(\"MYSQL_PASSWORD\"))\n\tif err != nil {\n\t\tt.Fatalf(\"create: %s\", err)\n\t}\n\ttestStorage(t, s)\n}\n\nfunc TestNameString(t *testing.T) {\n\ts, _ := newMem(\"test\", \"\", \"\", \"\")\n\ts = WithPrefix(s, \"a/\")\n\ts = WithPrefix(s, \"b/\")\n\tif s.String() != \"mem://test/a/b/\" {\n\t\tt.Fatalf(\"name with two prefix does not match: %s\", s.String())\n\t}\n}\n\nfunc TestEtcd(t *testing.T) { //skip mutate\n\tif os.Getenv(\"ETCD_ADDR\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newEtcd(os.Getenv(\"ETCD_ADDR\"), \"\", \"\", \"\")\n\ttestStorage(t, s)\n}\n\n//func TestCeph(t *testing.T) {\n//\tif os.Getenv(\"CEPH_ENDPOINT\") == \"\" {\n//\t\tt.SkipNow()\n//\t}\n//\ts, _ := newCeph(os.Getenv(\"CEPH_ENDPOINT\"), os.Getenv(\"CEPH_CLUSTER\"), os.Getenv(\"CEPH_USER\"))\n//\ttestStorage(t, s)\n//}\n\nfunc TestEOS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"EOS_ENDPOINT\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newEos(os.Getenv(\"EOS_ENDPOINT\"), os.Getenv(\"EOS_ACCESS_KEY\"), os.Getenv(\"EOS_SECRET_KEY\"), \"\")\n\ttestStorage(t, s)\n}\n\nfunc TestWASABI(t *testing.T) { //skip mutate\n\tif os.Getenv(\"WASABI_ENDPOINT\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newWasabi(os.Getenv(\"WASABI_ENDPOINT\"), os.Getenv(\"WASABI_ACCESS_KEY\"), os.Getenv(\"WASABI_SECRET_KEY\"), \"\")\n\ttestStorage(t, s)\n}\n\nfunc TestIBMCOS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"IBMCOS_ENDPOINT\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ts, _ := newIBMCOS(os.Getenv(\"IBMCOS_ENDPOINT\"), os.Getenv(\"IBMCOS_ACCESS_KEY\"), os.Getenv(\"IBMCOS_SECRET_KEY\"), \"\")\n\ttestStorage(t, s)\n}\n\nfunc TestTOS(t *testing.T) { //skip mutate\n\tif os.Getenv(\"TOS_ENDPOINT\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\ttos, err := newTOS(os.Getenv(\"TOS_ENDPOINT\"), os.Getenv(\"TOS_ACCESS_KEY\"), os.Getenv(\"TOS_SECRET_KEY\"), \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create: %s\", err)\n\t}\n\ttestStorage(t, tos)\n}\n\nfunc TestDragonfly(t *testing.T) { //skip mutate\n\tif os.Getenv(\"DRAGONFLY_ENDPOINT\") == \"\" {\n\t\tt.SkipNow()\n\t}\n\tdragonfly, err := newDragonfly(os.Getenv(\"DRAGONFLY_ENDPOINT\"), \"\", \"\", \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create: %s\", err)\n\t}\n\ttestStorage(t, dragonfly)\n}\n\nfunc TestCifs(t *testing.T) { //skip mutate\n\tif os.Getenv(\"CIFS_ADDR\") == \"\" {\n\t\tfmt.Println(\"skip CIFS test\")\n\t\tt.SkipNow()\n\t}\n\tcifs, err := newCifs(os.Getenv(\"CIFS_ADDR\"), os.Getenv(\"CIFS_USER\"), os.Getenv(\"CIFS_PASSWORD\"), \"\")\n\tif err != nil {\n\t\tt.Fatalf(\"create: %s\", err)\n\t}\n\ttestStorage(t, cifs)\n}\n\n// func TestBunny(t *testing.T) { //skip mutate\n// \tif os.Getenv(\"BUNNY_ENDPOINT\") == \"\" {\n// \t\tt.SkipNow()\n// \t}\n// \tbunny, err := newBunny(os.Getenv(\"BUNNY_ENDPOINT\"), \"\", os.Getenv(\"BUNNY_SECRET_KEY\"), \"\")\n// \tif err != nil {\n// \t\tt.Fatalf(\"create: %s\", err)\n// \t}\n// \ttestStorage(t, bunny)\n// }\n\nfunc TestMain(m *testing.M) {\n\tif envFile := os.Getenv(\"JUICEFS_ENV_FILE_FOR_TEST\"); envFile != \"\" {\n\t\t// schema: S3 AWS_ENDPOINT=xxxxx\n\t\tif _, err := os.Stat(envFile); err == nil {\n\t\t\tfile, _ := os.ReadFile(envFile)\n\t\t\tfor _, line := range strings.Split(strings.TrimSpace(string(file)), \"\\n\") {\n\t\t\t\tif envkv := strings.SplitN(line, \"=\", 2); len(envkv) == 2 {\n\t\t\t\t\tif err := os.Setenv(envkv[0], envkv[1]); err != nil {\n\t\t\t\t\t\tlogger.Errorf(\"set env %s=%s error\", envkv[0], envkv[1])\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tm.Run()\n}\n" }, { "path": "pkg/object/obs.go", "content": "//go:build !noobs\n// +build !noobs\n\n/*\n * JuiceFS, Copyright 2019 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/md5\"\n\t\"encoding/base64\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"golang.org/x/net/http/httpproxy\"\n)\n\nconst obsDefaultRegion = \"cn-north-1\"\n\ntype obsClient struct {\n\tbucket string\n\tregion string\n\tcheckEtag bool\n\tsc string\n\tc *obs.ObsClient\n}\n\nfunc (s *obsClient) String() string {\n\treturn fmt.Sprintf(\"obs://%s/\", s.bucket)\n}\n\nfunc (s *obsClient) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 100 << 10,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (s *obsClient) Create(ctx context.Context) error {\n\tparams := &obs.CreateBucketInput{}\n\tparams.Bucket = s.bucket\n\tparams.Location = s.region\n\tparams.AvailableZone = \"3az\"\n\tparams.StorageClass = obs.StorageClassType(s.sc)\n\t_, err := s.c.CreateBucket(params)\n\tif err != nil && isExists(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\nfunc getStorageClassStr(sc obs.StorageClassType) string {\n\tif sc == \"\" {\n\t\treturn string(obs.StorageClassStandard)\n\t} else {\n\t\treturn string(sc)\n\t}\n}\nfunc (s *obsClient) Head(ctx context.Context, key string) (Object, error) {\n\tparams := &obs.GetObjectMetadataInput{\n\t\tBucket: s.bucket,\n\t\tKey: key,\n\t}\n\tr, err := s.c.GetObjectMetadata(params)\n\tif err != nil {\n\t\tif e, ok := err.(obs.ObsError); ok && e.BaseModel.StatusCode == http.StatusNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\n\treturn &obj{\n\t\tkey,\n\t\tr.ContentLength,\n\t\tr.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\tgetStorageClassStr(r.StorageClass),\n\t}, nil\n}\n\nfunc (s *obsClient) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tparams := &obs.GetObjectInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = key\n\tvar resp *obs.GetObjectOutput\n\tvar err error\n\trangeStr := getRange(off, limit)\n\tif rangeStr != \"\" {\n\t\tresp, err = s.c.GetObject(params, obs.WithHeader(obs.HEADER_RANGE, []string{rangeStr}))\n\t} else {\n\t\tresp, err = s.c.GetObject(params)\n\t}\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.RequestId).SetStorageClass(getStorageClassStr(resp.StorageClass))\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err = checkGetStatus(resp.StatusCode, rangeStr != \"\"); err != nil {\n\t\t_ = resp.Body.Close()\n\t\treturn nil, err\n\t}\n\treturn resp.Body, nil\n}\n\nfunc (s *obsClient) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tvar body io.ReadSeeker\n\tvar vlen int64\n\tvar sum []byte\n\tif b, ok := in.(io.ReadSeeker); ok {\n\t\tvar err error\n\t\th := md5.New()\n\t\tbuf := bufPool.Get().(*[]byte)\n\t\tdefer bufPool.Put(buf)\n\t\tvlen, err = io.CopyBuffer(h, in, *buf)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err = b.Seek(0, io.SeekStart)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tsum = h.Sum(nil)\n\t\tbody = b\n\t} else {\n\t\tdata, err := io.ReadAll(in)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvlen = int64(len(data))\n\t\ts := md5.Sum(data)\n\t\tsum = s[:]\n\t\tbody = bytes.NewReader(data)\n\t}\n\tmimeType := utils.GuessMimeType(key)\n\tparams := &obs.PutObjectInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = key\n\tparams.Body = body\n\tparams.ContentLength = vlen\n\tparams.ContentMD5 = base64.StdEncoding.EncodeToString(sum[:])\n\tparams.ContentType = mimeType\n\tparams.StorageClass = obs.StorageClassType(s.sc)\n\tresp, err := s.c.PutObject(params)\n\tif err == nil && s.checkEtag && strings.Trim(resp.ETag, \"\\\"\") != obs.Hex(sum) {\n\t\terr = fmt.Errorf(\"unexpected ETag: %s != %s\", strings.Trim(resp.ETag, \"\\\"\"), obs.Hex(sum))\n\t}\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.RequestId).SetStorageClass(getStorageClassStr(resp.StorageClass))\n\t}\n\treturn err\n}\n\nfunc (s *obsClient) Copy(ctx context.Context, dst, src string) error {\n\tparams := &obs.CopyObjectInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = dst\n\tparams.CopySourceBucket = s.bucket\n\tparams.CopySourceKey = src\n\tparams.StorageClass = obs.StorageClassType(s.sc)\n\t_, err := s.c.CopyObject(params)\n\treturn err\n}\n\nfunc (s *obsClient) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tparams := obs.DeleteObjectInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = key\n\tresp, err := s.c.DeleteObject(¶ms)\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.RequestId)\n\t}\n\treturn err\n}\n\nfunc (s *obsClient) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tinput := &obs.ListObjectsInput{\n\t\tBucket: s.bucket,\n\t\tMarker: start,\n\t}\n\tinput.Prefix = prefix\n\tinput.MaxKeys = int(limit)\n\tinput.Delimiter = delimiter\n\tinput.EncodingType = \"url\"\n\tresp, err := s.c.ListObjects(input)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(resp.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\t// Obs SDK listObjects method already decodes the object key.\n\t\to := resp.Contents[i]\n\t\tobjs[i] = &obj{o.Key, o.Size, o.LastModified, strings.HasSuffix(o.Key, \"/\"), string(o.StorageClass)}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range resp.CommonPrefixes {\n\t\t\tprefix, err := obs.UrlDecode(p)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode commonPrefixes %s\", p)\n\t\t\t}\n\t\t\tobjs = append(objs, &obj{prefix, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, resp.IsTruncated, resp.NextMarker, nil\n}\n\nfunc (s *obsClient) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (s *obsClient) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tparams := &obs.InitiateMultipartUploadInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = key\n\tparams.StorageClass = obs.StorageClassType(s.sc)\n\tresp, err := s.c.InitiateMultipartUpload(params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: resp.UploadId, MinPartSize: 5 << 20, MaxCount: 10000}, nil\n}\n\nfunc (s *obsClient) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\tparams := &obs.UploadPartInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = key\n\tparams.UploadId = uploadID\n\tparams.Body = bytes.NewReader(body)\n\tparams.PartNumber = num\n\tparams.PartSize = int64(len(body))\n\tsum := md5.Sum(body)\n\tparams.ContentMD5 = base64.StdEncoding.EncodeToString(sum[:])\n\tresp, err := s.c.UploadPart(params)\n\tif err == nil && s.checkEtag && strings.Trim(resp.ETag, \"\\\"\") != obs.Hex(sum[:]) {\n\t\terr = fmt.Errorf(\"unexpected ETag: %s != %s\", strings.Trim(resp.ETag, \"\\\"\"), obs.Hex(sum[:]))\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: resp.ETag}, err\n}\n\nfunc (s *obsClient) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tresp, err := s.c.CopyPart(&obs.CopyPartInput{\n\t\tBucket: s.bucket,\n\t\tKey: key,\n\t\tUploadId: uploadID,\n\t\tPartNumber: num,\n\t\tCopySourceBucket: s.bucket,\n\t\tCopySourceKey: srcKey,\n\t\tCopySourceRangeStart: off,\n\t\tCopySourceRangeEnd: off + size - 1,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: resp.ETag}, err\n}\n\nfunc (s *obsClient) AbortUpload(ctx context.Context, key string, uploadID string) {\n\tparams := &obs.AbortMultipartUploadInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = key\n\tparams.UploadId = uploadID\n\t_, _ = s.c.AbortMultipartUpload(params)\n}\n\nfunc (s *obsClient) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\tparams := &obs.CompleteMultipartUploadInput{}\n\tparams.Bucket = s.bucket\n\tparams.Key = key\n\tparams.UploadId = uploadID\n\tfor i := range parts {\n\t\tparams.Parts = append(params.Parts, obs.Part{ETag: parts[i].ETag, PartNumber: parts[i].Num})\n\t}\n\t_, err := s.c.CompleteMultipartUpload(params)\n\treturn err\n}\n\nfunc (s *obsClient) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tinput := &obs.ListMultipartUploadsInput{\n\t\tBucket: s.bucket,\n\t\tKeyMarker: marker,\n\t}\n\n\tresult, err := s.c.ListMultipartUploads(input)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tparts[i] = &PendingPart{u.Key, u.UploadId, u.Initiated}\n\t}\n\tvar nextMarker string\n\tif result.NextKeyMarker != \"\" {\n\t\tnextMarker = result.NextKeyMarker\n\t}\n\treturn parts, nextMarker, nil\n}\n\nfunc (s *obsClient) SetStorageClass(sc string) error {\n\ts.sc = sc\n\treturn nil\n}\n\nfunc autoOBSEndpoint(bucketName, accessKey, secretKey, token string) (string, error) {\n\tregion := obsDefaultRegion\n\tif r := os.Getenv(\"HWCLOUD_DEFAULT_REGION\"); r != \"\" {\n\t\tregion = r\n\t}\n\tendpoint := fmt.Sprintf(\"https://obs.%s.myhuaweicloud.com\", region)\n\n\tobsCli, err := obs.New(accessKey, secretKey, endpoint, obs.WithSecurityToken(token))\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer obsCli.Close()\n\n\tresult, err := obsCli.ListBuckets(&obs.ListBucketsInput{QueryLocation: true})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tfor _, bucket := range result.Buckets {\n\t\tif bucket.Name == bucketName {\n\t\t\tlogger.Debugf(\"Get location of bucket %q: %s\", bucketName, bucket.Location)\n\t\t\treturn fmt.Sprintf(\"obs.%s.myhuaweicloud.com\", bucket.Location), nil\n\t\t}\n\t}\n\treturn \"\", fmt.Errorf(\"bucket %q does not exist\", bucketName)\n}\n\nfunc newOBS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"invalid endpoint %s: %q\", endpoint, err)\n\t}\n\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\tbucketName := hostParts[0]\n\tif len(hostParts) > 1 {\n\t\tendpoint = fmt.Sprintf(\"%s://%s\", uri.Scheme, hostParts[1])\n\t}\n\n\tif accessKey == \"\" {\n\t\taccessKey = os.Getenv(\"HWCLOUD_ACCESS_KEY\")\n\t\tsecretKey = os.Getenv(\"HWCLOUD_SECRET_KEY\")\n\t}\n\n\tvar region string\n\tif len(hostParts) == 1 {\n\t\tif endpoint, err = autoOBSEndpoint(bucketName, accessKey, secretKey, token); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"cannot get location of bucket %s: %q\", bucketName, err)\n\t\t}\n\t\tif !strings.HasPrefix(endpoint, \"http\") {\n\t\t\tendpoint = fmt.Sprintf(\"%s://%s\", uri.Scheme, endpoint)\n\t\t}\n\t} else {\n\t\tregion = strings.Split(hostParts[1], \".\")[1]\n\t}\n\n\t// Use proxy setting from environment variables: HTTP_PROXY, HTTPS_PROXY, NO_PROXY\n\tif uri, err = url.ParseRequestURI(endpoint); err != nil {\n\t\treturn nil, fmt.Errorf(\"invalid endpoint %s: %q\", endpoint, err)\n\t}\n\tproxyURL, err := httpproxy.FromEnvironment().ProxyFunc()(uri)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"get proxy url for endpoint: %s error: %q\", endpoint, err)\n\t}\n\tvar urlString string\n\tif proxyURL != nil {\n\t\turlString = proxyURL.String()\n\t}\n\n\t// Empty proxy url string has no effect\n\t// there is a bug in the retry of PUT (did not call Seek(0,0) before retry), so disable the retry here\n\tc, err := obs.New(accessKey, secretKey, endpoint, obs.WithSecurityToken(token),\n\t\tobs.WithProxyUrl(urlString), obs.WithMaxRetryCount(0), obs.WithHttpTransport(httpClient.Transport.(*http.Transport)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"fail to initialize OBS: %q\", err)\n\t}\n\tvar checkEtag bool\n\tif _, err = c.GetBucketEncryption(bucketName); err != nil {\n\t\tif obsError, ok := err.(obs.ObsError); ok && obsError.Code == \"NoSuchEncryptionConfiguration\" {\n\t\t\tcheckEtag = true\n\t\t} else if !ok || obsError.Code != \"NoSuchBucket\" {\n\t\t\tlogger.Warnf(\"get bucket encryption: %q\", err)\n\t\t}\n\t}\n\treturn &obsClient{bucket: bucketName, region: region, checkEtag: checkEtag, c: c}, nil\n}\n\nfunc init() {\n\tRegister(\"obs\", newOBS)\n}\n" }, { "path": "pkg/object/oos.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n)\n\ntype oos struct {\n\ts3client\n}\n\nfunc (s *oos) String() string {\n\treturn fmt.Sprintf(\"oos://%s/\", s.s3client.bucket)\n}\n\nfunc (s *oos) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tMinPartSize: 5 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (s *oos) Create(ctx context.Context) error {\n\t_, _, _, err := s.List(ctx, \"\", \"\", \"\", \"\", 1, true)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"please create bucket %s manually\", s.s3client.bucket)\n\t}\n\treturn err\n}\n\nfunc (s *oos) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\tobjs, hasMore, nextMarker, err := s.s3client.List(ctx, prefix, start, token, delimiter, limit, followLink)\n\tif start != \"\" && len(objs) > 0 && objs[0].Key() == start {\n\t\tobjs = objs[1:]\n\t}\n\treturn objs, hasMore, nextMarker, err\n}\n\nfunc newOOS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\thostParts := strings.Split(uri.Host, \".\")\n\tbucket := hostParts[0]\n\tregion := hostParts[1][4:]\n\tendpoint = uri.Scheme + \"://\" + uri.Host[len(bucket)+1:]\n\tforcePathStyle := !strings.Contains(strings.ToLower(endpoint), \"xstore.ctyun.cn\")\n\n\tawsCfg, err := config.LoadDefaultConfig(ctx,\n\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to load config: %s\", err)\n\t}\n\tclient := s3.NewFromConfig(awsCfg, func(options *s3.Options) {\n\t\toptions.EndpointOptions.DisableHTTPS = !ssl\n\t\toptions.Region = region\n\t\toptions.UsePathStyle = forcePathStyle\n\t\toptions.HTTPClient = httpClient\n\t\toptions.BaseEndpoint = aws.String(endpoint)\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\treturn &oos{s3client{bucket: bucket, s3: client, region: region}}, nil\n}\n\nfunc init() {\n\tRegister(\"oos\", newOOS)\n}\n" }, { "path": "pkg/object/oss.go", "content": "//go:build !nooss\n// +build !nooss\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss\"\n\t\"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials\"\n\topenapicred \"github.com/aliyun/credentials-go/credentials\"\n)\n\nconst ossDefaultRegionID = \"cn-hangzhou\"\n\ntype ossClient struct {\n\tclient *oss.Client\n\tbucket string\n\tsc string\n}\n\nfunc (o *ossClient) String() string {\n\treturn fmt.Sprintf(\"oss://%s/\", o.bucket)\n}\n\nfunc (o *ossClient) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: int(oss.MinPartSize),\n\t\tMaxPartSize: oss.MaxPartSize,\n\t\tMaxPartCount: int(oss.MaxUploadParts),\n\t}\n}\n\nfunc (o *ossClient) Create(ctx context.Context) error {\n\tvar configuration *oss.CreateBucketConfiguration\n\tif o.sc != \"\" {\n\t\tconfiguration = &oss.CreateBucketConfiguration{\n\t\t\tStorageClass: oss.StorageClassType(o.sc),\n\t\t}\n\t}\n\t_, err := o.client.PutBucket(ctx, &oss.PutBucketRequest{\n\t\tBucket: &o.bucket,\n\t\tCreateBucketConfiguration: configuration,\n\t})\n\tif err != nil && isExists(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (o *ossClient) Head(ctx context.Context, key string) (Object, error) {\n\tinfo, err := o.client.HeadObject(ctx, &oss.HeadObjectRequest{\n\t\tBucket: &o.bucket,\n\t\tKey: &key,\n\t})\n\tif err != nil {\n\t\tvar svcErr *oss.ServiceError\n\t\tif errors.As(err, &svcErr) && svcErr.StatusCode == http.StatusNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tkey,\n\t\tinfo.ContentLength,\n\t\toss.ToTime(info.LastModified),\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\toss.ToString(info.StorageClass),\n\t}, nil\n}\n\nfunc (o *ossClient) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (resp io.ReadCloser, err error) {\n\tvar result *oss.GetObjectResult\n\tvar reqId string\n\tvar sc string\n\tresult, err = o.client.GetObject(ctx, &oss.GetObjectRequest{\n\t\tBucket: &o.bucket,\n\t\tKey: &key,\n\t\tRange: oss.HTTPRange{Offset: off, Count: limit}.FormatHTTPRange(),\n\t\tRangeBehavior: oss.Ptr(\"standard\"),\n\t})\n\tif err != nil {\n\t\tvar svcErr *oss.ServiceError\n\t\tif errors.As(err, &svcErr) {\n\t\t\treqId = svcErr.RequestID\n\t\t}\n\t} else {\n\t\treqId = result.ResultCommon.Headers.Get(oss.HeaderOssRequestID)\n\t\tsc = oss.ToString(result.StorageClass)\n\t\tif off > 0 || limit > 0 {\n\t\t\tresp = result.Body\n\t\t} else {\n\t\t\tresp = verifyChecksum(result.Body,\n\t\t\t\tresult.Headers.Get(oss.HeaderOssMetaPrefix+checksumAlgr),\n\t\t\t\tresult.ContentLength)\n\t\t}\n\t}\n\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(reqId)\n\tattrs.SetStorageClass(sc)\n\treturn\n}\n\nfunc (o *ossClient) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\treq := &oss.PutObjectRequest{\n\t\tBucket: &o.bucket,\n\t\tKey: &key,\n\t\tStorageClass: oss.StorageClassType(o.sc),\n\t\tBody: in,\n\t}\n\tif ins, ok := in.(io.ReadSeeker); ok {\n\t\treq.Metadata = make(map[string]string)\n\t\treq.Metadata[oss.HeaderOssMetaPrefix+checksumAlgr] = generateChecksum(ins)\n\t}\n\tvar reqId string\n\tresult, err := o.client.PutObject(ctx, req)\n\tif err != nil {\n\t\tvar svcErr *oss.ServiceError\n\t\tif errors.As(err, &svcErr) {\n\t\t\treqId = svcErr.RequestID\n\t\t}\n\t} else {\n\t\treqId = result.Headers.Get(oss.HeaderOssRequestID)\n\t}\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(reqId).SetStorageClass(o.sc)\n\treturn err\n}\n\nfunc (o *ossClient) Copy(ctx context.Context, dst, src string) error {\n\tvar req = &oss.CopyObjectRequest{\n\t\tSourceBucket: &o.bucket,\n\t\tBucket: &o.bucket,\n\t\tSourceKey: &src,\n\t\tKey: &dst,\n\t\tStorageClass: oss.StorageClassType(o.sc),\n\t}\n\t_, err := o.client.CopyObject(ctx, req)\n\treturn err\n}\n\nfunc (o *ossClient) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tresult, err := o.client.DeleteObject(ctx, &oss.DeleteObjectRequest{\n\t\tBucket: &o.bucket,\n\t\tKey: &key,\n\t})\n\tvar reqId string\n\tif err != nil {\n\t\tvar svcErr *oss.ServiceError\n\t\tif errors.As(err, &svcErr) {\n\t\t\treqId = svcErr.RequestID\n\t\t}\n\t} else {\n\t\treqId = result.Headers.Get(oss.HeaderOssRequestID)\n\t}\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetRequestID(reqId)\n\treturn err\n}\n\nfunc (o *ossClient) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\tresult, err := o.client.ListObjectsV2(ctx, &oss.ListObjectsV2Request{\n\t\tBucket: &o.bucket,\n\t\tPrefix: &prefix,\n\t\tStartAfter: &start,\n\t\tContinuationToken: &token,\n\t\tDelimiter: &delimiter,\n\t\tMaxKeys: int32(limit),\n\t})\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(result.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\to := result.Contents[i]\n\t\tobjs[i] = &obj{oss.ToString(o.Key), o.Size, oss.ToTime(o.LastModified), strings.HasSuffix(oss.ToString(o.Key), \"/\"), oss.ToString(o.StorageClass)}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, o := range result.CommonPrefixes {\n\t\t\tobjs = append(objs, &obj{oss.ToString(o.Prefix), 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, result.IsTruncated, oss.ToString(result.NextContinuationToken), nil\n}\n\nfunc (o *ossClient) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (o *ossClient) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tresult, err := o.client.InitiateMultipartUpload(ctx, &oss.InitiateMultipartUploadRequest{\n\t\tBucket: &o.bucket,\n\t\tKey: &key,\n\t\tStorageClass: oss.StorageClassType(o.sc),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: oss.ToString(result.UploadId), MinPartSize: 4 << 20, MaxCount: 10000}, nil\n}\n\nfunc (o *ossClient) UploadPart(ctx context.Context, key string, uploadID string, num int, data []byte) (*Part, error) {\n\tr, err := o.client.UploadPart(ctx, &oss.UploadPartRequest{\n\t\tBucket: &o.bucket,\n\t\tUploadId: &uploadID,\n\t\tKey: &key,\n\t\tBody: bytes.NewReader(data),\n\t\tPartNumber: int32(num),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: oss.ToString(r.ETag)}, nil\n}\n\nfunc (o *ossClient) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tpartCopy, err := o.client.UploadPartCopy(ctx, &oss.UploadPartCopyRequest{\n\t\tSourceBucket: &o.bucket,\n\t\tBucket: &o.bucket,\n\t\tSourceKey: &srcKey,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t\tPartNumber: int32(num),\n\t\tRange: oss.HTTPRange{Offset: off, Count: size}.FormatHTTPRange(),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: oss.ToString(partCopy.ETag)}, nil\n}\n\nfunc (o *ossClient) AbortUpload(ctx context.Context, key string, uploadID string) {\n\t_, _ = o.client.AbortMultipartUpload(ctx, &oss.AbortMultipartUploadRequest{\n\t\tBucket: &o.bucket,\n\t\tUploadId: &uploadID,\n\t\tKey: &key,\n\t})\n}\n\nfunc (o *ossClient) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\toparts := make([]oss.UploadPart, len(parts))\n\tfor i, p := range parts {\n\t\toparts[i].PartNumber = int32(p.Num)\n\t\toparts[i].ETag = &p.ETag\n\t}\n\t_, err := o.client.CompleteMultipartUpload(ctx, &oss.CompleteMultipartUploadRequest{\n\t\tBucket: &o.bucket,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t\tCompleteMultipartUpload: &oss.CompleteMultipartUpload{\n\t\t\tParts: oparts,\n\t\t},\n\t})\n\treturn err\n}\n\nfunc (o *ossClient) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tresult, err := o.client.ListParts(ctx, &oss.ListPartsRequest{\n\t\tBucket: &o.bucket,\n\t\tKey: &marker,\n\t})\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Parts))\n\tfor i, u := range result.Parts {\n\t\tparts[i] = &PendingPart{oss.ToString(result.Key), oss.ToString(result.UploadId), oss.ToTime(u.LastModified)}\n\t}\n\treturn parts, string(result.NextPartNumberMarker), nil\n}\n\nfunc (o *ossClient) SetStorageClass(sc string) error {\n\to.sc = sc\n\treturn nil\n}\n\nfunc autoOSSEndpoint(bucketName string, provider credentials.CredentialsProvider) (string, error) {\n\tvar err error\n\tregionID := ossDefaultRegionID\n\tif rid := os.Getenv(\"ALICLOUD_REGION_ID\"); rid != \"\" {\n\t\tregionID = rid\n\t}\n\tconfig := oss.NewConfig()\n\tconfig.CredentialsProvider = provider\n\tconfig.Region = ®ionID\n\tclient := oss.NewClient(config)\n\tvar info *oss.GetBucketInfoResult\n\tinfo, err = client.GetBucketInfo(ctx, &oss.GetBucketInfoRequest{\n\t\tBucket: &bucketName,\n\t})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\t// try oss internal endpoint\n\tclient2 := oss.NewClient(oss.NewConfig().\n\t\tWithEndpoint(oss.ToString(info.BucketInfo.IntranetEndpoint)).\n\t\tWithCredentialsProvider(provider).\n\t\tWithRegion(regionID))\n\tif _, err := client2.GetBucketInfo(ctx, &oss.GetBucketInfoRequest{Bucket: &bucketName}); err == nil {\n\t\treturn \"http://\" + oss.ToString(info.BucketInfo.IntranetEndpoint), err\n\t}\n\treturn \"https://\" + oss.ToString(info.BucketInfo.ExtranetEndpoint), nil\n}\n\nfunc newOSS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\tbucketName := hostParts[0]\n\n\tvar domain string\n\tif len(hostParts) > 1 {\n\t\tdomain = uri.Scheme + \"://\" + hostParts[1]\n\t}\n\t// try environment variable\n\tif accessKey == \"\" {\n\t\taccessKey = os.Getenv(\"ALICLOUD_ACCESS_KEY_ID\")\n\t\tsecretKey = os.Getenv(\"ALICLOUD_ACCESS_KEY_SECRET\")\n\t\ttoken = os.Getenv(\"SECURITY_TOKEN\")\n\t}\n\tvar provider credentials.CredentialsProvider\n\tif accessKey == \"\" {\n\t\t// use default credential chain https://github.com/aliyun/credentials-go?tab=readme-ov-file#credential-provider-chain\n\t\tdefaultCred, _ := openapicred.NewCredential(nil)\n\t\tprovider = credentials.CredentialsProviderFunc(func(ctx context.Context) (credentials.Credentials, error) {\n\t\t\t// return the old certificate before its expiration and obtain a new certificate when the old certificate expires\n\t\t\tcred, err := defaultCred.GetCredential()\n\t\t\tif err != nil {\n\t\t\t\treturn credentials.Credentials{}, err\n\t\t\t}\n\t\t\treturn credentials.Credentials{\n\t\t\t\tAccessKeyID: *cred.AccessKeyId,\n\t\t\t\tAccessKeySecret: *cred.AccessKeySecret,\n\t\t\t\tSecurityToken: *cred.SecurityToken,\n\t\t\t}, nil\n\t\t})\n\t} else {\n\t\tprovider = credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)\n\t}\n\n\tif domain == \"\" {\n\t\tif domain, err = autoOSSEndpoint(bucketName, provider); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unable to get endpoint of bucket %s: %s\", bucketName, err)\n\t\t}\n\t\tlogger.Debugf(\"use endpoint %s\", domain)\n\t}\n\tvar regionID string\n\tvar useV4 bool\n\tif regionID = os.Getenv(\"ALICLOUD_REGION_ID\"); regionID == \"\" {\n\t\tindex := strings.Index(domain, \".\")\n\t\tif index <= 0 {\n\t\t\treturn nil, fmt.Errorf(\"invalid endpoint: %q\", domain)\n\t\t}\n\t\tif strings.HasSuffix(domain, \".aliyuncs.com\") {\n\t\t\tif strings.Contains(domain, \".privatelink.\") {\n\t\t\t\t// .oss..privatelink.aliyuncs.com\n\t\t\t\tparts := strings.Split(domain, \".\")\n\t\t\t\tif len(parts) < 3 {\n\t\t\t\t\treturn nil, fmt.Errorf(\"invalid private link endpoint: %q\", domain)\n\t\t\t\t}\n\t\t\t\tregionID = parts[2]\n\t\t\t\tuseV4 = true\n\t\t\t} else {\n\t\t\t\t// oss-.aliyuncs.com\n\t\t\t\t// oss--internal.aliyuncs.com\n\t\t\t\told := strings.TrimPrefix(strings.TrimPrefix(domain[:index], \"http://\"), \"https://\")\n\t\t\t\tregionID = strings.TrimPrefix(old, \"oss-\")\n\t\t\t\tregionID = strings.TrimSuffix(regionID, \"-internal\")\n\t\t\t\tregionID = strings.TrimSuffix(regionID, \"-vpc\")\n\t\t\t\tuseV4 = old != regionID\n\t\t\t}\n\t\t}\n\t}\n\tconfig := oss.LoadDefaultConfig()\n\tconfig.Endpoint = oss.Ptr(domain)\n\tif useV4 {\n\t\tconfig.WithSignatureVersion(oss.SignatureVersionV4)\n\t\tconfig.Region = oss.Ptr(regionID)\n\t} else {\n\t\tconfig.WithSignatureVersion(oss.SignatureVersionV1)\n\t}\n\tconfig.UsePathStyle = oss.Ptr(strings.Contains(domain, \".privatelink.\"))\n\tconfig.RetryMaxAttempts = oss.Ptr(1)\n\tconfig.ConnectTimeout = oss.Ptr(time.Second * 2)\n\tconfig.ReadWriteTimeout = oss.Ptr(time.Second * 5)\n\tenableChecksum := strings.EqualFold(uri.Query().Get(\"disable-checksum\"), \"false\")\n\tif enableChecksum {\n\t\tlogger.Infof(\"default CRC checksum is enabled\")\n\t}\n\tconfig.DisableUploadCRC64Check = oss.Ptr(!enableChecksum)\n\tconfig.DisableDownloadCRC64Check = oss.Ptr(!enableChecksum)\n\tconfig.UserAgent = &UserAgent\n\tconfig.HttpClient = httpClient\n\tconfig.CredentialsProvider = provider\n\tclient := oss.NewClient(config)\n\to := &ossClient{client: client, bucket: bucketName}\n\treturn o, nil\n}\n\nfunc init() {\n\tRegister(\"oss\", newOSS)\n}\n" }, { "path": "pkg/object/prefix.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"time\"\n)\n\ntype withPrefix struct {\n\tos ObjectStorage\n\tprefix string\n}\n\n// WithPrefix return an object storage that add a prefix to keys.\nfunc WithPrefix(os ObjectStorage, prefix string) ObjectStorage {\n\treturn &withPrefix{os, prefix}\n}\n\nfunc (s *withPrefix) SetStorageClass(sc string) error {\n\tif o, ok := s.os.(SupportStorageClass); ok {\n\t\treturn o.SetStorageClass(sc)\n\t}\n\treturn notSupported\n}\n\nfunc (s *withPrefix) Symlink(oldName, newName string) error {\n\tif w, ok := s.os.(SupportSymlink); ok {\n\t\treturn w.Symlink(oldName, s.prefix+newName)\n\t}\n\treturn notSupported\n}\n\nfunc (s *withPrefix) Readlink(name string) (string, error) {\n\tif w, ok := s.os.(SupportSymlink); ok {\n\t\treturn w.Readlink(s.prefix + name)\n\t}\n\treturn \"\", notSupported\n}\n\nfunc (p *withPrefix) String() string {\n\treturn fmt.Sprintf(\"%s%s\", p.os, p.prefix)\n}\n\nfunc (p *withPrefix) Limits() Limits {\n\treturn p.os.Limits()\n}\n\nfunc (p *withPrefix) Create(ctx context.Context) error {\n\treturn p.os.Create(ctx)\n}\n\ntype withFile struct {\n\tFile\n\tkey string\n}\n\nfunc (f *withFile) Key() string { return f.key }\n\ntype withObj struct {\n\tObject\n\tkey string\n}\n\nfunc (o *withObj) Key() string { return o.key }\n\nfunc (p *withPrefix) updateKey(o Object) Object {\n\tkey := o.Key()\n\tif len(key) < len(p.prefix) {\n\t\treturn o\n\t}\n\tkey = key[len(p.prefix):]\n\tswitch po := o.(type) {\n\tcase *obj:\n\t\tpo.key = key\n\tcase *file:\n\t\tpo.key = key\n\tcase File:\n\t\to = &withFile{po, key}\n\tcase Object:\n\t\to = &withObj{po, key}\n\t}\n\treturn o\n}\n\nfunc (p *withPrefix) Head(ctx context.Context, key string) (Object, error) {\n\to, err := p.os.Head(ctx, p.prefix+key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn p.updateKey(o), nil\n}\n\nfunc (p *withPrefix) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tif off > 0 && limit < 0 {\n\t\treturn nil, fmt.Errorf(\"invalid range: %d-%d\", off, limit)\n\t}\n\treturn p.os.Get(ctx, p.prefix+key, off, limit, getters...)\n}\n\nfunc (p *withPrefix) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\treturn p.os.Put(ctx, p.prefix+key, in, getters...)\n}\n\nfunc (p *withPrefix) Copy(ctx context.Context, dst, src string) error {\n\treturn p.os.Copy(ctx, dst, src)\n}\n\nfunc (p *withPrefix) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\treturn p.os.Delete(ctx, p.prefix+key, getters...)\n}\n\nfunc (p *withPrefix) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif start != \"\" {\n\t\tstart = p.prefix + start\n\t}\n\tobjs, hasMore, nextMarker, err := p.os.List(ctx, p.prefix+prefix, start, token, delimiter, limit, followLink)\n\tfor i, o := range objs {\n\t\tobjs[i] = p.updateKey(o)\n\t}\n\treturn objs, hasMore, nextMarker, err\n}\n\nfunc (p *withPrefix) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\tif marker != \"\" {\n\t\tmarker = p.prefix + marker\n\t}\n\tr, err := p.os.ListAll(ctx, p.prefix+prefix, marker, followLink)\n\tif err != nil {\n\t\treturn r, err\n\t}\n\tr2 := make(chan Object, 10240)\n\tgo func() {\n\t\tfor o := range r {\n\t\t\tif o != nil && o.Key() != \"\" {\n\t\t\t\to = p.updateKey(o)\n\t\t\t}\n\t\t\tr2 <- o\n\t\t}\n\t\tclose(r2)\n\t}()\n\treturn r2, nil\n}\n\nfunc (p *withPrefix) Chmod(path string, mode os.FileMode) error {\n\tif fs, ok := p.os.(FileSystem); ok {\n\t\treturn fs.Chmod(p.prefix+path, mode)\n\t}\n\treturn notSupported\n}\n\nfunc (p *withPrefix) Chown(path string, owner, group string) error {\n\tif fs, ok := p.os.(FileSystem); ok {\n\t\treturn fs.Chown(p.prefix+path, owner, group)\n\t}\n\treturn notSupported\n}\n\nfunc (p *withPrefix) Chtimes(key string, mtime time.Time) error {\n\tif fs, ok := p.os.(FileSystem); ok {\n\t\treturn fs.Chtimes(p.prefix+key, mtime)\n\t}\n\treturn notSupported\n}\n\nfunc (p *withPrefix) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\treturn p.os.CreateMultipartUpload(ctx, p.prefix+key)\n}\n\nfunc (p *withPrefix) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\treturn p.os.UploadPart(ctx, p.prefix+key, uploadID, num, body)\n}\n\nfunc (s *withPrefix) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\treturn s.os.UploadPartCopy(ctx, s.prefix+key, uploadID, num, s.prefix+srcKey, off, size)\n}\n\nfunc (p *withPrefix) AbortUpload(ctx context.Context, key string, uploadID string) {\n\tp.os.AbortUpload(ctx, p.prefix+key, uploadID)\n}\n\nfunc (p *withPrefix) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\treturn p.os.CompleteUpload(ctx, p.prefix+key, uploadID, parts)\n}\n\nfunc (p *withPrefix) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tparts, nextMarker, err := p.os.ListUploads(ctx, marker)\n\tfor _, part := range parts {\n\t\tpart.Key = part.Key[len(p.prefix):]\n\t}\n\treturn parts, nextMarker, err\n}\n\nvar _ ObjectStorage = (*withPrefix)(nil)\n\nfunc IsFileSystem(object ObjectStorage) bool {\n\tif o, ok := object.(*withPrefix); ok {\n\t\tobject = o.os\n\t}\n\t_, ok := object.(FileSystem)\n\treturn ok\n}\n" }, { "path": "pkg/object/qingstor.go", "content": "//go:build !noqingstore\n// +build !noqingstore\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/qingstor/qingstor-sdk-go/v4/config\"\n\t\"github.com/qingstor/qingstor-sdk-go/v4/request/errors\"\n\tqs \"github.com/qingstor/qingstor-sdk-go/v4/service\"\n)\n\ntype qingstor struct {\n\tbucket *qs.Bucket\n\tsc string\n}\n\nfunc (q *qingstor) String() string {\n\treturn fmt.Sprintf(\"qingstor://%s/\", *q.bucket.Properties.BucketName)\n}\n\nfunc (q *qingstor) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 4 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (q *qingstor) Create(ctx context.Context) error {\n\t_, err := q.bucket.PutWithContext(ctx)\n\tif err != nil && strings.Contains(err.Error(), \"bucket_already_exists\") {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (q *qingstor) Head(ctx context.Context, key string) (Object, error) {\n\tr, err := q.bucket.HeadObjectWithContext(ctx, key, nil)\n\tif err != nil {\n\t\tif e, ok := err.(*errors.QingStorError); ok && e.StatusCode == http.StatusNotFound {\n\t\t\treturn nil, os.ErrNotExist\n\t\t}\n\t}\n\treturn &obj{\n\t\tkey,\n\t\t*r.ContentLength,\n\t\t*r.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t*r.XQSStorageClass,\n\t}, nil\n}\n\nfunc (q *qingstor) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tinput := &qs.GetObjectInput{}\n\trangeStr := getRange(off, limit)\n\tif rangeStr != \"\" {\n\t\tinput.Range = &rangeStr\n\t}\n\toutput, err := q.bucket.GetObjectWithContext(ctx, key, input)\n\tif output != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(aws.ToString(output.RequestID))\n\t\tif output.XQSStorageClass != nil {\n\t\t\tattrs.SetStorageClass(*output.XQSStorageClass)\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err = checkGetStatus(*output.StatusCode, rangeStr != \"\"); err != nil {\n\t\t_ = output.Body.Close()\n\t\treturn nil, err\n\t}\n\treturn output.Body, nil\n}\n\nfunc findLen(in io.Reader) (io.Reader, int64, error) {\n\tvar vlen int64\n\tswitch v := in.(type) {\n\tcase *bytes.Buffer:\n\t\tvlen = int64(v.Len())\n\tcase *bytes.Reader:\n\t\tvlen = int64(v.Len())\n\tcase *strings.Reader:\n\t\tvlen = int64(v.Len())\n\tcase *os.File:\n\t\tst, err := v.Stat()\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\tvlen = st.Size()\n\tcase io.ReadSeeker:\n\t\tvar err error\n\t\tvlen, err = v.Seek(0, 2)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\tif _, err = v.Seek(0, 0); err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\tdefault:\n\t\td, err := io.ReadAll(in)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\tvlen = int64(len(d))\n\t\tin = bytes.NewBuffer(d)\n\t}\n\treturn in, vlen, nil\n}\n\nfunc (q *qingstor) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tbody, vlen, err := findLen(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\tmimeType := utils.GuessMimeType(key)\n\tinput := &qs.PutObjectInput{\n\t\tBody: body,\n\t\tContentLength: &vlen,\n\t\tContentType: &mimeType,\n\t}\n\tif q.sc != \"\" {\n\t\tinput.XQSStorageClass = &q.sc\n\t}\n\tout, err := q.bucket.PutObjectWithContext(ctx, key, input)\n\tif out != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(aws.ToString(out.RequestID)).SetStorageClass(q.sc)\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\tif *out.StatusCode != 201 {\n\t\treturn fmt.Errorf(\"unexpected code: %d\", *out.StatusCode)\n\t}\n\treturn nil\n}\n\nfunc (q *qingstor) Copy(ctx context.Context, dst, src string) error {\n\tsource := fmt.Sprintf(\"/%s/%s\", *q.bucket.Properties.BucketName, src)\n\tinput := &qs.PutObjectInput{\n\t\tXQSCopySource: &source,\n\t}\n\tif q.sc != \"\" {\n\t\tinput.XQSStorageClass = &q.sc\n\t}\n\tout, err := q.bucket.PutObjectWithContext(ctx, dst, input)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif *out.StatusCode != 201 {\n\t\treturn fmt.Errorf(\"unexpected code: %d\", *out.StatusCode)\n\t}\n\treturn nil\n}\n\nfunc (q *qingstor) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\toutput, err := q.bucket.DeleteObjectWithContext(ctx, key)\n\tif output != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(aws.ToString(output.RequestID))\n\t}\n\treturn err\n}\n\nfunc (q *qingstor) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\tlimit_ := int(limit)\n\tinput := &qs.ListObjectsInput{\n\t\tPrefix: &prefix,\n\t\tMarker: &start,\n\t\tLimit: &limit_,\n\t}\n\tif delimiter != \"\" {\n\t\tinput.Delimiter = &delimiter\n\t}\n\tout, err := q.bucket.ListObjectsWithContext(ctx, input)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(out.Keys)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\tk := out.Keys[i]\n\t\tobjs[i] = &obj{\n\t\t\t*k.Key,\n\t\t\t*k.Size,\n\t\t\ttime.Unix(int64(*k.Modified), 0),\n\t\t\tstrings.HasSuffix(*k.Key, \"/\"),\n\t\t\t*k.StorageClass,\n\t\t}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range out.CommonPrefixes {\n\t\t\tobjs = append(objs, &obj{*p, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, *out.HasMore, *out.NextMarker, nil\n}\n\nfunc (q *qingstor) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (q *qingstor) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tvar input qs.InitiateMultipartUploadInput\n\tif q.sc != \"\" {\n\t\tinput.XQSStorageClass = &q.sc\n\t}\n\tr, err := q.bucket.InitiateMultipartUploadWithContext(ctx, key, &input)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: *r.UploadID, MinPartSize: 4 << 20, MaxCount: 10000}, nil\n}\n\nfunc (q *qingstor) UploadPart(ctx context.Context, key string, uploadID string, num int, data []byte) (*Part, error) {\n\tinput := &qs.UploadMultipartInput{\n\t\tUploadID: &uploadID,\n\t\tPartNumber: &num,\n\t\tBody: bytes.NewReader(data),\n\t}\n\tr, err := q.bucket.UploadMultipartWithContext(ctx, key, input)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, Size: len(data), ETag: strings.Trim(*r.ETag, \"\\\"\")}, nil\n}\n\nfunc (q *qingstor) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tinput := &qs.UploadMultipartInput{\n\t\tUploadID: &uploadID,\n\t\tPartNumber: &num,\n\t\tXQSCopySource: aws.String(fmt.Sprintf(\"/%s/%s\", *q.bucket.Properties.BucketName, srcKey)),\n\t\tXQSCopyRange: aws.String(fmt.Sprintf(\"bytes=%d-%d\", off, off+size-1)),\n\t}\n\tr, err := q.bucket.UploadMultipartWithContext(ctx, key, input)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, Size: int(size), ETag: strings.Trim(*r.ETag, \"\\\"\")}, nil\n}\n\nfunc (q *qingstor) AbortUpload(ctx context.Context, key string, uploadID string) {\n\tinput := &qs.AbortMultipartUploadInput{\n\t\tUploadID: &uploadID,\n\t}\n\t_, _ = q.bucket.AbortMultipartUploadWithContext(ctx, key, input)\n}\n\nfunc (q *qingstor) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\toparts := make([]*qs.ObjectPartType, len(parts))\n\tfor i := range parts {\n\t\toparts[i] = &qs.ObjectPartType{\n\t\t\tPartNumber: &parts[i].Num,\n\t\t\tEtag: &parts[i].ETag,\n\t\t}\n\t}\n\tinput := &qs.CompleteMultipartUploadInput{\n\t\tUploadID: &uploadID,\n\t\tObjectParts: oparts,\n\t}\n\t_, err := q.bucket.CompleteMultipartUploadWithContext(ctx, key, input)\n\treturn err\n}\n\nfunc (q *qingstor) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tinput := &qs.ListMultipartUploadsInput{\n\t\tKeyMarker: &marker,\n\t}\n\tresult, err := q.bucket.ListMultipartUploadsWithContext(ctx, input)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tparts[i] = &PendingPart{*u.Key, *u.UploadID, *u.Created}\n\t}\n\tvar nextMarker string\n\tif result.NextKeyMarker != nil {\n\t\tnextMarker = *result.NextKeyMarker\n\t}\n\treturn parts, nextMarker, nil\n}\n\nfunc (q *qingstor) SetStorageClass(sc string) error {\n\tq.sc = sc\n\treturn nil\n}\n\nfunc newQingStor(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\tvar bucketName, zone, host string\n\tif !strings.HasSuffix(uri.Host, \"qingstor.com\") {\n\t\t// support private cloud\n\t\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\t\tbucketName, zone, host = hostParts[0], \"\", hostParts[1]\n\t} else {\n\t\thostParts := strings.SplitN(uri.Host, \".\", 3)\n\t\tbucketName, zone, host = hostParts[0], hostParts[1], hostParts[2]\n\t}\n\tconf, err := config.New(accessKey, secretKey)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Can't load config: %s\", err.Error())\n\t}\n\tconf.Host = host\n\tconf.Protocol = uri.Scheme\n\tif uri.Scheme == \"http\" {\n\t\tconf.Port = 80\n\t} else {\n\t\tconf.Port = 443\n\t}\n\tconf.Connection = httpClient\n\tqsService, _ := qs.Init(conf)\n\tbucket, _ := qsService.Bucket(bucketName, zone)\n\treturn &qingstor{bucket: bucket}, nil\n}\n\nfunc init() {\n\tRegister(\"qingstor\", newQingStor)\n}\n" }, { "path": "pkg/object/qiniu.go", "content": "//go:build !noqiniu && !nos3\n// +build !noqiniu,!nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n\t\"github.com/qiniu/go-sdk/v7/auth\"\n\t\"github.com/qiniu/go-sdk/v7/storage\"\n)\n\ntype qiniu struct {\n\ts3client\n\tbm *storage.BucketManager\n\tcred *auth.Credentials\n\tcfg *storage.Config\n\tmarker string\n}\n\nfunc (q *qiniu) String() string {\n\treturn fmt.Sprintf(\"qiniu://%s/\", q.bucket)\n}\n\nfunc (q *qiniu) SetStorageClass(_ string) error {\n\treturn notSupported\n}\n\nfunc (q *qiniu) Limits() Limits {\n\treturn Limits{}\n}\n\nfunc (q *qiniu) download(key string, off, limit int64) (io.ReadCloser, error) {\n\tdeadline := time.Now().Add(time.Second * 3600).Unix()\n\turl := storage.MakePrivateURL(q.cred, os.Getenv(\"QINIU_DOMAIN\"), key, deadline)\n\treq, err := http.NewRequest(\"GET\", url, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnow := time.Now().UTC().Format(http.TimeFormat)\n\treq.Header.Add(\"Date\", now)\n\tif off > 0 || limit > 0 {\n\t\tif limit > 0 {\n\t\t\treq.Header.Add(\"Range\", fmt.Sprintf(\"bytes=%d-%d\", off, off+limit-1))\n\t\t} else {\n\t\t\treq.Header.Add(\"Range\", fmt.Sprintf(\"bytes=%d-\", off))\n\t\t}\n\t}\n\tresp, err := httpClient.Do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif resp.StatusCode != 200 && resp.StatusCode != 206 {\n\t\treturn nil, fmt.Errorf(\"Status code: %d\", resp.StatusCode)\n\t}\n\treturn resp.Body, nil\n}\n\nvar notexist = \"no such file or directory\"\n\nfunc (q *qiniu) Head(ctx context.Context, key string) (Object, error) {\n\tr, err := q.bm.Stat(q.bucket, key)\n\tif err != nil {\n\t\tif strings.Contains(err.Error(), notexist) {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\n\tmtime := time.Unix(0, r.PutTime*100)\n\treturn &obj{\n\t\tkey,\n\t\tr.Fsize,\n\t\tmtime,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\"\",\n\t}, nil\n}\n\nfunc (q *qiniu) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tif strings.HasPrefix(key, \"/\") && os.Getenv(\"QINIU_DOMAIN\") != \"\" {\n\t\treturn q.download(key, off, limit)\n\t}\n\treturn q.s3client.Get(ctx, key, off, limit, getters...)\n}\n\nfunc (q *qiniu) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tbody, vlen, err := findLen(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\tputPolicy := storage.PutPolicy{Scope: q.bucket + \":\" + key}\n\tupToken := putPolicy.UploadToken(q.cred)\n\tformUploader := storage.NewFormUploader(q.cfg)\n\tvar ret storage.PutRet\n\treturn formUploader.Put(ctx, &ret, upToken, key, body, vlen, nil)\n}\n\nfunc (q *qiniu) Copy(ctx context.Context, dst, src string) error {\n\treturn q.bm.Copy(q.bucket, src, q.bucket, dst, true)\n}\n\nfunc (q *qiniu) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\treturn nil, notSupported\n}\n\nfunc (q *qiniu) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\terr := q.bm.Delete(q.bucket, key)\n\tif err != nil && strings.Contains(err.Error(), notexist) {\n\t\treturn nil\n\t}\n\treturn err\n}\n\nfunc (q *qiniu) List(ctx context.Context, prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\tentries, prefixes, markerOut, hasNext, err := q.bm.ListFiles(q.bucket, prefix, delimiter, token, int(limit))\n\tif len(entries) > 0 || err == io.EOF {\n\t\t// ignore error if returned something\n\t\terr = nil\n\t}\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(entries)\n\tobjs := make([]Object, 0, n)\n\tfor i := 0; i < n; i++ {\n\t\tentry := entries[i]\n\t\tif entry.Key <= startAfter {\n\t\t\tcontinue\n\t\t}\n\t\tmtime := entry.PutTime / 10000000\n\t\tobjs = append(objs, &obj{entry.Key, entry.Fsize, time.Unix(mtime, 0), strings.HasSuffix(entry.Key, \"/\"), \"\"})\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range prefixes {\n\t\t\tif p <= startAfter {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tobjs = append(objs, &obj{p, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\tif len(objs) == 0 {\n\t\thasNext = false\n\t\tmarkerOut = \"\"\n\t}\n\treturn objs, hasNext, markerOut, nil\n}\n\nfunc newQiniu(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\tbucket := hostParts[0]\n\tendpoint = hostParts[1]\n\tvar region string\n\tif strings.HasPrefix(endpoint, \"s3\") {\n\t\t// private region\n\t\tregion = endpoint[strings.Index(endpoint, \"-\")+1 : strings.Index(endpoint, \".\")]\n\t} else if strings.HasPrefix(endpoint, \"qvm-\") {\n\t\tregion = \"cn-east-1\" // internal\n\t} else if strings.HasPrefix(endpoint, \"qvm-z1\") {\n\t\tregion = \"cn-north-1\"\n\t} else {\n\t\tregion = endpoint[:strings.LastIndex(endpoint, \"-\")]\n\t}\n\n\tawsCfg, err := config.LoadDefaultConfig(ctx,\n\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to load config: %s\", err)\n\t}\n\tclient := s3.NewFromConfig(awsCfg, func(options *s3.Options) {\n\t\toptions.BaseEndpoint = aws.String(uri.Scheme + \"://\" + endpoint)\n\t\toptions.Region = region\n\t\toptions.EndpointOptions.DisableHTTPS = uri.Scheme == \"http\"\n\t\toptions.UsePathStyle = true\n\t\toptions.HTTPClient = httpClient\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\ts3c := s3client{bucket: bucket, s3: client, region: region}\n\tcfg := storage.Config{\n\t\tUseHTTPS: uri.Scheme == \"https\",\n\t}\n\tzone, err := storage.GetZone(accessKey, bucket)\n\tif err != nil {\n\t\tdomain := strings.SplitN(endpoint, \"-\", 2)[1]\n\t\tzone = &storage.Zone{\n\t\t\tRsHost: \"rs-\" + domain,\n\t\t\tRsfHost: \"rsf-\" + domain,\n\t\t\tApiHost: \"api-\" + domain,\n\t\t\tIovipHost: \"io-\" + domain,\n\t\t\tSrcUpHosts: []string{\"up-\" + domain},\n\t\t}\n\t} else if strings.HasPrefix(endpoint, \"qvm-z1\") {\n\t\tzone.SrcUpHosts = []string{\"free-qvm-z1-zz.qiniup.com\"}\n\t} else if strings.HasPrefix(endpoint, \"qvm-\") {\n\t\tzone.SrcUpHosts = []string{\"free-qvm-z0-xs.qiniup.com\"}\n\t}\n\tcfg.Zone = zone\n\tcred := auth.New(accessKey, secretKey)\n\tbucketManager := storage.NewBucketManager(cred, &cfg)\n\treturn &qiniu{s3c, bucketManager, cred, &cfg, \"\"}, nil\n}\n\nfunc init() {\n\tRegister(\"qiniu\", newQiniu)\n}\n" }, { "path": "pkg/object/redis.go", "content": "//go:build !noredis\n// +build !noredis\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// redisStore stores data chunks into Redis.\ntype redisStore struct {\n\tDefaultObjectStorage\n\trdb redis.UniversalClient\n\turi string\n}\n\nfunc (r *redisStore) String() string {\n\treturn r.uri + \"/\"\n}\n\nfunc (r *redisStore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tdata, err := r.rdb.Get(ctx, key).Bytes()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif off > int64(len(data)) {\n\t\toff = int64(len(data))\n\t}\n\tdata = data[off:]\n\tif limit > 0 && limit < int64(len(data)) {\n\t\tdata = data[:limit]\n\t}\n\treturn io.NopCloser(bytes.NewBuffer(data)), nil\n}\n\nfunc (r *redisStore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tdata, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn r.rdb.Set(ctx, key, data, 0).Err()\n}\n\nfunc (r *redisStore) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\treturn r.rdb.Del(ctx, key).Err()\n}\n\nfunc (t *redisStore) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\tvar scanCli []redis.UniversalClient\n\tvar m sync.Mutex\n\tif c, ok := t.rdb.(*redis.ClusterClient); ok {\n\t\terr := c.ForEachMaster(ctx, func(ctx context.Context, client *redis.Client) error {\n\t\t\tm.Lock()\n\t\t\tdefer m.Unlock()\n\t\t\tscanCli = append(scanCli, client)\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tscanCli = append(scanCli, t.rdb)\n\t}\n\tbatch := 1000\n\tvar objs = make(chan Object, batch)\n\tvar keyList []string\n\tvar cursor uint64\n\tfor _, mCli := range scanCli {\n\t\tfor {\n\t\t\t// FIXME: this will be really slow for many objects\n\t\t\tkeys, c, err := mCli.Scan(ctx, cursor, prefix+\"*\", int64(batch)).Result()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"redis scan error, coursor %d: %s\", cursor, err)\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tfor _, key := range keys {\n\t\t\t\tif key > marker {\n\t\t\t\t\tkeyList = append(keyList, key)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif c == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tcursor = c\n\t\t}\n\t}\n\tsort.Strings(keyList)\n\n\tgo func() {\n\t\tdefer close(objs)\n\t\tlKeyList := len(keyList)\n\t\tfor start := 0; start < lKeyList; start += batch {\n\t\t\tend := start + batch\n\t\t\tif end > lKeyList {\n\t\t\t\tend = lKeyList\n\t\t\t}\n\n\t\t\tp := t.rdb.Pipeline()\n\t\t\tfor _, key := range keyList[start:end] {\n\t\t\t\tp.StrLen(ctx, key)\n\t\t\t}\n\t\t\tcmds, err := p.Exec(ctx)\n\t\t\tif err != nil {\n\t\t\t\tobjs <- nil\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tnow := time.Now()\n\t\t\tfor idx, cmd := range cmds {\n\t\t\t\tif intCmd, ok := cmd.(*redis.IntCmd); ok {\n\t\t\t\t\tsize, err := intCmd.Result()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tobjs <- nil\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tif size == 0 {\n\t\t\t\t\t\texist, err := t.rdb.Exists(ctx, keyList[start:end][idx]).Result()\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tobjs <- nil\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif exist == 0 {\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// FIXME: mtime\n\t\t\t\t\tobjs <- &obj{keyList[start:end][idx], size, now, strings.HasSuffix(keyList[start:end][idx], \"/\"), \"\"}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\treturn objs, nil\n}\n\nfunc (t *redisStore) Head(ctx context.Context, key string) (Object, error) {\n\tdata, err := t.rdb.Get(ctx, key).Bytes()\n\tif err != nil {\n\t\tif err == redis.Nil {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tkey,\n\t\tint64(len(data)),\n\t\ttime.Now(),\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\"\",\n\t}, err\n}\n\nfunc newRedis(uri, user, passwd, token string) (ObjectStorage, error) {\n\tu, err := url.Parse(uri)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"url parse %s: %s\", uri, err)\n\t}\n\thosts := u.Host\n\topt, err := redis.ParseURL(u.String())\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"redis parse %s: %s\", uri, err)\n\t}\n\tif user != \"\" {\n\t\topt.Username = user\n\t}\n\tif passwd != \"\" {\n\t\topt.Password = passwd\n\t}\n\tif opt.MaxRetries == 0 {\n\t\topt.MaxRetries = -1 // Redis use -1 to disable retries\n\t}\n\tvar rdb redis.UniversalClient\n\tif strings.Contains(hosts, \",\") && strings.Index(hosts, \",\") < strings.Index(hosts, \":\") {\n\t\tvar fopt redis.FailoverOptions\n\t\tps := strings.Split(hosts, \",\")\n\t\tfopt.MasterName = ps[0]\n\t\tfopt.SentinelAddrs = ps[1:]\n\t\t_, port, _ := net.SplitHostPort(fopt.SentinelAddrs[len(fopt.SentinelAddrs)-1])\n\t\tif port == \"\" {\n\t\t\tport = \"26379\"\n\t\t}\n\t\tfor i, addr := range fopt.SentinelAddrs {\n\t\t\th, p, e := net.SplitHostPort(addr)\n\t\t\tif e != nil {\n\t\t\t\tfopt.SentinelAddrs[i] = net.JoinHostPort(addr, port)\n\t\t\t} else if p == \"\" {\n\t\t\t\tfopt.SentinelAddrs[i] = net.JoinHostPort(h, port)\n\t\t\t}\n\t\t}\n\t\tfopt.SentinelPassword = os.Getenv(\"SENTINEL_PASSWORD_FOR_OBJ\")\n\t\tfopt.DB = opt.DB\n\t\tfopt.Username = opt.Username\n\t\tfopt.Password = opt.Password\n\t\tfopt.TLSConfig = opt.TLSConfig\n\t\tfopt.MaxRetries = opt.MaxRetries\n\t\tfopt.MinRetryBackoff = opt.MinRetryBackoff\n\t\tfopt.MaxRetryBackoff = opt.MaxRetryBackoff\n\t\tfopt.ReadTimeout = opt.ReadTimeout\n\t\tfopt.WriteTimeout = opt.WriteTimeout\n\t\trdb = redis.NewFailoverClient(&fopt)\n\t} else {\n\t\tif !strings.Contains(hosts, \",\") {\n\t\t\tc := redis.NewClient(opt)\n\t\t\tinfo, err := c.ClusterInfo(context.Background()).Result()\n\t\t\tif err != nil && strings.Contains(err.Error(), \"cluster mode\") || err == nil && strings.Contains(info, \"cluster_state:\") {\n\t\t\t\tlogger.Infof(\"redis %s is in cluster mode\", hosts)\n\t\t\t} else {\n\t\t\t\trdb = c\n\t\t\t}\n\t\t}\n\t\tif rdb == nil {\n\t\t\tvar copt redis.ClusterOptions\n\t\t\tcopt.Addrs = strings.Split(hosts, \",\")\n\t\t\tcopt.MaxRedirects = 1\n\t\t\tcopt.Username = opt.Username\n\t\t\tcopt.Password = opt.Password\n\t\t\tcopt.TLSConfig = opt.TLSConfig\n\t\t\tcopt.MaxRetries = opt.MaxRetries\n\t\t\tcopt.MinRetryBackoff = opt.MinRetryBackoff\n\t\t\tcopt.MaxRetryBackoff = opt.MaxRetryBackoff\n\t\t\tcopt.ReadTimeout = opt.ReadTimeout\n\t\t\tcopt.WriteTimeout = opt.WriteTimeout\n\t\t\trdb = redis.NewClusterClient(&copt)\n\t\t}\n\t}\n\tu.User = new(url.Userinfo)\n\treturn &redisStore{DefaultObjectStorage{}, rdb, u.String()}, nil\n}\n\nfunc init() {\n\tRegister(\"redis\", newRedis)\n}\n" }, { "path": "pkg/object/response_attrs.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nconst DefaultStorageClass = \"STANDARD\"\n\ntype SupportStorageClass interface {\n\tSetStorageClass(sc string) error\n}\n\n// A generic way to get attributes from different object storage clients\ntype ResponseAttrs struct {\n\tstorageClass *string\n\trequestID *string\n\trequestSize *int64\n\t// other interested attrs can be added here\n}\n\nfunc (r *ResponseAttrs) SetRequestID(id string) *ResponseAttrs {\n\tif r.requestID != nil { // Will be nil if caller is not interested in this attribute\n\t\t*r.requestID = id\n\t}\n\treturn r\n}\n\nfunc (r *ResponseAttrs) SetStorageClass(sc string) *ResponseAttrs {\n\tif r.storageClass != nil && sc != \"\" { // Don't overwrite default storage class\n\t\t*r.storageClass = sc\n\t}\n\treturn r\n}\n\nfunc (r *ResponseAttrs) GetRequestSize() int64 {\n\tif r.requestSize != nil {\n\t\treturn *r.requestSize\n\t}\n\treturn -1\n}\n\ntype AttrGetter func(attrs *ResponseAttrs)\n\nfunc WithRequestID(id *string) AttrGetter {\n\treturn func(attrs *ResponseAttrs) {\n\t\tattrs.requestID = id\n\t}\n}\n\nfunc WithStorageClass(sc *string) AttrGetter {\n\treturn func(attrs *ResponseAttrs) {\n\t\tattrs.storageClass = sc\n\t}\n}\n\nfunc WithRequestSize(size *int64) AttrGetter {\n\treturn func(attrs *ResponseAttrs) {\n\t\tattrs.requestSize = size\n\t}\n}\n\nfunc ApplyGetters(getters ...AttrGetter) ResponseAttrs {\n\tvar attrs ResponseAttrs\n\tfor _, getter := range getters {\n\t\tgetter(&attrs)\n\t}\n\treturn attrs\n}\n" }, { "path": "pkg/object/response_attrs_test.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst reqIDExample = \"c30c0107cd3a073f6607cd3a-ac103aa8-1rqU4w-PuO-cs-tos-front-azc-2\"\n\nfunc apiCall(getters ...AttrGetter) {\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetStorageClass(\"STANDARD\")\n\tattrs.SetRequestID(reqIDExample)\n\treturn\n}\n\nfunc Test_api_call(t *testing.T) {\n\tvar reqID, sc string\n\n\tapiCall(WithRequestID(&reqID), WithStorageClass(&sc))\n\tassert.Equalf(t, reqIDExample, reqID, \"expected %q, got %q\", reqIDExample, reqID)\n\tassert.Equalf(t, \"STANDARD\", sc, \"expected %q, got %q\", \"STANDARD\", sc)\n\n\tattrs := ApplyGetters(WithStorageClass(&sc))\n\tattrs.SetStorageClass(\"\") // Won't overwrite by empty string\n\tassert.Equalf(t, \"STANDARD\", sc, \"expected %q, got %q\", \"STANDARD\", sc)\n}\n" }, { "path": "pkg/object/restful.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/viki-org/dnscache\"\n)\n\nvar resolver = dnscache.New(time.Minute)\nvar httpClient *http.Client\n\nfunc splitIPsByVersion(ips []net.IP) ([]net.IP, []net.IP) {\n\tipv6 := make([]net.IP, 0, len(ips))\n\tipv4 := make([]net.IP, 0, len(ips))\n\tfor _, ip := range ips {\n\t\tif ip.To4() == nil {\n\t\t\tipv6 = append(ipv6, ip)\n\t\t} else {\n\t\t\tipv4 = append(ipv4, ip)\n\t\t}\n\t}\n\treturn ipv6, ipv4\n}\n\n// dialParallel is adapted from the Go standard library.\n// Copyright 2010 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\nfunc dialParallel(ctx context.Context, dialer *net.Dialer, network string, primaries, fallbacks []net.IP, port string) (net.Conn, error) {\n\tif len(fallbacks) == 0 {\n\t\treturn dialRandom(ctx, dialer, network, primaries, port)\n\t}\n\n\treturned := make(chan struct{})\n\tdefer close(returned)\n\n\ttype dialResult struct {\n\t\tnet.Conn\n\t\terror\n\t\tprimary bool\n\t\tdone bool\n\t}\n\tresults := make(chan dialResult) // unbuffered\n\n\tstartRacer := func(ctx context.Context, primary bool) {\n\t\tras := primaries\n\t\tif !primary {\n\t\t\tras = fallbacks\n\t\t}\n\t\tc, err := dialRandom(ctx, dialer, network, ras, port)\n\t\tselect {\n\t\tcase results <- dialResult{Conn: c, error: err, primary: primary, done: true}:\n\t\tcase <-returned:\n\t\t\tif c != nil {\n\t\t\t\tc.Close()\n\t\t\t}\n\t\t}\n\t}\n\n\tvar primary, fallback dialResult\n\n\t// Start the main racer.\n\tprimaryCtx, primaryCancel := context.WithCancel(ctx)\n\tdefer primaryCancel()\n\tgo startRacer(primaryCtx, true)\n\n\t// Start the timer for the fallback racer.\n\tfallbackTimer := time.NewTimer(300 * time.Millisecond)\n\tdefer fallbackTimer.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-fallbackTimer.C:\n\t\t\tfallbackCtx, fallbackCancel := context.WithCancel(ctx)\n\t\t\tdefer fallbackCancel()\n\t\t\tgo startRacer(fallbackCtx, false)\n\n\t\tcase res := <-results:\n\t\t\tif res.error == nil {\n\t\t\t\treturn res.Conn, nil\n\t\t\t}\n\t\t\tif res.primary {\n\t\t\t\tprimary = res\n\t\t\t} else {\n\t\t\t\tfallback = res\n\t\t\t}\n\t\t\tif primary.done && fallback.done {\n\t\t\t\treturn nil, errors.Join(primary.error, fallback.error)\n\t\t\t}\n\t\t\tif res.primary && fallbackTimer.Stop() {\n\t\t\t\t// If we were able to stop the timer, that means it\n\t\t\t\t// was running (hadn't yet started the fallback), but\n\t\t\t\t// we just got an error on the primary path, so start\n\t\t\t\t// the fallback immediately (in 0 nanoseconds).\n\t\t\t\tfallbackTimer.Reset(0)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc dialRandom(ctx context.Context, dialer *net.Dialer, network string, ips []net.IP, port string) (net.Conn, error) {\n\tvar lastErr error\n\tn := len(ips)\n\tif n == 0 {\n\t\treturn nil, fmt.Errorf(\"no addresses to dial\")\n\t}\n\tfirst := rand.Intn(n)\n\tfor i := 0; i < n; i++ {\n\t\tip := ips[(first+i)%n]\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn nil, ctx.Err()\n\t\tdefault:\n\t\t}\n\t\tconn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))\n\t\tif err == nil {\n\t\t\treturn conn, nil\n\t\t}\n\t\tlastErr = err\n\t}\n\treturn nil, lastErr\n}\n\nfunc init() {\n\tdialer := &net.Dialer{Timeout: time.Second * 10}\n\thttpClient = &http.Client{\n\t\tTransport: &http.Transport{\n\t\t\tProxy: http.ProxyFromEnvironment,\n\t\t\tTLSHandshakeTimeout: time.Second * 20,\n\t\t\tResponseHeaderTimeout: time.Second * 30,\n\t\t\tIdleConnTimeout: time.Second * 300,\n\t\t\tMaxIdleConnsPerHost: 500,\n\t\t\tReadBufferSize: 32 << 10,\n\t\t\tWriteBufferSize: 32 << 10,\n\t\t\tDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {\n\t\t\t\thost, port, err := net.SplitHostPort(address)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif ip := net.ParseIP(host); ip != nil {\n\t\t\t\t\treturn dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))\n\t\t\t\t}\n\t\t\t\tips, err := resolver.Fetch(host)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif len(ips) == 0 {\n\t\t\t\t\treturn nil, &net.DNSError{Err: \"no such host\", Name: host, IsNotFound: true}\n\t\t\t\t}\n\t\t\t\tipv6, ipv4 := splitIPsByVersion(ips)\n\t\t\t\treturn dialParallel(ctx, dialer, network, ipv6, ipv4, port)\n\t\t\t},\n\t\t\tDisableCompression: true,\n\t\t\tTLSClientConfig: &tls.Config{},\n\t\t},\n\t\tTimeout: time.Hour,\n\t}\n}\n\nfunc GetHttpClient() *http.Client {\n\treturn httpClient\n}\n\nfunc cleanup(response *http.Response) {\n\tif response != nil && response.Body != nil {\n\t\t_, _ = io.Copy(io.Discard, response.Body)\n\t\t_ = response.Body.Close()\n\t}\n}\n\ntype RestfulStorage struct {\n\tDefaultObjectStorage\n\tendpoint string\n\taccessKey string\n\tsecretKey string\n\tsignName string\n\tsigner func(*http.Request, string, string, string)\n}\n\nfunc (s *RestfulStorage) String() string {\n\treturn s.endpoint\n}\n\nvar HEADER_NAMES = []string{\"Content-MD5\", \"Content-Type\", \"Date\"}\n\nfunc (s *RestfulStorage) request(ctx context.Context, method, key string, body io.Reader, headers map[string]string) (*http.Response, error) {\n\turi := s.endpoint + \"/\" + key\n\treq, err := http.NewRequestWithContext(ctx, method, uri, body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif f, ok := body.(*os.File); ok {\n\t\tst, err := f.Stat()\n\t\tif err == nil {\n\t\t\treq.ContentLength = st.Size()\n\t\t}\n\t}\n\tnow := time.Now().UTC().Format(http.TimeFormat)\n\treq.Header.Add(\"Date\", now)\n\tfor key := range headers {\n\t\treq.Header.Add(key, headers[key])\n\t}\n\ts.signer(req, s.accessKey, s.secretKey, s.signName)\n\treturn httpClient.Do(req)\n}\n\nfunc parseError(resp *http.Response) error {\n\tdata, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"request failed: %s\", err)\n\t}\n\treturn fmt.Errorf(\"status: %v, message: %s\", resp.StatusCode, string(data))\n}\n\nfunc (s *RestfulStorage) Head(ctx context.Context, key string) (Object, error) {\n\tresp, err := s.request(ctx, \"HEAD\", key, nil, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn nil, os.ErrNotExist\n\t}\n\tdefer cleanup(resp)\n\tif resp.StatusCode != 200 {\n\t\treturn nil, parseError(resp)\n\t}\n\n\tlastModified := resp.Header.Get(\"Last-Modified\")\n\tif lastModified == \"\" {\n\t\treturn nil, fmt.Errorf(\"cannot get last modified time\")\n\t}\n\tmtime, _ := time.Parse(time.RFC1123, lastModified)\n\treturn &obj{\n\t\tkey,\n\t\tresp.ContentLength,\n\t\tmtime,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\"\",\n\t}, nil\n}\n\nfunc getRange(off, limit int64) string {\n\tif off > 0 || limit > 0 {\n\t\tif limit > 0 {\n\t\t\treturn fmt.Sprintf(\"bytes=%d-%d\", off, off+limit-1)\n\t\t} else {\n\t\t\treturn fmt.Sprintf(\"bytes=%d-\", off)\n\t\t}\n\t}\n\treturn \"\"\n}\n\nfunc checkGetStatus(statusCode int, partial bool) error {\n\tvar expected = http.StatusOK\n\tif partial {\n\t\texpected = http.StatusPartialContent\n\t}\n\tif statusCode != expected {\n\t\treturn fmt.Errorf(\"expected status code %d, but got %d\", expected, statusCode)\n\t}\n\treturn nil\n}\n\nfunc (s *RestfulStorage) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\theaders := make(map[string]string)\n\tif off > 0 || limit > 0 {\n\t\theaders[\"Range\"] = getRange(off, limit)\n\t}\n\tresp, err := s.request(ctx, \"GET\", key, nil, headers)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif resp.StatusCode != 200 && resp.StatusCode != 206 {\n\t\treturn nil, parseError(resp)\n\t}\n\tif err = checkGetStatus(resp.StatusCode, len(headers) > 0); err != nil {\n\t\t_ = resp.Body.Close()\n\t\treturn nil, err\n\t}\n\treturn resp.Body, nil\n}\n\nfunc (u *RestfulStorage) Put(ctx context.Context, key string, body io.Reader, getters ...AttrGetter) error {\n\tresp, err := u.request(ctx, \"PUT\", key, body, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer cleanup(resp)\n\tif resp.StatusCode != 201 && resp.StatusCode != 200 {\n\t\treturn parseError(resp)\n\t}\n\treturn nil\n}\n\nfunc (s *RestfulStorage) Copy(ctx context.Context, dst, src string) error {\n\tin, err := s.Get(ctx, src, 0, -1)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer in.Close()\n\td, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn s.Put(ctx, dst, bytes.NewReader(d))\n}\n\nfunc (s *RestfulStorage) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tresp, err := s.request(ctx, \"DELETE\", key, nil, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer cleanup(resp)\n\tif resp.StatusCode != 204 && resp.StatusCode != 404 {\n\t\treturn parseError(resp)\n\t}\n\treturn nil\n}\n\nfunc (s *RestfulStorage) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\treturn nil, false, \"\", notSupported\n}\n\nvar _ ObjectStorage = (*RestfulStorage)(nil)\n" }, { "path": "pkg/object/restful_test.go", "content": "/*\n * JuiceFS, Copyright 2026 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n)\n\n// startTCPListener starts a TCP listener on the given address and returns it.\n// The listener accepts connections in background and immediately closes them.\nfunc startTCPListener(t *testing.T, addr string) net.Listener {\n\tt.Helper()\n\tln, err := net.Listen(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to listen on %s: %v\", addr, err)\n\t}\n\tgo func() {\n\t\tfor {\n\t\t\tc, err := ln.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tc.Close()\n\t\t}\n\t}()\n\treturn ln\n}\n\nfunc getPort(t *testing.T, ln net.Listener) string {\n\tt.Helper()\n\t_, port, err := net.SplitHostPort(ln.Addr().String())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn port\n}\n\nfunc TestDialParallel_OnlyPrimaries(t *testing.T) {\n\tln := startTCPListener(t, \"127.0.0.1:0\")\n\tdefer ln.Close()\n\tport := getPort(t, ln)\n\n\tdialer := &net.Dialer{Timeout: 2 * time.Second}\n\tconn, err := dialParallel(context.Background(), dialer, \"tcp\",\n\t\t[]net.IP{net.ParseIP(\"127.0.0.1\")}, nil, port)\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\tconn.Close()\n}\n\nfunc TestDialParallel_OnlyFallbacks(t *testing.T) {\n\t// Bug reproduced: empty primaries should not panic\n\tln := startTCPListener(t, \"127.0.0.1:0\")\n\tdefer ln.Close()\n\tport := getPort(t, ln)\n\n\tdialer := &net.Dialer{Timeout: 2 * time.Second}\n\tconn, err := dialParallel(context.Background(), dialer, \"tcp\",\n\t\tnil, []net.IP{net.ParseIP(\"127.0.0.1\")}, port)\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\tconn.Close()\n}\n\nfunc TestDialParallel_PrimaryFailsFast_FallbackSucceeds(t *testing.T) {\n\t// Primary (IPv6 ::1) has no listener → fails fast (connection refused)\n\t// Fallback (127.0.0.1) has a listener → succeeds\n\tln := startTCPListener(t, \"127.0.0.1:0\")\n\tdefer ln.Close()\n\tport := getPort(t, ln)\n\n\tdialer := &net.Dialer{Timeout: 2 * time.Second}\n\tconn, err := dialParallel(context.Background(), dialer, \"tcp\",\n\t\t[]net.IP{net.ParseIP(\"::1\")}, // primary - will fail (no listener on ::1:port)\n\t\t[]net.IP{net.ParseIP(\"127.0.0.1\")}, // fallback - has listener\n\t\tport)\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\tconn.Close()\n}\n\nfunc TestDialParallel_BothFail(t *testing.T) {\n\tdialer := &net.Dialer{Timeout: 500 * time.Millisecond}\n\t_, err := dialParallel(context.Background(), dialer, \"tcp\",\n\t\t[]net.IP{net.ParseIP(\"::1\")},\n\t\t[]net.IP{net.ParseIP(\"127.0.0.1\")},\n\t\t\"0\")\n\tif err == nil {\n\t\tt.Fatal(\"expected error when both groups fail, got nil\")\n\t}\n}\n\nfunc TestSplitIPsByVersion(t *testing.T) {\n\tips := []net.IP{\n\t\tnet.ParseIP(\"127.0.0.1\"),\n\t\tnet.ParseIP(\"::1\"),\n\t\tnet.ParseIP(\"10.0.0.1\"),\n\t\tnet.ParseIP(\"fe80::1\"),\n\t}\n\tv6, v4 := splitIPsByVersion(ips)\n\tif len(v6) != 2 {\n\t\tt.Errorf(\"expected 2 IPv6, got %d\", len(v6))\n\t}\n\tif len(v4) != 2 {\n\t\tt.Errorf(\"expected 2 IPv4, got %d\", len(v4))\n\t}\n}\n" }, { "path": "pkg/object/s3.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"os\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/aws/middleware\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3/types\"\n\t\"github.com/aws/smithy-go\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n)\n\nconst awsDefaultRegion = \"us-east-1\"\nconst s3RequestIDKey = \"X-Amz-Request-Id\"\n\ntype s3client struct {\n\ts3 *s3.Client\n\tbucket string\n\tregion string\n\tsc string\n\tdisableChecksum bool\n}\n\nfunc (s *s3client) String() string {\n\tif s.s3.Options().BaseEndpoint != nil {\n\t\tendpoint := *s.s3.Options().BaseEndpoint\n\t\tif idx := strings.Index(endpoint, \"://\"); idx >= 0 {\n\t\t\tendpoint = endpoint[idx+3:]\n\t\t}\n\t\treturn fmt.Sprintf(\"s3://%s/%s/\", endpoint, s.bucket)\n\t}\n\treturn fmt.Sprintf(\"s3://%s/\", s.bucket)\n}\n\nfunc (s *s3client) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 5 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc isExists(err error) bool {\n\tmsg := err.Error()\n\treturn strings.Contains(msg, \"BucketAlreadyExists\") || strings.Contains(msg, \"BucketAlreadyOwnedByYou\")\n}\n\nfunc (s *s3client) Create(ctx context.Context) error {\n\tif _, _, _, err := s.List(ctx, \"\", \"\", \"\", \"\", 1, true); err == nil {\n\t\treturn nil\n\t}\n\t_, err := s.s3.CreateBucket(ctx, &s3.CreateBucketInput{\n\t\tBucket: &s.bucket,\n\t\tCreateBucketConfiguration: &types.CreateBucketConfiguration{\n\t\t\tLocationConstraint: types.BucketLocationConstraint(s.region),\n\t\t}})\n\tif err != nil && isExists(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (s *s3client) Head(ctx context.Context, key string) (Object, error) {\n\tparam := s3.HeadObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\tr, err := s.s3.HeadObject(ctx, ¶m)\n\tif err != nil {\n\t\tvar notFound *types.NotFound\n\t\tif errors.As(err, ¬Found) {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tkey,\n\t\t*r.ContentLength,\n\t\t*r.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\tstring(r.StorageClass),\n\t}, nil\n}\n\nfunc (s *s3client) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tparams := &s3.GetObjectInput{Bucket: &s.bucket, Key: &key}\n\tif off > 0 || limit > 0 {\n\t\tvar r string\n\t\tif limit > 0 {\n\t\t\tr = fmt.Sprintf(\"bytes=%d-%d\", off, off+limit-1)\n\t\t} else {\n\t\t\tr = fmt.Sprintf(\"bytes=%d-\", off)\n\t\t}\n\t\tparams.Range = &r\n\t}\n\tattrs := ApplyGetters(getters...)\n\tresp, err := s.s3.GetObject(ctx, params)\n\tif err != nil {\n\t\tvar re s3.ResponseError\n\t\tif errors.As(err, &re) {\n\t\t\tattrs.SetRequestID(re.ServiceRequestID())\n\t\t}\n\t\treturn nil, err\n\t}\n\tif reqID, ok := middleware.GetRequestIDMetadata(resp.ResultMetadata); ok {\n\t\tattrs.SetRequestID(reqID)\n\t}\n\tif off == 0 && limit == -1 && !s.disableChecksum {\n\t\tcs := resp.Metadata[strings.ToLower(checksumAlgr)]\n\t\tif cs != \"\" && resp.ContentLength != nil {\n\t\t\tresp.Body = verifyChecksum(resp.Body, cs, *resp.ContentLength)\n\t\t}\n\t}\n\tattrs.SetStorageClass(string(resp.StorageClass))\n\treturn resp.Body, nil\n}\n\nfunc (s *s3client) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tvar body io.ReadSeeker\n\tif b, ok := in.(io.ReadSeeker); ok {\n\t\tbody = b\n\t} else {\n\t\tdata, err := io.ReadAll(in)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tbody = bytes.NewReader(data)\n\t}\n\tmimeType := utils.GuessMimeType(key)\n\tparams := &s3.PutObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tBody: body,\n\t\tContentType: &mimeType,\n\t\tStorageClass: types.StorageClass(s.sc),\n\t\tChecksumAlgorithm: \"\", // X-Amz-Content-Sha256: UNSIGNED-PAYLOAD\n\t}\n\tif !s.disableChecksum {\n\t\tchecksum := generateChecksum(body)\n\t\tparams.Metadata = map[string]string{checksumAlgr: checksum}\n\t}\n\tattrs := ApplyGetters(getters...)\n\tattrs.SetStorageClass(s.sc)\n\tresp, err := s.s3.PutObject(ctx, params)\n\tif err != nil {\n\t\tvar re s3.ResponseError\n\t\tif errors.As(err, &re) {\n\t\t\tattrs.SetRequestID(re.ServiceRequestID())\n\t\t}\n\t\treturn err\n\t}\n\tif reqID, ok := middleware.GetRequestIDMetadata(resp.ResultMetadata); ok {\n\t\tattrs.SetRequestID(reqID)\n\t}\n\treturn err\n}\n\nfunc (s *s3client) Copy(ctx context.Context, dst, src string) error {\n\tsrc = s.bucket + \"/\" + src\n\tparams := &s3.CopyObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &dst,\n\t\tCopySource: &src,\n\t\tStorageClass: types.StorageClass(s.sc),\n\t}\n\t_, err := s.s3.CopyObject(ctx, params)\n\treturn err\n}\n\nfunc (s *s3client) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tparam := s3.DeleteObjectInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t}\n\tresp, err := s.s3.DeleteObject(ctx, ¶m)\n\tattrs := ApplyGetters(getters...)\n\tif err != nil {\n\t\tvar re s3.ResponseError\n\t\tif errors.As(err, &re) {\n\t\t\tattrs.SetRequestID(re.ServiceRequestID())\n\t\t}\n\t\tif strings.Contains(err.Error(), \"NoSuchKey\") {\n\t\t\terr = nil\n\t\t}\n\t} else {\n\t\tif reqID, ok := middleware.GetRequestIDMetadata(resp.ResultMetadata); ok {\n\t\t\tattrs.SetRequestID(reqID)\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (s *s3client) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\tparam := s3.ListObjectsV2Input{\n\t\tBucket: &s.bucket,\n\t\tPrefix: &prefix,\n\t\tMaxKeys: aws.Int32(int32(limit)),\n\t\tEncodingType: types.EncodingTypeUrl,\n\t\tStartAfter: aws.String(start),\n\t\tDelimiter: aws.String(delimiter),\n\t}\n\tif token != \"\" {\n\t\tparam.ContinuationToken = aws.String(token)\n\t}\n\tresp, err := s.s3.ListObjectsV2(ctx, ¶m)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(resp.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\to := resp.Contents[i]\n\t\toKey, err := decodeKey(*o.Key, aws.String(string(resp.EncodingType)))\n\t\tif err != nil {\n\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode key %s\", *o.Key)\n\t\t}\n\t\tif !strings.HasPrefix(oKey, prefix) || oKey < start {\n\t\t\treturn nil, false, \"\", fmt.Errorf(\"found invalid key %s from List, prefix: %s, marker: %s\", oKey, prefix, start)\n\t\t}\n\t\tobjs[i] = &obj{\n\t\t\toKey,\n\t\t\t*o.Size,\n\t\t\t*o.LastModified,\n\t\t\tstrings.HasSuffix(oKey, \"/\"),\n\t\t\tstring(o.StorageClass),\n\t\t}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range resp.CommonPrefixes {\n\t\t\tprefix, err := decodeKey(*p.Prefix, aws.String(string(resp.EncodingType)))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, false, \"\", errors.WithMessagef(err, \"failed to decode commonPrefixes %s\", *p.Prefix)\n\t\t\t}\n\t\t\tobjs = append(objs, &obj{prefix, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\tvar isTruncated bool\n\tif resp.IsTruncated != nil {\n\t\tisTruncated = *resp.IsTruncated\n\t}\n\tvar nextMarker string\n\tif resp.NextContinuationToken != nil {\n\t\tnextMarker = *resp.NextContinuationToken\n\t}\n\treturn objs, isTruncated, nextMarker, nil\n}\n\nfunc (s *s3client) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (s *s3client) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tparams := &s3.CreateMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tStorageClass: types.StorageClass(s.sc),\n\t}\n\tresp, err := s.s3.CreateMultipartUpload(ctx, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: *resp.UploadId, MinPartSize: 5 << 20, MaxCount: 10000}, nil\n}\n\nfunc (s *s3client) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\tparams := &s3.UploadPartInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t\tBody: bytes.NewReader(body),\n\t\tPartNumber: aws.Int32(int32(num)),\n\t\tChecksumAlgorithm: \"\", // X-Amz-Content-Sha256: UNSIGNED-PAYLOAD\n\t}\n\tresp, err := s.s3.UploadPart(ctx, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: *resp.ETag}, nil\n}\n\nfunc (s *s3client) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tresp, err := s.s3.UploadPartCopy(ctx, &s3.UploadPartCopyInput{\n\t\tBucket: aws.String(s.bucket),\n\t\tCopySource: aws.String(s.bucket + \"/\" + srcKey),\n\t\tCopySourceRange: aws.String(fmt.Sprintf(\"bytes=%d-%d\", off, off+size-1)),\n\t\tKey: aws.String(key),\n\t\tPartNumber: aws.Int32(int32(num)),\n\t\tUploadId: aws.String(uploadID),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: *resp.CopyPartResult.ETag}, nil\n}\n\nfunc (s *s3client) AbortUpload(ctx context.Context, key string, uploadID string) {\n\tparams := &s3.AbortMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t}\n\t_, _ = s.s3.AbortMultipartUpload(ctx, params)\n}\n\nfunc (s *s3client) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\tvar s3Parts []types.CompletedPart\n\tfor i := range parts {\n\t\ts3Parts = append(s3Parts, types.CompletedPart{ETag: &parts[i].ETag, PartNumber: aws.Int32(int32(parts[i].Num))})\n\t}\n\tparams := &s3.CompleteMultipartUploadInput{\n\t\tBucket: &s.bucket,\n\t\tKey: &key,\n\t\tUploadId: &uploadID,\n\t\tMultipartUpload: &types.CompletedMultipartUpload{Parts: s3Parts},\n\t}\n\t_, err := s.s3.CompleteMultipartUpload(ctx, params)\n\treturn err\n}\n\nfunc (s *s3client) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tinput := &s3.ListMultipartUploadsInput{\n\t\tBucket: aws.String(s.bucket),\n\t\tKeyMarker: aws.String(marker),\n\t}\n\n\tresult, err := s.s3.ListMultipartUploads(ctx, input)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tparts[i] = &PendingPart{*u.Key, *u.UploadId, *u.Initiated}\n\t}\n\tvar nextMarker string\n\tif result.NextKeyMarker != nil {\n\t\tnextMarker = *result.NextKeyMarker\n\t}\n\treturn parts, nextMarker, nil\n}\n\nfunc (s *s3client) SetStorageClass(sc string) error {\n\ts.sc = sc\n\treturn nil\n}\n\nfunc autoS3Region(bucketName, accessKey, secretKey string) (string, error) {\n\tvar cfg aws.Config\n\tvar err error\n\tif accessKey != \"\" {\n\t\tcfg, err = config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, \"\")))\n\t} else {\n\t\tcfg, err = config.LoadDefaultConfig(ctx)\n\t}\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tcfg.HTTPClient = httpClient\n\tvar regions []string\n\tif r := os.Getenv(\"AWS_DEFAULT_REGION\"); r != \"\" {\n\t\tregions = []string{r}\n\t} else {\n\t\tregions = []string{awsDefaultRegion, \"cn-north-1\"}\n\t}\n\tvar result *s3.GetBucketLocationOutput\n\tfor _, r := range regions {\n\t\t// try to get bucket location\n\t\tcfg.Region = r\n\t\tclient := s3.NewFromConfig(cfg)\n\t\tresult, err = client.GetBucketLocation(ctx, &s3.GetBucketLocationInput{\n\t\t\tBucket: aws.String(bucketName),\n\t\t})\n\t\tif err == nil {\n\t\t\tlogger.Debugf(\"Get location of bucket %q from region %q endpoint success: %s\",\n\t\t\t\tbucketName, r, result.LocationConstraint)\n\t\t\treturn string(result.LocationConstraint), nil\n\t\t}\n\t\t// continue to try other regions if the credentials are invalid, otherwise stop trying.\n\t\tvar err1 *smithy.GenericAPIError\n\t\tif errors.As(err, &err1) {\n\t\t\tif err1.Code != \"SignatureDoesNotMatch\" && err1.Code != \"InvalidAccessKeyId\" {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\t\t}\n\t\tlogger.Debugf(\"Fail to get location of bucket %q from region %q endpoint: %s\", bucketName, r, err)\n\t}\n\treturn \"\", err\n}\n\nfunc parseRegion(endpoint string) string {\n\tif strings.HasPrefix(endpoint, \"s3-\") || strings.HasPrefix(endpoint, \"s3.\") {\n\t\tendpoint = endpoint[3:]\n\t}\n\tif strings.HasPrefix(endpoint, \"dualstack\") {\n\t\tendpoint = endpoint[len(\"dualstack.\"):]\n\t}\n\tif endpoint == \"amazonaws.com\" {\n\t\tendpoint = awsDefaultRegion + \".\" + endpoint\n\t}\n\tregion := strings.Split(endpoint, \".\")[0]\n\tif region == \"external-1\" {\n\t\tregion = awsDefaultRegion\n\t}\n\treturn region\n}\n\nfunc defaultPathStyle() bool {\n\tv := os.Getenv(\"JFS_S3_VHOST_STYLE\")\n\treturn v == \"\" || v == \"0\" || v == \"false\"\n}\n\nvar oracleCompileRegexp = `.*\\.compat.objectstorage\\.(.*)\\.oraclecloud\\.com`\nvar OVHCompileRegexp = `^s3\\.(\\w*)(\\.\\w*)?\\.cloud\\.ovh\\.net$`\n\nfunc newS3(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tif len(strings.Split(endpoint, \".\")) > 1 && !strings.HasSuffix(endpoint, \".amazonaws.com\") {\n\t\t\tendpoint = fmt.Sprintf(\"http://%s\", endpoint)\n\t\t} else {\n\t\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t\t}\n\t}\n\tendpoint = strings.Trim(endpoint, \"/\")\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err.Error())\n\t}\n\n\tvar (\n\t\tbucketName string\n\t\tregion string\n\t\tep string\n\t)\n\n\tif uri.Path != \"\" {\n\t\t// [ENDPOINT]/[BUCKET]\n\t\tpathParts := strings.Split(uri.Path, \"/\")\n\t\tbucketName = pathParts[1]\n\t\tif strings.Contains(uri.Host, \".amazonaws.com\") {\n\t\t\t// standard s3\n\t\t\t// s3-[REGION].[REST_OF_ENDPOINT]/[BUCKET]\n\t\t\t// s3.[REGION].amazonaws.com[.cn]/[BUCKET]\n\t\t\tendpoint = uri.Host\n\t\t\tregion = parseRegion(endpoint)\n\t\t} else {\n\t\t\t// compatible s3\n\t\t\tep = uri.Host\n\t\t}\n\t} else {\n\t\t// [BUCKET].[ENDPOINT]\n\t\thostParts := strings.SplitN(uri.Host, \".\", 2)\n\t\tif len(hostParts) == 1 {\n\t\t\t// take endpoint as bucketname\n\t\t\tbucketName = hostParts[0]\n\t\t\tif region, err = autoS3Region(bucketName, accessKey, secretKey); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"Can't guess your region for bucket %s: %s\", bucketName, err)\n\t\t\t}\n\t\t} else {\n\t\t\t// get region or endpoint\n\t\t\tif strings.Contains(uri.Host, \".amazonaws.com\") {\n\t\t\t\tvpcCompile := regexp.MustCompile(`^.*\\.(.*)\\.vpce\\.amazonaws\\.com`)\n\t\t\t\t// vpc link\n\t\t\t\tif vpcCompile.MatchString(uri.Host) {\n\t\t\t\t\tbucketName = hostParts[0]\n\t\t\t\t\tep = hostParts[1]\n\t\t\t\t\tif submatch := vpcCompile.FindStringSubmatch(uri.Host); len(submatch) == 2 {\n\t\t\t\t\t\tregion = submatch[1]\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// standard s3\n\t\t\t\t\t// [BUCKET].s3-[REGION].[REST_OF_ENDPOINT]\n\t\t\t\t\t// [BUCKET].s3.[REGION].amazonaws.com[.cn]\n\t\t\t\t\thostParts = strings.SplitN(uri.Host, \".s3\", 2)\n\t\t\t\t\tbucketName = hostParts[0]\n\t\t\t\t\tendpoint = \"s3\" + hostParts[1]\n\t\t\t\t\tregion = parseRegion(endpoint)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// compatible s3\n\t\t\t\tbucketName = hostParts[0]\n\t\t\t\tep = hostParts[1]\n\n\t\t\t\tfor _, compileRegexp := range []string{oracleCompileRegexp, OVHCompileRegexp} {\n\t\t\t\t\tcompile := regexp.MustCompile(compileRegexp)\n\t\t\t\t\tif compile.MatchString(ep) {\n\t\t\t\t\t\tif submatch := compile.FindStringSubmatch(ep); len(submatch) >= 2 {\n\t\t\t\t\t\t\tregion = submatch[1]\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif region == \"\" {\n\t\tregion = os.Getenv(\"AWS_REGION\")\n\t}\n\tif region == \"\" {\n\t\tregion = os.Getenv(\"AWS_DEFAULT_REGION\")\n\t}\n\tif region == \"\" {\n\t\tregion = awsDefaultRegion\n\t}\n\tvar optFns []func(*s3.Options)\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\toptFns = append(optFns, func(options *s3.Options) {\n\t\toptions.EndpointOptions.DisableHTTPS = !ssl\n\t\toptions.Region = region\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\n\tdisable100Continue := strings.EqualFold(uri.Query().Get(\"disable-100-continue\"), \"true\")\n\tif disable100Continue {\n\t\tlogger.Infof(\"HTTP header 100-Continue is disabled\")\n\t\toptFns = append(optFns, func(options *s3.Options) {\n\t\t\toptions.ContinueHeaderThresholdBytes = -1\n\t\t})\n\t}\n\tdisableChecksum := strings.EqualFold(uri.Query().Get(\"disable-checksum\"), \"true\")\n\tif disableChecksum {\n\t\tlogger.Infof(\"default CRC checksum is disabled\")\n\t}\n\n\tif ep != \"\" {\n\t\toptFns = append(optFns, func(options *s3.Options) {\n\t\t\toptions.BaseEndpoint = aws.String(uri.Scheme + \"://\" + ep)\n\t\t\toptions.UsePathStyle = defaultPathStyle()\n\t\t})\n\t}\n\tvar cfg aws.Config\n\tif accessKey == \"anonymous\" {\n\t\tcfg, err = config.LoadDefaultConfig(ctx,\n\t\t\tconfig.WithCredentialsProvider(aws.AnonymousCredentials{}))\n\t} else if accessKey != \"\" {\n\t\tcfg, err = config.LoadDefaultConfig(ctx,\n\t\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\t} else {\n\t\tcfg, err = config.LoadDefaultConfig(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcfg.HTTPClient = httpClient\n\tclient := s3.NewFromConfig(cfg, optFns...)\n\treturn &s3client{bucket: bucketName, s3: client, disableChecksum: disableChecksum, region: region}, nil\n}\n\nfunc init() {\n\tRegister(\"s3\", newS3)\n}\n" }, { "path": "pkg/object/s3_test.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc Test_s3client_full_string(t *testing.T) {\n\ttests := []struct {\n\t\tendpoint string\n\t\twant string\n\t}{\n\t\t{endpoint: \"s3.compatible.site/bucket\", want: \"s3://s3.compatible.site/bucket/\"},\n\t\t{endpoint: \"http://s3.compatible.site/bucket\", want: \"s3://s3.compatible.site/bucket/\"},\n\t\t{endpoint: \"s3://s3.compatible.site/bucket\", want: \"s3://s3.compatible.site/bucket/\"},\n\t\t{endpoint: \"https://mybucket.s3.us-east-2.amazonaws.com\", want: \"s3://mybucket/\"},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.endpoint, func(t *testing.T) {\n\t\t\tstor, err := newS3(tt.endpoint, \"\", \"\", \"\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"newS3() error = %v\", err)\n\t\t\t}\n\t\t\tassert.Equalf(t, tt.want, stor.String(), \"Display full address of s3 compatible object storage\")\n\t\t})\n\t}\n}\n" }, { "path": "pkg/object/scw.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n)\n\ntype scw struct {\n\ts3client\n}\n\nfunc (s *scw) String() string {\n\treturn fmt.Sprintf(\"scw://%s/\", s.s3client.bucket)\n}\n\nfunc (s *scw) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 5 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 1000,\n\t}\n}\n\nfunc newScw(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\thostParts := strings.Split(uri.Host, \".\")\n\tbucket := hostParts[0]\n\tregion := hostParts[2]\n\tendpoint = uri.Scheme + \"://\" + uri.Host[len(bucket)+1:]\n\tif accessKey == \"\" {\n\t\taccessKey = os.Getenv(\"SCW_ACCESS_KEY\")\n\t}\n\tif secretKey == \"\" {\n\t\tsecretKey = os.Getenv(\"SCW_SECRET_KEY\")\n\t}\n\n\tawsCfg, err := config.LoadDefaultConfig(ctx,\n\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to load config: %s\", err)\n\t}\n\tclient := s3.NewFromConfig(awsCfg, func(options *s3.Options) {\n\t\toptions.Region = region\n\t\toptions.BaseEndpoint = aws.String(endpoint)\n\t\toptions.EndpointOptions.DisableHTTPS = !ssl\n\t\toptions.UsePathStyle = false\n\t\toptions.HTTPClient = httpClient\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\treturn &scw{s3client{bucket: bucket, s3: client, region: region}}, nil\n}\n\nfunc init() {\n\tRegister(\"scw\", newScw)\n}\n" }, { "path": "pkg/object/sftp.go", "content": "//go:build !nosftp\n// +build !nosftp\n\n// Part of this file is borrowed from Rclone under MIT license:\n// https://github.com/ncw/rclone/blob/master/backend/sftp/sftp.go\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"os/user\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/pkg/sftp\"\n\t\"golang.org/x/crypto/ssh\"\n\t\"golang.org/x/crypto/ssh/agent\"\n\t\"golang.org/x/crypto/ssh/knownhosts\"\n\t\"golang.org/x/term\"\n)\n\n// conn encapsulates an ssh client and corresponding sftp client\ntype conn struct {\n\tsshClient *ssh.Client\n\tsftpClient *sftp.Client\n\terr chan error\n}\n\n// Wait for connection to close\nfunc (c *conn) wait() {\n\tc.err <- c.sshClient.Conn.Wait()\n}\n\n// Closes the connection\nfunc (c *conn) close() error {\n\tsftpErr := c.sftpClient.Close()\n\tsshErr := c.sshClient.Close()\n\tif sftpErr != nil {\n\t\treturn sftpErr\n\t}\n\treturn sshErr\n}\n\n// Returns an error if closed\nfunc (c *conn) closed() error {\n\tselect {\n\tcase err := <-c.err:\n\t\treturn err\n\tdefault:\n\t}\n\treturn nil\n}\n\ntype sftpStore struct {\n\tDefaultObjectStorage\n\thost string\n\tport string\n\troot string\n\tconfig *ssh.ClientConfig\n\tpoolMu sync.Mutex\n\tpool []*conn\n}\n\n// Open a new connection to the SFTP server.\nfunc (f *sftpStore) sftpConnection() (c *conn, err error) {\n\tc = &conn{\n\t\terr: make(chan error, 1),\n\t}\n\tconn, err := net.Dial(\"tcp\", net.JoinHostPort(f.host, f.port))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsshc, chans, reqs, err := ssh.NewClientConn(conn, net.JoinHostPort(f.host, f.port), f.config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tc.sshClient = ssh.NewClient(sshc, chans, reqs)\n\tc.sftpClient, err = sftp.NewClient(c.sshClient)\n\tif err != nil {\n\t\t_ = c.sshClient.Close()\n\t\treturn nil, errors.Wrap(err, \"couldn't initialise SFTP\")\n\t}\n\tgo c.wait()\n\treturn c, nil\n}\n\n// Get an SFTP connection from the pool, or open a new one\nfunc (f *sftpStore) getSftpConnection() (c *conn, err error) {\n\tf.poolMu.Lock()\n\tfor len(f.pool) > 0 {\n\t\tc = f.pool[0]\n\t\tf.pool = f.pool[1:]\n\t\terr := c.closed()\n\t\tif err == nil {\n\t\t\tbreak\n\t\t}\n\t\tc = nil\n\t}\n\tf.poolMu.Unlock()\n\tif c != nil {\n\t\treturn c, nil\n\t}\n\treturn f.sftpConnection()\n}\n\n// Return an SFTP connection to the pool\n//\n// It nils the pointed to connection out so it can't be reused\n//\n// if err is not nil then it checks the connection is alive using a\n// Getwd request\nfunc (f *sftpStore) putSftpConnection(c *conn, err error) {\n\tif err != nil {\n\t\t// work out if this is an expected error\n\t\tunderlyingErr := errors.Cause(err)\n\t\tisRegularError := false\n\t\tswitch underlyingErr {\n\t\tcase os.ErrNotExist:\n\t\t\tisRegularError = true\n\t\tdefault:\n\t\t\tswitch underlyingErr.(type) {\n\t\t\tcase *sftp.StatusError, *os.PathError:\n\t\t\t\tisRegularError = true\n\t\t\t}\n\t\t}\n\t\t// If not a regular SFTP error code then check the connection\n\t\tif !isRegularError {\n\t\t\t_, nopErr := c.sftpClient.Getwd()\n\t\t\tif nopErr != nil {\n\t\t\t\t_ = c.close()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\tf.poolMu.Lock()\n\tf.pool = append(f.pool, c)\n\tf.poolMu.Unlock()\n}\n\nfunc (f *sftpStore) String() string {\n\treturn fmt.Sprintf(\"sftp://%s@%s:%s\", f.config.User, f.host, f.root)\n}\n\n// always preserve suffix `/` for directory key\nfunc (f *sftpStore) path(key string) string {\n\treturn f.root + key\n}\n\nfunc (f *sftpStore) Head(ctx context.Context, key string) (Object, error) {\n\tc, err := f.getSftpConnection()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\n\tinfo, err := c.sftpClient.Lstat(f.path(key))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar isSymlink bool\n\tif info.Mode()&os.ModeSymlink != 0 {\n\t\tisSymlink = true\n\t\tinfo, err = c.sftpClient.Stat(f.path(key))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn f.fileInfo(key, info, isSymlink), nil\n}\n\nfunc (f *sftpStore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tc, err := f.getSftpConnection()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\n\tp := f.path(key)\n\tff, err := c.sftpClient.Open(p)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfinfo, err := ff.Stat()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif finfo.IsDir() || off >= finfo.Size() {\n\t\t_ = ff.Close()\n\t\treturn io.NopCloser(bytes.NewBuffer([]byte{})), nil\n\t}\n\n\tif limit > 0 {\n\t\treturn &SectionReaderCloser{\n\t\t\tSectionReader: io.NewSectionReader(ff, off, limit),\n\t\t\tCloser: ff,\n\t\t}, nil\n\t}\n\treturn ff, err\n}\n\nfunc (f *sftpStore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) (err error) {\n\tc, err := f.getSftpConnection()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\n\tp := f.path(key)\n\tif strings.HasSuffix(p, dirSuffix) {\n\t\treturn c.sftpClient.MkdirAll(p)\n\t}\n\tif err := c.sftpClient.MkdirAll(filepath.Dir(p)); err != nil {\n\t\treturn err\n\t}\n\n\tvar tmp string\n\tif PutInplace {\n\t\ttmp = p\n\t} else {\n\t\tname := path.Base(p)\n\t\tif len(name) > 200 {\n\t\t\tname = name[:200]\n\t\t}\n\t\ttmp = TmpFilePath(p, name)\n\t\tdefer func() {\n\t\t\tif err != nil {\n\t\t\t\t_ = c.sftpClient.Remove(tmp)\n\t\t\t}\n\t\t}()\n\t}\n\tff, err := c.sftpClient.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC)\n\tif err != nil {\n\t\treturn err\n\t}\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\t_, err = io.CopyBuffer(ff, in, *buf)\n\tif err != nil {\n\t\t_ = ff.Close()\n\t\treturn err\n\t}\n\terr = ff.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !PutInplace {\n\t\t_ = c.sftpClient.Remove(p)\n\t\treturn c.sftpClient.Rename(tmp, p)\n\t}\n\treturn nil\n}\n\nfunc (f *sftpStore) Chtimes(key string, mtime time.Time) (err error) {\n\tvar c *conn\n\tc, err = f.getSftpConnection()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\t// fixme: 1. The Chtimes of sftp always follows link 2. Only pass the mtime field to avoid updating atime\n\t// ref: https://datatracker.ietf.org/doc/html/draft-ietf-secsh-filexfer-13#section-8.6\n\treturn c.sftpClient.Chtimes(f.path(key), mtime, mtime)\n}\n\nfunc (f *sftpStore) Chmod(key string, mode os.FileMode) (err error) {\n\tvar c *conn\n\tc, err = f.getSftpConnection()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\treturn c.sftpClient.Chmod(f.path(key), mode)\n}\n\nfunc (f *sftpStore) Chown(key string, owner, group string) (err error) {\n\tvar c *conn\n\tc, err = f.getSftpConnection()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\tuid := utils.LookupUser(owner)\n\tgid := utils.LookupGroup(group)\n\tif uid == -1 || gid == -1 {\n\t\treturn fmt.Errorf(\"user(%s):group(%s) not found\", owner, group)\n\t}\n\treturn c.sftpClient.Chown(f.path(key), uid, gid)\n}\n\nfunc (f *sftpStore) Symlink(oldName, newName string) error {\n\tc, err := f.getSftpConnection()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\tp := f.path(newName)\n\terr = c.sftpClient.Symlink(oldName, p)\n\tif err != nil && os.IsNotExist(err) {\n\t\t_ = c.sftpClient.MkdirAll(filepath.Dir(p))\n\t\terr = c.sftpClient.Symlink(oldName, p)\n\t}\n\treturn err\n}\n\nfunc (f *sftpStore) Readlink(name string) (link string, err error) {\n\tc, err := f.getSftpConnection()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\treturn c.sftpClient.ReadLink(f.path(name))\n}\n\nfunc (f *sftpStore) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tc, err := f.getSftpConnection()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\terr = c.sftpClient.Remove(strings.TrimRight(f.path(key), dirSuffix))\n\tif err != nil && os.IsNotExist(err) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (f *sftpStore) sortByName(c *sftp.Client, path string, fis []os.FileInfo, followLink bool) []*mEntry {\n\tmEntries := make([]*mEntry, len(fis))\n\tfor i, e := range fis {\n\t\tisSymlink := e.Mode()&os.ModeSymlink != 0\n\t\tif e.IsDir() {\n\t\t\tmEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}\n\t\t} else if isSymlink && followLink {\n\t\t\tvar fi os.FileInfo\n\t\t\tp := path + e.Name()\n\t\t\tfi, err := c.Stat(p)\n\t\t\tif err != nil {\n\t\t\t\tmEntries[i] = &mEntry{e, e.Name(), nil, true}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tname := e.Name()\n\t\t\tif fi.IsDir() {\n\t\t\t\tname = e.Name() + dirSuffix\n\t\t\t}\n\t\t\tmEntries[i] = &mEntry{e, name, fi, false}\n\t\t} else {\n\t\t\tmEntries[i] = &mEntry{e, e.Name(), nil, isSymlink}\n\t\t}\n\t}\n\tsort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })\n\treturn mEntries\n}\n\nfunc (f *sftpStore) fileInfo(key string, fi os.FileInfo, isSymlink bool) Object {\n\towner, group := getOwnerGroup(fi)\n\tff := &file{\n\t\tobj{key, fi.Size(), fi.ModTime(), fi.IsDir(), \"\"},\n\t\towner,\n\t\tgroup,\n\t\tfi.Mode(),\n\t\tisSymlink,\n\t}\n\tif fi.IsDir() {\n\t\tif key != \"\" && !strings.HasSuffix(key, \"/\") {\n\t\t\tff.key += \"/\"\n\t\t}\n\t\tff.size = 0\n\t}\n\treturn ff\n}\n\nfunc (f *sftpStore) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\n\tc, err := f.getSftpConnection()\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\n\tvar objs []Object\n\tdir := f.path(prefix)\n\tif !strings.HasSuffix(dir, \"/\") {\n\t\tdir = filepath.Dir(dir)\n\t\tif !strings.HasSuffix(dir, dirSuffix) {\n\t\t\tdir += dirSuffix\n\t\t}\n\t} else if marker == \"\" {\n\t\tobj, err := f.Head(ctx, prefix)\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil, false, \"\", nil\n\t\t\t}\n\t\t\treturn nil, false, \"\", err\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\tinfos, err := c.sftpClient.ReadDir(dir)\n\tif err != nil {\n\t\tif os.IsPermission(err) {\n\t\t\tlogger.Warnf(\"skip %s: %s\", dir, err)\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\tif os.IsNotExist(err) {\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\n\tentries := f.sortByName(c.sftpClient, dir, infos, followLink)\n\tfor _, e := range entries {\n\t\tp := path.Join(dir, e.Name())\n\t\tif e.IsDir() {\n\t\t\tp = p + \"/\"\n\t\t}\n\t\tif !strings.HasPrefix(p, f.root) {\n\t\t\tcontinue\n\t\t}\n\t\tkey := p[len(f.root):]\n\t\tif !strings.HasPrefix(key, prefix) || (marker != \"\" && key <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\tinfo := e.Info()\n\t\tf := toFile(key, info, e.isSymlink, getOwnerGroup)\n\t\tobjs = append(objs, f)\n\t\tif len(objs) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc sshInteractive(user, instruction string, questions []string, echos []bool) (answers []string, err error) {\n\tif len(questions) == 0 {\n\t\tfmt.Print(user, instruction)\n\t} else {\n\t\tanswers = make([]string, len(questions))\n\t\tfor i, q := range questions {\n\t\t\tfmt.Print(q)\n\t\t\tvar ans []byte\n\t\t\tif echos[i] {\n\t\t\t\t_, err = fmt.Scanln(&answers[i])\n\t\t\t} else {\n\t\t\t\tans, err = term.ReadPassword(int(syscall.Stdin))\n\t\t\t\tanswers[i] = string(ans)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"read password: %s\", err)\n\t\t\t}\n\t\t}\n\t}\n\treturn answers, nil\n}\n\nfunc unescape(original string) string {\n\tif escaped, err := url.QueryUnescape(original); err != nil {\n\t\tlogger.Warnf(\"unescape(%s) error: %s\", original, err)\n\t\treturn original\n\t} else {\n\t\treturn escaped\n\t}\n}\n\nfunc newSftp(endpoint, username, pass, token string) (ObjectStorage, error) {\n\tidx := strings.LastIndex(endpoint, \":\")\n\thost, port, err := net.SplitHostPort(endpoint[:idx])\n\tif err != nil && strings.Contains(err.Error(), \"missing port\") {\n\t\thost, port, err = net.SplitHostPort(endpoint[:idx] + \":22\")\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to parse host from endpoint (%s): %q\", endpoint, err)\n\t}\n\troot := filepath.Clean(endpoint[idx+1:])\n\tif runtime.GOOS == \"windows\" {\n\t\troot = strings.Replace(root, \"\\\\\", \"/\", -1)\n\t}\n\t// append suffix `/` removed by filepath.Clean()\n\tif strings.HasSuffix(endpoint[idx+1:], dirSuffix) {\n\t\troot = root + dirSuffix\n\t}\n\n\tif username == \"\" {\n\t\tu, _ := user.Current()\n\t\tif u != nil {\n\t\t\tusername = u.Username\n\t\t}\n\t}\n\tusername = unescape(username)\n\tvar auth []ssh.AuthMethod\n\tif pass != \"\" {\n\t\tauth = append(auth, ssh.Password(unescape(pass)))\n\t}\n\n\tvar signers []ssh.Signer\n\tif privateKeyPath := os.Getenv(\"SSH_PRIVATE_KEY_PATH\"); privateKeyPath != \"\" {\n\t\tkey, err := os.ReadFile(privateKeyPath)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unable to read private key, error: %v\", err)\n\t\t}\n\t\tsigner, err := ssh.ParsePrivateKey(key)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unable to parse private key, error: %v\", err)\n\t\t}\n\t\tsigners = append(signers, signer)\n\t} else {\n\t\thome := filepath.Join(os.Getenv(\"HOME\"), \".ssh\")\n\t\tvar algo = []string{\"rsa\", \"dsa\", \"ecdsa\", \"ecdsa_sk\", \"ed25519\", \"xmss\"}\n\t\tfor _, a := range algo {\n\t\t\tkey, err := os.ReadFile(filepath.Join(home, \"id_\"+a))\n\t\t\tif err != nil {\n\t\t\t\tkey, err = os.ReadFile(filepath.Join(home, \"id_\"+a+\"-cert\"))\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\tsigner, err := ssh.ParsePrivateKey(key)\n\t\t\t\tif err == nil {\n\t\t\t\t\tsigners = append(signers, signer)\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Debugf(\"load private key %s: %s\", filepath.Join(home, \"id_\"+a), err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tsocket := os.Getenv(\"SSH_AUTH_SOCK\")\n\tif socket != \"\" {\n\t\tconn, err := net.Dial(\"unix\", socket)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Failed to open SSH_AUTH_SOCK: %v\", err)\n\t\t} else {\n\t\t\tagent := agent.NewClient(conn)\n\t\t\tsigner, err := agent.Signers()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"load signer from agent: %s\", err)\n\t\t\t} else {\n\t\t\t\tsigners = append(signers, signer...)\n\t\t\t}\n\t\t}\n\t}\n\tif len(signers) > 0 {\n\t\tauth = append(auth, ssh.PublicKeys(signers...))\n\t}\n\n\tif pass == \"\" {\n\t\tauth = append(auth, ssh.KeyboardInteractive(sshInteractive))\n\t}\n\tvar hostKeyCallback ssh.HostKeyCallback\n\tif kn := os.Getenv(\"SSH_KNOWN_HOSTS\"); kn != \"\" {\n\t\tvar err error\n\t\thostKeyCallback, err = knownhosts.New(kn)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\thostKeyCallback = ssh.InsecureIgnoreHostKey()\n\t}\n\n\tconfig := &ssh.ClientConfig{\n\t\tUser: username,\n\t\tHostKeyCallback: hostKeyCallback,\n\t\tTimeout: time.Second * 3,\n\t\tAuth: auth,\n\t}\n\tf := &sftpStore{\n\t\thost: host,\n\t\tport: port,\n\t\troot: root,\n\t\tconfig: config,\n\t}\n\n\tc, err := f.getSftpConnection()\n\tif err != nil && strings.Contains(err.Error(), \"unable to authenticate\") &&\n\t\tpass == \"\" && os.Getenv(\"SSH_PRIVATE_KEY_PATH\") == \"\" {\n\t\tfmt.Printf(\"%s@%s's password: \", username, host)\n\t\tvar password []byte\n\t\tpassword, err = term.ReadPassword(int(syscall.Stdin))\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Read password: %s\", err.Error())\n\t\t}\n\t\tf.config.Auth = append(f.config.Auth, ssh.Password(string(password)))\n\t\tc, err = f.getSftpConnection()\n\t}\n\tif err != nil {\n\t\tlogger.Errorf(\"connect to %s failed: %s\", host, err)\n\t\treturn nil, err\n\t}\n\tdefer func() { f.putSftpConnection(c, err) }()\n\n\treturn f, nil\n}\n\nfunc init() {\n\tRegister(\"sftp\", newSftp)\n}\n" }, { "path": "pkg/object/sharding.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"container/heap\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"hash/fnv\"\n\t\"io\"\n\t\"strings\"\n\t\"time\"\n)\n\ntype sharded struct {\n\tDefaultObjectStorage\n\tstores []ObjectStorage\n}\n\nfunc (s *sharded) String() string {\n\treturn fmt.Sprintf(\"shard%d://%s\", len(s.stores), s.stores[0])\n}\n\nfunc (s *sharded) Limits() Limits {\n\tl := s.stores[0].Limits()\n\tl.IsSupportUploadPartCopy = false\n\treturn l\n}\n\nfunc (s *sharded) Create(ctx context.Context) error {\n\tfor _, o := range s.stores {\n\t\tif err := o.Create(ctx); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (s *sharded) pick(key string) ObjectStorage {\n\th := fnv.New32a()\n\t_, _ = h.Write([]byte(key))\n\ti := h.Sum32() % uint32(len(s.stores))\n\treturn s.stores[i]\n}\n\nfunc (s *sharded) Head(ctx context.Context, key string) (Object, error) {\n\treturn s.pick(key).Head(ctx, key)\n}\n\nfunc (s *sharded) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\treturn s.pick(key).Get(ctx, key, off, limit, getters...)\n}\n\nfunc (s *sharded) Put(ctx context.Context, key string, body io.Reader, getters ...AttrGetter) error {\n\treturn s.pick(key).Put(ctx, key, body, getters...)\n}\n\nfunc (s *sharded) Copy(ctx context.Context, dst, src string) error {\n\treturn notSupported\n}\n\nfunc (s *sharded) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\treturn s.pick(key).Delete(ctx, key, getters...)\n}\n\nfunc (s *sharded) SetStorageClass(sc string) error {\n\tvar err = notSupported\n\tfor _, o := range s.stores {\n\t\tif os, ok := o.(SupportStorageClass); ok {\n\t\t\terr = os.SetStorageClass(sc)\n\t\t}\n\t}\n\treturn err\n}\n\nconst maxResults = 10000\n\n// ListAll lists all keys that starts at marker from object storage.\nfunc ListAll(ctx context.Context, store ObjectStorage, prefix, marker string, followLink, sort bool) (<-chan Object, error) {\n\tif ch, err := store.ListAll(ctx, prefix, marker, followLink); err == nil {\n\t\treturn ch, nil\n\t} else if !errors.Is(err, notSupported) {\n\t\treturn nil, err\n\t}\n\n\tstartTime := time.Now()\n\tout := make(chan Object, maxResults)\n\tlogger.Debugf(\"Listing objects from %s marker %q\", store, marker)\n\tobjs, hasMore, nextToken, err := store.List(ctx, prefix, marker, \"\", \"\", maxResults, followLink)\n\tif errors.Is(err, notSupported) {\n\t\treturn ListAllWithDelimiter(ctx, store, prefix, marker, \"\", followLink)\n\t}\n\tif err != nil {\n\t\tlogger.Errorf(\"Can't list %s: %s\", store, err.Error())\n\t\treturn nil, err\n\t}\n\tlogger.Debugf(\"Found %d object from %s in %s\", len(objs), store, time.Since(startTime))\n\tgo func() {\n\t\tdefer close(out)\n\t\tlastkey := \"\"\n\t\tfirst := true\n\t\tfor {\n\t\t\tfor _, obj := range objs {\n\t\t\t\tkey := obj.Key()\n\t\t\t\tif sort && !first && key <= lastkey {\n\t\t\t\t\tlogger.Errorf(\"The keys are out of order: marker %q, last %q current %q\", marker, lastkey, key)\n\t\t\t\t\tout <- nil\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tlastkey = key\n\t\t\t\tout <- obj\n\t\t\t\tfirst = false\n\t\t\t}\n\t\t\tif !hasMore {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tmarker = lastkey\n\t\t\tstartTime = time.Now()\n\t\t\tlogger.Debugf(\"Continue listing objects from %s marker %q\", store, marker)\n\t\t\tvar nextToken2 string\n\t\t\tobjs, hasMore, nextToken2, err = store.List(ctx, prefix, marker, nextToken, \"\", maxResults, followLink)\n\t\t\tfor err != nil {\n\t\t\t\tlogger.Warnf(\"Fail to list: %s, retry again\", err.Error())\n\t\t\t\t// slow down\n\t\t\t\ttime.Sleep(time.Millisecond * 100)\n\t\t\t\tobjs, hasMore, nextToken, err = store.List(ctx, prefix, marker, nextToken, \"\", maxResults, followLink)\n\t\t\t}\n\t\t\tnextToken = nextToken2\n\t\t\tlogger.Debugf(\"Found %d object from %s in %s\", len(objs), store, time.Since(startTime))\n\t\t}\n\t}()\n\treturn out, nil\n}\n\ntype nextKey struct {\n\to Object\n\tch <-chan Object\n}\n\ntype nextObjects struct {\n\tos []nextKey\n}\n\nfunc (s *nextObjects) Len() int { return len(s.os) }\nfunc (s *nextObjects) Less(i, j int) bool { return s.os[i].o.Key() < s.os[j].o.Key() }\nfunc (s *nextObjects) Swap(i, j int) { s.os[i], s.os[j] = s.os[j], s.os[i] }\nfunc (s *nextObjects) Push(o interface{}) { s.os = append(s.os, o.(nextKey)) }\nfunc (s *nextObjects) Pop() interface{} {\n\to := s.os[len(s.os)-1]\n\ts.os = s.os[:len(s.os)-1]\n\treturn o\n}\n\nfunc (s *sharded) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\theads := &nextObjects{make([]nextKey, 0)}\n\tfor i := range s.stores {\n\t\tch, err := ListAll(ctx, s.stores[i], prefix, marker, followLink, true)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"list %s: %s\", s.stores[i], err)\n\t\t}\n\t\tfirst := <-ch\n\t\tif first != nil {\n\t\t\theads.Push(nextKey{first, ch})\n\t\t}\n\t}\n\theap.Init(heads)\n\n\tout := make(chan Object, 1000)\n\tgo func() {\n\t\tfor heads.Len() > 0 {\n\t\t\tn := heap.Pop(heads).(nextKey)\n\t\t\tout <- n.o\n\t\t\to := <-n.ch\n\t\t\tif o != nil {\n\t\t\t\theap.Push(heads, nextKey{o, n.ch})\n\t\t\t}\n\t\t}\n\t\tclose(out)\n\t}()\n\treturn out, nil\n}\n\nfunc (s *sharded) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\treturn s.pick(key).CreateMultipartUpload(ctx, key)\n}\n\nfunc (s *sharded) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\treturn s.pick(key).UploadPart(ctx, key, uploadID, num, body)\n}\n\nfunc (s *sharded) AbortUpload(ctx context.Context, key string, uploadID string) {\n\ts.pick(key).AbortUpload(ctx, key, uploadID)\n}\n\nfunc (s *sharded) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\treturn s.pick(key).CompleteUpload(ctx, key, uploadID, parts)\n}\n\nfunc NewSharded(name, endpoint, ak, sk, token string, shards int) (ObjectStorage, error) {\n\tstores := make([]ObjectStorage, shards)\n\tvar err error\n\tfor i := range stores {\n\t\tep := fmt.Sprintf(endpoint, i)\n\t\tif strings.HasSuffix(ep, \"%!(EXTRA int=0)\") {\n\t\t\treturn nil, fmt.Errorf(\"can not generate different endpoint using %s\", endpoint)\n\t\t}\n\t\tstores[i], err = CreateStorage(name, ep, ak, sk, token)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn &sharded{stores: stores}, nil\n}\n" }, { "path": "pkg/object/space.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n)\n\ntype space struct {\n\ts3client\n}\n\nfunc (s *space) String() string {\n\treturn fmt.Sprintf(\"space://%s/\", s.s3client.bucket)\n}\n\nfunc (s *space) Limits() Limits {\n\treturn s.s3client.Limits()\n}\n\nfunc (s *space) SetStorageClass(sc string) error {\n\treturn notSupported\n}\n\nfunc newSpace(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, _ := url.ParseRequestURI(endpoint)\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\thostParts := strings.Split(uri.Host, \".\")\n\tbucket := hostParts[0]\n\tregion := hostParts[1]\n\tendpoint = uri.Scheme + \"://\" + uri.Host[len(bucket)+1:]\n\n\tawsCfg, err := config.LoadDefaultConfig(ctx,\n\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to load config: %s\", err)\n\t}\n\tclient := s3.NewFromConfig(awsCfg, func(options *s3.Options) {\n\t\toptions.Region = region\n\t\toptions.BaseEndpoint = aws.String(endpoint)\n\t\toptions.EndpointOptions.DisableHTTPS = !ssl\n\t\toptions.UsePathStyle = false\n\t\toptions.HTTPClient = httpClient\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\treturn &space{s3client{bucket: bucket, s3: client, region: region}}, nil\n}\n\nfunc init() {\n\tRegister(\"space\", newSpace)\n}\n" }, { "path": "pkg/object/sql.go", "content": "//go:build !nosqlite || !nomysql || !nopg\n// +build !nosqlite !nomysql !nopg\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"database/sql\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/sirupsen/logrus\"\n\t\"xorm.io/xorm\"\n\t\"xorm.io/xorm/log\"\n\t\"xorm.io/xorm/names\"\n)\n\ntype sqlStore struct {\n\tDefaultObjectStorage\n\tdb *xorm.Engine\n\taddr string\n}\n\ntype blob struct {\n\tId int64 `xorm:\"pk bigserial\"`\n\tKey []byte `xorm:\"notnull unique(blob) varbinary(255) \"`\n\tSize int64 `xorm:\"notnull\"`\n\tModified time.Time `xorm:\"notnull updated\"`\n\tData []byte `xorm:\"mediumblob\"`\n}\n\nfunc (s *sqlStore) String() string {\n\tdriver := s.db.DriverName()\n\tif driver == \"pgx\" {\n\t\tdriver = \"postgres\"\n\t}\n\treturn fmt.Sprintf(\"%s://%s/\", driver, s.addr)\n}\n\nfunc (s *sqlStore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tvar b = blob{Key: []byte(key)}\n\t// TODO: range\n\tok, err := s.db.Get(&b)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif !ok {\n\t\treturn nil, os.ErrNotExist\n\t}\n\tif off > int64(len(b.Data)) {\n\t\toff = int64(len(b.Data))\n\t}\n\tdata := b.Data[off:]\n\tif limit > 0 && limit < int64(len(data)) {\n\t\tdata = data[:limit]\n\t}\n\treturn io.NopCloser(bytes.NewBuffer(data)), nil\n}\n\nfunc (s *sqlStore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\td, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar n int64\n\tnow := time.Now()\n\tb := blob{Key: []byte(key), Data: d, Size: int64(len(d)), Modified: now}\n\tif name := s.db.DriverName(); name == \"postgres\" || name == \"pgx\" {\n\t\tvar r sql.Result\n\t\tr, err = s.db.Exec(\"INSERT INTO jfs_blob(key, size,modified, data) VALUES(?, ?, ?,? ) \"+\n\t\t\t\"ON CONFLICT (key) DO UPDATE SET size=?,data=?\", []byte(key), b.Size, now, d, b.Size, d)\n\t\tif err == nil {\n\t\t\tn, err = r.RowsAffected()\n\t\t}\n\t} else {\n\t\tn, err = s.db.Insert(&b)\n\t\tif err != nil || n == 0 {\n\t\t\tn, err = s.db.Update(&b, &blob{Key: []byte(key)})\n\t\t}\n\t}\n\tif err == nil && n == 0 {\n\t\terr = errors.New(\"not inserted or updated\")\n\t}\n\treturn err\n}\n\nfunc (s *sqlStore) Head(ctx context.Context, key string) (Object, error) {\n\tvar b = blob{Key: []byte(key)}\n\tok, err := s.db.Cols(\"key\", \"modified\", \"size\").Get(&b)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif !ok {\n\t\treturn nil, os.ErrNotExist\n\t}\n\treturn &obj{\n\t\tkey,\n\t\tb.Size,\n\t\tb.Modified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\"\",\n\t}, nil\n}\n\nfunc (s *sqlStore) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\t_, err := s.db.Delete(&blob{Key: []byte(key)})\n\treturn err\n}\n\nfunc (s *sqlStore) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif marker == \"\" {\n\t\tmarker = prefix\n\t}\n\t// todo\n\tif delimiter != \"\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tvar bs []blob\n\terr := s.db.Where(\"`key` > ?\", []byte(marker)).Limit(int(limit)).Cols(\"`key`\", \"size\", \"modified\").OrderBy(\"`key`\").Find(&bs)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tvar objs []Object\n\tfor _, b := range bs {\n\t\tif strings.HasPrefix(string(b.Key), prefix) {\n\t\t\tobjs = append(objs, &obj{\n\t\t\t\tkey: string(b.Key),\n\t\t\t\tsize: b.Size,\n\t\t\t\tmtime: b.Modified,\n\t\t\t\tisDir: strings.HasSuffix(string(b.Key), \"/\"),\n\t\t\t})\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc newSQLStore(driver, addr, user, password string) (ObjectStorage, error) {\n\tvar err error\n\turi := addr\n\tif user != \"\" {\n\t\turi = user + \":\" + password + \"@\" + addr\n\t}\n\tvar searchPath string\n\tif driver == \"postgres\" {\n\t\turi = \"postgres://\" + uri\n\t\tdriver = \"pgx\"\n\n\t\tparse, err := url.Parse(uri)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"parse url %s failed: %s\", uri, err)\n\t\t}\n\t\tsearchPath = parse.Query().Get(\"search_path\")\n\t\tif searchPath != \"\" {\n\t\t\tif len(strings.Split(searchPath, \",\")) > 1 {\n\t\t\t\treturn nil, fmt.Errorf(\"currently, only one schema is supported in search_path\")\n\t\t\t}\n\t\t}\n\t}\n\tengine, err := xorm.NewEngine(driver, uri)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"open %s: %s\", uri, err)\n\t}\n\tswitch logger.Level { // make xorm less verbose\n\tcase logrus.TraceLevel:\n\t\tengine.SetLogLevel(log.LOG_DEBUG)\n\tcase logrus.DebugLevel:\n\t\tengine.SetLogLevel(log.LOG_INFO)\n\tcase logrus.InfoLevel, logrus.WarnLevel:\n\t\tengine.SetLogLevel(log.LOG_WARNING)\n\tcase logrus.ErrorLevel:\n\t\tengine.SetLogLevel(log.LOG_ERR)\n\tdefault:\n\t\tengine.SetLogLevel(log.LOG_OFF)\n\t}\n\tif searchPath != \"\" {\n\t\tengine.SetSchema(searchPath)\n\t}\n\tengine.SetTableMapper(names.NewPrefixMapper(engine.GetTableMapper(), \"jfs_\"))\n\tif err := engine.Sync2(new(blob)); err != nil {\n\t\treturn nil, fmt.Errorf(\"create table blob: %s\", err)\n\t}\n\treturn &sqlStore{DefaultObjectStorage{}, engine, addr}, nil\n}\n\nfunc removeScheme(addr string) string {\n\tp := strings.Index(addr, \"://\")\n\tif p > 0 {\n\t\taddr = addr[p+3:]\n\t}\n\treturn addr\n}\n" }, { "path": "pkg/object/sql_mysql.go", "content": "//go:build !nomysql\n// +build !nomysql\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t_ \"github.com/go-sql-driver/mysql\"\n)\n\nfunc init() {\n\tRegister(\"mysql\", func(addr, user, pass, token string) (ObjectStorage, error) {\n\t\treturn newSQLStore(\"mysql\", removeScheme(addr), user, pass)\n\t})\n}\n" }, { "path": "pkg/object/sql_pg.go", "content": "//go:build !nopg\n// +build !nopg\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t_ \"github.com/jackc/pgx/v5/stdlib\"\n)\n\nfunc init() {\n\tRegister(\"postgres\", func(addr, user, pass, token string) (ObjectStorage, error) {\n\t\treturn newSQLStore(\"postgres\", removeScheme(addr), user, pass)\n\t})\n}\n" }, { "path": "pkg/object/sql_sqlite.go", "content": "//go:build !nosqlite\n// +build !nosqlite\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t_ \"github.com/mattn/go-sqlite3\"\n)\n\nfunc init() {\n\tRegister(\"sqlite3\", func(addr, user, pass, token string) (ObjectStorage, error) {\n\t\treturn newSQLStore(\"sqlite3\", removeScheme(addr), user, pass)\n\t})\n}\n" }, { "path": "pkg/object/swift.go", "content": "//go:build !noswift\n// +build !noswift\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/ncw/swift/v2\"\n)\n\ntype swiftOSS struct {\n\tDefaultObjectStorage\n\tconn *swift.Connection\n\tregion string\n\tstorageUrl string\n\tcontainer string\n}\n\nfunc (s *swiftOSS) String() string {\n\treturn fmt.Sprintf(\"swift://%s/\", s.container)\n}\n\nfunc (s *swiftOSS) Create(ctx context.Context) error {\n\t// No error is returned if it already exists but the metadata if any will be updated.\n\treturn s.conn.ContainerCreate(ctx, s.container, nil)\n}\n\nfunc (s *swiftOSS) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\theaders := make(map[string]string)\n\tif off > 0 || limit > 0 {\n\t\tif limit > 0 {\n\t\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-%d\", off, off+limit-1)\n\t\t} else {\n\t\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-\", off)\n\t\t}\n\t}\n\tf, _, err := s.conn.ObjectOpen(ctx, s.container, key, true, headers)\n\treturn f, err\n}\n\nfunc (s *swiftOSS) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tmimeType := utils.GuessMimeType(key)\n\t_, err := s.conn.ObjectPut(ctx, s.container, key, in, true, \"\", mimeType, nil)\n\treturn err\n}\n\nfunc (s *swiftOSS) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\terr := s.conn.ObjectDelete(ctx, s.container, key)\n\tif err != nil && errors.Is(err, swift.ObjectNotFound) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (s *swiftOSS) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif limit > 10000 {\n\t\tlimit = 10000\n\t}\n\tvar delimiter_ rune\n\tif delimiter != \"\" {\n\t\tif len([]rune(delimiter)) == 1 {\n\t\t\tdelimiter_ = []rune(delimiter)[0]\n\t\t} else {\n\t\t\treturn nil, false, \"\", fmt.Errorf(\"delimiter should be a rune but now is %s\", delimiter)\n\t\t}\n\t}\n\tobjects, err := s.conn.Objects(ctx, s.container, &swift.ObjectsOpts{Prefix: prefix, Marker: marker, Delimiter: delimiter_, Limit: int(limit)})\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tvar objs = make([]Object, len(objects))\n\tfor i, o := range objects {\n\t\t// https://docs.openstack.org/swift/latest/api/pseudo-hierarchical-folders-directories.html\n\t\tif delimiter != \"\" && o.PseudoDirectory {\n\t\t\tobjs[i] = &obj{o.SubDir, 0, time.Unix(0, 0), true, \"\"}\n\t\t} else {\n\t\t\tobjs[i] = &obj{o.Name, o.Bytes, o.LastModified, strings.HasSuffix(o.Name, \"/\"), \"\"}\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc (s *swiftOSS) Head(ctx context.Context, key string) (Object, error) {\n\tobject, _, err := s.conn.Object(ctx, s.container, key)\n\tif err != nil {\n\t\tif err == swift.ObjectNotFound {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tkey,\n\t\tobject.Bytes,\n\t\tobject.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\"\",\n\t}, err\n}\n\nfunc newSwiftOSS(endpoint, username, apiKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"http://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tif uri.Scheme != \"http\" && uri.Scheme != \"https\" {\n\t\treturn nil, fmt.Errorf(\"Invalid uri.Scheme: %s\", uri.Scheme)\n\t}\n\n\thostSlice := strings.SplitN(uri.Host, \".\", 2)\n\tif len(hostSlice) != 2 {\n\t\treturn nil, fmt.Errorf(\"Invalid host: %s\", uri.Host)\n\t}\n\tcontainer := hostSlice[0]\n\thost := hostSlice[1]\n\n\t// current only support V1 authentication\n\tauthURL := uri.Scheme + \"://\" + host + \"/auth/v1.0\"\n\n\tconn := swift.Connection{\n\t\tUserName: username,\n\t\tApiKey: apiKey,\n\t\tAuthToken: token,\n\t\tAuthUrl: authURL,\n\t\tTransport: httpClient.Transport.(*http.Transport),\n\t}\n\terr = conn.Authenticate(context.Background())\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Auth: %s\", err)\n\t}\n\treturn &swiftOSS{DefaultObjectStorage{}, &conn, conn.Region, conn.StorageUrl, container}, nil\n}\n\nfunc init() {\n\tRegister(\"swift\", newSwiftOSS)\n}\n" }, { "path": "pkg/object/tikv.go", "content": "//go:build !notikv\n// +build !notikv\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\tplog \"github.com/pingcap/log\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/tikv/client-go/v2/config\"\n\t\"github.com/tikv/client-go/v2/rawkv\"\n)\n\ntype tikv struct {\n\tDefaultObjectStorage\n\tc *rawkv.Client\n\taddr string\n}\n\nfunc (t *tikv) String() string {\n\treturn fmt.Sprintf(\"tikv://%s/\", t.addr)\n}\n\nfunc (t *tikv) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\td, err := t.c.Get(ctx, []byte(key))\n\tif len(d) == 0 {\n\t\terr = os.ErrNotExist\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif off > int64(len(d)) {\n\t\toff = int64(len(d))\n\t}\n\tdata := d[off:]\n\tif limit > 0 && limit < int64(len(data)) {\n\t\tdata = data[:limit]\n\t}\n\treturn io.NopCloser(bytes.NewBuffer(data)), nil\n}\n\nfunc (t *tikv) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\td, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn t.c.Put(ctx, []byte(key), d)\n}\n\nfunc (t *tikv) Head(ctx context.Context, key string) (Object, error) {\n\tdata, err := t.c.Get(ctx, []byte(key))\n\tif err == nil && data == nil {\n\t\treturn nil, os.ErrNotExist\n\t}\n\treturn &obj{\n\t\tkey,\n\t\tint64(len(data)),\n\t\ttime.Now(),\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\t\"\",\n\t}, err\n}\n\nfunc (t *tikv) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\treturn t.c.Delete(ctx, []byte(key))\n}\n\nfunc (t *tikv) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tif marker == \"\" {\n\t\tmarker = prefix\n\t}\n\tif limit > int64(rawkv.MaxRawKVScanLimit) {\n\t\tlimit = int64(rawkv.MaxRawKVScanLimit)\n\t}\n\t// TODO: key only\n\tkeys, vs, err := t.c.Scan(ctx, []byte(marker), nil, int(limit))\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tvar objs = make([]Object, len(keys))\n\tmtime := time.Now()\n\tfor i, k := range keys {\n\t\t// FIXME: mtime\n\t\tobjs[i] = &obj{string(k), int64(len(vs[i])), mtime, strings.HasSuffix(string(k), \"/\"), \"\"}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc newTiKV(endpoint, accesskey, secretkey, token string) (ObjectStorage, error) {\n\tvar plvl string // TiKV (PingCap) uses uber-zap logging, make it less verbose\n\tswitch logger.Level {\n\tcase logrus.TraceLevel:\n\t\tplvl = \"debug\"\n\tcase logrus.DebugLevel:\n\t\tplvl = \"info\"\n\tcase logrus.InfoLevel, logrus.WarnLevel:\n\t\tplvl = \"warn\"\n\tcase logrus.ErrorLevel:\n\t\tplvl = \"error\"\n\tdefault:\n\t\tplvl = \"dpanic\"\n\t}\n\tl, prop, _ := plog.InitLogger(&plog.Config{Level: plvl})\n\tplog.ReplaceGlobals(l, prop)\n\n\tif !strings.HasPrefix(endpoint, \"tikv://\") {\n\t\tendpoint = \"tikv://\" + endpoint\n\t}\n\ttUrl, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tpds := strings.Split(tUrl.Host, \",\")\n\tfor i, pd := range pds {\n\t\tpd = strings.TrimSpace(pd)\n\t\tif !strings.Contains(pd, \":\") {\n\t\t\tpd += \":2379\"\n\t\t}\n\t\tpds[i] = pd\n\t}\n\n\tq := tUrl.Query()\n\tc, err := rawkv.NewClient(context.TODO(), pds, config.NewSecurity(\n\t\tq.Get(\"ca\"),\n\t\tq.Get(\"cert\"),\n\t\tq.Get(\"key\"),\n\t\tstrings.Split(q.Get(\"verify-cn\"), \",\")))\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &tikv{c: c, addr: tUrl.Host}, nil\n}\n\nfunc init() {\n\tRegister(\"tikv\", newTiKV)\n}\n" }, { "path": "pkg/object/tos.go", "content": "//go:build !tos\n\n/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/volcengine/ve-tos-golang-sdk/v2/tos\"\n\t\"github.com/volcengine/ve-tos-golang-sdk/v2/tos/codes\"\n\t\"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum\"\n)\n\ntype tosClient struct {\n\tbucket string\n\tsc string\n\tclient *tos.ClientV2\n}\n\nfunc (t *tosClient) String() string {\n\treturn fmt.Sprintf(\"tos://%s/\", t.bucket)\n}\n\nfunc (t *tosClient) Limits() Limits {\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: true,\n\t\tMinPartSize: 4 << 20,\n\t\tMaxPartSize: 5 << 30,\n\t\tMaxPartCount: 10000,\n\t}\n}\n\nfunc (t *tosClient) Create(ctx context.Context) error {\n\t_, err := t.client.CreateBucketV2(ctx, &tos.CreateBucketV2Input{Bucket: t.bucket, StorageClass: enum.StorageClassType(t.sc)})\n\tif e, ok := err.(*tos.TosServerError); ok {\n\t\tif e.Code == codes.BucketAlreadyOwnedByYou || e.Code == codes.BucketAlreadyExists {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn err\n}\n\nfunc (t *tosClient) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\trangeStr := getRange(off, limit)\n\tresp, err := t.client.GetObjectV2(ctx, &tos.GetObjectV2Input{\n\t\tBucket: t.bucket,\n\t\tKey: key,\n\t\tRange: rangeStr, // When Range and RangeStart & RangeEnd appear together, range is preferred\n\t})\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.RequestID).SetStorageClass(string(resp.StorageClass))\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err = checkGetStatus(resp.StatusCode, rangeStr != \"\"); err != nil {\n\t\t_ = resp.Content.Close()\n\t\treturn nil, err\n\t}\n\tif off == 0 && limit == -1 {\n\t\tv, _ := resp.Meta.Get(checksumAlgr)\n\t\tresp.Content = verifyChecksum(resp.Content, v, resp.ContentLength)\n\t}\n\n\treturn resp.Content, nil\n}\n\nfunc (t *tosClient) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tvar meta map[string]string\n\tif ins, ok := in.(io.ReadSeeker); ok {\n\t\tmeta = map[string]string{\n\t\t\tchecksumAlgr: generateChecksum(ins),\n\t\t}\n\t}\n\tresp, err := t.client.PutObjectV2(ctx, &tos.PutObjectV2Input{\n\t\tPutObjectBasicInput: tos.PutObjectBasicInput{\n\t\t\tBucket: t.bucket,\n\t\t\tKey: key,\n\t\t\tStorageClass: enum.StorageClassType(t.sc),\n\t\t\tMeta: meta,\n\t\t},\n\t\tContent: in,\n\t})\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.RequestID).SetStorageClass(t.sc)\n\t}\n\treturn err\n}\n\nfunc (t *tosClient) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tresp, err := t.client.DeleteObjectV2(ctx, &tos.DeleteObjectV2Input{\n\t\tBucket: t.bucket,\n\t\tKey: key,\n\t})\n\tif resp != nil {\n\t\tattrs := ApplyGetters(getters...)\n\t\tattrs.SetRequestID(resp.RequestID)\n\t}\n\treturn err\n}\n\nfunc (t *tosClient) Head(ctx context.Context, key string) (Object, error) {\n\thead, err := t.client.HeadObjectV2(ctx, &tos.HeadObjectV2Input{Bucket: t.bucket, Key: key})\n\tif err != nil {\n\t\tif e, ok := err.(*tos.TosServerError); ok {\n\t\t\tif e.StatusCode == http.StatusNotFound {\n\t\t\t\terr = os.ErrNotExist\n\t\t\t}\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tkey,\n\t\thead.ContentLength,\n\t\thead.LastModified,\n\t\tstrings.HasSuffix(key, \"/\"),\n\t\tstring(head.StorageClass),\n\t}, err\n}\n\nfunc (t *tosClient) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tresp, err := t.client.ListObjectsType2(ctx, &tos.ListObjectsType2Input{\n\t\tBucket: t.bucket,\n\t\tDelimiter: delimiter,\n\t\tPrefix: prefix,\n\t\tStartAfter: start,\n\t\tMaxKeys: int(limit),\n\t\tContinuationToken: token,\n\t})\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tn := len(resp.Contents)\n\tobjs := make([]Object, n)\n\tfor i := 0; i < n; i++ {\n\t\to := resp.Contents[i]\n\t\tif !strings.HasPrefix(o.Key, prefix) || o.Key <= start {\n\t\t\treturn nil, false, \"\", fmt.Errorf(\"found invalid key %s from List, prefix: %s, marker: %s\", o.Key, prefix, start)\n\t\t}\n\t\tobjs[i] = &obj{\n\t\t\to.Key,\n\t\t\to.Size,\n\t\t\to.LastModified,\n\t\t\tstrings.HasSuffix(o.Key, \"/\"),\n\t\t\tstring(o.StorageClass),\n\t\t}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, p := range resp.CommonPrefixes {\n\t\t\tobjs = append(objs, &obj{p.Prefix, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\treturn objs, resp.IsTruncated, resp.NextContinuationToken, nil\n}\n\nfunc (t *tosClient) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error) {\n\treturn nil, notSupported\n}\n\nfunc (t *tosClient) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tresp, err := t.client.CreateMultipartUploadV2(ctx, &tos.CreateMultipartUploadV2Input{\n\t\tBucket: t.bucket,\n\t\tKey: key,\n\t\tStorageClass: enum.StorageClassType(t.sc),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: resp.UploadID, MinPartSize: 5 << 20, MaxCount: 10000}, nil\n}\n\nfunc (t *tosClient) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error) {\n\tresp, err := t.client.UploadPartV2(ctx, &tos.UploadPartV2Input{\n\t\tUploadPartBasicInput: tos.UploadPartBasicInput{\n\t\t\tBucket: t.bucket,\n\t\t\tKey: key,\n\t\t\tUploadID: uploadID,\n\t\t\tPartNumber: num,\n\t\t},\n\t\tContent: bytes.NewReader(body),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: resp.ETag}, nil\n}\n\nfunc (t *tosClient) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error) {\n\tresp, err := t.client.UploadPartCopyV2(ctx, &tos.UploadPartCopyV2Input{\n\t\tBucket: t.bucket,\n\t\tKey: key,\n\t\tUploadID: uploadID,\n\t\tPartNumber: num,\n\t\tSrcBucket: t.bucket,\n\t\tSrcKey: srcKey,\n\t\tCopySourceRange: fmt.Sprintf(\"bytes=%d-%d\", off, off+size-1),\n\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Part{Num: num, ETag: resp.ETag}, nil\n}\n\nfunc (t *tosClient) AbortUpload(ctx context.Context, key string, uploadID string) {\n\t_, _ = t.client.AbortMultipartUpload(ctx, &tos.AbortMultipartUploadInput{\n\t\tBucket: t.bucket,\n\t\tKey: key,\n\t\tUploadID: uploadID,\n\t})\n}\n\nfunc (t *tosClient) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\tvar tosParts []tos.UploadedPartV2\n\tfor i := range parts {\n\t\ttosParts = append(tosParts, tos.UploadedPartV2{ETag: parts[i].ETag, PartNumber: parts[i].Num})\n\t}\n\t_, err := t.client.CompleteMultipartUploadV2(ctx, &tos.CompleteMultipartUploadV2Input{\n\t\tBucket: t.bucket,\n\t\tKey: key,\n\t\tUploadID: uploadID,\n\t\tParts: tosParts,\n\t})\n\treturn err\n}\n\nfunc (t *tosClient) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tresult, err := t.client.ListMultipartUploadsV2(ctx, &tos.ListMultipartUploadsV2Input{Bucket: t.bucket})\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tparts := make([]*PendingPart, len(result.Uploads))\n\tfor i, u := range result.Uploads {\n\t\tparts[i] = &PendingPart{u.Key, u.UploadID, u.Initiated}\n\t}\n\tvar nextMarker string\n\tif result.NextKeyMarker != \"\" {\n\t\tnextMarker = result.NextKeyMarker\n\t}\n\treturn parts, nextMarker, nil\n}\n\nfunc (t *tosClient) Copy(ctx context.Context, dst, src string) error {\n\t_, err := t.client.CopyObject(ctx, &tos.CopyObjectInput{\n\t\tSrcBucket: t.bucket,\n\t\tBucket: t.bucket,\n\t\tSrcKey: src,\n\t\tKey: dst,\n\t\tStorageClass: enum.StorageClassType(t.sc),\n\t})\n\treturn err\n}\n\nfunc (t *tosClient) SetStorageClass(sc string) error {\n\tt.sc = sc\n\treturn nil\n}\n\nfunc newTOS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"invalid endpoint: %v, error: %v\", endpoint, err)\n\t}\n\tdisableChecksum := strings.EqualFold(uri.Query().Get(\"disable-checksum\"), \"true\")\n\tif disableChecksum {\n\t\tlogger.Infof(\"default CRC checksum is disabled\")\n\t}\n\thostParts := strings.SplitN(uri.Host, \".\", 3)\n\tcredentials := tos.NewStaticCredentials(accessKey, secretKey)\n\tcredentials.WithSecurityToken(token)\n\tcli, err := tos.NewClientV2(\n\t\thostParts[1]+\".\"+hostParts[2],\n\t\ttos.WithRegion(strings.TrimPrefix(hostParts[1], \"tos-\")),\n\t\ttos.WithCredentials(credentials),\n\t\ttos.WithEnableVerifySSL(httpClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify),\n\t\ttos.WithEnableCRC(!disableChecksum))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &tosClient{bucket: hostParts[0], client: cli}, nil\n}\n\nfunc init() {\n\tRegister(\"tos\", newTOS)\n}\n" }, { "path": "pkg/object/ufile.go", "content": "//go:build !noufile\n// +build !noufile\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/hmac\"\n\t\"crypto/sha1\"\n\t\"encoding/base64\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\ntype ufile struct {\n\tRestfulStorage\n}\n\nfunc (u *ufile) String() string {\n\turi, _ := url.ParseRequestURI(u.endpoint)\n\treturn fmt.Sprintf(\"ufile://%s/\", uri.Host)\n}\n\nfunc (u *ufile) Limits() Limits {\n\t// only support 4MB part size and max object size: 5TB\n\treturn Limits{\n\t\tIsSupportMultipartUpload: true,\n\t\tIsSupportUploadPartCopy: false,\n\t\tMinPartSize: 4 << 20,\n\t\tMaxPartSize: 4 << 20,\n\t\tMaxPartCount: 1310720,\n\t}\n}\n\nfunc ufileSigner(req *http.Request, accessKey, secretKey, signName string) {\n\tif accessKey == \"\" {\n\t\treturn\n\t}\n\ttoSign := req.Method + \"\\n\"\n\tfor _, n := range HEADER_NAMES {\n\t\ttoSign += req.Header.Get(n) + \"\\n\"\n\t}\n\tbucket := strings.Split(req.URL.Host, \".\")[0]\n\tkey := req.URL.Path\n\t// Hack for UploadHit\n\tif len(req.URL.RawQuery) > 0 {\n\t\tvs, _ := url.ParseQuery(req.URL.RawQuery)\n\t\tif _, ok := vs[\"FileName\"]; ok {\n\t\t\tkey = \"/\" + vs.Get(\"FileName\")\n\t\t}\n\t}\n\ttoSign += \"/\" + bucket + key\n\th := hmac.New(sha1.New, []byte(secretKey))\n\t_, _ = h.Write([]byte(toSign))\n\tsig := base64.StdEncoding.EncodeToString(h.Sum(nil))\n\ttoken := signName + \" \" + accessKey + \":\" + sig\n\treq.Header.Add(\"Authorization\", token)\n}\n\nfunc (u *ufile) Create(ctx context.Context) error {\n\turi, _ := url.ParseRequestURI(u.endpoint)\n\tparts := strings.Split(uri.Host, \".\")\n\tname := parts[0]\n\tregion := parts[1] // www.cn-bj.ufileos.com\n\tif region == \"ufile\" {\n\t\tregion = parts[2] // www.ufile.cn-north-02.ucloud.cn\n\t}\n\tif strings.HasPrefix(region, \"internal\") {\n\t\t// www.internal-hk-01.ufileos.cn\n\t\t// www.internal-cn-gd-02.ufileos.cn\n\t\tps := strings.Split(region, \"-\")\n\t\tregion = strings.Join(ps[1:len(ps)-1], \"-\")\n\t}\n\n\tquery := url.Values{}\n\tquery.Add(\"Action\", \"CreateBucket\")\n\tquery.Add(\"BucketName\", name)\n\tquery.Add(\"PublicKey\", u.accessKey)\n\tquery.Add(\"Region\", region)\n\n\t// generate signature\n\ttoSign := fmt.Sprintf(\"ActionCreateBucketBucketName%sPublicKey%sRegion%s\",\n\t\tname, u.accessKey, region)\n\n\tsum := sha1.Sum([]byte(toSign + u.secretKey))\n\tsig := hex.EncodeToString(sum[:])\n\tquery.Add(\"Signature\", sig)\n\n\treq, err := http.NewRequest(\"GET\", \"https://api.ucloud.cn/?\"+query.Encode(), nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tresp, err := httpClient.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\terr = parseError(resp)\n\tif strings.Contains(err.Error(), \"duplicate bucket name\") ||\n\t\tstrings.Contains(err.Error(), \"CreateBucketResponse\") {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc (u *ufile) parseResp(resp *http.Response, out interface{}) error {\n\tdefer resp.Body.Close()\n\tvar data []byte\n\tif resp.ContentLength <= 0 || resp.ContentLength > (1<<31) {\n\t\td, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdata = d\n\t} else {\n\t\tdata = make([]byte, resp.ContentLength)\n\t\tif _, err := io.ReadFull(resp.Body, data); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif resp.StatusCode != 200 {\n\t\treturn fmt.Errorf(\"status: %v, message: %s\", resp.StatusCode, string(data))\n\t}\n\terr := json.Unmarshal(data, out)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc copyObj(ctx context.Context, store ObjectStorage, dst, src string) error {\n\tin, err := store.Get(ctx, src, 0, -1)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer in.Close()\n\td, err := io.ReadAll(in)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn store.Put(ctx, dst, bytes.NewReader(d))\n}\n\nfunc (u *ufile) Copy(ctx context.Context, dst, src string) error {\n\tresp, err := u.request(ctx, \"HEAD\", src, nil, nil)\n\tif err != nil {\n\t\treturn copyObj(ctx, u, dst, src)\n\t}\n\tif resp.StatusCode != 200 {\n\t\treturn copyObj(ctx, u, dst, src)\n\t}\n\n\tetag := resp.Header[\"Etag\"]\n\tif len(etag) < 1 {\n\t\treturn copyObj(ctx, u, dst, src)\n\t}\n\thash := etag[0][1 : len(etag[0])-1]\n\tlens := resp.Header[\"Content-Length\"]\n\tif len(lens) < 1 {\n\t\treturn copyObj(ctx, u, dst, src)\n\t}\n\turi := fmt.Sprintf(\"uploadhit?Hash=%s&FileName=%s&FileSize=%s\", hash, dst, lens[0])\n\tresp, err = u.request(ctx, \"POST\", uri, nil, nil)\n\tif err != nil {\n\t\treturn copyObj(ctx, u, dst, src)\n\t}\n\tdefer cleanup(resp)\n\tif resp.StatusCode != 200 {\n\t\treturn copyObj(ctx, u, dst, src)\n\t}\n\treturn nil\n}\n\ntype ContentsItem struct {\n\tKey string\n\tSize string\n\tLastModified int\n\tCreateTime int\n\tStorageClass string\n\tETag string\n}\n\ntype CommonPrefixesItem struct {\n\tPrefix string\n}\n\n// uFileListObjectsOutput presents output for ListObjects.\ntype uFileListObjectsOutput struct {\n\tMaxkeys string `json:\"MaxKeys,omitempty\"`\n\tDelimiter string `json:\"Delimiter,omitempty\"`\n\tNextMarker string `json:\"NextMarker,omitempty\"`\n\tIsTruncated bool `json:\"IsTruncated,omitempty\"`\n\n\t// Object keys\n\tContents []*ContentsItem `json:\"Contents,omitempty\"`\n\tCommonPrefixes []*CommonPrefixesItem `json:\"CommonPrefixes,omitempty\"`\n}\n\nfunc (u *ufile) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\tquery := url.Values{}\n\tquery.Add(\"prefix\", prefix)\n\tquery.Add(\"marker\", start)\n\tquery.Add(\"delimiter\", delimiter)\n\tif limit > 1000 {\n\t\tlimit = 1000\n\t}\n\tquery.Add(\"max-keys\", strconv.Itoa(int(limit)))\n\tresp, err := u.request(ctx, \"GET\", \"?listobjects&\"+query.Encode(), nil, nil)\n\tif err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\n\tvar out uFileListObjectsOutput\n\tif err := u.parseResp(resp, &out); err != nil {\n\t\treturn nil, false, \"\", err\n\t}\n\tobjs := make([]Object, len(out.Contents))\n\tfor i, item := range out.Contents {\n\t\tsize_, _ := strconv.ParseInt(item.Size, 10, 64)\n\t\tobjs[i] = &obj{item.Key, size_, time.Unix(int64(item.LastModified), 0), strings.HasSuffix(item.Key, \"/\"), \"\"}\n\t}\n\tif delimiter != \"\" {\n\t\tfor _, item := range out.CommonPrefixes {\n\t\t\tobjs = append(objs, &obj{item.Prefix, 0, time.Unix(0, 0), true, \"\"})\n\t\t}\n\t\tsort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })\n\t}\n\t// This is a bug in ufile, NextMarker is not the last one after sorting.\n\tvar lastKey string\n\tif len(objs) > 0 {\n\t\tlastKey = objs[len(objs)-1].Key()\n\t}\n\treturn objs, out.IsTruncated, lastKey, nil\n}\n\ntype ufileCreateMultipartUploadResult struct {\n\tUploadId string\n\tBlkSize int\n\tBucket string\n\tKey string\n}\n\nfunc (u *ufile) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error) {\n\tresp, err := u.request(ctx, \"POST\", key+\"?uploads\", nil, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar out ufileCreateMultipartUploadResult\n\tif err := u.parseResp(resp, &out); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &MultipartUpload{UploadID: out.UploadId, MinPartSize: out.BlkSize, MaxCount: 1000000}, nil\n}\n\nfunc (u *ufile) UploadPart(ctx context.Context, key string, uploadID string, num int, data []byte) (*Part, error) {\n\t// UFile require the PartNumber to start from 0 (continuous)\n\tnum--\n\tpath := fmt.Sprintf(\"%s?uploadId=%s&partNumber=%d\", key, uploadID, num)\n\tresp, err := u.request(ctx, \"PUT\", path, bytes.NewReader(data), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer cleanup(resp)\n\tif resp.StatusCode != 200 {\n\t\treturn nil, fmt.Errorf(\"UploadPart: %s\", parseError(resp).Error())\n\t}\n\tetags := resp.Header[\"Etag\"]\n\tif len(etags) < 1 {\n\t\treturn nil, errors.New(\"No ETag\")\n\t}\n\treturn &Part{Num: num, Size: len(data), ETag: strings.Trim(etags[0], \"\\\"\")}, nil\n}\n\nfunc (u *ufile) AbortUpload(ctx context.Context, key string, uploadID string) {\n\t_, _ = u.request(ctx, \"DELETE\", key+\"?uploads=\"+uploadID, nil, nil)\n}\n\nfunc (u *ufile) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error {\n\tetags := make([]string, len(parts))\n\tfor i, p := range parts {\n\t\tetags[i] = p.ETag\n\t}\n\tresp, err := u.request(ctx, \"POST\", key+\"?uploadId=\"+uploadID, bytes.NewReader([]byte(strings.Join(etags, \",\"))), nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer cleanup(resp)\n\tif resp.StatusCode != 200 {\n\t\treturn fmt.Errorf(\"CompleteMultipart: %s\", parseError(resp).Error())\n\t}\n\treturn nil\n}\n\ntype ufileUpload struct {\n\tFileName string\n\tUploadId string\n\tStartTime int\n}\n\ntype ufileListMultipartUploadsResult struct {\n\tRetCode int\n\tErrMsg string\n\tNextMarker string\n\tDataSet []*ufileUpload\n}\n\nfunc (u *ufile) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error) {\n\tquery := url.Values{}\n\tquery.Add(\"muploadid\", \"\")\n\tquery.Add(\"prefix\", \"\")\n\tquery.Add(\"marker\", marker)\n\tquery.Add(\"limit\", strconv.Itoa(1000))\n\tresp, err := u.request(ctx, \"GET\", \"?\"+query.Encode(), nil, nil)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tvar out ufileListMultipartUploadsResult\n\t// FIXME: invalid auth\n\tif err := u.parseResp(resp, &out); err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tif out.RetCode != 0 {\n\t\treturn nil, \"\", errors.New(out.ErrMsg)\n\t}\n\tparts := make([]*PendingPart, len(out.DataSet))\n\tfor i, u := range out.DataSet {\n\t\tparts[i] = &PendingPart{u.FileName, u.UploadId, time.Unix(int64(u.StartTime), 0)}\n\t}\n\treturn parts, out.NextMarker, nil\n}\n\nfunc newUFile(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\treturn &ufile{RestfulStorage{DefaultObjectStorage{}, endpoint, accessKey, secretKey, \"UCloud\", ufileSigner}}, nil\n}\n\nfunc init() {\n\tRegister(\"ufile\", newUFile)\n}\n" }, { "path": "pkg/object/wasabi.go", "content": "//go:build !nos3\n// +build !nos3\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\tsmithymiddleware \"github.com/aws/smithy-go/middleware\"\n)\n\ntype wasabi struct {\n\ts3client\n}\n\nfunc (s *wasabi) String() string {\n\treturn fmt.Sprintf(\"wasabi://%s/\", s.s3client.bucket)\n}\n\nfunc (s *wasabi) SetStorageClass(_ string) error {\n\treturn notSupported\n}\n\nfunc newWasabi(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"https://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tssl := strings.ToLower(uri.Scheme) == \"https\"\n\thostParts := strings.Split(uri.Host, \".\")\n\tbucket := hostParts[0]\n\tregion := hostParts[2]\n\tendpoint = uri.Scheme + \"://\" + uri.Host[len(bucket)+1:]\n\n\tawsCfg, err := config.LoadDefaultConfig(ctx,\n\t\tconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, token)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to load config: %s\", err)\n\t}\n\tclient := s3.NewFromConfig(awsCfg, func(options *s3.Options) {\n\t\toptions.Region = region\n\t\toptions.BaseEndpoint = aws.String(endpoint)\n\t\toptions.EndpointOptions.DisableHTTPS = !ssl\n\t\toptions.UsePathStyle = false\n\t\toptions.HTTPClient = httpClient\n\t\toptions.APIOptions = append(options.APIOptions, func(stack *smithymiddleware.Stack) error {\n\t\t\treturn v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack)\n\t\t})\n\t\toptions.RetryMaxAttempts = 1\n\t})\n\treturn &wasabi{s3client{bucket: bucket, s3: client, region: region}}, nil\n}\n\nfunc init() {\n\tRegister(\"wasabi\", newWasabi)\n}\n" }, { "path": "pkg/object/webdav.go", "content": "//go:build !nowebdav\n// +build !nowebdav\n\n/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage object\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/studio-b12/gowebdav\"\n)\n\ntype webdav struct {\n\tDefaultObjectStorage\n\tendpoint *url.URL\n\tc *gowebdav.Client\n}\n\nfunc (w *webdav) String() string {\n\treturn fmt.Sprintf(\"webdav://%s/\", w.endpoint.Host)\n}\n\nfunc (w *webdav) Create(ctx context.Context) error {\n\treturn nil\n}\n\nfunc (w *webdav) Head(ctx context.Context, key string) (Object, error) {\n\tinfo, err := w.c.Stat(key)\n\tif err != nil {\n\t\tif gowebdav.IsErrNotFound(err) {\n\t\t\terr = os.ErrNotExist\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn &obj{\n\t\tkey,\n\t\tinfo.Size(),\n\t\tinfo.ModTime(),\n\t\tinfo.IsDir(),\n\t\t\"\",\n\t}, nil\n}\n\nfunc (w *webdav) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {\n\tif off == 0 && limit <= 0 {\n\t\treturn w.c.ReadStream(key)\n\t}\n\treturn w.c.ReadStreamRange(key, off, limit)\n}\n\nfunc (w *webdav) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error {\n\tif key == \"\" {\n\t\treturn nil\n\t}\n\tif strings.HasSuffix(key, dirSuffix) {\n\t\treturn w.c.MkdirAll(key, 0)\n\t}\n\treturn w.c.WriteStream(key, in, 0)\n}\n\nfunc (w *webdav) Delete(ctx context.Context, key string, getters ...AttrGetter) error {\n\tinfo, err := w.c.Stat(key)\n\tif gowebdav.IsErrNotFound(err) {\n\t\treturn nil\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\tif info.IsDir() {\n\t\tinfos, err := w.c.ReadDir(key)\n\t\tif err != nil {\n\t\t\tif gowebdav.IsErrNotFound(err) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif len(infos) != 0 {\n\t\t\treturn fmt.Errorf(\"%s is non-empty directory\", key)\n\t\t}\n\t}\n\treturn w.c.Remove(key)\n}\n\nfunc (w *webdav) Copy(ctx context.Context, dst, src string) error {\n\treturn w.c.Copy(src, dst, true)\n}\n\ntype webDAVFile struct {\n\tos.FileInfo\n\tname string\n}\n\nfunc (w webDAVFile) Name() string {\n\treturn w.name\n}\n\nfunc (w *webdav) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {\n\tif delimiter != \"/\" {\n\t\treturn nil, false, \"\", notSupported\n\t}\n\n\troot := \"/\" + prefix\n\tvar objs []Object\n\tif !strings.HasSuffix(root, dirSuffix) {\n\t\t// If the root is not ends with `/`, we'll list the directory root resides.\n\t\troot = path.Dir(root)\n\t\tif !strings.HasSuffix(root, dirSuffix) {\n\t\t\troot += dirSuffix\n\t\t}\n\t}\n\n\tinfos, err := w.c.ReadDir(root)\n\tif err != nil {\n\t\tif gowebdav.IsErrCode(err, http.StatusForbidden) {\n\t\t\tlogger.Warnf(\"skip %s: %s\", root, err)\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\tif gowebdav.IsErrNotFound(err) {\n\t\t\treturn nil, false, \"\", nil\n\t\t}\n\t\treturn nil, false, \"\", err\n\t}\n\tsortedInfos := make([]os.FileInfo, len(infos))\n\tfor idx, o := range infos {\n\t\tif o.IsDir() {\n\t\t\tsortedInfos[idx] = &webDAVFile{name: o.Name() + dirSuffix, FileInfo: o}\n\t\t} else {\n\t\t\tsortedInfos[idx] = o\n\t\t}\n\t}\n\tsort.Slice(sortedInfos, func(i, j int) bool {\n\t\treturn sortedInfos[i].Name() < sortedInfos[j].Name()\n\t})\n\tfor _, info := range sortedInfos {\n\t\tkey := root[1:] + info.Name()\n\t\tif !strings.HasPrefix(key, prefix) || (marker != \"\" && key <= marker) {\n\t\t\tcontinue\n\t\t}\n\t\tobjs = append(objs, &obj{\n\t\t\tkey,\n\t\t\tinfo.Size(),\n\t\t\tinfo.ModTime(),\n\t\t\tinfo.IsDir(),\n\t\t\t\"\",\n\t\t})\n\t\tif len(objs) == int(limit) {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn generateListResult(objs, limit)\n}\n\nfunc newWebDAV(endpoint, user, passwd, token string) (ObjectStorage, error) {\n\tif !strings.Contains(endpoint, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"http://%s\", endpoint)\n\t}\n\turi, err := url.ParseRequestURI(endpoint)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid endpoint %s: %s\", endpoint, err)\n\t}\n\tif uri.Path == \"\" {\n\t\turi.Path = \"/\"\n\t}\n\tc := gowebdav.NewClient(uri.String(), user, passwd)\n\tc.SetTransport(httpClient.Transport)\n\treturn &webdav{endpoint: uri, c: c}, nil\n}\n\nfunc init() {\n\tRegister(\"webdav\", newWebDAV)\n}\n" }, { "path": "pkg/sync/cluster.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sync\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/dustin/go-humanize\"\n\t\"github.com/oliverisaac/shellescape\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\n// Stat has the counters to represent the progress.\ntype Stat struct {\n\tCopied int64 // the number of copied files\n\tCopiedBytes int64 // total amount of copied data in bytes\n\tChecked int64 // the number of checked files\n\tCheckedBytes int64 // total amount of checked data in bytes\n\tDeleted int64 // the number of deleted files\n\tSkipped int64 // the number of files skipped\n\tSkippedBytes int64 // total amount of skipped data in bytes\n\tFailed int64 // the number of files that fail to copy\n\tDelayDelDir []string // the directories that need to be deleted\n}\n\nfunc updateStats(r *Stat) {\n\tcopied.IncrInt64(r.Copied)\n\tcopiedBytes.IncrInt64(r.CopiedBytes)\n\tif checked != nil {\n\t\tchecked.IncrInt64(r.Checked)\n\t\tcheckedBytes.IncrInt64(r.CheckedBytes)\n\t}\n\tif deleted != nil {\n\t\tdeleted.IncrInt64(r.Deleted)\n\t}\n\tskipped.IncrInt64(r.Skipped)\n\tskippedBytes.IncrInt64(r.SkippedBytes)\n\tif failed != nil {\n\t\tfailed.IncrInt64(r.Failed)\n\t}\n\thandled.IncrInt64(r.Copied + r.Deleted + r.Skipped + r.Failed)\n}\n\nfunc httpRequest(url string, body []byte) (ans []byte, err error) {\n\tmethod := \"GET\"\n\tif body != nil {\n\t\tmethod = \"POST\"\n\t}\n\treq, err := http.NewRequest(method, url, bytes.NewReader(body))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar resp *http.Response\n\tresp, err = http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer resp.Body.Close()\n\treturn io.ReadAll(resp.Body)\n}\n\nvar sendStatMu sync.Mutex\n\nfunc sendStats(addr string) {\n\tsendStatMu.Lock()\n\tdefer sendStatMu.Unlock()\n\tvar r Stat\n\tr.Skipped = skipped.Current()\n\tr.SkippedBytes = skippedBytes.Current()\n\tr.Copied = copied.Current()\n\tr.CopiedBytes = copiedBytes.Current()\n\tsrcDelayDelMu.Lock()\n\tr.DelayDelDir = srcDelayDel\n\tsrcDelayDel = make([]string, 0)\n\tsrcDelayDelMu.Unlock()\n\tif checked != nil {\n\t\tr.Checked = checked.Current()\n\t\tr.CheckedBytes = checkedBytes.Current()\n\t}\n\tif deleted != nil {\n\t\tr.Deleted = deleted.Current()\n\t}\n\tif failed != nil {\n\t\tr.Failed = failed.Current()\n\t}\n\td, _ := json.Marshal(r)\n\tans, err := httpRequest(fmt.Sprintf(\"http://%s/stats\", addr), d)\n\tif err != nil || string(ans) != \"OK\" {\n\t\tsrcDelayDelMu.Lock()\n\t\tsrcDelayDel = append(srcDelayDel, r.DelayDelDir...)\n\t\tsrcDelayDelMu.Unlock()\n\t\tif errors.Is(err, syscall.ECONNREFUSED) {\n\t\t\tlogger.Errorf(\"the management process has been stopped, so the worker process now exits\")\n\t\t\tos.Exit(1)\n\t\t}\n\t\tlogger.Errorf(\"update stats: %s %s\", string(ans), err)\n\t} else {\n\t\tskipped.IncrInt64(-r.Skipped)\n\t\tskippedBytes.IncrInt64(-r.SkippedBytes)\n\t\tcopied.IncrInt64(-r.Copied)\n\t\tcopiedBytes.IncrInt64(-r.CopiedBytes)\n\t\tif checked != nil {\n\t\t\tchecked.IncrInt64(-r.Checked)\n\t\t\tcheckedBytes.IncrInt64(-r.CheckedBytes)\n\t\t}\n\t\tif deleted != nil {\n\t\t\tdeleted.IncrInt64(-r.Deleted)\n\t\t}\n\t\tif failed != nil {\n\t\t\tfailed.IncrInt64(-r.Failed)\n\t\t}\n\t}\n}\n\nfunc startManager(config *Config, tasks <-chan object.Object) (string, error) {\n\thttp.HandleFunc(\"/fetch\", func(w http.ResponseWriter, req *http.Request) {\n\t\tvar objs []object.Object\n\t\tvar total int64\n\t\tobj, ok := <-tasks\n\t\tif !ok {\n\t\t\t_, _ = w.Write([]byte(\"[]\"))\n\t\t\treturn\n\t\t}\n\t\tobjs = append(objs, obj)\n\t\ttotal += obj.Size()\n\tLOOP:\n\t\tfor len(objs) < 100 && total < 400<<20 {\n\t\t\tselect {\n\t\t\tcase obj = <-tasks:\n\t\t\t\tif obj == nil {\n\t\t\t\t\tbreak LOOP\n\t\t\t\t}\n\t\t\t\tobjs = append(objs, obj)\n\t\t\t\ttotal += obj.Size()\n\t\t\tdefault:\n\t\t\t\tbreak LOOP\n\t\t\t}\n\t\t}\n\t\td, err := marshalObjects(objs)\n\t\tif err != nil {\n\t\t\thttp.Error(w, err.Error(), http.StatusInternalServerError)\n\t\t\treturn\n\t\t}\n\t\tlogger.Debugf(\"send %d objects(%s) to %s\", len(objs), humanize.IBytes(uint64(total)), req.RemoteAddr)\n\t\t_, _ = w.Write(d)\n\t})\n\thttp.HandleFunc(\"/stats\", func(w http.ResponseWriter, req *http.Request) {\n\t\tif req.Method != \"POST\" {\n\t\t\thttp.Error(w, \"POST required\", http.StatusBadRequest)\n\t\t\treturn\n\t\t}\n\t\td, err := io.ReadAll(req.Body)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"read: %s\", err)\n\t\t\treturn\n\t\t}\n\t\tvar r Stat\n\t\terr = json.Unmarshal(d, &r)\n\t\tif err != nil {\n\t\t\thttp.Error(w, err.Error(), http.StatusBadRequest)\n\t\t\treturn\n\t\t}\n\t\tupdateStats(&r)\n\t\tsrcDelayDelMu.Lock()\n\t\tsrcDelayDel = append(srcDelayDel, r.DelayDelDir...)\n\t\tsrcDelayDelMu.Unlock()\n\t\tlogger.Debugf(\"receive stats %+v from %s\", r, req.RemoteAddr)\n\t\t_, _ = w.Write([]byte(\"OK\"))\n\t})\n\tvar addr string\n\tu, err := url.Parse(\"ssh://\" + config.Workers[0])\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"invalid worker address %s: %s\", config.Workers[0], err)\n\t}\n\tif config.ManagerAddr != \"\" {\n\t\taddr = config.ManagerAddr\n\t\tif strings.HasPrefix(addr, \":\") || strings.Contains(addr, \"0.0.0.0\") {\n\t\t\tip, err := utils.GetLocalIp(net.JoinHostPort(u.Host, \"22\"))\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", fmt.Errorf(\"get local ip: %s\", err)\n\t\t\t}\n\t\t\taddr = ip + addr\n\t\t}\n\t} else {\n\t\tip, err := utils.GetLocalIp(net.JoinHostPort(u.Host, \"22\"))\n\t\tif err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"not found local ip: %s\", err)\n\t\t}\n\t\tlogger.Debugf(\"Use local ip %s\", ip)\n\t\taddr = ip\n\t}\n\n\tif !strings.Contains(addr, \":\") {\n\t\taddr += \":\"\n\t}\n\n\tl, err := net.Listen(\"tcp\", addr)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"listen: %s\", err)\n\t}\n\tlogger.Infof(\"Listen at %s\", l.Addr())\n\tgo func() { _ = http.Serve(l, nil) }()\n\treturn l.Addr().String(), nil\n}\n\nfunc findSelfPath() (string, error) {\n\tprogram := os.Args[0]\n\tif strings.Contains(program, \"/\") {\n\t\tpath, err := filepath.Abs(program)\n\t\tif err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"resolve path %s: %s\", program, err)\n\t\t}\n\t\treturn path, nil\n\t}\n\tfor _, searchPath := range strings.Split(os.Getenv(\"PATH\"), \":\") {\n\t\tif searchPath != \"\" {\n\t\t\tp := filepath.Join(searchPath, program)\n\t\t\tif _, err := os.Stat(p); err == nil {\n\t\t\t\treturn p, nil\n\t\t\t}\n\t\t}\n\t}\n\treturn \"\", fmt.Errorf(\"can't find path for %s\", program)\n}\n\nfunc launchWorker(address string, config *Config, wg *sync.WaitGroup) {\n\tworkers := strings.Split(strings.Join(config.Workers, \",\"), \",\")\n\tfor _, host := range workers {\n\t\twg.Add(1)\n\t\tgo func(host string) {\n\t\t\tdefer wg.Done()\n\t\t\t// copy\n\t\t\tpath, err := findSelfPath()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"find self path: %s\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\trpath := filepath.Join(\"/tmp\", filepath.Base(path))\n\t\t\tcmd := exec.Command(\"rsync\", \"-a\", \"-e\", \"ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no\", path, host+\":\"+rpath)\n\t\t\toutput, err := cmd.CombinedOutput()\n\t\t\tlogger.Debugf(\"exec: %s,err: %s\", cmd.String(), string(output))\n\t\t\tif err != nil {\n\t\t\t\t// fallback to scp\n\t\t\t\tcmd = exec.Command(\"scp\", \"-o\", \"StrictHostKeyChecking=no\", \"-o\", \"PasswordAuthentication=no\", path, host+\":\"+rpath)\n\t\t\t\toutput, err = cmd.CombinedOutput()\n\t\t\t\tlogger.Debugf(\"exec: %s,err: %s\", cmd.String(), string(output))\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"copy itself to %s: %s\", host, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// launch itself\n\t\t\tvar args = []string{host}\n\t\t\t// set env\n\t\t\tvar printEnv []string\n\t\t\tfor k, v := range config.Env {\n\t\t\t\targs = append(args, fmt.Sprintf(\"%s=%s\", k, v))\n\t\t\t\tif strings.Contains(k, \"SECRET\") ||\n\t\t\t\t\tstrings.Contains(k, \"TOKEN\") ||\n\t\t\t\t\tstrings.Contains(k, \"PASSWORD\") ||\n\t\t\t\t\tstrings.Contains(k, \"AZURE_STORAGE_CONNECTION_STRING\") ||\n\t\t\t\t\tstrings.Contains(k, \"JFS_RSA_PASSPHRASE\") {\n\t\t\t\t\tv = \"******\"\n\t\t\t\t}\n\t\t\t\tprintEnv = append(printEnv, fmt.Sprintf(\"%s=%s\", k, v))\n\t\t\t}\n\n\t\t\targs = append(args, rpath)\n\t\t\targs = append(args, os.Args[1:]...)\n\t\t\targs = append(args, \"--manager\", address)\n\t\t\tif !config.Verbose && !config.Quiet {\n\t\t\t\targs = append(args, \"-q\")\n\t\t\t}\n\t\t\tvar argsBk = make([]string, len(args))\n\t\t\tcopy(argsBk, args)\n\t\t\tfor i, s := range printEnv {\n\t\t\t\targsBk[i+1] = s\n\t\t\t}\n\t\t\tlogger.Debugf(\"launch worker command args: [ssh, %s]\", strings.Join(shellescape.EscapeArgs(argsBk), \", \"))\n\t\t\tcmd = exec.Command(\"ssh\", shellescape.EscapeArgs(args)...)\n\t\t\tcmd.Stdin = os.Stdin\n\t\t\tstderr, err := cmd.StderrPipe()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"redirect stderr: %s\", err)\n\t\t\t}\n\t\t\terr = cmd.Start()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"start itself at %s: %s\", host, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tlogger.Infof(\"launch a worker on %s\", host)\n\t\t\tvar finished = make(chan struct{})\n\t\t\tvar logRe = regexp.MustCompile(`^.*<([A-Z]+)>: (.*)`)\n\t\t\tgo func() {\n\t\t\t\tr := bufio.NewReader(stderr)\n\t\t\t\tfor {\n\t\t\t\t\tline, err := r.ReadString('\\n')\n\t\t\t\t\tif err != nil || len(line) == 0 {\n\t\t\t\t\t\tfinished <- struct{}{}\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tline = strings.TrimSuffix(line, \"\\n\")\n\n\t\t\t\t\tvar level, content string\n\t\t\t\t\tif matches := logRe.FindStringSubmatch(line); len(matches) >= 3 {\n\t\t\t\t\t\tlevel = matches[1]\n\t\t\t\t\t\tcontent = matches[2]\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlevel = \"INFO\"\n\t\t\t\t\t\tcontent = line\n\t\t\t\t\t}\n\n\t\t\t\t\tswitch level {\n\t\t\t\t\tcase \"ERROR\":\n\t\t\t\t\t\tlogger.Errorf(\"[%s] %s\", host, content)\n\t\t\t\t\tcase \"WARNING\":\n\t\t\t\t\t\tlogger.Warnf(\"[%s] %s\", host, content)\n\t\t\t\t\tcase \"DEBUG\":\n\t\t\t\t\t\tlogger.Debugf(\"[%s] %s\", host, content)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tlogger.Infof(\"[%s] %s\", host, content)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\t\terr = cmd.Wait()\n\t\t\t<-finished\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"%s: %s\", host, err)\n\t\t\t}\n\t\t}(host)\n\t}\n}\n\nfunc marshalObjects(objs []object.Object) ([]byte, error) {\n\tvar arr []map[string]interface{}\n\tfor _, o := range objs {\n\t\tnsize := o.Size()\n\t\to = withoutSize(o)\n\t\tobj := object.MarshalObject(o)\n\t\tif nsize != o.Size() {\n\t\t\tobj[\"nsize\"] = nsize\n\t\t}\n\t\tarr = append(arr, obj)\n\t}\n\treturn json.MarshalIndent(arr, \"\", \" \")\n}\n\nfunc unmarshalObjects(d []byte) ([]object.Object, error) {\n\tvar arr []map[string]interface{}\n\terr := json.Unmarshal(d, &arr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar objs []object.Object\n\tfor _, m := range arr {\n\t\tobj := object.UnmarshalObject(m)\n\t\tif nsize, ok := m[\"nsize\"]; ok {\n\t\t\tobj = withSize(obj, int64(nsize.(float64)))\n\t\t}\n\t\tobjs = append(objs, obj)\n\t}\n\treturn objs, nil\n}\n\nfunc fetchJobs(tasks chan<- object.Object, config *Config) {\n\tfor {\n\t\turl := fmt.Sprintf(\"http://%s/fetch\", config.Manager)\n\t\tans, err := httpRequest(url, nil)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"fetch jobs: %s\", err)\n\t\t\ttime.Sleep(time.Second)\n\t\t\tcontinue\n\t\t}\n\t\tvar jobs []object.Object\n\t\tjobs, err = unmarshalObjects(ans)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Unmarshal %s: %s\", string(ans), err)\n\t\t\ttime.Sleep(time.Second)\n\t\t\tcontinue\n\t\t}\n\t\tlogger.Debugf(\"got %d jobs\", len(jobs))\n\t\tif len(jobs) == 0 {\n\t\t\tlogger.Infof(\"no more jobs\")\n\t\t\tbreak\n\t\t}\n\t\tfor _, obj := range jobs {\n\t\t\ttasks <- obj\n\t\t}\n\t}\n\tclose(tasks)\n}\n" }, { "path": "pkg/sync/cluster_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage sync\n\nimport (\n\t\"os\"\n\t\"os/user\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n)\n\ntype obj struct {\n\tkey string\n\tsize int64\n\tmtime time.Time\n\tisDir bool\n\tisSymlink bool\n}\n\nfunc (o *obj) Key() string { return o.key }\nfunc (o *obj) Size() int64 { return o.size }\nfunc (o *obj) Mtime() time.Time { return o.mtime }\nfunc (o *obj) IsDir() bool { return o.isDir }\nfunc (o *obj) IsSymlink() bool { return o.isSymlink }\nfunc (o *obj) StorageClass() string { return \"\" }\n\ntype file struct {\n\tobj\n}\n\nfunc (o *file) Owner() string { return \"\" }\nfunc (o *file) Group() string { return \"\" }\nfunc (o *file) Mode() os.FileMode { return 0 }\n\nfunc TestCluster(t *testing.T) {\n\t// manager\n\tworkerAddr := \"127.0.0.1\"\n\tif u, err := user.Current(); err != nil {\n\t\tlogger.Warnf(\"Failed to get current user: %v\", err)\n\t} else if u.Username != \"\" {\n\t\tworkerAddr = u.Username + \"@\" + workerAddr\n\t}\n\ttodo := make(chan object.Object, 100)\n\tvar conf Config\n\tconf.Workers = []string{workerAddr}\n\taddr, err := startManager(&conf, todo)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// sendStats(addr)\n\t// worker\n\tconf.Manager = addr\n\tmytodo := make(chan object.Object, 100)\n\tgo fetchJobs(mytodo, &conf)\n\n\ttodo <- &obj{key: \"test\"}\n\tclose(todo)\n\n\tobj := <-mytodo\n\tif obj.Key() != \"test\" {\n\t\tt.Fatalf(\"expect test but got %s\", obj.Key())\n\t}\n\tif _, ok := <-mytodo; ok {\n\t\tt.Fatalf(\"should end\")\n\t}\n}\n\nfunc TestMarshal(t *testing.T) {\n\tmtime := time.Now()\n\tvar objs = []object.Object{\n\t\t&obj{key: \"test\", mtime: mtime},\n\t\twithSize(&obj{key: \"test1\", size: 100}, -4),\n\t\twithSize(&file{obj{key: \"test2\", size: 200}}, -1),\n\t\twithSize(&file{obj{key: \"test3\", size: 200, isSymlink: true}}, -1),\n\t}\n\td, err := marshalObjects(objs)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tobjs2, e := unmarshalObjects(d)\n\tif e != nil {\n\t\tt.Fatal(e)\n\t}\n\tif objs2[0].Key() != \"test\" {\n\t\tt.Fatalf(\"expect test but got %s\", objs2[0].Key())\n\t}\n\tif !objs2[0].Mtime().Equal(objs[0].Mtime()) {\n\t\tt.Fatalf(\"expect %s but got %s\", mtime, objs2[0].Mtime())\n\t}\n\tif objs2[1].Key() != \"test1\" || objs2[1].Size() != -4 || withoutSize(objs2[1]).Size() != 100 {\n\t\tt.Fatalf(\"expect withSize but got %s\", objs2[1].Key())\n\t}\n\tif objs2[2].Key() != \"test2\" || objs2[2].Size() != -1 || withoutSize(objs2[2]).Size() != 200 {\n\t\tt.Fatalf(\"expect withFSize but got %s\", objs2[2].Key())\n\t}\n\tif objs2[3].Key() != \"test3\" || objs2[3].Size() != -1 || withoutSize(objs2[3]).Size() != 200 && objs2[3].IsSymlink() != true {\n\t\tt.Fatalf(\"expect withFSize but got %s\", objs2[3].Key())\n\t}\n}\n" }, { "path": "pkg/sync/config.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sync\n\nimport (\n\t\"math\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/spf13/cast\"\n\t\"github.com/urfave/cli/v2\"\n)\n\ntype Config struct {\n\tStorageClass string\n\tStart string\n\tEnd string\n\tThreads int\n\tUpdate bool\n\tForceUpdate bool\n\tPerms bool\n\tMaxFailure int64\n\tDry bool\n\tDeleteSrc bool\n\tDeleteDst bool\n\tMatchFullPath bool\n\tDirs bool\n\tExclude []string\n\tInclude []string\n\tExisting bool\n\tIgnoreExisting bool\n\tLinks bool\n\tInplace bool\n\tLimit int64\n\tManager string\n\tWorkers []string\n\tManagerAddr string\n\tListThreads int\n\tListDepth int\n\tBWLimit int64\n\tTrafficControlURL string\n\tNoHTTPS bool\n\tVerbose bool\n\tQuiet bool\n\tCheckAll bool\n\tCheckNew bool\n\tCheckChange bool\n\tMaxSize int64\n\tMinSize int64\n\tMaxAge time.Duration\n\tMinAge time.Duration\n\tStartTime time.Time\n\tEndTime time.Time\n\tEnv map[string]string\n\n\tFilesFrom string\n\n\trules []rule\n\tconcurrentList chan int\n\tRegisterer prometheus.Registerer\n}\n\nconst JFS_UMASK = \"JFS_UMASK\"\n\nfunc envList() []string {\n\treturn []string{\n\t\t\"ACCESS_KEY\",\n\t\t\"SECRET_KEY\",\n\t\t\"SESSION_TOKEN\",\n\n\t\t\"MINIO_ACCESS_KEY\",\n\t\t\"MINIO_SECRET_KEY\",\n\t\t\"MINIO_REGION\",\n\n\t\t\"META_PASSWORD\",\n\t\t\"REDIS_PASSWORD\",\n\t\t\"SENTINEL_PASSWORD\",\n\t\t\"SENTINEL_PASSWORD_FOR_OBJ\",\n\n\t\t\"AZURE_STORAGE_CONNECTION_STRING\",\n\n\t\t\"BDCLOUD_DEFAULT_REGION\",\n\t\t\"BDCLOUD_ACCESS_KEY\",\n\t\t\"BDCLOUD_SECRET_KEY\",\n\n\t\t\"COS_SECRETID\",\n\t\t\"COS_SECRETKEY\",\n\n\t\t\"EOS_ACCESS_KEY\",\n\t\t\"EOS_SECRET_KEY\",\n\t\t\"EOS_TOKEN\",\n\n\t\t\"GOOGLE_CLOUD_PROJECT\",\n\n\t\t\"HADOOP_USER_NAME\",\n\t\t\"HADOOP_SUPER_USER\",\n\t\t\"HADOOP_SUPER_GROUP\",\n\t\t\"HADOOP_CONF_DIR\",\n\t\t\"HADOOP_HOME\",\n\t\t\"KRB5_CONFIG\",\n\t\t\"KRB5CCNAME\",\n\t\t\"KRB5KEYTAB\",\n\t\t\"KRB5KEYTAB_BASE64\",\n\t\t\"KRB5PRINCIPAL\",\n\n\t\t\"AWS_REGION\",\n\t\t\"AWS_DEFAULT_REGION\",\n\n\t\t\"HWCLOUD_DEFAULT_REGION\",\n\t\t\"HWCLOUD_ACCESS_KEY\",\n\t\t\"HWCLOUD_SECRET_KEY\",\n\n\t\t\"ALICLOUD_REGION_ID\",\n\t\t\"ALICLOUD_ACCESS_KEY_ID\",\n\t\t\"ALICLOUD_ACCESS_KEY_SECRET\",\n\t\t\"SECURITY_TOKEN\",\n\n\t\t\"CEPH_ADMIN_SOCKET\",\n\t\t\"CEPH_LOG_FILE\",\n\n\t\t\"QINIU_DOMAIN\",\n\n\t\t\"SCW_ACCESS_KEY\",\n\t\t\"SCW_SECRET_KEY\",\n\n\t\t\"SSH_PRIVATE_KEY_PATH\",\n\t\t\"SSH_AUTH_SOCK\",\n\n\t\t\"JFS_RSA_PASSPHRASE\",\n\t\t\"PYROSCOPE_AUTH_TOKEN\",\n\t\t\"DISPLAY_PROGRESSBAR\",\n\t\t\"CGOFUSE_TRACE\",\n\t\t\"JUICEFS_DEBUG\",\n\t\t\"JUICEFS_LOGLEVEL\",\n\t}\n}\n\nfunc NewConfigFromCli(c *cli.Context) *Config {\n\tif c.Int64(\"limit\") < -1 {\n\t\tlogger.Fatal(\"limit should not be less than -1\")\n\t}\n\tvar startTime, endTime time.Time\n\tvar err error\n\tif c.IsSet(\"start-time\") {\n\t\tstartTime, err = cast.ToTimeInDefaultLocationE(c.String(\"start-time\"), time.Local)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"failed to parse start time: %v\", err)\n\t\t}\n\t}\n\tif c.IsSet(\"end-time\") {\n\t\tendTime, err = cast.ToTimeInDefaultLocationE(c.String(\"end-time\"), time.Local)\n\t\tif err != nil {\n\t\t\tlogger.Fatalf(\"failed to parse end time: %v\", err)\n\t\t}\n\t}\n\tcfg := &Config{\n\t\tStorageClass: c.String(\"storage-class\"),\n\t\tStart: c.String(\"start\"),\n\t\tEnd: c.String(\"end\"),\n\t\tThreads: c.Int(\"threads\"),\n\t\tListThreads: c.Int(\"list-threads\"),\n\t\tListDepth: c.Int(\"list-depth\"),\n\t\tUpdate: c.Bool(\"update\"),\n\t\tForceUpdate: c.Bool(\"force-update\"),\n\t\tPerms: c.Bool(\"perms\"),\n\t\tDirs: c.Bool(\"dirs\"),\n\t\tDry: c.Bool(\"dry\"),\n\t\tMaxFailure: c.Int64(\"max-failure\"),\n\t\tDeleteSrc: c.Bool(\"delete-src\"),\n\t\tDeleteDst: c.Bool(\"delete-dst\"),\n\t\tExclude: c.StringSlice(\"exclude\"),\n\t\tInclude: c.StringSlice(\"include\"),\n\t\tMatchFullPath: c.Bool(\"match-full-path\"),\n\t\tExisting: c.Bool(\"existing\"),\n\t\tIgnoreExisting: c.Bool(\"ignore-existing\"),\n\t\tLinks: c.Bool(\"links\"),\n\t\tInplace: c.Bool(\"inplace\"),\n\t\tLimit: c.Int64(\"limit\"),\n\t\tWorkers: c.StringSlice(\"worker\"),\n\t\tManagerAddr: c.String(\"manager-addr\"),\n\t\tManager: c.String(\"manager\"),\n\t\tBWLimit: utils.ParseMbps(c, \"bwlimit\"),\n\t\tTrafficControlURL: c.String(\"traffic-control-url\"),\n\t\tNoHTTPS: c.Bool(\"no-https\"),\n\t\tVerbose: c.Bool(\"verbose\"),\n\t\tQuiet: c.Bool(\"quiet\"),\n\t\tCheckAll: c.Bool(\"check-all\"),\n\t\tCheckNew: c.Bool(\"check-new\"),\n\t\tCheckChange: c.Bool(\"check-change\"),\n\t\tMaxSize: int64(utils.ParseBytes(c, \"max-size\", 'B')),\n\t\tMinSize: int64(utils.ParseBytes(c, \"min-size\", 'B')),\n\t\tMaxAge: utils.Duration(c.String(\"max-age\")),\n\t\tMinAge: utils.Duration(c.String(\"min-age\")),\n\t\tStartTime: startTime,\n\t\tEndTime: endTime,\n\t\tFilesFrom: c.String(\"files-from\"),\n\t\tEnv: make(map[string]string),\n\t}\n\tif !c.IsSet(\"max-size\") {\n\t\tcfg.MaxSize = math.MaxInt64\n\t}\n\tif cfg.MinSize > cfg.MaxSize {\n\t\tlogger.Fatal(\"min-size should not be larger than max-size\")\n\t}\n\tif cfg.MaxAge > 0 && cfg.MinAge > cfg.MaxAge {\n\t\tlogger.Fatal(\"min-age should not be larger than max-age\")\n\t}\n\tif cfg.Threads <= 0 {\n\t\tlogger.Warnf(\"threads should be larger than 0, reset it to 1\")\n\t\tcfg.Threads = 1\n\t}\n\tfor _, key := range envList() {\n\t\tif os.Getenv(key) != \"\" {\n\t\t\tcfg.Env[key] = os.Getenv(key)\n\t\t}\n\t}\n\t// pass all the variable that contains \"JFS\"\n\tfor _, ekv := range os.Environ() {\n\t\tkey := strings.Split(ekv, \"=\")[0]\n\t\tif strings.Contains(key, \"JFS\") {\n\t\t\tcfg.Env[key] = os.Getenv(key)\n\t\t}\n\t}\n\t// pass umask to workers\n\tcfg.Env[JFS_UMASK] = strconv.Itoa(utils.GetUmask())\n\n\t// workers: set umask for the current process\n\tif umask := os.Getenv(JFS_UMASK); umask != \"\" {\n\t\tutils.SetUmask(cast.ToInt(umask))\n\t}\n\n\treturn cfg\n}\n" }, { "path": "pkg/sync/download.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sync\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n)\n\ntype parallelDownloader struct {\n\tsync.Mutex\n\tnotify *sync.Cond\n\tsrc object.ObjectStorage\n\tkey string\n\tfsize int64\n\tblockSize int64\n\tconcurrent chan int\n\tbuffers map[int64]*[]byte\n\toff int64\n\terr error\n}\n\nfunc (r *parallelDownloader) hasErr() bool {\n\tr.Lock()\n\tdefer r.Unlock()\n\treturn r.err != nil\n}\n\nfunc (r *parallelDownloader) setErr(err error) {\n\tr.Lock()\n\tdefer r.Unlock()\n\tr.err = err\n}\n\nconst downloadBufSize = 10 << 20\n\nvar downloadBufPool = sync.Pool{\n\tNew: func() interface{} {\n\t\tbuf := make([]byte, 0, downloadBufSize)\n\t\treturn &buf\n\t},\n}\n\nfunc (r *parallelDownloader) download() {\n\tfor off := int64(0); off < r.fsize; off += r.blockSize {\n\t\tr.concurrent <- 1\n\t\tgo func(off int64) {\n\t\t\tvar size = r.blockSize\n\t\t\tif off+r.blockSize > r.fsize {\n\t\t\t\tsize = r.fsize - off\n\t\t\t}\n\t\t\tvar saved bool\n\t\t\tif !r.hasErr() {\n\t\t\t\tif limiter != nil {\n\t\t\t\t\tlimiter.Wait(size)\n\t\t\t\t}\n\t\t\t\tvar in io.ReadCloser\n\t\t\t\te := try(3, func() error {\n\t\t\t\t\tvar err error\n\t\t\t\t\tin, err = r.src.Get(context.Background(), r.key, off, size)\n\t\t\t\t\treturn err\n\t\t\t\t})\n\t\t\t\tif e != nil {\n\t\t\t\t\tr.setErr(e)\n\t\t\t\t} else { //nolint:typecheck\n\t\t\t\t\tdefer in.Close()\n\t\t\t\t\tp := downloadBufPool.Get().(*[]byte)\n\t\t\t\t\t*p = (*p)[:size]\n\t\t\t\t\t_, e = io.ReadFull(in, *p)\n\t\t\t\t\tif e != nil {\n\t\t\t\t\t\tr.setErr(e)\n\t\t\t\t\t\tdownloadBufPool.Put(p)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tr.Lock()\n\t\t\t\t\t\tif r.buffers != nil {\n\t\t\t\t\t\t\tr.buffers[off] = p\n\t\t\t\t\t\t\tsaved = true\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tdownloadBufPool.Put(p)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tr.Unlock()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !saved {\n\t\t\t\t<-r.concurrent\n\t\t\t}\n\t\t\tr.notify.Signal()\n\t\t}(off)\n\t}\n}\n\nfunc (r *parallelDownloader) Read(b []byte) (int, error) {\n\tif len(b) == 0 {\n\t\treturn 0, nil\n\t}\n\tif r.off >= r.fsize {\n\t\treturn 0, io.EOF\n\t}\n\toff := r.off / r.blockSize * r.blockSize\n\tr.Lock()\n\tfor r.err == nil && r.buffers[off] == nil {\n\t\tr.notify.Wait()\n\t}\n\tp := r.buffers[off]\n\tr.Unlock()\n\tif p == nil {\n\t\treturn 0, r.err\n\t}\n\tn := copy(b, (*p)[r.off-off:])\n\tr.off += int64(n)\n\tif r.off == off+int64(len(*p)) {\n\t\tdownloadBufPool.Put(p)\n\t\tr.Lock()\n\t\tdelete(r.buffers, off)\n\t\tr.Unlock()\n\t\t<-r.concurrent\n\t}\n\tif copiedBytes != nil {\n\t\tcopiedBytes.IncrInt64(int64(n))\n\t}\n\treturn n, nil\n}\n\nfunc (r *parallelDownloader) Close() {\n\tr.Lock()\n\tdefer r.Unlock()\n\tfor _, p := range r.buffers {\n\t\tdownloadBufPool.Put(p)\n\t}\n\tr.buffers = nil\n\tif r.err == nil {\n\t\tr.err = errors.New(\"closed\")\n\t}\n}\n\nfunc newParallelDownloader(store object.ObjectStorage, key string, size int64, bSize int64, concurrent chan int) *parallelDownloader {\n\tif bSize < 1 {\n\t\tpanic(\"concurrent and blockSize must be positive integer\")\n\t}\n\tdown := ¶llelDownloader{\n\t\tsrc: store,\n\t\tkey: key,\n\t\tfsize: size,\n\t\tblockSize: bSize,\n\t\tconcurrent: concurrent,\n\t\tbuffers: make(map[int64]*[]byte),\n\t}\n\tdown.notify = sync.NewCond(down)\n\tgo down.download()\n\treturn down\n}\n" }, { "path": "pkg/sync/download_test.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sync\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nfunc TestDownload(t *testing.T) {\n\tkey := \"testDownload\"\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/download/\", \"\", \"\", \"\")\n\tt.Cleanup(func() {\n\t\tos.RemoveAll(\"/tmp/download/\")\n\t})\n\ttype config struct {\n\t\tconcurrent int\n\t\tfsize int64\n\t}\n\ttype tcase struct {\n\t\tconfig\n\t\ttfunc func(t *testing.T, pr *parallelDownloader, content []byte)\n\t}\n\n\ttcases := []tcase{\n\t\t{config: config{fsize: downloadBufSize*3 + 100, concurrent: 4}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tres, err := io.ReadAll(pr)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !bytes.Equal(res, content) {\n\t\t\t\tt.Fatalf(\"get wrong content by io.ReadAll\")\n\t\t\t}\n\t\t}},\n\n\t\t{config: config{fsize: 97340326, concurrent: 4}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tres, err := io.ReadAll(pr)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !bytes.Equal(res, content) {\n\t\t\t\tt.Fatalf(\"get wrong content by io.ReadAll\")\n\t\t\t}\n\t\t}},\n\n\t\t{config: config{fsize: downloadBufSize*3 + 100, concurrent: 5}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tres, err := io.ReadAll(pr)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !bytes.Equal(res, content) {\n\t\t\t\tt.Fatalf(\"get wrong content by io.ReadAll\")\n\t\t\t}\n\t\t}},\n\n\t\t{config: config{fsize: 1, concurrent: 5}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tres := make([]byte, 1)\n\t\t\tn, err := pr.Read(res)\n\t\t\tif err != nil || n != 1 || res[0] != content[0] {\n\t\t\t\tt.Fatalf(\"read 1 byte should succeed\")\n\t\t\t}\n\t\t\tn, err = pr.Read(res)\n\t\t\tif err != io.EOF || n != 0 {\n\t\t\t\tt.Fatalf(\"err should be io.EOF or n should equal 0, but got %s %d\", err, n)\n\t\t\t}\n\t\t}},\n\n\t\t{config: config{fsize: 2, concurrent: 5}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tres := make([]byte, 1)\n\t\t\tn, err := pr.Read(res)\n\t\t\tif err != nil || n != 1 || res[0] != content[0] {\n\t\t\t\tt.Fatalf(\"read 1 byte should succeed\")\n\t\t\t}\n\t\t\tn, err = pr.Read(res)\n\t\t\tif err != nil || n != 1 || res[0] != content[1] {\n\t\t\t\tt.Fatalf(\"read 1 byte should succeed\")\n\t\t\t}\n\t\t\tn, err = pr.Read(res)\n\t\t\tif err != io.EOF || n != 0 {\n\t\t\t\tt.Fatalf(\"err should be io.EOF or n should equal 0, but got %s %d\", err, n)\n\t\t\t}\n\t\t}},\n\n\t\t{config: config{fsize: 2, concurrent: 1}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tres := make([]byte, 1)\n\t\t\tn, err := pr.Read(res)\n\n\t\t\tif err != nil || n != 1 || res[0] != content[0] {\n\t\t\t\tt.Fatalf(\"read 1 byte should succeed\")\n\t\t\t}\n\t\t\tn, err = pr.Read(res)\n\t\t\tif err != nil || n != 1 || res[0] != content[1] {\n\t\t\t\tt.Fatalf(\"read 1 byte should succeed\")\n\t\t\t}\n\t\t\tn, err = pr.Read(res)\n\t\t\tif err != io.EOF || n != 0 {\n\t\t\t\tt.Fatalf(\"err should be io.EOF or n should equal 0, but got %s %d\", err, n)\n\t\t\t}\n\t\t}},\n\n\t\t{config: config{fsize: downloadBufSize * 20, concurrent: 3}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tresSize := 4 * downloadBufSize\n\t\t\tres := make([]byte, 4*downloadBufSize)\n\t\t\tn, err := io.ReadFull(pr, res)\n\n\t\t\tif err != nil || n != resSize || res[0] != content[0] {\n\t\t\t\tt.Fatalf(\"read %v byte should succeed, but got %d, %s\", resSize, n, err)\n\t\t\t}\n\t\t\tn, err = io.ReadFull(pr, res)\n\t\t\tif err != nil || n != resSize || res[0] != content[resSize] {\n\t\t\t\tt.Fatalf(\"read %v byte should succeed, but got %d, %s\", resSize, n, err)\n\t\t\t}\n\t\t\t_ = a.Delete(ctx, key)\n\t\t\tn, err = io.ReadFull(pr, res)\n\t\t\tn, err = io.ReadFull(pr, res)\n\t\t\tif !os.IsNotExist(err) {\n\t\t\t\tt.Fatalf(\"err should be ErrNotExist, but got %d, %s\", n, err)\n\t\t\t}\n\t\t}},\n\n\t\t{config: config{fsize: 0, concurrent: 5}, tfunc: func(t *testing.T, pr *parallelDownloader, content []byte) {\n\t\t\tdefer pr.Close()\n\t\t\tres := make([]byte, 1)\n\t\t\tn, err := pr.Read(res)\n\t\t\tif err != io.EOF || n != 0 {\n\t\t\t\tt.Fatalf(\"err should be io.EOF or n should equal 0, but got %s %d\", err, n)\n\t\t\t}\n\t\t}},\n\t}\n\n\tfor _, c := range tcases {\n\t\tcontent := make([]byte, c.config.fsize)\n\t\tutils.RandRead(content)\n\t\t_ = a.Put(ctx, key, bytes.NewReader(content))\n\t\tc.tfunc(t, newParallelDownloader(a, key, c.config.fsize, downloadBufSize, make(chan int, c.concurrent)), content)\n\t}\n\n\tdownloader := newParallelDownloader(a, \"notExist\", 10*downloadBufSize, downloadBufSize, make(chan int, 5))\n\tres := make([]byte, 1)\n\tn, err := downloader.Read(res)\n\tif !os.IsNotExist(err) || n != 0 {\n\t\tt.Fatalf(\"err should be ErrNotExist or n should equal 0, but got %s %d\", err, n)\n\t}\n}\n" }, { "path": "pkg/sync/sync.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sync\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\t\"unicode\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juju/ratelimit\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/vimeo/go-util/crc32combine\"\n)\n\n// The max number of key per listing request\nconst (\n\tmaxResults = 1000\n\tdefaultPartSize = 5 << 20\n\tbufferSize = 32 << 10\n\tmaxBlock = defaultPartSize * 2\n\tmarkDeleteSrc = -1\n\tmarkDeleteDst = -2\n\tmarkCopyPerms = -3\n\tmarkChecksum = -4\n)\n\nvar (\n\thandled *utils.Bar\n\tpending *utils.Bar\n\tcopied, copiedBytes *utils.Bar\n\tchecked, checkedBytes *utils.Bar\n\tskipped, skippedBytes *utils.Bar\n\texcluded, excludedBytes *utils.Bar\n\textra, extraBytes *utils.Bar\n\tdeleted, failed *utils.Bar\n\tlistedPrefix *utils.Bar\n\tconcurrent chan int\n\tlimiter *mixedLimiter\n\ttotalHandled atomic.Int64\n)\n\ntype mixedLimiter struct {\n\tglobal *globalLimit\n\tlocal *ratelimit.Bucket\n}\n\nfunc (l *mixedLimiter) Wait(count int64) {\n\tif l.local != nil {\n\t\tl.local.Wait(count)\n\t}\n\tif l.global != nil {\n\t\tl.global.wait(count)\n\t}\n}\n\ntype globalLimit struct {\n\tsync.Mutex\n\tbalance int64\n\tdue time.Time\n\tneed int64\n\twaiters []*sync.Cond\n\n\taddress string\n}\ntype req struct {\n\t// Positive numbers indicate a request, negative numbers indicate a payback.\n\tBytes int64 `json:\"bytes\"`\n}\n\ntype resp struct {\n\tGranted int64 `json:\"granted\"` // bytes\n\tExpired int64 `json:\"expired\"` // Millisecond\n}\n\nfunc (l *globalLimit) request(ask int64) (int64, int64, error) {\n\tr := req{Bytes: ask}\n\tdata, err := json.Marshal(r)\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\tresult, err := http.Post(l.address, \"application/json\", bytes.NewReader(data))\n\tif err != nil || result.StatusCode != http.StatusOK {\n\t\tvar status string\n\t\tif result != nil {\n\t\t\tstatus = http.StatusText(result.StatusCode)\n\t\t}\n\t\tlogger.Errorf(\"request traffic control %s failed: %s, http status: %s\", l.address, err, status)\n\t\treturn 0, 0, err\n\t}\n\tdefer result.Body.Close()\n\tcontent, err := io.ReadAll(result.Body)\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\tres := resp{}\n\tif err := json.Unmarshal(content, &res); err != nil {\n\t\treturn 0, 0, err\n\t}\n\treturn res.Granted, res.Expired, nil\n}\n\nfunc (l *globalLimit) wait(bytes int64) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tif bytes <= 0 || l.balance >= bytes && len(l.waiters) == 0 {\n\t\tl.balance -= bytes\n\t\treturn\n\t}\n\tl.need += bytes\n\n\tvar me = sync.NewCond(l)\n\tl.waiters = append(l.waiters, me)\n\tfor l.waiters[0] != me {\n\t\tme.Wait()\n\t}\n\n\tif l.balance < bytes {\n\t\t// request credit for other waiters together\n\t\task := l.need - l.balance\n\t\tif ask >= bytes*10 {\n\t\t\t// don't wait for too long\n\t\t\task = bytes * 10\n\t\t}\n\t\tl.Unlock()\n\t\tgranted, expire, err := l.request(ask)\n\t\tl.Lock()\n\t\tif err == nil {\n\t\t\tl.balance += granted\n\t\t\tl.due = time.Now().Add(time.Millisecond * time.Duration(expire))\n\t\t\tlogger.Debugf(\"grant %d from %s until %s\", granted, l.address, l.due)\n\t\t}\n\t}\n\n\tl.balance -= bytes\n\tl.need -= bytes\n\tl.waiters = l.waiters[1:]\n\tif len(l.waiters) > 0 {\n\t\tl.waiters[0].Signal()\n\t}\n}\n\nfunc (l *globalLimit) checkBalance() {\n\tnow := time.Now()\n\tl.Lock()\n\tif l.balance > 0 && l.need == 0 && l.due.Before(now) {\n\t\tpayback := l.balance\n\t\tif payback > 1<<30 {\n\t\t\tpayback = 1 << 30\n\t\t}\n\t\tl.balance -= payback\n\t\tl.Unlock()\n\t\t_, _, _ = l.request(-payback)\n\t} else {\n\t\tl.Unlock()\n\t}\n}\n\nvar crcTable = crc32.MakeTable(crc32.Castagnoli)\nvar logger = utils.GetLogger(\"juicefs\")\nvar ctx = context.Background()\n\nfunc incrTotal(n int64) {\n\ttotalHandled.Add(n)\n}\n\nfunc incrHandled(n int) {\n\told := totalHandled.Swap(0)\n\thandled.IncrTotal(old)\n\thandled.IncrBy(n)\n}\n\ntype chksumReader struct {\n\tio.Reader\n\tchksum uint32\n\tcal bool\n}\n\nfunc (r *chksumReader) Read(p []byte) (n int, err error) {\n\tn, err = r.Reader.Read(p)\n\tif r.cal {\n\t\tr.chksum = crc32.Update(r.chksum, crcTable, p[:n])\n\t}\n\treturn\n}\n\ntype chksumWithSz struct {\n\tchksum uint32\n\tsize int64\n}\n\n// human readable bytes size\nfunc formatSize(bytes int64) string {\n\tunits := [7]string{\" \", \"K\", \"M\", \"G\", \"T\", \"P\", \"E\"}\n\tif bytes < 1024 {\n\t\treturn fmt.Sprintf(\"%v B\", bytes)\n\t}\n\tz := 0\n\tv := float64(bytes)\n\tfor v > 1024.0 {\n\t\tz++\n\t\tv /= 1024.0\n\t}\n\treturn fmt.Sprintf(\"%.2f %siB\", v, units[z])\n}\n\n// ListAll on all the keys that starts at marker from object storage.\nfunc ListAll(store object.ObjectStorage, prefix, start, end string, followLink bool) (<-chan object.Object, error) {\n\tstartTime := time.Now()\n\tlogger.Debugf(\"Iterating objects from %s with prefix %s start %q\", store, prefix, start)\n\n\tout := make(chan object.Object, maxResults*10)\n\n\t// As the result of object storage's List method doesn't include the marker key,\n\t// we try List the marker key separately.\n\tif start != \"\" && strings.HasPrefix(start, prefix) {\n\t\tif obj, err := store.Head(ctx, start); err == nil {\n\t\t\tlogger.Debugf(\"Found start key: %s from %s in %s\", start, store, time.Since(startTime))\n\t\t\tout <- obj\n\t\t}\n\t}\n\n\tif ch, err := store.ListAll(ctx, prefix, start, followLink); err == nil {\n\t\tgo func() {\n\t\t\tfor obj := range ch {\n\t\t\t\tif obj != nil && end != \"\" && obj.Key() > end {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tout <- obj\n\t\t\t}\n\t\t\tclose(out)\n\t\t}()\n\t\treturn out, nil\n\t} else if !errors.Is(err, utils.ENOTSUP) {\n\t\treturn nil, err\n\t}\n\n\tmarker := start\n\tlogger.Debugf(\"Listing objects from %s marker %q\", store, marker)\n\n\tobjs, hasMore, nextToken, err := store.List(ctx, prefix, marker, \"\", \"\", maxResults, followLink)\n\tif errors.Is(err, utils.ENOTSUP) {\n\t\treturn object.ListAllWithDelimiter(ctx, store, prefix, start, end, followLink)\n\t}\n\tif err != nil {\n\t\tlogger.Errorf(\"Can't list %s: %s\", store, err.Error())\n\t\treturn nil, err\n\t}\n\tlogger.Debugf(\"Found %d object from %s in %s\", len(objs), store, time.Since(startTime))\n\tgo func() {\n\t\tlastkey := \"\"\n\t\tfirst := true\n\tEND:\n\t\tfor {\n\t\t\tfor _, obj := range objs {\n\t\t\t\tkey := obj.Key()\n\t\t\t\tif !first && key <= lastkey {\n\t\t\t\t\tlogger.Errorf(\"The keys are out of order: marker %q, last %q current %q\", marker, lastkey, key)\n\t\t\t\t\tout <- nil\n\t\t\t\t\tbreak END\n\t\t\t\t}\n\t\t\t\tif end != \"\" && key > end {\n\t\t\t\t\tbreak END\n\t\t\t\t}\n\t\t\t\tlastkey = key\n\t\t\t\t// logger.Debugf(\"key: %s\", key)\n\t\t\t\tout <- obj\n\t\t\t\tfirst = false\n\t\t\t}\n\t\t\tif !hasMore {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tmarker = lastkey\n\t\t\tstartTime = time.Now()\n\t\t\tlogger.Debugf(\"Continue listing objects from %s marker %q\", store, marker)\n\t\t\tvar nextToken2 string\n\t\t\tobjs, hasMore, nextToken2, err = store.List(ctx, prefix, marker, nextToken, \"\", maxResults, followLink)\n\t\t\tcount := 0\n\t\t\tfor err != nil && count < 3 {\n\t\t\t\tlogger.Warnf(\"Fail to list: %s, retry again\", err.Error())\n\t\t\t\t// slow down\n\t\t\t\ttime.Sleep(time.Millisecond * 100)\n\n\t\t\t\tobjs, hasMore, nextToken2, err = store.List(ctx, prefix, marker, nextToken, \"\", maxResults, followLink)\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tlogger.Debugf(\"Found %d object from %s in %s\", len(objs), store, time.Since(startTime))\n\t\t\tif err != nil {\n\t\t\t\t// Telling that the listing has failed\n\t\t\t\tout <- nil\n\t\t\t\tlogger.Errorf(\"Fail to list after %s: %s\", marker, err.Error())\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tnextToken = nextToken2\n\t\t\tif len(objs) > 0 && objs[0].Key() == marker {\n\t\t\t\t// workaround from a object store that is not compatible to S3.\n\t\t\t\tobjs = objs[1:]\n\t\t\t}\n\t\t}\n\t\tclose(out)\n\t}()\n\treturn out, nil\n}\n\nvar bufPool = sync.Pool{\n\tNew: func() interface{} {\n\t\tbuf := make([]byte, bufferSize)\n\t\treturn &buf\n\t},\n}\n\nfunc shouldRetry(err error) bool {\n\tif err == nil || errors.Is(err, utils.ErrSkipped) || errors.Is(err, utils.ErrExtlink) {\n\t\treturn false\n\t}\n\n\tvar eno syscall.Errno\n\tif errors.As(err, &eno) {\n\t\tswitch eno {\n\t\tcase syscall.EAGAIN, syscall.EINTR, syscall.EBUSY, syscall.ETIMEDOUT, syscall.EIO:\n\t\t\treturn true\n\t\tdefault:\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc try(n int, f func() error) (err error) {\n\tfor i := 0; i < n; i++ {\n\t\terr = f()\n\t\tif !shouldRetry(err) {\n\t\t\treturn\n\t\t}\n\t\tlogger.Debugf(\"Try %d failed: %s\", i+1, err)\n\t\ttime.Sleep(time.Second * time.Duration(i*i))\n\t}\n\treturn\n}\n\nfunc deleteObj(storage object.ObjectStorage, key string, dry bool) {\n\tif dry {\n\t\tlogger.Debugf(\"Will delete %s from %s\", key, storage)\n\t\tdeleted.Increment()\n\t\treturn\n\t}\n\tstart := time.Now()\n\tif err := try(3, func() error { return storage.Delete(ctx, key) }); err == nil {\n\t\tdeleted.Increment()\n\t\tlogger.Debugf(\"Deleted %s from %s in %s\", key, storage, time.Since(start))\n\t} else {\n\t\tfailed.Increment()\n\t\tlogger.Errorf(\"Failed to delete %s from %s in %s: %s\", key, storage, time.Since(start), err)\n\t}\n}\n\nfunc needCopyPerms(o1, o2 object.Object) bool {\n\tf1 := o1.(object.File)\n\tf2 := o2.(object.File)\n\treturn f2.Mode() != f1.Mode() || f2.Owner() != f1.Owner() || f2.Group() != f1.Group()\n}\n\nfunc copyPerms(dst object.ObjectStorage, obj object.Object, config *Config) {\n\tstart := time.Now()\n\tkey := obj.Key()\n\tfi := obj.(object.File)\n\tif !fi.IsSymlink() || !config.Links {\n\t\t// chmod needs to be executed after chown, because chown will change setuid setgid to be invalid.\n\t\tif err := dst.(object.FileSystem).Chown(key, fi.Owner(), fi.Group()); err != nil {\n\t\t\tlogger.Warnf(\"Chown %s to (%s,%s): %s\", key, fi.Owner(), fi.Group(), err)\n\t\t}\n\t\tif err := dst.(object.FileSystem).Chmod(key, fi.Mode()); err != nil {\n\t\t\tlogger.Warnf(\"Chmod %s to %o: %s\", key, fi.Mode(), err)\n\t\t}\n\t}\n\tlogger.Debugf(\"Copied permissions (%s:%s:%s) for %s in %s\", fi.Owner(), fi.Group(), fi.Mode(), key, time.Since(start))\n}\n\nfunc calPartChksum(objStor object.ObjectStorage, key string, abort chan struct{}, offset, length int64) (uint32, error) {\n\tif limiter != nil {\n\t\tlimiter.Wait(length)\n\t}\n\tselect {\n\tcase <-abort:\n\t\treturn 0, fmt.Errorf(\"aborted\")\n\tcase concurrent <- 1:\n\t\tdefer func() {\n\t\t\t<-concurrent\n\t\t}()\n\t}\n\tin, err := objStor.Get(ctx, key, offset, length)\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"dest get: %s\", err)\n\t}\n\tdefer in.Close()\n\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\tvar chksum uint32\n\tfor left := int(length); left > 0; left -= bufferSize {\n\t\tbs := bufferSize\n\t\tif left < bufferSize {\n\t\t\tbs = left\n\t\t}\n\t\t*buf = (*buf)[:bs]\n\t\tif _, err = io.ReadFull(in, *buf); err != nil {\n\t\t\treturn 0, fmt.Errorf(\"dest read: %s\", err)\n\t\t}\n\t\tchksum = crc32.Update(chksum, crcTable, *buf)\n\t}\n\treturn chksum, nil\n}\n\nfunc calObjChksum(objStor object.ObjectStorage, key string, abort chan struct{}, obj object.Object) (uint32, error) {\n\tvar err error\n\tvar chksum uint32\n\tif obj.Size() < maxBlock {\n\t\treturn calPartChksum(objStor, key, abort, 0, obj.Size())\n\t}\n\tn := int((obj.Size()-1)/defaultPartSize) + 1\n\terrs := make(chan error, n)\n\tchksums := make([]chksumWithSz, n)\n\tfor i := 0; i < n; i++ {\n\t\tgo func(num int) {\n\t\t\tsz := int64(defaultPartSize)\n\t\t\tif num == n-1 {\n\t\t\t\tsz = obj.Size() - int64(num)*defaultPartSize\n\t\t\t}\n\t\t\tchksum, err := calPartChksum(objStor, key, abort, int64(num)*defaultPartSize, sz)\n\t\t\tchksums[num] = chksumWithSz{chksum, sz}\n\t\t\terrs <- err\n\t\t}(i)\n\t}\n\tfor i := 0; i < n; i++ {\n\t\tif err = <-errs; err != nil {\n\t\t\tclose(abort)\n\t\t\tbreak\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tchksum = chksums[0].chksum\n\tfor i := 1; i < n; i++ {\n\t\tchksum = crc32combine.CRC32Combine(crc32.Castagnoli, chksum, chksums[i].chksum, chksums[i].size)\n\t}\n\treturn chksum, nil\n}\n\nfunc compObjPartBinary(src, dst object.ObjectStorage, key string, abort chan struct{}, offset, length int64) error {\n\tif limiter != nil {\n\t\tlimiter.Wait(length)\n\t}\n\tselect {\n\tcase <-abort:\n\t\treturn fmt.Errorf(\"aborted\")\n\tcase concurrent <- 1:\n\t\tdefer func() {\n\t\t\t<-concurrent\n\t\t}()\n\t}\n\tin, err := src.Get(ctx, key, offset, length)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"src get: %s\", err)\n\t}\n\tdefer in.Close()\n\tin2, err := dst.Get(ctx, key, offset, length)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"dest get: %s\", err)\n\t}\n\tdefer in2.Close()\n\n\tbuf := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf)\n\tbuf2 := bufPool.Get().(*[]byte)\n\tdefer bufPool.Put(buf2)\n\tfor left := int(length); left > 0; left -= bufferSize {\n\t\tbs := bufferSize\n\t\tif left < bufferSize {\n\t\t\tbs = left\n\t\t}\n\t\t*buf = (*buf)[:bs]\n\t\t*buf2 = (*buf2)[:bs]\n\t\tif _, err = io.ReadFull(in, *buf); err != nil {\n\t\t\treturn fmt.Errorf(\"src read: %s\", err)\n\t\t}\n\t\tif _, err = io.ReadFull(in2, *buf2); err != nil {\n\t\t\treturn fmt.Errorf(\"dest read: %s\", err)\n\t\t}\n\t\tif !bytes.Equal(*buf, *buf2) {\n\t\t\treturn fmt.Errorf(\"bytes not equal\")\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc compObjBinary(src, dst object.ObjectStorage, key string, abort chan struct{}, obj object.Object) (bool, error) {\n\tvar err error\n\tif obj.Size() < maxBlock {\n\t\terr = compObjPartBinary(src, dst, key, abort, 0, obj.Size())\n\t} else {\n\t\tn := int((obj.Size()-1)/defaultPartSize) + 1\n\t\terrs := make(chan error, n)\n\t\tfor i := 0; i < n; i++ {\n\t\t\tgo func(num int) {\n\t\t\t\tsz := int64(defaultPartSize)\n\t\t\t\tif num == n-1 {\n\t\t\t\t\tsz = obj.Size() - int64(num)*defaultPartSize\n\t\t\t\t}\n\t\t\t\terrs <- compObjPartBinary(src, dst, key, abort, int64(num)*defaultPartSize, sz)\n\t\t\t}(i)\n\t\t}\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif err = <-errs; err != nil {\n\t\t\t\tclose(abort)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tequal := false\n\tif err != nil && err.Error() == \"bytes not equal\" {\n\t\terr = nil\n\t} else {\n\t\tequal = err == nil\n\t}\n\treturn equal, err\n}\n\nfunc doCheckSum(src, dst object.ObjectStorage, key string, srcChksumPtr *uint32, obj object.Object, config *Config, equal *bool) error {\n\tif obj.IsSymlink() && config.Links && (config.CheckAll || config.CheckNew) {\n\t\tvar srcLink, dstLink string\n\t\tvar err error\n\t\tif s, ok := src.(object.SupportSymlink); ok {\n\t\t\tif srcLink, err = s.Readlink(key); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif s, ok := dst.(object.SupportSymlink); ok {\n\t\t\tif dstLink, err = s.Readlink(key); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\t*equal = srcLink == dstLink && srcLink != \"\" && dstLink != \"\"\n\t\treturn nil\n\t}\n\tabort := make(chan struct{})\n\tvar err error\n\tif srcChksumPtr != nil {\n\t\tvar srcChksum uint32\n\t\tvar dstChksum uint32\n\t\tsrcChksum = *srcChksumPtr\n\t\tdstChksum, err = calObjChksum(dst, key, abort, obj)\n\t\tif err == nil {\n\t\t\t*equal = srcChksum == dstChksum\n\t\t} else {\n\t\t\t*equal = false\n\t\t}\n\t} else {\n\t\t*equal, err = compObjBinary(src, dst, key, abort, obj)\n\t}\n\treturn err\n}\n\nfunc checkSum(src, dst object.ObjectStorage, key string, srcChksum *uint32, obj object.Object, config *Config) (bool, error) {\n\tstart := time.Now()\n\tvar equal bool\n\terr := try(3, func() error { return doCheckSum(src, dst, key, srcChksum, obj, config, &equal) })\n\tif err == nil {\n\t\tchecked.Increment()\n\t\tcheckedBytes.IncrInt64(obj.Size())\n\t\tif equal {\n\t\t\tlogger.Debugf(\"Checked %s OK (and equal) in %s,\", key, time.Since(start))\n\t\t} else {\n\t\t\tlogger.Warnf(\"Checked %s OK (but NOT equal) in %s,\", key, time.Since(start))\n\t\t}\n\t} else {\n\t\tlogger.Errorf(\"Failed to check %s in %s: %s\", key, time.Since(start), err)\n\t}\n\treturn equal, err\n}\n\nvar fastStreamRead = map[string]struct{}{\"file\": {}, \"hdfs\": {}, \"jfs\": {}, \"gluster\": {}}\nvar streamWrite = map[string]struct{}{\"file\": {}, \"hdfs\": {}, \"sftp\": {}, \"gs\": {}, \"wasb\": {}, \"ceph\": {}, \"swift\": {}, \"webdav\": {}, \"jfs\": {}, \"gluster\": {}}\nvar readInMem = map[string]struct{}{\"mem\": {}, \"etcd\": {}, \"redis\": {}, \"tikv\": {}, \"mysql\": {}, \"postgres\": {}, \"sqlite3\": {}}\n\nfunc inMap(obj object.ObjectStorage, m map[string]struct{}) bool {\n\t_, ok := m[strings.Split(obj.String(), \"://\")[0]]\n\treturn ok\n}\n\nfunc doCopySingle(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error) {\n\tif size > maxBlock && !inMap(dst, readInMem) && !inMap(src, fastStreamRead) {\n\t\tvar err error\n\t\tvar in io.Reader\n\t\tdowner := newParallelDownloader(src, key, size, downloadBufSize, concurrent)\n\t\tdefer downer.Close()\n\t\tif inMap(dst, streamWrite) {\n\t\t\tin = downer\n\t\t} else {\n\t\t\tvar f *os.File\n\t\t\t// download the object into disk\n\t\t\tif f, err = os.CreateTemp(\"\", \"rep\"); err != nil {\n\t\t\t\tlogger.Warnf(\"create temp file: %s\", err)\n\t\t\t\treturn doCopySingle0(src, dst, key, size, calChksum)\n\t\t\t}\n\t\t\t_ = os.Remove(f.Name()) // will be deleted after Close()\n\t\t\tdefer f.Close()\n\t\t\tbuf := bufPool.Get().(*[]byte)\n\t\t\tdefer bufPool.Put(buf)\n\t\t\t// hide f.ReadFrom to avoid discarding buf\n\t\t\tif _, err = io.CopyBuffer(struct{ io.Writer }{f}, downer, *buf); err == nil {\n\t\t\t\t_, err = f.Seek(0, 0)\n\t\t\t\tin = f\n\t\t\t}\n\t\t}\n\t\tr := &chksumReader{in, 0, calChksum}\n\t\tif err == nil {\n\t\t\terr = dst.Put(ctx, key, r)\n\t\t}\n\t\tif err != nil {\n\t\t\tif _, e := src.Head(ctx, key); os.IsNotExist(e) {\n\t\t\t\tlogger.Debugf(\"Head src %s: %s\", key, err)\n\t\t\t\terr = utils.ErrSkipped\n\t\t\t}\n\t\t}\n\t\treturn r.chksum, err\n\t}\n\treturn doCopySingle0(src, dst, key, size, calChksum)\n}\n\nfunc doCopySingle0(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error) {\n\tconcurrent <- 1\n\tdefer func() {\n\t\t<-concurrent\n\t}()\n\tvar in io.ReadCloser\n\tvar err error\n\tif size == 0 {\n\t\tif key == \"\" && !object.IsFileSystem(dst) {\n\t\t\tps := strings.SplitN(dst.String(), \"/\", 4)\n\t\t\tif len(ps) == 4 && ps[3] == \"\" {\n\t\t\t\tlogger.Warnf(\"empty key is not support by %s, ignore it\", dst)\n\t\t\t\treturn 0, nil\n\t\t\t}\n\t\t}\n\t\tif object.IsFileSystem(src) {\n\t\t\t// for check permissions\n\t\t\tr, err := src.Get(ctx, key, 0, -1)\n\t\t\tif err != nil {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t\t_ = r.Close()\n\t\t}\n\t\tin = io.NopCloser(bytes.NewReader(nil))\n\t} else {\n\t\tin, err = src.Get(ctx, key, 0, size)\n\t\tif err != nil {\n\t\t\tif _, e := src.Head(ctx, key); os.IsNotExist(e) {\n\t\t\t\tlogger.Debugf(\"Head src %s: %s\", key, err)\n\t\t\t\terr = utils.ErrSkipped\n\t\t\t}\n\t\t\treturn 0, err\n\t\t}\n\t}\n\tr := &chksumReader{in, 0, calChksum}\n\tdefer in.Close()\n\terr = dst.Put(ctx, key, &withProgress{r})\n\treturn r.chksum, err\n}\n\ntype withProgress struct {\n\tr io.Reader\n}\n\nfunc (w *withProgress) Read(b []byte) (int, error) {\n\tif limiter != nil {\n\t\tlimiter.Wait(int64(len(b)))\n\t}\n\tn, err := w.r.Read(b)\n\tcopiedBytes.IncrInt64(int64(n))\n\treturn n, err\n}\n\nfunc dynAlloc(size int) []byte {\n\tzeros := utils.PowerOf2(size)\n\tb := *dynPools[zeros].Get().(*[]byte)\n\tif cap(b) < size {\n\t\tpanic(fmt.Sprintf(\"%d < %d\", cap(b), size))\n\t}\n\treturn b[:size]\n}\n\nfunc dynFree(b []byte) {\n\tdynPools[utils.PowerOf2(cap(b))].Put(&b)\n}\n\nvar dynPools []*sync.Pool\n\nfunc init() {\n\tdynPools = make([]*sync.Pool, 33) // 1 - 8G\n\tfor i := 0; i < 33; i++ {\n\t\tfunc(bits int) {\n\t\t\tdynPools[i] = &sync.Pool{\n\t\t\t\tNew: func() interface{} {\n\t\t\t\t\tb := make([]byte, 1< partSize*int64(upload.MaxCount) {\n\t\tpartSize = size / int64(upload.MaxCount)\n\t\tpartSize = ((partSize-1)>>20 + 1) << 20 // align to MB\n\t}\n\treturn partSize\n}\n\nfunc doCopyRange(src, dst object.ObjectStorage, key string, off, size int64, upload *object.MultipartUpload, num int, abort chan struct{}, calChksum bool) (*object.Part, uint32, error) {\n\tselect {\n\tcase <-abort:\n\t\treturn nil, 0, fmt.Errorf(\"aborted\")\n\tcase concurrent <- 1:\n\t\tdefer func() {\n\t\t\t<-concurrent\n\t\t}()\n\t}\n\n\tlimits := dst.Limits()\n\tif size <= 32<<20 || !limits.IsSupportUploadPartCopy {\n\t\treturn doUploadPart(src, dst, key, off, size, key, upload.UploadID, num, calChksum)\n\t}\n\n\ttmpkey := fmt.Sprintf(\"%s.part%d\", key, num)\n\tvar up *object.MultipartUpload\n\tvar err error\n\terr = try(3, func() error {\n\t\tup, err = dst.CreateMultipartUpload(ctx, tmpkey)\n\t\treturn err\n\t})\n\tif err != nil {\n\t\treturn nil, 0, fmt.Errorf(\"range(%d,%d): %s\", off, size, err)\n\t}\n\n\tpartSize := choosePartSize(up, size)\n\tn := int((size-1)/partSize) + 1\n\tlogger.Debugf(\"Copying data of %s (range: %d,%d) as %d parts (size: %d): %s\", key, off, size, n, partSize, up.UploadID)\n\tparts := make([]*object.Part, n)\n\tvar tmpChksum uint32\n\tfirst := true\n\n\tfor i := 0; i < n; i++ {\n\t\tsz := partSize\n\t\tif i == n-1 {\n\t\t\tsz = size - int64(i)*partSize\n\t\t}\n\t\tselect {\n\t\tcase <-abort:\n\t\t\tdst.AbortUpload(ctx, tmpkey, up.UploadID)\n\t\t\treturn nil, 0, fmt.Errorf(\"aborted\")\n\t\tdefault:\n\t\t}\n\t\tvar chksum uint32\n\t\tparts[i], chksum, err = doUploadPart(src, dst, key, off+int64(i)*partSize, sz, tmpkey, up.UploadID, i, calChksum)\n\t\tif err != nil {\n\t\t\tdst.AbortUpload(ctx, tmpkey, up.UploadID)\n\t\t\treturn nil, 0, fmt.Errorf(\"range(%d,%d): %s\", off, size, err)\n\t\t}\n\t\tif calChksum {\n\t\t\tif first {\n\t\t\t\ttmpChksum = chksum\n\t\t\t\tfirst = false\n\t\t\t} else {\n\t\t\t\ttmpChksum = crc32combine.CRC32Combine(crc32.Castagnoli, tmpChksum, chksum, sz)\n\t\t\t}\n\t\t}\n\t}\n\n\terr = try(3, func() error { return dst.CompleteUpload(ctx, tmpkey, up.UploadID, parts) })\n\tif err != nil {\n\t\tdst.AbortUpload(ctx, tmpkey, up.UploadID)\n\t\treturn nil, 0, fmt.Errorf(\"multipart: %s\", err)\n\t}\n\tvar part *object.Part\n\terr = try(3, func() error {\n\t\tpart, err = dst.UploadPartCopy(ctx, key, upload.UploadID, num+1, tmpkey, 0, size)\n\t\treturn err\n\t})\n\t_ = dst.Delete(ctx, tmpkey)\n\treturn part, tmpChksum, err\n}\n\nfunc doCopyMultiple(src, dst object.ObjectStorage, key string, size int64, upload *object.MultipartUpload, calChksum bool) (uint32, error) {\n\tlimits := dst.Limits()\n\tif size > limits.MaxPartSize*int64(upload.MaxCount) {\n\t\treturn 0, fmt.Errorf(\"object size %d is too large to copy\", size)\n\t}\n\n\tpartSize := choosePartSize(upload, size)\n\tn := int((size-1)/partSize) + 1\n\tlogger.Debugf(\"Copying data of %s as %d parts (size: %d): %s\", key, n, partSize, upload.UploadID)\n\tabort := make(chan struct{})\n\tparts := make([]*object.Part, n)\n\terrs := make(chan error, n)\n\tchksums := make([]chksumWithSz, n)\n\tvar err error\n\n\tfor i := 0; i < n; i++ {\n\t\tgo func(num int) {\n\t\t\tsz := partSize\n\t\t\tif num == n-1 {\n\t\t\t\tsz = size - int64(num)*partSize\n\t\t\t}\n\t\t\tvar copyErr error\n\t\t\tvar chksum uint32\n\t\t\tparts[num], chksum, copyErr = doCopyRange(src, dst, key, int64(num)*partSize, sz, upload, num, abort, calChksum)\n\t\t\tchksums[num] = chksumWithSz{chksum, sz}\n\t\t\terrs <- copyErr\n\t\t}(i)\n\t}\n\n\tfor i := 0; i < n; i++ {\n\t\tif err = <-errs; err != nil {\n\t\t\tclose(abort)\n\t\t\tbreak\n\t\t}\n\t}\n\tif err == nil {\n\t\terr = try(3, func() error { return dst.CompleteUpload(ctx, key, upload.UploadID, parts) })\n\t}\n\tif err != nil {\n\t\tdst.AbortUpload(ctx, key, upload.UploadID)\n\t\treturn 0, fmt.Errorf(\"multipart: %s\", err)\n\t}\n\tvar chksum uint32\n\tif calChksum {\n\t\tchksum = chksums[0].chksum\n\t\tfor i := 1; i < n; i++ {\n\t\t\tchksum = crc32combine.CRC32Combine(crc32.Castagnoli, chksum, chksums[i].chksum, chksums[i].size)\n\t\t}\n\t}\n\n\treturn chksum, nil\n}\n\nfunc InitForCopyData() {\n\tconcurrent = make(chan int, 10)\n\tprogress := utils.NewProgress(true)\n\tcopied = progress.AddCountSpinner(\"Copied objects\")\n\tcopiedBytes = progress.AddByteSpinner(\"Copied bytes\")\n}\n\nfunc CopyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error) {\n\tstart := time.Now()\n\tvar err error\n\tvar srcChksum uint32\n\tif size < maxBlock {\n\t\terr = try(3, func() (err error) {\n\t\t\tsrcChksum, err = doCopySingle(src, dst, key, size, calChksum)\n\t\t\treturn\n\t\t})\n\t} else {\n\t\tvar upload *object.MultipartUpload\n\t\tif upload, err = dst.CreateMultipartUpload(ctx, key); err == nil {\n\t\t\tsrcChksum, err = doCopyMultiple(src, dst, key, size, upload, calChksum)\n\t\t} else if err == utils.ENOTSUP {\n\t\t\terr = try(3, func() (err error) {\n\t\t\t\tsrcChksum, err = doCopySingle(src, dst, key, size, calChksum)\n\t\t\t\treturn\n\t\t\t})\n\t\t} else { // other error retry\n\t\t\tif err = try(2, func() error {\n\t\t\t\tupload, err = dst.CreateMultipartUpload(ctx, key)\n\t\t\t\treturn err\n\t\t\t}); err == nil {\n\t\t\t\tsrcChksum, err = doCopyMultiple(src, dst, key, size, upload, calChksum)\n\t\t\t}\n\t\t}\n\t}\n\tif err == nil {\n\t\tlogger.Debugf(\"Copied data of %s (%d bytes) in %s\", key, size, time.Since(start))\n\t} else {\n\t\tlogger.Errorf(\"Failed to copy data of %s in %s: %s\", key, time.Since(start), err)\n\t}\n\treturn srcChksum, err\n}\n\ntype holder struct {\n\tdone chan struct{}\n}\n\nvar muHolder sync.Mutex\nvar holders []*holder\n\nfunc fetchTask(tasks chan object.Object) (t object.Object, done func()) {\n\tmuHolder.Lock()\n\tdefer muHolder.Unlock()\n\tif len(holders) > 0 {\n\t\th := holders[len(holders)-1]\n\t\tholders = holders[:len(holders)-1]\n\t\tmuHolder.Unlock()\n\t\t<-h.done\n\t\tmuHolder.Lock()\n\t}\n\tif t = <-tasks; t == nil {\n\t\treturn nil, func() {}\n\t}\n\tsize := t.Size()\n\tif size == markChecksum {\n\t\tsize = withoutSize(t).Size()\n\t}\n\tif size >= maxBlock*2 {\n\t\tdone := make(chan struct{})\n\t\th := &holder{done: done}\n\t\tn := min(int(size)/maxBlock, 20)\n\t\tfor i := 1; i < n; i++ {\n\t\t\tholders = append(holders, h)\n\t\t}\n\t\treturn t, func() { close(done) }\n\t} else {\n\t\treturn t, func() {}\n\t}\n}\n\nfunc worker(tasks chan object.Object, src, dst object.ObjectStorage, config *Config) {\n\tfor {\n\t\tobj, done := fetchTask(tasks)\n\t\tif obj == nil {\n\t\t\tbreak\n\t\t}\n\t\tkey := obj.Key()\n\t\tswitch obj.Size() {\n\t\tcase markDeleteSrc:\n\t\t\tdeleteObj(src, key, config.Dry)\n\t\tcase markDeleteDst:\n\t\t\tdeleteObj(dst, key, config.Dry)\n\t\tcase markCopyPerms:\n\t\t\tif config.Dry {\n\t\t\t\tlogger.Debugf(\"Will copy permissions for %s\", key)\n\t\t\t} else {\n\t\t\t\tcopyPerms(dst, withoutSize(obj), config)\n\t\t\t}\n\t\t\tcopied.Increment()\n\t\tcase markChecksum:\n\t\t\tif config.Dry {\n\t\t\t\tlogger.Debugf(\"Will compare checksum for %s\", key)\n\t\t\t\tchecked.Increment()\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tobj = withoutSize(obj)\n\t\t\tif equal, err := checkSum(src, dst, key, nil, obj, config); err != nil {\n\t\t\t\tfailed.Increment()\n\t\t\t\tbreak\n\t\t\t} else if equal {\n\t\t\t\tif config.DeleteSrc {\n\t\t\t\t\tif obj.IsDir() {\n\t\t\t\t\t\tsrcDelayDelMu.Lock()\n\t\t\t\t\t\tsrcDelayDel = append(srcDelayDel, key)\n\t\t\t\t\t\tsrcDelayDelMu.Unlock()\n\t\t\t\t\t} else {\n\t\t\t\t\t\tdeleteObj(src, key, false)\n\t\t\t\t\t}\n\t\t\t\t} else if config.Perms && (!obj.IsSymlink() || !config.Links) {\n\t\t\t\t\tif o, e := dst.Head(ctx, key); e == nil {\n\t\t\t\t\t\tif needCopyPerms(obj, o) {\n\t\t\t\t\t\t\tcopyPerms(dst, obj, config)\n\t\t\t\t\t\t\tcopied.Increment()\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tskipped.Increment()\n\t\t\t\t\t\t\tskippedBytes.IncrInt64(obj.Size())\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlogger.Warnf(\"Failed to head object %s: %s\", key, e)\n\t\t\t\t\t\tfailed.Increment()\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tskipped.Increment()\n\t\t\t\t\tskippedBytes.IncrInt64(obj.Size())\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// checkSum not equal, copy the object\n\t\t\tfallthrough\n\t\tdefault:\n\t\t\tif config.Dry {\n\t\t\t\tlogger.Debugf(\"Will copy %s (%d bytes)\", obj.Key(), obj.Size())\n\t\t\t\tcopied.Increment()\n\t\t\t\tcopiedBytes.IncrInt64(obj.Size())\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tvar err error\n\t\t\tvar srcChksum uint32\n\n\t\t\tif config.Links && obj.IsSymlink() {\n\t\t\t\tif err = copyLink(src, dst, key); err != nil {\n\t\t\t\t\tlogger.Errorf(\"copy link failed: %s\", err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tsrcChksum, err = CopyData(src, dst, key, obj.Size(), config.CheckAll || config.CheckNew)\n\t\t\t}\n\t\t\tif errors.Is(err, utils.ErrExtlink) {\n\t\t\t\tlogger.Warnf(\"Skip external link %s: %s\", key, err)\n\t\t\t\terr = utils.ErrSkipped\n\t\t\t}\n\n\t\t\tif err == nil && config.CheckChange {\n\t\t\t\terr = checkChange(src, dst, obj, key, config)\n\t\t\t}\n\n\t\t\tif err == nil && (config.CheckAll || config.CheckNew) {\n\t\t\t\tvar equal bool\n\t\t\t\tif equal, err = checkSum(src, dst, key, &srcChksum, obj, config); err == nil && !equal {\n\t\t\t\t\terr = fmt.Errorf(\"checksums of copied object %s don't match\", key)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\tif mc, ok := dst.(object.MtimeChanger); ok {\n\t\t\t\t\tif err = mc.Chtimes(obj.Key(), obj.Mtime()); err != nil && !errors.Is(err, utils.ENOTSUP) {\n\t\t\t\t\t\tlogger.Warnf(\"Update mtime of %s: %s\", key, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif config.Perms {\n\t\t\t\t\tcopyPerms(dst, obj, config)\n\t\t\t\t}\n\t\t\t\tcopied.Increment()\n\t\t\t} else if errors.Is(err, utils.ErrSkipped) {\n\t\t\t\tskipped.Increment()\n\t\t\t} else {\n\t\t\t\tfailed.Increment()\n\t\t\t\tlogger.Errorf(\"Failed to copy object %s: %s\", key, err)\n\t\t\t}\n\t\t}\n\t\tincrHandled(1)\n\t\tdone()\n\t}\n}\n\nfunc checkChange(src, dst object.ObjectStorage, obj object.Object, key string, config *Config) error {\n\tif obj == nil || config.Links && obj.IsSymlink() {\n\t\treturn nil // ignore symlink\n\t}\n\tif cur, err := src.Head(ctx, key); err == nil {\n\t\tif !config.CheckAll && !config.CheckNew {\n\t\t\tchecked.Increment()\n\t\t\tcheckedBytes.IncrInt64(obj.Size())\n\t\t}\n\t\tequal := cur.Size() == obj.Size()\n\t\tif equal && !cur.Mtime().Equal(obj.Mtime()) {\n\t\t\t// Head of an object may not return the millisecond part of mtime as List\n\t\t\tequal = cur.Mtime().Unix() == obj.Mtime().Unix() && cur.Mtime().UnixMilli()%1000 == 0\n\t\t}\n\t\tif !equal {\n\t\t\treturn fmt.Errorf(\"%s changed during sync. Original: size=%d, mtime=%s; Current: size=%d, mtime=%s\",\n\t\t\t\tcur.Key(), obj.Size(), obj.Mtime(), cur.Size(), cur.Mtime())\n\t\t}\n\t\tif dstObj, err := dst.Head(ctx, key); err == nil {\n\t\t\tif cur.Size() != dstObj.Size() {\n\t\t\t\treturn fmt.Errorf(\"copied %s size mismatch: original=%d, current=%d\", key, obj.Size(), dstObj.Size())\n\t\t\t}\n\t\t\treturn nil\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"check %s in %s: %s\", key, dst, err)\n\t\t}\n\t} else if errors.Is(err, os.ErrNotExist) {\n\t\treturn fmt.Errorf(\"object %s was removed during sync\", key)\n\t} else {\n\t\treturn fmt.Errorf(\"check %s in %s: %s\", key, src, err)\n\t}\n}\n\nfunc copyLink(src object.ObjectStorage, dst object.ObjectStorage, key string) error {\n\tif p, err := src.(object.SupportSymlink).Readlink(key); err != nil {\n\t\treturn err\n\t} else {\n\t\tif err := dst.Delete(ctx, key); err != nil {\n\t\t\tlogger.Debugf(\"Deleted %s from %s \", key, dst)\n\t\t\treturn err\n\t\t}\n\t\t// TODO: use relative path based on option\n\t\treturn dst.(object.SupportSymlink).Symlink(p, key)\n\t}\n}\n\ntype objWithSize struct {\n\tobject.Object\n\tnsize int64\n}\n\nfunc (o *objWithSize) Size() int64 {\n\treturn o.nsize\n}\n\ntype fileWithSize struct {\n\tobject.File\n\tnsize int64\n}\n\nfunc (o *fileWithSize) Size() int64 {\n\treturn o.nsize\n}\n\nfunc withSize(o object.Object, nsize int64) object.Object {\n\tif f, ok := o.(object.File); ok {\n\t\treturn &fileWithSize{f, nsize}\n\t}\n\treturn &objWithSize{o, nsize}\n}\n\nfunc withoutSize(o object.Object) object.Object {\n\tswitch w := o.(type) {\n\tcase *objWithSize:\n\t\treturn w.Object\n\tcase *fileWithSize:\n\t\treturn w.File\n\t}\n\treturn o\n}\n\nvar dstDelayDelMu sync.Mutex\nvar dstDelayDel []string\nvar srcDelayDelMu sync.Mutex\nvar srcDelayDel []string\n\nfunc handleExtraObject(tasks chan<- object.Object, dstobj object.Object, config *Config) bool {\n\tincrTotal(1)\n\tif !config.DeleteDst || !config.Dirs && dstobj.IsDir() || config.Limit == 0 {\n\t\tlogger.Debug(\"Ignore extra object\", dstobj.Key())\n\t\textra.Increment()\n\t\textraBytes.IncrInt64(dstobj.Size())\n\t\treturn false\n\t}\n\tconfig.Limit--\n\tif dstobj.IsDir() {\n\t\tdstDelayDelMu.Lock()\n\t\tdstDelayDel = append(dstDelayDel, dstobj.Key())\n\t\tdstDelayDelMu.Unlock()\n\t} else {\n\t\ttasks <- withSize(dstobj, markDeleteDst)\n\t}\n\treturn config.Limit == 0\n}\n\nfunc startSingleProducer(tasks chan<- object.Object, src, dst object.ObjectStorage, prefix string, config *Config) error {\n\tstart, end := config.Start, config.End\n\tlogger.Debugf(\"maxResults: %d, defaultPartSize: %d, maxBlock: %d\", maxResults, defaultPartSize, maxBlock)\n\n\tsrckeys, err := ListAll(src, prefix, start, end, !config.Links)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"list %s: %s\", src, err)\n\t}\n\n\tvar dstkeys <-chan object.Object\n\tif config.ForceUpdate {\n\t\tt := make(chan object.Object)\n\t\tclose(t)\n\t\tdstkeys = t\n\t} else {\n\t\tdstkeys, err = ListAll(dst, prefix, start, end, !config.Links)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"list %s: %s\", dst, err)\n\t\t}\n\t}\n\treturn produce(tasks, srckeys, dstkeys, config)\n}\n\nfunc produce(tasks chan<- object.Object, srckeys, dstkeys <-chan object.Object, config *Config) error {\n\tsrckeys = filter(srckeys, config.rules, config)\n\tdstkeys = filter(dstkeys, config.rules, config)\n\tvar dstobj object.Object\n\tvar (\n\t\tskip, skipBytes int64\n\t\tlastUpdate time.Time\n\t)\n\tflushProgress := func() {\n\t\tskipped.IncrInt64(skip)\n\t\tskippedBytes.IncrInt64(skipBytes)\n\t\tincrHandled(int(skip))\n\t\tskip, skipBytes = 0, 0\n\t}\n\tdefer flushProgress()\n\tskipIt := func(obj object.Object) {\n\t\tskip++\n\t\tskipBytes += obj.Size()\n\t\tif skip > 100 || time.Since(lastUpdate) > time.Millisecond*100 {\n\t\t\tlastUpdate = time.Now()\n\t\t\tflushProgress()\n\t\t}\n\t}\n\tfor obj := range srckeys {\n\t\tif obj == nil {\n\t\t\treturn fmt.Errorf(\"listing failed, stop syncing, waiting for pending ones\")\n\t\t}\n\t\tif !config.Dirs && obj.IsDir() {\n\t\t\tlogger.Debug(\"Ignore directory \", obj.Key())\n\t\t\tcontinue\n\t\t}\n\t\tif config.Limit >= 0 {\n\t\t\tif config.Limit == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tconfig.Limit--\n\t\t}\n\t\tincrTotal(1)\n\n\t\tif dstobj != nil && obj.Key() > dstobj.Key() {\n\t\t\tif handleExtraObject(tasks, dstobj, config) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tdstobj = nil\n\t\t}\n\t\tif dstobj == nil {\n\t\t\tfor dstobj = range dstkeys {\n\t\t\t\tif dstobj == nil {\n\t\t\t\t\treturn fmt.Errorf(\"listing failed, stop syncing, waiting for pending ones\")\n\t\t\t\t}\n\t\t\t\tif obj.Key() <= dstobj.Key() {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif handleExtraObject(tasks, dstobj, config) {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tdstobj = nil\n\t\t\t}\n\t\t}\n\n\t\t// FIXME: there is a race when source is modified during coping\n\t\tif dstobj == nil || obj.Key() < dstobj.Key() {\n\t\t\tif config.Existing {\n\t\t\t\tskipIt(obj)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttasks <- obj\n\t\t} else { // obj.key == dstobj.key\n\t\t\tif config.IgnoreExisting {\n\t\t\t\tskipIt(obj)\n\t\t\t\tdstobj = nil\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif config.ForceUpdate ||\n\t\t\t\t(config.Update && obj.Mtime().Unix() > dstobj.Mtime().Unix()) ||\n\t\t\t\t(!config.Update && obj.Size() != dstobj.Size()) {\n\t\t\t\ttasks <- obj\n\t\t\t} else if config.Update && obj.Mtime().Unix() < dstobj.Mtime().Unix() {\n\t\t\t\tskipIt(obj)\n\t\t\t} else if config.CheckAll { // two objects are likely the same\n\t\t\t\ttasks <- withSize(obj, markChecksum)\n\t\t\t} else if config.DeleteSrc {\n\t\t\t\tif obj.IsDir() {\n\t\t\t\t\tsrcDelayDelMu.Lock()\n\t\t\t\t\tsrcDelayDel = append(srcDelayDel, obj.Key())\n\t\t\t\t\tsrcDelayDelMu.Unlock()\n\t\t\t\t} else {\n\t\t\t\t\ttasks <- withSize(obj, markDeleteSrc)\n\t\t\t\t}\n\t\t\t} else if config.Perms && needCopyPerms(obj, dstobj) {\n\t\t\t\ttasks <- withSize(obj, markCopyPerms)\n\t\t\t} else {\n\t\t\t\tskipIt(obj)\n\t\t\t}\n\t\t\tdstobj = nil\n\t\t}\n\t}\n\tif config.DeleteDst {\n\t\tif dstobj != nil {\n\t\t\tif handleExtraObject(tasks, dstobj, config) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfor dstobj = range dstkeys {\n\t\t\tif dstobj == nil {\n\t\t\t\treturn fmt.Errorf(\"listing failed, stop syncing, waiting for pending ones\")\n\t\t\t}\n\t\t\tif handleExtraObject(tasks, dstobj, config) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\ntype rule struct {\n\tpattern string\n\tinclude bool\n}\n\nfunc parseRule(name, p string) rule {\n\tif runtime.GOOS == \"windows\" {\n\t\tp = strings.Replace(p, \"\\\\\", \"/\", -1)\n\t}\n\treturn rule{pattern: p, include: name == \"-include\"}\n}\n\nfunc parseIncludeRules(args []string) (rules []rule) {\n\tl := len(args)\n\tfor i, a := range args {\n\t\tif strings.HasPrefix(a, \"--\") {\n\t\t\ta = a[1:]\n\t\t}\n\t\tif l-1 > i && (a == \"-include\" || a == \"-exclude\") {\n\t\t\tif _, err := path.Match(args[i+1], \"xxxx\"); err != nil {\n\t\t\t\tlogger.Warnf(\"ignore invalid pattern: %s %s\", a, args[i+1])\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trules = append(rules, parseRule(a, args[i+1]))\n\t\t} else if strings.HasPrefix(a, \"-include=\") || strings.HasPrefix(a, \"-exclude=\") {\n\t\t\tif s := strings.Split(a, \"=\"); len(s) == 2 && s[1] != \"\" {\n\t\t\t\tif _, err := path.Match(s[1], \"xxxx\"); err != nil {\n\t\t\t\t\tlogger.Warnf(\"ignore invalid pattern: %s\", a)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\trules = append(rules, parseRule(s[0], s[1]))\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\nfunc filterKey(o object.Object, now time.Time, rules []rule, config *Config) bool {\n\tvar ok bool = true\n\tif !o.IsDir() && !o.IsSymlink() {\n\t\tok = o.Size() >= int64(config.MinSize) && o.Size() <= int64(config.MaxSize)\n\t\tif ok && config.MaxAge > 0 {\n\t\t\tok = o.Mtime().After(now.Add(-config.MaxAge))\n\t\t}\n\t\tif ok && config.MinAge > 0 {\n\t\t\tok = o.Mtime().Before(now.Add(-config.MinAge))\n\t\t}\n\t\tif ok && !config.StartTime.IsZero() {\n\t\t\tok = o.Mtime().After(config.StartTime)\n\t\t}\n\t\tif ok && !config.EndTime.IsZero() {\n\t\t\tok = o.Mtime().Before(config.EndTime)\n\t\t}\n\t}\n\tif ok {\n\t\tif config.MatchFullPath {\n\t\t\tok = matchFullPath(rules, o.Key())\n\t\t} else {\n\t\t\tok = matchLeveledPath(rules, o.Key())\n\t\t}\n\t}\n\treturn ok\n}\n\nfunc filter(keys <-chan object.Object, rules []rule, config *Config) <-chan object.Object {\n\tr := make(chan object.Object)\n\tnow := time.Now()\n\tgo func() {\n\t\tfor o := range keys {\n\t\t\tif o == nil {\n\t\t\t\t// Telling that the listing has failed\n\t\t\t\tr <- nil\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif filterKey(o, now, rules, config) {\n\t\t\t\tr <- o\n\t\t\t} else {\n\t\t\t\tlogger.Debugf(\"exclude %s size: %d, mtime: %s\", o.Key(), o.Size(), o.Mtime())\n\t\t\t\texcluded.Increment()\n\t\t\t\texcludedBytes.IncrInt64(o.Size())\n\t\t\t}\n\t\t}\n\t\tclose(r)\n\t}()\n\treturn r\n}\n\nfunc matchTwoStar(p string, s []string) bool {\n\tif len(s) == 0 {\n\t\treturn p == \"*\"\n\t}\n\tidx := strings.Index(p, \"**\")\n\tif idx == -1 {\n\t\tok, _ := path.Match(p, strings.Join(s, \"/\"))\n\t\treturn ok\n\t}\n\tok, _ := path.Match(p[:idx+1], s[0])\n\tif !ok {\n\t\treturn false\n\t}\n\tfor i := 0; i <= len(s); i++ {\n\t\ttp := p[idx+1:]\n\t\tif i == 0 {\n\t\t\ttp = p[:idx] + p[idx+1:]\n\t\t}\n\t\tif matchTwoStar(tp, s[i:]) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc matchPrefix(p, s []string) bool {\n\tif len(p) == 0 || len(s) == 0 {\n\t\treturn len(p) == len(s)\n\t}\n\tfirst := p[0]\n\tn := len(s)\n\tswitch {\n\tcase first == \"***\":\n\t\treturn true\n\tcase strings.Contains(first, \"**\"):\n\t\tfor i := 1; i <= n; i++ {\n\t\t\tif matchTwoStar(first, s[:i]) && matchPrefix(p[1:], s[i:]) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\tdefault:\n\t\tok, _ := path.Match(first, s[0])\n\t\treturn ok && matchPrefix(p[1:], s[1:])\n\t}\n}\n\nfunc matchSuffix(p, s []string) bool {\n\tif len(p) == 0 {\n\t\treturn true\n\t}\n\tlast := p[len(p)-1]\n\tif len(s) == 0 {\n\t\treturn len(p) == 1 && (last == \"***\" || last == \"**\")\n\t}\n\tprefix := p[:len(p)-1]\n\tn := len(s)\n\tswitch {\n\tcase last == \"***\":\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif matchSuffix(prefix, s[:i]) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\tcase strings.Contains(last, \"**\"):\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif matchTwoStar(last, s[i:]) && matchSuffix(prefix, s[:i]) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\tdefault:\n\t\tok, _ := path.Match(last, s[n-1])\n\t\treturn ok && matchSuffix(prefix, s[:n-1])\n\t}\n}\n\nfunc matchFullPath(rules []rule, key string) bool {\n\tps := strings.Split(key, \"/\")\n\tfor _, rule := range rules {\n\t\tp := strings.Split(rule.pattern, \"/\")\n\t\tvar ok bool\n\t\tif p[0] == \"\" {\n\t\t\tif ps[0] != \"\" {\n\t\t\t\tp = p[1:]\n\t\t\t}\n\t\t\tok = matchPrefix(p, ps)\n\t\t} else {\n\t\t\tok = matchSuffix(p, ps)\n\t\t}\n\t\tif ok {\n\t\t\tif rule.include {\n\t\t\t\tbreak // try next level\n\t\t\t} else {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\treturn true\n}\n\n// Consistent with rsync behavior, the matching order is adjusted according to the order of the \"include\" and \"exclude\" options\nfunc matchLeveledPath(rules []rule, key string) bool {\n\tparts := strings.Split(key, \"/\")\n\tfor i := range parts {\n\t\tif parts[i] == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, rule := range rules {\n\t\t\tps := parts[:i+1]\n\t\t\tp := strings.Split(rule.pattern, \"/\")\n\t\t\tif i < len(parts)-1 && (p[len(p)-1] == \"\" || p[len(p)-1] == \"***\") {\n\t\t\t\tps = append(append([]string{}, ps...), \"\") // don't overwrite parts\n\t\t\t}\n\t\t\tvar ok bool\n\t\t\tif p[0] == \"\" {\n\t\t\t\tif ps[0] != \"\" {\n\t\t\t\t\tp = p[1:]\n\t\t\t\t}\n\t\t\t\tok = matchPrefix(p, ps)\n\t\t\t} else {\n\t\t\t\tok = matchSuffix(p, ps)\n\t\t\t}\n\t\t\tif ok {\n\t\t\t\tif rule.include {\n\t\t\t\t\tbreak // try next level\n\t\t\t\t} else {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn true\n}\n\nfunc listCommonPrefix(store object.ObjectStorage, prefix string, cp chan object.Object, followLink bool) (chan object.Object, error) {\n\tvar total []object.Object\n\tvar objs []object.Object\n\tvar err error\n\tvar nextToken string\n\tvar marker string\n\tvar hasMore bool\n\tvar thisListMaxResults int64 = maxResults\n\tif strings.HasPrefix(store.String(), \"file://\") || strings.HasPrefix(store.String(), \"nfs://\") ||\n\t\tstrings.HasPrefix(store.String(), \"gluster://\") || strings.HasPrefix(store.String(), \"jfs://\") ||\n\t\tstrings.HasPrefix(store.String(), \"hdfs://\") || strings.HasPrefix(store.String(), \"webdav://\") {\n\t\tthisListMaxResults = math.MaxInt64\n\t}\n\tfor {\n\t\tobjs, hasMore, nextToken, err = store.List(ctx, prefix, marker, nextToken, \"/\", thisListMaxResults, followLink)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(objs) > 0 {\n\t\t\ttotal = append(total, objs...)\n\t\t\tmarker = objs[len(objs)-1].Key()\n\t\t}\n\t\tif !hasMore {\n\t\t\tbreak\n\t\t}\n\t}\n\tsrckeys := make(chan object.Object, 1000)\n\tgo func() {\n\t\tdefer close(srckeys)\n\t\tfor _, o := range total {\n\t\t\tif o.IsDir() && o.Key() > prefix {\n\t\t\t\tif cp != nil {\n\t\t\t\t\tcp <- o\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tsrckeys <- o\n\t\t\t}\n\t\t}\n\t}()\n\treturn srckeys, nil\n}\n\nfunc produceFromList(tasks chan<- object.Object, src, dst object.ObjectStorage, config *Config) error {\n\tf, err := os.Open(config.FilesFrom)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"open %s: %s\", config.FilesFrom, err)\n\t}\n\tdefer f.Close()\n\n\tprefixs := make(chan string, config.Threads)\n\tvar wg sync.WaitGroup\n\twg.Add(config.Threads)\n\tfor i := 0; i < config.Threads; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor key := range prefixs {\n\t\t\t\tif !strings.HasSuffix(key, \"/\") {\n\t\t\t\t\tif err := produceSingleObject(tasks, src, dst, key, config); err == nil {\n\t\t\t\t\t\tlistedPrefix.Increment()\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t} else if errors.Is(err, errDirSuffix) {\n\t\t\t\t\t\tkey += \"/\"\n\t\t\t\t\t} else if os.IsNotExist(err) {\n\t\t\t\t\t\tatomic.AddInt64(&ignoreFiles, 1)\n\t\t\t\t\t\tlistedPrefix.Increment()\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tlogger.Debugf(\"start listing prefix %s\", key)\n\t\t\t\terr = startProducer(tasks, src, dst, key, config.ListDepth, config)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"list prefix %s: %s\", key, err)\n\t\t\t\t\tfailed.Increment()\n\t\t\t\t}\n\t\t\t\tlistedPrefix.Increment()\n\t\t\t}\n\t\t}()\n\t}\n\n\tscanner := bufio.NewScanner(f)\n\tfor scanner.Scan() {\n\t\tkey := scanner.Text()\n\t\tif key == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\ttrimKey := strings.TrimRightFunc(key, unicode.IsSpace)\n\t\tif trimKey != key {\n\t\t\tlogger.Infof(\"found a prefix with a space character:%q\", key)\n\t\t}\n\t\tprefixs <- trimKey\n\t}\n\tclose(prefixs)\n\n\twg.Wait()\n\tlistedPrefix.Done()\n\treturn nil\n}\n\nvar errDirSuffix = errors.New(\"dir miss suffix '/'\")\nvar ignoreFiles int64\n\nfunc produceSingleObject(tasks chan<- object.Object, src, dst object.ObjectStorage, key string, config *Config) error {\n\tobj, err := src.Head(ctx, key)\n\tif err != nil {\n\t\tlogger.Warnf(\"head %s from %s: %s\", key, src, err)\n\t\treturn err\n\t}\n\tif obj.IsDir() {\n\t\t// only `files-from` will hit this case\n\t\tif !strings.HasSuffix(key, \"/\") {\n\t\t\treturn errDirSuffix\n\t\t}\n\t\tif !config.Dirs {\n\t\t\treturn nil\n\t\t}\n\t}\n\tvar srckeys = make(chan object.Object, 1)\n\tsrckeys <- obj\n\tclose(srckeys)\n\tif dobj, e := dst.Head(ctx, key); e == nil || os.IsNotExist(e) {\n\t\tvar dstkeys = make(chan object.Object, 1)\n\t\tif dobj != nil {\n\t\t\tdstkeys <- dobj\n\t\t}\n\t\tclose(dstkeys)\n\t\tlogger.Debugf(\"produce single key %s\", key)\n\t\t_ = produce(tasks, srckeys, dstkeys, config)\n\t\treturn nil\n\t} else {\n\t\tlogger.Warnf(\"head %s from %s: %s\", key, dst, e)\n\t\terr = e\n\t}\n\treturn err\n}\n\nfunc startProducer(tasks chan<- object.Object, src, dst object.ObjectStorage, prefix string, listDepth int, config *Config) error {\n\tconfig.concurrentList <- 1\n\tdefer func() {\n\t\t<-config.concurrentList\n\t}()\n\tif config.Limit == 1 && len(config.rules) == 0 {\n\t\tif produceSingleObject(tasks, src, dst, prefix, config) == nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\tif config.ListThreads <= 1 || listDepth <= 0 {\n\t\treturn startSingleProducer(tasks, src, dst, prefix, config)\n\t}\n\n\tcommonPrefix := make(chan object.Object, 1000)\n\tdone := make(chan bool)\n\tgo func() {\n\t\tdefer close(done)\n\t\tvar mu sync.Mutex\n\t\tprocessing := make(map[string]bool)\n\t\tvar wg sync.WaitGroup\n\t\tdefer wg.Wait()\n\t\tfor c := range commonPrefix {\n\t\t\tmu.Lock()\n\t\t\tif processing[c.Key()] {\n\t\t\t\tmu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tprocessing[c.Key()] = true\n\t\t\tmu.Unlock()\n\n\t\t\tif len(config.rules) > 0 && !matchLeveledPath(config.rules, c.Key()) {\n\t\t\t\tlogger.Infof(\"exclude prefix %s\", c.Key())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif c.Key() < config.Start {\n\t\t\t\tlogger.Infof(\"ignore prefix %s\", c.Key())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif config.End != \"\" && c.Key() > config.End {\n\t\t\t\tlogger.Infof(\"ignore prefix %s\", c.Key())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\twg.Add(1)\n\t\t\tgo func(prefix string) {\n\t\t\t\tdefer wg.Done()\n\t\t\t\terr := startProducer(tasks, src, dst, prefix, listDepth-1, config)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Errorf(\"list prefix %s: %s\", prefix, err)\n\t\t\t\t\tfailed.Increment()\n\t\t\t\t}\n\t\t\t}(c.Key())\n\t\t}\n\t}()\n\n\tsrckeys, err := listCommonPrefix(src, prefix, commonPrefix, !config.Links)\n\tif err == utils.ENOTSUP {\n\t\treturn startSingleProducer(tasks, src, dst, prefix, config)\n\t} else if err != nil {\n\t\treturn fmt.Errorf(\"list %s with delimiter: %s\", src, err)\n\t}\n\tvar dcp chan object.Object\n\tif config.DeleteDst {\n\t\tdcp = commonPrefix // search common prefix in dst\n\t}\n\tvar dstkeys <-chan object.Object\n\tif config.ForceUpdate {\n\t\tt := make(chan object.Object)\n\t\tclose(t)\n\t\tdstkeys = t\n\t} else {\n\t\tdstkeys, err = listCommonPrefix(dst, prefix, dcp, !config.Links)\n\t\tif err == utils.ENOTSUP {\n\t\t\treturn startSingleProducer(tasks, src, dst, prefix, config)\n\t\t} else if err != nil {\n\t\t\treturn fmt.Errorf(\"list %s with delimiter: %s\", dst, err)\n\t\t}\n\t}\n\t// sync returned objects\n\tif err := produce(tasks, srckeys, dstkeys, config); err != nil {\n\t\treturn err\n\t}\n\t// consume all the keys from dst\n\tfor range dstkeys {\n\t}\n\tclose(commonPrefix)\n\n\t<-config.concurrentList\n\t<-done\n\tconfig.concurrentList <- 1\n\treturn nil\n}\n\n// Sync syncs all the keys between to object storage\nfunc Sync(src, dst object.ObjectStorage, config *Config) error {\n\tif strings.HasPrefix(src.String(), \"file://\") && strings.HasPrefix(dst.String(), \"file://\") {\n\t\tmajor, minor := utils.GetKernelVersion()\n\t\t// copy_file_range() system call first appeared in Linux 4.5, and reworked in 5.3\n\t\t// Go requires kernel >= 5.3 to use copy_file_range(), see:\n\t\t// https://github.com/golang/go/blob/go1.17.11/src/internal/poll/copy_file_range_linux.go#L58-L66\n\t\tif major > 5 || (major == 5 && minor >= 3) {\n\t\t\td1 := utils.GetDev(src.String()[7:]) // remove prefix \"file://\"\n\t\t\td2 := utils.GetDev(dst.String()[7:])\n\t\t\tif d1 != -1 && d1 == d2 {\n\t\t\t\tobject.TryCFR = true\n\t\t\t}\n\t\t}\n\t}\n\n\tif config.Inplace {\n\t\tobject.PutInplace = true\n\t}\n\n\tvar bufferSize = 10240\n\tif config.Manager != \"\" {\n\t\t// No support for work-stealing, so workers shouldnot buffer tasks to prevent piling up in their own queues, which could cause imbalance among workers.\n\t\tbufferSize = 1\n\t}\n\ttasks := make(chan object.Object, bufferSize)\n\twg := sync.WaitGroup{}\n\tconcurrent = make(chan int, config.Threads)\n\tvar localLimit *ratelimit.Bucket\n\tif config.BWLimit > 0 {\n\t\tbps := float64(config.BWLimit*1e6/8) * 0.85 // 15% overhead\n\t\tlocalLimit = ratelimit.NewBucketWithRate(bps, int64(bps)/10)\n\t}\n\tvar gLimit *globalLimit\n\tif config.TrafficControlURL != \"\" {\n\t\tgLimit = &globalLimit{address: config.TrafficControlURL}\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t\t\tgLimit.checkBalance()\n\t\t\t}\n\t\t}()\n\t}\n\tif localLimit != nil || gLimit != nil {\n\t\tlimiter = &mixedLimiter{\n\t\t\tglobal: gLimit,\n\t\t\tlocal: localLimit,\n\t\t}\n\t}\n\n\tprogress := utils.NewProgress(config.Verbose || config.Quiet || config.Manager != \"\")\n\thandled = progress.AddCountBar(\"Scanned objects\", 0)\n\texcluded = progress.AddCountSpinner(\"Excluded objects\")\n\texcludedBytes = progress.AddByteSpinner(\"Excluded bytes\")\n\tskipped = progress.AddCountSpinner(\"Skipped objects\")\n\tskippedBytes = progress.AddByteSpinner(\"Skipped bytes\")\n\textra = progress.AddCountSpinner(\"Extra objects\")\n\textraBytes = progress.AddByteSpinner(\"Extra bytes\")\n\tpending = progress.AddCountSpinner(\"Pending objects\")\n\tcopied = progress.AddCountSpinner(\"Copied objects\")\n\tcopiedBytes = progress.AddByteSpinner(\"Copied bytes\")\n\tif config.CheckAll || config.CheckNew || config.CheckChange {\n\t\tchecked = progress.AddCountSpinner(\"Checked objects\")\n\t\tcheckedBytes = progress.AddByteSpinner(\"Checked bytes\")\n\t}\n\tif config.DeleteSrc || config.DeleteDst {\n\t\tdeleted = progress.AddCountSpinner(\"Deleted objects\")\n\t}\n\n\tsyncExitFunc := func() error {\n\t\tif config.Manager == \"\" {\n\t\t\tval := atomic.LoadInt64(&ignoreFiles)\n\t\t\tif val > 0 {\n\t\t\t\tlogger.Infof(\"Ignored %d non-existent paths from the file list\", val)\n\t\t\t}\n\t\t\tpending.SetCurrent(0)\n\t\t\tincrHandled(0)\n\t\t\ttotal := handled.GetTotal()\n\t\t\tprogress.Done()\n\n\t\t\tmsg := fmt.Sprintf(\"Found: %d, excluded: %d (%s), skipped: %d (%s), copied: %d (%s), extra: %d (%s)\", total,\n\t\t\t\texcluded.Current(), formatSize(excludedBytes.Current()),\n\t\t\t\tskipped.Current(), formatSize(skippedBytes.Current()),\n\t\t\t\tcopied.Current(), formatSize(copiedBytes.Current()),\n\t\t\t\textra.Current(), formatSize(extraBytes.Current()))\n\t\t\tif checked != nil {\n\t\t\t\tmsg += fmt.Sprintf(\", checked: %d (%s)\", checked.Current(), formatSize(checkedBytes.Current()))\n\t\t\t}\n\t\t\tif deleted != nil {\n\t\t\t\tmsg += fmt.Sprintf(\", deleted: %d\", deleted.Current())\n\t\t\t}\n\t\t\tif failed != nil {\n\t\t\t\tmsg += fmt.Sprintf(\", failed: %d\", failed.Current())\n\t\t\t}\n\t\t\tif total-handled.Current()-extra.Current() > 0 {\n\t\t\t\tmsg += fmt.Sprintf(\", lost: %d\", total-handled.Current())\n\t\t\t}\n\t\t\tlogger.Info(msg)\n\n\t\t\tif failed != nil {\n\t\t\t\tif n := failed.Current(); n > 0 || total > handled.Current()+extra.Current() {\n\t\t\t\t\treturn fmt.Errorf(\"failed to handle %d objects\", n+total-handled.Current()-extra.Current())\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tsendStats(config.Manager)\n\t\t\tfor len(srcDelayDel) > 0 {\n\t\t\t\tsendStats(config.Manager)\n\t\t\t}\n\t\t\tlogger.Infof(\"This worker process has already completed its tasks\")\n\t\t}\n\t\treturn nil\n\t}\n\n\tif !config.Dry {\n\t\tfailed = progress.AddCountSpinner(\"Failed objects\")\n\t\tif config.MaxFailure > 0 {\n\t\t\tgo func() {\n\t\t\t\tfor {\n\t\t\t\t\tif failed.Current() >= config.MaxFailure {\n\t\t\t\t\t\tlogger.Infof(\"the maximum error limit of %d was reached, stop now\", config.MaxFailure)\n\t\t\t\t\t\t_ = syncExitFunc()\n\t\t\t\t\t\tos.Exit(1)\n\t\t\t\t\t}\n\t\t\t\t\ttime.Sleep(time.Millisecond * 100)\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t}\n\n\tif config.Manager == \"\" && config.FilesFrom != \"\" {\n\t\tlistedPrefix = progress.AddCountSpinner(\"Prefix\")\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\tpending.SetCurrent(int64(len(tasks)))\n\t\t\ttime.Sleep(time.Millisecond * 100)\n\t\t}\n\t}()\n\n\tinitSyncMetrics(config)\n\tfor i := 0; i < config.Threads; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tworker(tasks, src, dst, config)\n\t\t}()\n\t}\n\n\tif len(config.Exclude) > 0 {\n\t\tconfig.rules = parseIncludeRules(os.Args)\n\t}\n\n\tif config.Manager == \"\" {\n\t\tif len(config.Workers) > 0 {\n\t\t\taddr, err := startManager(config, tasks)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tlaunchWorker(addr, config, &wg)\n\t\t}\n\t\tlogger.Infof(\"Syncing from %s to %s\", src, dst)\n\t\tif config.Start != \"\" {\n\t\t\tlogger.Infof(\"first key: %q\", config.Start)\n\t\t}\n\t\tif config.End != \"\" {\n\t\t\tlogger.Infof(\"last key: %q\", config.End)\n\t\t}\n\t\tconfig.concurrentList = make(chan int, config.ListThreads)\n\t\tvar err error\n\t\tif config.FilesFrom != \"\" {\n\t\t\terr = produceFromList(tasks, src, dst, config)\n\t\t} else {\n\t\t\terr = startProducer(tasks, src, dst, \"\", config.ListDepth, config)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tclose(tasks)\n\t} else {\n\t\tgo fetchJobs(tasks, config)\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tsendStats(config.Manager)\n\t\t\t\ttime.Sleep(time.Second)\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\n\tif config.Manager == \"\" {\n\t\tdelayDelFunc := func(storage object.ObjectStorage, keys []string) {\n\t\t\tif len(keys) > 0 {\n\t\t\t\tlogger.Infof(\"delete %d dirs from %s\", len(keys), storage)\n\t\t\t\tsort.Strings(keys)\n\t\t\t}\n\t\t\tfor i := len(keys) - 1; i >= 0; i-- {\n\t\t\t\tincrHandled(1)\n\t\t\t\tdeleteObj(storage, keys[i], config.Dry)\n\t\t\t}\n\t\t}\n\t\tdelWg := sync.WaitGroup{}\n\n\t\tdelWg.Add(1)\n\t\tgo func() {\n\t\t\tdelayDelFunc(src, srcDelayDel)\n\t\t\tdelWg.Done()\n\t\t}()\n\t\tdelWg.Add(1)\n\t\tgo func() {\n\t\t\tdelayDelFunc(dst, dstDelayDel)\n\t\t\tdelWg.Done()\n\t\t}()\n\t\tdelWg.Wait()\n\t}\n\treturn syncExitFunc()\n}\n\nfunc initSyncMetrics(config *Config) {\n\tif config.Registerer != nil {\n\t\tconfig.Registerer.MustRegister(\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"scanned\",\n\t\t\t\tHelp: \"Scanned objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(handled.Total())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"excluded\",\n\t\t\t\tHelp: \"Excluded objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(excluded.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"excluded_bytes\",\n\t\t\t\tHelp: \"Excluded bytes\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(copied.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"extra\",\n\t\t\t\tHelp: \"Extra objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(excluded.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"extra_bytes\",\n\t\t\t\tHelp: \"Extra bytes\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(copied.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"handled\",\n\t\t\t\tHelp: \"Handled objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(handled.Current())\n\t\t\t}),\n\t\t\tprometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\t\t\tName: \"pending\",\n\t\t\t\tHelp: \"Pending objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(pending.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"copied\",\n\t\t\t\tHelp: \"Copied objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(copied.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"copied_bytes\",\n\t\t\t\tHelp: \"Copied bytes\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(copiedBytes.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"skipped\",\n\t\t\t\tHelp: \"Skipped objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(skipped.Current())\n\t\t\t}),\n\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"skipped_bytes\",\n\t\t\t\tHelp: \"Skipped bytes\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(skippedBytes.Current())\n\t\t\t}),\n\t\t)\n\t\tif failed != nil {\n\t\t\tconfig.Registerer.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"failed\",\n\t\t\t\tHelp: \"Failed objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(failed.Current())\n\t\t\t}))\n\t\t}\n\t\tif deleted != nil {\n\t\t\tconfig.Registerer.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"deleted\",\n\t\t\t\tHelp: \"Deleted objects\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(deleted.Current())\n\t\t\t}))\n\t\t}\n\t\tif checked != nil && checkedBytes != nil {\n\t\t\tconfig.Registerer.MustRegister(\n\t\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\t\tName: \"checked\",\n\t\t\t\t\tHelp: \"Checked objects\",\n\t\t\t\t}, func() float64 {\n\t\t\t\t\treturn float64(checked.Current())\n\t\t\t\t}),\n\t\t\t\tprometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\t\tName: \"checked_bytes\",\n\t\t\t\t\tHelp: \"Checked bytes\",\n\t\t\t\t}, func() float64 {\n\t\t\t\t\treturn float64(checkedBytes.Current())\n\t\t\t\t}))\n\t\t}\n\t\tif listedPrefix != nil {\n\t\t\tconfig.Registerer.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{\n\t\t\t\tName: \"Prefix\",\n\t\t\t\tHelp: \"listed prefix\",\n\t\t\t}, func() float64 {\n\t\t\t\treturn float64(listedPrefix.Current())\n\t\t\t}))\n\t\t}\n\t}\n}\n" }, { "path": "pkg/sync/sync_test.go", "content": "/*\n * JuiceFS, Copyright 2018 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage sync\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n)\n\nfunc collectAll(c <-chan object.Object) []string {\n\tr := make([]string, 0)\n\tfor s := range c {\n\t\tr = append(r, s.Key())\n\t}\n\treturn r\n}\n\n// nolint:errcheck\nfunc TestIterator(t *testing.T) {\n\tm, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tm.Put(ctx, \"a\", bytes.NewReader([]byte(\"a\")))\n\tm.Put(ctx, \"b\", bytes.NewReader([]byte(\"a\")))\n\tm.Put(ctx, \"aa\", bytes.NewReader([]byte(\"a\")))\n\tm.Put(ctx, \"c\", bytes.NewReader([]byte(\"a\")))\n\n\tch, _ := ListAll(m, \"\", \"a\", \"b\", true)\n\tkeys := collectAll(ch)\n\tif len(keys) != 3 {\n\t\tt.Fatalf(\"length should be 3, but got %d\", len(keys))\n\t}\n\tif !reflect.DeepEqual(keys, []string{\"a\", \"aa\", \"b\"}) {\n\t\tt.Fatalf(\"result wrong: %s\", keys)\n\t}\n\n\t// Single object\n\ts, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\ts.Put(ctx, \"a\", bytes.NewReader([]byte(\"a\")))\n\tch, _ = ListAll(s, \"\", \"\", \"\", true)\n\tkeys = collectAll(ch)\n\tif !reflect.DeepEqual(keys, []string{\"a\"}) {\n\t\tt.Fatalf(\"result wrong: %s\", keys)\n\t}\n}\n\nfunc TestIeratorSingleEmptyKey(t *testing.T) {\n\t// utils.SetLogLevel(logrus.DebugLevel)\n\n\t// Construct mem storage\n\ts, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\terr := s.Put(ctx, \"abc\", bytes.NewReader([]byte(\"abc\")))\n\tif err != nil {\n\t\tt.Fatalf(\"Put error: %q\", err)\n\t}\n\n\t// Simulate command line prefix in SRC or DST\n\ts = object.WithPrefix(s, \"abc\")\n\tch, _ := ListAll(s, \"\", \"\", \"\", true)\n\tkeys := collectAll(ch)\n\tif !reflect.DeepEqual(keys, []string{\"\"}) {\n\t\tt.Fatalf(\"result wrong: %s\", keys)\n\t}\n}\n\nfunc deepEqualWithOutMtime(a, b object.Object) bool {\n\treturn a.IsDir() == b.IsDir() && a.Key() == b.Key() && a.Size() == b.Size() &&\n\t\tmath.Abs(a.Mtime().Sub(b.Mtime()).Seconds()) < 1\n}\n\n// nolint:errcheck\nfunc TestSync(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a\")\n\t\t_ = os.RemoveAll(\"/tmp/b\")\n\t}()\n\tconfig := &Config{\n\t\tStart: \"\",\n\t\tEnd: \"\",\n\t\tThreads: 50,\n\t\tListThreads: 1,\n\t\tUpdate: true,\n\t\tPerms: true,\n\t\tDry: false,\n\t\tDeleteSrc: false,\n\t\tLimit: -1,\n\t\tDeleteDst: false,\n\t\tExclude: []string{\"c*\"},\n\t\tInclude: []string{\"a[1-9]\", \"a*\"},\n\t\tMaxSize: math.MaxInt64,\n\t\tVerbose: false,\n\t\tQuiet: true,\n\t}\n\tos.Args = []string{\"--include\", \"a[1-9]\", \"--exclude\", \"a*\", \"--exclude\", \"c*\"}\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a/\", \"\", \"\", \"\")\n\ta.Put(ctx, \"a1\", bytes.NewReader([]byte(\"a1\")))\n\ta.Put(ctx, \"a2\", bytes.NewReader([]byte(\"a2\")))\n\ta.Put(ctx, \"abc\", bytes.NewReader([]byte(\"abc\")))\n\ta.Put(ctx, \"c1\", bytes.NewReader([]byte(\"c1\")))\n\ta.Put(ctx, \"c2\", bytes.NewReader([]byte(\"c2\")))\n\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b/\", \"\", \"\", \"\")\n\tb.Put(ctx, \"a1\", bytes.NewReader([]byte(\"a1\")))\n\tb.Put(ctx, \"ba\", bytes.NewReader([]byte(\"a1\")))\n\n\t// Copy a2\n\tif err := Sync(a, b, config); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\tif c := copied.Current(); c != 1 {\n\t\tt.Fatalf(\"should copy 1 keys, but got %d\", c)\n\t}\n\n\tif err := Sync(a, b, config); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\t// No copy occurred\n\tif c := copied.Current(); c != 0 {\n\t\tt.Fatalf(\"should copy 0 keys, but got %d\", c)\n\t}\n\n\t// Now a: {\"a1\", \"a2\", \"abc\", \"c1\", \"c2\"}, b: {\"a1\", \"a2\", \"ba\"}\n\t// Copy \"ba\" from b to a\n\tos.Args = []string{}\n\tconfig.Exclude = nil\n\tconfig.rules = nil\n\tif err := Sync(b, a, config); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\tif c := copied.Current(); c != 1 {\n\t\tt.Fatalf(\"should copy 1 keys, but got %d\", c)\n\t}\n\t// Now a: {\"a1\", \"a2\", \"abc\", \"ba\", \"c1\", \"c2\"}, b: {\"a1\", \"a2\", \"ba\"}\n\taRes, _ := ListAll(a, \"\", \"\", \"\", true)\n\tbRes, _ := ListAll(b, \"\", \"\", \"\", true)\n\n\tvar aObjs, bObjs []object.Object\n\tfor obj := range aRes {\n\t\taObjs = append(aObjs, obj)\n\t}\n\tfor obj := range bRes {\n\t\tbObjs = append(bObjs, obj)\n\t}\n\n\tif !deepEqualWithOutMtime(aObjs[1], bObjs[1]) {\n\t\tt.FailNow()\n\t}\n\n\tif !deepEqualWithOutMtime(aObjs[4], bObjs[len(bObjs)-1]) {\n\t\tt.Fatalf(\"expect %+v but got %+v\", aObjs[4], bObjs[len(bObjs)-1])\n\t}\n\t// Test --force-update option\n\tconfig.ForceUpdate = true\n\t// Forcibly copy {\"a1\", \"a2\", \"abc\",\"c1\",\"c2\",\"ba\"} from a to b.\n\tif err := Sync(a, b, config); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n}\n\n// nolint:errcheck\nfunc TestSyncIncludeAndExclude(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a\")\n\t\t_ = os.RemoveAll(\"/tmp/b\")\n\t}()\n\tconfig := &Config{\n\t\tStart: \"\",\n\t\tEnd: \"\",\n\t\tThreads: 50,\n\t\tListThreads: 1,\n\t\tUpdate: true,\n\t\tPerms: true,\n\t\tDry: false,\n\t\tDeleteSrc: false,\n\t\tDeleteDst: false,\n\t\tVerbose: false,\n\t\tLimit: -1,\n\t\tQuiet: true,\n\t\tMaxSize: math.MaxInt64,\n\t\tExclude: []string{\"1\"},\n\t}\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a/\", \"\", \"\", \"\")\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b/\", \"\", \"\", \"\")\n\n\tsimple := []string{\"a1/z1/z2\", \"a2\", \"ab1\", \"ab2\", \"b1\", \"b2\", \"c1\", \"c2\"}\n\ttestCases := []struct {\n\t\tsrcKey, args, want []string\n\t}{\n\t\t{\n\t\t\tsrcKey: simple,\n\t\t\targs: []string{\"--include\", \"xx*\", \"--include\", \"xxx*\"},\n\t\t\twant: []string{\"a1/\", \"a1/z1/\", \"a1/z1/z2\", \"a2\", \"ab1\", \"ab2\", \"b1\", \"b2\", \"c1\", \"c2\"},\n\t\t},\n\t\t{\n\t\t\tsrcKey: simple,\n\t\t\targs: []string{\"--exclude\", \"a*\", \"--exclude\", \"c*\"},\n\t\t\twant: []string{\"b1\", \"b2\"},\n\t\t},\n\t\t{\n\t\t\tsrcKey: simple,\n\t\t\targs: []string{\"--exclude\", \"a[1-2]\", \"--include\", \"a*\"},\n\t\t\twant: []string{\"ab1\", \"ab2\", \"b1\", \"b2\", \"c1\", \"c2\"},\n\t\t},\n\t\t{\n\t\t\tsrcKey: simple,\n\t\t\targs: []string{\"--exclude\", \"ab?\", \"--include\", \"a*\"},\n\t\t\twant: []string{\"a1/\", \"a1/z1/\", \"a1/z1/z2\", \"a2\", \"b1\", \"b2\", \"c1\", \"c2\"},\n\t\t},\n\t\t{\n\t\t\tsrcKey: simple,\n\t\t\targs: []string{\"--include\", \"a*\", \"--exclude\", \"c*\"},\n\t\t\twant: []string{\"a1/\", \"a1/z1/\", \"a1/z1/z2\", \"a2\", \"ab1\", \"ab2\", \"b1\", \"b2\"},\n\t\t},\n\t\t{\n\t\t\tsrcKey: simple,\n\t\t\targs: []string{\"--exclude\", \"a*\", \"--exclude\", \"c*\"},\n\t\t\twant: []string{\"b1\", \"b2\"},\n\t\t},\n\t\t{\n\t\t\tsrcKey: []string{\"a1/b1/c1\", \"a1/b1/c2\", \"a1/b2/c1\", \"a1/b2/c2\", \"a2/b1/c2\", \"a3/b2/c2\", \"a4\"},\n\t\t\targs: []string{\"--exclude\", \"a*/b[1-2]/c1\", \"--exclude\", \"a4\"},\n\t\t\twant: []string{\"a1/\", \"a1/b1/\", \"a1/b1/c2\", \"a1/b2/\", \"a1/b2/c2\", \"a2/\", \"a2/b1/\", \"a2/b1/c2\", \"a3/\", \"a3/b2/\", \"a3/b2/c2\"},\n\t\t},\n\t}\n\n\tfor _, testCase := range testCases {\n\t\t_ = os.RemoveAll(\"/tmp/a/\")\n\t\t_ = os.RemoveAll(\"/tmp/b/\")\n\t\tos.Args = testCase.args\n\t\tfor _, k := range testCase.srcKey {\n\t\t\ta.Put(ctx, k, bytes.NewReader([]byte(k)))\n\t\t}\n\t\tif err := Sync(a, b, config); err != nil {\n\t\t\tt.Fatalf(\"sync: %s\", err)\n\t\t}\n\n\t\tbRes, _ := ListAll(b, \"\", \"\", \"\", true)\n\t\tvar bKeys []string\n\t\tfor obj := range bRes {\n\t\t\tbKeys = append(bKeys, obj.Key())\n\t\t}\n\t\tif !reflect.DeepEqual(bKeys[1:], testCase.want) {\n\t\t\tt.Errorf(\"sync args %v, want %v, but get %v\", os.Args, testCase.want, bKeys)\n\t\t}\n\t}\n}\n\nfunc TestParseRules(t *testing.T) {\n\ttests := []struct {\n\t\targs []string\n\t\twantRules []rule\n\t}{\n\t\t{\n\t\t\targs: []string{\"--include\", \"a\"},\n\t\t\twantRules: []rule{{pattern: \"a\", include: true}},\n\t\t},\n\t\t{\n\t\t\targs: []string{\"--exclude\", \"a\", \"--include\", \"b\"},\n\t\t\twantRules: []rule{{pattern: \"a\"}, {pattern: \"b\", include: true}},\n\t\t},\n\t\t{\n\t\t\targs: []string{\"--include\", \"a\", \"--test\", \"t\", \"--exclude\", \"b\"},\n\t\t\twantRules: []rule{{pattern: \"a\", include: true}, {pattern: \"b\"}},\n\t\t},\n\t\t{\n\t\t\targs: []string{\"--include\", \"a\", \"--test\", \"t\", \"--exclude\"},\n\t\t\twantRules: []rule{{pattern: \"a\", include: true}},\n\t\t},\n\t\t{\n\t\t\targs: []string{\"--include\", \"a\", \"--exclude\", \"b\", \"--include\", \"c\", \"--exclude\", \"d\"},\n\t\t\twantRules: []rule{{pattern: \"a\", include: true}, {pattern: \"b\"}, {pattern: \"c\", include: true}, {pattern: \"d\"}},\n\t\t},\n\t\t{\n\t\t\targs: []string{\"--include\", \"a\", \"--include\", \"b\", \"--test\", \"--exclude\", \"c\", \"--exclude\", \"d\"},\n\t\t\twantRules: []rule{{pattern: \"a\", include: true}, {pattern: \"b\", include: true}, {pattern: \"c\"}, {pattern: \"d\"}},\n\t\t},\n\t\t{\n\t\t\targs: []string{\"--include=a\", \"--include=b\", \"--exclude=c\", \"--exclude=d\", \"--test=aaa\"},\n\t\t\twantRules: []rule{{pattern: \"a\", include: true}, {pattern: \"b\", include: true}, {pattern: \"c\"}, {pattern: \"d\"}},\n\t\t},\n\t\t{\n\t\t\targs: []string{\"-include=a\", \"--test\", \"t\", \"--include=b\", \"--exclude=c\", \"-exclude=\"},\n\t\t\twantRules: []rule{{pattern: \"a\", include: true}, {pattern: \"b\", include: true}, {pattern: \"c\"}},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tif gotRules := parseIncludeRules(tt.args); !reflect.DeepEqual(gotRules, tt.wantRules) {\n\t\t\tt.Errorf(\"got %+v, want %+v\", gotRules, tt.wantRules)\n\t\t}\n\t}\n}\n\nfunc TestSyncLink(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a\")\n\t\t_ = os.RemoveAll(\"/tmp/b\")\n\t}()\n\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a/\", \"\", \"\", \"\")\n\ta.Put(ctx, \"a1\", bytes.NewReader([]byte(\"test\")))\n\tas := a.(object.SupportSymlink)\n\tas.Symlink(\"/tmp/a/a1\", \"l1\")\n\tas.Symlink(\"./../a1\", \"d1/l2\")\n\tas.Symlink(\"./../notExist\", \"l3\")\n\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b/\", \"\", \"\", \"\")\n\tbs := b.(object.SupportSymlink)\n\tbs.Symlink(\"/tmp/b/a1\", \"l1\")\n\n\tif err := Sync(a, b, &Config{\n\t\tThreads: 50,\n\t\tUpdate: true,\n\t\tPerms: true,\n\t\tListThreads: 1,\n\t\tLinks: true,\n\t\tQuiet: true,\n\t\tLimit: -1,\n\t\tForceUpdate: true,\n\t\tMaxSize: math.MaxInt64,\n\t}); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\n\tl1, err := bs.Readlink(\"l1\")\n\tif err != nil || l1 != \"/tmp/a/a1\" {\n\t\tt.Fatalf(\"readlink: %s content: %s\", err, l1)\n\t}\n\tcontent, err := b.Get(ctx, \"l1\", 0, -1)\n\tif err != nil {\n\t\tt.Fatalf(\"get content failed: %s\", err)\n\t}\n\tif c, err := io.ReadAll(content); err != nil || string(c) != \"test\" {\n\t\tt.Fatalf(\"read content failed: err %s content %s\", err, string(c))\n\t}\n\n\tl2, err := bs.Readlink(\"d1/l2\")\n\tif err != nil || l2 != \"./../a1\" {\n\t\tt.Fatalf(\"readlink: %s\", err)\n\t}\n\tcontent, err = b.Get(ctx, \"d1/l2\", 0, -1)\n\tif err != nil {\n\t\tt.Fatalf(\"content failed: %s\", err)\n\t}\n\tif c, err := io.ReadAll(content); err != nil || string(c) != \"test\" {\n\t\tt.Fatalf(\"read content failed: err %s content %s\", err, string(c))\n\t}\n\n\tl3, err := bs.Readlink(\"l3\")\n\tif err != nil || l3 != \"./../notExist\" {\n\t\tt.Fatalf(\"readlink: %s\", err)\n\t}\n}\n\nfunc TestSyncLinkWithOutFollow(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a\")\n\t\t_ = os.RemoveAll(\"/tmp/b\")\n\t}()\n\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a/\", \"\", \"\", \"\")\n\ta.Put(ctx, \"a1\", bytes.NewReader([]byte(\"test\")))\n\tas := a.(object.SupportSymlink)\n\tas.Symlink(\"/tmp/a/a1\", \"l1\")\n\tas.Symlink(\"./../notExist\", \"l3\")\n\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b/\", \"\", \"\", \"\")\n\n\tif err := Sync(a, b, &Config{\n\t\tThreads: 50,\n\t\tListThreads: 1,\n\t\tUpdate: true,\n\t\tPerms: true,\n\t\tQuiet: true,\n\t\tForceUpdate: true,\n\t\tLimit: -1,\n\t\tMaxSize: math.MaxInt64,\n\t}); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\tcontent, err := b.Get(ctx, \"l1\", 0, -1)\n\tif err != nil {\n\t\tt.Fatalf(\"get content error: %s\", err)\n\t}\n\tif c, err := io.ReadAll(content); err != nil || string(c) != \"test\" {\n\t\tt.Fatalf(\"read content error: %s\", err)\n\t}\n\n\tif lstat, err := os.Lstat(\"/tmp/b/l1\"); err != nil && lstat.Mode()&os.ModeSymlink != 0 {\n\t\tt.Fatalf(\"should follow link\")\n\t}\n\tif _, err := os.Stat(\"/tmp/b/l3\"); !os.IsNotExist(err) {\n\t\tt.Fatalf(\"should not copy broken link\")\n\t}\n}\n\nfunc TestSingleLink(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a\")\n\t\t_ = os.RemoveAll(\"/tmp/b\")\n\t}()\n\t_ = os.Symlink(\"/tmp/aa\", \"/tmp/a\")\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a\", \"\", \"\", \"\")\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b\", \"\", \"\", \"\")\n\tif err := Sync(a, b, &Config{\n\t\tThreads: 50,\n\t\tListThreads: 1,\n\t\tUpdate: true,\n\t\tPerms: true,\n\t\tLinks: true,\n\t\tQuiet: true,\n\t\tLimit: -1,\n\t\tMaxSize: math.MaxInt64,\n\t\tForceUpdate: true,\n\t}); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\treadlink, _ := os.Readlink(\"/tmp/a\")\n\treadlink2, err := os.Readlink(\"/tmp/b\")\n\tif err != nil {\n\t\tt.Fatalf(\"sync err: %v\", err)\n\t}\n\n\tif readlink != readlink2 || readlink != \"/tmp/aa\" {\n\t\tt.Fatalf(\"sync link failed\")\n\t}\n}\n\nfunc TestSyncCheckAllLink(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a\")\n\t\t_ = os.RemoveAll(\"/tmp/b\")\n\t}()\n\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a/\", \"\", \"\", \"\")\n\ta.Put(ctx, \"a1\", bytes.NewReader([]byte(\"test\")))\n\tas := a.(object.SupportSymlink)\n\tas.Symlink(\"/tmp/a/a1\", \"l1\")\n\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b/\", \"\", \"\", \"\")\n\tbs := b.(object.SupportSymlink)\n\tbs.Symlink(\"/tmp/b/a1\", \"l1\")\n\n\tif err := Sync(a, b, &Config{\n\t\tThreads: 50,\n\t\tPerms: true,\n\t\tLinks: true,\n\t\tQuiet: true,\n\t\tListThreads: 1,\n\t\tLimit: -1,\n\t\tMaxSize: math.MaxInt64,\n\t\tCheckAll: true,\n\t}); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\n\tl1, err := bs.Readlink(\"l1\")\n\tif err != nil || l1 != \"/tmp/a/a1\" {\n\t\tt.Fatalf(\"readlink: %s content: %s\", err, l1)\n\t}\n\tcontent, err := b.Get(ctx, \"l1\", 0, -1)\n\tif err != nil {\n\t\tt.Fatalf(\"get content failed: %s\", err)\n\t}\n\tif c, err := io.ReadAll(content); err != nil || string(c) != \"test\" {\n\t\tt.Fatalf(\"read content failed: err %s content %s\", err, string(c))\n\t}\n}\n\nfunc TestSyncCheckNewLink(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a\")\n\t\t_ = os.RemoveAll(\"/tmp/b\")\n\t}()\n\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a/\", \"\", \"\", \"\")\n\ta.Put(ctx, \"a1\", bytes.NewReader([]byte(\"test\")))\n\tas := a.(object.SupportSymlink)\n\tas.Symlink(\"/tmp/a/a1\", \"l1\")\n\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b/\", \"\", \"\", \"\")\n\tbs := b.(object.SupportSymlink)\n\n\tif err := Sync(a, b, &Config{\n\t\tThreads: 50,\n\t\tPerms: true,\n\t\tLinks: true,\n\t\tQuiet: true,\n\t\tListThreads: 1,\n\t\tLimit: -1,\n\t\tMaxSize: math.MaxInt64,\n\t\tCheckNew: true,\n\t}); err != nil {\n\t\tt.Fatalf(\"sync: %s\", err)\n\t}\n\n\tl1, err := bs.Readlink(\"l1\")\n\tif err != nil || l1 != \"/tmp/a/a1\" {\n\t\tt.Fatalf(\"readlink: %s content: %s\", err, l1)\n\t}\n\tcontent, err := b.Get(ctx, \"l1\", 0, -1)\n\tif err != nil {\n\t\tt.Fatalf(\"get content failed: %s\", err)\n\t}\n\tif c, err := io.ReadAll(content); err != nil || string(c) != \"test\" {\n\t\tt.Fatalf(\"read content failed: err %s content %s\", err, string(c))\n\t}\n}\n\nfunc TestLimits(t *testing.T) {\n\tdefer func() {\n\t\t_ = os.RemoveAll(\"/tmp/a/\")\n\t\t_ = os.RemoveAll(\"/tmp/b/\")\n\t\t_ = os.RemoveAll(\"/tmp/c/\")\n\t}()\n\ta, _ := object.CreateStorage(\"file\", \"/tmp/a/\", \"\", \"\", \"\")\n\tb, _ := object.CreateStorage(\"file\", \"/tmp/b/\", \"\", \"\", \"\")\n\tc, _ := object.CreateStorage(\"file\", \"/tmp/c/\", \"\", \"\", \"\")\n\tput := func(storage object.ObjectStorage, keys []string) {\n\t\tfor _, key := range keys {\n\t\t\tif key != \"\" {\n\t\t\t\t_ = storage.Put(ctx, key, bytes.NewReader([]byte{}))\n\t\t\t}\n\t\t}\n\t}\n\tcommonKeys := []string{\"\", \"a1\", \"a2\", \"a3\", \"a4\", \"a5\", \"a6\"}\n\tput(a, commonKeys)\n\tput(c, []string{\"c1\", \"c2\", \"c3\"})\n\ttype subConfig struct {\n\t\tdst object.ObjectStorage\n\t\tlimit int64\n\t\tdeleteDst bool\n\t\texpectedKeys []string\n\t}\n\ttestCases := []subConfig{\n\t\t{b, 2, false, []string{\"\", \"a1\", \"a2\"}},\n\t\t{b, -1, false, commonKeys},\n\t\t{b, 0, false, commonKeys},\n\t\t{c, 7, true, append(commonKeys, \"c2\", \"c3\")},\n\t}\n\tconfig := &Config{\n\t\tThreads: 50,\n\t\tUpdate: true,\n\t\tPerms: true,\n\t\tMaxSize: math.MaxInt64,\n\t\tListThreads: 1,\n\t}\n\tsetConfig := func(config *Config, subC subConfig) {\n\t\tconfig.Limit = subC.limit\n\t\tconfig.DeleteDst = subC.deleteDst\n\t}\n\n\tfor _, tcase := range testCases {\n\t\tsetConfig(config, tcase)\n\t\tif err := Sync(a, tcase.dst, config); err != nil {\n\t\t\tt.Fatalf(\"sync: %s\", err)\n\t\t}\n\n\t\tall, err := ListAll(tcase.dst, \"\", \"\", \"\", true)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"list all b: %s\", err)\n\t\t}\n\n\t\terr = testKeysEqual(all, tcase.expectedKeys)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"testKeysEqual fail: %s\", err)\n\t\t}\n\t}\n}\n\nfunc testKeysEqual(objsCh <-chan object.Object, expectedKeys []string) error {\n\tvar gottenKeys []string\n\tfor obj := range objsCh {\n\t\tgottenKeys = append(gottenKeys, obj.Key())\n\t}\n\tif len(gottenKeys) != len(expectedKeys) {\n\t\treturn fmt.Errorf(\"expected {%s}, got {%s}\", strings.Join(expectedKeys, \", \"),\n\t\t\tstrings.Join(gottenKeys, \", \"))\n\t}\n\n\tfor idx, key := range gottenKeys {\n\t\tif key != expectedKeys[idx] {\n\t\t\treturn fmt.Errorf(\"expected {%s}, got {%s}\", strings.Join(expectedKeys, \", \"),\n\t\t\t\tstrings.Join(gottenKeys, \", \"))\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc TestMatchObjects(t *testing.T) {\n\ttype tcase struct {\n\t\trules []rule\n\t\tkey string\n\t\twant bool\n\t}\n\ttests := []tcase{\n\t\t{rules: []rule{{pattern: \"a*\"}}, key: \"a1\"},\n\t\t{rules: []rule{{pattern: \"a*/b*\"}}, key: \"a1/b1\"},\n\t\t{rules: []rule{{pattern: \"/a*\"}}, key: \"/a1\"},\n\t\t{rules: []rule{{pattern: \"/a\"}}, key: \"/a1\", want: true},\n\t\t{rules: []rule{{pattern: \"/a/b/c\"}}, key: \"/a1\", want: true},\n\t\t{rules: []rule{{pattern: \"a*/b?\"}}, key: \"a1/b1/c2/d1\"},\n\t\t{rules: []rule{{pattern: \"a*/b?/\"}}, key: \"a1/\", want: true},\n\t\t{rules: []rule{{pattern: \"a*/b?/c.txt\"}}, key: \"a1/b1\", want: true},\n\t\t{rules: []rule{{pattern: \"a*/b?/\"}}, key: \"a1/b1/\"},\n\t\t{rules: []rule{{pattern: \"a*/b?/\"}}, key: \"a1/b1/c.txt\"},\n\t\t{rules: []rule{{pattern: \"a*/\"}}, key: \"a1/b1\"},\n\t\t{rules: []rule{{pattern: \"a*/b*/\"}}, key: \"a1/b1/c1/d.txt/\"},\n\t\t{rules: []rule{{pattern: \"/a*/b*\"}}, key: \"/a1/b1/c1/d.txt/\"},\n\t\t{rules: []rule{{pattern: \"a*/b*/c\"}}, key: \"a1/b1/c1/d.txt/\", want: true},\n\t\t{rules: []rule{{pattern: \"a\"}}, key: \"a/b/c/d/\"},\n\t\t{rules: []rule{{pattern: \"a.go\", include: true}, {pattern: \"pkg\"}}, key: \"a/pkg/c/a.go\"},\n\t\t{rules: []rule{{pattern: \"a\"}, {pattern: \"pkg\", include: true}}, key: \"a/pkg/c/a.go\"},\n\t\t{rules: []rule{{pattern: \"a.go\", include: true}, {pattern: \"pkg\"}}, key: \"\", want: true},\n\t\t{rules: []rule{{pattern: \"a\", include: true}, {pattern: \"b/\"}, {pattern: \"c\", include: true}}, key: \"a/b/c\"},\n\t\t{rules: []rule{{pattern: \"a/\", include: true}, {pattern: \"a\"}}, key: \"a/b\", want: true},\n\t\t{rules: []rule{{pattern: \"/***\"}}, key: \"a\"},\n\t\t{rules: []rule{{pattern: \"/***\"}}, key: \"a/b\"},\n\t\t{rules: []rule{{pattern: \"/a/***\"}}, key: \"a/\"},\n\t\t{rules: []rule{{pattern: \"/a/***\"}}, key: \"a/b\"},\n\t\t{rules: []rule{{pattern: \"/a/***\"}}, key: \"a/b/c\"},\n\t\t{rules: []rule{{pattern: \"/a/***\"}}, key: \"b/a/\", want: true},\n\t\t{rules: []rule{{pattern: \"a/***\"}}, key: \"a/\"},\n\t\t{rules: []rule{{pattern: \"a/***\"}}, key: \"a/b\"},\n\t\t{rules: []rule{{pattern: \"a/***\"}}, key: \"a/b/c\"},\n\t\t{rules: []rule{{pattern: \"a/***\"}}, key: \"d/a/b/c\"},\n\t\t{rules: []rule{{pattern: \"a/***\"}}, key: \"a\", want: true},\n\t\t{rules: []rule{{pattern: \"a/***\"}}, key: \"ba\", want: true},\n\t\t{rules: []rule{{pattern: \"a/***\"}}, key: \"ba/\", want: true},\n\t\t{rules: []rule{{pattern: \"*/a/***\"}}, key: \"/a/\"},\n\t\t{rules: []rule{{pattern: \"*/a/***\"}}, key: \"b/a/\"},\n\t\t{rules: []rule{{pattern: \"*/a/***\"}}, key: \"b/a/c\"},\n\t\t{rules: []rule{{pattern: \"/*/a/***\"}}, key: \"/b/a/\"},\n\t\t{rules: []rule{{pattern: \"/*/a/***\"}}, key: \"/b/a/c\"},\n\t\t{rules: []rule{{pattern: \"/*/a/***\"}}, key: \"c/b/a/\", want: true},\n\t\t{rules: []rule{{pattern: \"a/**/b\"}}, key: \"a/c/b\"},\n\t\t{rules: []rule{{pattern: \"a/**/b\"}}, key: \"a/c/d/b\"},\n\t\t{rules: []rule{{pattern: \"a/**/b\"}}, key: \"a/c/d/e/b\"},\n\t\t{rules: []rule{{pattern: \"/**/b\"}}, key: \"a/c/b\"},\n\t\t{rules: []rule{{pattern: \"/**/b\"}}, key: \"a/c/d/b/\"},\n\t\t{rules: []rule{{pattern: \"a**/b\"}}, key: \"a/c/d/b/\"},\n\t\t{rules: []rule{{pattern: \"a**/b\"}}, key: \"a/c/d/ab/\", want: true},\n\t\t{rules: []rule{{pattern: \"a**b\"}}, key: \"a/c/d/b/\"},\n\t\t{rules: []rule{{pattern: \"a**b\"}}, key: \"b/c/d/b/\", want: true},\n\t\t{rules: []rule{{pattern: \"a?**\"}}, key: \"a/a\", want: true},\n\t\t{rules: []rule{{pattern: \"**a\"}}, key: \"a\"},\n\t\t{rules: []rule{{pattern: \"a**\"}}, key: \"a\"},\n\t\t{rules: []rule{{pattern: \"a**a\"}}, key: \"a\", want: true},\n\t\t{rules: []rule{{pattern: \"aa**a\"}}, key: \"aa\", want: true},\n\t\t{rules: []rule{{pattern: \"**/d2/**a\"}}, key: \"/d2/d3/1a\"},\n\t\t{rules: []rule{{pattern: \"**/d2/**a\"}}, key: \"d2/d3/1a\"},\n\t\t{rules: []rule{{pattern: \"a/**/a\"}}, key: \"a\", want: true},\n\t\t{rules: []rule{{pattern: \"a/**/a\"}}, key: \"a/\", want: true},\n\t\t{rules: []rule{{pattern: \"**aa**\", include: true}, {pattern: \"a\"}}, key: \"aa/a\", want: true},\n\t}\n\tfor _, c := range tests {\n\t\tif got := matchLeveledPath(c.rules, c.key); got != c.want {\n\t\t\tt.Errorf(\"matchKey(%+v, %s) = %v, want %v\", c.rules, c.key, got, c.want)\n\t\t}\n\t}\n}\n\nfunc TestMatchFullPatch(t *testing.T) {\n\ttype tcase struct {\n\t\trules []rule\n\t\tkey string\n\t}\n\tmatchedCases := []tcase{\n\t\t{rules: []rule{{pattern: \"a\"}}, key: \"b/a\"},\n\t\t{rules: []rule{{pattern: \"a*\"}}, key: \"a1\"},\n\t\t{rules: []rule{{pattern: \"a*/b*\"}}, key: \"a1/b1\"},\n\t\t{rules: []rule{{pattern: \"/a*\"}}, key: \"/a1\"},\n\t\t{rules: []rule{{pattern: \"a*/b?/\"}}, key: \"a1/b1/\"},\n\t\t{rules: []rule{{pattern: \"a/**/b\"}}, key: \"a/c/b\"},\n\t\t{rules: []rule{{pattern: \"a/**/b\"}}, key: \"a/c/d/b\"},\n\t\t{rules: []rule{{pattern: \"a/**/b\"}}, key: \"a/c/d/e/b\"},\n\t\t{rules: []rule{{pattern: \"/**/b\"}}, key: \"a/c/b\"},\n\t\t{rules: []rule{{pattern: \"a**/b\"}}, key: \"a/c/d/b\"},\n\t\t{rules: []rule{{pattern: \"a**b\"}}, key: \"a/c/d/b\"},\n\t\t{rules: []rule{{pattern: \"**a\"}}, key: \"a\"},\n\t\t{rules: []rule{{pattern: \"a**\"}}, key: \"a\"},\n\t\t{rules: []rule{{pattern: \"**/d2/**a\"}}, key: \"/d2/d3/1a\"},\n\t\t{rules: []rule{{pattern: \"**/d2/**a\"}}, key: \"d2/d3/1a\"},\n\t}\n\tfor _, c := range matchedCases {\n\t\tif got := matchFullPath(c.rules, c.key); got != false {\n\t\t\tt.Errorf(\"matchKey(%+v, %s) = %v, want %v\", c.rules, c.key, got, false)\n\t\t}\n\t}\n\tunmatchedCases := []tcase{\n\t\t{rules: []rule{{pattern: \"/a\"}}, key: \"/a1\"},\n\t\t{rules: []rule{{pattern: \"a*/b?\"}}, key: \"a1/b1/c2/d1\"},\n\t\t{rules: []rule{{pattern: \"/a/b/c\"}}, key: \"/a1\"},\n\t\t{rules: []rule{{pattern: \"a*/b?/\"}}, key: \"a1/\"},\n\t\t{rules: []rule{{pattern: \"a*/b?/c.txt\"}}, key: \"a1/b1\"},\n\t\t{rules: []rule{{pattern: \"a*/b?/\"}}, key: \"a1/b1/c.txt\"},\n\t\t{rules: []rule{{pattern: \"a*/\"}}, key: \"a1/b1\"},\n\t\t{rules: []rule{{pattern: \"a*/b*/\"}}, key: \"a1/b1/c1/d.txt/\"},\n\t\t{rules: []rule{{pattern: \"/a*/b*\"}}, key: \"/a1/b1/c1/d.txt/\"},\n\t\t{rules: []rule{{pattern: \"a\"}}, key: \"a/b/c/d/\"},\n\t\t{rules: []rule{{pattern: \"a*/b*/c\"}}, key: \"a1/b1/c1/d.txt/\"},\n\t\t{rules: []rule{{pattern: \"a**/b\"}}, key: \"a/c/d/ab/\"},\n\t\t{rules: []rule{{pattern: \"a**b\"}}, key: \"b/c/d/b\"},\n\t\t{rules: []rule{{pattern: \"/**/b\"}}, key: \"a/c/d/b/\"},\n\t\t{rules: []rule{{pattern: \"a?**\"}}, key: \"a/a\"},\n\t\t{rules: []rule{{pattern: \"a**a\"}}, key: \"a\"},\n\t\t{rules: []rule{{pattern: \"aa**a\"}}, key: \"aa\"},\n\t\t{rules: []rule{{pattern: \"a/**/a\"}}, key: \"a\"},\n\t\t{rules: []rule{{pattern: \"a/**/a\"}}, key: \"a/\"},\n\t\t{rules: []rule{{pattern: \"**aa**\", include: true}, {pattern: \"a\"}}, key: \"aa/a\"},\n\t}\n\tfor _, c := range unmatchedCases {\n\t\tif got := matchFullPath(c.rules, c.key); got != true {\n\t\t\tt.Errorf(\"matchKey(%+v, %s) = %v, want %v\", c.rules, c.key, got, true)\n\t\t}\n\t}\n}\n\nfunc TestParseFilterRule(t *testing.T) {\n\ttype tcase struct {\n\t\targs []string\n\t\trules []rule\n\t}\n\tcases := []tcase{\n\t\t{[]string{\"--include\", \"a\"}, []rule{{pattern: \"a\", include: true}}},\n\t\t{[]string{\"--exclude\", \"a\", \"--include\", \"b\"}, []rule{{pattern: \"a\"}, {pattern: \"b\", include: true}}},\n\t\t{[]string{\"--include\", \"a\", \"--test\", \"t\", \"--exclude\", \"b\"}, []rule{{pattern: \"a\", include: true}, {pattern: \"b\"}}},\n\t\t{[]string{\"--include=a\", \"--test\", \"t\", \"--exclude\"}, []rule{{pattern: \"a\", include: true}}},\n\t\t{[]string{\"--include\", \"a\", \"--test\", \"t\", \"--exclude\"}, []rule{{pattern: \"a\", include: true}}},\n\t\t{[]string{\"-include=\", \"a\", \"--test\", \"t\", \"--exclude=*\"}, []rule{{pattern: \"*\"}}},\n\t}\n\n\tfor _, c := range cases {\n\t\tif got := parseIncludeRules(c.args); !reflect.DeepEqual(got, c.rules) {\n\t\t\tt.Errorf(\"parseIncludeRules(%+v) = %v, want %v\", c.args, got, c.rules)\n\t\t}\n\t}\n}\n\ntype mockObject struct {\n\tsize int64\n\tmtime time.Time\n}\n\nfunc (o *mockObject) Key() string { return \"\" }\nfunc (o *mockObject) IsDir() bool { return false }\nfunc (o *mockObject) IsSymlink() bool { return false }\nfunc (o *mockObject) Size() int64 { return o.size }\nfunc (o *mockObject) Mtime() time.Time { return o.mtime }\nfunc (o *mockObject) StorageClass() string { return \"\" }\n\nfunc TestFilterSizeAndAge(t *testing.T) {\n\tconfig := &Config{\n\t\tMaxSize: 100,\n\t\tMinSize: 10,\n\t\tMaxAge: time.Second * 100,\n\t\tMinAge: time.Second * 10,\n\t}\n\tnow := time.Now()\n\tif !filterKey(&mockObject{10, now.Add(-time.Second * 15)}, now, nil, config) {\n\t\tt.Fatalf(\"filterKey failed\")\n\t}\n\tif filterKey(&mockObject{200, now.Add(-time.Second * 200)}, now, nil, config) {\n\t\tt.Fatalf(\"filterKey should fail\")\n\t}\n\n\tconfig = &Config{\n\t\tMaxSize: math.MaxInt64,\n\t\tStartTime: time.Now().Add(-time.Hour),\n\t\tEndTime: time.Now().Add(-time.Minute),\n\t}\n\tif !filterKey(&mockObject{200, now.Add(-time.Minute * 30)}, now, nil, config) {\n\t\tt.Fatalf(\"filterKey fail\")\n\t}\n\n\tif filterKey(&mockObject{200, now.Add(-time.Hour * 2)}, now, nil, config) {\n\t\tt.Fatalf(\"filterKey should fail\")\n\t}\n}\n" }, { "path": "pkg/usage/usage.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage usage\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nvar reportUrl = \"https://juicefs.com/report-usage\"\n\nvar logger = utils.GetLogger(\"juicefs\")\n\ntype usage struct {\n\tVolumeID string `json:\"volumeID\"`\n\tSessionID int64 `json:\"sessionID\"`\n\tUsedSpace int64 `json:\"usedBytes\"`\n\tUsedInodes int64 `json:\"usedInodes\"`\n\tVersion string `json:\"version\"`\n\tUptime int64 `json:\"uptime\"`\n\tMetaEngine string `json:\"metaEngine\"` // type of meta engine\n\tDataStore string `json:\"dataStore\"` // type of object store\n}\n\nfunc sendUsage(u usage) error {\n\tbody, err := json.Marshal(u)\n\tif err != nil {\n\t\treturn err\n\t}\n\treq, err := http.NewRequest(\"POST\", reportUrl, bytes.NewReader(body))\n\tif err != nil {\n\t\treturn err\n\t}\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\treturn fmt.Errorf(\"got %s\", resp.Status)\n\t}\n\t_, err = io.ReadAll(resp.Body)\n\treturn err\n}\n\n// ReportUsage will send anonymous usage data to juicefs.com to help the team\n// understand how the community is using it. You can use `--no-usage-report`\n// to disable this.\nfunc ReportUsage(m meta.Meta, version string) {\n\tctx := meta.Background()\n\tvar u usage\n\tif format, err := m.Load(false); err == nil {\n\t\tu.VolumeID = format.UUID\n\t\tu.DataStore = format.Storage\n\t}\n\tu.MetaEngine = m.Name()\n\tu.SessionID = int64(rand.Uint32())\n\tu.Version = version\n\tvar start = time.Now()\n\tfor {\n\t\tvar totalSpace, availSpace, iused, iavail uint64\n\t\t_ = m.StatFS(ctx, meta.RootInode, &totalSpace, &availSpace, &iused, &iavail)\n\t\tu.Uptime = int64(time.Since(start).Seconds())\n\t\tu.UsedSpace = int64(totalSpace - availSpace)\n\t\tu.UsedInodes = int64(iused)\n\n\t\tif err := sendUsage(u); err != nil {\n\t\t\tlogger.Debugf(\"send usage: %s\", err)\n\t\t}\n\t\ttime.Sleep(time.Hour)\n\t}\n}\n" }, { "path": "pkg/usage/usage_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage usage\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\n// nolint:errcheck\nfunc TestUsageReport(t *testing.T) {\n\t// invalid addr\n\treportUrl = \"http://127.0.0.1/report-usage\"\n\tm := meta.NewClient(\"memkv://\", nil)\n\tformat := &meta.Format{\n\t\tName: \"test\",\n\t\tBlockSize: 4096,\n\t\tCapacity: 1 << 30,\n\t\tDirStats: true,\n\t}\n\t_ = m.Init(format, true)\n\tgo ReportUsage(m, \"unittest\")\n\t// wait for it to report to unavailable address, it should not panic.\n\ttime.Sleep(time.Millisecond * 100)\n\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer l.Close()\n\n\tmux := http.NewServeMux()\n\tvar u usage\n\tdone := make(chan bool)\n\tmux.HandleFunc(\"/report-usage\", func(rw http.ResponseWriter, r *http.Request) {\n\t\td, _ := io.ReadAll(r.Body)\n\t\t_ = json.Unmarshal(d, &u)\n\t\t_, _ = rw.Write([]byte(\"OK\"))\n\t\tdone <- true\n\t})\n\tgo http.Serve(l, mux)\n\n\taddr := l.Addr().String()\n\treportUrl = fmt.Sprintf(\"http://%s/report-usage\", addr)\n\tgo ReportUsage(m, \"unittest\")\n\n\tdeadline := time.NewTimer(time.Second * 3)\n\tselect {\n\tcase <-done:\n\t\tif u.MetaEngine != \"memkv\" {\n\t\t\tt.Fatalf(\"unexpected meta engine: %s\", u.MetaEngine)\n\t\t}\n\t\tif u.Version != \"unittest\" {\n\t\t\tt.Fatalf(\"unexpected version: %s\", u.Version)\n\t\t}\n\tcase <-deadline.C:\n\t\tt.Fatalf(\"no report after 3 seconds\")\n\t}\n\ttime.Sleep(time.Millisecond * 100) // wait for the client to finish\n}\n" }, { "path": "pkg/utils/alloc.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"math/bits\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nvar used int64\n\n// Alloc returns size bytes memory from Go heap.\nfunc Alloc(size int) []byte {\n\tb := Alloc0(size)\n\tatomic.AddInt64(&used, int64(cap(b)))\n\treturn b\n}\n\n// Alloc returns size bytes memory from Go heap.\nfunc Alloc0(size int) []byte {\n\tzeros := PowerOf2(size)\n\tb := *pools[zeros].Get().(*[]byte)\n\tif cap(b) < size {\n\t\tpanic(fmt.Sprintf(\"%d < %d\", cap(b), size))\n\t}\n\treturn b[:size]\n}\n\n// Free returns memory to Go heap.\nfunc Free(b []byte) {\n\t// buf could be zero length\n\tatomic.AddInt64(&used, -int64(cap(b)))\n\tFree0(b)\n}\n\n// Free returns memory to Go heap.\nfunc Free0(b []byte) {\n\t// buf could be zero length\n\tpools[PowerOf2(cap(b))].Put(&b)\n}\n\n// AllocMemory returns the allocated memory\nfunc AllocMemory() int64 {\n\treturn atomic.LoadInt64(&used)\n}\n\nvar pools []*sync.Pool\n\n// PowerOf2 returns the smallest power of 2 that is >= s\nfunc PowerOf2(s int) int {\n\tif s <= 0 {\n\t\treturn 0\n\t}\n\t// Find position of the most significant bit (MSB)\n\treturn bits.Len(uint(s - 1))\n}\n\nfunc init() {\n\tpools = make([]*sync.Pool, 34) // 1 - 8G\n\tfor i := 0; i < 34; i++ {\n\t\tfunc(bits int) {\n\t\t\tpools[i] = &sync.Pool{\n\t\t\t\tNew: func() interface{} {\n\t\t\t\t\tb := make([]byte, 1< 1000 {\n\t\tt.Fatal(\"time is not accurate\")\n\t}\n\tc1 := Clock()\n\tc2 := Clock()\n\tif c2 < c1 {\n\t\tt.Fatalf(\"clock is not monotonic: %s > %s\", c1, c2)\n\t}\n}\n\nfunc BenchmarkNow(b *testing.B) {\n\tvar now time.Time\n\tfor i := 0; i < b.N; i++ {\n\t\tnow = Now()\n\t}\n\tlog.Print(now)\n}\n\nfunc BenchmarkClock(b *testing.B) {\n\tvar now time.Duration\n\tfor i := 0; i < b.N; i++ {\n\t\tnow = Clock()\n\t}\n\tlog.Print(now)\n}\n" }, { "path": "pkg/utils/clock_unix.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport \"time\"\n\nvar started = time.Now()\n\nfunc Now() time.Time {\n\treturn time.Now()\n}\n\nfunc Clock() time.Duration {\n\treturn time.Since(started)\n}\n" }, { "path": "pkg/utils/clock_windows.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"syscall\"\n\t\"time\"\n\t\"unsafe\"\n)\n\ntype clock struct {\n\tt time.Time\n\ttick time.Duration\n}\n\nvar last *clock\n\nfunc Now() time.Time {\n\tc := last\n\treturn c.t.Add(Clock() - c.tick)\n}\n\n// Clock returns the number of milliseconds that have elapsed since the program\n// was started.\nvar Clock func() time.Duration\n\nfunc init() {\n\tQPCTimer := func() func() time.Duration {\n\t\tlib, _ := syscall.LoadLibrary(\"kernel32.dll\")\n\t\tqpc, _ := syscall.GetProcAddress(lib, \"QueryPerformanceCounter\")\n\t\tqpf, _ := syscall.GetProcAddress(lib, \"QueryPerformanceFrequency\")\n\t\tif qpc == 0 || qpf == 0 {\n\t\t\treturn nil\n\t\t}\n\n\t\tvar freq, start uint64\n\t\tsyscall.Syscall(qpf, 1, uintptr(unsafe.Pointer(&freq)), 0, 0)\n\t\tsyscall.Syscall(qpc, 1, uintptr(unsafe.Pointer(&start)), 0, 0)\n\t\tif freq <= 0 {\n\t\t\treturn nil\n\t\t}\n\n\t\tfreqns := float64(freq) / 1e9\n\t\treturn func() time.Duration {\n\t\t\tvar now uint64\n\t\t\tsyscall.Syscall(qpc, 1, uintptr(unsafe.Pointer(&now)), 0, 0)\n\t\t\treturn time.Duration(float64(now-start) / freqns)\n\t\t}\n\t}\n\tif Clock = QPCTimer(); Clock == nil {\n\t\t// Fallback implementation\n\t\tstart := time.Now()\n\t\tClock = func() time.Duration { return time.Since(start) }\n\t}\n\tlast = &clock{time.Now(), Clock()}\n\tgo func() {\n\t\tfor {\n\t\t\tlast = &clock{time.Now(), Clock()}\n\t\t\ttime.Sleep(time.Hour)\n\t\t}\n\t}()\n}\n" }, { "path": "pkg/utils/cond.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"sync\"\n\t\"time\"\n)\n\n// Cond is similar to sync.Cond, but you can wait with a timeout.\ntype Cond struct {\n\tL sync.Locker\n\tsignal chan struct{}\n}\n\n// Signal wakes up a waiter.\n// It's required for the caller to hold L.\nfunc (c *Cond) Signal() {\n\tselect {\n\tcase c.signal <- struct{}{}:\n\tdefault:\n\t}\n}\n\n// Broadcast wake up all the waiters.\n// It's required for the caller to hold L.\nfunc (c *Cond) Broadcast() {\n\tclose(c.signal)\n\tc.signal = make(chan struct{})\n}\n\nvar timerPool = sync.Pool{}\n\n// WaitWithTimeout wait for a signal or a period of timeout eclipsed.\n// returns true in case of timeout else false\nfunc (c *Cond) WaitWithTimeout(d time.Duration) bool {\n\tch := c.signal\n\tc.L.Unlock()\n\tvar t *time.Timer\n\tif e := timerPool.Get(); e == nil {\n\t\tt = time.NewTimer(d)\n\t} else {\n\t\tt = e.(*time.Timer)\n\t\tt.Reset(d)\n\t}\n\tdefer func() {\n\t\ttimerPool.Put(t)\n\t\tc.L.Lock()\n\t}()\n\tselect {\n\tcase <-ch:\n\t\tif !t.Stop() {\n\t\t\t<-t.C\n\t\t}\n\t\treturn false\n\tcase <-t.C:\n\t\treturn true\n\t}\n}\n\n// NewCond creates a Cond.\nfunc NewCond(lock sync.Locker) *Cond {\n\treturn &Cond{lock, make(chan struct{})}\n}\n" }, { "path": "pkg/utils/cond_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestCond(t *testing.T) {\n\t// test Wait and Signal\n\tvar m sync.Mutex\n\tc := NewCond(&m)\n\tvar ready bool\n\tstart := time.Now()\n\tgo func() {\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tm.Lock()\n\t\t\tready = true\n\t\t\tc.Signal()\n\t\t\tfor ready {\n\t\t\t\tc.WaitWithTimeout(time.Millisecond * 100)\n\t\t\t}\n\t\t\tm.Unlock()\n\t\t}\n\t}()\n\tfor i := 0; i < 10; i++ {\n\t\tm.Lock()\n\t\tfor !ready {\n\t\t\tc.WaitWithTimeout(time.Millisecond * 100)\n\t\t}\n\t\tready = false\n\t\tc.Signal()\n\t\tm.Unlock()\n\t}\n\tif ready {\n\t\tt.Fatalf(\"the work should finish with ready = false\")\n\t}\n\tif time.Since(start) > time.Second {\n\t\tt.Fatalf(\"the work should finish in 1 second\")\n\t}\n\n\t// test WaitWithTimeout\n\tdone := make(chan bool)\n\tvar timeout bool\n\tgo func() {\n\t\tm.Lock()\n\t\tdefer m.Unlock()\n\t\ttimeout = c.WaitWithTimeout(time.Millisecond * 10)\n\t\tdone <- true\n\t}()\n\tselect {\n\tcase <-done:\n\t\tif !timeout {\n\t\t\tt.Fatalf(\"it should timeout\")\n\t\t}\n\tcase <-time.NewTimer(time.Second).C:\n\t\tt.Fatalf(\"wait did not return after 1 second\")\n\t}\n\n\t// test Broadcast to wake up all goroutines\n\tvar N = 1000\n\tdone2 := make(chan bool, N)\n\tvar wg2 sync.WaitGroup\n\tfor i := 0; i < N; i++ {\n\t\twg2.Add(1)\n\t\tgo func() {\n\t\t\tm.Lock()\n\t\t\twg2.Done()\n\t\t\ttimeout := c.WaitWithTimeout(time.Second)\n\t\t\tm.Unlock()\n\t\t\tdone2 <- timeout\n\t\t}()\n\t}\n\twg2.Wait()\n\tm.Lock()\n\tc.Broadcast()\n\tm.Unlock()\n\tdeadline := time.NewTimer(time.Millisecond * 500)\n\tfor i := 0; i < N; i++ {\n\t\tselect {\n\t\tcase timeout := <-done2:\n\t\t\tif timeout {\n\t\t\t\tt.Fatalf(\"cond should not timeout\")\n\t\t\t}\n\t\tcase <-deadline.C:\n\t\t\tt.Fatalf(\"not all goroutines wakeup in 500 ms; i %d\", i)\n\t\t}\n\t}\n}\n" }, { "path": "pkg/utils/errors.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"errors\"\n\t\"syscall\"\n)\n\nvar (\n\tENOTSUP = errors.New(\"not supported\")\n\tErrFuncTimeout = errors.New(\"function timeout\")\n\tErrSkipped = errors.New(\"skipped\")\n\tErrExtlink = syscall.Errno(1000)\n)\n" }, { "path": "pkg/utils/general.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"math/rand\"\n\t\"time\"\n)\n\nfunc SleepWithJitter(d time.Duration) {\n\ttime.Sleep(JitterIt(d))\n}\n\nfunc JitterIt[T float64 | time.Duration](d T) T {\n\tj := int64(d / 20) // +- 5%\n\treturn d + T(rand.Int63n(2*j+1)-j)\n}\n" }, { "path": "pkg/utils/humanize.go", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"errors\"\n\t\"strconv\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nfunc ParseBytes(ctx *cli.Context, key string, unit byte) uint64 {\n\tstr := ctx.String(key)\n\tif len(str) == 0 {\n\t\treturn 0\n\t}\n\treturn ParseBytesStr(key, str, unit)\n}\n\nfunc ParseBytesStr(key, str string, unit byte) uint64 {\n\ts := str\n\tif c := s[len(s)-1]; c < '0' || c > '9' {\n\t\tunit = c\n\t\ts = s[:len(s)-1]\n\t}\n\tval, err := strconv.ParseFloat(s, 64)\n\tif err == nil {\n\t\tvar shift int\n\t\tswitch unit {\n\t\tcase 'B':\n\t\tcase 'k', 'K':\n\t\t\tshift = 10\n\t\tcase 'm', 'M':\n\t\t\tshift = 20\n\t\tcase 'g', 'G':\n\t\t\tshift = 30\n\t\tcase 't', 'T':\n\t\t\tshift = 40\n\t\tcase 'p', 'P':\n\t\t\tshift = 50\n\t\tcase 'e', 'E':\n\t\t\tshift = 60\n\t\tdefault:\n\t\t\terr = errors.New(\"invalid unit\")\n\t\t}\n\t\tval *= float64(uint64(1) << shift)\n\t}\n\tif err != nil {\n\t\tlogger.Fatalf(\"Invalid value \\\"%s\\\" for \\\"%s\\\": %s\", str, key, err)\n\t}\n\treturn uint64(val)\n}\n\nfunc ParseMbps(ctx *cli.Context, key string) int64 {\n\tstr := ctx.String(key)\n\tif len(str) == 0 {\n\t\treturn 0\n\t}\n\n\treturn ParseMbpsStr(key, str)\n}\n\nfunc ParseMbpsStr(key, str string) int64 {\n\ts := str\n\tvar unit byte = 'M'\n\tif c := s[len(s)-1]; c < '0' || c > '9' {\n\t\tunit = c\n\t\ts = s[:len(s)-1]\n\t}\n\tval, err := strconv.ParseFloat(s, 64)\n\tif err == nil {\n\t\tswitch unit {\n\t\tcase 'm', 'M':\n\t\tcase 'g', 'G':\n\t\t\tval *= 1e3\n\t\tcase 't', 'T':\n\t\t\tval *= 1e6\n\t\tcase 'p', 'P':\n\t\t\tval *= 1e9\n\t\tdefault:\n\t\t\terr = errors.New(\"invalid unit\")\n\t\t}\n\t}\n\tif err != nil {\n\t\tlogger.Fatalf(\"Invalid value \\\"%s\\\" for \\\"%s\\\"\", str, key)\n\t}\n\treturn int64(val)\n}\n\nfunc Mbps(val int64) string {\n\tv := float64(val)\n\tif v < 1e3 {\n\t\treturn strconv.FormatFloat(v, 'f', 1, 64) + \" Mbps\"\n\t} else if v < 1e6 {\n\t\treturn strconv.FormatFloat(v/1e3, 'f', 1, 64) + \" Gbps\"\n\t} else if v < 1e9 {\n\t\treturn strconv.FormatFloat(v/1e6, 'f', 1, 64) + \" Tbps\"\n\t}\n\treturn strconv.FormatFloat(v/1e9, 'f', 1, 64) + \" Pbps\"\n}\n" }, { "path": "pkg/utils/logger.go", "content": "// Copyright 2015 Ka-Hing Cheung\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/sirupsen/logrus\"\n)\n\nvar mu sync.Mutex\nvar loggers = make(map[string]*logHandle)\n\nvar syslogHook logrus.Hook\nvar framePlaceHolder = runtime.Frame{Function: \"???\", File: \"???\", Line: 0}\n\ntype logHandle struct {\n\tlogrus.Logger\n\n\tname string\n\tlogid string\n\tpid int\n\tlvl *logrus.Level\n\tcolorful bool\n}\n\nfunc (l *logHandle) Format(e *logrus.Entry) ([]byte, error) {\n\tlvl := e.Level\n\tif l.lvl != nil {\n\t\tlvl = *l.lvl\n\t}\n\tlvlStr := strings.ToUpper(lvl.String())\n\tif l.colorful {\n\t\tvar color int\n\t\tswitch lvl {\n\t\tcase logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:\n\t\t\tcolor = 31 // RED\n\t\tcase logrus.WarnLevel:\n\t\t\tcolor = 33 // YELLOW\n\t\tcase logrus.InfoLevel:\n\t\t\tcolor = 34 // BLUE\n\t\tdefault: // logrus.TraceLevel, logrus.DebugLevel\n\t\t\tcolor = 35 // MAGENTA\n\t\t}\n\t\tlvlStr = fmt.Sprintf(\"\\033[1;%dm%s\\033[0m\", color, lvlStr)\n\t}\n\tconst timeFormat = \"2006/01/02 15:04:05.000000\"\n\tcaller := e.Caller\n\tif caller == nil { // for unknown reason, sometimes e.Caller is nil\n\t\tcaller = &framePlaceHolder\n\t}\n\tstr := fmt.Sprintf(\"%s%v %s[%d] <%v>: %v [%s@%s:%d]\",\n\t\tl.logid,\n\t\te.Time.Format(timeFormat),\n\t\tl.name,\n\t\tl.pid,\n\t\tlvlStr,\n\t\tstrings.TrimRight(e.Message, \"\\n\"),\n\t\tMethodName(caller.Function),\n\t\tpath.Base(caller.File),\n\t\tcaller.Line)\n\n\tif len(e.Data) != 0 {\n\t\tstr += \" \" + fmt.Sprint(e.Data)\n\t}\n\tif !strings.HasSuffix(str, \"\\n\") {\n\t\tstr += \"\\n\"\n\t}\n\treturn []byte(str), nil\n}\n\n// Returns a human-readable method name, removing internal markers added by Go\nfunc MethodName(fullFuncName string) string {\n\tfirstSlash := strings.Index(fullFuncName, \"/\")\n\tif firstSlash != -1 && firstSlash < len(fullFuncName)-1 {\n\t\tfullFuncName = fullFuncName[firstSlash+1:]\n\t}\n\tlastDot := strings.LastIndex(fullFuncName, \".\")\n\tif lastDot == -1 || lastDot == len(fullFuncName)-1 {\n\t\treturn fullFuncName\n\t}\n\tmethod := fullFuncName[lastDot+1:]\n\t// avoid func1\n\tif strings.HasPrefix(method, \"func\") && method[4] >= '0' && method[4] <= '9' {\n\t\tcandidate := MethodName(fullFuncName[:lastDot])\n\t\tif candidate != \"\" {\n\t\t\tmethod = candidate\n\t\t}\n\t}\n\t// avoid init.3\n\tif len(method) == 1 && method[0] >= '0' && method[0] <= '9' {\n\t\tcandidate := MethodName(fullFuncName[:lastDot])\n\t\tif candidate != \"\" {\n\t\t\tmethod = candidate\n\t\t}\n\t}\n\treturn method\n}\n\n// for aws.Logger\nfunc (l *logHandle) Log(args ...interface{}) {\n\tl.Debugln(args...)\n}\n\nfunc newLogger(name string) *logHandle {\n\tl := &logHandle{Logger: *logrus.New(), name: name, pid: os.Getpid(), colorful: SupportANSIColor(os.Stderr.Fd())}\n\tl.Formatter = l\n\tif syslogHook != nil {\n\t\tl.AddHook(syslogHook)\n\t}\n\tl.SetReportCaller(true)\n\treturn l\n}\n\n// GetLogger returns a logger mapped to `name`\nfunc GetLogger(name string) *logHandle {\n\tmu.Lock()\n\tdefer mu.Unlock()\n\n\tif logger, ok := loggers[name]; ok {\n\t\treturn logger\n\t}\n\tlogger := newLogger(name)\n\tloggers[name] = logger\n\treturn logger\n}\n\n// SetLogLevel sets Level to all the loggers in the map\nfunc SetLogLevel(lvl logrus.Level) {\n\tmu.Lock()\n\tdefer mu.Unlock()\n\tfor _, logger := range loggers {\n\t\tlogger.Level = lvl\n\t}\n}\n\nfunc DisableLogColor() {\n\tmu.Lock()\n\tdefer mu.Unlock()\n\tfor _, logger := range loggers {\n\t\tlogger.colorful = false\n\t}\n}\n\nfunc SetOutFile(name string) {\n\tfile, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)\n\tif err != nil {\n\t\treturn\n\t}\n\tmu.Lock()\n\tdefer mu.Unlock()\n\tfor _, logger := range loggers {\n\t\tlogger.SetOutput(file)\n\t\tlogger.colorful = false\n\t}\n}\n\nfunc SetOutput(w io.Writer) {\n\tmu.Lock()\n\tdefer mu.Unlock()\n\tfor _, logger := range loggers {\n\t\tlogger.SetOutput(w)\n\t}\n}\n\nfunc SetLogID(id string) {\n\tmu.Lock()\n\tdefer mu.Unlock()\n\tfor _, logger := range loggers {\n\t\tlogger.logid = id\n\t}\n}\n" }, { "path": "pkg/utils/logger_syslog.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"log/syslog\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/sirupsen/logrus\"\n\tlogrus_syslog \"github.com/sirupsen/logrus/hooks/syslog\"\n)\n\ntype logLine struct {\n\tlevel logrus.Level\n\tmsg string\n}\n\ntype SyslogHook struct {\n\t*logrus_syslog.SyslogHook\n\tbuffer chan logLine\n}\n\nfunc (hook *SyslogHook) flush() {\n\tfor l := range hook.buffer {\n\t\tline := l.msg\n\t\tvar err error\n\t\tswitch l.level {\n\t\tcase logrus.PanicLevel:\n\t\t\terr = hook.Writer.Crit(line)\n\t\tcase logrus.FatalLevel:\n\t\t\terr = hook.Writer.Crit(line)\n\t\tcase logrus.ErrorLevel:\n\t\t\terr = hook.Writer.Err(line)\n\t\tcase logrus.WarnLevel:\n\t\t\terr = hook.Writer.Warning(line)\n\t\tcase logrus.InfoLevel:\n\t\t\terr = hook.Writer.Info(line)\n\t\tcase logrus.DebugLevel:\n\t\t\terr = hook.Writer.Debug(line)\n\t\t}\n\t\tif err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"write to syslog: %v, level: %s, line: %s\", err, l.level, line)\n\t\t}\n\t}\n}\n\nfunc (hook *SyslogHook) Fire(entry *logrus.Entry) error {\n\tline, err := entry.String()\n\tif err != nil {\n\t\tfmt.Fprintf(os.Stderr, \"Unable to read entry, %v\", err)\n\t\treturn err\n\t}\n\n\tselect {\n\tcase hook.buffer <- logLine{entry.Level, line[27:]}: // drop the timestamp\n\t\treturn nil\n\tdefault:\n\t\tfmt.Fprintf(os.Stderr, \"buffer of syslog is full, drop: %s\", line)\n\t\treturn fmt.Errorf(\"buffer is full\")\n\t}\n}\n\nvar once sync.Once\n\nfunc InitLoggers(logToSyslog bool) {\n\tif logToSyslog {\n\t\tonce.Do(func() {\n\t\t\thook, err := logrus_syslog.NewSyslogHook(\"\", \"\", syslog.LOG_DEBUG|syslog.LOG_USER, \"\")\n\t\t\tif err != nil {\n\t\t\t\t// println(\"Unable to connect to local syslog daemon\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tsyslogHook = &SyslogHook{hook, make(chan logLine, 1024)}\n\t\t\tgo syslogHook.(*SyslogHook).flush()\n\n\t\t\tfor _, l := range loggers {\n\t\t\t\tl.AddHook(syslogHook)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "pkg/utils/logger_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/sirupsen/logrus\"\n)\n\nfunc TestLogger(t *testing.T) {\n\t_ = GetLogger(\"test\")\n\tf, err := os.CreateTemp(\"\", \"test_logger\")\n\tif err != nil {\n\t\tt.Fatalf(\"temp file: %s\", err)\n\t}\n\tdefer f.Close()\n\tSetOutFile(\"\") // invalid\n\tSetOutFile(f.Name())\n\tInitLoggers(true)\n\tSetLogID(\"testid\")\n\n\tSetLogLevel(logrus.TraceLevel)\n\tSetLogLevel(logrus.DebugLevel)\n\tSetLogLevel(logrus.InfoLevel)\n\tSetLogLevel(logrus.ErrorLevel)\n\tSetLogLevel(logrus.FatalLevel)\n\tSetLogLevel(logrus.WarnLevel)\n\tlogger := GetLogger(\"test\")\n\tlogger.Info(\"info level\")\n\tlogger.Debug(\"debug level\")\n\tlogger.Warnf(\"warn level\")\n\tlogger.Error(\"error level\")\n\n\td, _ := os.ReadFile(f.Name())\n\ts := string(d)\n\tif strings.Contains(s, \"info level\") || strings.Contains(s, \"debug level\") {\n\t\tt.Fatalf(\"info/debug should not be logged: %s\", s)\n\t} else if !strings.Contains(s, \"warn level\") || !strings.Contains(s, \"error level\") {\n\t\tt.Fatalf(\"warn/error should be logged: %s\", s)\n\t} else if !strings.Contains(s, \"testid\") {\n\t\tt.Fatalf(\"logid \\\"testid\\\" should be logged: %s\", s)\n\t}\n}\n\nfunc TestMethodName(t *testing.T) {\n\ttype args struct {\n\t\tfullFuncName string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant string\n\t}{{\n\t\tname: \"main\",\n\t\targs: args{\n\t\t\tfullFuncName: \"cmd.Main\",\n\t\t},\n\t\twant: \"Main\",\n\t}, {\n\t\tname: \"nested method\",\n\t\targs: args{\n\t\t\tfullFuncName: \"github.com/juicedata/juicefs/cmd.watchdog.func1\",\n\t\t},\n\t\twant: \"watchdog\",\n\t}, {\n\t\tname: \"multiple inits\",\n\t\targs: args{\n\t\t\tfullFuncName: \"github.com/juicedata/juicefs/pkg/utils.init.3.func1\",\n\t\t},\n\t\twant: \"init\",\n\t}}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := MethodName(tt.args.fullFuncName); got != tt.want {\n\t\t\t\tt.Errorf(\"MethodName() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "pkg/utils/logger_windows.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nfunc InitLoggers(logToSyslog bool) {}\n" }, { "path": "pkg/utils/memusage.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"strconv\"\n\t\"syscall\"\n)\n\nfunc MemoryUsage() (virt, rss uint64) {\n\tstat, err := os.ReadFile(\"/proc/self/stat\")\n\tif err == nil {\n\t\tstats := bytes.Split(stat, []byte(\" \"))\n\t\tif len(stats) >= 24 {\n\t\t\tv, _ := strconv.ParseUint(string(stats[22]), 10, 64)\n\t\t\tr, _ := strconv.ParseUint(string(stats[23]), 10, 64)\n\t\t\treturn v, r * 4096\n\t\t}\n\t}\n\n\tvar ru syscall.Rusage\n\terr = syscall.Getrusage(syscall.RUSAGE_SELF, &ru)\n\tif err == nil {\n\t\treturn uint64(ru.Maxrss), uint64(ru.Maxrss)\n\t}\n\treturn\n}\n" }, { "path": "pkg/utils/memusage_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport \"testing\"\n\nfunc TestMemUsage(t *testing.T) {\n\tvirt, rss := MemoryUsage()\n\tif virt < (1<<20) || rss < (1<<20) || rss > (100<<20) {\n\t\tt.Fatalf(\"invalid memory usage: virt %d, rss %d\", virt, rss)\n\t}\n}\n" }, { "path": "pkg/utils/memusage_windows.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"unsafe\"\n\n\t\"golang.org/x/sys/windows\"\n)\n\ntype PROCESS_MEMORY_COUNTERS struct {\n\tCB uint32\n\tPageFaultCount uint32\n\tPeakWorkingSetSize uint64\n\tWorkingSetSize uint64\n\tQuotaPeakPagedPoolUsage uint64\n\tQuotaPagedPoolUsage uint64\n\tQuotaPeakNonPagedPoolUsage uint64\n\tQuotaNonPagedPoolUsage uint64\n\tPagefileUsage uint64\n\tPeakPagefileUsage uint64\n}\n\nvar (\n\tmodpsapi = windows.NewLazySystemDLL(\"psapi.dll\")\n\tprocGetProcessMemoryInfo = modpsapi.NewProc(\"GetProcessMemoryInfo\")\n)\n\nfunc getMemoryInfo(pid int32) (PROCESS_MEMORY_COUNTERS, error) {\n\tvar mem PROCESS_MEMORY_COUNTERS\n\tc, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid))\n\tif err != nil {\n\t\treturn mem, err\n\t}\n\tdefer windows.CloseHandle(c)\n\tif err := getProcessMemoryInfo(c, &mem); err != nil {\n\t\treturn mem, err\n\t}\n\n\treturn mem, err\n}\n\nfunc getProcessMemoryInfo(h windows.Handle, mem *PROCESS_MEMORY_COUNTERS) (err error) {\n\tr1, _, e1 := syscall.Syscall(procGetProcessMemoryInfo.Addr(), 3, uintptr(h), uintptr(unsafe.Pointer(mem)), uintptr(unsafe.Sizeof(*mem)))\n\tif r1 == 0 {\n\t\tif e1 != 0 {\n\t\t\terr = error(e1)\n\t\t} else {\n\t\t\terr = syscall.EINVAL\n\t\t}\n\t}\n\treturn\n}\n\nfunc MemoryUsage() (virt, rss uint64) {\n\tc, err := getMemoryInfo(int32(os.Getpid()))\n\tif err == nil {\n\t\treturn c.PeakWorkingSetSize, c.WorkingSetSize\n\t}\n\treturn 0, 0\n}\n" }, { "path": "pkg/utils/proc_title.go", "content": "//go:build !nogspt\n// +build !nogspt\n\n/*\n * JuiceFS, Copyright 2026 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"strings\"\n\n\t\"github.com/erikdubbelboer/gspt\"\n)\n\nfunc SetProcTitle(args []string) {\n\tgspt.SetProcTitle(strings.Join(args, \" \"))\n}\n" }, { "path": "pkg/utils/proc_title_noop.go", "content": "//go:build nogspt\n// +build nogspt\n\n/*\n * JuiceFS, Copyright 2026 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nfunc SetProcTitle(args []string) {\n\t// noop: gspt is excluded from this build to prevent argv modification\n\t// when libjfs.so is loaded as a shared library (e.g. by the Java SDK).\n}\n" }, { "path": "pkg/utils/progress.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/vbauerster/mpb/v7\"\n\t\"github.com/vbauerster/mpb/v7/decor\"\n)\n\ntype Progress struct {\n\t*mpb.Progress\n\tQuiet bool\n\tbars []*mpb.Bar\n}\n\ntype Bar struct {\n\ttotal int64\n\t*mpb.Bar\n}\n\nfunc (b *Bar) IncrTotal(n int64) {\n\ttotal := atomic.AddInt64(&b.total, n)\n\tb.Bar.SetTotal(total, false)\n}\n\nfunc (b *Bar) SetTotal(total int64) {\n\tatomic.StoreInt64(&b.total, total)\n\tb.Bar.SetTotal(total, false)\n}\n\nfunc (b *Bar) GetTotal() int64 {\n\treturn atomic.LoadInt64(&b.total)\n}\n\nfunc (b *Bar) Done() {\n\tb.Bar.SetTotal(0, true)\n}\n\ntype DoubleSpinner struct {\n\tcount *mpb.Bar\n\tbytes *mpb.Bar\n}\n\nfunc (s *DoubleSpinner) IncrInt64(size int64) {\n\ts.count.Increment()\n\ts.bytes.IncrInt64(size)\n}\n\nfunc (s *DoubleSpinner) Done() {\n\ts.count.SetTotal(0, true)\n\ts.bytes.SetTotal(0, true)\n}\n\nfunc (s *DoubleSpinner) Current() (int64, int64) {\n\treturn s.count.Current(), s.bytes.Current()\n}\n\nfunc (s *DoubleSpinner) SetCurrent(count, bytes int64) {\n\ts.count.SetCurrent(count)\n\ts.bytes.SetCurrent(bytes)\n}\n\nfunc NewProgress(quiet bool) *Progress {\n\tvar p *Progress\n\tif quiet || os.Getenv(\"DISPLAY_PROGRESSBAR\") == \"false\" || !isatty.IsTerminal(os.Stdout.Fd()) {\n\t\tp = &Progress{mpb.New(mpb.WithWidth(64), mpb.WithOutput(nil)), true, nil}\n\t} else {\n\t\tp = &Progress{mpb.New(mpb.WithWidth(64)), false, nil}\n\t\tif isatty.IsTerminal(os.Stderr.Fd()) {\n\t\t\tSetOutput(p)\n\t\t}\n\t}\n\treturn p\n}\n\nfunc (p *Progress) AddCountBar(name string, total int64) *Bar {\n\tstartTime := time.Now()\n\tvar speedMsg, usedMsg string\n\tb := p.Progress.AddBar(0, // disable triggerComplete\n\t\tmpb.PrependDecorators(\n\t\t\tdecor.Name(name+\": \", decor.WCSyncWidth),\n\t\t\tdecor.CountersNoUnit(\"%d/%d\"),\n\t\t),\n\t\tmpb.AppendDecorators(\n\t\t\tdecor.OnComplete(decor.AverageSpeed(0, \" %.1f/s\", decor.WCSyncWidthR), \"\"),\n\t\t\tdecor.Any(func(s decor.Statistics) string {\n\t\t\t\tif s.Completed && speedMsg == \"\" {\n\t\t\t\t\tspeed := float64(s.Current) / time.Since(startTime).Seconds()\n\t\t\t\t\tspeedMsg = fmt.Sprintf(\" %.1f/s\", speed)\n\t\t\t\t}\n\t\t\t\treturn speedMsg\n\t\t\t}, decor.WCSyncWidthR),\n\t\t\tdecor.OnComplete(decor.Name(\" ETA: \", decor.WCSyncWidthR), \"\"),\n\t\t\tdecor.OnComplete(\n\t\t\t\tdecor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR), \"\",\n\t\t\t),\n\t\t\tdecor.Any(func(s decor.Statistics) string {\n\t\t\t\tif s.Completed && usedMsg == \"\" {\n\t\t\t\t\tusedMsg = \" used: \" + (time.Since(startTime)).String()\n\t\t\t\t}\n\t\t\t\treturn usedMsg\n\t\t\t}, decor.WCSyncWidthR),\n\t\t),\n\t)\n\tb.SetTotal(total, false)\n\tp.bars = append(p.bars, b)\n\treturn &Bar{Bar: b, total: total}\n}\n\nfunc newSpinner() mpb.BarFiller {\n\tspinnerStyle := []string{\"⠋\", \"⠙\", \"⠹\", \"⠸\", \"⠼\", \"⠴\", \"⠦\", \"⠧\", \"⠇\", \"⠏\"}\n\tfor i, s := range spinnerStyle {\n\t\tspinnerStyle[i] = \"\\033[1;32m\" + s + \"\\033[0m\"\n\t}\n\treturn mpb.NewBarFiller(mpb.SpinnerStyle(spinnerStyle...))\n}\n\nfunc (p *Progress) AddCountSpinner(name string) *Bar {\n\tdecors := []decor.Decorator{\n\t\tdecor.Name(name+\": \", decor.WCSyncWidth),\n\t\tdecor.Merge(decor.CurrentNoUnit(\"%d\", decor.WCSyncSpaceR), decor.WCSyncSpaceR),\n\t}\n\tdecors = append(decors, decor.AverageSpeed(0, \" %.1f/s\", decor.WCSyncSpaceR))\n\tb := p.Progress.Add(0, newSpinner(),\n\t\tmpb.PrependDecorators(decors...),\n\t\tmpb.BarFillerClearOnComplete(),\n\t)\n\tp.bars = append(p.bars, b)\n\treturn &Bar{Bar: b}\n}\n\nfunc (p *Progress) AddByteSpinner(name string) *Bar {\n\tdecors := []decor.Decorator{\n\t\tdecor.Name(name+\": \", decor.WCSyncWidth),\n\t\tdecor.CurrentKibiByte(\"% .1f\", decor.WCSyncSpaceR),\n\t\tdecor.CurrentNoUnit(\"(%d Bytes)\", decor.WCSyncSpaceR),\n\t}\n\t// FIXME: maybe use EWMA speed\n\tdecors = append(decors, decor.AverageSpeed(decor.UnitKiB, \" % .1f\", decor.WCSyncSpaceR))\n\tb := p.Progress.Add(0, newSpinner(),\n\t\tmpb.PrependDecorators(decors...),\n\t\tmpb.BarFillerClearOnComplete(),\n\t)\n\tp.bars = append(p.bars, b)\n\treturn &Bar{Bar: b}\n}\n\nfunc (p *Progress) AddIoSpeedBar(name string, total int64) *Bar {\n\tb := p.Progress.Add(0,\n\t\tmpb.NewBarFiller(mpb.BarStyle()),\n\t\tmpb.PrependDecorators(\n\t\t\tdecor.Name(name+\": \", decor.WCSyncWidth),\n\t\t\tdecor.CountersKibiByte(\"% .1f / % .1f\"),\n\t\t),\n\t\tmpb.AppendDecorators(\n\t\t\tdecor.OnComplete(decor.Percentage(decor.WC{W: 5}), \"done\"),\n\t\t\tdecor.OnComplete(\n\t\t\t\tdecor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 6}), \"\",\n\t\t\t),\n\t\t),\n\t)\n\tb.SetTotal(total, false)\n\tp.bars = append(p.bars, b)\n\treturn &Bar{Bar: b}\n}\n\nfunc (p *Progress) AddDoubleSpinner(name string) *DoubleSpinner {\n\treturn &DoubleSpinner{\n\t\tp.AddCountSpinner(name).Bar,\n\t\tp.AddByteSpinner(name).Bar,\n\t}\n}\n\nfunc (p *Progress) AddDoubleSpinnerTwo(countName, sizeName string) *DoubleSpinner {\n\treturn &DoubleSpinner{\n\t\tp.AddCountSpinner(countName).Bar,\n\t\tp.AddByteSpinner(sizeName).Bar,\n\t}\n}\n\nfunc (p *Progress) Done() {\n\tfor _, b := range p.bars {\n\t\tif !b.Completed() {\n\t\t\tb.SetTotal(0, true)\n\t\t}\n\t}\n\tp.Progress.Wait()\n\tSetOutput(os.Stderr)\n}\n\nfunc MockProgress() (*Progress, *Bar) {\n\tprogress := NewProgress(true)\n\tbar := progress.AddCountBar(\"Mock\", 0)\n\treturn progress, bar\n}\n" }, { "path": "pkg/utils/progress_test.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestProgresBar(t *testing.T) {\n\tp := NewProgress(true)\n\tbar := p.AddCountBar(\"Bar\", 0)\n\tcp := p.AddCountSpinner(\"Spinner\")\n\tbp := p.AddByteSpinner(\"Spinner\")\n\tbar.SetTotal(50)\n\tfor i := 0; i < 100; i++ {\n\t\ttime.Sleep(time.Millisecond)\n\t\tbar.Increment()\n\t\tif i%2 == 0 {\n\t\t\tbar.IncrTotal(1)\n\t\t\tcp.Increment()\n\t\t\tbp.IncrInt64(1024)\n\t\t}\n\t}\n\tbar.Done()\n\tp.Done()\n\tif bar.Current() != 100 || cp.Current() != 50 || bp.Current() != 50*1024 {\n\t\tt.Fatalf(\"Final values: bar %d, count %d, bytes: %d\", bar.Current(), cp.Current(), bp.Current())\n\t}\n\n\tp = NewProgress(true)\n\tdp := p.AddDoubleSpinner(\"Spinner\")\n\tgo func() {\n\t\tfor i := 0; i < 100; i++ {\n\t\t\ttime.Sleep(time.Millisecond)\n\t\t\tdp.IncrInt64(1024)\n\t\t}\n\t\tdp.Done()\n\t}()\n\tp.Wait()\n\tif c, b := dp.Current(); c != 100 || b != 102400 {\n\t\tt.Fatalf(\"Final values: count %d, bytes %d\", c, b)\n\t}\n}\n" }, { "path": "pkg/utils/rusage.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport \"syscall\"\n\ntype Rusage struct {\n\tsyscall.Rusage\n}\n\n// GetUtime returns the user time in seconds.\nfunc (ru *Rusage) GetUtime() float64 {\n\treturn float64(ru.Utime.Sec) + float64(ru.Utime.Usec)/1e6\n}\n\n// GetStime returns the system time in seconds.\nfunc (ru *Rusage) GetStime() float64 {\n\treturn float64(ru.Stime.Sec) + float64(ru.Stime.Usec)/1e6\n}\n\n// GetRusage returns CPU usage of current process.\nfunc GetRusage() *Rusage {\n\tvar ru syscall.Rusage\n\t_ = syscall.Getrusage(syscall.RUSAGE_SELF, &ru)\n\treturn &Rusage{ru}\n}\n" }, { "path": "pkg/utils/rusage_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestRUsage(t *testing.T) {\n\t//u := GetRusage()\n\tvar s string\n\tfor i := 0; i < 1000; i++ {\n\t\ts += time.Now().String()\n\t}\n\t// don't optimize the loop\n\tif len(s) < 10 {\n\t\tpanic(\"unreachable\")\n\t}\n\t_ = GetRusage()\n\t// cancelled due to high machine load\n\t//if u2.GetUtime()-u.GetUtime() < 0.0001 {\n\t//\tt.Fatalf(\"invalid utime: %f\", u2.GetStime()-u.GetStime())\n\t//}\n}\n" }, { "path": "pkg/utils/rusage_windows.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport \"golang.org/x/sys/windows\"\n\ntype Rusage struct {\n\tkernel windows.Filetime\n\tuser windows.Filetime\n}\n\nfunc (ru *Rusage) GetUtime() float64 {\n\treturn float64((int64(ru.user.HighDateTime)<<32)+int64(ru.user.LowDateTime)) / 10 / 1e6\n}\n\nfunc (ru *Rusage) GetStime() float64 {\n\treturn float64((int64(ru.kernel.HighDateTime)<<32)+int64(ru.kernel.LowDateTime)) / 10 / 1e6\n}\n\nfunc GetRusage() *Rusage {\n\th := windows.CurrentProcess()\n\tvar creation, exit, kernel, user windows.Filetime\n\terr := windows.GetProcessTimes(h, &creation, &exit, &kernel, &user)\n\tif err == nil {\n\t\treturn &Rusage{kernel, user}\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/utils/utils.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"context\"\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"mime\"\n\t\"net\"\n\t\"os\"\n\t\"os/user\"\n\t\"path\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/mattn/go-isatty\"\n)\n\n// Exists checks if the file/folder in given path exists\nfunc Exists(path string) bool {\n\t_, err := os.Stat(path)\n\treturn err == nil || !os.IsNotExist(err) //skip mutate\n}\n\n// SplitDir splits a path with default path list separator or comma.\nfunc SplitDir(d string) []string {\n\tdd := strings.Split(d, string(os.PathListSeparator))\n\tif len(dd) == 1 {\n\t\tdd = strings.Split(dd[0], \",\")\n\t}\n\treturn dd\n}\n\n// GetLocalIp get the local ip used to access remote address.\nfunc GetLocalIp(address string) (string, error) {\n\tconn, err := net.Dial(\"udp\", address)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tip, _, err := net.SplitHostPort(conn.LocalAddr().String())\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn ip, nil\n}\n\nfunc FindLocalIPs(allowedInterfaces ...string) ([]net.IP, error) {\n\tifaces, err := net.Interfaces()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Build a set of allowed interface names for fast lookup\n\tallowedSet := make(map[string]bool)\n\tfor _, name := range allowedInterfaces {\n\t\tallowedSet[name] = true\n\t}\n\tcheckAllowed := len(allowedSet) > 0\n\n\tvar ips []net.IP\n\tfor _, iface := range ifaces {\n\t\tif iface.Flags&net.FlagUp == 0 {\n\t\t\tcontinue // interface down\n\t\t}\n\t\tif iface.Flags&net.FlagLoopback != 0 {\n\t\t\tcontinue // loopback interface\n\t\t}\n\t\t// Filter by interface name if allowedInterfaces is specified\n\t\tif checkAllowed && !allowedSet[iface.Name] {\n\t\t\tcontinue\n\t\t}\n\t\taddrs, err := iface.Addrs()\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, addr := range addrs {\n\t\t\tvar ip net.IP\n\t\t\tswitch v := addr.(type) {\n\t\t\tcase *net.IPNet:\n\t\t\t\tip = v.IP\n\t\t\tcase *net.IPAddr:\n\t\t\t\tip = v.IP\n\t\t\t}\n\t\t\tif len(ip) > 0 && !ip.IsLoopback() {\n\t\t\t\tips = append(ips, ip)\n\t\t\t}\n\t\t}\n\t}\n\treturn ips, nil\n}\n\nfunc WithTimeout(pCtx context.Context, f func(context.Context) error, timeout time.Duration) error {\n\tvar done = make(chan int, 1)\n\tvar t = time.NewTimer(timeout)\n\tvar err error\n\tctx, cancel := context.WithCancel(pCtx)\n\tgo func() {\n\t\terr = f(ctx)\n\t\tdone <- 1\n\t}()\n\tselect {\n\tcase <-ctx.Done():\n\t\terr = ctx.Err()\n\t\tt.Stop()\n\tcase <-done:\n\t\tt.Stop()\n\tcase <-t.C:\n\t\terr = fmt.Errorf(\"timeout after %s: %w\", timeout, ErrFuncTimeout)\n\t}\n\tcancel()\n\treturn err\n}\n\nfunc RemovePassword(uri string) string {\n\tp := strings.LastIndex(uri, \"@\")\n\tif p < 0 {\n\t\treturn uri\n\t}\n\tsp := strings.Index(uri, \"://\") + 3\n\tif sp == 2 {\n\t\tsp = 0\n\t}\n\tcp := strings.Index(uri[sp:], \":\")\n\tif cp < 0 || sp+cp > p {\n\t\treturn uri\n\t}\n\treturn uri[:sp+cp] + \":****\" + uri[p:]\n}\n\nfunc GuessMimeType(key string) string {\n\tmimeType := mime.TypeByExtension(path.Ext(key))\n\tif !strings.ContainsRune(mimeType, '/') {\n\t\tmimeType = \"application/octet-stream\"\n\t}\n\treturn mimeType\n}\n\nfunc StringContains(s []string, e string) bool {\n\tfor _, item := range s {\n\t\tif item == e {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc FormatBytes(n uint64) string {\n\tif n < 1024 {\n\t\treturn fmt.Sprintf(\"%d Bytes\", n)\n\t}\n\tunits := []string{\"K\", \"M\", \"G\", \"T\", \"P\", \"E\"}\n\tm := n\n\ti := 0\n\tfor ; i < len(units)-1 && m >= 1<<20; i++ {\n\t\tm = m >> 10\n\t}\n\treturn fmt.Sprintf(\"%.2f %siB (%d Bytes)\", float64(m)/1024.0, units[i], n)\n}\n\nfunc SupportANSIColor(fd uintptr) bool {\n\treturn isatty.IsTerminal(fd) && runtime.GOOS != \"windows\"\n}\n\nfunc RandRead(buf []byte) {\n\tif _, err := rand.Read(buf); err != nil {\n\t\tlogger.Fatalf(\"Generate random content: %s\", err)\n\t}\n}\n\nvar uids = make(map[int]string)\nvar gids = make(map[int]string)\nvar users = make(map[string]int)\nvar groups = make(map[string]int)\nvar mutex sync.Mutex\n\nvar logger = GetLogger(\"juicefs\")\n\nfunc UserName(uid int) string {\n\tmutex.Lock()\n\tdefer mutex.Unlock()\n\tname, ok := uids[uid]\n\tif !ok {\n\t\tif u, err := user.LookupId(strconv.Itoa(uid)); err == nil {\n\t\t\tname = u.Username\n\t\t} else {\n\t\t\tlogger.Warnf(\"lookup uid %d: %s\", uid, err)\n\t\t\tname = strconv.Itoa(uid)\n\t\t}\n\t\tuids[uid] = name\n\t}\n\treturn name\n}\n\nfunc GroupName(gid int) string {\n\tmutex.Lock()\n\tdefer mutex.Unlock()\n\tname, ok := gids[gid]\n\tif !ok {\n\t\tif g, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil {\n\t\t\tname = g.Name\n\t\t} else {\n\t\t\tlogger.Warnf(\"lookup gid %d: %s\", gid, err)\n\t\t\tname = strconv.Itoa(gid)\n\t\t}\n\t\tgids[gid] = name\n\t}\n\treturn name\n}\n\nfunc LookupUser(name string) int {\n\tmutex.Lock()\n\tdefer mutex.Unlock()\n\tif u, ok := users[name]; ok {\n\t\treturn u\n\t}\n\tvar uid = -1\n\tif u, err := user.Lookup(name); err == nil {\n\t\tuid, _ = strconv.Atoi(u.Uid)\n\t} else {\n\t\tif g, e := strconv.Atoi(name); e == nil {\n\t\t\tuid = g\n\t\t} else {\n\t\t\tlogger.Warnf(\"lookup user %s: %s\", name, err)\n\t\t}\n\t}\n\tusers[name] = uid\n\treturn uid\n}\n\nfunc LookupGroup(name string) int {\n\tmutex.Lock()\n\tdefer mutex.Unlock()\n\tif u, ok := groups[name]; ok {\n\t\treturn u\n\t}\n\tvar gid = -1\n\tif u, err := user.LookupGroup(name); err == nil {\n\t\tgid, _ = strconv.Atoi(u.Gid)\n\t} else {\n\t\tif g, e := strconv.Atoi(name); e == nil {\n\t\t\tgid = g\n\t\t} else {\n\t\t\tlogger.Warnf(\"lookup group %s: %s\", name, err)\n\t\t}\n\t}\n\tgroups[name] = gid\n\treturn gid\n}\n\nfunc Duration(s string) time.Duration {\n\tif s == \"\" {\n\t\treturn 0\n\t}\n\tv, err := strconv.ParseFloat(s, 64)\n\tif err == nil {\n\t\treturn time.Microsecond * time.Duration(v*1e6)\n\t}\n\n\terr = nil\n\tvar d time.Duration\n\tp := strings.Index(s, \"d\")\n\tif p >= 0 {\n\t\tv, err = strconv.ParseFloat(s[:p], 64)\n\t}\n\tif err == nil && s[p+1:] != \"\" {\n\t\td, err = time.ParseDuration(s[p+1:])\n\t}\n\n\tif err != nil {\n\t\tlogger.Warnf(\"Invalid duration value: %s, setting it to 0\", s)\n\t\treturn 0\n\t}\n\treturn d + time.Hour*time.Duration(v*24)\n}\n" }, { "path": "pkg/utils/utils_darwin.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"os/exec\"\n)\n\nfunc GetKernelVersion() (major, minor int) { return }\n\nfunc GetSysInfo() string {\n\tvar (\n\t\tkernel string\n\t\tosVersion []byte\n\t\thardware []byte\n\t)\n\n\tkernel, _ = GetKernelInfo()\n\n\tosVersion, _ = exec.Command(\"sw_vers\").Output()\n\n\thardware, _ = exec.Command(\"system_profiler\", \"SPMemoryDataType\", \"SPStorageDataType\").Output()\n\n\treturn fmt.Sprintf(`\nKernel: \n%s\nOS: \n%s\nHardware: \n%s`, kernel, string(osVersion), string(hardware))\n}\n\nfunc SetIOFlusher() {}\n\nfunc DisableTHP() {}\n\nfunc AdjustOOMKiller(score int) {}\n" }, { "path": "pkg/utils/utils_linux.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\nfunc GetKernelVersion() (major, minor int) {\n\tvar uname syscall.Utsname\n\tif err := syscall.Uname(&uname); err == nil {\n\t\tbuf := make([]byte, 0, 65) // Utsname.Release [65]int8\n\t\tfor _, v := range uname.Release {\n\t\t\tif v == 0x00 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tbuf = append(buf, byte(v))\n\t\t}\n\t\tps := strings.SplitN(string(buf), \".\", 3)\n\t\tif len(ps) < 2 {\n\t\t\treturn\n\t\t}\n\t\tif major, err = strconv.Atoi(ps[0]); err != nil {\n\t\t\treturn\n\t\t}\n\t\tminor, _ = strconv.Atoi(ps[1])\n\t}\n\treturn\n}\n\nfunc GetSysInfo() string {\n\tvar (\n\t\tkernel []byte\n\t\tosVersion []byte\n\t\terr error\n\t)\n\n\tkernel, _ = exec.Command(\"cat\", \"/proc/version\").Output()\n\n\tif osVersion, err = exec.Command(\"lsb_release\", \"-a\").Output(); err != nil {\n\t\tosVersion, _ = exec.Command(\"cat\", \"/etc/os-release\").Output()\n\t}\n\n\treturn fmt.Sprintf(`\nKernel: \n%s\nOS: \n%s`, kernel, osVersion)\n}\n\nfunc SetIOFlusher() {\n\terr := unix.Prctl(unix.PR_SET_IO_FLUSHER, 1, 0, 0, 0)\n\tif errors.Is(err, unix.EPERM) {\n\t\tlogger.Warn(\"CAP_SYS_RESOURCE is needed for PR_SET_IO_FLUSHER\")\n\t} else if errors.Is(err, unix.EINVAL) {\n\t\tlogger.Info(\"PR_SET_IO_FLUSHER, which is introduced by Linux 5.6, is not supported by the running kernel\")\n\t}\n}\n\n// Disable transparent huge page\nfunc DisableTHP() {\n\tfor {\n\t\terr := unix.Prctl(unix.PR_SET_THP_DISABLE, 1, 0, 0, 0)\n\t\tif err == nil {\n\t\t\tlogger.Info(\"Disabled transparent hugepage\")\n\t\t\tbreak\n\t\t}\n\n\t\tif errors.Is(err, unix.EINTR) {\n\t\t\tcontinue\n\t\t} else {\n\t\t\tlogger.Warnf(\"Failed to disable transparent huge page: %s\", err)\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// AdjustOOMKiller: change oom_score_adj to avoid OOM-killer\nfunc AdjustOOMKiller(score int) {\n\tif os.Getuid() != 0 {\n\t\treturn\n\t}\n\tf, err := os.OpenFile(\"/proc/self/oom_score_adj\", os.O_WRONLY, 0666)\n\tif err != nil {\n\t\tif !os.IsNotExist(err) {\n\t\t\tprintln(err)\n\t\t}\n\t\treturn\n\t}\n\tdefer f.Close()\n\t_, err = f.WriteString(strconv.Itoa(score))\n\tif err != nil {\n\t\tprintln(\"adjust OOM score:\", err)\n\t}\n}\n" }, { "path": "pkg/utils/utils_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"context\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\n// mutate_test_job_number: 2\n// checksum 9cb13bb28aa7918edaf4f0f4ca92eea5\n// checksum 05debda2840d31bac0ab5c20c5510591\nfunc TestMin(t *testing.T) {\n\tassertEqual(t, min(1, 2), 1)\n\tassertEqual(t, min(-1, -2), -2)\n\tassertEqual(t, min(0, 0), 0)\n}\n\nfunc TestExists(t *testing.T) {\n\tassertEqual(t, Exists(\"/\"), true)\n\tassertEqual(t, Exists(\"/not_exist_path\"), false)\n}\n\nfunc TestSplitDir(t *testing.T) {\n\tassertEqual(t, SplitDir(\"/a:/b\"), []string{\"/a\", \"/b\"})\n\tassertEqual(t, SplitDir(\"a,/b\"), []string{\"a\", \"/b\"})\n\tassertEqual(t, SplitDir(\"/a;b\"), []string{\"/a;b\"})\n\tassertEqual(t, SplitDir(\"a/b\"), []string{\"a/b\"})\n}\n\nfunc TestGetInode(t *testing.T) {\n\t_, err := GetFileInode(\"\")\n\tif err == nil {\n\t\tt.Fatalf(\"invalid path should fail\")\n\t}\n\tino, err := GetFileInode(\"/\")\n\tif err != nil {\n\t\tt.Fatalf(\"get file inode: %s\", err)\n\t} else if ino > 2 {\n\t\tt.Fatalf(\"inode of root should be 1/2, but got %d\", ino)\n\t}\n}\n\nfunc TestLocalIp(t *testing.T) {\n\t_, err := GetLocalIp(\"127.0.0.1\")\n\tif err == nil {\n\t\tt.Fatalf(\"should fail with invalid address\")\n\t}\n\tip, err := GetLocalIp(\"127.0.0.1:22\")\n\tif err != nil {\n\t\tt.Fatalf(\"get local ip: %s\", err)\n\t}\n\tif ip != \"127.0.0.1\" {\n\t\tt.Fatalf(\"local ip should be 127.0.0.1, bug got %s\", ip)\n\t}\n}\n\nfunc TestFindLocalIPs(t *testing.T) {\n\t// Test without interface filter (should return all IPs)\n\tips, err := FindLocalIPs()\n\tif err != nil {\n\t\tt.Fatalf(\"FindLocalIPs failed: %s\", err)\n\t}\n\tif len(ips) == 0 {\n\t\tt.Logf(\"Warning: No network interfaces found (this might be expected in some environments)\")\n\t}\n\n\t// Test with non-existent interface filter (should return no IPs)\n\tips, err = FindLocalIPs(\"nonexistent_interface_12345\")\n\tif err != nil {\n\t\tt.Fatalf(\"FindLocalIPs with filter failed: %s\", err)\n\t}\n\tif len(ips) != 0 {\n\t\tt.Fatalf(\"Expected 0 IPs with non-existent interface, got %d\", len(ips))\n\t}\n\n\t// Test with multiple interface filters\n\tips, err = FindLocalIPs(\"eth0\", \"en0\", \"lo0\")\n\tif err != nil {\n\t\tt.Fatalf(\"FindLocalIPs with multiple filters failed: %s\", err)\n\t}\n\t// We don't assert length here since it depends on the system\n\tt.Logf(\"Found %d IPs with eth0/en0/lo0 filter\", len(ips))\n}\n\nfunc TestTimeout(t *testing.T) {\n\terr := WithTimeout(context.TODO(), func(context.Context) error {\n\t\treturn nil\n\t}, time.Millisecond*10)\n\tif err != nil {\n\t\tt.Fatalf(\"fast function should return nil\")\n\t}\n\terr = WithTimeout(context.TODO(), func(context.Context) error {\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\treturn nil\n\t}, time.Millisecond*10)\n\tif err == nil || !strings.HasPrefix(err.Error(), \"timeout after\") {\n\t\tt.Fatalf(\"slow function should be timeout: %s\", err)\n\t}\n}\n\nfunc TestRemovePassword(t *testing.T) {\n\ttestCase := []struct {\n\t\turi string\n\t\texpected string\n\t}{\n\t\t{\"redis://:password@localhost:6379/0\",\n\t\t\t\"redis://:****@localhost:6379/0\",\n\t\t},\n\t\t{\"redis://:pass@word@localhost:6379/0\",\n\t\t\t\"redis://:****@localhost:6379/0\",\n\t\t},\n\t\t{\":password@localhost:6379/0\",\n\t\t\t\":****@localhost:6379/0\",\n\t\t},\n\t\t{\"oss://ak:sk@zhijian-test2.oss-cn-hangzhou.aliyuncs.com\",\n\t\t\t\"oss://ak:****@zhijian-test2.oss-cn-hangzhou.aliyuncs.com\",\n\t\t},\n\t\t{\"/tmp/file\",\n\t\t\t\"/tmp/file\",\n\t\t},\n\t\t{\"file:///tmp/file\",\n\t\t\t\"file:///tmp/file\",\n\t\t},\n\t\t{\"sftp:///tmp/file\",\n\t\t\t\"sftp:///tmp/file\",\n\t\t},\n\t}\n\tfor _, tc := range testCase {\n\t\tassertEqual(t, RemovePassword(tc.uri), tc.expected)\n\t}\n}\n" }, { "path": "pkg/utils/utils_unix.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\nfunc GetCurrentUID() int {\n\treturn os.Getuid()\n}\n\nfunc GetCurrentGID() int {\n\treturn os.Getgid()\n}\n\nfunc GetCurrentUserSIDStr() string {\n\treturn \"\"\n}\n\nfunc GetCurrentUserGroupSIDStr() string {\n\treturn \"\"\n}\n\nfunc IsWinAdminOrElevatedPrivilege() bool {\n\treturn false\n}\n\nfunc GetFileInode(path string) (uint64, error) {\n\tfi, err := os.Stat(path)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn sst.Ino, nil\n\t}\n\treturn 0, nil\n}\n\nfunc GetFileInodeNotFollow(path string) (uint64, error) {\n\tfi, err := os.Lstat(path)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn sst.Ino, nil\n\t}\n\treturn 0, nil\n}\n\nfunc GetDev(fpath string) int { // ID of device containing file\n\tfi, err := os.Stat(fpath)\n\tif err != nil {\n\t\treturn -1\n\t}\n\tif sst, ok := fi.Sys().(*syscall.Stat_t); ok {\n\t\treturn int(sst.Dev)\n\t}\n\treturn -1\n}\n\nfunc GetKernelInfo() (string, error) {\n\tkernel, err := exec.Command(\"uname\", \"-a\").Output()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\t// Ignore hostname information\n\ttmp := strings.Split(string(kernel), \" \")\n\tresult := strings.Join(append(tmp[:1], tmp[2:]...), \" \")\n\treturn result, nil\n}\n\nfunc GetUmask() int {\n\tumask := syscall.Umask(0)\n\tsyscall.Umask(umask)\n\treturn umask\n}\n\nfunc SetUmask(umask int) int {\n\treturn syscall.Umask(umask)\n}\n\nfunc ErrnoName(err syscall.Errno) string {\n\terrName := unix.ErrnoName(err)\n\tif errName == \"\" {\n\t\terrName = strconv.Itoa(int(err))\n\t}\n\treturn errName\n}\n" }, { "path": "pkg/utils/utils_windows.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"syscall\"\n\n\t\"github.com/juicedata/juicefs/pkg/win\"\n\t\"golang.org/x/sys/windows\"\n)\n\nfunc GetCurrentUID() int {\n\treturn win.GetCurrentUID()\n}\n\nfunc GetCurrentGID() int {\n\treturn win.GetCurrentGID()\n}\n\nfunc GetCurrentUserSIDStr() string {\n\tsid, err := win.GetCurrentUserSID()\n\tif err != nil {\n\t\tlogger.Warnf(\"failed to get sid for current user, %s\", err)\n\t\treturn \"\"\n\t}\n\n\treturn fmt.Sprintf(\"%s (%s)\", sid.String(), win.GetSidName(sid, true))\n}\n\nfunc GetCurrentUserGroupSIDStr() string {\n\tsid, err := win.GetCurrentUserPrimaryGroupSID()\n\tif err != nil {\n\t\tlogger.Warnf(\"failed to get sid for current user, %s\", err)\n\t\treturn \"\"\n\t}\n\n\treturn fmt.Sprintf(\"%s (%s)\", sid.String(), win.GetSidName(sid, true))\n}\n\nfunc IsWinAdminOrElevatedPrivilege() bool {\n\tuid := GetCurrentUID()\n\tif uid == win.AdministratorUIDFromFUSE {\n\t\treturn true\n\t}\n\televated, err := win.IsProcessElevated()\n\tif err != nil {\n\t\tlogger.Warnf(\"failed to determine if process is elevated, %s\", err)\n\t\treturn false\n\t}\n\treturn elevated\n}\n\nfunc getFileInode(path string, follow bool) (uint64, error) {\n\tpathU16, err := windows.UTF16PtrFromString(path)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tvar flagsAndAttributes uint32 = windows.FILE_FLAG_BACKUP_SEMANTICS\n\tif !follow {\n\t\tflagsAndAttributes |= windows.FILE_FLAG_OPEN_REPARSE_POINT\n\t}\n\tfd, err := windows.CreateFile(pathU16, windows.GENERIC_READ, windows.FILE_SHARE_READ, nil, windows.OPEN_EXISTING, flagsAndAttributes, 0)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tdefer windows.Close(fd)\n\tvar data windows.ByHandleFileInformation\n\terr = windows.GetFileInformationByHandle(fd, &data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn uint64(data.FileIndexHigh)<<32 + uint64(data.FileIndexLow), nil\n}\n\nfunc GetFileInode(path string) (uint64, error) {\n\treturn getFileInode(path, true)\n}\n\nfunc GetFileInodeNotFollow(path string) (uint64, error) {\n\treturn getFileInode(path, false)\n}\n\nfunc GetKernelVersion() (major, minor int) { return }\n\nfunc GetDev(fpath string) int { return -1 }\n\nfunc GetSysInfo() string {\n\tsysInfo, _ := exec.Command(\"systeminfo\").Output()\n\treturn string(sysInfo)\n}\n\nfunc GetUmask() int { return 0 }\n\nfunc SetUmask(umask int) int {\n\treturn 0\n}\n\nfunc ErrnoName(err syscall.Errno) string {\n\treturn strconv.Itoa(int(err))\n}\n" }, { "path": "pkg/version/.gitattributes", "content": "version.go export-subst\n" }, { "path": "pkg/version/version.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n// Reference: https://semver.org; NOT strictly followed.\npackage version\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n)\n\nvar (\n\trevision = \"$Format:%h$\" // value is assigned in Makefile\n\trevisionDate = \"$Format:%as$\"\n\tver = Semver{\n\t\tmajor: 1,\n\t\tminor: 4,\n\t\tpatch: 0,\n\t\tpreRelease: \"dev\",\n\t\tbuild: fmt.Sprintf(\"%s.%s\", revisionDate, revision),\n\t}\n)\n\ntype Semver struct {\n\tmajor, minor, patch uint64\n\tpreRelease, build string\n}\n\nfunc (s *Semver) String() string {\n\tpr := s.preRelease\n\tif pr != \"\" {\n\t\tpr = \"-\" + pr\n\t}\n\tif strings.Contains(s.build, \"Format\") {\n\t\ts.build = \"unknown\"\n\t}\n\treturn fmt.Sprintf(\"%d.%d.%d%s+%s\", s.major, s.minor, s.patch, pr, s.build)\n}\n\nfunc Version() string {\n\treturn ver.String()\n}\n\nfunc SetVersion(v string) {\n\tver = *Parse(v)\n}\n\nfunc GetVersion() Semver {\n\treturn ver\n}\n\nfunc CompareVersions(v1, v2 *Semver) (int, error) {\n\tif v1 == nil || v2 == nil {\n\t\treturn 0, fmt.Errorf(\"v1 %v and v2 %v can't be nil\", v1, v2)\n\t}\n\tvar less bool\n\tif v1.major != v2.major {\n\t\tless = v1.major < v2.major\n\t} else if v1.minor != v2.minor {\n\t\tless = v1.minor < v2.minor\n\t} else if v1.patch != v2.patch {\n\t\tless = v1.patch < v2.patch\n\t} else if v1.preRelease != v2.preRelease {\n\t\tless = v1.preRelease < v2.preRelease\n\t\tif v1.preRelease == \"\" || v2.preRelease == \"\" {\n\t\t\tless = !less\n\t\t}\n\t} else {\n\t\treturn 0, nil\n\t}\n\tif less {\n\t\treturn -1, nil\n\t} else {\n\t\treturn 1, nil\n\t}\n}\n\nfunc Parse(vs string) *Semver {\n\tif p := strings.Index(vs, \"+\"); p > 0 {\n\t\tvs = vs[:p] // ignore build information\n\t}\n\tvar v Semver\n\tif p := strings.Index(vs, \"-\"); p > 0 {\n\t\tv.preRelease = vs[p+1:]\n\t\tvs = vs[:p]\n\t}\n\n\tps := strings.Split(vs, \".\")\n\tif len(ps) > 3 {\n\t\treturn nil\n\t}\n\tvar err error\n\tif v.major, err = strconv.ParseUint(ps[0], 10, 64); err != nil {\n\t\treturn nil\n\t}\n\tif len(ps) > 1 {\n\t\tif v.minor, err = strconv.ParseUint(ps[1], 10, 64); err != nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\tif len(ps) > 2 {\n\t\tif v.patch, err = strconv.ParseUint(ps[2], 10, 64); err != nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn &v\n}\n" }, { "path": "pkg/version/version_test.go", "content": "/*\n * JuiceFS, Copyright 2022 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage version\n\nimport \"testing\"\n\nfunc TestVersion(t *testing.T) {\n\tver = Semver{\n\t\tmajor: 1,\n\t\tminor: 0,\n\t\tpatch: 0,\n\t\tbuild: \"2022-02-22.f4692af9\",\n\t}\n\tif v := Version(); v != \"1.0.0+2022-02-22.f4692af9\" {\n\t\tt.Fatalf(\"Version %s != expected 1.0.0+2022-02-22.f4692af9\", v)\n\t}\n\tif _, err := CompareVersions(&ver, Parse(\"\")); err == nil {\n\t\tt.Fatalf(\"Expect failed to parse empty string\")\n\t}\n\tif _, err := CompareVersions(&ver, Parse(\"0.1.2.3\")); err == nil {\n\t\tt.Fatalf(\"Expect failed to parse string \\\"0.1.2.3\\\"\")\n\t}\n\n\tcases := []struct {\n\t\tvs string\n\t\texpect int\n\t}{\n\t\t{\"0.9+foo.bar\", 1},\n\t\t{\"0.9.10\", 1},\n\t\t{\"1.0-beta+baz\", 1},\n\t\t{\"1\", 0},\n\t\t{\"1.1\", -1},\n\t\t{\"2.0.0-alpha\", -1},\n\t}\n\tfor _, c := range cases {\n\t\tif r, _ := CompareVersions(&ver, Parse(c.vs)); r != c.expect {\n\t\t\tt.Fatalf(\"Failed case: %+v\", c)\n\t\t}\n\t}\n\n\tver.preRelease = \"beta\"\n\tif v := Version(); v != \"1.0.0-beta+2022-02-22.f4692af9\" {\n\t\tt.Fatalf(\"Version %s != expected 1.0.0-beta+2022-02-22.f4692af9\", v)\n\t}\n\tcases[2].expect = 0\n\tcases[3].expect = -1\n\tfor _, c := range cases {\n\t\tif r, _ := CompareVersions(&ver, Parse(c.vs)); r != c.expect {\n\t\t\tt.Fatalf(\"Failed case: %+v\", c)\n\t\t}\n\t}\n}\n" }, { "path": "pkg/vfs/accesslog.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nvar (\n\topsDurationsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"fuse_ops_durations_histogram_seconds\",\n\t\tHelp: \"Operations latency distributions.\",\n\t\tBuckets: prometheus.ExponentialBuckets(0.00001, 1.8, 29), // should cover range of `objectReqsHistogram`\n\t})\n\topsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{\n\t\tName: \"fuse_ops_total\",\n\t\tHelp: \"Total number of operations.\",\n\t}, []string{\"method\"})\n\topsDurations = prometheus.NewCounterVec(prometheus.CounterOpts{\n\t\tName: \"fuse_ops_durations_seconds\",\n\t\tHelp: \"Operations latency in seconds.\",\n\t}, []string{\"method\"})\n\topsIOErrors = prometheus.NewCounterVec(prometheus.CounterOpts{\n\t\tName: \"fuse_ops_io_errors\",\n\t\tHelp: \"Number of IO errors.\",\n\t}, []string{\"errno\"})\n)\n\ntype logReader struct {\n\tsync.Mutex\n\tbuffer chan []byte\n\tlast []byte\n}\n\nvar (\n\treaderLock sync.RWMutex\n\treaders map[uint64]*logReader\n)\n\nfunc init() {\n\treaders = make(map[uint64]*logReader)\n}\n\nfunc logit(ctx Context, method string, err syscall.Errno, format string, args ...interface{}) {\n\tused := ctx.Duration()\n\topsDurationsHistogram.Observe(used.Seconds())\n\topsTotal.WithLabelValues(method).Inc()\n\topsDurations.WithLabelValues(method).Add(used.Seconds())\n\tif err != 0 {\n\t\topsIOErrors.WithLabelValues(utils.ErrnoName(err)).Inc()\n\t}\n\treaderLock.RLock()\n\tdefer readerLock.RUnlock()\n\tif len(readers) == 0 && used < time.Second*10 {\n\t\treturn\n\t}\n\tfor i, a := range args {\n\t\tswitch v := a.(type) {\n\t\tcase string:\n\t\t\tif !strconv.CanBackquote(v) {\n\t\t\t\targs[i] = strings.Trim(strconv.Quote(v), \"\\\"\")\n\t\t\t}\n\t\t}\n\t}\n\tcmd := fmt.Sprintf(method+\" \"+format, args...)\n\tt := utils.Now()\n\tts := t.Format(\"2006.01.02 15:04:05.000000\")\n\tcmd += fmt.Sprintf(\" - %s <%.6f>\", strerr(err), used.Seconds())\n\tif ctx.Pid() != 0 && used >= time.Second*10 {\n\t\tlogger.Infof(\"slow operation: %s\", cmd)\n\t}\n\tline := []byte(fmt.Sprintf(\"%s [uid:%d,gid:%d,pid:%d] %s\\n\", ts, ctx.Uid(), ctx.Gid(), ctx.Pid(), cmd))\n\n\tfor _, r := range readers {\n\t\tselect {\n\t\tcase r.buffer <- line:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc openAccessLog(fh uint64) uint64 {\n\treaderLock.Lock()\n\tdefer readerLock.Unlock()\n\treaders[fh] = &logReader{buffer: make(chan []byte, 10240)}\n\treturn fh\n}\n\nfunc closeAccessLog(fh uint64) {\n\treaderLock.Lock()\n\tdefer readerLock.Unlock()\n\tdelete(readers, fh)\n}\n\nfunc readAccessLog(fh uint64, buf []byte) int {\n\treaderLock.RLock()\n\tr, ok := readers[fh]\n\treaderLock.RUnlock()\n\tif !ok {\n\t\treturn 0\n\t}\n\tr.Lock()\n\tdefer r.Unlock()\n\tvar n int\n\tif len(r.last) > 0 {\n\t\tn = copy(buf, r.last)\n\t\tr.last = r.last[n:]\n\t}\n\tvar t = time.NewTimer(time.Second)\n\tdefer t.Stop()\n\tfor n < len(buf) {\n\t\tselect {\n\t\tcase line := <-r.buffer:\n\t\t\tl := copy(buf[n:], line)\n\t\t\tn += l\n\t\t\tif l < len(line) {\n\t\t\t\tr.last = line[l:]\n\t\t\t}\n\t\tcase <-t.C:\n\t\t\tif n == 0 {\n\t\t\t\tn = copy(buf, \"#\\n\")\n\t\t\t}\n\t\t\treturn n\n\t\t}\n\t}\n\treturn n\n}\n" }, { "path": "pkg/vfs/accesslog_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\nfunc TestAccessLog(t *testing.T) {\n\topenAccessLog(1)\n\tdefer closeAccessLog(1)\n\n\tctx := NewLogContext(meta.NewContext(10, 1, []uint32{2}))\n\tlogit(ctx, \"method\", 0, \"test\")\n\n\tn := readAccessLog(2, nil)\n\tif n != 0 {\n\t\tt.Fatalf(\"invalid fd\")\n\t}\n\n\tnow := time.Now()\n\t// partial read\n\tbuf := make([]byte, 1024)\n\tn = readAccessLog(1, buf[:10])\n\tif n != 10 {\n\t\tt.Fatalf(\"partial read: %d\", n)\n\t}\n\tif time.Since(now) > time.Millisecond*10 {\n\t\tt.Fatalf(\"should not block\")\n\t}\n\n\t// read whole line, block for 1 second\n\tn = readAccessLog(1, buf[10:])\n\tif n != 66 {\n\t\tt.Fatalf(\"partial read: %d\", n)\n\t}\n\tlogs := string(buf[:10+n])\n\n\t// check format\n\tts, err := time.Parse(\"2006.01.02 15:04:05.000000\", logs[:26])\n\tif err != nil {\n\t\tt.Fatalf(\"invalid time %s: %s\", logs, err)\n\t}\n\tif now.Sub(ts.Local()) > time.Millisecond*10 {\n\t\tt.Fatalf(\"stale time: %s now: %s\", ts, time.Now())\n\t}\n\tif logs[26:len(logs)-4] != \" [uid:1,gid:2,pid:10] method test - OK <0.0000\" {\n\t\tt.Fatalf(\"unexpected log: %q\", logs[26:])\n\t}\n\n\t// block read\n\tn = readAccessLog(1, buf)\n\tif n != 2 || string(buf[:2]) != \"#\\n\" {\n\t\tt.Fatalf(\"expected line: %q\", string(buf[:n]))\n\t}\n}\n" }, { "path": "pkg/vfs/backup.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"compress/gzip\"\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"sort\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\tosync \"github.com/juicedata/juicefs/pkg/sync\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nvar (\n\tLastBackupTimeG = prometheus.NewGauge(prometheus.GaugeOpts{\n\t\tName: \"last_successful_backup\",\n\t\tHelp: \"Last successful backup.\",\n\t})\n\tLastBackupDurationG = prometheus.NewGauge(prometheus.GaugeOpts{\n\t\tName: \"last_backup_duration\",\n\t\tHelp: \"Last backup duration.\",\n\t})\n)\n\n// Backup metadata periodically in the object storage\nfunc Backup(m meta.Meta, blob object.ObjectStorage, interval time.Duration, skipTrash bool) {\n\tctx := meta.Background()\n\tkey := \"lastBackup\"\n\tfor {\n\t\tutils.SleepWithJitter(interval / 10)\n\t\tvar value []byte\n\t\tif st := m.GetXattr(ctx, 0, key, &value); st != 0 && st != meta.ENOATTR {\n\t\t\tlogger.Warnf(\"getxattr inode 1 key %s: %s\", key, st)\n\t\t\tcontinue\n\t\t}\n\t\tvar last time.Time\n\t\tvar err error\n\t\tif len(value) > 0 {\n\t\t\tlast, err = time.Parse(time.RFC3339, string(value))\n\t\t}\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"parse time value %s: %s\", value, err)\n\t\t\tcontinue\n\t\t}\n\t\tif now := time.Now(); now.Sub(last) >= interval {\n\t\t\tvar iused, dummy uint64\n\t\t\t_ = m.StatFS(ctx, meta.RootInode, &dummy, &dummy, &iused, &dummy)\n\t\t\tif interval <= time.Hour {\n\t\t\t\tif iused > 1e6 {\n\t\t\t\t\tlogger.Warnf(\"backup metadata skipped because of too many inodes: %d %s; \"+\n\t\t\t\t\t\t\"you may increase `--backup-meta` to enable it again\", iused, interval)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tif st := m.SetXattr(ctx, 0, key, []byte(now.Format(time.RFC3339)), meta.XattrCreateOrReplace); st != 0 {\n\t\t\t\tlogger.Warnf(\"setxattr inode 1 key %s: %s\", key, st)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif iused >= 1e5 {\n\t\t\t\tlogger.Infof(\"backup metadata started, inodes=%d\", iused)\n\t\t\t}\n\t\t\tif fpath, err := backup(m, blob, now, iused < 1e5, skipTrash); err == nil {\n\t\t\t\tgo cleanupBackups(blob, now) // only cleanup on success\n\t\t\t\tLastBackupTimeG.Set(float64(now.UnixNano()) / 1e9)\n\t\t\t\tlogger.Infof(\"backup metadata succeed, fast mode: %v, path: %q, used %s\", iused < 1e5, fpath, time.Since(now))\n\t\t\t} else {\n\t\t\t\tlogger.Warnf(\"backup metadata failed: %s\", err)\n\t\t\t}\n\t\t\tLastBackupDurationG.Set(time.Since(now).Seconds())\n\t\t} else {\n\t\t\tLastBackupDurationG.Set(0)\n\t\t}\n\t}\n}\n\nfunc backup(m meta.Meta, blob object.ObjectStorage, now time.Time, fast, skipTrash bool) (string, error) {\n\tname := \"dump-\" + now.UTC().Format(\"2006-01-02-150405\") + \".json.gz\"\n\tlocalDir := os.TempDir()\n\tif !strings.HasSuffix(localDir, \"/\") {\n\t\tlocalDir += \"/\"\n\t}\n\tfp, err := os.Create(filepath.Join(localDir, \"meta\", name))\n\tif errors.Is(err, syscall.ENOENT) || (errors.Is(err, syscall.ENOTDIR) && runtime.GOOS == \"windows\") {\n\t\tif err = os.MkdirAll(filepath.Join(localDir, \"meta\"), 0755); err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tfp, err = os.Create(filepath.Join(localDir, \"meta\", name))\n\t}\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer os.Remove(fp.Name())\n\tdefer fp.Close()\n\tzw, _ := gzip.NewWriterLevel(fp, gzip.BestSpeed)\n\tvar threads = 2\n\tif m.Name() == \"tikv\" {\n\t\tthreads = 10\n\t}\n\terr = m.DumpMeta(zw, 0, threads, false, fast, skipTrash) // force dump the whole tree\n\t_ = zw.Close()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tsize, err := fp.Seek(0, io.SeekCurrent)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tfpath := \"meta/\" + name\n\tdisk, err := object.CreateStorage(\"file\", localDir, \"\", \"\", \"\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tosync.InitForCopyData()\n\t_, err = osync.CopyData(disk, blob, fpath, size, true)\n\treturn blob.String() + fpath, err\n}\n\nfunc cleanupBackups(blob object.ObjectStorage, now time.Time) {\n\tblob = object.WithPrefix(blob, \"meta/\")\n\tch, err := object.ListAll(context.TODO(), blob, \"\", \"\", true, false)\n\tif err != nil {\n\t\tlogger.Warnf(\"listAll prefix meta/: %s\", err)\n\t\treturn\n\t}\n\tvar objs []string\n\tfor o := range ch {\n\t\tif o == nil {\n\t\t\tlogger.Warnf(\"list failed, skip cleanup\")\n\t\t\treturn\n\t\t}\n\t\tif !o.IsDir() {\n\t\t\tobjs = append(objs, o.Key())\n\t\t}\n\t}\n\n\ttoDel := rotate(objs, now)\n\tfor _, o := range toDel {\n\t\tif err = blob.Delete(context.Background(), o); err != nil {\n\t\t\tlogger.Warnf(\"delete object %s: %s\", o, err)\n\t\t}\n\t}\n}\n\n// Cleanup policy:\n// 1. keep all backups within 2 days\n// 2. keep one backup each day within 2 weeks\n// 3. keep one backup each week within 2 months\n// 4. keep one backup each month within 2 years\n// 5. delete backups older than 2 years\nfunc rotate(objs []string, now time.Time) []string {\n\tvar days = 2\n\tcutoff := now.UTC().AddDate(-2, 0, 0)\n\tedge := now.UTC().AddDate(0, 0, -days)\n\tnext := func() {\n\t\tif days < 14 {\n\t\t\tdays++\n\t\t\tedge = edge.AddDate(0, 0, -1)\n\t\t} else if days < 60 {\n\t\t\tdays += 7\n\t\t\tedge = edge.AddDate(0, 0, -7)\n\t\t} else {\n\t\t\tdays += 30\n\t\t\tedge = edge.AddDate(0, 0, -30)\n\t\t}\n\t}\n\n\tvar toDel, within []string\n\tsort.Strings(objs)\n\tfor i := len(objs) - 1; i >= 0; i-- {\n\t\tif len(objs[i]) != 30 { // len(\"dump-2006-01-02-150405.json.gz\")\n\t\t\tlogger.Warnf(\"bad object for metadata backup %s: length %d\", objs[i], len(objs[i]))\n\t\t\tcontinue\n\t\t}\n\t\tts, err := time.Parse(\"2006-01-02-150405\", objs[i][5:22])\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"bad object for metadata backup %s: %s\", objs[i], err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif ts.Before(cutoff) {\n\t\t\ttoDel = append(toDel, objs[:i+1]...)\n\t\t\tbreak\n\t\t}\n\n\t\tif ts.Before(edge) {\n\t\t\tif l := len(within); l > 0 { // keep the earliest one\n\t\t\t\ttoDel = append(toDel, within[:l-1]...)\n\t\t\t\twithin = within[:0]\n\t\t\t}\n\t\t\tfor next(); ts.Before(edge); next() {\n\t\t\t}\n\t\t\twithin = append(within, objs[i])\n\t\t} else if days > 2 {\n\t\t\twithin = append(within, objs[i])\n\t\t}\n\t}\n\tif l := len(within); l > 0 {\n\t\ttoDel = append(toDel, within[:l-1]...)\n\t}\n\treturn toDel\n}\n" }, { "path": "pkg/vfs/backup_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/object\"\n)\n\nfunc TestRotate(t *testing.T) {\n\tformat := func(ts time.Time) string {\n\t\treturn \"dump-\" + ts.UTC().Format(\"2006-01-02-150405\") + \".json.gz\"\n\t}\n\n\tnow := time.Now()\n\tobjs := make([]string, 0, 25)\n\tfor cursor, i := now.AddDate(0, 0, -100), 0; i <= 200; i++ { // one backup for every half day\n\t\tobjs = append(objs, format(cursor))\n\t\ttoDel := rotate(objs, cursor)\n\t\tfor _, d := range toDel {\n\t\t\tfor j, k := range objs {\n\t\t\t\tif k == d {\n\t\t\t\t\tobjs = append(objs[:j], objs[j+1:]...)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tcursor = cursor.Add(time.Duration(12) * time.Hour)\n\t}\n\n\texpect := make([]string, 0, 25)\n\texpect = append(expect, format(now.AddDate(0, 0, -100)))\n\tfor days := 65; days > 14; days -= 7 {\n\t\texpect = append(expect, format(now.AddDate(0, 0, -days)))\n\t}\n\tfor days := 13; days > 2; days-- {\n\t\texpect = append(expect, format(now.AddDate(0, 0, -days)))\n\t}\n\tfor i := 4; i >= 0; i-- {\n\t\texpect = append(expect, format(now.Add(time.Duration(-i*12)*time.Hour)))\n\t}\n\n\tif len(objs) != len(expect) {\n\t\tt.Fatalf(\"length of objs %d != length of expect %d\", len(objs), len(expect))\n\t}\n\tfor i, o := range objs {\n\t\tif o != expect[i] {\n\t\t\tt.Fatalf(\"obj %s != expect %s\", o, expect[i])\n\t\t}\n\t}\n}\n\nfunc TestBackup(t *testing.T) {\n\tv, blob := createTestVFS(nil, \"\")\n\tgo Backup(v.Meta, blob, time.Millisecond*100, false)\n\ttime.Sleep(time.Millisecond * 100)\n\n\tblob = object.WithPrefix(blob, \"meta/\")\n\tkc, _ := object.ListAll(context.TODO(), blob, \"\", \"\", true, false)\n\tvar keys []string\n\tfor obj := range kc {\n\t\tkeys = append(keys, obj.Key())\n\t}\n\tif len(keys) < 1 {\n\t\tt.Fatalf(\"there should be at least 1 backup file\")\n\t}\n}\n" }, { "path": "pkg/vfs/compact.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nvar (\n\tcompactSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"compact_size_histogram_bytes\",\n\t\tHelp: \"Distribution of size of compacted data in bytes.\",\n\t\tBuckets: prometheus.ExponentialBuckets(1024, 2, 16),\n\t})\n)\n\nfunc readSlice(store chunk.ChunkStore, s *meta.Slice, page *chunk.Page, off int) error {\n\tbuf := page.Data\n\tread := 0\n\treader := store.NewReader(s.Id, int(s.Size))\n\tfor read < len(buf) {\n\t\tp := page.Slice(read, len(buf)-read)\n\t\tn, err := reader.ReadAt(context.Background(), p, off+int(s.Off))\n\t\tp.Release()\n\t\tif n == 0 && err != nil {\n\t\t\treturn err\n\t\t}\n\t\tread += n\n\t\toff += n\n\t}\n\treturn nil\n}\n\nfunc Compact(conf chunk.Config, store chunk.ChunkStore, slices []meta.Slice, id uint64) error {\n\tfor utils.AllocMemory()-store.UsedMemory() > int64(conf.BufferSize)*3/2 {\n\t\ttime.Sleep(time.Millisecond * 100)\n\t}\n\tvar size uint32\n\tfor _, s := range slices {\n\t\tsize += s.Len\n\t}\n\tcompactSizeHistogram.Observe(float64(size))\n\tlogger.Debugf(\"compact %d slices (%d bytes) to new slice %d\", len(slices), size, id)\n\n\twriter := store.NewWriter(id)\n\twriter.SetWriteback(false)\n\n\tvar pos int\n\tfor i, s := range slices {\n\t\tif s.Id == 0 {\n\t\t\t_, err := writer.WriteAt(make([]byte, int(s.Len)), int64(pos))\n\t\t\tif err != nil {\n\t\t\t\twriter.Abort()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tpos += int(s.Len)\n\t\t\tcontinue\n\t\t}\n\t\tvar read int\n\t\tfor read < int(s.Len) {\n\t\t\tl := min(conf.BlockSize, int(s.Len)-read)\n\t\t\tp := chunk.NewOffPage(l)\n\t\t\tif err := readSlice(store, &slices[i], p, read); err != nil {\n\t\t\t\tlogger.Debugf(\"can't compact to slice %d, retry later, read %d: %s\", id, i, err)\n\t\t\t\tp.Release()\n\t\t\t\twriter.Abort()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t_, err := writer.WriteAt(p.Data, int64(pos+read))\n\t\t\tp.Release()\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"can't compact to slice %d, retry later, write: %s\", id, err)\n\t\t\t\twriter.Abort()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tread += l\n\t\t\tif pos+read >= conf.BlockSize {\n\t\t\t\tif err = writer.FlushTo(pos + read); err != nil {\n\t\t\t\t\tpanic(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tpos += int(s.Len)\n\t}\n\terr := writer.Finish(pos)\n\tif err != nil {\n\t\twriter.Abort()\n\t}\n\treturn err\n}\n" }, { "path": "pkg/vfs/compact_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n)\n\nfunc TestCompact(t *testing.T) {\n\tcconf := chunk.Config{\n\t\tBlockSize: 256 * 1024,\n\t\tCompress: \"lz4\",\n\t\tMaxUpload: 2,\n\t\tMaxDownload: 200,\n\t\tBufferSize: 30 << 20,\n\t\tCacheSize: 10 << 20,\n\t\tCacheDir: \"memory\",\n\t}\n\tblob, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tstore := chunk.NewCachedStore(blob, cconf, nil)\n\n\t// prepare the slices\n\tvar slices []meta.Slice\n\tvar total int\n\tfor i := 0; i < 100; i++ {\n\t\tbuf := make([]byte, 100+i*100)\n\t\tfor j := range buf {\n\t\t\tbuf[j] = byte(i)\n\t\t}\n\t\tcid := uint64(i)\n\t\tw := store.NewWriter(cid)\n\t\tif n, e := w.WriteAt(buf, 0); e != nil {\n\t\t\tt.Fatalf(\"write chunk %d: %s\", cid, e)\n\t\t} else {\n\t\t\ttotal += n\n\t\t}\n\t\tif e := w.Finish(len(buf)); e != nil {\n\t\t\tt.Fatalf(\"flush chunk %d: %s\", cid, e)\n\t\t}\n\t\tslices = append(slices, meta.Slice{Id: cid, Size: uint32(len(buf)), Len: uint32(len(buf))})\n\t}\n\n\t// compact\n\tvar cid uint64 = 1000\n\terr := Compact(cconf, store, slices, cid)\n\tif err != nil {\n\t\tt.Fatalf(\"compact %d slices : %s\", len(slices), err)\n\t}\n\n\t// verify result\n\tr := store.NewReader(cid, total)\n\tvar off int\n\tfor i := 0; i < 100; i++ {\n\t\tbuf := make([]byte, 100+i*100)\n\t\tpage := chunk.NewPage(buf)\n\t\tn, err := r.ReadAt(context.Background(), page, off)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"read chunk %d at %d: %s\", cid, off, err)\n\t\t} else if n != len(buf) {\n\t\t\tt.Fatalf(\"short read: %d\", n)\n\t\t}\n\t\tfor j := range buf {\n\t\t\tif buf[j] != byte(i) {\n\t\t\t\tt.Fatalf(\"invalid byte at %d: %d !=%d\", j, buf[j], i)\n\t\t\t}\n\t\t}\n\t\toff += len(buf)\n\t\tdefer page.Release()\n\t}\n\n\t// failed\n\t_ = store.Remove(1, 200)\n\terr = Compact(cconf, store, slices, cid)\n\tif err == nil {\n\t\tt.Fatalf(\"compact should fail with read but got nil\")\n\t}\n\n\t// TODO: inject write failure\n}\n" }, { "path": "pkg/vfs/fill.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"fmt\"\n\t\"path\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\ntype _file struct {\n\tino Ino\n\tsize uint64\n}\n\ntype CacheAction uint8\n\nfunc (act CacheAction) String() string {\n\tswitch act {\n\tcase WarmupCache:\n\t\treturn \"warmup cache\"\n\tcase EvictCache:\n\t\treturn \"evict cache\"\n\tcase CheckCache:\n\t\treturn \"check cache\"\n\t}\n\treturn \"unknown operation\"\n}\n\nconst (\n\tWarmupCache CacheAction = iota\n\tEvictCache\n\tCheckCache = 2\n)\n\ntype CacheFiller struct {\n\tconf *Config\n\tmeta meta.Meta\n\tstore chunk.ChunkStore\n}\n\nfunc NewCacheFiller(conf *Config, meta meta.Meta, store chunk.ChunkStore) *CacheFiller {\n\treturn &CacheFiller{\n\t\tconf: conf,\n\t\tmeta: meta,\n\t\tstore: store,\n\t}\n}\n\ntype token struct{}\n\nfunc (c *CacheFiller) cacheFile(ctx meta.Context, action CacheAction, resp *CacheResponse, concurrent chan token, wg *sync.WaitGroup, f _file) {\n\tconcurrent <- token{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer func() {\n\t\t\t<-concurrent\n\t\t\twg.Done()\n\t\t}()\n\n\t\tif f.ino == 0 {\n\t\t\tlogger.Warnf(\"%s got inode 0\", action)\n\t\t\treturn\n\t\t}\n\n\t\tvar handler sliceHandler\n\t\tswitch action {\n\t\tcase WarmupCache:\n\t\t\thandler = func(s meta.Slice) error {\n\t\t\t\treturn c.store.FillCache(s.Id, s.Size)\n\t\t\t}\n\n\t\t\tif c.conf.Meta.OpenCache > 0 {\n\t\t\t\tif err := c.meta.Open(ctx, f.ino, syscall.O_RDONLY, &meta.Attr{}); err != 0 {\n\t\t\t\t\tlogger.Errorf(\"Inode %d could be opened: %s\", f.ino, err)\n\t\t\t\t}\n\t\t\t\t_ = c.meta.Close(ctx, f.ino)\n\t\t\t}\n\t\tcase EvictCache:\n\t\t\thandler = func(s meta.Slice) error {\n\t\t\t\treturn c.store.EvictCache(s.Id, s.Size)\n\t\t\t}\n\t\tcase CheckCache:\n\t\t\tblockHandler := func(exists bool, loc string, size int) {\n\t\t\t\tif exists {\n\t\t\t\t\tresp.Lock()\n\t\t\t\t\tresp.Locations[loc] += uint64(size)\n\t\t\t\t\tresp.Unlock()\n\t\t\t\t} else {\n\t\t\t\t\tatomic.AddUint64(&resp.MissBytes, uint64(size))\n\t\t\t\t}\n\t\t\t}\n\t\t\thandler = func(s meta.Slice) error {\n\t\t\t\treturn c.store.CheckCache(s.Id, s.Size, blockHandler)\n\t\t\t}\n\t\t}\n\n\t\titer := newSliceIterator(ctx, c.meta, f.ino, f.size, resp)\n\t\terr := iter.Iterate(handler, concurrent)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"%s error : %s\", action, err)\n\t\t}\n\n\t\tatomic.AddUint64(&resp.FileCount, 1)\n\t}()\n}\n\nfunc (c *CacheFiller) Cache(ctx meta.Context, action CacheAction, paths []string, threads int, resp *CacheResponse) {\n\tif resp == nil {\n\t\tresp = &CacheResponse{Locations: make(map[string]uint64)}\n\t}\n\tstart := time.Now()\n\ttodo := make(chan _file, 20*threads)\n\n\tconcurrent := make(chan token, threads)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor f := range todo {\n\t\t\tif ctx.Canceled() {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tc.cacheFile(ctx, action, resp, concurrent, &wg, f)\n\t\t}\n\t}()\n\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tfor _, p := range paths {\n\t\tif st := c.resolve(ctx, p, &inode, attr); st != 0 {\n\t\t\tlogger.Warnf(\"Failed to resolve path %s: %s\", p, st)\n\t\t\tcontinue\n\t\t}\n\t\tlogger.Debugf(\"path %s\", p)\n\t\tif attr.Typ == meta.TypeDirectory {\n\t\t\tc.walkDir(ctx, inode, todo)\n\t\t} else if attr.Typ == meta.TypeFile {\n\t\t\t_ = sendFile(ctx, todo, _file{inode, attr.Length})\n\t\t}\n\t\tif ctx.Canceled() {\n\t\t\tbreak\n\t\t}\n\t}\n\tclose(todo)\n\twg.Wait()\n\n\tif ctx.Canceled() {\n\t\tlogger.Infof(\"%s cancelled\", action)\n\t}\n\tlogger.Infof(\"%s %d paths in %s\", action, len(paths), time.Since(start))\n}\n\nfunc sendFile(ctx meta.Context, todo chan _file, f _file) error {\n\tselect {\n\tcase todo <- f:\n\t\treturn nil\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\t}\n}\n\nfunc (c *CacheFiller) resolve(ctx meta.Context, p string, inode *Ino, attr *Attr) syscall.Errno {\n\tvar inodePrefix = \"inode:\"\n\tif strings.HasPrefix(p, inodePrefix) {\n\t\ti, err := strconv.ParseUint(p[len(inodePrefix):], 10, 64)\n\t\tif err == nil {\n\t\t\t*inode = meta.Ino(i)\n\t\t\treturn c.meta.GetAttr(ctx, meta.Ino(i), attr)\n\t\t}\n\t}\n\tp = strings.Trim(p, \"/\")\n\terr := c.meta.Resolve(ctx, 1, p, inode, attr)\n\tif err != syscall.ENOTSUP {\n\t\treturn err\n\t}\n\n\t// Fallback to the default implementation that calls `meta.Lookup` for each directory along the path.\n\t// It might be slower for deep directories, but it works for every meta that implements `Lookup`.\n\tparent := Ino(1)\n\tss := strings.Split(p, \"/\")\n\tfor i, name := range ss {\n\t\tif len(name) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tif parent == meta.RootInode && i == len(ss)-1 && IsSpecialName(name) {\n\t\t\t*inode, attr = GetInternalNodeByName(name)\n\t\t\tparent = *inode\n\t\t\tbreak\n\t\t}\n\t\tif i > 0 {\n\t\t\tif err = c.meta.Access(ctx, parent, MODE_MASK_R|MODE_MASK_X, attr); err != 0 {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif err = c.meta.Lookup(ctx, parent, name, inode, attr, false); err != 0 {\n\t\t\treturn err\n\t\t}\n\t\tif attr.Typ == meta.TypeSymlink {\n\t\t\tvar buf []byte\n\t\t\tif err = c.meta.ReadLink(ctx, *inode, &buf); err != 0 {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ttarget := string(buf)\n\t\t\tif strings.HasPrefix(target, \"/\") || strings.Contains(target, \"://\") {\n\t\t\t\treturn syscall.ENOTSUP\n\t\t\t}\n\t\t\ttarget = path.Join(strings.Join(ss[:i], \"/\"), target)\n\t\t\tif err = c.resolve(ctx, target, inode, attr); err != 0 {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tparent = *inode\n\t}\n\tif parent == meta.RootInode {\n\t\t*inode = parent\n\t\tif err = c.meta.GetAttr(ctx, *inode, attr); err != 0 {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc (c *CacheFiller) walkDir(ctx meta.Context, inode Ino, todo chan _file) {\n\tpending := make([]Ino, 1)\n\tpending[0] = inode\n\tfor len(pending) > 0 {\n\t\tl := len(pending)\n\t\tl--\n\t\tinode = pending[l]\n\t\tpending = pending[:l]\n\t\tvar entries []*meta.Entry\n\t\tr := c.meta.Readdir(ctx, inode, 1, &entries)\n\t\tif r == 0 {\n\t\t\tfor _, f := range entries {\n\t\t\t\tname := string(f.Name)\n\t\t\t\tif name == \".\" || name == \"..\" {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif f.Attr.Typ == meta.TypeDirectory {\n\t\t\t\t\tpending = append(pending, f.Inode)\n\t\t\t\t} else if f.Attr.Typ != meta.TypeSymlink {\n\t\t\t\t\t_ = sendFile(ctx, todo, _file{f.Inode, f.Attr.Length})\n\t\t\t\t}\n\t\t\t\tif ctx.Canceled() {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Warnf(\"readdir %d: %s\", inode, r)\n\t\t}\n\t}\n}\n\ntype sliceIterator struct {\n\tctx meta.Context\n\tmClient meta.Meta\n\tino Ino\n\tchunkCnt uint32\n\tstat *CacheResponse\n\n\terr error\n\tnextChunkIndex uint32\n\tnextSliceIndex uint64\n\tslices []meta.Slice\n}\n\ntype sliceHandler func(s meta.Slice) error\n\nfunc (iter *sliceIterator) hasNext() bool {\n\tif iter.err != nil {\n\t\tlogger.Error(iter.err)\n\t\titer.err = nil\n\t}\n\n\tif iter.ctx.Canceled() {\n\t\titer.err = iter.ctx.Err()\n\t\treturn false\n\t}\n\n\tfor iter.nextSliceIndex >= uint64(len(iter.slices)) {\n\t\tif iter.nextChunkIndex >= iter.chunkCnt {\n\t\t\treturn false\n\t\t}\n\n\t\titer.slices = nil\n\t\titer.nextSliceIndex = 0\n\t\tif st := iter.mClient.Read(iter.ctx, iter.ino, iter.nextChunkIndex, &iter.slices); st != 0 {\n\t\t\titer.err = fmt.Errorf(\"get slices of inode %d index %d error: %d\", iter.ino, iter.nextChunkIndex, st)\n\t\t\tlogger.Error(iter.err)\n\t\t\treturn false\n\t\t}\n\t\titer.nextChunkIndex++\n\t}\n\n\treturn true\n}\n\nfunc (iter *sliceIterator) next() meta.Slice {\n\ts := iter.slices[iter.nextSliceIndex]\n\titer.nextSliceIndex++\n\treturn s\n}\n\nfunc (iter *sliceIterator) Iterate(handler sliceHandler, concurrent chan token) error {\n\tif handler == nil {\n\t\treturn fmt.Errorf(\"handler not set\")\n\t}\n\tvar wg sync.WaitGroup\n\tfor iter.hasNext() {\n\t\ts := iter.next()\n\t\tif s.Id == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tatomic.AddUint64(&iter.stat.SliceCount, 1)\n\t\tatomic.AddUint64(&iter.stat.TotalBytes, uint64(s.Size))\n\n\t\tselect {\n\t\tcase concurrent <- token{}:\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer func() {\n\t\t\t\t\t<-concurrent\n\t\t\t\t\twg.Done()\n\t\t\t\t}()\n\t\t\t\tif err := handler(s); err != nil {\n\t\t\t\t\titer.err = fmt.Errorf(\"inode %d slice %d : %w\", iter.ino, s.Id, err)\n\t\t\t\t}\n\t\t\t}()\n\t\tdefault:\n\t\t\tif err := handler(s); err != nil {\n\t\t\t\titer.err = fmt.Errorf(\"inode %d slice %d : %w\", iter.ino, s.Id, err)\n\t\t\t}\n\t\t}\n\t}\n\twg.Wait()\n\treturn iter.err\n}\n\nfunc newSliceIterator(ctx meta.Context, mClient meta.Meta, ino Ino, size uint64, stat *CacheResponse) *sliceIterator {\n\treturn &sliceIterator{\n\t\tctx: ctx,\n\t\tmClient: mClient,\n\t\tino: ino,\n\t\tstat: stat,\n\n\t\tnextSliceIndex: 0,\n\t\tnextChunkIndex: 0,\n\t\tchunkCnt: uint32((size + meta.ChunkSize - 1) / meta.ChunkSize),\n\t}\n}\n" }, { "path": "pkg/vfs/fill_test.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\nfunc TestFill(t *testing.T) {\n\tv, _ := createTestVFS(nil, \"\")\n\tctx := NewLogContext(meta.Background())\n\tentry, _ := v.Mkdir(ctx, 1, \"test\", 0777, 022)\n\tfe, fh, _ := v.Create(ctx, entry.Inode, \"file\", 0644, 0, uint32(os.O_WRONLY))\n\t_ = v.Write(ctx, fe.Inode, []byte(\"hello\"), 0, fh)\n\t_ = v.Flush(ctx, fe.Inode, fh, 0)\n\tv.Release(ctx, fe.Inode, fh)\n\t_, _ = v.Symlink(ctx, \"test/file\", 1, \"sym\")\n\t_, _ = v.Symlink(ctx, \"/tmp/testfile\", 1, \"sym2\")\n\t_, _ = v.Symlink(ctx, \"testfile\", 1, \"sym3\")\n\n\t// normal cases\n\tv.cacheFiller.Cache(meta.Background(), WarmupCache, []string{\"/test/file\", \"/test\", \"/sym\", \"/\"}, 2, nil)\n\n\t// remove chunk\n\tvar slices []meta.Slice\n\t_ = v.Meta.Read(meta.Background(), fe.Inode, 0, &slices)\n\tfor _, s := range slices {\n\t\t_ = v.Store.Remove(s.Id, int(s.Size))\n\t}\n\t// bad cases\n\tv.cacheFiller.Cache(meta.Background(), WarmupCache, []string{\"/test/file\", \"/sym2\", \"/sym3\", \"/.stats\", \"/not_exists\"}, 2, nil)\n}\n" }, { "path": "pkg/vfs/handle.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"os\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\ntype handle struct {\n\tsync.Mutex\n\tinode Ino\n\tfh uint64\n\n\t// for dir\n\tdirHandler meta.DirHandler\n\treadAt time.Time\n\n\t// for file\n\tflags uint32\n\tlocks uint8\n\tflockOwner uint64 // kernel 3.1- does not pass lock_owner in release()\n\tofdOwner uint64 // OFD lock\n\treader FileReader\n\twriter FileWriter\n\tops []Context\n\n\t// rwlock\n\twriting uint32\n\treaders uint32\n\twriters uint32\n\tcond *utils.Cond\n\n\t// internal files\n\toff uint64\n\tdata []byte\n\tpending []byte\n\tbctx meta.Context\n}\n\nfunc (h *handle) Write(buf []byte) (int, error) {\n\th.Lock()\n\tdefer h.Unlock()\n\th.data = append(h.data, buf...)\n\treturn len(buf), nil\n}\n\nfunc (h *handle) addOp(ctx Context) {\n\th.Lock()\n\tdefer h.Unlock()\n\th.ops = append(h.ops, ctx)\n}\n\nfunc (h *handle) removeOp(ctx Context) {\n\th.Lock()\n\tdefer h.Unlock()\n\tfor i, c := range h.ops {\n\t\tif c == ctx {\n\t\t\th.ops[i] = h.ops[len(h.ops)-1]\n\t\t\th.ops = h.ops[:len(h.ops)-1]\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (h *handle) cancelOp(pid uint32) {\n\tif pid == 0 {\n\t\treturn\n\t}\n\th.Lock()\n\tdefer h.Unlock()\n\tfor _, c := range h.ops {\n\t\tif c.Pid() == pid || c.Pid() > 0 && c.Duration() > time.Second {\n\t\t\tc.Cancel()\n\t\t}\n\t}\n}\n\nfunc (h *handle) Rlock(ctx Context) bool {\n\th.Lock()\n\tfor (h.writing | h.writers) != 0 {\n\t\tif h.cond.WaitWithTimeout(time.Second) && ctx.Canceled() {\n\t\t\th.Unlock()\n\t\t\tlogger.Warnf(\"read lock %d interrupted\", h.inode)\n\t\t\treturn false\n\t\t}\n\t}\n\th.readers++\n\th.Unlock()\n\th.addOp(ctx)\n\treturn true\n}\n\nfunc (h *handle) Runlock() {\n\th.Lock()\n\th.readers--\n\tif h.readers == 0 {\n\t\th.cond.Broadcast()\n\t}\n\th.Unlock()\n}\n\nfunc (h *handle) Wlock(ctx Context) bool {\n\th.Lock()\n\th.writers++\n\tfor (h.readers | h.writing) != 0 {\n\t\tif h.cond.WaitWithTimeout(time.Second) && ctx.Canceled() {\n\t\t\th.writers--\n\t\t\th.Unlock()\n\t\t\tlogger.Warnf(\"write lock %d interrupted\", h.inode)\n\t\t\treturn false\n\t\t}\n\t}\n\th.writers--\n\th.writing = 1\n\th.Unlock()\n\th.addOp(ctx)\n\treturn true\n}\n\nfunc (h *handle) Wunlock() {\n\th.Lock()\n\th.writing = 0\n\th.cond.Broadcast()\n\th.Unlock()\n}\n\nfunc (h *handle) Close() {\n\tif h.reader != nil {\n\t\th.reader.Close(meta.Background())\n\t\th.reader = nil\n\t}\n\tif h.writer != nil {\n\t\t_ = h.writer.Close(meta.Background())\n\t\th.writer = nil\n\t}\n}\n\nfunc (v *VFS) newHandle(inode Ino, readOnly bool) *handle {\n\tv.hanleM.Lock()\n\tdefer v.hanleM.Unlock()\n\tvar lowBits uint64\n\tif readOnly {\n\t\tlowBits = 1\n\t}\n\tfor v.handleIno[v.nextfh] > 0 || v.nextfh&1 != lowBits {\n\t\tv.nextfh++ // skip recovered fd\n\t}\n\tfh := v.nextfh\n\th := &handle{inode: inode, fh: fh}\n\tv.nextfh++\n\th.cond = utils.NewCond(h)\n\tv.handles[inode] = append(v.handles[inode], h)\n\treturn h\n}\n\nfunc (v *VFS) findAllHandles(inode Ino) []*handle {\n\tv.hanleM.Lock()\n\tdefer v.hanleM.Unlock()\n\ths := v.handles[inode]\n\tif len(hs) <= 1 {\n\t\treturn hs\n\t}\n\t// copy hs so it will not be modified by releaseHandle\n\ths2 := make([]*handle, len(hs))\n\tcopy(hs2, hs)\n\treturn hs2\n}\n\nconst O_RECOVERED = 1 << 31 // is recovered fd\n\nfunc (v *VFS) findHandle(inode Ino, fh uint64) *handle {\n\tv.hanleM.Lock()\n\tdefer v.hanleM.Unlock()\n\tfor _, f := range v.handles[inode] {\n\t\tif f.fh == fh {\n\t\t\treturn f\n\t\t}\n\t}\n\tif fh&1 == 1 && inode != controlInode {\n\t\tf := &handle{inode: inode, fh: fh, flags: O_RECOVERED}\n\t\tf.cond = utils.NewCond(f)\n\t\tv.handles[inode] = append(v.handles[inode], f)\n\t\tif v.handleIno[fh] == 0 {\n\t\t\tv.handleIno[fh] = inode\n\t\t}\n\t\treturn f\n\t}\n\treturn nil\n}\n\nfunc (v *VFS) releaseHandle(inode Ino, fh uint64) {\n\tv.hanleM.Lock()\n\tdefer v.hanleM.Unlock()\n\ths := v.handles[inode]\n\tfor i, f := range hs {\n\t\tif f.fh == fh {\n\t\t\tif hs[i].dirHandler != nil {\n\t\t\t\ths[i].dirHandler.Close()\n\t\t\t\ths[i].dirHandler = nil\n\t\t\t}\n\t\t\tif i+1 < len(hs) {\n\t\t\t\ths[i] = hs[len(hs)-1]\n\t\t\t}\n\t\t\tif len(hs) > 1 {\n\t\t\t\tv.handles[inode] = hs[:len(hs)-1]\n\t\t\t} else {\n\t\t\t\tdelete(v.handles, inode)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (v *VFS) newFileHandle(inode Ino, length uint64, flags uint32) uint64 {\n\th := v.newHandle(inode, (flags&O_ACCMODE) == syscall.O_RDONLY)\n\th.Lock()\n\tdefer h.Unlock()\n\th.flags = flags\n\tswitch flags & O_ACCMODE {\n\tcase syscall.O_RDONLY:\n\t\th.reader = v.reader.Open(inode, length)\n\tcase syscall.O_WRONLY: // FUSE writeback_cache mode need reader even for WRONLY\n\t\tfallthrough\n\tcase syscall.O_RDWR:\n\t\th.reader = v.reader.Open(inode, length)\n\t\th.writer = v.writer.Open(inode, length)\n\t}\n\treturn h.fh\n}\n\nfunc (v *VFS) releaseFileHandle(ino Ino, fh uint64) {\n\th := v.findHandle(ino, fh)\n\tif h != nil {\n\t\tv.releaseHandle(ino, fh)\n\t\th.Lock()\n\t\tfor (h.writing | h.writers | h.readers) != 0 {\n\t\t\th.cond.WaitWithTimeout(time.Millisecond * 100)\n\t\t}\n\t\th.Unlock()\n\t\th.Close()\n\t}\n}\n\nfunc (v *VFS) invalidateDirHandle(parent Ino, name string, inode Ino, attr *Attr) {\n\tv.hanleM.Lock()\n\ths := v.handles[parent]\n\tv.hanleM.Unlock()\n\tfor _, h := range hs {\n\t\th.Lock()\n\t\tif h.dirHandler != nil {\n\t\t\tif inode > 0 {\n\t\t\t\th.dirHandler.Insert(inode, name, attr)\n\t\t\t} else {\n\t\t\t\th.dirHandler.Delete(name)\n\t\t\t}\n\t\t}\n\t\th.Unlock()\n\t}\n}\n\ntype state struct {\n\tHandler map[uint64]saveHandle\n\tNextFh uint64\n}\n\ntype saveHandle struct {\n\tInode uint64\n\tLength uint64\n\tFlags uint32\n\tUseLocks uint8\n\tFlockOwner uint64\n\tOff uint64\n\tData string\n}\n\nfunc (v *VFS) dumpAllHandles(path string) (err error) {\n\tv.hanleM.Lock()\n\tdefer v.hanleM.Unlock()\n\tvar vfsState state\n\tvfsState.Handler = make(map[uint64]saveHandle)\n\tfor ino, hs := range v.handles {\n\t\tif ino == controlInode {\n\t\t\t// the job is lost, can't be recovered\n\t\t\tcontinue\n\t\t}\n\t\tfor _, h := range hs {\n\t\t\th.Lock()\n\t\t\tif ino == logInode {\n\t\t\t\treaderLock.RLock()\n\t\t\t\treader := readers[h.fh]\n\t\t\t\treaderLock.RUnlock()\n\t\t\t\tif reader == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\treader.Lock()\n\t\t\tOUTER:\n\t\t\t\tfor {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase line := <-reader.buffer:\n\t\t\t\t\t\treader.last = append(reader.last, line...)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tbreak OUTER\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\th.data = reader.last\n\t\t\t\treader.Unlock()\n\t\t\t}\n\t\t\tvar length uint64\n\t\t\tif h.writer != nil {\n\t\t\t\tlength = h.writer.GetLength()\n\t\t\t\terr := h.writer.Flush(meta.Background())\n\t\t\t\tif err != 0 {\n\t\t\t\t\tlogger.Errorf(\"flush writer of %d: %s\", ino, err)\n\t\t\t\t}\n\t\t\t} else if h.reader != nil {\n\t\t\t\tlength = h.reader.GetLength()\n\t\t\t}\n\t\t\ts := saveHandle{\n\t\t\t\tInode: uint64(h.inode),\n\t\t\t\tLength: length,\n\t\t\t\tFlags: h.flags,\n\t\t\t\tUseLocks: h.locks,\n\t\t\t\tFlockOwner: h.flockOwner,\n\t\t\t\tOff: h.off,\n\t\t\t\tData: hex.EncodeToString(h.data),\n\t\t\t}\n\t\t\th.Unlock()\n\t\t\tvfsState.Handler[h.fh] = s\n\t\t}\n\t}\n\tvfsState.NextFh = v.nextfh\n\td, err := json.Marshal(vfsState)\n\tif err != nil {\n\t\treturn err\n\t}\n\tf, err := os.Create(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer f.Close()\n\t_, err = f.Write(d)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn\n}\n\nfunc (v *VFS) loadAllHandles(path string) error {\n\tf, err := os.Open(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer f.Close()\n\td, err := io.ReadAll(f)\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar vfsState state\n\terr = json.Unmarshal(d, &vfsState)\n\tif err != nil {\n\t\treturn err\n\t}\n\tv.hanleM.Lock()\n\tdefer v.hanleM.Unlock()\n\tfor fh, s := range vfsState.Handler {\n\t\tdata, err := hex.DecodeString(s.Data)\n\t\tif err != nil {\n\t\t\tlogger.Warnf(\"decode data for inode %d: %s\", s.Inode, err)\n\t\t}\n\t\th := &handle{\n\t\t\tinode: Ino(s.Inode),\n\t\t\tfh: fh,\n\t\t\tflags: s.Flags,\n\t\t\tlocks: s.UseLocks,\n\t\t\tflockOwner: s.FlockOwner,\n\t\t\toff: s.Off,\n\t\t}\n\t\th.cond = utils.NewCond(h)\n\t\tv.handles[h.inode] = append(v.handles[h.inode], h)\n\t\tv.handleIno[fh] = h.inode\n\t\tif s.Inode == logInode {\n\t\t\topenAccessLog(fh)\n\t\t\treaders[fh].last = data\n\t\t\tcontinue\n\t\t}\n\t\th.data = data\n\t\tswitch s.Flags & O_ACCMODE {\n\t\tcase syscall.O_RDONLY:\n\t\t\th.reader = v.reader.Open(h.inode, s.Length)\n\t\tcase syscall.O_WRONLY: // FUSE writeback_cache mode need reader even for WRONLY\n\t\t\tfallthrough\n\t\tcase syscall.O_RDWR:\n\t\t\th.reader = v.reader.Open(h.inode, s.Length)\n\t\t\th.writer = v.writer.Open(h.inode, s.Length)\n\t\t}\n\t}\n\tif len(v.handleIno) > 0 {\n\t\tlogger.Infof(\"load %d handles from %s\", len(v.handleIno), path)\n\t}\n\tv.nextfh = vfsState.NextFh\n\t// _ = os.Remove(path)\n\treturn nil\n}\n" }, { "path": "pkg/vfs/helpers.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"fmt\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\nconst (\n\tMODE_MASK_R = 4\n\tMODE_MASK_W = 2\n\tMODE_MASK_X = 1\n)\n\nfunc strerr(errno syscall.Errno) string {\n\tif errno == 0 {\n\t\treturn \"OK\"\n\t}\n\treturn errno.Error()\n}\n\nvar typestr = map[uint16]byte{\n\tsyscall.S_IFSOCK: 's',\n\tsyscall.S_IFLNK: 'l',\n\tsyscall.S_IFREG: '-',\n\tsyscall.S_IFBLK: 'b',\n\tsyscall.S_IFDIR: 'd',\n\tsyscall.S_IFCHR: 'c',\n\tsyscall.S_IFIFO: 'f',\n\t0: '?',\n}\n\ntype smode uint16\n\nfunc (mode smode) String() string {\n\ts := []byte(\"?rwxrwxrwx\")\n\ts[0] = typestr[uint16(mode)&(syscall.S_IFMT&0xffff)]\n\tif (mode & syscall.S_ISUID) != 0 {\n\t\ts[3] = 's'\n\t}\n\tif (mode & syscall.S_ISGID) != 0 {\n\t\ts[6] = 's'\n\t}\n\tif (mode & syscall.S_ISVTX) != 0 {\n\t\ts[9] = 't'\n\t}\n\tfor i := uint16(0); i < 9; i++ {\n\t\tif (mode & (1 << i)) == 0 {\n\t\t\tif s[9-i] == 's' || s[9-i] == 't' {\n\t\t\t\ts[9-i] &= 0xDF\n\t\t\t} else {\n\t\t\t\ts[9-i] = '-'\n\t\t\t}\n\t\t}\n\t}\n\treturn string(s)\n}\n\n// Entry is an alias of meta.Entry, which is used to generate the string\n// representation lazily.\ntype Entry meta.Entry\n\nfunc (entry *Entry) String() string {\n\tif entry == nil {\n\t\treturn \"\"\n\t}\n\tif entry.Attr == nil {\n\t\treturn fmt.Sprintf(\" (%d)\", entry.Inode)\n\t}\n\ta := entry.Attr\n\tmode := a.SMode()\n\treturn fmt.Sprintf(\" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])\",\n\t\tentry.Inode, smode(mode), mode, a.Nlink, a.Uid, a.Gid,\n\t\ta.Atime, a.Mtime, a.Ctime, a.Length)\n}\n\n// LogContext is an interface to add duration on meta.Context.\ntype LogContext interface {\n\tmeta.Context\n\tDuration() time.Duration\n}\n\ntype logContext struct {\n\tmeta.Context\n\tstart time.Time\n}\n\nfunc (ctx *logContext) Duration() time.Duration {\n\treturn time.Since(ctx.start)\n}\n\n// NewLogContext creates an LogContext starting from now.\nfunc NewLogContext(ctx meta.Context) LogContext {\n\treturn &logContext{ctx, time.Now()}\n}\n" }, { "path": "pkg/vfs/helpers_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"syscall\"\n\t\"testing\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\ntype smodeCase struct {\n\tmode uint16\n\tstr string\n}\n\nvar cases = []smodeCase{\n\t{syscall.S_IFDIR | 00755, \"drwxr-xr-x\"},\n\t{syscall.S_IFREG | 01644, \"-rw-r--r-T\"},\n\t{syscall.S_IFLNK | 03755, \"lrwxr-sr-t\"},\n\t{syscall.S_IFSOCK | 06700, \"srws--S---\"},\n}\n\nfunc TestSmode(t *testing.T) {\n\tfor _, s := range cases {\n\t\tres := smode(s.mode).String()\n\t\tif res != s.str {\n\t\t\tt.Fatalf(\"str of %o: %s != %s\", s.mode, res, s.str)\n\t\t}\n\t}\n}\n\nfunc TestEntryString(t *testing.T) {\n\tvar e *Entry\n\tif e.String() != \"\" {\n\t\tt.Fatalf(\"empty entry should be ''\")\n\t}\n\te = &Entry{Inode: 2, Name: []byte(\"test\")}\n\tif e.String() != \" (2)\" {\n\t\tt.Fatalf(\"empty entry should be ` (2)`\")\n\t}\n\n\te.Attr = &meta.Attr{\n\t\tTyp: meta.TypeFile,\n\t\tMode: 01755,\n\t\tNlink: 1,\n\t\tUid: 2,\n\t\tGid: 3,\n\t\tAtime: 4,\n\t\tMtime: 5,\n\t\tCtime: 6,\n\t\tLength: 7,\n\t}\n\tif e.String() != \" (2,[-rwxr-xr-t:0101755,1,2,3,4,5,6,7])\" {\n\t\tt.Fatalf(\"string of entry is not expected: %s\", e.String())\n\t}\n}\n\nfunc TestError(t *testing.T) {\n\tif strerr(0) != \"OK\" {\n\t\tt.Fatalf(\"expect 'OK' but got %q\", strerr(0))\n\t}\n\tif strerr(syscall.EACCES) != \"permission denied\" {\n\t\tt.Fatalf(\"expect 'Access denied', but got %q\", strerr(syscall.EACCES))\n\t}\n}\n" }, { "path": "pkg/vfs/internal.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tio_prometheus_client \"github.com/prometheus/client_model/go\"\n)\n\nconst (\n\tminInternalNode = 0x7FFFFFFF00000000\n\tlogInode = minInternalNode + 1\n\tcontrolInode = minInternalNode + 2\n\tStatsInode = minInternalNode + 3\n\tConfigInode = minInternalNode + 4\n\ttrashInode = meta.TrashInode\n)\n\nvar controlMutex sync.Mutex\nvar controlHandlers = make(map[uint32]uint64)\n\nfunc (v *VFS) getControlHandle(pid uint32) uint64 {\n\tcontrolMutex.Lock()\n\tdefer controlMutex.Unlock()\n\tfh := controlHandlers[pid]\n\tif fh == 0 {\n\t\th := v.newHandle(controlInode, false)\n\t\tfh = h.fh\n\t\tcontrolHandlers[pid] = fh\n\t}\n\treturn fh\n}\n\nfunc (v *VFS) releaseControlHandle(pid uint32) {\n\tcontrolMutex.Lock()\n\tdefer controlMutex.Unlock()\n\tfh := controlHandlers[pid]\n\tif fh != 0 {\n\t\tv.releaseHandle(controlInode, fh)\n\t\tdelete(controlHandlers, pid)\n\t}\n}\n\ntype internalNode struct {\n\tinode Ino\n\tname string\n\tattr *Attr\n}\n\nvar internalNodes = []*internalNode{\n\t{controlInode, \".control\", &Attr{Mode: 0666}},\n\t{logInode, \".accesslog\", &Attr{Mode: 0400}},\n\t{StatsInode, \".stats\", &Attr{Mode: 0444}},\n\t{ConfigInode, \".config\", &Attr{Mode: 0400}},\n\t{trashInode, meta.TrashName, &Attr{Mode: 0555}},\n}\n\nfunc init() {\n\tuid := uint32(utils.GetCurrentUID())\n\tgid := uint32(utils.GetCurrentGID())\n\tnow := time.Now().Unix()\n\tfor _, v := range internalNodes {\n\t\tif v.inode == trashInode {\n\t\t\tv.attr.Typ = meta.TypeDirectory\n\t\t\tv.attr.Nlink = 2\n\t\t} else {\n\t\t\tv.attr.Typ = meta.TypeFile\n\t\t\tv.attr.Nlink = 1\n\t\t\tv.attr.Uid = uid\n\t\t\tv.attr.Gid = gid\n\t\t}\n\t\tv.attr.Atime = now\n\t\tv.attr.Mtime = now\n\t\tv.attr.Ctime = now\n\t\tv.attr.Full = true\n\t}\n}\n\nfunc IsSpecialNode(ino Ino) bool {\n\treturn ino >= minInternalNode\n}\n\nfunc IsSpecialName(name string) bool {\n\tif name[0] != '.' {\n\t\treturn false\n\t}\n\tfor _, n := range internalNodes {\n\t\tif name == n.name {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc getInternalNode(ino Ino) *internalNode {\n\tfor _, n := range internalNodes {\n\t\tif ino == n.inode {\n\t\t\treturn n\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc GetInternalNodeByName(name string) (Ino, *Attr) {\n\tn := getInternalNodeByName(name)\n\tif n != nil {\n\t\treturn n.inode, n.attr\n\t}\n\treturn 0, nil\n}\n\nfunc getInternalNodeByName(name string) *internalNode {\n\tif name[0] != '.' {\n\t\treturn nil\n\t}\n\tfor _, n := range internalNodes {\n\t\tif name == n.name {\n\t\t\treturn n\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc CollectMetrics(registry *prometheus.Registry) []byte {\n\tif registry == nil {\n\t\treturn []byte(\"\")\n\t}\n\tmfs, err := registry.Gather()\n\tif err != nil {\n\t\tlogger.Errorf(\"collect metrics: %s\", err)\n\t\treturn nil\n\t}\n\tw := bytes.NewBuffer(nil)\n\tformat := func(v float64) string {\n\t\treturn strconv.FormatFloat(v, 'f', -1, 64)\n\t}\n\tfor _, mf := range mfs {\n\t\tfor _, m := range mf.Metric {\n\t\t\tvar name = *mf.Name\n\t\t\tfor _, l := range m.Label {\n\t\t\t\tif *l.Name == \"method\" || *l.Name == \"errno\" {\n\t\t\t\t\tname += \"_\" + *l.Value\n\t\t\t\t}\n\t\t\t}\n\t\t\tswitch *mf.Type {\n\t\t\tcase io_prometheus_client.MetricType_GAUGE:\n\t\t\t\t_, _ = fmt.Fprintf(w, \"%s %s\\n\", name, format(*m.Gauge.Value))\n\t\t\tcase io_prometheus_client.MetricType_COUNTER:\n\t\t\t\t_, _ = fmt.Fprintf(w, \"%s %s\\n\", name, format(*m.Counter.Value))\n\t\t\tcase io_prometheus_client.MetricType_HISTOGRAM:\n\t\t\t\t_, _ = fmt.Fprintf(w, \"%s_total %d\\n\", name, *m.Histogram.SampleCount)\n\t\t\t\t_, _ = fmt.Fprintf(w, \"%s_sum %s\\n\", name, format(*m.Histogram.SampleSum))\n\t\t\tcase io_prometheus_client.MetricType_SUMMARY:\n\t\t\t}\n\t\t}\n\t}\n\treturn w.Bytes()\n}\n\nfunc writeProgress(item1, item2 *uint64, out io.Writer, done chan struct{}) {\n\twb := utils.NewBuffer(17)\n\twb.Put8(meta.CPROGRESS)\n\tif item2 == nil {\n\t\titem2 = new(uint64)\n\t}\n\tticker := time.NewTicker(time.Millisecond * 300)\n\tfor {\n\t\tselect {\n\t\tcase <-ticker.C:\n\t\t\twb.Put64(atomic.LoadUint64(item1))\n\t\t\twb.Put64(atomic.LoadUint64(item2))\n\t\t\t_, _ = out.Write(wb.Bytes())\n\t\t\twb.Seek(1)\n\t\tcase <-done:\n\t\t\tticker.Stop()\n\t\t\tif *item1 > 0 || *item2 > 0 {\n\t\t\t\twb.Put64(atomic.LoadUint64(item1))\n\t\t\t\twb.Put64(atomic.LoadUint64(item2))\n\t\t\t\t_, _ = out.Write(wb.Bytes())\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n\ntype obj struct {\n\tkey string\n\tsize, off, len uint32\n}\n\nfunc (v *VFS) calcObjects(id uint64, size, offset, length uint32) []*obj {\n\tif id == 0 {\n\t\treturn []*obj{{\"\", size, offset, length}}\n\t}\n\tif length == 0 || offset+length > size {\n\t\tlogger.Warnf(\"Corrupt slice id %d size %d offset %d length %d\", id, size, offset, length)\n\t\treturn nil\n\t}\n\tbsize := uint32(v.Conf.Chunk.BlockSize)\n\tvar prefix string\n\tif v.Conf.Chunk.HashPrefix {\n\t\tprefix = fmt.Sprintf(\"%s/chunks/%02X/%v/%v\", v.Conf.Format.Name, id%256, id/1000/1000, id)\n\t} else {\n\t\tprefix = fmt.Sprintf(\"%s/chunks/%v/%v/%v\", v.Conf.Format.Name, id/1000/1000, id/1000, id)\n\t}\n\tfirst := offset / bsize\n\tlast := (offset + length - 1) / bsize\n\tobjs := make([]*obj, 0, last-first+1)\n\tfor indx := first; indx <= last; indx++ {\n\t\tobjs = append(objs, &obj{fmt.Sprintf(\"%s_%d_%d\", prefix, indx, bsize), bsize, 0, bsize})\n\t}\n\tfo, lo := objs[0], objs[len(objs)-1]\n\tfo.off = offset - first*bsize\n\tfo.len = fo.size - fo.off\n\tif (last+1)*bsize > size {\n\t\tlo.size = size - last*bsize\n\t\tlo.key = fmt.Sprintf(\"%s_%d_%d\", prefix, last, lo.size)\n\t}\n\tlo.len = (offset + length) - last*bsize - lo.off\n\n\treturn objs\n}\n\ntype InfoResponse struct {\n\tIno Ino\n\tFailed bool\n\tReason string\n\tSummary meta.Summary\n\tPaths []string\n\tChunks []*chunkSlice\n\tObjects []*chunkObj\n\tPLocks []meta.PLockItem\n\tFLocks []meta.FLockItem\n}\n\ntype SummaryReponse struct {\n\tErrno syscall.Errno\n\tTree meta.TreeSummary\n}\n\ntype CacheResponse struct {\n\tsync.Mutex\n\tFileCount uint64\n\tSliceCount uint64\n\tTotalBytes uint64\n\tMissBytes uint64 // for check op\n\tLocations map[string]uint64\n}\n\nfunc (resp *CacheResponse) Add(other *CacheResponse) {\n\tresp.FileCount += other.FileCount\n\tresp.TotalBytes += other.TotalBytes\n\tresp.SliceCount += other.SliceCount\n\tresp.MissBytes += other.MissBytes\n\tfor k, bytes := range other.Locations {\n\t\tresp.Locations[k] += bytes\n\t}\n}\n\ntype chunkSlice struct {\n\tChunkIndex uint64\n\tmeta.Slice\n}\n\ntype chunkObj struct {\n\tChunkIndex uint64\n\tKey string\n\tSize, Off, Len uint32\n}\n\nfunc (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, out io.Writer) {\n\tswitch cmd {\n\tcase meta.Rmr:\n\t\tdone := make(chan struct{})\n\t\tinode := Ino(r.Get64())\n\t\tname := string(r.Get(int(r.Get8())))\n\t\tvar skipTrash bool\n\t\tvar numThreads int = meta.RmrDefaultThreads\n\t\tif r.HasMore() {\n\t\t\tskipTrash = r.Get8()&1 != 0\n\t\t}\n\t\tif r.HasMore() {\n\t\t\tnumThreads = int(r.Get8())\n\t\t}\n\t\tvar count uint64\n\t\tvar st syscall.Errno\n\t\tgo func() {\n\t\t\tlogger.Infof(\"Start to rmr %d/%s, workers=%d, skipTrash=%v\", inode, name, numThreads, skipTrash)\n\t\t\tst = v.Meta.Remove(ctx, inode, name, skipTrash, numThreads, &count)\n\t\t\tif st != 0 {\n\t\t\t\tlogger.Errorf(\"remove %d/%s: %s\", inode, name, st)\n\t\t\t}\n\t\t\tclose(done)\n\t\t}()\n\t\twriteProgress(&count, nil, out, done)\n\t\tif st == 0 && v.InvalidateEntry != nil {\n\t\t\tif st := v.InvalidateEntry(inode, name); st != 0 {\n\t\t\t\tlogger.Warnf(\"Invalidate entry %d/%s: %s\", inode, name, st)\n\t\t\t}\n\t\t}\n\t\t_, _ = out.Write([]byte{uint8(st)})\n\tcase meta.Clone:\n\t\tdone := make(chan struct{})\n\t\tsrcIno := Ino(r.Get64())\n\t\tsrcParentIno := Ino(r.Get64())\n\t\tdstParentIno := Ino(r.Get64())\n\t\tdstName := string(r.Get(int(r.Get8())))\n\t\tumask := r.Get16()\n\t\tcmode := r.Get8()\n\t\tvar concurrency uint8 = meta.CLONE_DEFAULT_CONCURRENCY // default for backward compatibility\n\t\tif r.HasMore() {\n\t\t\tconcurrency = r.Get8()\n\t\t}\n\t\tvar count, total uint64\n\t\tvar eno syscall.Errno\n\t\tgo func() {\n\t\t\tlogger.Infof(\"Start to clone %d/%d to %d/%s, cmode=%d, umask=%d, concurrency=%d\", srcParentIno, srcIno, dstParentIno, dstName, cmode, umask, concurrency)\n\t\t\tif eno = v.Meta.Clone(ctx, srcParentIno, srcIno, dstParentIno, dstName, cmode, umask, concurrency, &count, &total); eno != 0 {\n\t\t\t\tlogger.Errorf(\"clone failed srcIno:%d,dstParentIno:%d,dstName:%s,cmode:%d,umask:%d,concurrency:%d,eno:%v\", srcIno, dstParentIno, dstName, cmode, umask, concurrency, eno)\n\t\t\t}\n\t\t\tclose(done)\n\t\t}()\n\n\t\twriteProgress(&count, &total, out, done)\n\t\t_, _ = out.Write([]byte{uint8(eno)})\n\n\tcase meta.LegacyInfo:\n\t\tvar summary meta.Summary\n\t\tinode := Ino(r.Get64())\n\t\tvar recursive uint8 = 1\n\t\tif r.HasMore() {\n\t\t\trecursive = r.Get8()\n\t\t}\n\t\tvar raw bool\n\t\tif r.HasMore() {\n\t\t\traw = r.Get8() != 0\n\t\t}\n\t\tlogger.Infof(\"Start to get legacy info of %d, recursive=%d\", inode, recursive)\n\n\t\twb := utils.NewBuffer(4)\n\t\tr := v.Meta.GetSummary(ctx, inode, &summary, recursive != 0, true)\n\t\tif r != 0 {\n\t\t\tmsg := r.Error()\n\t\t\twb.Put32(uint32(len(msg)))\n\t\t\t_, _ = out.Write(append(wb.Bytes(), msg...))\n\t\t\treturn\n\t\t}\n\t\tvar w = bytes.NewBuffer(nil)\n\t\tfmt.Fprintf(w, \" inode: %d\\n\", inode)\n\t\tfmt.Fprintf(w, \" files: %d\\n\", summary.Files)\n\t\tfmt.Fprintf(w, \" dirs: %d\\n\", summary.Dirs)\n\t\tfmt.Fprintf(w, \" length: %s\\n\", utils.FormatBytes(summary.Length))\n\t\tfmt.Fprintf(w, \" size: %s\\n\", utils.FormatBytes(summary.Size))\n\t\tps := v.Meta.GetPaths(ctx, inode)\n\t\tswitch len(ps) {\n\t\tcase 0:\n\t\t\tfmt.Fprintf(w, \" path: %s\\n\", \"unknown\")\n\t\tcase 1:\n\t\t\tfmt.Fprintf(w, \" path: %s\\n\", ps[0])\n\t\tdefault:\n\t\t\tfmt.Fprintf(w, \" paths:\\n\")\n\t\t\tfor _, p := range ps {\n\t\t\t\tfmt.Fprintf(w, \"\\t%s\\n\", p)\n\t\t\t}\n\t\t}\n\t\tif summary.Files == 1 && summary.Dirs == 0 {\n\t\t\tif raw {\n\t\t\t\tfmt.Fprintf(w, \" chunks:\\n\")\n\t\t\t} else {\n\t\t\t\tfmt.Fprintf(w, \"objects:\\n\")\n\t\t\t}\n\t\t\tfor indx := uint64(0); indx*meta.ChunkSize < summary.Length; indx++ {\n\t\t\t\tvar cs []meta.Slice\n\t\t\t\t_ = v.Meta.Read(ctx, inode, uint32(indx), &cs)\n\t\t\t\tfor _, c := range cs {\n\t\t\t\t\tif raw {\n\t\t\t\t\t\tfmt.Fprintf(w, \"\\t%d:\\t%d\\t%d\\t%d\\t%d\\n\", indx, c.Id, c.Size, c.Off, c.Len)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfor _, o := range v.calcObjects(c.Id, c.Size, c.Off, c.Len) {\n\t\t\t\t\t\t\tfmt.Fprintf(w, \"\\t%d:\\t%s\\t%d\\t%d\\t%d\\n\", indx, o.key, o.size, o.off, o.len)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\twb.Put32(uint32(w.Len()))\n\t\t_, _ = out.Write(append(wb.Bytes(), w.Bytes()...))\n\tcase meta.InfoV2:\n\t\tinode := Ino(r.Get64())\n\t\tinfo := &InfoResponse{\n\t\t\tIno: inode,\n\t\t}\n\n\t\tvar recursive uint8 = 1\n\t\tif r.HasMore() {\n\t\t\trecursive = r.Get8()\n\t\t}\n\t\tvar raw bool\n\t\tif r.HasMore() {\n\t\t\traw = r.Get8() != 0\n\t\t}\n\t\tvar strict bool\n\t\tif r.HasMore() {\n\t\t\tstrict = r.Get8() != 0\n\t\t}\n\n\t\tdone := make(chan struct{})\n\t\tvar r syscall.Errno\n\t\tgo func() {\n\t\t\tlogger.Infof(\"Start to get info v2 of %d, recursive=%d\", inode, recursive)\n\t\t\tr = v.Meta.GetSummary(ctx, inode, &info.Summary, recursive != 0, strict)\n\t\t\tclose(done)\n\t\t}()\n\t\twriteProgress(&info.Summary.Files, &info.Summary.Size, out, done)\n\t\tif r != 0 {\n\t\t\tinfo.Failed = true\n\t\t\tinfo.Reason = r.Error()\n\t\t} else {\n\t\t\tinfo.Paths = v.Meta.GetPaths(ctx, inode)\n\t\t\tif info.Summary.Files == 1 && info.Summary.Dirs == 0 {\n\t\t\t\tfor indx := uint64(0); indx*meta.ChunkSize < info.Summary.Length; indx++ {\n\t\t\t\t\tvar cs []meta.Slice\n\t\t\t\t\t_ = v.Meta.Read(ctx, inode, uint32(indx), &cs)\n\t\t\t\t\tfor _, c := range cs {\n\t\t\t\t\t\tif raw {\n\t\t\t\t\t\t\tinfo.Chunks = append(info.Chunks, &chunkSlice{indx, c})\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tfor _, o := range v.calcObjects(c.Id, c.Size, c.Off, c.Len) {\n\t\t\t\t\t\t\t\tinfo.Objects = append(info.Objects, &chunkObj{indx, o.key, o.size, o.off, o.len})\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar err error\n\t\t\tif info.PLocks, info.FLocks, err = v.Meta.ListLocks(ctx, inode); err != nil {\n\t\t\t\tinfo.Failed = true\n\t\t\t\tinfo.Reason = err.Error()\n\t\t\t}\n\t\t}\n\t\tdata, err := json.Marshal(info)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"marshal info response: %v\", err)\n\t\t\t_, _ = out.Write([]byte{byte(syscall.EIO & 0xff)})\n\t\t\treturn\n\t\t}\n\t\tw := utils.NewBuffer(uint32(1 + 4 + len(data)))\n\t\tw.Put8(meta.CDATA)\n\t\tw.Put32(uint32(len(data)))\n\t\tw.Put(data)\n\t\t_, _ = out.Write(w.Bytes())\n\tcase meta.OpSummary:\n\t\tinode := Ino(r.Get64())\n\t\ttree := meta.TreeSummary{\n\t\t\tInode: inode,\n\t\t\tPath: \"\",\n\t\t\tType: meta.TypeDirectory,\n\t\t}\n\n\t\tvar depth uint8 = 3\n\t\tif r.HasMore() {\n\t\t\tdepth = r.Get8()\n\t\t}\n\t\tvar topN uint8 = 10\n\t\tif r.HasMore() {\n\t\t\ttopN = r.Get8()\n\t\t}\n\t\tvar strict bool\n\t\tif r.HasMore() {\n\t\t\tstrict = r.Get8() != 0\n\t\t}\n\n\t\tdone := make(chan struct{})\n\t\tvar files, size uint64\n\t\tvar r syscall.Errno\n\t\tgo func() {\n\t\t\tlogger.Infof(\"Start to get summary of %d, depth=%d, topN=%d\", inode, depth, topN)\n\t\t\tr = v.Meta.GetTreeSummary(ctx, &tree, depth, topN, strict,\n\t\t\t\tfunc(count, bytes uint64) {\n\t\t\t\t\tatomic.AddUint64(&files, count)\n\t\t\t\t\tatomic.AddUint64(&size, bytes)\n\t\t\t\t})\n\t\t\tclose(done)\n\t\t}()\n\t\twriteProgress(&files, &size, out, done)\n\t\tdata, err := json.Marshal(&SummaryReponse{r, tree})\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"marshal summary response: %v\", err)\n\t\t\t_, _ = out.Write([]byte{byte(syscall.EIO & 0xff)})\n\t\t\treturn\n\t\t}\n\t\tw := utils.NewBuffer(uint32(1 + 4 + len(data)))\n\t\tw.Put8(meta.CDATA)\n\t\tw.Put32(uint32(len(data)))\n\t\tw.Put(data)\n\t\t_, _ = out.Write(w.Bytes())\n\tcase meta.CompactPath:\n\t\tinode := Ino(r.Get64())\n\t\tcoCnt := r.Get16()\n\n\t\tdone := make(chan struct{})\n\t\tvar totalChunks, currChunks uint64\n\t\tvar eno syscall.Errno\n\t\tgo func() {\n\t\t\tlogger.Infof(\"Start to compact %d with %d workers\", inode, coCnt)\n\t\t\teno = v.Meta.Compact(ctx, inode, int(coCnt), func() {\n\t\t\t\tatomic.AddUint64(&totalChunks, 1)\n\t\t\t}, func() {\n\t\t\t\tatomic.AddUint64(&currChunks, 1)\n\t\t\t})\n\t\t\tclose(done)\n\t\t}()\n\n\t\twriteProgress(&totalChunks, &currChunks, out, done)\n\t\t_, _ = out.Write([]byte{uint8(eno)})\n\n\tcase meta.FillCache:\n\t\tpaths := strings.Split(string(r.Get(int(r.Get32()))), \"\\n\")\n\t\tconcurrent := r.Get16()\n\t\tbackground := r.Get8()\n\n\t\taction := WarmupCache\n\t\tif r.HasMore() {\n\t\t\taction = CacheAction(r.Get8())\n\t\t}\n\n\t\tlogger.Infof(\"Start to %s %d paths with %d workers, background=%d\", action, len(paths), concurrent, background)\n\t\tstat := &CacheResponse{Locations: make(map[string]uint64)}\n\t\tif background == 0 {\n\t\t\tdone := make(chan struct{})\n\t\t\tgo func() {\n\t\t\t\tv.cacheFiller.Cache(ctx, action, paths, int(concurrent), stat)\n\t\t\t\tclose(done)\n\t\t\t}()\n\t\t\twriteProgress(&stat.FileCount, &stat.TotalBytes, out, done)\n\t\t} else {\n\t\t\tgo v.cacheFiller.Cache(meta.NewContext(ctx.Pid(), ctx.Uid(), ctx.Gids()), action, paths, int(concurrent), nil)\n\t\t}\n\t\tdata, err := json.Marshal(stat)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"marshal response error: %v\", err)\n\t\t\t_, _ = out.Write([]byte{byte(syscall.EIO & 0xff)})\n\t\t\treturn\n\t\t}\n\t\tw := utils.NewBuffer(uint32(1 + 4 + len(data)))\n\t\tw.Put8(meta.CDATA)\n\t\tw.Put32(uint32(len(data)))\n\t\tw.Put(data)\n\t\t_, _ = out.Write(w.Bytes())\n\tdefault:\n\t\tlogger.Warnf(\"unknown message type: %d\", cmd)\n\t\t_, _ = out.Write([]byte{byte(syscall.EINVAL & 0xff)})\n\t}\n}\n" }, { "path": "pkg/vfs/reader.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"runtime\"\n\t\"sort\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\n/*\n * state of sliceReader\n *\n * <-- REFRESH\n * | |\n * NEW -> BUSY -> READY\n * | |\n * BREAK ---> INVALID\n */\nconst (\n\tNEW = iota\n\tBUSY\n\tREFRESH\n\tBREAK\n\tREADY\n\tINVALID\n)\n\nconst readSessions = 2\n\nvar readBufferUsed atomic.Int64\n\ntype sstate uint8\n\nfunc (m sstate) valid() bool { return m != BREAK && m != INVALID }\n\nvar stateNames = []string{\"NEW\", \"BUSY\", \"REFRESH\", \"BREAK\", \"READY\", \"INVALID\"}\n\nfunc (m sstate) String() string {\n\tif m <= INVALID {\n\t\treturn stateNames[m]\n\t}\n\tpanic(\"\")\n}\n\ntype FileReader interface {\n\tRead(ctx meta.Context, off uint64, buf []byte) (int, syscall.Errno)\n\tGetLength() uint64\n\tClose(ctx meta.Context)\n}\n\ntype DataReader interface {\n\tOpen(inode Ino, length uint64) FileReader\n\tTruncate(inode Ino, length uint64)\n\tInvalidate(inode Ino, off, length uint64)\n}\n\ntype frange struct {\n\toff uint64\n\tlen uint64\n}\n\nfunc (r *frange) String() string { return fmt.Sprintf(\"[%d,%d,%d)\", r.off, r.len, r.end()) }\nfunc (r *frange) end() uint64 { return r.off + r.len }\nfunc (r *frange) contain(p uint64) bool { return r.off < p && p < r.end() }\nfunc (r *frange) overlap(a *frange) bool { return a.off < r.end() && r.off < a.end() }\nfunc (r *frange) include(a *frange) bool { return r.off <= a.off && a.end() <= r.end() }\n\n// protected by file\ntype sliceReader struct {\n\tctx context.Context\n\tcancel context.CancelFunc\n\tfile *fileReader\n\tblock *frange\n\tstate sstate\n\tpage *chunk.Page\n\tindx uint32\n\tcurrentPos uint32\n\tlastAccess time.Time\n\tcond *utils.Cond\n\tnext *sliceReader\n\tprev **sliceReader\n\trefs uint16\n}\n\nfunc (s *sliceReader) delay(delay time.Duration) {\n\ttime.AfterFunc(delay, s.run)\n}\n\nfunc (s *sliceReader) done(err syscall.Errno, delay time.Duration) {\n\tf := s.file\n\tswitch s.state {\n\tcase BUSY:\n\t\ts.state = NEW // failed\n\tcase BREAK:\n\t\ts.state = INVALID\n\tcase REFRESH:\n\t\ts.state = NEW\n\t}\n\tif err != 0 {\n\t\tif !f.closing {\n\t\t\tlogger.Errorf(\"read file %d: %s\", f.inode, err)\n\t\t}\n\t\tf.err = err\n\t}\n\tif f.shouldStop() {\n\t\ts.state = INVALID\n\t}\n\n\tswitch s.state {\n\tcase NEW:\n\t\ts.delay(delay)\n\tcase READY:\n\t\ts.cond.Broadcast()\n\tcase INVALID:\n\t\tif s.refs == 0 {\n\t\t\ts.delete()\n\t\t\tif f.closing && f.slices == nil {\n\t\t\t\tf.r.Lock()\n\t\t\t\tif f.refs == 0 {\n\t\t\t\t\tf.delete()\n\t\t\t\t}\n\t\t\t\tf.r.Unlock()\n\t\t\t}\n\t\t} else {\n\t\t\ts.cond.Broadcast()\n\t\t}\n\t}\n\truntime.Goexit()\n}\n\nfunc retry_time(trycnt uint32) time.Duration {\n\tif trycnt < 30 {\n\t\treturn time.Millisecond * time.Duration((trycnt-1)*300+1)\n\t}\n\treturn time.Second * 10\n}\n\nfunc (s *sliceReader) run() {\n\tf := s.file\n\tf.Lock()\n\tdefer f.Unlock()\n\tif s.state != NEW || f.shouldStop() {\n\t\ts.done(0, 0)\n\t}\n\ts.state = BUSY\n\tindx := s.indx\n\tinode := f.inode\n\tf.Unlock()\n\n\tvar slices []meta.Slice\n\terr := f.r.m.Read(meta.Background(), inode, indx, &slices)\n\tf.Lock()\n\tlength := f.length\n\tif s.state != BUSY || f.shouldStop() {\n\t\ts.done(0, 0)\n\t}\n\tif err == syscall.ENOENT {\n\t\ts.done(err, 0)\n\t} else if err != 0 {\n\t\tf.tried++\n\t\ttrycnt := f.tried\n\t\tif trycnt > f.r.maxRetries {\n\t\t\ts.done(syscall.EIO, 0)\n\t\t} else {\n\t\t\ts.done(0, retry_time(trycnt))\n\t\t}\n\t}\n\n\ts.currentPos = 0\n\tif s.block.off > length {\n\t\ts.block.len = 0\n\t\ts.state = READY\n\t\ts.done(0, 0)\n\t} else if s.block.end() > length {\n\t\ts.block.len = length - s.block.off\n\t}\n\tneed := s.block.len\n\tf.Unlock()\n\n\tp := s.page.Slice(0, int(need))\n\tdefer p.Release()\n\tvar n int\n\n\tctx := context.WithValue(s.ctx, meta.CtxKey(\"inode\"), inode) // Output inode in log for debugging\n\tn = f.r.Read(ctx, p, slices, (uint32(s.block.off))%meta.ChunkSize)\n\n\tf.Lock()\n\tif s.state != BUSY || f.shouldStop() {\n\t\ts.done(0, 0)\n\t}\n\tif n == int(need) {\n\t\ts.state = READY\n\t\ts.currentPos = uint32(n)\n\t\ts.file.tried = 0\n\t\ts.lastAccess = time.Now()\n\t\ts.done(0, 0)\n\t} else {\n\t\ts.currentPos = 0 // start again from beginning\n\t\terr = syscall.EIO\n\t\tf.tried++\n\t\t_ = f.r.m.InvalidateChunkCache(meta.Background(), inode, indx)\n\t\tif f.tried > f.r.maxRetries {\n\t\t\ts.done(err, 0)\n\t\t} else {\n\t\t\ts.done(0, retry_time(f.tried))\n\t\t}\n\t}\n}\n\nfunc (s *sliceReader) invalidate() {\n\tswitch s.state {\n\tcase NEW:\n\tcase BUSY:\n\t\ts.state = REFRESH\n\t\t// TODO cancel ongoing read\n\tcase READY:\n\t\tif s.refs > 0 {\n\t\t\ts.state = NEW\n\t\t\tgo s.run()\n\t\t} else {\n\t\t\ts.state = INVALID\n\t\t\ts.delete() // nobody wants it anymore, so delete it\n\t\t}\n\t}\n}\n\nfunc (s *sliceReader) drop() {\n\tif s.state <= BREAK {\n\t\tif s.refs == 0 {\n\t\t\ts.state = BREAK\n\t\t\ts.cancel()\n\t\t}\n\t} else {\n\t\tif s.refs == 0 {\n\t\t\ts.delete() // nobody wants it anymore, so delete it\n\t\t} else if s.state == READY {\n\t\t\ts.state = INVALID // somebody still using it, so mark it for removal\n\t\t}\n\t}\n}\n\nfunc (s *sliceReader) delete() {\n\t*(s.prev) = s.next\n\tif s.next != nil {\n\t\ts.next.prev = s.prev\n\t} else {\n\t\ts.file.last = s.prev\n\t}\n\treadBufferUsed.Add(-int64(cap(s.page.Data)))\n\ts.page.Release()\n}\n\ntype session struct {\n\tlastOffset uint64\n\ttotal uint64\n\treadahead uint64\n\tatime time.Time\n}\n\ntype fileReader struct {\n\t// protected by itself\n\tinode Ino\n\tlength uint64\n\terr syscall.Errno\n\ttried uint32\n\tsessions [readSessions]session\n\tslices *sliceReader\n\tlast **sliceReader\n\n\tsync.Mutex\n\tclosing bool\n\n\t// protected by r\n\trefs uint16\n\tnext *fileReader\n\tr *dataReader\n}\n\nfunc (f *fileReader) GetLength() uint64 {\n\tf.Lock()\n\tdefer f.Unlock()\n\treturn f.length\n}\n\n// protected by f\nfunc (f *fileReader) newSlice(block *frange) *sliceReader {\n\ts := &sliceReader{}\n\ts.ctx, s.cancel = context.WithCancel(context.Background())\n\ts.file = f\n\ts.lastAccess = time.Now()\n\ts.indx = uint32(block.off / meta.ChunkSize)\n\ts.block = &frange{block.off, block.len} // random read\n\tblockend := (block.off/f.r.blockSize + 1) * f.r.blockSize\n\tif s.block.end() > f.length {\n\t\ts.block.len = f.length - s.block.off\n\t}\n\tif s.block.end() > blockend {\n\t\ts.block.len = blockend - s.block.off\n\t}\n\tblock.off = s.block.end()\n\tblock.len -= s.block.len\n\ts.page = chunk.NewOffPage(int(s.block.len))\n\ts.cond = utils.NewCond(&f.Mutex)\n\ts.prev = f.last\n\t*(f.last) = s\n\tf.last = &(s.next)\n\tgo s.run()\n\treadBufferUsed.Add(int64(cap(s.page.Data)))\n\treturn s\n}\n\nfunc (f *fileReader) delete() {\n\tr := f.r\n\ti := r.files[f.inode]\n\tif i == f {\n\t\tif i.next != nil {\n\t\t\tr.files[f.inode] = i.next\n\t\t} else {\n\t\t\tdelete(r.files, f.inode)\n\t\t}\n\t} else {\n\t\tfor i != nil {\n\t\t\tif i.next == f {\n\t\t\t\ti.next = f.next\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ti = i.next\n\t\t}\n\t}\n\tf.next = nil\n}\n\nfunc (f *fileReader) acquire() {\n\tf.r.Lock()\n\tdefer f.r.Unlock()\n\tf.refs++\n}\n\nfunc (f *fileReader) release() {\n\tf.r.Lock()\n\tdefer f.r.Unlock()\n\tf.refs--\n\tif f.refs == 0 && f.slices == nil {\n\t\tf.delete()\n\t}\n}\n\nfunc (f *fileReader) guessSession(block *frange) int {\n\tidx := -1\n\tvar closestOff uint64\n\tfor i, ses := range f.sessions {\n\t\tif ses.lastOffset > closestOff && ses.lastOffset <= block.off && block.off <= ses.lastOffset+ses.readahead+f.r.blockSize {\n\t\t\tidx = i\n\t\t\tclosestOff = ses.lastOffset\n\t\t}\n\t}\n\tif idx == -1 {\n\t\tfor i, ses := range f.sessions {\n\t\t\tbt := ses.readahead / 8\n\t\t\tif bt < f.r.blockSize {\n\t\t\t\tbt = f.r.blockSize\n\t\t\t}\n\t\t\tmin := ses.lastOffset - bt\n\t\t\tif ses.lastOffset < bt {\n\t\t\t\tmin = 0\n\t\t\t}\n\t\t\tif min <= block.off && block.off < ses.lastOffset && (closestOff == 0 || ses.lastOffset < closestOff) {\n\t\t\t\tidx = i\n\t\t\t\tclosestOff = ses.lastOffset\n\t\t\t}\n\t\t}\n\t}\n\tif idx == -1 {\n\t\tfor i, ses := range f.sessions {\n\t\t\tif ses.total == 0 {\n\t\t\t\tidx = i\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif idx == -1 || ses.atime.Before(f.sessions[idx].atime) {\n\t\t\t\tidx = i\n\t\t\t}\n\t\t}\n\t\tf.sessions[idx].lastOffset = block.off\n\t\tf.sessions[idx].total = block.len\n\t\tf.sessions[idx].readahead = 0\n\t} else {\n\t\tif block.end() > f.sessions[idx].lastOffset {\n\t\t\tf.sessions[idx].total += block.end() - f.sessions[idx].lastOffset\n\t\t}\n\t}\n\tf.sessions[idx].atime = time.Now()\n\treturn idx\n}\n\nfunc (f *fileReader) checkReadahead(block *frange) int {\n\tidx := f.guessSession(block)\n\tses := &f.sessions[idx]\n\tseqdata := ses.total\n\treadahead := ses.readahead\n\tused := uint64(readBufferUsed.Load())\n\tif readahead == 0 && f.r.blockSize <= f.r.readAheadMax && (block.off == 0 || seqdata > block.len) { // begin with read-ahead turned on\n\t\tses.readahead = f.r.blockSize\n\t} else if readahead < f.r.readAheadMax && seqdata >= readahead && f.r.readAheadTotal > used+readahead*4 {\n\t\tses.readahead *= 2\n\t} else if readahead >= f.r.blockSize && (f.r.readAheadTotal < used+readahead/2 || seqdata < readahead/4) {\n\t\tses.readahead /= 2\n\t}\n\tif ses.readahead >= f.r.blockSize {\n\t\tahead := frange{block.end(), ses.readahead}\n\t\tf.readAhead(&ahead)\n\t}\n\tif block.end() > ses.lastOffset {\n\t\tses.lastOffset = block.end()\n\t}\n\treturn idx\n}\n\nfunc (f *fileReader) need(block *frange) bool {\n\tfor _, ses := range f.sessions {\n\t\tif ses.total == 0 {\n\t\t\tbreak\n\t\t}\n\t\tbt := ses.readahead / 8\n\t\tif bt < f.r.blockSize {\n\t\t\tbt = f.r.blockSize\n\t\t}\n\t\tb := &frange{ses.lastOffset - bt, ses.readahead*2 + f.r.blockSize*2}\n\t\tif ses.lastOffset < bt {\n\t\t\tb.off = 0\n\t\t}\n\t\tif block.overlap(b) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// cleanup unused requests\nfunc (f *fileReader) cleanupRequests(block *frange) {\n\tnow := time.Now()\n\tvar cnt int\n\tf.visit(func(s *sliceReader) bool {\n\t\tif !s.state.valid() ||\n\t\t\t!block.overlap(s.block) && (s.lastAccess.Add(time.Second*30).Before(now) || !f.need(s.block)) {\n\t\t\ts.drop()\n\t\t} else if !block.overlap(s.block) {\n\t\t\tcnt++\n\t\t}\n\t\treturn true\n\t})\n\tf.visit(func(s *sliceReader) bool {\n\t\tif !block.overlap(s.block) && cnt > f.r.maxRequests {\n\t\t\ts.drop()\n\t\t\tcnt--\n\t\t}\n\t\treturn cnt > f.r.maxRequests\n\t})\n}\n\nfunc (f *fileReader) releaseIdleBuffer() {\n\tf.Lock()\n\tdefer f.Unlock()\n\tnow := time.Now()\n\tvar idle = time.Minute\n\tused := readBufferUsed.Load()\n\tif used > int64(f.r.readAheadTotal) {\n\t\tidle /= time.Duration(used / int64(f.r.readAheadTotal))\n\t}\n\tf.visit(func(s *sliceReader) bool {\n\t\tif !s.state.valid() || s.lastAccess.Add(idle).Before(now) || !f.need(s.block) {\n\t\t\ts.drop()\n\t\t}\n\t\treturn true\n\t})\n}\n\nfunc (f *fileReader) splitRange(block *frange) []uint64 {\n\tranges := []uint64{block.off, block.end()}\n\tcontain := func(p uint64) bool {\n\t\tfor _, i := range ranges {\n\t\t\tif i == p {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n\tf.visit(func(s *sliceReader) bool {\n\t\tif s.state.valid() {\n\t\t\tif block.contain(s.block.off) && !contain(s.block.off) {\n\t\t\t\tranges = append(ranges, s.block.off)\n\t\t\t}\n\t\t\tif block.contain(s.block.end()) && !contain(s.block.end()) {\n\t\t\t\tranges = append(ranges, s.block.end())\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\tsort.Slice(ranges, func(i, j int) bool {\n\t\treturn ranges[i] < ranges[j]\n\t})\n\treturn ranges\n}\n\n// protected by f\nfunc (f *fileReader) readAhead(block *frange) {\n\tf.visit(func(r *sliceReader) bool {\n\t\tif r.state.valid() && r.block.off <= block.off && r.block.end() > block.off {\n\t\t\tif r.state == READY && block.len > f.r.blockSize && r.block.off == block.off && r.block.off%f.r.blockSize == 0 {\n\t\t\t\t// next block is ready, reduce readahead by a block\n\t\t\t\tblock.len -= f.r.blockSize / 2\n\t\t\t}\n\t\t\tif r.block.end() <= block.end() {\n\t\t\t\tblock.len = block.end() - r.block.end()\n\t\t\t} else {\n\t\t\t\tblock.len = 0\n\t\t\t}\n\t\t\tblock.off = r.block.end()\n\t\t}\n\t\treturn true\n\t})\n\tif block.len > 0 && block.off < f.length && uint64(readBufferUsed.Load()) < f.r.readAheadTotal {\n\t\tif block.len < f.r.blockSize {\n\t\t\tblock.len += f.r.blockSize - block.end()%f.r.blockSize // align to end of a block\n\t\t}\n\t\tf.newSlice(block)\n\t\tif block.len > 0 {\n\t\t\tf.readAhead(block)\n\t\t}\n\t}\n}\n\ntype req struct {\n\tfrange\n\ts *sliceReader\n}\n\nfunc (f *fileReader) prepareRequests(ranges []uint64) []*req {\n\tvar reqs []*req\n\tedges := len(ranges)\n\tfor i := 0; i < edges-1; i++ {\n\t\tvar added bool\n\t\tb := frange{ranges[i], ranges[i+1] - ranges[i]}\n\t\tf.visit(func(s *sliceReader) bool {\n\t\t\tif !added && s.state.valid() && s.block.include(&b) {\n\t\t\t\ts.refs++\n\t\t\t\ts.lastAccess = time.Now()\n\t\t\t\treqs = append(reqs, &req{frange{ranges[i] - s.block.off, b.len}, s})\n\t\t\t\tadded = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tif !added {\n\t\t\tfor b.len > 0 {\n\t\t\t\ts := f.newSlice(&b)\n\t\t\t\ts.refs++\n\t\t\t\treqs = append(reqs, &req{frange{0, s.block.len}, s})\n\t\t\t}\n\t\t}\n\t}\n\treturn reqs\n}\n\nfunc (f *fileReader) shouldStop() bool {\n\treturn f.err != 0 || f.closing\n}\n\nfunc (f *fileReader) waitForIO(ctx meta.Context, reqs []*req, buf []byte) (int, syscall.Errno) {\n\tstart := time.Now()\n\tfor _, req := range reqs {\n\t\ts := req.s\n\t\tfor s.state != READY && uint64(s.currentPos) < s.block.len {\n\t\t\tif s.cond.WaitWithTimeout(time.Second) {\n\t\t\t\tif ctx.Canceled() {\n\t\t\t\t\tlogger.Warnf(\"read %d interrupted after %s\", f.inode, time.Since(start))\n\t\t\t\t\treturn 0, syscall.EINTR\n\t\t\t\t}\n\t\t\t}\n\t\t\tif f.shouldStop() {\n\t\t\t\treturn 0, f.err\n\t\t\t}\n\t\t}\n\t}\n\n\tvar n int\n\tfor _, req := range reqs {\n\t\ts := req.s\n\t\tif req.off < s.block.len && s.block.off+req.off < f.length {\n\t\t\tif req.end() > s.block.len {\n\t\t\t\tlogger.Warnf(\"not enough bytes (%d < %d), restart read\", s.block.len, req.end())\n\t\t\t\treturn 0, syscall.EAGAIN\n\t\t\t}\n\t\t\tif s.block.off+req.end() > f.length {\n\t\t\t\treq.len = f.length - s.block.off - req.off\n\t\t\t}\n\t\t\tn += copy(buf[n:], s.page.Data[req.off:req.end()])\n\t\t}\n\t}\n\treturn n, 0\n}\n\nfunc (f *fileReader) Read(ctx meta.Context, offset uint64, buf []byte) (int, syscall.Errno) {\n\tif f.r.readBufferUsed() > f.r.bufferSize {\n\t\ttime.Sleep(time.Millisecond * 10) // slow down\n\t\tfor f.r.readBufferUsed() > f.r.bufferSize*2 { // readahead uses 80% of buffer, stop here to avoid OOM\n\t\t\ttime.Sleep(time.Millisecond * 100)\n\t\t}\n\t}\n\tf.Lock()\n\tdefer f.Unlock()\n\tf.acquire()\n\tdefer f.release()\n\n\tif f.shouldStop() {\n\t\treturn 0, f.err\n\t}\n\n\tsize := uint64(len(buf))\n\tif offset >= f.length || size == 0 {\n\t\treturn 0, 0\n\t}\n\tblock := &frange{offset, size}\n\tif block.end() > f.length {\n\t\tblock.len = f.length - block.off\n\t}\n\n\tf.cleanupRequests(block)\n\tvar lastBS uint64 = 32 << 10\n\tif block.off+lastBS > f.length {\n\t\tlastblock := frange{f.length - lastBS, lastBS}\n\t\tif f.length < lastBS {\n\t\t\tlastblock = frange{0, f.length}\n\t\t}\n\t\tf.readAhead(&lastblock)\n\t}\n\tranges := f.splitRange(block)\n\treqs := f.prepareRequests(ranges)\n\tdefer func() {\n\t\tfor _, req := range reqs {\n\t\t\ts := req.s\n\t\t\ts.refs--\n\t\t\tif s.refs == 0 && s.state == INVALID {\n\t\t\t\ts.delete()\n\t\t\t}\n\t\t}\n\t}()\n\tf.checkReadahead(block)\n\treturn f.waitForIO(ctx, reqs, buf)\n}\n\nfunc (f *fileReader) visit(fn func(s *sliceReader) bool) {\n\tvar next *sliceReader\n\tfor s := f.slices; s != nil; s = next {\n\t\tnext = s.next\n\t\tif !fn(s) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (f *fileReader) Close(ctx meta.Context) {\n\tf.Lock()\n\tf.closing = true\n\tf.visit(func(s *sliceReader) bool {\n\t\ts.drop()\n\t\treturn true\n\t})\n\tf.release()\n\tf.Unlock()\n}\n\ntype dataReader struct {\n\tsync.Mutex\n\tm meta.Meta\n\tstore chunk.ChunkStore\n\tfiles map[Ino]*fileReader\n\tblockSize uint64\n\tbufferSize int64\n\treadAheadMax uint64\n\treadAheadTotal uint64\n\tmaxRequests int\n\tmaxRetries uint32\n}\n\nfunc NewDataReader(conf *Config, m meta.Meta, store chunk.ChunkStore) DataReader {\n\tvar readAheadTotal = 256 << 20\n\tif conf.Chunk.BufferSize > 0 {\n\t\treadAheadTotal = int(conf.Chunk.BufferSize / 10 * 8) // 80% of total buffer\n\t}\n\treadAheadMax := min(conf.Chunk.Readahead, readAheadTotal)\n\tr := &dataReader{\n\t\tm: m,\n\t\tstore: store,\n\t\tfiles: make(map[Ino]*fileReader),\n\t\tblockSize: uint64(conf.Chunk.BlockSize),\n\t\tbufferSize: int64(conf.Chunk.BufferSize),\n\t\treadAheadTotal: uint64(readAheadTotal),\n\t\treadAheadMax: uint64(readAheadMax),\n\t\tmaxRequests: readAheadMax/conf.Chunk.BlockSize*readSessions + 1,\n\t\tmaxRetries: uint32(conf.Meta.Retries),\n\t}\n\tgo r.checkReadBuffer()\n\treturn r\n}\n\nfunc (r *dataReader) readBufferUsed() int64 {\n\tused := readBufferUsed.Load()\n\treturn used\n}\n\nfunc (r *dataReader) checkReadBuffer() {\n\tfor {\n\t\tr.Lock()\n\t\tfor _, f := range r.files {\n\t\t\tfor f != nil {\n\t\t\t\tr.Unlock()\n\t\t\t\tf.releaseIdleBuffer()\n\t\t\t\tr.Lock()\n\t\t\t\tf = f.next\n\t\t\t}\n\t\t}\n\t\tr.Unlock()\n\t\ttime.Sleep(time.Second)\n\t}\n}\n\nfunc (r *dataReader) Open(inode Ino, length uint64) FileReader {\n\tf := &fileReader{\n\t\tr: r,\n\t\tinode: inode,\n\t\tlength: length,\n\t}\n\tf.last = &(f.slices)\n\n\tr.Lock()\n\tf.refs = 1\n\tf.next = r.files[inode]\n\tr.files[inode] = f\n\tr.Unlock()\n\treturn f\n}\n\nfunc (r *dataReader) visit(inode Ino, fn func(*fileReader)) {\n\t// r could be hold inside f, so Unlock r first to avoid deadlock\n\tr.Lock()\n\tvar fs []*fileReader\n\tf := r.files[inode]\n\tfor f != nil {\n\t\tfs = append(fs, f)\n\t\tf = f.next\n\t}\n\tr.Unlock()\n\tfor _, f := range fs {\n\t\tf.Lock()\n\t\tfn(f)\n\t\tf.Unlock()\n\t}\n}\n\nfunc (r *dataReader) Truncate(inode Ino, length uint64) {\n\tr.visit(inode, func(f *fileReader) {\n\t\tif length < f.length {\n\t\t\tf.visit(func(s *sliceReader) bool {\n\t\t\t\tif s.block.off+s.block.len > length {\n\t\t\t\t\ts.invalidate()\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\t\tf.length = length\n\t})\n}\n\nfunc (r *dataReader) Invalidate(inode Ino, off, length uint64) {\n\tb := frange{off, length}\n\tr.visit(inode, func(f *fileReader) {\n\t\tif off+length > f.length {\n\t\t\tf.length = off + length\n\t\t}\n\t\tf.visit(func(s *sliceReader) bool {\n\t\t\tif b.overlap(s.block) {\n\t\t\t\ts.invalidate()\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t})\n}\n\nfunc (r *dataReader) readSlice(ctx context.Context, s *meta.Slice, page *chunk.Page, off int) error {\n\tbuf := page.Data\n\tread := 0\n\tif s.Id == 0 {\n\t\tfor read < len(buf) {\n\t\t\tbuf[read] = 0\n\t\t\tread++\n\t\t}\n\t\treturn nil\n\t}\n\n\treader := r.store.NewReader(s.Id, int(s.Size))\n\tfor read < len(buf) {\n\t\tp := page.Slice(read, len(buf)-read)\n\t\tn, err := reader.ReadAt(ctx, p, off+int(s.Off))\n\t\tp.Release()\n\t\tif n == 0 && err != nil {\n\t\t\tlogger.Warningf(\"fail to read sliceId %d (off:%d, size:%d, clen: %d, inode: %d): %s\",\n\t\t\t\ts.Id, off+int(s.Off), len(buf)-read, s.Size, ctx.Value(meta.CtxKey(\"inode\")), err)\n\t\t\treturn err\n\t\t}\n\t\tread += n\n\t\toff += n\n\t}\n\treturn nil\n}\n\nfunc (r *dataReader) Read(ctx context.Context, page *chunk.Page, slices []meta.Slice, offset uint32) int {\n\tif len(slices) > 16 {\n\t\treturn r.readManySlices(ctx, page, slices, offset)\n\t}\n\tread := 0\n\tvar pos uint32\n\terrs := make(chan error, 10)\n\twaits := 0\n\tbuf := page.Data\n\tsize := len(buf)\n\tfor i := 0; i < len(slices); i++ {\n\t\tif read < size && offset < pos+slices[i].Len {\n\t\t\ttoread := min(size-read, int(pos+slices[i].Len-offset))\n\t\t\tgo func(s *meta.Slice, p *chunk.Page, off, pos uint32) {\n\t\t\t\tdefer p.Release()\n\t\t\t\terrs <- r.readSlice(ctx, s, p, int(off))\n\t\t\t}(&slices[i], page.Slice(read, toread), offset-pos, pos)\n\t\t\tread += toread\n\t\t\toffset += uint32(toread)\n\t\t\twaits++\n\t\t}\n\t\tpos += slices[i].Len\n\t}\n\tfor read < size {\n\t\tbuf[read] = 0\n\t\tread++\n\t}\n\tvar err error\n\t// wait for all goroutine to return, otherwise they may access invalid memory\n\tfor waits > 0 {\n\t\tif e := <-errs; e != nil {\n\t\t\terr = e\n\t\t}\n\t\twaits--\n\t}\n\tif err != nil {\n\t\treturn 0\n\t}\n\treturn read\n}\n\nfunc (r *dataReader) readManySlices(ctx context.Context, page *chunk.Page, slices []meta.Slice, offset uint32) int {\n\tread := 0\n\tvar pos uint32\n\tvar err error\n\terrs := make(chan error, 10)\n\twaits := 0\n\tbuf := page.Data\n\tsize := len(buf)\n\tconcurrency := make(chan byte, 16)\n\nSLICES:\n\tfor i := 0; i < len(slices); i++ {\n\t\tif read < size && offset < pos+slices[i].Len {\n\t\t\ttoread := min(size-read, int(pos+slices[i].Len-offset))\n\t\tWAIT:\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase concurrency <- 1:\n\t\t\t\t\tbreak WAIT\n\t\t\t\tcase e := <-errs:\n\t\t\t\t\twaits--\n\t\t\t\t\tif e != nil {\n\t\t\t\t\t\terr = e\n\t\t\t\t\t\tbreak SLICES\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tgo func(s *meta.Slice, p *chunk.Page, off int, pos uint32) {\n\t\t\t\tdefer p.Release()\n\t\t\t\terrs <- r.readSlice(ctx, s, p, off)\n\t\t\t\t<-concurrency\n\t\t\t}(&slices[i], page.Slice(read, toread), int(offset-pos), pos)\n\n\t\t\tread += toread\n\t\t\toffset += uint32(toread)\n\t\t\twaits++\n\t\t}\n\t\tpos += slices[i].Len\n\t}\n\t// wait for all jobs done, otherwise they may access invalid memory\n\tfor waits > 0 {\n\t\tif e := <-errs; e != nil {\n\t\t\terr = e\n\t\t}\n\t\twaits--\n\t}\n\tif err != nil {\n\t\treturn 0\n\t}\n\tfor read < size {\n\t\tbuf[read] = 0\n\t\tread++\n\t}\n\treturn read\n}\n" }, { "path": "pkg/vfs/vfs.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"runtime\"\n\t\"sort\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\ntype Ino = meta.Ino\ntype Attr = meta.Attr\ntype Context = LogContext\n\nconst (\n\trootID = 1\n\tmaxName = meta.MaxName\n\tmaxSymlink = meta.MaxSymlink\n\tmaxFileSize = meta.ChunkSize << 31\n)\n\ntype Port struct {\n\tPrometheusAgent string `json:\",omitempty\"`\n\tDebugAgent string `json:\",omitempty\"`\n\tConsulAddr string `json:\",omitempty\"`\n\tPyroscopeAddr string `json:\",omitempty\"`\n}\n\n// FuseOptions contains options for fuse mount, keep the same structure with `fuse.MountOptions`\ntype FuseOptions struct {\n\tAllowOther bool\n\tOptions []string\n\tMaxBackground int\n\tMaxWrite int\n\tMaxReadAhead int\n\tIgnoreSecurityLabels bool // ignoring labels should be provided as a fusermount mount option.\n\tRememberInodes bool\n\tFsName string\n\tName string\n\tSingleThreaded bool\n\tDisableXAttrs bool\n\tDebug bool\n\tLogger *log.Logger `json:\"-\"`\n\tEnableLocks bool\n\tEnableSymlinkCaching bool `json:\",omitempty\"`\n\tExplicitDataCacheControl bool\n\tSyncRead bool `json:\",omitempty\"`\n\tDirectMount bool\n\tDirectMountStrict bool `json:\",omitempty\"`\n\tDirectMountFlags uintptr\n\tEnableAcl bool\n\tDisableReadDirPlus bool `json:\",omitempty\"`\n\tEnableReadDirPlusAuto bool\n\tEnableWriteback bool\n\tEnableIoctl bool `json:\",omitempty\"`\n\tDontUmask bool\n\tOtherCaps uint32\n\tNoAllocForRead bool\n\tTimeout time.Duration\n}\n\nfunc (o FuseOptions) StripOptions() FuseOptions {\n\toptions := o.Options\n\to.Options = make([]string, 0, len(o.Options))\n\tfor _, opt := range options {\n\t\tif opt == \"nonempty\" {\n\t\t\tcontinue\n\t\t}\n\t\to.Options = append(o.Options, opt)\n\t}\n\n\tsort.Strings(o.Options)\n\n\t// ignore these options because they won't be send to kernel\n\to.IgnoreSecurityLabels,\n\t\to.RememberInodes,\n\t\to.SingleThreaded,\n\t\to.DisableXAttrs,\n\t\to.Debug,\n\t\to.NoAllocForRead = false, false, false, false, false, false\n\n\t// ignore there options because they cannot be configured by users\n\to.Name = \"\"\n\to.MaxBackground = 0\n\to.MaxReadAhead = 0\n\to.DirectMount = false\n\to.DontUmask = false\n\to.Timeout = 0\n\treturn o\n}\n\ntype SecurityConfig struct {\n\tEnableCap bool\n\tEnableSELinux bool\n}\n\ntype Config struct {\n\tMeta *meta.Config\n\tFormat meta.Format\n\tChunk *chunk.Config\n\tSecurity *SecurityConfig\n\tPort *Port\n\tVersion string\n\tAttrTimeout time.Duration\n\tDirEntryTimeout time.Duration\n\tNegEntryTimeout time.Duration\n\tEntryTimeout time.Duration\n\tReaddirCache bool\n\tBackupMeta time.Duration\n\tBackupSkipTrash bool\n\tFastResolve bool `json:\",omitempty\"`\n\tAccessLog string `json:\",omitempty\"`\n\tSubdir string `json:\",omitempty\"`\n\tPrefixInternal bool\n\tHideInternal bool\n\tRootSquash *AnonymousAccount `json:\",omitempty\"`\n\tAllSquash *AnonymousAccount `json:\",omitempty\"`\n\tNonDefaultPermission bool `json:\",omitempty\"`\n\tUMask uint16\n\n\tPid int\n\tPPid int\n\tCommPath string `json:\",omitempty\"`\n\tStatePath string `json:\",omitempty\"`\n\tFuseOpts *FuseOptions `json:\",omitempty\"`\n\n\t// the mount point for current volume (to follow symlink)\n\tMountpoint string\n}\n\ntype AnonymousAccount struct {\n\tUid uint32\n\tGid uint32\n}\n\nvar (\n\treadSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"fuse_read_size_bytes\",\n\t\tHelp: \"size of read distributions.\",\n\t\tBuckets: prometheus.LinearBuckets(4096, 4096, 32),\n\t})\n\twrittenSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"fuse_written_size_bytes\",\n\t\tHelp: \"size of write distributions.\",\n\t\tBuckets: prometheus.LinearBuckets(4096, 4096, 32),\n\t})\n)\n\nfunc (v *VFS) Lookup(ctx Context, parent Ino, name string) (entry *meta.Entry, err syscall.Errno) {\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tif parent == rootID || name == internalNodes[0].name { // 0 is the control file\n\t\tn := getInternalNodeByName(name)\n\t\tif n != nil {\n\t\t\tentry = &meta.Entry{Inode: n.inode, Attr: n.attr}\n\t\t\treturn\n\t\t}\n\t}\n\tif IsSpecialNode(parent) && name == \".\" {\n\t\tif n := getInternalNode(parent); n != nil {\n\t\t\tentry = &meta.Entry{Inode: n.inode, Attr: n.attr}\n\t\t\treturn\n\t\t}\n\t}\n\tdefer func() {\n\t\tlogit(ctx, \"lookup\", err, \"(%d,%s):%s\", parent, name, (*Entry)(entry))\n\t}()\n\tif len(name) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\terr = v.Meta.Lookup(ctx, parent, name, &inode, attr, true)\n\tif err == 0 {\n\t\tentry = &meta.Entry{Inode: inode, Attr: attr}\n\t}\n\treturn\n}\n\nfunc (v *VFS) GetAttr(ctx Context, ino Ino, opened uint8) (entry *meta.Entry, err syscall.Errno) {\n\tif IsSpecialNode(ino) && getInternalNode(ino) != nil {\n\t\tn := getInternalNode(ino)\n\t\tentry = &meta.Entry{Inode: n.inode, Attr: n.attr}\n\t\treturn\n\t}\n\tdefer func() { logit(ctx, \"getattr\", err, \"(%d):%s\", ino, (*Entry)(entry)) }()\n\tvar attr = &Attr{}\n\terr = v.Meta.GetAttr(ctx, ino, attr)\n\tif err == 0 {\n\t\tentry = &meta.Entry{Inode: ino, Attr: attr}\n\t}\n\treturn\n}\n\nfunc get_filetype(mode uint16) uint8 {\n\tswitch mode & (syscall.S_IFMT & 0xffff) {\n\tcase syscall.S_IFIFO:\n\t\treturn meta.TypeFIFO\n\tcase syscall.S_IFSOCK:\n\t\treturn meta.TypeSocket\n\tcase syscall.S_IFLNK:\n\t\treturn meta.TypeSymlink\n\tcase syscall.S_IFREG:\n\t\treturn meta.TypeFile\n\tcase syscall.S_IFBLK:\n\t\treturn meta.TypeBlockDev\n\tcase syscall.S_IFDIR:\n\t\treturn meta.TypeDirectory\n\tcase syscall.S_IFCHR:\n\t\treturn meta.TypeCharDev\n\t}\n\treturn meta.TypeFile\n}\n\nfunc (v *VFS) Mknod(ctx Context, parent Ino, name string, mode uint16, cumask uint16, rdev uint32) (entry *meta.Entry, err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"mknod\", err, \"(%d,%s,%s:0%04o,0x%08X):%s\", parent, name, smode(mode), mode, rdev, (*Entry)(entry))\n\t}()\n\tif parent == rootID && IsSpecialName(name) {\n\t\terr = syscall.EEXIST\n\t\treturn\n\t}\n\tif len(name) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\t_type := get_filetype(mode)\n\tif _type == 0 {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\n\tvar inode Ino\n\tvar attr = &Attr{}\n\terr = v.Meta.Mknod(ctx, parent, name, _type, mode&07777, cumask, rdev, \"\", &inode, attr)\n\tif err == 0 {\n\t\tentry = &meta.Entry{Inode: inode, Attr: attr}\n\t\tv.invalidateDirHandle(parent, name, inode, attr)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Unlink(ctx Context, parent Ino, name string) (err syscall.Errno) {\n\treturn v.doUnlink(ctx, parent, name, false)\n}\n\nfunc (v *VFS) doUnlink(ctx Context, parent Ino, name string, skipTrash bool) (err syscall.Errno) {\n\tdefer func() { logit(ctx, \"unlink\", err, \"(%d,%s)\", parent, name) }()\n\tif parent == rootID && IsSpecialName(name) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif len(name) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\terr = v.Meta.Unlink(ctx, parent, name, skipTrash)\n\tif err == 0 {\n\t\tv.invalidateDirHandle(parent, name, 0, nil)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Mkdir(ctx Context, parent Ino, name string, mode uint16, cumask uint16) (entry *meta.Entry, err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"mkdir\", err, \"(%d,%s,%s:0%04o):%s\", parent, name, smode(mode), mode, (*Entry)(entry))\n\t}()\n\tif parent == rootID && IsSpecialName(name) {\n\t\terr = syscall.EEXIST\n\t\treturn\n\t}\n\tif len(name) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\n\tvar inode Ino\n\tvar attr = &Attr{}\n\terr = v.Meta.Mkdir(ctx, parent, name, mode, cumask, 0, &inode, attr)\n\tif err == 0 {\n\t\tentry = &meta.Entry{Inode: inode, Attr: attr}\n\t\tv.invalidateDirHandle(parent, name, inode, attr)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Rmdir(ctx Context, parent Ino, name string) (err syscall.Errno) {\n\tdefer func() { logit(ctx, \"rmdir\", err, \"(%d,%s)\", parent, name) }()\n\tif len(name) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\terr = v.Meta.Rmdir(ctx, parent, name)\n\tif err == 0 {\n\t\tv.invalidateDirHandle(parent, name, 0, nil)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Symlink(ctx Context, path string, parent Ino, name string) (entry *meta.Entry, err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"symlink\", err, \"(%d,%s,%s):%s\", parent, name, path, (*Entry)(entry))\n\t}()\n\tif parent == rootID && IsSpecialName(name) {\n\t\terr = syscall.EEXIST\n\t\treturn\n\t}\n\tif len(name) > maxName || len(path) >= maxSymlink {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\n\tvar inode Ino\n\tvar attr = &Attr{}\n\terr = v.Meta.Symlink(ctx, parent, name, path, &inode, attr)\n\tif err == 0 {\n\t\tentry = &meta.Entry{Inode: inode, Attr: attr}\n\t\tv.invalidateDirHandle(parent, name, inode, attr)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Readlink(ctx Context, ino Ino) (path []byte, err syscall.Errno) {\n\tdefer func() { logit(ctx, \"readlink\", err, \"(%d): (%s)\", ino, string(path)) }()\n\terr = v.Meta.ReadLink(ctx, ino, &path)\n\treturn\n}\n\nfunc (v *VFS) Rename(ctx Context, parent Ino, name string, newparent Ino, newname string, flags uint32) (err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"rename\", err, \"(%d,%s,%d,%s,%d)\", parent, name, newparent, newname, flags)\n\t}()\n\tif parent == rootID && IsSpecialName(name) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif newparent == rootID && IsSpecialName(newname) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif len(name) > maxName || len(newname) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\n\tvar inode Ino\n\tvar attr = &Attr{}\n\terr = v.Meta.Rename(ctx, parent, name, newparent, newname, flags, &inode, attr)\n\tif err == 0 {\n\t\tv.invalidateDirHandle(parent, name, 0, nil)\n\t\tv.invalidateDirHandle(newparent, newname, 0, nil)\n\t\tv.invalidateDirHandle(newparent, newname, inode, attr)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Link(ctx Context, ino Ino, newparent Ino, newname string) (entry *meta.Entry, err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"link\", err, \"(%d,%d,%s):%s\", ino, newparent, newname, (*Entry)(entry))\n\t}()\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif newparent == rootID && IsSpecialName(newname) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif len(newname) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\n\tvar attr = &Attr{}\n\terr = v.Meta.Link(ctx, ino, newparent, newname, attr)\n\tif err == 0 {\n\t\tentry = &meta.Entry{Inode: ino, Attr: attr}\n\t\tv.invalidateDirHandle(newparent, newname, ino, attr)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Opendir(ctx Context, ino Ino, flags uint32) (fh uint64, err syscall.Errno) {\n\tdefer func() { logit(ctx, \"opendir\", err, \"(%d) [fh:%d]\", ino, fh) }()\n\tif ctx.CheckPermission() {\n\t\tvar mmask uint8 = 0\n\t\tswitch flags & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) {\n\t\tcase syscall.O_RDONLY:\n\t\t\tmmask = MODE_MASK_R\n\t\tcase syscall.O_WRONLY:\n\t\t\tmmask = MODE_MASK_W\n\t\tcase syscall.O_RDWR:\n\t\t\tmmask = MODE_MASK_R | MODE_MASK_W\n\t\t}\n\t\tif err = v.Meta.Access(ctx, ino, mmask, nil); err != 0 {\n\t\t\treturn\n\t\t}\n\t}\n\tfh = v.newHandle(ino, true).fh\n\treturn\n}\n\nfunc (v *VFS) UpdateLength(inode Ino, attr *meta.Attr) {\n\tif attr.Full && attr.Typ == meta.TypeFile {\n\t\tlength := v.writer.GetLength(inode)\n\t\tif length > attr.Length {\n\t\t\tattr.Length = length\n\t\t}\n\t\tv.reader.Truncate(inode, attr.Length)\n\t}\n}\n\nfunc (v *VFS) Readdir(ctx Context, ino Ino, size uint32, off int, fh uint64, plus bool) (entries []*meta.Entry, readAt time.Time, err syscall.Errno) {\n\tdefer func() { logit(ctx, \"readdir\", err, \"(%d,%d,%d,%t): (%d)\", ino, size, off, plus, len(entries)) }()\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\th.Lock()\n\tdefer h.Unlock()\n\n\tif h.dirHandler == nil || off == 0 {\n\t\tif h.dirHandler != nil {\n\t\t\th.dirHandler.Close()\n\t\t\th.dirHandler = nil\n\t\t}\n\t\tvar initEntries []*meta.Entry\n\t\tif ino == rootID && !v.Conf.HideInternal {\n\t\t\tfor _, node := range internalNodes[1:] {\n\t\t\t\tinitEntries = append(initEntries, &meta.Entry{\n\t\t\t\t\tInode: node.inode,\n\t\t\t\t\tName: []byte(node.name),\n\t\t\t\t\tAttr: node.attr,\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t\th.readAt = time.Now()\n\t\tif h.dirHandler, err = v.Meta.NewDirHandler(ctx, ino, plus, initEntries); err != 0 {\n\t\t\tif plus && err == syscall.EACCES {\n\t\t\t\th.dirHandler, err = v.Meta.NewDirHandler(ctx, ino, false, initEntries)\n\t\t\t}\n\t\t\tif err != 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\tif entries, err = h.dirHandler.List(ctx, off); err != 0 {\n\t\treturn\n\t}\n\treadAt = h.readAt\n\tlogger.Debugf(\"readdir: [%d:%d] %d entries, offset=%d\", ino, fh, len(entries), off)\n\treturn\n}\n\nfunc (v *VFS) UpdateReaddirOffset(ctx Context, ino Ino, fh uint64, off int) {\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\treturn\n\t}\n\th.Lock()\n\tdefer h.Unlock()\n\tif h.dirHandler != nil {\n\t\th.dirHandler.Read(off)\n\t}\n}\n\nfunc (v *VFS) Releasedir(ctx Context, ino Ino, fh uint64) int {\n\tdefer logit(ctx, \"releasedir\", 0, \"(%d)\", ino)\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\treturn 0\n\t}\n\tv.ReleaseHandler(ino, fh)\n\treturn 0\n}\n\nconst O_TMPFILE = 020000000\n\nfunc (v *VFS) Create(ctx Context, parent Ino, name string, mode uint16, cumask uint16, flags uint32) (entry *meta.Entry, fh uint64, err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"create\", err, \"(%d,%s,%s:0%04o):%s [fh:%d]\", parent, name, smode(mode), mode, (*Entry)(entry), fh)\n\t}()\n\t// O_TMPFILE support\n\tdoUnlink := runtime.GOOS == \"linux\" && flags&O_TMPFILE != 0\n\tif doUnlink {\n\t\tname = fmt.Sprintf(\"tmpfile_%s\", uuid.New().String())\n\t}\n\tif parent == rootID && IsSpecialName(name) {\n\t\terr = syscall.EEXIST\n\t\treturn\n\t}\n\tif len(name) > maxName {\n\t\terr = syscall.ENAMETOOLONG\n\t\treturn\n\t}\n\n\tvar inode Ino\n\tvar attr = &Attr{}\n\tif runtime.GOOS == \"windows\" {\n\t\tattr.Flags = meta.FlagWindowsArchive\n\t}\n\terr = v.Meta.Create(ctx, parent, name, mode&07777, cumask, flags, &inode, attr)\n\tif runtime.GOOS == \"darwin\" && err == syscall.ENOENT {\n\t\terr = syscall.EACCES\n\t}\n\tif err == 0 {\n\t\tv.UpdateLength(inode, attr)\n\t\tfh = v.newFileHandle(inode, attr.Length, flags)\n\t\tentry = &meta.Entry{Inode: inode, Attr: attr}\n\t\tv.invalidateDirHandle(parent, name, inode, attr)\n\n\t\tif doUnlink {\n\t\t\tif flags&syscall.O_EXCL != 0 {\n\t\t\t\tlogger.Warnf(\"The O_EXCL is currently not supported for use with O_TMPFILE\")\n\t\t\t}\n\t\t\terr = v.doUnlink(ctx, parent, name, true)\n\t\t}\n\t}\n\treturn\n}\n\nfunc (v *VFS) Open(ctx Context, ino Ino, flags uint32) (entry *meta.Entry, fh uint64, err syscall.Errno) {\n\tdefer func() {\n\t\tif entry != nil {\n\t\t\tlogit(ctx, \"open\", err, \"(%d,%#x) [fh:%d]\", ino, flags, fh)\n\t\t} else {\n\t\t\tlogit(ctx, \"open\", err, \"(%d,%#x)\", ino, flags)\n\t\t}\n\t}()\n\tvar attr = &Attr{}\n\tif IsSpecialNode(ino) {\n\t\tif ino != controlInode && (flags&O_ACCMODE) != syscall.O_RDONLY {\n\t\t\terr = syscall.EACCES\n\t\t\treturn\n\t\t}\n\t\th := v.newHandle(ino, true)\n\t\tfh = h.fh\n\t\tn := getInternalNode(ino)\n\t\tif n == nil {\n\t\t\treturn\n\t\t}\n\t\tentry = &meta.Entry{Inode: ino, Attr: n.attr}\n\t\tswitch ino {\n\t\tcase logInode:\n\t\t\topenAccessLog(fh)\n\t\tcase StatsInode:\n\t\t\th.data = CollectMetrics(v.registry)\n\t\tcase ConfigInode:\n\t\t\tv.Conf.Format = v.Meta.GetFormat()\n\t\t\tif v.UpdateFormat != nil {\n\t\t\t\tv.UpdateFormat(&v.Conf.Format)\n\t\t\t}\n\t\t\tv.Conf.Format.RemoveSecret()\n\t\t\th.data, _ = json.MarshalIndent(v.Conf, \"\", \" \")\n\t\t\tentry.Attr.Length = uint64(len(h.data))\n\t\t}\n\t\treturn\n\t}\n\n\terr = v.Meta.Open(ctx, ino, flags, attr)\n\tif err == 0 {\n\t\tv.UpdateLength(ino, attr)\n\t\tfh = v.newFileHandle(ino, attr.Length, flags)\n\t\tentry = &meta.Entry{Inode: ino, Attr: attr}\n\t}\n\treturn\n}\n\nfunc (v *VFS) Truncate(ctx Context, ino Ino, size int64, fh uint64, attr *Attr) (err syscall.Errno) {\n\t// defer func() { logit(ctx, \"truncate (%d,%d): %s\", ino, size, strerr(err)) }()\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif size < 0 {\n\t\terr = syscall.EINVAL\n\t\treturn\n\t}\n\tif size >= maxFileSize {\n\t\terr = syscall.EFBIG\n\t\treturn\n\t}\n\ths := v.findAllHandles(ino)\n\tsort.Slice(hs, func(i, j int) bool { return hs[i].fh < hs[j].fh })\n\tfor _, h := range hs {\n\t\tif !h.Wlock(ctx) {\n\t\t\terr = syscall.EINTR\n\t\t\treturn\n\t\t}\n\t\tdefer func(h *handle) { h.Wunlock() }(h)\n\t}\n\t_ = v.writer.Flush(ctx, ino)\n\tif fh == 0 {\n\t\terr = v.Meta.Truncate(ctx, ino, 0, uint64(size), attr, false)\n\t} else {\n\t\th := v.findHandle(ino, fh)\n\t\tif h == nil {\n\t\t\terr = syscall.EBADF\n\t\t\treturn\n\t\t}\n\t\tif h.writer == nil {\n\t\t\terr = syscall.EACCES\n\t\t\treturn\n\t\t}\n\t\t// flags = 1 means the file is opened, so we don't need to check if it's in the trash\n\t\terr = v.Meta.Truncate(ctx, ino, 1, uint64(size), attr, true)\n\t}\n\tif err == 0 {\n\t\tv.writer.Truncate(ino, uint64(size))\n\t\tv.reader.Truncate(ino, uint64(size))\n\t\tv.invalidateAttr(ino)\n\t}\n\treturn err\n}\n\nfunc (v *VFS) ReleaseHandler(ino Ino, fh uint64) {\n\tv.releaseFileHandle(ino, fh)\n}\n\nfunc (v *VFS) Release(ctx Context, ino Ino, fh uint64) {\n\tvar err syscall.Errno\n\tdefer func() { logit(ctx, \"release\", err, \"(%d,%d)\", ino, fh) }()\n\tif IsSpecialNode(ino) {\n\t\tif ino == logInode {\n\t\t\tcloseAccessLog(fh)\n\t\t}\n\t\tv.releaseHandle(ino, fh)\n\t\treturn\n\t}\n\tif fh > 0 {\n\t\tf := v.findHandle(ino, fh)\n\t\tif f != nil {\n\t\t\tf.Lock()\n\t\t\tfor (f.writing | f.writers | f.readers) != 0 {\n\t\t\t\tif f.cond.WaitWithTimeout(time.Second) && ctx.Canceled() {\n\t\t\t\t\tf.Unlock()\n\t\t\t\t\tlogger.Warnf(\"write lock %d interrupted\", f.inode)\n\t\t\t\t\terr = syscall.EINTR\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tlocks := f.locks\n\t\t\tfowner := f.flockOwner\n\t\t\tpowner := f.ofdOwner\n\t\t\tf.Unlock()\n\t\t\tif f.writer != nil {\n\t\t\t\t_ = f.writer.Flush(ctx)\n\t\t\t\tv.invalidateAttr(ino)\n\t\t\t}\n\t\t\tif locks&1 != 0 {\n\t\t\t\t_ = v.Meta.Flock(ctx, ino, fowner^fh, F_UNLCK, false)\n\t\t\t}\n\t\t\tif locks&2 != 0 && powner != 0 {\n\t\t\t\t_ = v.Meta.Setlk(ctx, ino, powner, false, F_UNLCK, 0, 0x7FFFFFFFFFFFFFFF, 0)\n\t\t\t}\n\t\t}\n\t\t_ = v.Meta.Close(ctx, ino)\n\t\tgo v.releaseFileHandle(ino, fh) // after writes it waits for data sync, so do it after everything\n\t}\n}\n\nfunc hasReadPerm(flag uint32) bool {\n\treturn (flag & O_ACCMODE) != syscall.O_WRONLY\n}\n\nfunc (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n int, err syscall.Errno) {\n\tsize := uint32(len(buf))\n\tif IsSpecialNode(ino) {\n\t\tif ino == controlInode && runtime.GOOS == \"darwin\" {\n\t\t\tfh = v.getControlHandle(ctx.Pid())\n\t\t}\n\t\th := v.findHandle(ino, fh)\n\t\tif h == nil {\n\t\t\terr = syscall.EBADF\n\t\t\treturn\n\t\t}\n\t\tif len(h.data) == 0 {\n\t\t\tswitch ino {\n\t\t\tcase StatsInode:\n\t\t\t\th.data = CollectMetrics(v.registry)\n\t\t\tcase ConfigInode:\n\t\t\t\tv.Conf.Format = v.Meta.GetFormat()\n\t\t\t\tif v.UpdateFormat != nil {\n\t\t\t\t\tv.UpdateFormat(&v.Conf.Format)\n\t\t\t\t}\n\t\t\t\tv.Conf.Format.RemoveSecret()\n\t\t\t\th.data, _ = json.MarshalIndent(v.Conf, \"\", \" \")\n\t\t\t}\n\t\t}\n\n\t\tif ino == logInode {\n\t\t\tif h.flags&O_RECOVERED != 0 {\n\t\t\t\topenAccessLog(fh)\n\t\t\t}\n\t\t\tn = readAccessLog(fh, buf)\n\t\t} else {\n\t\t\tdefer func() { logit(ctx, \"read\", err, \"(%d,%d,%d,%d): %d\", ino, size, off, fh, n) }()\n\t\t\th.Lock()\n\t\t\tdefer h.Unlock()\n\t\t\tif off < h.off {\n\t\t\t\tlogger.Errorf(\"read dropped data from %s: %d < %d\", ino, off, h.off)\n\t\t\t\terr = syscall.EIO\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif int(off-h.off) < len(h.data) {\n\t\t\t\tn = copy(buf, h.data[off-h.off:])\n\t\t\t}\n\t\t\tif len(h.data) > 2<<20 && off-h.off > 1<<20 {\n\t\t\t\t// drop first part to avoid OOM\n\t\t\t\th.off += 1 << 20\n\t\t\t\th.data = h.data[1<<20:]\n\t\t\t}\n\t\t}\n\t\treturn\n\t}\n\n\tdefer func() {\n\t\treadSizeHistogram.Observe(float64(n))\n\t\tlogit(ctx, \"read\", err, \"(%d,%d,%d,%d): (%d)\", ino, size, off, fh, n)\n\t}()\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif h.flags&O_RECOVERED != 0 {\n\t\t// recovered\n\t\tvar attr Attr\n\t\terr = v.Meta.Open(ctx, ino, syscall.O_RDONLY, &attr)\n\t\tif err != 0 {\n\t\t\tv.releaseHandle(ino, fh)\n\t\t\terr = syscall.EBADF\n\t\t\treturn\n\t\t}\n\t\th.Lock()\n\t\tv.UpdateLength(ino, &attr)\n\t\th.flags = syscall.O_RDONLY\n\t\th.reader = v.reader.Open(h.inode, attr.Length)\n\t\th.Unlock()\n\t}\n\n\tif off >= maxFileSize || off+uint64(size) >= maxFileSize {\n\t\terr = syscall.EFBIG\n\t\treturn\n\t}\n\tif h.reader == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\n\t// there could be read operation for write-only if kernel writeback is enabled\n\tif v.Conf.FuseOpts != nil && !v.Conf.FuseOpts.EnableWriteback && !hasReadPerm(h.flags) {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif !h.Rlock(ctx) {\n\t\terr = syscall.EINTR\n\t\treturn\n\t}\n\tdefer h.Runlock()\n\n\t_ = v.writer.Flush(ctx, ino)\n\tn, err = h.reader.Read(ctx, off, buf)\n\tfor err == syscall.EAGAIN {\n\t\tn, err = h.reader.Read(ctx, off, buf)\n\t}\n\tif err == syscall.ENOENT {\n\t\terr = syscall.EBADF\n\t}\n\th.removeOp(ctx)\n\treturn\n}\n\nfunc (v *VFS) Write(ctx Context, ino Ino, buf []byte, off, fh uint64) (err syscall.Errno) {\n\tsize := uint64(len(buf))\n\tif ino == controlInode && runtime.GOOS == \"darwin\" {\n\t\tfh = v.getControlHandle(ctx.Pid())\n\t}\n\tdefer func() { logit(ctx, \"write\", err, \"(%d,%d,%d,%d)\", ino, size, off, fh) }()\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif off >= maxFileSize || off+size >= maxFileSize {\n\t\terr = syscall.EFBIG\n\t\treturn\n\t}\n\n\tif ino == controlInode {\n\t\th.Lock()\n\t\tdefer h.Unlock()\n\t\th.pending = append(h.pending, buf...)\n\t\trb := utils.ReadBuffer(h.pending)\n\t\tcmd := rb.Get32()\n\t\tsize := int(rb.Get32())\n\t\tif rb.Left() < size {\n\t\t\tlogger.Debugf(\"message not complete: %d %d > %d\", cmd, size, rb.Left())\n\t\t\treturn\n\t\t}\n\t\th.data = append(h.data, h.pending...)\n\t\th.pending = h.pending[:0]\n\t\tif rb.Left() == size {\n\t\t\th.bctx = meta.NewContext(ctx.Pid(), ctx.Uid(), ctx.Gids())\n\t\t\tgo v.handleInternalMsg(h.bctx, cmd, rb, h)\n\t\t} else {\n\t\t\tlogger.Warnf(\"broken message: %d %d < %d\", cmd, size, rb.Left())\n\t\t\th.data = append(h.data, uint8(syscall.EIO&0xff))\n\t\t}\n\t\treturn\n\t}\n\n\tif h.writer == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\n\tif !h.Wlock(ctx) {\n\t\terr = syscall.EINTR\n\t\treturn\n\t}\n\tdefer h.Wunlock()\n\n\terr = h.writer.Write(ctx, off, buf)\n\tif err == syscall.ENOENT || err == syscall.EPERM || err == syscall.EINVAL {\n\t\terr = syscall.EBADF\n\t}\n\th.removeOp(ctx)\n\n\tif err == 0 {\n\t\twrittenSizeHistogram.Observe(float64(len(buf)))\n\t\tv.reader.Invalidate(ino, off, size)\n\t\tv.invalidateAttr(ino)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Fallocate(ctx Context, ino Ino, mode uint8, off, size int64, fh uint64) (err syscall.Errno) {\n\tdefer func() { logit(ctx, \"fallocate\", err, \"(%d,%d,%d,%d)\", ino, mode, off, size) }()\n\tif off < 0 || size <= 0 {\n\t\terr = syscall.EINVAL\n\t\treturn\n\t}\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif off >= maxFileSize || off+size >= maxFileSize {\n\t\terr = syscall.EFBIG\n\t\treturn\n\t}\n\tif h.writer == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif !h.Wlock(ctx) {\n\t\terr = syscall.EINTR\n\t\treturn\n\t}\n\tdefer h.Wunlock()\n\tdefer h.removeOp(ctx)\n\n\terr = v.writer.Flush(ctx, ino)\n\tif err != 0 {\n\t\treturn\n\t}\n\tvar length uint64\n\terr = v.Meta.Fallocate(ctx, ino, mode, uint64(off), uint64(size), &length)\n\tif err == 0 {\n\t\tv.writer.Truncate(ino, length)\n\t\ts := size\n\t\tif off+size > int64(length) {\n\t\t\ts = int64(length) - off\n\t\t}\n\t\tif s > 0 {\n\t\t\tv.reader.Invalidate(ino, uint64(off), uint64(s))\n\t\t}\n\t\tv.invalidateAttr(ino)\n\t}\n\treturn\n}\n\nfunc (v *VFS) CopyFileRange(ctx Context, nodeIn Ino, fhIn, offIn uint64, nodeOut Ino, fhOut, offOut, size uint64, flags uint32) (copied uint64, err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"copy_file_range\", err, \"(%d,%d,%d,%d,%d,%d)\", nodeIn, offIn, nodeOut, offOut, size, flags)\n\t}()\n\tif IsSpecialNode(nodeIn) {\n\t\terr = syscall.ENOTSUP\n\t\treturn\n\t}\n\tif IsSpecialNode(nodeOut) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\thi := v.findHandle(nodeIn, fhIn)\n\tif fhIn == 0 || hi == nil || hi.inode != nodeIn {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tho := v.findHandle(nodeOut, fhOut)\n\tif fhOut == 0 || ho == nil || ho.inode != nodeOut {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif hi.reader == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif ho.writer == nil {\n\t\terr = syscall.EACCES\n\t\treturn\n\t}\n\tif offIn >= maxFileSize || offIn+size >= maxFileSize || offOut >= maxFileSize || offOut+size >= maxFileSize {\n\t\terr = syscall.EFBIG\n\t\treturn\n\t}\n\tif flags != 0 {\n\t\terr = syscall.EINVAL\n\t\treturn\n\t}\n\tif nodeIn == nodeOut && (offIn <= offOut && offOut < offIn+size || offOut <= offIn && offIn < offOut+size) {\n\t\terr = syscall.EINVAL // overlap\n\t\treturn\n\t}\n\n\tif !ho.Wlock(ctx) {\n\t\terr = syscall.EINTR\n\t\treturn\n\t}\n\tdefer ho.Wunlock()\n\tdefer ho.removeOp(ctx)\n\tif nodeIn != nodeOut {\n\t\tif !hi.Rlock(ctx) {\n\t\t\terr = syscall.EINTR\n\t\t\treturn\n\t\t}\n\t\tdefer hi.Runlock()\n\t\tdefer hi.removeOp(ctx)\n\t}\n\n\terr = v.writer.Flush(ctx, nodeIn)\n\tif err != 0 {\n\t\treturn\n\t}\n\terr = v.writer.Flush(ctx, nodeOut)\n\tif err != 0 {\n\t\treturn\n\t}\n\tvar length uint64\n\terr = v.Meta.CopyFileRange(ctx, nodeIn, offIn, nodeOut, offOut, size, flags, &copied, &length)\n\tif err == 0 {\n\t\tv.writer.Truncate(nodeOut, length)\n\t\tv.reader.Invalidate(nodeOut, offOut, size)\n\t\tv.invalidateAttr(nodeOut)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Flush(ctx Context, ino Ino, fh uint64, lockOwner uint64) (err syscall.Errno) {\n\tif ino == controlInode && runtime.GOOS == \"darwin\" {\n\t\tfh = v.getControlHandle(ctx.Pid())\n\t\tdefer v.releaseControlHandle(ctx.Pid())\n\t}\n\tdefer func() { logit(ctx, \"flush\", err, \"(%d,%d,%016X)\", ino, fh, lockOwner) }()\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif IsSpecialNode(ino) {\n\t\tif ino == controlInode && h.bctx != nil {\n\t\t\th.bctx.Cancel()\n\t\t}\n\t\treturn\n\t}\n\n\tif h.writer != nil {\n\t\tfor !h.Wlock(ctx) {\n\t\t\th.cancelOp(ctx.Pid())\n\t\t}\n\n\t\terr = h.writer.Flush(ctx)\n\t\tif err == syscall.ENOENT || err == syscall.EPERM || err == syscall.EINVAL {\n\t\t\terr = syscall.EBADF\n\t\t}\n\t\th.removeOp(ctx)\n\t\th.Wunlock()\n\t} else if h.reader != nil {\n\t\th.cancelOp(ctx.Pid())\n\t}\n\n\th.Lock()\n\tlocks := h.locks\n\tif lockOwner == h.ofdOwner {\n\t\th.ofdOwner = 0\n\t}\n\th.Unlock()\n\tif locks&2 != 0 {\n\t\t_ = v.Meta.Setlk(ctx, ino, lockOwner, false, F_UNLCK, 0, 0x7FFFFFFFFFFFFFFF, 0)\n\t}\n\treturn\n}\n\nfunc (v *VFS) Fsync(ctx Context, ino Ino, datasync int, fh uint64) (err syscall.Errno) {\n\tdefer func() { logit(ctx, \"fsync\", err, \"(%d,%d)\", ino, datasync) }()\n\tif IsSpecialNode(ino) {\n\t\treturn\n\t}\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\tif h.writer != nil {\n\t\tif !h.Wlock(ctx) {\n\t\t\treturn syscall.EINTR\n\t\t}\n\t\tdefer h.Wunlock()\n\t\tdefer h.removeOp(ctx)\n\n\t\terr = h.writer.Flush(ctx)\n\t\tif err == syscall.ENOENT || err == syscall.EPERM || err == syscall.EINVAL {\n\t\t\terr = syscall.EBADF\n\t\t}\n\t}\n\treturn\n}\n\nconst (\n\txattrMaxName = 255\n\txattrMaxSize = 65536\n)\n\nvar macSupportFlags = meta.XattrCreateOrReplace | meta.XattrCreate | meta.XattrReplace\n\nconst (\n\t_SECURITY_CAPABILITY = \"security.capability\"\n\t_SECURITY_SELINUX = \"security.selinux\"\n\t_SECURITY_ACL = \"system.posix_acl_access\"\n\t_SECURITY_ACL_DEFAULT = \"system.posix_acl_default\"\n)\n\nfunc isXattrEnabled(conf *Config, name string) bool {\n\tswitch name {\n\tcase _SECURITY_CAPABILITY:\n\t\treturn conf.Security != nil && conf.Security.EnableCap\n\tcase _SECURITY_SELINUX:\n\t\treturn conf.Security != nil && conf.Security.EnableSELinux\n\tcase _SECURITY_ACL, _SECURITY_ACL_DEFAULT:\n\t\treturn conf.Format.EnableACL\n\t}\n\treturn true\n}\n\nfunc (v *VFS) SetXattr(ctx Context, ino Ino, name string, value []byte, flags uint32) (err syscall.Errno) {\n\tdefer func() { logit(ctx, \"setxattr\", err, \"(%d,%s,%d,%d)\", ino, name, len(value), flags) }()\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif len(value) > xattrMaxSize {\n\t\tif runtime.GOOS == \"darwin\" {\n\t\t\terr = syscall.E2BIG\n\t\t} else {\n\t\t\terr = syscall.ERANGE\n\t\t}\n\t\treturn\n\t}\n\tif len(name) > xattrMaxName {\n\t\tif runtime.GOOS == \"darwin\" {\n\t\t\terr = syscall.EPERM\n\t\t} else {\n\t\t\terr = syscall.ERANGE\n\t\t}\n\t\treturn\n\t}\n\tif len(name) == 0 {\n\t\terr = syscall.EINVAL\n\t\treturn\n\t}\n\n\tif !isXattrEnabled(v.Conf, name) {\n\t\terr = syscall.ENOTSUP\n\t\treturn\n\t}\n\n\tif typ, ok := aclTypes[name]; ok {\n\t\tvar rule *acl.Rule\n\t\trule, err = decodeACL(value)\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t\terr = v.Meta.SetFacl(ctx, ino, typ, rule)\n\t\tv.invalidateAttr(ino)\n\t} else {\n\t\t// only retain supported flags\n\t\tif runtime.GOOS == \"darwin\" {\n\t\t\tflags &= uint32(macSupportFlags)\n\t\t}\n\t\terr = v.Meta.SetXattr(ctx, ino, name, value, flags)\n\t}\n\treturn\n}\n\nfunc (v *VFS) GetXattr(ctx Context, ino Ino, name string, size uint32) (value []byte, err syscall.Errno) {\n\tif !isXattrEnabled(v.Conf, name) {\n\t\terr = syscall.ENODATA\n\t\treturn\n\t}\n\n\tdefer func() { logit(ctx, \"getxattr\", err, \"(%d,%s,%d): (%d)\", ino, name, size, len(value)) }()\n\tif IsSpecialNode(ino) {\n\t\terr = meta.ENOATTR\n\t\treturn\n\t}\n\tif len(name) > xattrMaxName {\n\t\tif runtime.GOOS == \"darwin\" {\n\t\t\terr = syscall.EPERM\n\t\t} else {\n\t\t\terr = syscall.ERANGE\n\t\t}\n\t\treturn\n\t}\n\tif len(name) == 0 {\n\t\terr = syscall.EINVAL\n\t\treturn\n\t}\n\n\tif typ, ok := aclTypes[name]; ok {\n\t\trule := &acl.Rule{}\n\t\tif err = v.Meta.GetFacl(ctx, ino, typ, rule); err != 0 {\n\t\t\treturn nil, err\n\t\t}\n\t\tvalue = encodeACL(rule)\n\t} else {\n\t\terr = v.Meta.GetXattr(ctx, ino, name, &value)\n\t}\n\tif size > 0 && len(value) > int(size) {\n\t\terr = syscall.ERANGE\n\t}\n\treturn\n}\n\nfunc (v *VFS) ListXattr(ctx Context, ino Ino, size int) (data []byte, err syscall.Errno) {\n\tdefer func() { logit(ctx, \"listxattr\", err, \"(%d,%d): (%d)\", ino, size, len(data)) }()\n\tif IsSpecialNode(ino) {\n\t\terr = meta.ENOATTR\n\t\treturn\n\t}\n\terr = v.Meta.ListXattr(ctx, ino, &data)\n\tif size > 0 && len(data) > size {\n\t\terr = syscall.ERANGE\n\t}\n\treturn\n}\n\nfunc (v *VFS) RemoveXattr(ctx Context, ino Ino, name string) (err syscall.Errno) {\n\tdefer func() { logit(ctx, \"removexattr\", err, \"(%d,%s)\", ino, name) }()\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif len(name) > xattrMaxName {\n\t\tif runtime.GOOS == \"darwin\" {\n\t\t\terr = syscall.EPERM\n\t\t} else {\n\t\t\terr = syscall.ERANGE\n\t\t}\n\t\treturn\n\t}\n\tif len(name) == 0 {\n\t\terr = syscall.EINVAL\n\t\treturn\n\t}\n\n\tif !isXattrEnabled(v.Conf, name) {\n\t\terr = syscall.ENOTSUP\n\t\treturn\n\t}\n\n\tif typ, ok := aclTypes[name]; ok {\n\t\terr = v.Meta.SetFacl(ctx, ino, typ, acl.EmptyRule())\n\t} else {\n\t\terr = v.Meta.RemoveXattr(ctx, ino, name)\n\t}\n\n\treturn\n}\n\nvar logger = utils.GetLogger(\"juicefs\")\n\ntype VFS struct {\n\tConf *Config\n\tMeta meta.Meta\n\tStore chunk.ChunkStore\n\tInvalidateEntry func(parent meta.Ino, name string) syscall.Errno\n\tUpdateFormat func(*meta.Format)\n\treader DataReader\n\twriter DataWriter\n\tcacheFiller *CacheFiller\n\n\thandles map[Ino][]*handle\n\thandleIno map[uint64]Ino\n\thanleM sync.Mutex\n\tnextfh uint64\n\n\tmodM sync.Mutex\n\tmodifiedAt map[Ino]time.Time\n\n\tregistry *prometheus.Registry\n}\n\nfunc NewVFS(conf *Config, m meta.Meta, store chunk.ChunkStore, registerer prometheus.Registerer, registry *prometheus.Registry) *VFS {\n\treader := NewDataReader(conf, m, store)\n\twriter := NewDataWriter(conf, m, store, reader)\n\n\tv := &VFS{\n\t\tConf: conf,\n\t\tMeta: m,\n\t\tStore: store,\n\t\treader: reader,\n\t\twriter: writer,\n\t\tcacheFiller: NewCacheFiller(conf, m, store),\n\t\thandles: make(map[Ino][]*handle),\n\t\thandleIno: make(map[uint64]Ino),\n\t\tmodifiedAt: make(map[meta.Ino]time.Time),\n\t\tnextfh: 1,\n\t\tregistry: registry,\n\t}\n\n\tn := getInternalNode(ConfigInode)\n\tv.Conf.Format.RemoveSecret()\n\tdata, _ := json.MarshalIndent(v.Conf, \"\", \" \")\n\tn.attr.Length = uint64(len(data))\n\tif conf.Meta.Subdir != \"\" { // don't show trash directory\n\t\tinternalNodes = internalNodes[:len(internalNodes)-1]\n\t}\n\tif conf.PrefixInternal {\n\t\tfor _, n := range internalNodes {\n\t\t\tn.name = \".jfs\" + n.name\n\t\t}\n\t\tmeta.TrashName = \".jfs\" + meta.TrashName\n\t}\n\n\tstatePath := os.Getenv(\"_FUSE_STATE_PATH\")\n\tif statePath == \"\" {\n\t\tstatePath = fmt.Sprintf(\"/tmp/state%d.json\", os.Getppid())\n\t}\n\tif err := v.loadAllHandles(statePath); err != nil && !os.IsNotExist(err) {\n\t\tlogger.Errorf(\"load state from %s: %s\", statePath, err)\n\t}\n\t_ = os.Rename(statePath, statePath+\".bak\")\n\n\tgo v.cleanupModified()\n\tinitVFSMetrics(v, writer, reader, registerer)\n\treturn v\n}\n\nfunc (v *VFS) invalidateAttr(ino Ino) {\n\tv.modM.Lock()\n\tv.modifiedAt[ino] = time.Now()\n\tv.modM.Unlock()\n}\n\nfunc (v *VFS) ModifiedSince(ino Ino, start time.Time) bool {\n\tv.modM.Lock()\n\tt, ok := v.modifiedAt[ino]\n\tv.modM.Unlock()\n\treturn ok && t.After(start)\n}\n\nfunc (v *VFS) cleanupModified() {\n\tfor {\n\t\tv.modM.Lock()\n\t\texpire := time.Now().Add(time.Second * -30)\n\t\tvar cnt, deleted int\n\t\tfor i, t := range v.modifiedAt {\n\t\t\tif t.Before(expire) {\n\t\t\t\tdelete(v.modifiedAt, i)\n\t\t\t\tdeleted++\n\t\t\t}\n\t\t\tcnt++\n\t\t\tif cnt > 1000 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tv.modM.Unlock()\n\t\ttime.Sleep(time.Millisecond * time.Duration(1000*(cnt+1-deleted*2)/(cnt+1)))\n\t}\n}\n\nfunc (v *VFS) FlushAll(path string) (err error) {\n\tnow := time.Now()\n\tdefer func() {\n\t\tlogger.Infof(\"flush buffered data in %s: %v\", time.Since(now), err)\n\t}()\n\terr = v.writer.FlushAll()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif path == \"\" {\n\t\treturn nil\n\t}\n\treturn v.dumpAllHandles(path)\n}\n\nfunc initVFSMetrics(v *VFS, writer DataWriter, reader DataReader, registerer prometheus.Registerer) {\n\tif registerer == nil {\n\t\treturn\n\t}\n\thandlersGause := prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\tName: \"fuse_open_handlers\",\n\t\tHelp: \"number of open files and directories.\",\n\t}, func() float64 {\n\t\tv.hanleM.Lock()\n\t\tdefer v.hanleM.Unlock()\n\t\treturn float64(len(v.handles))\n\t})\n\t_ = registerer.Register(handlersGause)\n\tInitMemoryBufferMetrics(writer, reader, registerer)\n}\n\nfunc InitMemoryBufferMetrics(writer DataWriter, reader DataReader, registerer prometheus.Registerer) {\n\tusedBufferSize := prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\tName: \"used_buffer_size_bytes\",\n\t\tHelp: \"size of currently used buffer.\",\n\t}, func() float64 {\n\t\tif dw, ok := writer.(*dataWriter); ok {\n\t\t\treturn float64(dw.usedBufferSize())\n\t\t}\n\t\treturn 0.0\n\t})\n\tstoreCacheSize := prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\tName: \"store_cache_size_bytes\",\n\t\tHelp: \"size of store cache.\",\n\t}, func() float64 {\n\t\tif dw, ok := writer.(*dataWriter); ok {\n\t\t\treturn float64(dw.store.UsedMemory())\n\t\t}\n\t\treturn 0.0\n\t})\n\treadBufferMetric := prometheus.NewGaugeFunc(prometheus.GaugeOpts{\n\t\tName: \"used_read_buffer_size_bytes\",\n\t\tHelp: \"size of currently used buffer for read\",\n\t}, func() float64 {\n\t\tif dr, ok := reader.(*dataReader); ok {\n\t\t\treturn float64(dr.readBufferUsed())\n\t\t}\n\t\treturn 0.0\n\t})\n\t_ = registerer.Register(usedBufferSize)\n\t_ = registerer.Register(storeCacheSize)\n\t_ = registerer.Register(readBufferMetric)\n}\n\nfunc InitMetrics(registerer prometheus.Registerer) {\n\tif registerer == nil {\n\t\treturn\n\t}\n\tregisterer.MustRegister(readSizeHistogram)\n\tregisterer.MustRegister(writtenSizeHistogram)\n\tregisterer.MustRegister(opsDurationsHistogram)\n\tregisterer.MustRegister(opsTotal)\n\tregisterer.MustRegister(opsDurations)\n\tregisterer.MustRegister(opsIOErrors)\n\tregisterer.MustRegister(compactSizeHistogram)\n}\n\n// Linux ACL format:\n//\n//\tversion:8 (2)\n//\tflags:8 (0)\n//\tfiller:16\n//\tN * [ tag:16 perm:16 id:32 ]\n//\ttag:\n//\t 01 - user\n//\t 02 - named user\n//\t 04 - group\n//\t 08 - named group\n//\t 10 - mask\n//\t 20 - other\n\nfunc encodeACL(n *acl.Rule) []byte {\n\tlength := 4 + 24 + uint32(len(n.NamedUsers)+len(n.NamedGroups))*8\n\tif n.Mask != 0xFFFF {\n\t\tlength += 8\n\t}\n\tbuff := make([]byte, length)\n\tw := utils.NewNativeBuffer(buff)\n\tw.Put8(acl.Version) // version\n\tw.Put8(0) // flag\n\tw.Put16(0) // filler\n\twRule := func(tag, perm uint16, id uint32) {\n\t\tw.Put16(tag)\n\t\tw.Put16(perm)\n\t\tw.Put32(id)\n\t}\n\twRule(1, n.Owner, 0xFFFFFFFF)\n\tfor _, rule := range n.NamedUsers {\n\t\twRule(2, rule.Perm, rule.Id)\n\t}\n\twRule(4, n.Group, 0xFFFFFFFF)\n\tfor _, rule := range n.NamedGroups {\n\t\twRule(8, rule.Perm, rule.Id)\n\t}\n\tif n.Mask != 0xFFFF {\n\t\twRule(0x10, n.Mask, 0xFFFFFFFF)\n\t}\n\twRule(0x20, n.Other, 0xFFFFFFFF)\n\treturn buff\n}\n\nfunc decodeACL(buff []byte) (*acl.Rule, syscall.Errno) {\n\tlength := len(buff)\n\tif length < 4 || ((length % 8) != 4) || buff[0] != acl.Version {\n\t\treturn nil, syscall.EINVAL\n\t}\n\n\tn := acl.EmptyRule()\n\tr := utils.NewNativeBuffer(buff[4:])\n\tfor r.HasMore() {\n\t\ttag := r.Get16()\n\t\tperm := r.Get16()\n\t\tid := r.Get32()\n\t\tswitch tag {\n\t\tcase 1:\n\t\t\tif n.Owner != 0xFFFF {\n\t\t\t\treturn nil, syscall.EINVAL\n\t\t\t}\n\t\t\tn.Owner = perm\n\t\tcase 2:\n\t\t\tn.NamedUsers = append(n.NamedUsers, acl.Entry{Id: id, Perm: perm})\n\t\tcase 4:\n\t\t\tif n.Group != 0xFFFF {\n\t\t\t\treturn nil, syscall.EINVAL\n\t\t\t}\n\t\t\tn.Group = perm\n\t\tcase 8:\n\t\t\tn.NamedGroups = append(n.NamedGroups, acl.Entry{Id: id, Perm: perm})\n\t\tcase 0x10:\n\t\t\tif n.Mask != 0xFFFF {\n\t\t\t\treturn nil, syscall.EINVAL\n\t\t\t}\n\t\t\tn.Mask = perm\n\t\tcase 0x20:\n\t\t\tif n.Other != 0xFFFF {\n\t\t\t\treturn nil, syscall.EINVAL\n\t\t\t}\n\t\t\tn.Other = perm\n\t\t}\n\t}\n\tif n.Mask == 0xFFFF && len(n.NamedUsers)+len(n.NamedGroups) > 0 {\n\t\treturn nil, syscall.EINVAL\n\t}\n\treturn n, 0\n}\n\nvar aclTypes = map[string]uint8{\n\t_SECURITY_ACL: acl.TypeAccess,\n\t_SECURITY_ACL_DEFAULT: acl.TypeDefault,\n}\n" }, { "path": "pkg/vfs/vfs_test.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strings\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/stretchr/testify/require\"\n\t\"golang.org/x/sys/unix\"\n)\n\n// nolint:errcheck\n\nfunc createTestVFS(applyMetaConfOption func(metaConfig *meta.Config), metaUri string) (*VFS, object.ObjectStorage) {\n\tmp := \"/jfs\"\n\tmetaConf := meta.DefaultConf()\n\tmetaConf.MountPoint = mp\n\tif applyMetaConfOption != nil {\n\t\tapplyMetaConfOption(metaConf)\n\t}\n\tif metaUri == \"\" {\n\t\tmetaUri = \"memkv://\"\n\t}\n\tm := meta.NewClient(metaUri, metaConf)\n\tformat := &meta.Format{\n\t\tName: \"test\",\n\t\tUUID: uuid.New().String(),\n\t\tStorage: \"mem\",\n\t\tBlockSize: 4096,\n\t\tCompression: \"lz4\",\n\t\tDirStats: true,\n\t}\n\terr := m.Init(format, true)\n\tif err != nil {\n\t\tlog.Fatalf(\"setting: %s\", err)\n\t}\n\tconf := &Config{\n\t\tMeta: metaConf,\n\t\tFormat: *format,\n\t\tVersion: \"Juicefs\",\n\t\tChunk: &chunk.Config{\n\t\t\tBlockSize: format.BlockSize * 1024,\n\t\t\tCompress: format.Compression,\n\t\t\tMaxUpload: 2,\n\t\t\tMaxDownload: 200,\n\t\t\tBufferSize: 30 << 20,\n\t\t\tCacheSize: 10 << 20,\n\t\t\tCacheDir: \"memory\",\n\t\t},\n\t\tFuseOpts: &FuseOptions{},\n\t}\n\tblob, _ := object.CreateStorage(\"mem\", \"\", \"\", \"\", \"\")\n\tregistry := prometheus.NewRegistry() // replace default so only JuiceFS metrics are exposed\n\tregisterer := prometheus.WrapRegistererWithPrefix(\"juicefs_\",\n\t\tprometheus.WrapRegistererWith(prometheus.Labels{\"mp\": mp, \"vol_name\": format.Name}, registry))\n\tstore := chunk.NewCachedStore(blob, *conf.Chunk, registry)\n\treturn NewVFS(conf, m, store, registerer, registry), blob\n}\n\nfunc TestVFSBasic(t *testing.T) {\n\tv, _ := createTestVFS(nil, \"\")\n\tctx := NewLogContext(meta.NewContext(10, 1, []uint32{2, 3}))\n\n\tif st, e := v.StatFS(ctx, 1); e != 0 {\n\t\tt.Fatalf(\"statfs 1: %s\", e)\n\t} else if st.Total-st.Avail != 0 {\n\t\tt.Fatalf(\"used: %d\", st.Total-st.Avail)\n\t}\n\n\t// dirs\n\tde, e := v.Mkdir(ctx, 1, \"d1\", 0755, 0)\n\tif e != 0 {\n\t\tt.Fatalf(\"mkdir d1: %s\", e)\n\t}\n\tif _, e := v.Mkdir(ctx, de.Inode, \"d2\", 0755, 0); e != 0 {\n\t\tt.Fatalf(\"mkdir d1/d2: %s\", e)\n\t}\n\tif e := v.Rmdir(ctx, 1, \"d1\"); e != syscall.ENOTEMPTY {\n\t\tt.Fatalf(\"rmdir not empty: %s\", e)\n\t}\n\tif e := v.Rmdir(ctx, de.Inode, \"d2\"); e != 0 {\n\t\tt.Fatalf(\"rmdir d1/d2: %s\", e)\n\t}\n\n\t// files\n\tfe, e := v.Mknod(ctx, de.Inode, \"f1\", 0644|syscall.S_IFREG, 0, 0)\n\tif e != 0 {\n\t\tt.Fatalf(\"mknod d1/f1: %s\", e)\n\t}\n\tif e := v.Access(ctx, fe.Inode, unix.X_OK); e != syscall.EACCES {\n\t\tt.Fatalf(\"access d1/f1: %s\", e)\n\t}\n\tif _, e := v.SetAttr(ctx, fe.Inode, meta.SetAttrMtimeNow|meta.SetAttrAtimeNow, 0, 0, 0, 0, 0, 0, 0, 0, 0); e != 0 {\n\t\tt.Fatalf(\"setattr d1/f2 mtimeNow: %s\", e)\n\t}\n\tif fe2, e := v.SetAttr(ctx, fe.Inode, meta.SetAttrMode|meta.SetAttrUID|meta.SetAttrGID|meta.SetAttrAtime|meta.SetAttrMtime|meta.SetAttrSize, 0, 0755, 1, 3, 1234, 1234, 5678, 5678, 1024); e != 0 {\n\t\tt.Fatalf(\"setattr d1/f1: %s %d %d\", e, fe2.Attr.Gid, fe2.Attr.Length)\n\t} else if fe2.Attr.Mode != 0755 || fe2.Attr.Uid != 1 || fe2.Attr.Gid != 3 || fe2.Attr.Atime != 1234 || fe2.Attr.Atimensec != 5678 || fe2.Attr.Mtime != 1234 || fe2.Attr.Mtimensec != 5678 || fe2.Attr.Length != 1024 {\n\t\tt.Fatalf(\"setattr d1/f1: %+v\", fe2.Attr)\n\t}\n\tif e := v.Access(ctx, fe.Inode, unix.X_OK); e != 0 {\n\t\tt.Fatalf(\"access d1/f1: %s\", e)\n\t}\n\tif _, e := v.Link(ctx, fe.Inode, 1, \"f2\"); e != 0 {\n\t\tt.Fatalf(\"link f2->f1: %s\", e)\n\t}\n\tif fe, e := v.GetAttr(ctx, fe.Inode, 0); e != 0 || fe.Attr.Nlink != 2 {\n\t\tt.Fatalf(\"getattr d1/f2: %s %d\", e, fe.Attr.Nlink)\n\t}\n\tif e := v.Unlink(ctx, de.Inode, \"f1\"); e != 0 {\n\t\tt.Fatalf(\"unlink d1/f1: %s\", e)\n\t}\n\tif fe, e := v.Lookup(ctx, 1, \"f2\"); e != 0 || fe.Attr.Nlink != 1 {\n\t\tt.Fatalf(\"lookup f2: %s\", e)\n\t}\n\tif e := v.Rename(ctx, 1, \"f2\", 1, \"f3\", 0); e != 0 {\n\t\tt.Fatalf(\"rename f2 -> f3: %s\", e)\n\t}\n\tif fe, fh, e := v.Open(ctx, fe.Inode, syscall.O_RDONLY); e != 0 {\n\t\tt.Fatalf(\"open f3: %s\", e)\n\t} else if e := v.Flush(ctx, fe.Inode, fh, 0); e != 0 {\n\t\tt.Fatalf(\"close f3: %s\", e)\n\t} else {\n\t\tv.Release(ctx, fe.Inode, fh)\n\t}\n\n\t// symlink\n\tif fe, e := v.Symlink(ctx, \"f2\", 1, \"sym\"); e != 0 {\n\t\tt.Fatalf(\"symlink sym -> f2: %s\", e)\n\t} else if target, e := v.Readlink(ctx, fe.Inode); e != 0 || string(target) != \"f2\" {\n\t\tt.Fatalf(\"readlink sym: %s %s\", e, string(target))\n\t}\n\n\t// edge cases\n\tlongName := strings.Repeat(\"a\", 256)\n\tif _, e = v.Lookup(ctx, 1, longName); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"lookup long name\")\n\t}\n\tif _, _, e = v.Create(ctx, 1, longName, 0, 0, 0); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"create long name\")\n\t}\n\tif _, e = v.Mknod(ctx, 1, longName, 0, 0, 0); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"mknod long name\")\n\t}\n\tif _, e = v.Mkdir(ctx, 1, longName, 0, 0); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"mkdir long name\")\n\t}\n\tif _, e = v.Link(ctx, 2, 1, longName); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"link long name\")\n\t}\n\tif e = v.Unlink(ctx, 1, longName); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"unlink long name\")\n\t}\n\tif e = v.Rmdir(ctx, 1, longName); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"rmdir long name\")\n\t}\n\tif _, e = v.Symlink(ctx, \"\", 1, longName); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"symlink long name\")\n\t}\n\tif e = v.Rename(ctx, 1, \"a\", 1, longName, 0); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"rename long name\")\n\t}\n\tif e = v.Rename(ctx, 1, longName, 1, \"a\", 0); e != syscall.ENAMETOOLONG {\n\t\tt.Fatalf(\"rename long name\")\n\t}\n\n}\n\nfunc TestVFSIO(t *testing.T) {\n\tv, _ := createTestVFS(nil, \"\")\n\tctx := NewLogContext(meta.Background())\n\tfe, fh, e := v.Create(ctx, 1, \"file\", 0755, 0, syscall.O_RDWR)\n\tif e != 0 {\n\t\tt.Fatalf(\"create file: %s\", e)\n\t}\n\tif e = v.Fallocate(ctx, fe.Inode, 0, 0, 64<<10, fh); e != 0 {\n\t\tt.Fatalf(\"fallocate : %s\", e)\n\t}\n\tif e = v.Write(ctx, fe.Inode, []byte(\"hello\"), 0, fh); e != 0 {\n\t\tt.Fatalf(\"write file: %s\", e)\n\t}\n\tif e = v.Fsync(ctx, fe.Inode, 1, fh); e != 0 {\n\t\tt.Fatalf(\"fsync file: %s\", e)\n\t}\n\tif e = v.Write(ctx, fe.Inode, []byte(\"hello\"), 100<<20, fh); e != 0 {\n\t\tt.Fatalf(\"write file: %s\", e)\n\t}\n\tvar attr meta.Attr\n\tif e = v.Truncate(ctx, fe.Inode, (100<<20)+2, fh, &attr); e != 0 {\n\t\tt.Fatalf(\"truncate file: %s\", e)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, fe.Inode, fh, 10<<20, 10, 0); e != 0 || n != 10 {\n\t\tt.Fatalf(\"copyfilerange: %s %d\", e, n)\n\t}\n\tvar buf = make([]byte, 128<<10)\n\tif n, e := v.Read(ctx, fe.Inode, buf, 0, fh); e != 0 {\n\t\tt.Fatalf(\"read file: %s\", e)\n\t} else if n != len(buf) {\n\t\tt.Fatalf(\"short read file: %d != %d\", n, len(buf))\n\t} else if string(buf[:5]) != \"hello\" {\n\t\tt.Fatalf(\"unexpected data: %q\", string(buf[:5]))\n\t}\n\tif n, e := v.Read(ctx, fe.Inode, buf[:6], 10<<20, fh); e != 0 || n != 6 || string(buf[:n]) != \"hello\\x00\" {\n\t\tt.Fatalf(\"read file end: %s %d %s\", e, n, string(buf[:n]))\n\t}\n\tif n, e := v.Read(ctx, fe.Inode, buf, 100<<20, fh); e != 0 || n != 2 || string(buf[:n]) != \"he\" {\n\t\tt.Fatalf(\"read file end: %s %d %s\", e, n, string(buf[:n]))\n\t}\n\tif e = v.Flush(ctx, fe.Inode, fh, 0); e != 0 {\n\t\tt.Fatalf(\"flush file: %s\", e)\n\t}\n\n\t// edge cases\n\t_, fh2, _ := v.Open(ctx, fe.Inode, syscall.O_RDONLY)\n\t_, fh3, _ := v.Open(ctx, fe.Inode, syscall.O_WRONLY)\n\twHandle := v.findHandle(fe.Inode, fh3)\n\tif wHandle == nil {\n\t\tt.Fatalf(\"failed to find O_WRONLY handle\")\n\t}\n\twHandle.reader = nil\n\t// read\n\tif _, e = v.Read(ctx, fe.Inode, nil, 0, 0); e != syscall.EBADF {\n\t\tt.Fatalf(\"read bad fd: %s\", e)\n\t}\n\tif _, e = v.Read(ctx, fe.Inode, make([]byte, 1024), 0, fh3); e != syscall.EBADF {\n\t\tt.Fatalf(\"read write-only fd: %s\", e)\n\t}\n\tif _, e = v.Read(ctx, fe.Inode, nil, 1<<60, fh2); e != syscall.EFBIG {\n\t\tt.Fatalf(\"read off too big: %s\", e)\n\t}\n\t// write\n\tif e = v.Write(ctx, fe.Inode, nil, 0, 0); e != syscall.EBADF {\n\t\tt.Fatalf(\"write bad fd: %s\", e)\n\t}\n\tif e = v.Write(ctx, fe.Inode, nil, 1<<60, fh2); e != syscall.EFBIG {\n\t\tt.Fatalf(\"write off too big: %s\", e)\n\t}\n\tif e = v.Write(ctx, fe.Inode, make([]byte, 1024), 0, fh2); e != syscall.EBADF {\n\t\tt.Fatalf(\"write read-only fd: %s\", e)\n\t}\n\t// truncate\n\tif e = v.Truncate(ctx, fe.Inode, -1, 0, &meta.Attr{}); e != syscall.EINVAL {\n\t\tt.Fatalf(\"truncate invalid off,length: %s\", e)\n\t}\n\tif e = v.Truncate(ctx, fe.Inode, 1<<60, 0, &meta.Attr{}); e != syscall.EFBIG {\n\t\tt.Fatalf(\"truncate too large: %s\", e)\n\t}\n\t// fallocate\n\tif e = v.Fallocate(ctx, fe.Inode, 0, -1, -1, fh); e != syscall.EINVAL {\n\t\tt.Fatalf(\"fallocate invalid off,length: %s\", e)\n\t}\n\tif e = v.Fallocate(ctx, StatsInode, 0, 0, 1, fh); e != syscall.EPERM {\n\t\tt.Fatalf(\"fallocate invalid off,length: %s\", e)\n\t}\n\tif e = v.Fallocate(ctx, fe.Inode, 0, 0, 100, 0); e != syscall.EBADF {\n\t\tt.Fatalf(\"fallocate invalid off,length: %s\", e)\n\t}\n\tif e = v.Fallocate(ctx, fe.Inode, 0, 1<<60, 1<<60, fh); e != syscall.EFBIG {\n\t\tt.Fatalf(\"fallocate invalid off,length: %s\", e)\n\t}\n\tif e = v.Fallocate(ctx, fe.Inode, 0, 1<<10, 1<<20, fh2); e != syscall.EBADF {\n\t\tt.Fatalf(\"fallocate read-only fd: %s\", e)\n\t}\n\n\t// copy file range\n\tif n, e := v.CopyFileRange(ctx, StatsInode, fh, 0, fe.Inode, fh, 10<<20, 10, 0); e != syscall.ENOTSUP {\n\t\tt.Fatalf(\"copyfilerange internal file: %s %d\", e, n)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, StatsInode, fh, 10<<20, 10, 0); e != syscall.EPERM {\n\t\tt.Fatalf(\"copyfilerange internal file: %s %d\", e, n)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, 0, 0, fe.Inode, fh, 10<<20, 10, 0); e != syscall.EBADF {\n\t\tt.Fatalf(\"copyfilerange invalid fh: %s %d\", e, n)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, fe.Inode, 0, 10<<20, 10, 0); e != syscall.EBADF {\n\t\tt.Fatalf(\"copyfilerange invalid fh: %s %d\", e, n)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, fe.Inode, fh, 10<<20, 10, 1); e != syscall.EINVAL {\n\t\tt.Fatalf(\"copyfilerange invalid flag: %s %d\", e, n)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, fe.Inode, fh, 10<<20, 1<<50, 0); e != syscall.EINVAL {\n\t\tt.Fatalf(\"copyfilerange overlap: %s %d\", e, n)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, fe.Inode, fh, 1<<63, 1<<63, 0); e != syscall.EFBIG {\n\t\tt.Fatalf(\"copyfilerange too big file: %s %d\", e, n)\n\t}\n\tif n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, fe.Inode, fh2, 1<<20, 1<<10, 0); e != syscall.EACCES {\n\t\tt.Fatalf(\"copyfilerange too big file: %s %d\", e, n)\n\t}\n\n\t// sequntial write/read\n\tfor i := uint64(0); i < 1001; i++ {\n\t\tif e := v.Write(ctx, fe.Inode, make([]byte, 128<<10), i*(128<<10), fh); e != 0 {\n\t\t\tt.Fatalf(\"write big file: %s\", e)\n\t\t}\n\t}\n\tbuf = make([]byte, 128<<10)\n\tfor i := uint64(0); i < 1000; i++ {\n\t\tif n, e := v.Read(ctx, fe.Inode, buf, i*(128<<10), fh); e != 0 || n != (128<<10) {\n\t\t\tt.Fatalf(\"read big file: %s\", e)\n\t\t} else {\n\t\t\tfor j := 0; j < 128<<10; j++ {\n\t\t\t\tif buf[j] != 0 {\n\t\t\t\t\tt.Fatalf(\"read big file: %d %d\", j, buf[j])\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// many small write\n\tbuf = make([]byte, 5<<10)\n\tfor j := range buf {\n\t\tbuf[j] = 1\n\t}\n\tfor i := int64(32 - 1); i >= 0; i-- {\n\t\tif e := v.Write(ctx, fe.Inode, buf, uint64(i)*(4<<10), fh); e != 0 {\n\t\t\tt.Fatalf(\"write big file: %s\", e)\n\t\t}\n\t}\n\ttime.Sleep(time.Millisecond * 1500) // wait for it to be flushed\n\tbuf = make([]byte, 128<<10)\n\tif n, e := v.Read(ctx, fe.Inode, buf, 0, fh); e != 0 || n != (128<<10) {\n\t\tt.Fatalf(\"read big file: %s\", e)\n\t} else {\n\t\tfor j := range buf {\n\t\t\tif buf[j] != 1 {\n\t\t\t\tt.Fatalf(\"read big file: %d %d\", j, buf[j])\n\t\t\t}\n\t\t}\n\t}\n\n\tv.Release(ctx, fe.Inode, fh)\n}\n\nfunc TestVFSXattrs(t *testing.T) {\n\tv, _ := createTestVFS(nil, \"\")\n\tctx := NewLogContext(meta.Background())\n\tfe, e := v.Mkdir(ctx, 1, \"xattrs\", 0755, 0)\n\tif e != 0 {\n\t\tt.Fatalf(\"mkdir xattrs: %s\", e)\n\t}\n\t// normal cases\n\tif _, e := v.GetXattr(ctx, fe.Inode, \"test\", 0); e != meta.ENOATTR {\n\t\tt.Fatalf(\"getxattr not existed: %s\", e)\n\t}\n\tif e := v.SetXattr(ctx, fe.Inode, \"test\", []byte(\"value\"), 0); e != 0 {\n\t\tt.Fatalf(\"setxattr test: %s\", e)\n\t}\n\tif e = v.SetXattr(ctx, fe.Inode, \"test\", []byte(\"v1\"), meta.XattrCreate); e == 0 {\n\t\tt.Fatalf(\"setxattr test (create): %s\", e)\n\t}\n\tif v, e := v.ListXattr(ctx, fe.Inode, 100); e != 0 || string(v) != \"test\\x00\" {\n\t\tt.Fatalf(\"listxattr: %s %q\", e, string(v))\n\t}\n\tif v, e := v.GetXattr(ctx, fe.Inode, \"test\", 5); e != 0 || string(v) != \"value\" {\n\t\tt.Fatalf(\"getxattr test: %s %v\", e, v)\n\t}\n\tif e = v.SetXattr(ctx, fe.Inode, \"test\", []byte(\"v2\"), meta.XattrReplace); e != 0 {\n\t\tt.Fatalf(\"setxattr test (replace): %s\", e)\n\t}\n\tif v, e := v.GetXattr(ctx, fe.Inode, \"test\", 5); e != 0 || string(v) != \"v2\" {\n\t\tt.Fatalf(\"getxattr test: %s %v\", e, v)\n\t}\n\tif _, e := v.GetXattr(ctx, fe.Inode, \"test\", 1); e != syscall.ERANGE {\n\t\tt.Fatalf(\"getxattr large value: %s\", e)\n\t}\n\tif v, e := v.ListXattr(ctx, fe.Inode, 1); e != syscall.ERANGE {\n\t\tt.Fatalf(\"listxattr: %s %q\", e, string(v))\n\t}\n\tif e := v.RemoveXattr(ctx, fe.Inode, \"test\"); e != 0 {\n\t\tt.Fatalf(\"removexattr test: %s\", e)\n\t}\n\tif _, e := v.GetXattr(ctx, fe.Inode, \"test\", 0); e != meta.ENOATTR {\n\t\tt.Fatalf(\"getxattr not existed: %s\", e)\n\t}\n\tif v, e := v.ListXattr(ctx, fe.Inode, 100); e != 0 || string(v) != \"\" {\n\t\tt.Fatalf(\"listxattr: %s %q\", e, string(v))\n\t}\n\t// edge case\n\tif e = v.SetXattr(ctx, fe.Inode, \"\", []byte(\"v2\"), 0); e != syscall.EINVAL {\n\t\tt.Fatalf(\"setxattr long key: %s\", e)\n\t}\n\tif e = v.SetXattr(ctx, fe.Inode, strings.Repeat(\"test\", 100), []byte(\"v2\"), 0); e != syscall.EPERM && e != syscall.ERANGE {\n\t\tt.Fatalf(\"setxattr long key: %s\", e)\n\t}\n\tif e = v.SetXattr(ctx, fe.Inode, \"test\", make([]byte, 1<<20), 0); e != syscall.E2BIG && e != syscall.ERANGE {\n\t\tt.Fatalf(\"setxattr long key: %s\", e)\n\t}\n\tif e = v.SetXattr(ctx, fe.Inode, \"system.posix_acl_access\", []byte(\"v2\"), 0); e != syscall.ENOTSUP {\n\t\tt.Fatalf(\"setxattr long key: %s\", e)\n\t}\n\tif e = v.SetXattr(ctx, ConfigInode, \"test\", []byte(\"v2\"), 0); e != syscall.EPERM {\n\t\tt.Fatalf(\"setxattr long key: %s\", e)\n\t}\n\tif _, e := v.GetXattr(ctx, fe.Inode, \"\", 0); e != syscall.EINVAL {\n\t\tt.Fatalf(\"getxattr not existed: %s\", e)\n\t}\n\tif _, e := v.GetXattr(ctx, fe.Inode, strings.Repeat(\"test\", 100), 0); e == 0 {\n\t\tt.Fatalf(\"getxattr not existed: %s\", e)\n\t}\n\tif _, e := v.GetXattr(ctx, ConfigInode, \"test\", 0); e != meta.ENOATTR {\n\t\tt.Fatalf(\"getxattr not existed: %s\", e)\n\t}\n\tif _, e := v.GetXattr(ctx, fe.Inode, \"system.posix_acl_access\", 0); e != syscall.ENODATA {\n\t\tt.Fatalf(\"getxattr not existed: %s\", e)\n\t}\n\tif v, e := v.ListXattr(ctx, ConfigInode, 0); e != meta.ENOATTR {\n\t\tt.Fatalf(\"listxattr: %s %q\", e, string(v))\n\t}\n\tif e := v.RemoveXattr(ctx, fe.Inode, strings.Repeat(\"test\", 100)); e != syscall.EPERM && e != syscall.ERANGE {\n\t\tt.Fatalf(\"removexattr test: %s\", e)\n\t}\n\tif e := v.RemoveXattr(ctx, fe.Inode, \"\"); e != syscall.EINVAL {\n\t\tt.Fatalf(\"removexattr test: %s\", e)\n\t}\n\tif e := v.RemoveXattr(ctx, fe.Inode, \"system.posix_acl_access\"); e != syscall.ENOTSUP {\n\t\tt.Fatalf(\"removexattr test: %s\", e)\n\t}\n\tif e := v.RemoveXattr(ctx, ConfigInode, \"test\"); e != syscall.EPERM {\n\t\tt.Fatalf(\"removexattr test: %s\", e)\n\t}\n}\n\ntype accessCase struct {\n\tuid uint32\n\tgid uint32\n\tmode uint16\n\tr syscall.Errno\n}\n\nfunc TestAccessMode(t *testing.T) {\n\tvar attr = meta.Attr{\n\t\tUid: 1,\n\t\tGid: 2,\n\t\tMode: 0751,\n\t}\n\n\tcases := []accessCase{\n\t\t{0, 0, MODE_MASK_R | MODE_MASK_W | MODE_MASK_X, 0},\n\t\t{1, 3, MODE_MASK_R | MODE_MASK_W | MODE_MASK_X, 0},\n\t\t{2, 2, MODE_MASK_R | MODE_MASK_X, 0},\n\t\t{2, 2, MODE_MASK_W, syscall.EACCES},\n\t\t{3, 4, MODE_MASK_X, 0},\n\t\t{3, 4, MODE_MASK_R, syscall.EACCES},\n\t\t{3, 4, MODE_MASK_W, syscall.EACCES},\n\t}\n\tfor _, c := range cases {\n\t\tif e := accessTest(&attr, c.mode, c.uid, c.gid); e != c.r {\n\t\t\tt.Fatalf(\"expect %s on case %+v, but got %s\", c.r, c, e)\n\t\t}\n\t}\n}\n\nfunc assertEqual(t *testing.T, a interface{}, b interface{}) {\n\tif reflect.DeepEqual(a, b) {\n\t\treturn\n\t}\n\tmessage := fmt.Sprintf(\"%v != %v\", a, b)\n\tt.Fatal(message)\n}\n\nfunc TestSetattrStr(t *testing.T) {\n\tassertEqual(t, setattrStr(0, 0, 0, 0, 0, 0, 0), \"\")\n\tassertEqual(t, setattrStr(meta.SetAttrMode, 01755, 0, 0, 0, 0, 0), \"mode=?rwxr-xr-t:01755\")\n\tassertEqual(t, setattrStr(meta.SetAttrUID, 0, 1, 0, 0, 0, 0), \"uid=1\")\n\tassertEqual(t, setattrStr(meta.SetAttrGID, 0, 1, 2, 0, 0, 0), \"gid=2\")\n\tassertEqual(t, setattrStr(meta.SetAttrAtime, 0, 0, 0, -2, -1, 0), \"atime=NOW\")\n\tassertEqual(t, setattrStr(meta.SetAttrAtime, 0, 0, 0, 123, 123, 0), \"atime=123\")\n\tassertEqual(t, setattrStr(meta.SetAttrAtimeNow, 0, 0, 0, 0, 0, 0), \"atime=NOW\")\n\tassertEqual(t, setattrStr(meta.SetAttrMtime, 0, 0, 0, 0, -1, 0), \"mtime=NOW\")\n\tassertEqual(t, setattrStr(meta.SetAttrMtime, 0, 0, 0, 0, 123, 0), \"mtime=123\")\n\tassertEqual(t, setattrStr(meta.SetAttrMtimeNow, 0, 0, 0, 0, 0, 0), \"mtime=NOW\")\n\tassertEqual(t, setattrStr(meta.SetAttrSize, 0, 0, 0, 0, 0, 123), \"size=123\")\n\tassertEqual(t, setattrStr(meta.SetAttrUID|meta.SetAttrGID, 0, 1, 2, 0, 0, 0), \"uid=1,gid=2\")\n}\n\nfunc TestVFSLocks(t *testing.T) {\n\tv, _ := createTestVFS(nil, \"\")\n\tctx := NewLogContext(meta.Background())\n\tfe, fh, e := v.Create(ctx, 1, \"flock\", 0644, 0, syscall.O_RDWR)\n\tif e != 0 {\n\t\tt.Fatalf(\"create flock: %s\", e)\n\t}\n\t// flock\n\tif e = v.Flock(ctx, fe.Inode, fh, 123, 100, true); e != syscall.EINVAL {\n\t\tt.Fatalf(\"flock wr: %s\", e)\n\t}\n\tif e = v.Flock(ctx, fe.Inode, fh, 123, syscall.F_WRLCK, true); e != 0 {\n\t\tt.Fatalf(\"flock wr: %s\", e)\n\t}\n\tif e := v.Flock(ctx, fe.Inode, fh, 456, syscall.F_RDLCK, false); e != syscall.EAGAIN {\n\t\tt.Fatalf(\"flock rd: should block\")\n\t}\n\n\tdone := make(chan bool)\n\tgo func() {\n\t\t_ = v.Flock(ctx, fe.Inode, fh, 456, syscall.F_RDLCK, true)\n\t\tdone <- true\n\t}()\n\tif e := v.Flock(ctx, fe.Inode, fh, 123, syscall.F_UNLCK, true); e != 0 {\n\t\tt.Fatalf(\"flock unlock: %s\", e)\n\t}\n\tselect {\n\tcase <-done:\n\tcase <-time.NewTimer(time.Millisecond * 100).C:\n\t\tt.Fatalf(\"flock timeout on rdlock\")\n\t}\n\tif e := v.Flock(ctx, fe.Inode, fh, 456, syscall.F_UNLCK, true); e != 0 {\n\t\tt.Fatalf(\"flock unlock rd: %s\", e)\n\t}\n\n\t// posix lock\n\tif e = v.Setlk(ctx, fe.Inode, fh, 1, 0, 100, 100, 1, true); e != syscall.EINVAL {\n\t\tt.Fatalf(\"setlk: %s\", e)\n\t}\n\tif e = v.Setlk(ctx, fe.Inode, fh, 1, 0, 100, syscall.F_WRLCK, 1, true); e != 0 {\n\t\tt.Fatalf(\"setlk: %s\", e)\n\t}\n\tvar start, len uint64 = 10, 1000\n\tvar typ, pid uint32 = syscall.LOCK_UN, 10\n\tif e = v.Getlk(ctx, fe.Inode, fh, 2, &start, &len, &typ, &pid); e != syscall.EINVAL {\n\t\tt.Fatalf(\"getlk: %s\", e)\n\t}\n\ttyp = syscall.F_RDLCK\n\tif e = v.Getlk(ctx, fe.Inode, fh, 2, &start, &len, &typ, &pid); e != 0 {\n\t\tt.Fatalf(\"getlk: %s\", e)\n\t} else if start != 0 || len != 100 || typ != syscall.F_WRLCK || pid != 1 {\n\t\tt.Fatalf(\"getlk result: %d %d %d %d\", start, len, typ, pid)\n\t}\n\tif e = v.Setlk(ctx, fe.Inode, fh, 2, 10, 100, syscall.F_RDLCK, 10, false); e != syscall.EAGAIN {\n\t\tt.Fatalf(\"setlk rd: %s\", e)\n\t}\n\tgo func() {\n\t\t_ = v.Setlk(ctx, fe.Inode, fh, 2, 10, 100, syscall.F_RDLCK, 10, false)\n\t\tdone <- true\n\t}()\n\tif e = v.Setlk(ctx, fe.Inode, fh, 1, 10, 100, syscall.F_UNLCK, 1, true); e != 0 {\n\t\tt.Fatalf(\"setlk unlock: %s\", e)\n\t}\n\tselect {\n\tcase <-done:\n\tcase <-time.NewTimer(time.Millisecond * 100).C:\n\t\tt.Fatalf(\"setlk timeout on rdlock\")\n\t}\n\tif e = v.Setlk(ctx, fe.Inode, fh, 2, 0, 20, syscall.F_RDLCK, 10, false); e != syscall.EAGAIN {\n\t\tt.Fatalf(\"setlk rd: %s\", e)\n\t}\n\tif e = v.Setlk(ctx, fe.Inode, fh, 1, 0, 1000, syscall.F_UNLCK, 1, true); e != 0 {\n\t\tt.Fatalf(\"setlk unlock: %s\", e)\n\t}\n\tif e = v.Flush(ctx, fe.Inode, fh, 0); e != 0 {\n\t\tt.Fatalf(\"flush: %s\", e)\n\t}\n\tv.Release(ctx, fe.Inode, fh)\n\t// invalid fd\n\tif e = v.Flock(ctx, fe.Inode, 10, 123, syscall.F_WRLCK, true); e != syscall.EBADF {\n\t\tt.Fatalf(\"flock wr: %s\", e)\n\t}\n\tif e = v.Setlk(ctx, fe.Inode, 10, 1, 0, 1000, syscall.F_UNLCK, 1, true); e != syscall.EBADF {\n\t\tt.Fatalf(\"setlk unlock: %s\", e)\n\t}\n\tif e = v.Getlk(ctx, fe.Inode, 10, 2, &start, &len, &typ, &pid); e != syscall.EBADF {\n\t\tt.Fatalf(\"getlk: %s\", e)\n\t}\n\t// internal file\n\tfe, _ = v.Lookup(ctx, 1, \".stats\")\n\tif e = v.Flock(ctx, fe.Inode, 10, 123, syscall.F_WRLCK, true); e != syscall.EPERM {\n\t\tt.Fatalf(\"flock wr: %s\", e)\n\t}\n\tif e = v.Setlk(ctx, fe.Inode, 10, 1, 0, 1000, syscall.F_UNLCK, 1, true); e != syscall.EPERM {\n\t\tt.Fatalf(\"setlk unlock: %s\", e)\n\t}\n\tif e = v.Getlk(ctx, fe.Inode, 10, 2, &start, &len, &typ, &pid); e != syscall.EPERM {\n\t\tt.Fatalf(\"getlk: %s\", e)\n\t}\n}\n\nfunc TestInternalFile(t *testing.T) {\n\tv, _ := createTestVFS(nil, \"\")\n\tctx := NewLogContext(meta.Background())\n\t// list internal files\n\tfh, _ := v.Opendir(ctx, 1, 0)\n\tentries, _, e := v.Readdir(ctx, 1, 1024, 0, fh, true)\n\tif e != 0 {\n\t\tt.Fatalf(\"readdir 1: %s\", e)\n\t}\n\tinternalFiles := make(map[string]bool)\n\tfor _, e := range entries {\n\t\tif IsSpecialName(string(e.Name)) && e.Attr.Typ == meta.TypeFile {\n\t\t\tinternalFiles[string(e.Name)] = true\n\t\t}\n\t}\n\tif len(internalFiles) != 3 {\n\t\tt.Fatalf(\"there should be 3 internal files but got %d\", len(internalFiles))\n\t}\n\tv.Releasedir(ctx, 1, fh)\n\n\t// .config\n\tctx2 := NewLogContext(meta.NewContext(10, 111, []uint32{222}))\n\tfe, e := v.Lookup(ctx2, 1, \".config\")\n\tif e != 0 {\n\t\tt.Fatalf(\"lookup .config: %s\", e)\n\t}\n\tif e := v.Access(ctx2, fe.Inode, unix.R_OK); e != syscall.EACCES { // other user can't access .config\n\t\tt.Fatalf(\"access .config: %s\", e)\n\t}\n\tif _, e := v.GetAttr(ctx, fe.Inode, 0); e != 0 {\n\t\tt.Fatalf(\"getattr .config: %s\", e)\n\t}\n\t// ignore setattr on internal files\n\tif fe2, e := v.SetAttr(ctx, fe.Inode, meta.SetAttrUID, 0, 0, ctx2.Uid(), 0, 0, 0, 0, 0, 0); e != 0 || fe2.Attr.Uid != fe.Attr.Uid {\n\t\tt.Fatalf(\"can't setattr on internal files\")\n\t}\n\tif e = v.Unlink(ctx, 1, \".config\"); e != syscall.EPERM {\n\t\tt.Fatalf(\"should not unlink internal file\")\n\t}\n\tif _, _, e = v.Open(ctx, fe.Inode, syscall.O_WRONLY); e != syscall.EACCES {\n\t\tt.Fatalf(\"write .config: %s\", e)\n\t}\n\t_, fh, e = v.Open(ctx, fe.Inode, syscall.O_RDONLY)\n\tif e != 0 {\n\t\tt.Fatalf(\"open .config: %s\", e)\n\t}\n\tbuf := make([]byte, 10240)\n\tif _, e := v.Read(ctx, fe.Inode, buf, 0, 0); e != syscall.EBADF {\n\t\tt.Fatalf(\"read .config: %s\", e)\n\t}\n\tif n, e := v.Read(ctx, fe.Inode, buf, 0, fh); e != 0 {\n\t\tt.Fatalf(\"read .config: %s\", e)\n\t} else if !strings.Contains(string(buf[:n]), v.Conf.Format.UUID) {\n\t\tt.Fatalf(\"invalid config: %q\", string(buf[:n]))\n\t}\n\n\t// .stats\n\tfe, e = v.Lookup(ctx, 1, \".stats\")\n\tif e != 0 {\n\t\tt.Fatalf(\"lookup .stats: %s\", e)\n\t}\n\tif e := v.Access(ctx, fe.Inode, unix.W_OK); e != 0 { // root can do everything\n\t\tt.Fatalf(\"access .stats: %s\", e)\n\t}\n\tfe, fh, e = v.Open(ctx, fe.Inode, syscall.O_RDONLY)\n\tif e != 0 {\n\t\tt.Fatalf(\"open .stats: %s\", e)\n\t}\n\tdefer v.Release(ctx, fe.Inode, fh)\n\tdefer v.Flush(ctx, fe.Inode, fh, 0)\n\tbuf = make([]byte, 128<<10)\n\tn, e := v.Read(ctx, fe.Inode, buf[:4<<10], 0, fh)\n\tif e != 0 {\n\t\tt.Fatalf(\"read .stats: %s\", e)\n\t}\n\tif n == 4<<10 {\n\t\tif n2, e := v.Read(ctx, fe.Inode, buf[n:], uint64(n), fh); e != 0 {\n\t\t\tt.Fatalf(\"read .stats 2: %s\", e)\n\t\t} else {\n\t\t\tn += n2\n\t\t}\n\t}\n\tif !strings.Contains(string(buf[:n]), \"fuse_open_handlers\") {\n\t\tt.Fatalf(\".stats should contains `memory`, but got %s\", string(buf[:n]))\n\t}\n\tif e = v.Truncate(ctx, fe.Inode, 0, 1, &meta.Attr{}); e != syscall.EPERM {\n\t\tt.Fatalf(\"truncate .config: %s\", e)\n\t}\n\n\t// accesslog\n\tfe, e = v.Lookup(ctx, 1, \".accesslog\")\n\tif e != 0 {\n\t\tt.Fatalf(\"lookup .accesslog: %s\", e)\n\t}\n\tfe, fh, e = v.Open(ctx, fe.Inode, syscall.O_RDONLY)\n\tif e != 0 {\n\t\tt.Fatalf(\"open .accesslog: %s\", e)\n\t}\n\tif n, e = v.Read(ctx, fe.Inode, buf, 0, fh); e != 0 {\n\t\tt.Fatalf(\"read .accesslog: %s\", e)\n\t} else if !strings.Contains(string(buf[:n]), \"open (9223372032559808513\") {\n\t\tt.Fatalf(\"invalid access log: %q\", string(buf[:n]))\n\t}\n\t_ = v.Flush(ctx, fe.Inode, fh, 0)\n\tv.Release(ctx, fe.Inode, fh)\n\n\t// control messages\n\tfe, e = v.Lookup(ctx, 1, \".control\")\n\tif e != 0 {\n\t\tt.Fatalf(\"lookup .control: %s\", e)\n\t}\n\tfe, fh, e = v.Open(ctx, fe.Inode, syscall.O_RDWR)\n\tif e != 0 {\n\t\tt.Fatalf(\"open .stats: %s\", e)\n\t}\n\treadControl := func(resp []byte, off *uint64) (int, syscall.Errno) {\n\t\tfor {\n\t\t\tif n, errno := v.Read(ctx, fe.Inode, resp, *off, fh); n == 0 {\n\t\t\t\ttime.Sleep(time.Millisecond * 200)\n\t\t\t} else if n%17 == 0 {\n\t\t\t\t*off += uint64(n)\n\t\t\t\tcontinue\n\t\t\t} else if n%17 == 1 {\n\t\t\t\t*off += uint64(n / 17 * 17)\n\t\t\t\tresp[0] = resp[n-1]\n\t\t\t\treturn 1, errno\n\t\t\t} else {\n\t\t\t\treturn n, errno\n\t\t\t}\n\t\t}\n\t}\n\n\treadData := func(resp []byte, fileOff *uint64) ([]byte, syscall.Errno) {\n\t\tvar off uint64\n\t\tfor {\n\t\t\tn, errno := v.Read(ctx, fe.Inode, resp, *fileOff, fh)\n\t\t\tif errno != 0 {\n\t\t\t\treturn nil, errno\n\t\t\t}\n\t\t\tif n == 0 {\n\t\t\t\ttime.Sleep(time.Millisecond * 200)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t*fileOff += uint64(n)\n\t\t\tfor {\n\t\t\t\tif n == 1 {\n\t\t\t\t\treturn nil, syscall.Errno(resp[off])\n\t\t\t\t} else if off+17 <= uint64(n) && resp[off] == meta.CPROGRESS {\n\t\t\t\t\toff += 17\n\t\t\t\t} else if off+5 < uint64(n) && resp[off] == meta.CDATA {\n\t\t\t\t\tsize := binary.BigEndian.Uint32(resp[off+1 : off+5])\n\t\t\t\t\tif off+5+uint64(size) > uint64(n) {\n\t\t\t\t\t\tlogger.Errorf(\"Bad response off %d n %d: %v\", off, n, resp)\n\t\t\t\t\t\treturn nil, syscall.EIO\n\t\t\t\t\t}\n\t\t\t\t\treturn resp[off+5 : off+5+uint64(size)], 0\n\t\t\t\t} else {\n\t\t\t\t\tlogger.Errorf(\"Bad response off %d n %d: %v\", off, n, resp)\n\t\t\t\t\treturn nil, syscall.EIO\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// rmr\n\tbuf = make([]byte, 4+4+8+1+4)\n\tw := utils.FromBuffer(buf)\n\tw.Put32(meta.Rmr)\n\tw.Put32(13)\n\tw.Put64(1)\n\tw.Put8(4)\n\tw.Put([]byte(\"file\"))\n\tif e := v.Write(ctx, fe.Inode, w.Bytes(), 0, fh); e != 0 {\n\t\tt.Fatalf(\"write info: %s\", e)\n\t}\n\tvar off uint64 = uint64(len(buf))\n\tresp := make([]byte, 1024*10)\n\tif n, e := readControl(resp, &off); e != 0 || n != 1 {\n\t\tt.Fatalf(\"read result: %s %d\", e, n)\n\t} else if resp[0] != byte(syscall.ENOENT) {\n\t\tt.Fatalf(\"rmr result: %s\", string(buf[:n]))\n\t} else {\n\t\toff += uint64(n)\n\t}\n\t// legacy info\n\tbuf = make([]byte, 4+4+8)\n\tw = utils.FromBuffer(buf)\n\tw.Put32(meta.LegacyInfo)\n\tw.Put32(8)\n\tw.Put64(1)\n\tif e := v.Write(ctx, fe.Inode, w.Bytes(), off, fh); e != 0 {\n\t\tt.Fatalf(\"write legacy info: %s\", e)\n\t}\n\toff += uint64(len(buf))\n\tbuf = make([]byte, 1024*10)\n\tif n, e = readControl(buf, &off); e != 0 {\n\t\tt.Fatalf(\"read result: %s %d\", e, n)\n\t} else if !strings.Contains(string(buf[:n]), \"dirs:\") {\n\t\tt.Fatalf(\"legacy info result: %s\", string(buf[:n]))\n\t} else {\n\t\toff += uint64(n)\n\t}\n\t// info v2\n\tbuf = make([]byte, 4+4+8)\n\tw = utils.FromBuffer(buf)\n\tw.Put32(meta.InfoV2)\n\tw.Put32(8)\n\tw.Put64(1)\n\tif e := v.Write(ctx, fe.Inode, w.Bytes(), off, fh); e != 0 {\n\t\tt.Fatalf(\"write info v2: %s\", e)\n\t}\n\toff += uint64(len(buf))\n\tbuf = make([]byte, 1024*10)\n\tdata, e := readData(buf, &off)\n\tif e != 0 {\n\t\tt.Fatalf(\"read progress bar: %s %d\", e, n)\n\t}\n\n\tvar infoResp InfoResponse\n\tif e := json.Unmarshal(data, &infoResp); e != nil {\n\t\tt.Fatalf(\"unmarshal info v2: %s\", e)\n\t}\n\tif infoResp.Failed && infoResp.Reason != \"\" {\n\t\tt.Fatalf(\"info v2 result: %s\", infoResp.Reason)\n\t}\n\n\t// fill\n\tbuf = make([]byte, 4+4+8+1+1+2+1)\n\tw = utils.FromBuffer(buf)\n\tw.Put32(meta.FillCache)\n\tw.Put32(13)\n\tw.Put64(1)\n\tw.Put8(1)\n\tw.Put([]byte(\"/\"))\n\tw.Put16(2)\n\tw.Put8(0)\n\tif e := v.Write(ctx, fe.Inode, w.Bytes()[:10], 0, fh); e != 0 {\n\t\tt.Fatalf(\"write fill 1: %s\", e)\n\t}\n\tif e := v.Write(ctx, fe.Inode, w.Bytes()[10:], 0, fh); e != 0 {\n\t\tt.Fatalf(\"write fill 2: %s\", e)\n\t}\n\toff += uint64(len(buf))\n\tresp = make([]byte, 1024*10)\n\n\tdata, _ = json.Marshal(CacheResponse{Locations: make(map[string]uint64)})\n\texpectSize := 1 + 4 + len(data)\n\tif n, e = readControl(resp, &off); e != 0 || n != expectSize {\n\t\tt.Fatalf(\"read result: %s %d %d\", e, n, expectSize)\n\t}\n\n\toff += uint64(n)\n\n\t// invalid msg\n\tbuf = make([]byte, 4+4+2)\n\tw = utils.FromBuffer(buf)\n\tw.Put32(meta.Rmr)\n\tw.Put32(0)\n\tif e := v.Write(ctx, fe.Inode, buf, off, fh); e != 0 {\n\t\tt.Fatalf(\"write info: %s\", e)\n\t}\n\toff += uint64(len(buf))\n\tresp = make([]byte, 1024)\n\tif n, e := v.Read(ctx, fe.Inode, resp, off, fh); e != 0 || n != 1 {\n\t\tt.Fatalf(\"read result: %s %d\", e, n)\n\t} else if resp[0] != uint8(syscall.EIO) {\n\t\tt.Fatalf(\"result: %s\", string(resp[:n]))\n\t}\n}\n\nfunc TestReaddirCache(t *testing.T) {\n\tengines := map[string]string{\n\t\t\"kv\": \"memkv://\",\n\t\t\"db\": \"sqlite3://:memory:\",\n\t\t\"redis\": \"redis://127.0.0.1:6379/2\",\n\t}\n\tfor typ, metaUri := range engines {\n\t\ttestReaddirCache(t, metaUri, typ, 20)\n\t\ttestReaddirCache(t, metaUri, typ, 4096)\n\t}\n}\n\nfunc testReaddirCache(t *testing.T, metaUri string, typ string, batchNum int) {\n\tv, _ := createTestVFS(nil, metaUri)\n\tctx := NewLogContext(meta.Background())\n\n\told := meta.DirBatchNum\n\tmeta.DirBatchNum[typ] = batchNum\n\tdefer func() {\n\t\tmeta.DirBatchNum = old\n\t}()\n\n\tentry, st := v.Mkdir(ctx, 1, \"testdir\", 0777, 022)\n\tif st != 0 {\n\t\tt.Fatalf(\"mkdir testdir: %s\", st)\n\t}\n\tparent := entry.Inode\n\tfor i := 0; i <= 100; i++ {\n\t\t_, _ = v.Mkdir(ctx, parent, fmt.Sprintf(\"d%03d\", i), 0777, 022)\n\t}\n\n\tdefer func() {\n\t\tfor i := 0; i <= 120; i++ {\n\t\t\t_ = v.Rmdir(ctx, parent, fmt.Sprintf(\"d%03d\", i))\n\t\t}\n\t\t_ = v.Rmdir(ctx, 1, \"testdir\")\n\t}()\n\n\tfh, _ := v.Opendir(ctx, parent, 0)\n\tdefer v.Releasedir(ctx, parent, fh)\n\tinitNum, num := 2, 20\n\tvar files = make(map[string]bool)\n\t// read first 20\n\tentries, _, _ := v.Readdir(ctx, parent, 20, initNum, fh, true)\n\tfor _, e := range entries[:num] {\n\t\tfiles[string(e.Name)] = true\n\t}\n\n\toff := num + initNum\n\t{\n\t\tentries, _, _ = v.Readdir(ctx, parent, 20, off, fh, true) // read next 20\n\t\tv.UpdateReaddirOffset(ctx, parent, fh, off+1) // but readdir buffer is too full to return all entries\n\t\tname := fmt.Sprintf(\"d%03d\", off+2)\n\t\t_ = v.Rmdir(ctx, parent, name)\n\t\tentries, _, _ = v.Readdir(ctx, parent, 20, off, fh, true) // should only get 19 entries\n\t\tfor _, e := range entries {\n\t\t\tif string(e.Name) == name {\n\t\t\t\tt.Fatalf(\"dir %s should be deleted\", name)\n\t\t\t}\n\t\t}\n\t}\n\tv.UpdateReaddirOffset(ctx, parent, fh, off)\n\tfor i := 0; i < 100; i += 10 {\n\t\tname := fmt.Sprintf(\"d%03d\", i)\n\t\t_ = v.Rmdir(ctx, parent, name)\n\t\tdelete(files, name)\n\t}\n\tfor i := 100; i < 110; i++ {\n\t\t_, _ = v.Mkdir(ctx, parent, fmt.Sprintf(\"d%03d\", i), 0777, 022)\n\t\t_ = v.Rename(ctx, parent, fmt.Sprintf(\"d%03d\", i), parent, fmt.Sprintf(\"d%03d\", i+10), 0)\n\t\tdelete(files, fmt.Sprintf(\"d%03d\", i))\n\t}\n\tfor {\n\t\tentries, _, _ := v.Readdir(ctx, parent, 20, off, fh, true)\n\t\tif len(entries) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tif len(entries) > 20 {\n\t\t\tentries = entries[:20]\n\t\t}\n\t\tfor _, e := range entries {\n\t\t\tif e.Inode > 0 {\n\t\t\t\tfiles[string(e.Name)] = true\n\t\t\t} else {\n\t\t\t\tt.Logf(\"invalid entry %s\", e.Name)\n\t\t\t}\n\t\t}\n\t\toff += len(entries)\n\t\tv.UpdateReaddirOffset(ctx, parent, fh, off)\n\t}\n\tfor i := 0; i < 100; i += 10 {\n\t\tname := fmt.Sprintf(\"d%03d\", i)\n\t\tif _, ok := files[name]; ok {\n\t\t\tt.Fatalf(\"dir %s should be deleted\", name)\n\t\t}\n\t}\n\tfor i := 100; i < 110; i++ {\n\t\tname := fmt.Sprintf(\"d%03d\", i)\n\t\tif _, ok := files[name]; ok {\n\t\t\tt.Fatalf(\"dir %s should be deleted\", name)\n\t\t}\n\t}\n\tfor i := 110; i < 120; i++ {\n\t\tname := fmt.Sprintf(\"d%03d\", i)\n\t\tif _, ok := files[name]; !ok {\n\t\t\tt.Fatalf(\"dir %s should be added\", name)\n\t\t}\n\t}\n}\n\nfunc TestVFSReadDirSort(t *testing.T) {\n\tfor _, metaUri := range []string{\"\", \"sqlite3://\", \"redis://127.0.0.1:6379/2\"} {\n\t\ttestVFSReadDirSort(t, metaUri)\n\t}\n}\n\nfunc testVFSReadDirSort(t *testing.T, metaUri string) {\n\tv, _ := createTestVFS(func(metaConfig *meta.Config) {\n\t\tmetaConfig.SortDir = true\n\t}, metaUri)\n\tctx := NewLogContext(meta.Background())\n\tentry, st := v.Mkdir(ctx, 1, \"testdir\", 0777, 022)\n\tif st != 0 {\n\t\tt.Fatalf(\"mkdir testdir: %s\", st)\n\t}\n\tparent := entry.Inode\n\tfor i := 0; i < 100; i++ {\n\t\t_, _ = v.Mkdir(ctx, parent, fmt.Sprintf(\"d%d\", i), 0777, 022)\n\t}\n\tdefer func() {\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_ = v.Rmdir(ctx, parent, fmt.Sprintf(\"d%d\", i))\n\t\t}\n\t\t_ = v.Rmdir(ctx, 1, \"testdir\")\n\t}()\n\tfh, _ := v.Opendir(ctx, parent, 0)\n\tentries1, _, _ := v.Readdir(ctx, parent, 60, 10, fh, true)\n\tsorted := slices.IsSortedFunc(entries1, func(i, j *meta.Entry) int {\n\t\treturn strings.Compare(string(i.Name), string(j.Name))\n\t})\n\tif !sorted {\n\t\tt.Fatalf(\"read dir result should sorted\")\n\t}\n\tv.Releasedir(ctx, parent, fh)\n\n\tfh2, _ := v.Opendir(ctx, parent, 0)\n\tentries2, _, _ := v.Readdir(ctx, parent, 60, 10, fh, true)\n\tfor i := 0; i < len(entries1); i++ {\n\t\tif string(entries1[i].Name) != string(entries2[i].Name) {\n\t\t\tt.Fatalf(\"read dir result should be same\")\n\t\t}\n\t}\n\tv.Releasedir(ctx, parent, fh2)\n}\n\nfunc testReaddirBatch(t *testing.T, metaUri string, typ string, batchNum int) {\n\tn, extra := 5, 40\n\n\tv, _ := createTestVFS(nil, metaUri)\n\tctx := NewLogContext(meta.Background())\n\n\told := meta.DirBatchNum\n\tmeta.DirBatchNum[typ] = batchNum\n\tdefer func() {\n\t\tmeta.DirBatchNum = old\n\t}()\n\n\tentry, st := v.Mkdir(ctx, 1, \"testdir\", 0777, 022)\n\tif st != 0 {\n\t\tt.Fatalf(\"mkdir testdir: %s\", st)\n\t}\n\n\tparent := entry.Inode\n\tfor i := 0; i < n*batchNum+extra; i++ {\n\t\t_, _ = v.Mkdir(ctx, parent, fmt.Sprintf(\"d%d\", i), 0777, 022)\n\t}\n\tdefer func() {\n\t\tfor i := 0; i < n*batchNum+extra; i++ {\n\t\t\t_ = v.Rmdir(ctx, parent, fmt.Sprintf(\"d%d\", i))\n\t\t}\n\t\tv.Rmdir(ctx, 1, \"testdir\")\n\t}()\n\n\tfh, _ := v.Opendir(ctx, parent, 0)\n\tdefer v.Releasedir(ctx, parent, fh)\n\tentries1, _, _ := v.Readdir(ctx, parent, 0, 0, fh, true)\n\trequire.NotNil(t, entries1)\n\trequire.Equal(t, 2+batchNum, len(entries1)) // init entries: \".\" and \"..\"\n\n\tentries2, _, _ := v.Readdir(ctx, parent, 0, 2, fh, true)\n\trequire.NotNil(t, entries2)\n\trequire.Equal(t, batchNum, len(entries2))\n\n\tentries3, _, _ := v.Readdir(ctx, parent, 0, 2+batchNum, fh, true)\n\trequire.NotNil(t, entries3)\n\trequire.Equal(t, batchNum, len(entries3))\n\n\t// reach the end\n\tentries4, _, _ := v.Readdir(ctx, parent, 0, n*batchNum+extra+2, fh, true)\n\trequire.NotNil(t, entries4)\n\trequire.Equal(t, 0, len(entries4))\n\n\t// skip-style readdir\n\tentries5, _, _ := v.Readdir(ctx, parent, 0, n*batchNum+2, fh, true)\n\trequire.NotNil(t, entries5)\n\trequire.Equal(t, extra, len(entries5))\n\n\tentries6, _, _ := v.Readdir(ctx, parent, 0, 2, fh, true)\n\trequire.Equal(t, len(entries2), len(entries6))\n\tfor i := 0; i < len(entries2); i++ {\n\t\trequire.Equal(t, entries2[i].Inode, entries6[i].Inode)\n\t}\n\n\t// dir seak\n\tentries7, _, _ := v.Readdir(ctx, parent, 0, n*batchNum+2-20, fh, true)\n\trequire.True(t, reflect.DeepEqual(entries5, entries7[20:]))\n}\n\nfunc TestReadDirBatch(t *testing.T) {\n\tengines := map[string]string{\n\t\t\"kv\": \"memkv://\",\n\t\t\"db\": \"sqlite3://:memory:\",\n\t\t\"redis\": \"redis://127.0.0.1:6379/2\",\n\t}\n\tfor typ, metaUri := range engines {\n\t\ttestReaddirBatch(t, metaUri, typ, 100)\n\t\t// testReaddirBatch(t, metaUri, typ, 4096)\n\t}\n}\n\nfunc TestReaddir(t *testing.T) {\n\tengines := map[string]string{\n\t\t\"kv\": \"memkv://\",\n\t\t\"db\": \"sqlite3://:memory:\",\n\t\t\"redis\": \"redis://127.0.0.1:6379/2\",\n\t}\n\tfor typ, metaUri := range engines {\n\t\tbatchNum := meta.DirBatchNum[typ]\n\t\textra := rand.Intn(batchNum)\n\t\ttestReaddir(t, metaUri, 20, 0)\n\t\ttestReaddir(t, metaUri, 20, 5)\n\t\ttestReaddir(t, metaUri, 2*batchNum, 0)\n\t\ttestReaddir(t, metaUri, 2*batchNum, extra)\n\t}\n}\n\nfunc testReaddir(t *testing.T, metaUri string, dirNum int, offset int) {\n\tv, _ := createTestVFS(nil, metaUri)\n\tctx := NewLogContext(meta.Background())\n\n\tentry, st := v.Mkdir(ctx, 1, \"testdir\", 0777, 022)\n\tif st != 0 {\n\t\tt.Fatalf(\"mkdir testdir: %s\", st)\n\t}\n\n\tparent := entry.Inode\n\tfor i := 0; i < dirNum; i++ {\n\t\t_, _ = v.Mkdir(ctx, parent, fmt.Sprintf(\"d%d\", i), 0777, 022)\n\t}\n\tdefer func() {\n\t\tfor i := 0; i < dirNum; i++ {\n\t\t\t_ = v.Rmdir(ctx, parent, fmt.Sprintf(\"d%d\", i))\n\t\t}\n\t\tv.Rmdir(ctx, 1, \"testdir\")\n\t}()\n\n\tfh, _ := v.Opendir(ctx, parent, 0)\n\tdefer v.Releasedir(ctx, parent, fh)\n\n\treadAll := func(ctx Context, parent Ino, fh uint64, off int) []*meta.Entry {\n\t\tvar entries []*meta.Entry\n\t\tfor {\n\t\t\tents, _, st := v.Readdir(ctx, parent, 0, off, fh, true)\n\t\t\trequire.Equal(t, st, syscall.Errno(0))\n\t\t\tif len(ents) == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\toff += len(ents)\n\t\t\tentries = append(entries, ents...)\n\t\t}\n\t\treturn entries\n\t}\n\n\tentriesOne := readAll(ctx, parent, fh, offset)\n\tentriesTwo := readAll(ctx, parent, fh, offset)\n\trequire.True(t, reflect.DeepEqual(entriesOne, entriesTwo))\n}\n" }, { "path": "pkg/vfs/vfs_unix.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\nconst O_ACCMODE = syscall.O_ACCMODE\nconst F_UNLCK = syscall.F_UNLCK\n\ntype Statfs struct {\n\tTotal uint64\n\tAvail uint64\n\tFiles uint64\n\tFavail uint64\n}\n\nfunc (v *VFS) StatFS(ctx Context, ino Ino) (st *Statfs, err syscall.Errno) {\n\tvar totalspace, availspace, iused, iavail uint64\n\t_ = v.Meta.StatFS(ctx, ino, &totalspace, &availspace, &iused, &iavail)\n\tst = new(Statfs)\n\tst.Total = totalspace\n\tst.Avail = availspace\n\tst.Files = iused + iavail\n\tst.Favail = iavail\n\tlogit(ctx, \"statfs\", err, \"(%d): (%d,%d,%d,%d)\", ino, totalspace-availspace, availspace, iused, iavail)\n\treturn\n}\n\nfunc accessTest(attr *Attr, mmode uint16, uid uint32, gid uint32) syscall.Errno {\n\tif uid == 0 {\n\t\treturn 0\n\t}\n\tmode := attr.Mode\n\tvar effected uint16\n\tif uid == attr.Uid {\n\t\teffected = (mode >> 6) & 7\n\t} else {\n\t\teffected = mode & 7\n\t\tif gid == attr.Gid {\n\t\t\teffected = (mode >> 3) & 7\n\t\t}\n\t}\n\tif mmode&effected != mmode {\n\t\treturn syscall.EACCES\n\t}\n\treturn 0\n}\n\nfunc (v *VFS) Access(ctx Context, ino Ino, mask int) (err syscall.Errno) {\n\tdefer func() { logit(ctx, \"access\", err, \"(%d,0x%X)\", ino, mask) }()\n\tvar mmask uint16\n\tif mask&unix.R_OK != 0 {\n\t\tmmask |= MODE_MASK_R\n\t}\n\tif mask&unix.W_OK != 0 {\n\t\tmmask |= MODE_MASK_W\n\t}\n\tif mask&unix.X_OK != 0 {\n\t\tmmask |= MODE_MASK_X\n\t}\n\tif IsSpecialNode(ino) {\n\t\tnode := getInternalNode(ino)\n\t\tif node != nil {\n\t\t\terr = accessTest(node.attr, mmask, ctx.Uid(), ctx.Gid())\n\t\t\treturn\n\t\t}\n\t}\n\n\terr = v.Meta.Access(ctx, ino, uint8(mmask), nil)\n\treturn\n}\n\nfunc setattrStr(set int, mode, uid, gid uint32, atime, mtime int64, size uint64) string {\n\tvar sb strings.Builder\n\tif set&meta.SetAttrMode != 0 {\n\t\tsb.WriteString(fmt.Sprintf(\"mode=%s:0%04o,\", smode(uint16(mode)), mode&07777))\n\t}\n\tif set&meta.SetAttrUID != 0 {\n\t\tsb.WriteString(fmt.Sprintf(\"uid=%d,\", uid))\n\t}\n\tif set&meta.SetAttrGID != 0 {\n\t\tsb.WriteString(fmt.Sprintf(\"gid=%d,\", gid))\n\t}\n\n\tvar atimeStr string\n\tif set&meta.SetAttrAtimeNow != 0 || (set&meta.SetAttrAtime) != 0 && atime < 0 {\n\t\tatimeStr = \"NOW\"\n\t} else if set&meta.SetAttrAtime != 0 {\n\t\tatimeStr = strconv.FormatInt(atime, 10)\n\t}\n\tif atimeStr != \"\" {\n\t\tsb.WriteString(\"atime=\" + atimeStr + \",\")\n\t}\n\n\tvar mtimeStr string\n\tif set&meta.SetAttrMtimeNow != 0 || (set&meta.SetAttrMtime) != 0 && mtime < 0 {\n\t\tmtimeStr = \"NOW\"\n\t} else if set&meta.SetAttrMtime != 0 {\n\t\tmtimeStr = strconv.FormatInt(mtime, 10)\n\t}\n\tif mtimeStr != \"\" {\n\t\tsb.WriteString(\"mtime=\" + mtimeStr + \",\")\n\t}\n\n\tif set&meta.SetAttrSize != 0 {\n\t\tsizeStr := strconv.FormatUint(size, 10)\n\t\tsb.WriteString(\"size=\" + sizeStr + \",\")\n\t}\n\tr := sb.String()\n\tif len(r) > 1 {\n\t\tr = r[:len(r)-1] // drop last ,\n\t}\n\treturn r\n}\n\nfunc (v *VFS) SetAttr(ctx Context, ino Ino, set int, fh uint64, mode, uid, gid uint32, atime, mtime int64, atimensec, mtimensec uint32, size uint64) (entry *meta.Entry, err syscall.Errno) {\n\tstr := setattrStr(set, mode, uid, gid, atime, mtime, size)\n\tdefer func() {\n\t\tlogit(ctx, \"setattr\", err, \"(%d[%d],0x%X,[%s]):%s\", ino, fh, set, str, (*Entry)(entry))\n\t}()\n\tif IsSpecialNode(ino) {\n\t\tn := getInternalNode(ino)\n\t\tif n != nil {\n\t\t\tentry = &meta.Entry{Inode: ino, Attr: n.attr}\n\t\t} else {\n\t\t\terr = syscall.EPERM\n\t\t}\n\t\treturn\n\t}\n\tvar attr = &Attr{}\n\tif set&meta.SetAttrSize != 0 {\n\t\terr = v.Truncate(ctx, ino, int64(size), fh, attr)\n\t\tif err != 0 {\n\t\t\treturn\n\t\t}\n\t\tif (set &^ (meta.SetAttrSize | meta.SetAttrCtime | meta.SetAttrCtimeNow)) == 0 {\n\t\t\tv.UpdateLength(ino, attr)\n\t\t\tentry = &meta.Entry{Inode: ino, Attr: attr}\n\t\t\treturn\n\t\t}\n\t}\n\tif set&meta.SetAttrMode != 0 {\n\t\tattr.Mode = uint16(mode & 07777)\n\t}\n\tif set&meta.SetAttrUID != 0 {\n\t\tattr.Uid = uid\n\t}\n\tif set&meta.SetAttrGID != 0 {\n\t\tattr.Gid = gid\n\t}\n\tif set&meta.SetAttrAtime != 0 {\n\t\tattr.Atime = atime\n\t\tattr.Atimensec = atimensec\n\t}\n\tif set&meta.SetAttrMtime != 0 {\n\t\tattr.Mtime = mtime\n\t\tattr.Mtimensec = mtimensec\n\t}\n\tif set&meta.SetAttrMtime != 0 || set&meta.SetAttrMtimeNow != 0 {\n\t\tif ctx.CheckPermission() {\n\t\t\tif err = v.Meta.CheckSetAttr(ctx, ino, uint16(set), *attr); err != 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tif set&meta.SetAttrMtime != 0 {\n\t\t\tv.writer.UpdateMtime(ino, time.Unix(mtime, int64(mtimensec)))\n\t\t}\n\t\tif set&meta.SetAttrMtimeNow != 0 {\n\t\t\tv.writer.UpdateMtime(ino, time.Now())\n\t\t}\n\t}\n\n\terr = v.Meta.SetAttr(ctx, ino, uint16(set), 0, attr)\n\tif err == 0 {\n\t\tv.UpdateLength(ino, attr)\n\t\tentry = &meta.Entry{Inode: ino, Attr: attr}\n\t}\n\treturn\n}\n\ntype lockType uint32\n\nfunc (l lockType) String() string {\n\tswitch l {\n\tcase syscall.F_UNLCK:\n\t\treturn \"U\"\n\tcase syscall.F_RDLCK:\n\t\treturn \"R\"\n\tcase syscall.F_WRLCK:\n\t\treturn \"W\"\n\tdefault:\n\t\treturn \"X\"\n\t}\n}\n\nfunc (v *VFS) Getlk(ctx Context, ino Ino, fh uint64, owner uint64, start, len *uint64, typ *uint32, pid *uint32) (err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"getlk\", err, \"(%d,%d,%016X): (%d,%d,%s,%d)\", ino, fh, owner, *start, *len, lockType(*typ), *pid)\n\t}()\n\tif lockType(*typ).String() == \"X\" {\n\t\treturn syscall.EINVAL\n\t}\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\tif v.findHandle(ino, fh) == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\terr = v.Meta.Getlk(ctx, ino, owner, typ, start, len, pid)\n\treturn\n}\n\nfunc (v *VFS) Setlk(ctx Context, ino Ino, fh uint64, owner uint64, start, end uint64, typ uint32, pid uint32, block bool) (err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"setlk\", err, \"(%d,%d,%016X,%d,%d,%s,%t,%d)\", ino, fh, owner, start, end, lockType(typ), block, pid)\n\t}()\n\tif lockType(typ).String() == \"X\" {\n\t\treturn syscall.EINVAL\n\t}\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\th.addOp(ctx)\n\tdefer h.removeOp(ctx)\n\n\terr = v.Meta.Setlk(ctx, ino, owner, block, typ, start, end, pid)\n\tif err == 0 {\n\t\th.Lock()\n\t\tif typ != syscall.F_UNLCK {\n\t\t\th.locks |= 2\n\t\t\tif h.ofdOwner == 0 {\n\t\t\t\th.ofdOwner = owner\n\t\t\t}\n\t\t}\n\t\th.Unlock()\n\t}\n\treturn\n}\n\nfunc (v *VFS) Flock(ctx Context, ino Ino, fh uint64, owner uint64, typ uint32, block bool) (err syscall.Errno) {\n\tvar name string\n\tdefer func() { logit(ctx, \"flock\", err, \"(%d,%d,%016X,%s,%t)\", ino, fh, owner, name, block) }()\n\tswitch typ {\n\tcase syscall.F_RDLCK:\n\t\tname = \"LOCKSH\"\n\tcase syscall.F_WRLCK:\n\t\tname = \"LOCKEX\"\n\tcase syscall.F_UNLCK:\n\t\tname = \"UNLOCK\"\n\tdefault:\n\t\terr = syscall.EINVAL\n\t\treturn\n\t}\n\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\th := v.findHandle(ino, fh)\n\tif h == nil {\n\t\terr = syscall.EBADF\n\t\treturn\n\t}\n\th.addOp(ctx)\n\tdefer h.removeOp(ctx)\n\terr = v.Meta.Flock(ctx, ino, owner^fh, typ, block)\n\tif err == 0 {\n\t\th.Lock()\n\t\tif typ == syscall.F_UNLCK {\n\t\t\th.locks &= 2\n\t\t} else {\n\t\t\th.locks |= 1\n\t\t\th.flockOwner = owner\n\t\t}\n\t\th.Unlock()\n\t}\n\treturn\n}\n\nfunc (v *VFS) Ioctl(ctx Context, ino Ino, cmd uint32, arg uint64, bufIn, bufOut []byte) (err syscall.Errno) {\n\tconst (\n\t\tFS_IOC_GETFLAGS = 0x80086601\n\t\tFS_IOC_SETFLAGS = 0x40086602\n\t\tFS_IOC_GETFLAGS_32 = 0x80046601\n\t\tFS_IOC_SETFLAGS_32 = 0x40046602\n\t\tFS_IOC_FSGETXATTR = 0x801C581F\n\t)\n\tconst (\n\t\tFS_SECRM_FL = 0x00000001\n\t\tFS_IMMUTABLE_FL = 0x00000010\n\t\tFS_APPEND_FL = 0x00000020\n\t\tFS_XFLAG_IMMUTABLE = 0x00000008\n\t\tFS_XFLAG_APPEND = 0x00000010\n\t)\n\tdefer func() { logit(ctx, \"ioctl\", err, \"(%d,0x%X,0x%X,%v,%v)\", ino, cmd, arg, bufIn, bufOut) }()\n\tswitch cmd {\n\tdefault:\n\t\treturn syscall.ENOTTY\n\tcase FS_IOC_SETFLAGS, FS_IOC_GETFLAGS, FS_IOC_SETFLAGS_32, FS_IOC_GETFLAGS_32, FS_IOC_FSGETXATTR:\n\t}\n\tif IsSpecialNode(ino) {\n\t\treturn syscall.EPERM\n\t}\n\tvar attr = &Attr{}\n\tif cmd>>30 == 1 { // set\n\t\tvar iflag uint64\n\t\tif len(bufIn) == 8 {\n\t\t\tiflag = utils.NativeEndian.Uint64(bufIn)\n\t\t} else if len(bufIn) == 4 {\n\t\t\tiflag = uint64(utils.NativeEndian.Uint32(bufIn))\n\t\t} else {\n\t\t\treturn syscall.EINVAL\n\t\t}\n\t\tif ctx.CheckPermission() && ctx.Uid() != 0 && iflag&(FS_SECRM_FL|FS_IMMUTABLE_FL|FS_APPEND_FL) != 0 {\n\t\t\treturn syscall.EPERM\n\t\t}\n\t\tif (iflag & FS_SECRM_FL) != 0 {\n\t\t\tattr.Flags |= meta.FlagSkipTrash\n\t\t}\n\t\tif (iflag & FS_IMMUTABLE_FL) != 0 {\n\t\t\tattr.Flags |= meta.FlagImmutable\n\t\t}\n\t\tif (iflag & FS_APPEND_FL) != 0 {\n\t\t\tattr.Flags |= meta.FlagAppend\n\t\t}\n\t\tif iflag &= ^uint64(FS_SECRM_FL | FS_IMMUTABLE_FL | FS_APPEND_FL); iflag != 0 {\n\t\t\treturn syscall.ENOTSUP\n\t\t}\n\t\treturn v.Meta.SetAttr(ctx, ino, meta.SetAttrFlag, 0, attr)\n\t} else {\n\t\tif err = v.Meta.GetAttr(ctx, ino, attr); err != 0 {\n\t\t\treturn\n\t\t}\n\t\tvar iflag uint64\n\t\tif cmd>>8&0xFF == 'f' { // FS_IOC_GETFLAGS\n\t\t\tif (attr.Flags & meta.FlagSkipTrash) != 0 {\n\t\t\t\tiflag |= FS_SECRM_FL\n\t\t\t}\n\t\t\tif (attr.Flags & meta.FlagImmutable) != 0 {\n\t\t\t\tiflag |= FS_IMMUTABLE_FL\n\t\t\t}\n\t\t\tif (attr.Flags & meta.FlagAppend) != 0 {\n\t\t\t\tiflag |= FS_APPEND_FL\n\t\t\t}\n\t\t\tif len(bufOut) == 8 {\n\t\t\t\tutils.NativeEndian.PutUint64(bufOut, iflag)\n\t\t\t} else if len(bufOut) == 4 {\n\t\t\t\tutils.NativeEndian.PutUint32(bufOut, uint32(iflag))\n\t\t\t} else {\n\t\t\t\treturn syscall.EINVAL\n\t\t\t}\n\t\t} else { // 'X', FS_IOC_FSGETXATTR\n\t\t\tif (attr.Flags & meta.FlagImmutable) != 0 {\n\t\t\t\tiflag |= FS_XFLAG_IMMUTABLE\n\t\t\t}\n\t\t\tif (attr.Flags & meta.FlagAppend) != 0 {\n\t\t\t\tiflag |= FS_XFLAG_APPEND\n\t\t\t}\n\t\t\tif len(bufOut) == 28 {\n\t\t\t\tutils.NativeEndian.PutUint32(bufOut[:4], uint32(iflag))\n\t\t\t\tfor i := range bufOut[4:] {\n\t\t\t\t\tbufOut[4+i] = 0\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn syscall.EINVAL\n\t\t\t}\n\t\t}\n\t\treturn\n\t}\n}\n" }, { "path": "pkg/vfs/vfs_windows.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"syscall\"\n\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/winfsp/cgofuse/fuse\"\n)\n\nconst O_ACCMODE = uint32(fuse.O_ACCMODE)\nconst F_UNLCK = 0x01\n\nfunc (v *VFS) ChFlags(ctx Context, ino Ino, flags uint8) (err syscall.Errno) {\n\tdefer func() {\n\t\tlogit(ctx, \"chflags\", err, \"(%d):%d\", ino, flags)\n\t}()\n\tif IsSpecialNode(ino) {\n\t\terr = syscall.EPERM\n\t\treturn\n\t}\n\n\terr = syscall.EINVAL\n\tvar attr = &Attr{Flags: flags}\n\n\tif ctx.CheckPermission() {\n\t\tif err = v.Meta.CheckSetAttr(ctx, ino, meta.SetAttrFlag, *attr); err != 0 {\n\t\t\treturn\n\t\t}\n\t}\n\n\terr = v.Meta.SetAttr(ctx, ino, meta.SetAttrFlag, 0, attr)\n\treturn\n}\n" }, { "path": "pkg/vfs/writer.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage vfs\n\nimport (\n\t\"math/rand\"\n\t\"runtime\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nconst (\n\tflushDuration = time.Second * 5\n)\n\ntype FileWriter interface {\n\tWrite(ctx meta.Context, offset uint64, data []byte) syscall.Errno\n\tFlush(ctx meta.Context) syscall.Errno\n\tClose(ctx meta.Context) syscall.Errno\n\tGetLength() uint64\n\tTruncate(length uint64)\n}\n\ntype DataWriter interface {\n\tOpen(inode Ino, fleng uint64) FileWriter\n\tFlush(ctx meta.Context, inode Ino) syscall.Errno\n\tGetLength(inode Ino) uint64\n\tTruncate(inode Ino, length uint64)\n\tUpdateMtime(inode Ino, mtime time.Time)\n\tFlushAll() error\n}\n\ntype sliceWriter struct {\n\tid uint64\n\tchunk *chunkWriter\n\toff uint32\n\tlength uint32\n\tsoff uint32\n\tslen uint32\n\twriter chunk.Writer\n\tfreezed bool\n\tdone bool\n\terr syscall.Errno\n\tnotify *utils.Cond\n\tstarted time.Time\n\tlastMod time.Time\n}\n\nfunc (s *sliceWriter) prepareID(ctx meta.Context, retry bool) {\n\tf := s.chunk.file\n\tf.Lock()\n\tfor s.id == 0 {\n\t\tvar id uint64\n\t\tf.Unlock()\n\t\tst := f.w.m.NewSlice(ctx, &id)\n\t\tf.Lock()\n\t\tif st != 0 && st != syscall.EIO {\n\t\t\ts.err = st\n\t\t\tbreak\n\t\t}\n\t\tif !retry || st == 0 {\n\t\t\tif s.id == 0 {\n\t\t\t\ts.id = id\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tf.Unlock()\n\t\tlogger.Debugf(\"meta is not available: %s\", st)\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\tf.Lock()\n\t}\n\tif s.writer != nil && s.writer.ID() == 0 {\n\t\ts.writer.SetID(s.id)\n\t}\n\tf.Unlock()\n}\n\nfunc (s *sliceWriter) markDone() {\n\tf := s.chunk.file\n\tf.Lock()\n\ts.done = true\n\ts.notify.Signal()\n\tf.Unlock()\n}\n\n// freezed, no more data\nfunc (s *sliceWriter) flushData() {\n\tdefer s.markDone()\n\tif s.slen == 0 {\n\t\treturn\n\t}\n\ts.prepareID(meta.Background(), true)\n\tif s.err != 0 {\n\t\tlogger.Infof(\"flush inode: %v chunk: %d err: %s\", s.chunk.file.inode, s.id, s.err)\n\t\ts.writer.Abort()\n\t\treturn\n\t}\n\ts.length = s.slen\n\tif err := s.writer.Finish(int(s.length)); err != nil {\n\t\tlogger.Errorf(\"upload inode: %v chunk: %v (length: %v) fail: %s\", s.chunk.file.inode, s.id, s.length, err)\n\n\t\ts.writer.Abort()\n\t\ts.err = syscall.EIO\n\t}\n}\n\n// protected by s.chunk.file\nfunc (s *sliceWriter) write(ctx meta.Context, off uint32, data []uint8) syscall.Errno {\n\tf := s.chunk.file\n\t_, err := s.writer.WriteAt(data, int64(off))\n\tif err != nil {\n\t\tlogger.Warnf(\"write inode: %v chunk: %d off: %d %s\", s.chunk.file.inode, s.id, off, err)\n\t\treturn syscall.EIO\n\t}\n\tif off+uint32(len(data)) > s.slen {\n\t\ts.slen = off + uint32(len(data))\n\t}\n\ts.lastMod = time.Now()\n\tif s.slen == meta.ChunkSize {\n\t\ts.freezed = true\n\t\tgo s.flushData()\n\t} else if int(s.slen) >= f.w.blockSize {\n\t\tif s.id > 0 {\n\t\t\terr := s.writer.FlushTo(int(s.slen))\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"write inode: %v chunk: %d off: %d %s\", s.chunk.file.inode, s.id, off, err)\n\t\t\t\treturn syscall.EIO\n\t\t\t}\n\t\t}\n\t}\n\treturn 0\n}\n\ntype chunkWriter struct {\n\tindx uint32\n\tfile *fileWriter\n\tslices []*sliceWriter\n}\n\n// protected by file\nfunc (c *chunkWriter) findWritableSlice(pos uint32, size uint32) *sliceWriter {\n\tblockSize := uint32(c.file.w.blockSize)\n\tfor i := range c.slices {\n\t\ts := c.slices[len(c.slices)-1-i]\n\t\tif !s.freezed {\n\t\t\tflushoff := s.slen / blockSize * blockSize\n\t\t\tif pos >= s.off+flushoff && pos <= s.off+s.slen {\n\t\t\t\treturn s\n\t\t\t} else if i > 3 {\n\t\t\t\ts.freezed = true\n\t\t\t\tgo s.flushData()\n\t\t\t}\n\t\t}\n\t\tif pos < s.off+s.slen && s.off < pos+size {\n\t\t\t// overlaped\n\t\t\t// TODO: write into multiple slices\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *chunkWriter) commitThread() {\n\tf := c.file\n\tdefer f.w.free(f)\n\tf.Lock()\n\n\t// the slices should be committed in the order that are created\n\tfor len(c.slices) > 0 {\n\t\ts := c.slices[0]\n\t\tfor !s.done {\n\t\t\tif s.notify.WaitWithTimeout(time.Millisecond*100) && !s.freezed && time.Since(s.started) > flushDuration*2 {\n\t\t\t\ts.freezed = true\n\t\t\t\tgo s.flushData()\n\t\t\t}\n\t\t}\n\t\terr := s.err\n\t\tf.Unlock()\n\n\t\tif err == 0 {\n\t\t\tvar ss = meta.Slice{Id: s.id, Size: s.length, Off: s.soff, Len: s.slen}\n\t\t\terr = f.w.m.Write(meta.Background(), f.inode, c.indx, s.off, ss, s.lastMod)\n\t\t\tf.w.reader.Invalidate(f.inode, uint64(c.indx)*meta.ChunkSize+uint64(s.off), uint64(ss.Len))\n\t\t}\n\n\t\tf.Lock()\n\t\tif err != 0 {\n\t\t\tif err == syscall.ENOENT || err == syscall.ENOSPC || err == syscall.EDQUOT {\n\t\t\t\tgo func(id uint64, length int) {\n\t\t\t\t\t_ = f.w.store.Remove(id, length)\n\t\t\t\t}(s.id, int(s.length))\n\t\t\t} else {\n\t\t\t\tlogger.Warnf(\"write inode:%d error: %s\", f.inode, err)\n\t\t\t\terr = syscall.EIO\n\t\t\t}\n\t\t\tf.err = err\n\t\t\tlogger.Errorf(\"write inode:%d indx:%d %s\", f.inode, c.indx, err)\n\t\t}\n\t\tc.slices = c.slices[1:]\n\t}\n\tf.freeChunk(c)\n\tf.Unlock()\n}\n\ntype fileWriter struct {\n\tsync.Mutex\n\tw *dataWriter\n\n\tinode Ino\n\tlength uint64\n\terr syscall.Errno\n\tflushwaiting uint16\n\twritewaiting uint16\n\trefs uint16\n\tchunks map[uint32]*chunkWriter\n\n\tflushcond *utils.Cond // wait for chunks==nil (flush)\n\twritecond *utils.Cond // wait for flushwaiting==0 (write)\n}\n\n// protected by file\nfunc (f *fileWriter) findChunk(i uint32) *chunkWriter {\n\tc := f.chunks[i]\n\tif c == nil {\n\t\tc = &chunkWriter{indx: i, file: f}\n\t\tf.chunks[i] = c\n\t}\n\treturn c\n}\n\n// protected by file\nfunc (f *fileWriter) freeChunk(c *chunkWriter) {\n\tdelete(f.chunks, c.indx)\n\tif len(f.chunks) == 0 && f.flushwaiting > 0 {\n\t\tf.flushcond.Broadcast()\n\t}\n}\n\n// protected by file\nfunc (f *fileWriter) writeChunk(ctx meta.Context, indx uint32, off uint32, data []byte) syscall.Errno {\n\tc := f.findChunk(indx)\n\ts := c.findWritableSlice(off, uint32(len(data)))\n\tif s == nil {\n\t\ts = &sliceWriter{\n\t\t\tchunk: c,\n\t\t\toff: off,\n\t\t\twriter: f.w.store.NewWriter(0),\n\t\t\tnotify: utils.NewCond(&f.Mutex),\n\t\t\tstarted: time.Now(),\n\t\t}\n\t\tgo s.prepareID(meta.Background(), false)\n\t\tc.slices = append(c.slices, s)\n\t\tif len(c.slices) == 1 {\n\t\t\tf.w.Lock()\n\t\t\tf.refs++\n\t\t\tf.w.Unlock()\n\t\t\tgo c.commitThread()\n\t\t}\n\t}\n\treturn s.write(ctx, off-s.off, data)\n}\n\nfunc (f *fileWriter) totalSlices() int {\n\tvar cnt int\n\tf.Lock()\n\tfor _, c := range f.chunks {\n\t\tcnt += len(c.slices)\n\t}\n\tf.Unlock()\n\treturn cnt\n}\n\nfunc (w *dataWriter) usedBufferSize() int64 {\n\treturn utils.AllocMemory() - w.store.UsedMemory()\n}\n\nfunc (f *fileWriter) Write(ctx meta.Context, off uint64, data []byte) syscall.Errno {\n\tfor {\n\t\tif f.totalSlices() < 1000 {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(time.Millisecond)\n\t}\n\tif f.w.usedBufferSize() > f.w.bufferSize {\n\t\t// slow down\n\t\ttime.Sleep(time.Millisecond * 10)\n\t\tfor f.w.usedBufferSize() > f.w.bufferSize*2 {\n\t\t\ttime.Sleep(time.Millisecond * 100)\n\t\t}\n\t}\n\n\ts := time.Now()\n\tf.Lock()\n\tdefer f.Unlock()\n\tsize := uint64(len(data))\n\tf.writewaiting++\n\tfor f.flushwaiting > 0 {\n\t\tif f.writecond.WaitWithTimeout(time.Second) && ctx.Canceled() {\n\t\t\tf.writewaiting--\n\t\t\tlogger.Warnf(\"write %d interrupted after %d\", f.inode, time.Since(s))\n\t\t\treturn syscall.EINTR\n\t\t}\n\t}\n\tf.writewaiting--\n\n\tindx := uint32(off / meta.ChunkSize)\n\tpos := uint32(off % meta.ChunkSize)\n\tfor len(data) > 0 {\n\t\tn := uint32(len(data))\n\t\tif pos+n > meta.ChunkSize {\n\t\t\tn = meta.ChunkSize - pos\n\t\t}\n\t\tif st := f.writeChunk(ctx, indx, pos, data[:n]); st != 0 {\n\t\t\treturn st\n\t\t}\n\t\tdata = data[n:]\n\t\tindx++\n\t\tpos = (pos + n) % meta.ChunkSize\n\t}\n\tif off+size > f.length {\n\t\tf.length = off + size\n\t}\n\treturn f.err\n}\n\nfunc (f *fileWriter) updateMtime(t time.Time) {\n\tf.Lock()\n\tdefer f.Unlock()\n\tfor _, c := range f.chunks {\n\t\tfor _, s := range c.slices {\n\t\t\ts.lastMod = t\n\t\t}\n\t}\n}\n\nfunc (f *fileWriter) flush(ctx meta.Context, writeback bool) syscall.Errno {\n\ts := time.Now()\n\tf.Lock()\n\tdefer f.Unlock()\n\tf.flushwaiting++\n\n\tvar err syscall.Errno\n\tvar wait = time.Second * time.Duration((f.w.maxRetries+2)*(f.w.maxRetries+2)/2)\n\tif wait < time.Minute*5 {\n\t\twait = time.Minute * 5\n\t}\n\tvar deadline = time.Now().Add(wait)\n\tfor len(f.chunks) > 0 && err == 0 {\n\t\tfor _, c := range f.chunks {\n\t\t\tfor _, s := range c.slices {\n\t\t\t\tif !s.freezed {\n\t\t\t\t\ts.freezed = true\n\t\t\t\t\tgo s.flushData()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif f.flushcond.WaitWithTimeout(time.Second*3) && ctx.Canceled() && time.Since(s) > f.w.conf.Chunk.PutTimeout*2 {\n\t\t\tlogger.Warnf(\"flush %d interrupted after %d\", f.inode, time.Since(s))\n\t\t\terr = syscall.EINTR\n\t\t\tbreak\n\t\t}\n\t\tif time.Now().After(deadline) {\n\t\t\tlogger.Errorf(\"flush %d timeout after waited %s\", f.inode, wait)\n\t\t\tfor _, c := range f.chunks {\n\t\t\t\tfor _, s := range c.slices {\n\t\t\t\t\tlogger.Errorf(\"pending slice %d-%d: %+v\", f.inode, c.indx, *s)\n\t\t\t\t}\n\t\t\t}\n\t\t\tbuf := make([]byte, 1<<20)\n\t\t\tn := runtime.Stack(buf, true)\n\t\t\tlogger.Warnf(\"All goroutines (%d):\\n%s\", runtime.NumGoroutine(), buf[:n])\n\t\t\terr = syscall.EIO\n\t\t\tbreak\n\t\t}\n\t}\n\tf.flushwaiting--\n\tif f.flushwaiting == 0 && f.writewaiting > 0 {\n\t\tf.writecond.Broadcast()\n\t}\n\tif err == 0 {\n\t\terr = f.err\n\t}\n\treturn err\n}\n\nfunc (f *fileWriter) Flush(ctx meta.Context) syscall.Errno {\n\treturn f.flush(ctx, false)\n}\n\nfunc (f *fileWriter) Close(ctx meta.Context) syscall.Errno {\n\tdefer f.w.free(f)\n\treturn f.Flush(ctx)\n}\n\nfunc (f *fileWriter) GetLength() uint64 {\n\tf.Lock()\n\tdefer f.Unlock()\n\treturn f.length\n}\n\nfunc (f *fileWriter) Truncate(length uint64) {\n\tf.Lock()\n\tdefer f.Unlock()\n\t// TODO: truncate write buffer if length < f.length\n\tf.length = length\n}\n\ntype dataWriter struct {\n\tsync.Mutex\n\tm meta.Meta\n\tstore chunk.ChunkStore\n\tconf *Config\n\treader DataReader\n\tblockSize int\n\tbufferSize int64\n\tfiles map[Ino]*fileWriter\n\tmaxRetries uint32\n}\n\nfunc NewDataWriter(conf *Config, m meta.Meta, store chunk.ChunkStore, reader DataReader) DataWriter {\n\tw := &dataWriter{\n\t\tm: m,\n\t\tstore: store,\n\t\treader: reader,\n\t\tconf: conf,\n\t\tblockSize: conf.Chunk.BlockSize,\n\t\tbufferSize: int64(conf.Chunk.BufferSize),\n\t\tfiles: make(map[Ino]*fileWriter),\n\t\tmaxRetries: uint32(conf.Meta.Retries),\n\t}\n\tgo w.flushAll()\n\treturn w\n}\n\nfunc (w *dataWriter) flushAll() {\n\tfor {\n\t\tw.Lock()\n\t\tnow := time.Now()\n\t\tfor _, f := range w.files {\n\t\t\tf.refs++\n\t\t\tw.Unlock()\n\t\t\ttooMany := f.totalSlices() > 800\n\t\t\tf.Lock()\n\n\t\t\tlastBit := uint32(rand.Int() % 2) // choose half of chunks randomly\n\t\t\tfor i, c := range f.chunks {\n\t\t\t\ths := len(c.slices) / 2\n\t\t\t\tfor j, s := range c.slices {\n\t\t\t\t\tif !s.freezed && (now.Sub(s.started) > flushDuration || now.Sub(s.lastMod) > time.Second && now.Sub(s.started) > time.Second ||\n\t\t\t\t\t\ttooMany && i%2 == lastBit && j <= hs) {\n\t\t\t\t\t\ts.freezed = true\n\t\t\t\t\t\tgo s.flushData()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tf.Unlock()\n\t\t\tw.free(f)\n\t\t\tw.Lock()\n\t\t}\n\t\tw.Unlock()\n\t\ttime.Sleep(time.Millisecond * 100)\n\t}\n}\n\nfunc (w *dataWriter) Open(inode Ino, len uint64) FileWriter {\n\tw.Lock()\n\tdefer w.Unlock()\n\tf, ok := w.files[inode]\n\tif !ok {\n\t\tf = &fileWriter{\n\t\t\tw: w,\n\t\t\tinode: inode,\n\t\t\tlength: len,\n\t\t\tchunks: make(map[uint32]*chunkWriter),\n\t\t}\n\t\tf.flushcond = utils.NewCond(f)\n\t\tf.writecond = utils.NewCond(f)\n\t\tw.files[inode] = f\n\t}\n\tf.refs++\n\treturn f\n}\n\nfunc (w *dataWriter) find(inode Ino) *fileWriter {\n\tw.Lock()\n\tdefer w.Unlock()\n\treturn w.files[inode]\n}\n\nfunc (w *dataWriter) free(f *fileWriter) {\n\tw.Lock()\n\tdefer w.Unlock()\n\tf.refs--\n\tif f.refs == 0 {\n\t\tdelete(w.files, f.inode)\n\t}\n}\n\nfunc (w *dataWriter) Flush(ctx meta.Context, inode Ino) syscall.Errno {\n\tf := w.find(inode)\n\tif f != nil {\n\t\treturn f.Flush(ctx)\n\t}\n\treturn 0\n}\n\nfunc (w *dataWriter) GetLength(inode Ino) uint64 {\n\tf := w.find(inode)\n\tif f != nil {\n\t\treturn f.GetLength()\n\t}\n\treturn 0\n}\n\nfunc (w *dataWriter) Truncate(inode Ino, len uint64) {\n\tf := w.find(inode)\n\tif f != nil {\n\t\tf.Truncate(len)\n\t}\n}\n\nfunc (w *dataWriter) UpdateMtime(inode Ino, mtime time.Time) {\n\tf := w.find(inode)\n\tif f != nil {\n\t\tf.updateMtime(mtime)\n\t}\n}\n\nfunc (w *dataWriter) FlushAll() error {\n\tvar err error\n\tw.Lock()\n\tfor inode, ind := range w.files {\n\t\tind.refs++\n\t\tw.Unlock()\n\t\teno := ind.Flush(meta.Background())\n\t\tw.free(ind)\n\t\tif eno != 0 {\n\t\t\tlogger.Errorf(\"flush %s: %s\", inode, eno)\n\t\t\treturn eno\n\t\t}\n\t\tlogger.Debugf(\"Flush %d\", inode)\n\t\tw.Lock()\n\t}\n\tw.Unlock()\n\treturn err\n}\n" }, { "path": "pkg/win/ldap.go", "content": "//go:build windows\n// +build windows\n\n/*\n * JuiceFS, Copyright 2026 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage win\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"unsafe\"\n\n\t\"golang.org/x/sys/windows\"\n)\n\nvar (\n\tmodWldap32 = windows.NewLazySystemDLL(\"wldap32.dll\")\n\tprocLdapInitW = modWldap32.NewProc(\"ldap_initW\")\n\tprocLdapSetOptionW = modWldap32.NewProc(\"ldap_set_optionW\")\n\tprocLdapBindSW = modWldap32.NewProc(\"ldap_bind_sW\")\n\tprocLdapUnbind = modWldap32.NewProc(\"ldap_unbind\")\n\tprocLdapSearchSW = modWldap32.NewProc(\"ldap_search_sW\")\n\tprocLdapFirstEntryW = modWldap32.NewProc(\"ldap_first_entryW\")\n\tprocLdapGetValuesW = modWldap32.NewProc(\"ldap_get_valuesW\")\n\tprocLdapCountValuesW = modWldap32.NewProc(\"ldap_count_valuesW\")\n\tprocLdapValueFreeW = modWldap32.NewProc(\"ldap_value_freeW\")\n\tprocLdapMsgFreeW = modWldap32.NewProc(\"ldap_msgfreeW\")\n)\n\n// from winldap.h\nconst (\n\tLDAP_PORT = 389\n\tLDAP_SUCCESS = 0\n\tLDAP_OPT_SIGN = 0x95\n\tLDAP_OPT_ENCRYPT = 0x96\n\tLDAP_OPT_ON = 1\n\tLDAP_SCOPE_BASE = 0x00\n\tLDAP_SCOPE_ONELEVEL = 0x01\n\tLDAP_AUTH_NEGOTIATE = 0x0486 // LDAP_AUTH_OTHERKIND (0x86) | 0x0400\n)\n\nfunc LdapConnect(host string) (uintptr, error) {\n\thostPtr, err := windows.UTF16PtrFromString(host)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\thandle, _, _ := procLdapInitW.Call(\n\t\tuintptr(unsafe.Pointer(hostPtr)),\n\t\tuintptr(LDAP_PORT),\n\t)\n\tif handle == 0 {\n\t\treturn 0, fmt.Errorf(\"ldap_initW failed\")\n\t}\n\tprocLdapSetOptionW.Call(handle, uintptr(LDAP_OPT_SIGN), uintptr(LDAP_OPT_ON))\n\tprocLdapSetOptionW.Call(handle, uintptr(LDAP_OPT_ENCRYPT), uintptr(LDAP_OPT_ON))\n\n\tr1, _, _ := procLdapBindSW.Call(handle, 0, 0, uintptr(LDAP_AUTH_NEGOTIATE))\n\tif int32(r1) != LDAP_SUCCESS {\n\t\tprocLdapUnbind.Call(handle)\n\t\treturn 0, fmt.Errorf(\"ldap_bind_sW failed: %d\", r1)\n\t}\n\treturn handle, nil\n}\n\nfunc LdapClose(handle uintptr) {\n\tprocLdapUnbind.Call(handle)\n}\n\nfunc LdapGetValue(\n\thandle uintptr,\n\tbase string,\n\tscope uint32,\n\tfilter string,\n\tattribute string,\n) (string, error) {\n\tvar basePtr *uint16\n\tif base != \"\" {\n\t\tp, err := windows.UTF16PtrFromString(base)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tbasePtr = p\n\t}\n\tfilterPtr, err := windows.UTF16PtrFromString(filter)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tattrPtr, err := windows.UTF16PtrFromString(attribute)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tattrs := []uintptr{uintptr(unsafe.Pointer(attrPtr)), 0}\n\n\tvar msg uintptr\n\tr1, _, _ := procLdapSearchSW.Call(\n\t\thandle,\n\t\tuintptr(unsafe.Pointer(basePtr)),\n\t\tuintptr(scope),\n\t\tuintptr(unsafe.Pointer(filterPtr)),\n\t\tuintptr(unsafe.Pointer(&attrs[0])),\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&msg)),\n\t)\n\tif int32(r1) != LDAP_SUCCESS {\n\t\treturn \"\", fmt.Errorf(\"ldap_search_sW failed: %d\", r1)\n\t}\n\tdefer procLdapMsgFreeW.Call(msg)\n\n\tentry, _, _ := procLdapFirstEntryW.Call(handle, msg)\n\tif entry == 0 {\n\t\treturn \"\", fmt.Errorf(\"no entries found\")\n\t}\n\tvals, _, _ := procLdapGetValuesW.Call(handle, entry, uintptr(unsafe.Pointer(attrPtr)))\n\tif vals == 0 {\n\t\treturn \"\", fmt.Errorf(\"no attribute values\")\n\t}\n\tdefer procLdapValueFreeW.Call(vals)\n\tcnt, _, _ := procLdapCountValuesW.Call(vals)\n\tif cnt == 0 {\n\t\treturn \"\", fmt.Errorf(\"no attribute values\")\n\t}\n\tfirstPtr := *(*uintptr)(unsafe.Pointer(vals))\n\tvalue := windows.UTF16PtrToString((*uint16)(unsafe.Pointer(firstPtr)))\n\treturn value, nil\n}\n\nfunc LdapGetDefaultNamingContext(handle uintptr) (string, error) {\n\treturn LdapGetValue(handle, \"\", LDAP_SCOPE_BASE, \"(objectClass=*)\", \"defaultNamingContext\")\n}\n\nfunc LdapGetTrustPosixOffset(\n\thandle uintptr,\n\tcontext string,\n\tdomain string,\n) (string, error) {\n\tisFlat := !strings.Contains(domain, \".\")\n\tbase := fmt.Sprintf(\"CN=System,%s\", context)\n\tvar filter string\n\tif isFlat {\n\t\tfilter = fmt.Sprintf(\"(&(objectClass=trustedDomain)(flatName=%s))\", domain)\n\t} else {\n\t\tfilter = fmt.Sprintf(\"(&(objectClass=trustedDomain)(name=%s))\", domain)\n\t}\n\treturn LdapGetValue(handle, base, LDAP_SCOPE_ONELEVEL, filter, \"trustPosixOffset\")\n}\n" }, { "path": "pkg/win/sid.go", "content": "//go:build windows\n// +build windows\n\n/*\n * JuiceFS, Copyright 2026 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage win\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"syscall\"\n\t\"unsafe\"\n\n\t\"golang.org/x/sys/windows\"\n)\n\nvar (\n\tmodadvapi32 = windows.NewLazySystemDLL(\"advapi32.dll\")\n\tprocLsaOpenPolicy = modadvapi32.NewProc(\"LsaOpenPolicy\")\n\tprocLsaQueryInformationPolicy = modadvapi32.NewProc(\"LsaQueryInformationPolicy\")\n\tprocLsaFreeMemory = modadvapi32.NewProc(\"LsaFreeMemory\")\n\tprocLsaClose = modadvapi32.NewProc(\"LsaClose\")\n\n\tnetapi32 = windows.NewLazySystemDLL(\"netapi32.dll\")\n\n\t//https://learn.microsoft.com/en-us/windows/win32/api/dsgetdc/nf-dsgetdc-dsenumeratedomaintrustsw\n\tprocDsEnumerateDomainTrustsW = netapi32.NewProc(\"DsEnumerateDomainTrustsW\")\n\tprocNetApiBufferFree = netapi32.NewProc(\"NetApiBufferFree\")\n)\n\nvar trustedDomains []trustedDomain\n\ntype LSA_OBJECT_ATTRIBUTES struct {\n\tLength uint32\n\tRootDirectory windows.Handle\n\tObjectName uintptr\n\tAttributes uint32\n\tSecurityDescriptor uintptr\n\tSecurityQualityOfService uintptr\n}\n\nvar primaryDomainSid *windows.SID = nil\nvar accountDomainSid *windows.SID = nil\n\nconst (\n\tPolicyAccountDomainInformation = 5\n\tPolicyDnsDomainInformation = 12\n)\n\nconst (\n\tPOLICY_VIEW_LOCAL_INFORMATION = 0x00000001\n\tPOLICY_VIEW_AUDIT_INFORMATION = 0x00000002\n\tPOLICY_GET_PRIVATE_INFORMATION = 0x00000004\n\tPOLICY_TRUST_ADMIN = 0x00000008\n\tPOLICY_CREATE_ACCOUNT = 0x00000010\n\tPOLICY_CREATE_SECRET = 0x00000020\n\tPOLICY_CREATE_PRIVILEGE = 0x00000040\n\tPOLICY_SET_DEFAULT_QUOTA_LIMITS = 0x00000080\n\tPOLICY_SET_AUDIT_REQUIREMENTS = 0x00000100\n\tPOLICY_AUDIT_LOG_ADMIN = 0x00000200\n\tPOLICY_SERVER_ADMIN = 0x00000400\n\tPOLICY_LOOKUP_NAMES = 0x00000800\n\tPOLICY_NOTIFICATION = 0x00001000\n)\n\nconst (\n\tAdministratorUIDFromFUSE = 197108 // This is calcuated from the SID of Administrator user on Windows. //0x30000 + 500\n\tAdminstratorsGIDFromFUSE = 544 // S-1-5-32-544\n\tSystemUIDFromFUSE = 18 // S-1-5-32-18\n)\n\ntype UNICODE_STRING struct {\n\tLength uint16\n\tMaximumLength uint16\n\tBuffer *uint16\n}\n\n// https://learn.microsoft.com/en-us/windows/win32/api/lsalookup/ns-lsalookup-policy_account_domain_info\ntype POLICY_ACCOUNT_DOMAIN_INFO struct {\n\tDomainName UNICODE_STRING\n\tDomainSid *windows.SID\n}\n\ntype GUID struct {\n\tData1 uint32\n\tData2 uint16\n\tData3 uint16\n\tData4 [8]byte\n}\n\n// https://learn.microsoft.com/en-us/windows/win32/api/lsalookup/ns-lsalookup-policy_dns_domain_info\ntype POLICY_DNS_DOMAIN_INFO struct {\n\tName UNICODE_STRING\n\tDnsDomainName UNICODE_STRING\n\tDnsForestName UNICODE_STRING\n\tDomainGuid GUID\n\tSid *windows.SID\n}\n\nconst (\n\tDS_DOMAIN_DIRECT_INBOUND = 0x0001\n\tDS_DOMAIN_DIRECT_OUTBOUND = 0x0002\n\tDS_DOMAIN_IN_FOREST = 0x0008\n)\n\ntype DS_DOMAIN_TRUSTSW struct {\n\tNetbiosDomainName *uint16 // LPWSTR\n\tDnsDomainName *uint16 // LPWSTR\n\tFlags uint32 // ULONG\n\tParentIndex uint32 // ULONG\n\tTrustType uint32 // ULONG\n\tTrustAttributes uint32 // ULONG\n\tDomainSid *windows.SID // PSID\n\tDomainGuid windows.GUID // GUID\n}\n\ntype trustedDomain struct {\n\tDomainSid *windows.SID\n\tNetbiosDomainName *uint16\n\tDnsDomainName *uint16\n\tTrustPosixOffset uint32\n}\n\nfunc IsRelativeSid(sid1 *windows.SID, sid2 *windows.SID) bool {\n\tif sid1 == nil || sid2 == nil {\n\t\treturn sid1 == sid2\n\t}\n\n\t// Check if the SIDs have the same revision, we have to do it by ourself\n\t// since windows.SID does not expose the revision field directly.\n\trev1 := *(*uint8)(unsafe.Pointer(sid1))\n\trev2 := *(*uint8)(unsafe.Pointer(sid2))\n\tif rev1 != rev2 {\n\t\treturn false\n\t}\n\n\tauth1 := sid1.IdentifierAuthority()\n\tauth2 := sid2.IdentifierAuthority()\n\tfor i := 0; i < len(auth1.Value); i++ {\n\t\tif auth1.Value[i] != auth2.Value[i] {\n\t\t\treturn false\n\t\t}\n\t}\n\n\tcnt1 := sid1.SubAuthorityCount()\n\tcnt2 := sid2.SubAuthorityCount()\n\tif cnt1+1 != cnt2 {\n\t\treturn false\n\t}\n\n\tfor i := uint8(0); i < cnt1; i++ {\n\t\tif sid1.SubAuthority(uint32(i)) != sid2.SubAuthority(uint32(i)) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// initializeTrustPosixOffsets queries LDAP and sets TrustPosixOffset for each trusted domain.\nfunc initializeTrustPosixOffsets() error {\n\thandle, err := LdapConnect(\"\") // empty string means default server\n\tif err != nil {\n\t\treturn fmt.Errorf(\"LdapConnect failed: %w\", err)\n\t}\n\tdefer LdapClose(handle)\n\n\tdefaultNC, err := LdapGetDefaultNamingContext(handle)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"LdapGetDefaultNamingContext failed: %w\", err)\n\t}\n\n\t// For each trusted domain, get trustPosixOffset\n\tfor i := range trustedDomains {\n\t\tdomain := windows.UTF16PtrToString(trustedDomains[i].DnsDomainName)\n\t\toffsetStr, err := LdapGetTrustPosixOffset(handle, defaultNC, domain)\n\t\tif err == nil {\n\t\t\tif val, err := strconv.ParseUint(offsetStr, 10, 32); err == nil {\n\t\t\t\ttrustedDomains[i].TrustPosixOffset = uint32(val)\n\t\t\t}\n\t\t}\n\t}\n\n\t// If trustPosixOffset looks wrong, fix it up using Cygwin magic value 0xfe500000\n\tfor i := range trustedDomains {\n\t\tif trustedDomains[i].TrustPosixOffset < 0x100000 {\n\t\t\ttrustedDomains[i].TrustPosixOffset = 0xfe500000\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc init() {\n\tif runtime.GOOS != \"windows\" {\n\t\treturn\n\t}\n\n\tvar objAttr LSA_OBJECT_ATTRIBUTES\n\tobjAttr.Length = uint32(unsafe.Sizeof(objAttr))\n\n\tvar policyHandle windows.Handle\n\tr1, _, _ := procLsaOpenPolicy.Call(\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&objAttr)),\n\t\tuintptr(POLICY_VIEW_LOCAL_INFORMATION),\n\t\tuintptr(unsafe.Pointer(&policyHandle)),\n\t)\n\tif windows.NTStatus(r1) != windows.STATUS_SUCCESS {\n\t\treturn\n\t}\n\tdefer procLsaClose.Call(uintptr(policyHandle))\n\n\t// Get the account domain SID\n\tvar acctInfoPtr uintptr\n\tr1, _, _ = procLsaQueryInformationPolicy.Call(\n\t\tuintptr(policyHandle),\n\t\tuintptr(PolicyAccountDomainInformation),\n\t\tuintptr(unsafe.Pointer(&acctInfoPtr)),\n\t)\n\tif windows.NTStatus(r1) == windows.STATUS_SUCCESS && acctInfoPtr != 0 {\n\t\tdefer procLsaFreeMemory.Call(acctInfoPtr)\n\t\tinfo := (*POLICY_ACCOUNT_DOMAIN_INFO)(unsafe.Pointer(acctInfoPtr))\n\t\tif info.DomainSid != nil {\n\t\t\tif sidCopy, err := info.DomainSid.Copy(); err == nil {\n\t\t\t\taccountDomainSid = sidCopy\n\t\t\t}\n\t\t}\n\t}\n\n\t// Get the primary domain SID\n\tvar primInfoPtr uintptr\n\tr1, _, _ = procLsaQueryInformationPolicy.Call(\n\t\tuintptr(policyHandle),\n\t\tuintptr(PolicyDnsDomainInformation),\n\t\tuintptr(unsafe.Pointer(&primInfoPtr)),\n\t)\n\tif windows.NTStatus(r1) == windows.STATUS_SUCCESS && primInfoPtr != 0 {\n\t\tdefer procLsaFreeMemory.Call(primInfoPtr)\n\t\tinfo2 := (*POLICY_DNS_DOMAIN_INFO)(unsafe.Pointer(primInfoPtr))\n\t\tif info2.Sid != nil {\n\t\t\tif sidCopy, err := info2.Sid.Copy(); err == nil {\n\t\t\t\tprimaryDomainSid = sidCopy\n\t\t\t}\n\t\t}\n\t}\n\n\t// QUERY trusted domains\n\tvar domainsPtr uintptr\n\tvar domainCount uint32\n\tr1, _, _ = procDsEnumerateDomainTrustsW.Call(\n\t\t0,\n\t\tuintptr(DS_DOMAIN_DIRECT_INBOUND|DS_DOMAIN_DIRECT_OUTBOUND|DS_DOMAIN_IN_FOREST),\n\t\tuintptr(unsafe.Pointer(&domainsPtr)),\n\t\tuintptr(unsafe.Pointer(&domainCount)),\n\t)\n\tif r1 != 0 || domainsPtr == 0 {\n\t\treturn\n\t}\n\tdefer procNetApiBufferFree.Call(domainsPtr)\n\n\tentrySize := unsafe.Sizeof(DS_DOMAIN_TRUSTSW{})\n\tbase := domainsPtr\n\trealCount := 0\n\tfor i := 0; i < int(domainCount); i++ {\n\t\tdom := (*DS_DOMAIN_TRUSTSW)(unsafe.Pointer(base + uintptr(i)*entrySize))\n\t\tif dom.DomainSid == nil ||\n\t\t\t(dom.NetbiosDomainName == nil && dom.DnsDomainName == nil) ||\n\t\t\twindows.EqualSid(dom.DomainSid, primaryDomainSid) {\n\t\t\tcontinue\n\t\t}\n\t\trealCount++\n\t}\n\n\ttrustedDomains = make([]trustedDomain, 0, realCount)\n\tfor i := 0; i < int(domainCount); i++ {\n\t\tdom := (*DS_DOMAIN_TRUSTSW)(unsafe.Pointer(base + uintptr(i)*entrySize))\n\t\tif dom.DomainSid == nil ||\n\t\t\t(dom.NetbiosDomainName == nil && dom.DnsDomainName == nil) ||\n\t\t\twindows.EqualSid(dom.DomainSid, primaryDomainSid) {\n\t\t\tcontinue\n\t\t}\n\n\t\tsidCopy, err := dom.DomainSid.Copy()\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\ttrustedDomains = append(trustedDomains, trustedDomain{\n\t\t\tDomainSid: sidCopy,\n\t\t\tNetbiosDomainName: dom.NetbiosDomainName,\n\t\t\tDnsDomainName: dom.DnsDomainName,\n\t\t\tTrustPosixOffset: 0,\n\t\t})\n\t}\n\n\tif len(trustedDomains) != 0 {\n\t\tinitializeTrustPosixOffsets()\n\t}\n}\n\nfunc ConvertSidStrToUid(sidStr string) (int, error) {\n\tsid, err := windows.StringToSid(sidStr)\n\tif err != nil {\n\t\treturn -1, err\n\t}\n\tret := convertSidToUid(sid)\n\tif ret < 0 {\n\t\treturn -1, fmt.Errorf(\"invalid uid %d for sid %s\", ret, sidStr)\n\t}\n\treturn ret, nil\n}\n\nfunc convertSidToUid(sid *windows.SID) int {\n\tif sid == nil || !sid.IsValid() {\n\t\treturn -1\n\t}\n\n\tsubAuthCount := sid.SubAuthorityCount()\n\tif subAuthCount == 0 {\n\t\treturn -1\n\t}\n\n\t// SID FORMAT: https://learn.microsoft.com/en-us/windows-server/identity/ad-ds/manage/understand-security-identifiers\n\t// S-VERSION-IDENTIFIER_AUTHORITY-SUBAUTHORITY1-SUBAUTHORITY2-...-SUBAUTHORITYn(RID)\n\t// SUBAUTHORITY1-SUBAUTHORITY2 also known as Domain Identifier\n\n\trid := sid.SubAuthority(uint32(subAuthCount - 1))\n\tsubAuth0 := sid.SubAuthority(0)\n\tauth := sid.IdentifierAuthority()\n\n\tret := -1\n\n\tif auth == windows.SECURITY_NT_AUTHORITY {\n\t\t// windows.SECURITY_NT_AUTHORITY: 5\n\t\tif subAuthCount == 1 {\n\t\t\t// well-known SIDs\n\t\t\tret = int(rid)\n\t\t} else if subAuthCount == 2 && subAuth0 == 32 {\n\t\t\t// well-known SIDs\n\t\t\tret = int(rid) // BUILTIN domain\n\t\t} else if subAuthCount >= 2 && subAuth0 == 5 {\n\t\t\t// ignore\n\t\t} else if subAuthCount >= 5 && subAuth0 == 21 {\n\t\t\tif primaryDomainSid != nil && IsRelativeSid(primaryDomainSid, sid) {\n\t\t\t\t// Accounts from the machine's primary domain:\n\t\t\t\tret = 0x100000 + int(rid)\n\t\t\t} else if accountDomainSid != nil && IsRelativeSid(accountDomainSid, sid) {\n\t\t\t\t// Accounts from the local machine's user DB (SAM):\n\t\t\t\tret = 0x30000 + int(rid)\n\t\t\t} else {\n\t\t\t\t// Accounts from a trusted domain of the machine's primary domain:\n\t\t\t\tfor _, dom := range trustedDomains {\n\t\t\t\t\tif IsRelativeSid(dom.DomainSid, sid) {\n\t\t\t\t\t\tret = int(dom.TrustPosixOffset) + int(rid)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else if subAuthCount == 2 {\n\t\t\t// Other well-known SIDs in the NT_AUTHORITY domain (S-1-5-X-RID):\n\t\t\tret = 0x1000 + int(subAuth0) + int(rid)\n\t\t}\n\t} else if auth == windows.SECURITY_MANDATORY_LABEL_AUTHORITY {\n\t\t// windows.SECURITY_MANDATORY_LABEL_AUTHORITY: 16\n\t\tret = 0x60000 + int(rid)\n\t} else if auth.Value[5] != 0 || rid != 65534 {\n\t\t// Other well-known SIDs:\n\t\tret = 0x10000 + 0x100*int(auth.Value[5]) + int(rid)\n\t}\n\n\tif ret == -1 {\n\t\tret = 65534 // fallback to unmapped SID\n\t}\n\n\treturn ret\n}\n\nfunc GetCurrentUserSID() (*windows.SID, error) {\n\tvar token windows.Token\n\terr := windows.OpenProcessToken(windows.CurrentProcess(), windows.TOKEN_QUERY, &token)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer token.Close()\n\n\tvar requiredLen uint32\n\terr = windows.GetTokenInformation(token, windows.TokenUser, nil, 0, &requiredLen)\n\tif err != windows.ERROR_INSUFFICIENT_BUFFER {\n\t\treturn nil, err\n\t}\n\n\tbuf := make([]byte, requiredLen)\n\terr = windows.GetTokenInformation(token, windows.TokenUser, &buf[0], requiredLen, &requiredLen)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tuserInfo := (*windows.Tokenuser)(unsafe.Pointer(&buf[0]))\n\treturn userInfo.User.Sid, nil\n}\n\nfunc GetCurrentUserPrimaryGroupSID() (*windows.SID, error) {\n\tvar token windows.Token\n\terr := windows.OpenProcessToken(windows.CurrentProcess(), windows.TOKEN_QUERY, &token)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer token.Close()\n\n\tvar requiredLen uint32\n\terr = windows.GetTokenInformation(token, windows.TokenPrimaryGroup, nil, 0, &requiredLen)\n\tif err != windows.ERROR_INSUFFICIENT_BUFFER {\n\t\treturn nil, err\n\t}\n\n\tbuf := make([]byte, requiredLen)\n\terr = windows.GetTokenInformation(token, windows.TokenPrimaryGroup, &buf[0], requiredLen, &requiredLen)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tgroupInfo := (*windows.Tokenprimarygroup)(unsafe.Pointer(&buf[0]))\n\treturn groupInfo.PrimaryGroup, nil\n}\n\nfunc GetCurrentUID() int {\n\t// convert sid to uid, this function have the same procedure with FspPosixMapSidToUid to keep consistencywin\n\t// https://cygwin.com/cygwin-ug-net/ntsec.html\n\n\tsid, err := GetCurrentUserSID()\n\tif err != nil {\n\t\treturn -1\n\t}\n\n\treturn convertSidToUid(sid)\n}\n\nfunc GetCurrentGID() int {\n\tsid, err := GetCurrentUserPrimaryGroupSID()\n\tif err != nil {\n\t\treturn -1\n\t}\n\n\treturn convertSidToUid(sid)\n}\n\nfunc GetCurrentGroupName() string {\n\tsid, err := GetCurrentUserPrimaryGroupSID()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\treturn GetSidName(sid, false)\n}\n\nfunc GetSidName(sid *windows.SID, withDomain bool) string {\n\tvar nameLen, domLen, sidType uint32\n\n\terr := windows.LookupAccountSid(\n\t\tnil, sid,\n\t\tnil, &nameLen,\n\t\tnil, &domLen,\n\t\t&sidType,\n\t)\n\tif err != windows.ERROR_INSUFFICIENT_BUFFER {\n\t\treturn sid.String()\n\t}\n\n\tname := make([]uint16, nameLen)\n\tdom := make([]uint16, domLen)\n\n\terr = windows.LookupAccountSid(\n\t\tnil, sid,\n\t\t&name[0], &nameLen,\n\t\t&dom[0], &domLen,\n\t\t&sidType,\n\t)\n\tif err != nil {\n\t\treturn sid.String()\n\t}\n\n\taccount := syscall.UTF16ToString(name)\n\tif withDomain {\n\t\tdomain := syscall.UTF16ToString(dom)\n\t\treturn domain + `\\` + account\n\t}\n\n\treturn account\n}\n\nfunc IsProcessElevated() (bool, error) {\n\tvar token windows.Token\n\terr := windows.OpenProcessToken(windows.CurrentProcess(), windows.TOKEN_QUERY, &token)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tdefer token.Close()\n\n\t// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-token_elevation\n\ttype tokenElevation struct {\n\t\tTokenIsElevated uint32\n\t}\n\n\tvar elevation tokenElevation\n\tvar outLen uint32\n\terr = windows.GetTokenInformation(token, windows.TokenElevation, (*byte)(unsafe.Pointer(&elevation)), uint32(unsafe.Sizeof(elevation)), &outLen)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn elevation.TokenIsElevated != 0, nil\n}\n" }, { "path": "pkg/winfsp/log.go", "content": "//go:build windows\n// +build windows\n\n/*\n * JuiceFS, Copyright 2026 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage winfsp\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n)\n\nconst RotateAccessLog = 300 << 20 // 300 MiB\n\nfunc (j *juice) log(ctx fs.LogContext, format string, args ...interface{}) {\n\tvar failed bool\n\tfor _, a := range args {\n\t\tif eno, ok := a.(syscall.Errno); ok && eno == syscall.EIO {\n\t\t\tfailed = true\n\t\t}\n\t}\n\tj.logM.Lock()\n\tbuffer := j.logBuffer\n\tj.logM.Unlock()\n\tif buffer == nil && !failed {\n\t\treturn\n\t}\n\tnow := utils.Now()\n\tcmd := fmt.Sprintf(format, args...)\n\tts := now.Format(\"2006.01.02 15:04:05.000000\")\n\tused := ctx.Duration()\n\tcmd += fmt.Sprintf(\" <%.6f>\", used.Seconds())\n\tline := fmt.Sprintf(\"%s [uid:%d,gid:%d,pid:%d] %s\\n\", ts, ctx.Uid(), ctx.Gid(), ctx.Pid(), cmd)\n\tif failed {\n\t\tlogger.Errorf(\"failed operation: %s\", line)\n\t}\n\tif buffer == nil {\n\t\treturn\n\t}\n\tselect {\n\tcase buffer <- line:\n\tdefault:\n\t\tlogger.Debugf(\"log dropped: %s\", line[:len(line)-1])\n\t}\n}\n\nfunc (fs *juice) flushLog(f *os.File, path string, rotateCount int) {\n\tbuf := make([]byte, 0, 128<<10)\n\tvar lastcheck = time.Now()\n\tnumFiles := rotateCount\n\n\tfor {\n\t\tline := <-fs.logBuffer\n\t\tbuf = append(buf[:0], []byte(line)...)\n\tLOOP:\n\t\tfor len(buf) < (128 << 10) {\n\t\t\tselect {\n\t\t\tcase line = <-fs.logBuffer:\n\t\t\t\tbuf = append(buf, []byte(line)...)\n\t\t\tdefault:\n\t\t\t\tbreak LOOP\n\t\t\t}\n\t\t}\n\t\t_, err := f.Write(buf)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"write access log: %s\", err)\n\t\t\tbreak\n\t\t}\n\t\tif lastcheck.Add(time.Minute).After(time.Now()) {\n\t\t\tcontinue\n\t\t}\n\t\tlastcheck = time.Now()\n\t\tfi, err := f.Stat()\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"stat access log: %s\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif fi.Size() > RotateAccessLog {\n\t\t\t_ = f.Close()\n\t\t\tfi, err = os.Stat(path)\n\t\t\tif err == nil && fi.Size() > RotateAccessLog {\n\t\t\t\ttmp := fmt.Sprintf(\"%s.%p\", path, fs)\n\t\t\t\tif os.Rename(path, tmp) == nil {\n\t\t\t\t\tfor i := numFiles - 1; i > 0; i-- {\n\t\t\t\t\t\t_ = os.Rename(path+\".\"+strconv.Itoa(i), path+\".\"+strconv.Itoa(i+1))\n\t\t\t\t\t}\n\t\t\t\t\t_ = os.Rename(tmp, path+\".1\")\n\t\t\t\t} else {\n\t\t\t\t\tfi, err = os.Stat(path)\n\t\t\t\t\tif err == nil && fi.Size() > RotateAccessLog*int64(numFiles) {\n\t\t\t\t\t\tlogger.Infof(\"can't rename %s, truncate it\", path)\n\t\t\t\t\t\t_ = os.Truncate(path, 0)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tf, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"open %s: %s\", path, err)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t_ = os.Chmod(path, 0666)\n\t\t}\n\t}\n}\n" }, { "path": "pkg/winfsp/winfs.go", "content": "//go:build windows\n// +build windows\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage winfsp\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\t\"unicode\"\n\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/juicedata/juicefs/pkg/win\"\n\t\"github.com/winfsp/cgofuse/fuse\"\n\t\"golang.org/x/sys/windows/registry\"\n\n\t\"github.com/urfave/cli/v2\"\n)\n\nvar logger = utils.GetLogger(\"juicefs\")\n\nconst invalidFileHandle = uint64(0xffffffffffffffff)\n\ntype Ino = meta.Ino\n\ntype handleInfo struct {\n\tino meta.Ino\n\tcacheAttr *meta.Attr\n\tattrExpiredAt time.Time\n}\n\ntype juice struct {\n\tfuse.FileSystemBase\n\tsync.RWMutex\n\tconf *vfs.Config\n\tvfs *vfs.VFS\n\tfs *fs.FileSystem\n\thost *fuse.FileSystemHost\n\thandlers map[uint64]handleInfo\n\tbadfd map[uint64]uint64\n\tinoHandleMap map[meta.Ino][]uint64\n\n\tasRoot bool\n\tdelayClose int\n\tenabledGetPath bool\n\tdisableSymlink bool\n\treaddirBatchSize int\n\tadminAsRoot bool\n\n\tlogM sync.Mutex\n\tlogBuffer chan string\n\n\tattrCacheTimeout time.Duration\n}\n\n// Init is called when the file system is created.\nfunc (j *juice) Init() {\n\tj.handlers = make(map[uint64]handleInfo)\n\tj.badfd = make(map[uint64]uint64)\n\tj.inoHandleMap = make(map[meta.Ino][]uint64)\n}\n\nfunc (j *juice) newContext() vfs.LogContext {\n\tif j.asRoot {\n\t\treturn vfs.NewLogContext(meta.Background())\n\t}\n\tuid, gid, pid := fuse.Getcontext()\n\tif uid == 0xffffffff || uid == win.SystemUIDFromFUSE {\n\t\tuid = 0\n\t}\n\tif gid == 0xffffffff || gid == win.SystemUIDFromFUSE {\n\t\tgid = 0\n\t}\n\tif j.adminAsRoot && uid == win.AdministratorUIDFromFUSE {\n\t\t// gid is basically unused on Windows, so we just check the uid here and set the gid as well\n\t\tuid = 0\n\t\tgid = 0\n\t}\n\n\tif pid == -1 {\n\t\tpid = 0\n\t}\n\tctx := meta.NewContext(uint32(pid), uid, []uint32{gid})\n\treturn vfs.NewLogContext(ctx)\n}\n\n// Statfs gets file system statistics.\nfunc (j *juice) Statfs(path string, stat *fuse.Statfs_t) int {\n\tctx := j.newContext()\n\t// defer trace(path)(stat)\n\tvar totalspace, availspace, iused, iavail uint64\n\tj.fs.Meta().StatFS(ctx, meta.RootInode, &totalspace, &availspace, &iused, &iavail)\n\tvar bsize uint64 = 4096\n\tblocks := totalspace / bsize\n\tbavail := availspace / bsize\n\tstat.Namemax = 255\n\tstat.Frsize = 4096\n\tstat.Bsize = bsize\n\tstat.Blocks = blocks\n\tstat.Bfree = bavail\n\tstat.Bavail = bavail\n\tstat.Files = iused + iavail\n\tstat.Ffree = iavail\n\tstat.Favail = iavail\n\treturn 0\n}\n\nfunc errorconv(err syscall.Errno) int {\n\t// convert based on the error.i file in winfsp project\n\tswitch err {\n\tcase syscall.EACCES:\n\t\treturn -fuse.EACCES\n\tcase syscall.EEXIST:\n\t\treturn -fuse.EEXIST\n\tcase syscall.ENOENT, syscall.ENOTDIR:\n\t\treturn -fuse.ENOENT\n\tcase syscall.ECANCELED:\n\t\treturn -fuse.EINTR\n\tcase syscall.EIO:\n\t\treturn -fuse.EIO\n\tcase syscall.EINVAL:\n\t\treturn -fuse.EINVAL\n\tcase syscall.EBADFD:\n\t\treturn -fuse.EBADF\n\tcase syscall.EDQUOT:\n\t\treturn -fuse.ENOSPC\n\tcase syscall.EBUSY:\n\t\treturn -fuse.EBUSY\n\tcase syscall.ENOTEMPTY:\n\t\treturn -fuse.ENOTEMPTY\n\tcase syscall.ENAMETOOLONG:\n\t\treturn -fuse.ENAMETOOLONG\n\tcase syscall.ERROR_HANDLE_EOF:\n\t\treturn -fuse.ENODATA\n\t}\n\n\treturn -int(err)\n}\n\nfunc fuseFlagToSyscall(flag int) int {\n\tvar ret int\n\n\tif flag&fuse.O_RDONLY != 0 {\n\t\tret |= syscall.O_RDONLY\n\t}\n\tif flag&fuse.O_WRONLY != 0 {\n\t\tret |= syscall.O_WRONLY\n\t}\n\tif flag&fuse.O_RDWR != 0 {\n\t\tret |= syscall.O_RDWR\n\t}\n\tif flag&fuse.O_APPEND != 0 {\n\t\tret |= syscall.O_APPEND\n\t}\n\tif flag&fuse.O_CREAT != 0 {\n\t\tret |= syscall.O_CREAT\n\t}\n\tif flag&fuse.O_EXCL != 0 {\n\t\tret |= syscall.O_EXCL\n\t}\n\tif flag&fuse.O_TRUNC != 0 {\n\t\tret |= syscall.O_TRUNC\n\t}\n\treturn ret\n\n}\n\n// Mknod creates a file node.\nfunc (j *juice) Mknod(p string, mode uint32, dev uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Mknod (%s, %d, %d): %d\", p, mode, dev, e) }()\n\tparent, err := j.fs.Open(ctx, path.Dir(p), 0)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t\treturn\n\t}\n\t_, errno := j.vfs.Mknod(ctx, parent.Inode(), path.Base(p), uint16(mode), 0, uint32(dev))\n\te = errorconv(errno)\n\tif e == 0 {\n\t\tj.fs.InvalidateEntry(parent.Inode(), path.Base(p))\n\t}\n\treturn\n}\n\n// Mkdir creates a directory.\nfunc (j *juice) Mkdir(path string, mode uint32) (e int) {\n\tif path == \"/.UMOUNTIT\" {\n\t\tlogger.Infof(\"Umount %s ...\", j.conf.Meta.MountPoint)\n\t\tgo j.host.Unmount()\n\t\treturn -fuse.ENOENT\n\t}\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Mkdir (%s, %d): %d\", path, mode, e) }()\n\te = errorconv(j.fs.Mkdir(ctx, path, uint16(mode), 0))\n\treturn\n}\n\n// Unlink removes a file.\nfunc (j *juice) Unlink(path string) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Unlink (%s): %d\", path, e) }()\n\te = errorconv(j.fs.Delete(ctx, path))\n\treturn\n}\n\n// Rmdir removes a directory.\nfunc (j *juice) Rmdir(path string) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Rmdir (%s): %d\", path, e) }()\n\te = errorconv(j.fs.Delete(ctx, path))\n\treturn\n}\n\nfunc (j *juice) Symlink(target string, newpath string) (e int) {\n\treturn -fuse.ENOSYS\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Symlink (%s, %s): %d\", target, newpath, e) }()\n\tparent, err := j.fs.Open(ctx, path.Dir(newpath), 0)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t\treturn\n\t}\n\t_, errno := j.vfs.Symlink(ctx, target, parent.Inode(), path.Base(newpath))\n\te = errorconv(errno)\n\treturn\n}\n\nfunc (j *juice) Readlink(path string) (e int, target string) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Readlink (%s): (%d, %s)\", path, e, target) }()\n\tif path == \"/\" && j.disableSymlink {\n\t\te = -fuse.ENOSYS\n\t\treturn\n\t}\n\tfi, err := j.fs.Lstat(ctx, path)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t\treturn\n\t}\n\tt, errno := j.vfs.Readlink(ctx, fi.Inode())\n\te = errorconv(errno)\n\ttarget = string(t)\n\treturn\n}\n\n// Rename renames a file.\nfunc (j *juice) Rename(oldpath string, newpath string) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Rename (%s, %s): %d\", oldpath, newpath, e) }()\n\te = errorconv(j.fs.Rename(ctx, oldpath, newpath, 0))\n\treturn\n}\n\n// Chmod changes the permission bits of a file.\nfunc (j *juice) Chmod(path string, mode uint32) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Chmod (%s, %d): %d\", path, mode, e) }()\n\tf, err := j.fs.Open(ctx, path, 0)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t\treturn\n\t}\n\te = errorconv(f.Chmod(ctx, uint16(mode)))\n\tif e == 0 {\n\t\tj.invalidateAttrCache(f.Inode())\n\t}\n\treturn\n}\n\n// Chown changes the owner and group of a file.\nfunc (j *juice) Chown(path string, uid uint32, gid uint32) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Chown (%s, %d, %d): %d\", path, uid, gid, e) }()\n\tf, err := j.fs.Open(ctx, path, 0)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t\treturn\n\t}\n\tif runtime.GOOS == \"windows\" {\n\t\t// FIXME: don't change ownership in windows\n\t\treturn 0\n\t}\n\tinfo, _ := f.Stat()\n\tif uid == 0xffffffff {\n\t\tuid = uint32(info.(*fs.FileStat).Uid())\n\t}\n\tif gid == 0xffffffff {\n\t\tgid = uint32(info.(*fs.FileStat).Gid())\n\t}\n\te = errorconv(f.Chown(ctx, uid, gid))\n\treturn\n}\n\n// Utimens changes the access and modification times of a file.\nfunc (j *juice) Utimens(path string, tmsp []fuse.Timespec) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Utimens (%s, %v): %d\", path, tmsp, e) }()\n\tf, err := j.fs.Open(ctx, path, 0)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t} else {\n\t\te = errorconv(f.Utime2(ctx, tmsp[0].Sec, tmsp[0].Nsec, tmsp[1].Sec, tmsp[1].Nsec))\n\t\tif e == 0 {\n\t\t\tj.invalidateAttrCache(f.Inode())\n\t\t}\n\t}\n\treturn\n}\n\n// Create creates and opens a file.\n// The flags are a combination of the fuse.O_* constants.\nfunc (j *juice) Create(p string, flags int, mode uint32) (e int, fh uint64) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Create (%s, %d, %d): (%d, %d)\", p, flags, mode, e, fh) }()\n\tparent, err := j.fs.Open(ctx, path.Dir(p), 0)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t\treturn\n\t}\n\n\tentry, fh, errno := j.vfs.Create(ctx, parent.Inode(), path.Base(p), uint16(mode), 0, uint32(fuseFlagToSyscall(flags)))\n\tif errno == 0 {\n\t\tj.Lock()\n\t\tj.handlers[fh] = handleInfo{\n\t\t\tino: entry.Inode,\n\t\t\tcacheAttr: entry.Attr,\n\t\t\tattrExpiredAt: time.Now().Add(j.conf.AttrTimeout),\n\t\t}\n\t\tj.inoHandleMap[entry.Inode] = append(j.inoHandleMap[entry.Inode], fh)\n\t\tj.Unlock()\n\t}\n\te = errorconv(errno)\n\tif e == 0 {\n\t\tj.fs.InvalidateEntry(parent.Inode(), path.Base(p))\n\t}\n\treturn\n}\n\n// Open opens a file.\n// The flags are a combination of the fuse.O_* constants.\nfunc (j *juice) Open(path string, flags int) (e int, fh uint64) {\n\tvar fi fuse.FileInfo_t\n\tfi.Flags = fuseFlagToSyscall(flags)\n\te = j.OpenEx(path, &fi)\n\tfh = fi.Fh\n\treturn\n}\n\n// Open opens a file.\n// The flags are a combination of the fuse.O_* constants.\nfunc (j *juice) OpenEx(p string, fi *fuse.FileInfo_t) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Open (%s, %d): (%d, %d)\", p, fi.Flags, e, fi.Fh) }()\n\tino := meta.Ino(0)\n\tif strings.HasSuffix(p, \"/.control\") {\n\t\tino, _ = vfs.GetInternalNodeByName(\".control\")\n\t\tif ino == 0 {\n\t\t\te = -fuse.ENOENT\n\t\t\treturn\n\t\t}\n\t} else if filename := path.Base(p); vfs.IsSpecialName(filename) && path.Dir(p) == \"/\" {\n\t\tino, _ = vfs.GetInternalNodeByName(filename)\n\t\tif ino == 0 {\n\t\t\te = -fuse.ENOENT\n\t\t\treturn\n\t\t}\n\t} else {\n\t\tf, err := j.fs.Open(ctx, p, 0)\n\t\tif err != 0 {\n\t\t\te = -fuse.ENOENT\n\t\t\treturn\n\t\t}\n\t\tino = f.Inode()\n\t}\n\n\tentry, fh, errno := j.vfs.Open(ctx, ino, uint32(fuseFlagToSyscall(fi.Flags)))\n\tif errno == 0 {\n\t\tfi.Fh = fh\n\t\tif vfs.IsSpecialNode(ino) {\n\t\t\tfi.DirectIo = true\n\t\t} else {\n\t\t\tfi.KeepCache = entry.Attr.KeepCache\n\t\t}\n\t\tj.Lock()\n\t\tj.handlers[fh] = handleInfo{\n\t\t\tino: ino,\n\t\t\tcacheAttr: entry.Attr,\n\t\t\tattrExpiredAt: time.Now().Add(j.conf.AttrTimeout),\n\t\t}\n\t\tj.inoHandleMap[ino] = append(j.inoHandleMap[ino], fh)\n\t\tj.Unlock()\n\t}\n\te = errorconv(errno)\n\treturn\n}\n\nfunc (j *juice) attrToStat(inode Ino, attr *meta.Attr, stat *fuse.Stat_t) {\n\tstat.Ino = uint64(inode)\n\tstat.Mode = attr.SMode()\n\tstat.Uid = attr.Uid\n\tstat.Gid = attr.Gid\n\n\tif stat.Uid == 0 {\n\t\tif j.adminAsRoot {\n\t\t\tstat.Uid = win.AdministratorUIDFromFUSE\n\t\t} else {\n\t\t\tstat.Uid = win.SystemUIDFromFUSE\n\t\t}\n\t}\n\tif stat.Gid == 0 && j.adminAsRoot {\n\t\tif j.adminAsRoot {\n\t\t\tstat.Gid = win.AdminstratorsGIDFromFUSE\n\t\t} else {\n\t\t\tstat.Gid = win.SystemUIDFromFUSE\n\t\t}\n\t}\n\n\tstat.Birthtim.Sec = attr.Atime\n\tstat.Birthtim.Nsec = int64(attr.Atimensec)\n\tstat.Atim.Sec = attr.Atime\n\tstat.Atim.Nsec = int64(attr.Atimensec)\n\tstat.Mtim.Sec = attr.Mtime\n\tstat.Mtim.Nsec = int64(attr.Mtimensec)\n\tstat.Ctim.Sec = attr.Ctime\n\tstat.Ctim.Nsec = int64(attr.Ctimensec)\n\tstat.Nlink = attr.Nlink\n\tvar rdev uint32\n\tvar size, blocks uint64\n\tswitch attr.Typ {\n\tcase meta.TypeDirectory:\n\t\tfallthrough\n\tcase meta.TypeSymlink:\n\t\tfallthrough\n\tcase meta.TypeFile:\n\t\tsize = attr.Length\n\t\tblocks = (size + 0xffff) / 0x10000\n\t\tstat.Blksize = 0x10000\n\tcase meta.TypeBlockDev:\n\t\tfallthrough\n\tcase meta.TypeCharDev:\n\t\trdev = attr.Rdev\n\t}\n\tstat.Size = int64(size)\n\tstat.Blocks = int64(blocks)\n\tstat.Rdev = uint64(rdev)\n\tif attr.Flags&meta.FlagImmutable != 0 {\n\t\tstat.Flags |= fuse.UF_READONLY\n\t}\n\tif attr.Flags&meta.FlagWindowsHidden != 0 {\n\t\tstat.Flags |= fuse.UF_HIDDEN\n\t}\n\tif attr.Flags&meta.FlagWindowsSystem != 0 {\n\t\tstat.Flags |= fuse.UF_SYSTEM\n\t}\n\tif attr.Flags&meta.FlagWindowsArchive != 0 {\n\t\tstat.Flags |= fuse.UF_ARCHIVE\n\t}\n}\n\nfunc (j *juice) h2i(fh *uint64) meta.Ino {\n\tdefer j.RUnlock()\n\tj.RLock()\n\n\tentry := j.handlers[*fh]\n\tif entry.ino == 0 {\n\t\tnewfh := j.badfd[*fh]\n\t\tif newfh != 0 {\n\t\t\tentry = j.handlers[newfh]\n\t\t\tif entry.ino > 0 {\n\t\t\t\t*fh = newfh\n\t\t\t}\n\t\t}\n\t}\n\treturn entry.ino\n}\n\nfunc (j *juice) reopen(p string, fh *uint64) meta.Ino {\n\te, newfh := j.Open(p, os.O_RDWR)\n\tif e != 0 {\n\t\treturn 0\n\t}\n\tj.Lock()\n\tdefer j.Unlock()\n\tj.badfd[*fh] = newfh\n\t*fh = newfh\n\treturn j.handlers[newfh].ino\n}\n\n// Getattr gets file attributes.\nfunc (j *juice) getAttrForSpFile(ctx vfs.LogContext, p string, stat *fuse.Stat_t, fh uint64) (e int) {\n\tparentDir := path.Dir(p)\n\t_, err := j.fs.Stat(ctx, parentDir)\n\tif err != 0 {\n\t\te = -fuse.ENOENT\n\t\treturn\n\t}\n\n\tfilename := path.Base(p)\n\tinode, attr := vfs.GetInternalNodeByName(filename)\n\tif inode == 0 {\n\t\te = -fuse.ENOENT\n\t\treturn\n\t}\n\n\tj.vfs.UpdateLength(inode, attr)\n\n\tattr.Gid = ctx.Gid()\n\tattr.Uid = ctx.Uid()\n\n\tj.attrToStat(inode, attr, stat)\n\treturn\n}\n\nfunc (j *juice) invalidateAttrCache(ino meta.Ino) {\n\tif j.attrCacheTimeout == 0 || ino == 0 {\n\t\treturn\n\t}\n\tj.fs.InvalidateAttr(ino) // invalidate the attrcache in fs layer\n\tj.Lock()\n\tdefer j.Unlock()\n\n\thandlers := j.inoHandleMap[ino]\n\tfor _, fh := range handlers {\n\t\tif cache, ok := j.handlers[fh]; ok {\n\t\t\tcache.cacheAttr = nil\n\t\t\tcache.attrExpiredAt = time.Time{}\n\t\t\tj.handlers[fh] = cache\n\t\t}\n\t}\n}\n\nfunc (j *juice) getAttrFromCache(fh uint64) (entry *meta.Entry) {\n\tif j.attrCacheTimeout == 0 || fh == invalidFileHandle {\n\t\treturn nil\n\t}\n\tj.RLock()\n\tdefer j.RUnlock()\n\tif cache, ok := j.handlers[fh]; ok && cache.cacheAttr != nil {\n\t\tif time.Now().Before(cache.attrExpiredAt) {\n\t\t\tentry = &meta.Entry{\n\t\t\t\tInode: cache.ino,\n\t\t\t\tAttr: cache.cacheAttr,\n\t\t\t}\n\t\t\treturn entry\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (j *juice) setAttrCache(fh uint64, attr *meta.Attr) {\n\tif j.attrCacheTimeout == 0 || fh == invalidFileHandle {\n\t\treturn\n\t}\n\n\tj.Lock()\n\tdefer j.Unlock()\n\n\tif cache, ok := j.handlers[fh]; ok {\n\t\tcache.cacheAttr = attr\n\t\tcache.attrExpiredAt = time.Now().Add(j.attrCacheTimeout)\n\t\tj.handlers[fh] = cache\n\t}\n}\n\nfunc (j *juice) getAttr(ctx vfs.Context, fh uint64, ino Ino, opened uint8) (entry *meta.Entry, err syscall.Errno) {\n\tif entry := j.getAttrFromCache(fh); entry != nil {\n\t\treturn entry, 0\n\t}\n\n\tif entry, err = j.vfs.GetAttr(ctx, ino, opened); err != 0 {\n\t\treturn nil, err\n\t}\n\n\tj.setAttrCache(fh, entry.Attr)\n\n\treturn entry, 0\n}\n\n// Getattr gets file attributes.\nfunc (j *juice) Getattr(p string, stat *fuse.Stat_t, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Getattr (%s, %d): %d\", p, fh, e) }()\n\tino := j.h2i(&fh)\n\n\tif ino == 0 {\n\t\t// special case for .control file\n\t\tif strings.HasSuffix(p, \"/.control\") {\n\t\t\te = j.getAttrForSpFile(ctx, p, stat, fh)\n\t\t\treturn\n\t\t} else if vfs.IsSpecialName(path.Base(p)) && path.Dir(p) == \"/\" {\n\t\t\te = j.getAttrForSpFile(ctx, p, stat, fh)\n\t\t\treturn\n\t\t}\n\n\t\tfi, err := j.fs.Lstat(ctx, p)\n\t\tif err != 0 {\n\t\t\t// Known issue: If the parent directory is not exists, the Windows api such as\n\t\t\t// GetFileAttributeX expects the ERROR_PATH_NOT_FOUND returned.\n\t\t\t// However, the fuse api has no such error code defined.\n\t\t\te = -fuse.ENOENT\n\t\t\treturn\n\t\t}\n\t\tino = fi.Inode()\n\t\tentry := fi.Attr()\n\t\tif entry != nil {\n\t\t\tj.vfs.UpdateLength(ino, entry)\n\t\t\tj.attrToStat(ino, entry, stat)\n\t\t\treturn\n\t\t}\n\t}\n\n\tentry, errrno := j.getAttr(ctx, fh, ino, 0)\n\tif errrno != 0 {\n\t\te = errorconv(errrno)\n\t\treturn\n\t}\n\tj.vfs.UpdateLength(entry.Inode, entry.Attr)\n\tj.attrToStat(entry.Inode, entry.Attr, stat)\n\treturn\n}\n\n// Truncate changes the size of a file.\nfunc (j *juice) Truncate(path string, size int64, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Truncate (%s, %d, %d): %d\", path, size, fh, e) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\te = -fuse.EBADF\n\t\treturn\n\t}\n\te = errorconv(j.vfs.Truncate(ctx, ino, size, 0, nil))\n\tif e == 0 {\n\t\tj.invalidateAttrCache(ino)\n\t}\n\treturn\n}\n\n// Read reads data from a file.\nfunc (j *juice) Read(path string, buf []byte, off int64, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Read (%s, %d, %d, %d): %d\", path, len(buf), off, fh, e) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\tlogger.Warnf(\"read from released fd %d for %s, re-open it\", fh, path)\n\t\tino = j.reopen(path, &fh)\n\t}\n\tif ino == 0 {\n\t\te = -fuse.EBADF\n\t\treturn\n\t}\n\tn, err := j.vfs.Read(ctx, ino, buf, uint64(off), fh)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t\treturn\n\t}\n\treturn n\n}\n\n// Write writes data to a file.\nfunc (j *juice) Write(path string, buff []byte, off int64, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Write (%s, %d, %d, %d): %d\", path, len(buff), off, fh, e) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\tlogger.Warnf(\"write to released fd %d for %s, re-open it\", fh, path)\n\t\tino = j.reopen(path, &fh)\n\t}\n\tif ino == 0 {\n\t\te = -fuse.EBADF\n\t\treturn\n\t}\n\terrno := j.vfs.Write(ctx, ino, buff, uint64(off), fh)\n\tif errno != 0 {\n\t\te = errorconv(errno)\n\t} else {\n\t\te = len(buff)\n\t}\n\n\treturn\n}\n\n// Flush flushes cached file data.\nfunc (j *juice) Flush(path string, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Flush (%s, %d): %d\", path, fh, e) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\te = -fuse.EBADF\n\t\treturn\n\t}\n\te = errorconv(j.vfs.Flush(ctx, ino, fh, 0))\n\treturn\n}\n\nfunc (j *juice) cleanInoHandlerMap(ino meta.Ino, fh uint64) {\n\thandles := j.inoHandleMap[ino]\n\tfor i, handle := range handles {\n\t\tif handle == fh {\n\t\t\tj.inoHandleMap[ino] = append(handles[:i], handles[i+1:]...)\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(j.inoHandleMap[ino]) == 0 {\n\t\tdelete(j.inoHandleMap, ino)\n\t}\n}\n\n// Release closes an open file.\nfunc (j *juice) Release(path string, fh uint64) int {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Release (%s, %d)\", path, fh) }()\n\torig := fh\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\tlogger.Warnf(\"release invalid fd %d for %s\", fh, path)\n\t\treturn -fuse.EBADF\n\t}\n\tgo func() {\n\t\ttime.Sleep(time.Second * time.Duration(j.delayClose))\n\t\tj.Lock()\n\t\tdelete(j.handlers, fh)\n\t\tj.cleanInoHandlerMap(ino, fh)\n\t\tif orig != fh {\n\t\t\tdelete(j.badfd, orig)\n\t\t\tj.cleanInoHandlerMap(ino, orig)\n\t\t}\n\t\tj.Unlock()\n\t\tj.vfs.Release(j.newContext(), ino, fh)\n\t}()\n\treturn 0\n}\n\n// Fsync synchronizes file contents.\nfunc (j *juice) Fsync(path string, datasync bool, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Fsync (%s, %t, %d): %d\", path, datasync, fh, e) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\te = -fuse.EBADF\n\t} else {\n\t\te = errorconv(j.vfs.Fsync(ctx, ino, 1, fh))\n\t}\n\treturn\n}\n\n// Opendir opens a directory.\nfunc (j *juice) Opendir(path string) (e int, fh uint64) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Opendir (%s): (%d, %d)\", path, e, fh) }()\n\tf, err := j.fs.Open(ctx, path, 0)\n\tif err != 0 {\n\t\te = -fuse.ENOENT\n\t\treturn\n\t}\n\tfh, errno := j.vfs.Opendir(ctx, f.Inode(), 0)\n\tif errno == 0 {\n\t\tj.Lock()\n\t\tj.handlers[fh] = handleInfo{\n\t\t\tino: f.Inode(),\n\t\t}\n\t\tj.inoHandleMap[f.Inode()] = append(j.inoHandleMap[f.Inode()], fh)\n\n\t\tj.Unlock()\n\t}\n\te = errorconv(errno)\n\treturn\n}\n\n// Readdir reads a directory.\nfunc (j *juice) Readdir(path string,\n\tfill func(name string, stat *fuse.Stat_t, ofst int64) bool,\n\tofst int64, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Readdir (%s, %d, %d): %d\", path, ofst, fh, e) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\te = -fuse.EBADF\n\t\treturn\n\t}\n\n\tcurrentOffset := int(ofst)\n\n\tfor {\n\t\tentries, readAt, err := j.vfs.Readdir(ctx, ino, uint32(j.readdirBatchSize), currentOffset, fh, true)\n\t\tif err != 0 {\n\t\t\te = errorconv(err)\n\t\t\treturn\n\t\t}\n\n\t\tif len(entries) == 0 {\n\t\t\t// Some meta engines may return entries less than batch size\n\t\t\t// so we only break when no entries are returned\n\t\t\tbreak\n\t\t}\n\n\t\tvar st fuse.Stat_t\n\t\tvar ok bool\n\t\tvar full = true\n\t\t// all the entries should have same format\n\t\tfor _, e := range entries {\n\t\t\tif !e.Attr.Full {\n\t\t\t\tfull = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tfor _, e := range entries {\n\t\t\tname := string(e.Name)\n\t\t\tif full {\n\t\t\t\tif j.vfs.ModifiedSince(e.Inode, readAt) {\n\t\t\t\t\tif e2, err := j.vfs.GetAttr(ctx, e.Inode, 0); err == 0 {\n\t\t\t\t\t\te.Attr = e2.Attr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tj.vfs.UpdateLength(e.Inode, e.Attr)\n\t\t\t\tj.attrToStat(e.Inode, e.Attr, &st)\n\t\t\t\tok = fill(name, &st, 0)\n\t\t\t} else {\n\t\t\t\tok = fill(name, nil, 0)\n\t\t\t}\n\t\t\tif !ok {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tcurrentOffset += len(entries)\n\t}\n\treturn\n}\n\n// Releasedir closes an open directory.\nfunc (j *juice) Releasedir(path string, fh uint64) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Releasedir (%s, %d): %d\", path, fh, e) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\te = -fuse.EBADF\n\t\treturn\n\t}\n\tj.Lock()\n\tdelete(j.handlers, fh)\n\tj.cleanInoHandlerMap(ino, fh)\n\tj.Unlock()\n\te = -int(j.vfs.Releasedir(ctx, ino, fh))\n\treturn\n}\n\nfunc (j *juice) Chflags(path string, flags uint32) (e int) {\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Chflags (%s, %d): %d\", path, flags, e) }()\n\tfi, err := j.fs.Stat(ctx, path)\n\tif err != 0 {\n\t\te = -fuse.ENOENT\n\t\treturn\n\t}\n\n\tvar flagSet uint8\n\tif flags&fuse.UF_READONLY != 0 {\n\t\tflagSet |= meta.FlagImmutable\n\t}\n\tif flags&fuse.UF_HIDDEN != 0 {\n\t\tflagSet |= meta.FlagWindowsHidden\n\t}\n\tif flags&fuse.UF_SYSTEM != 0 {\n\t\tflagSet |= meta.FlagWindowsSystem\n\t}\n\tif flags&fuse.UF_ARCHIVE != 0 {\n\t\tflagSet |= meta.FlagWindowsArchive\n\t}\n\n\tino := fi.Inode()\n\terr = j.vfs.ChFlags(ctx, ino, flagSet)\n\tif err != 0 {\n\t\te = errorconv(err)\n\t} else {\n\t\tj.invalidateAttrCache(ino)\n\t}\n\n\treturn\n}\n\nfunc (j *juice) Getpath(p string, fh uint64) (e int, ret string) {\n\tif !j.enabledGetPath {\n\t\tret = p\n\t\treturn\n\t}\n\n\tif strings.HasSuffix(p, \"/.control\") {\n\t\tret = p\n\t\treturn\n\t} else if vfs.IsSpecialName(path.Base(p)) && path.Dir(p) == \"/\" {\n\t\tret = p\n\t\treturn\n\t}\n\n\tctx := j.newContext()\n\tdefer func() { j.log(ctx, \"Getpath (%s, %d): (%d, %s)\", p, fh, e, ret) }()\n\tino := j.h2i(&fh)\n\tif ino == 0 {\n\t\tfi, err := j.fs.Stat(ctx, p)\n\t\tif err != 0 {\n\t\t\te = errorconv(err)\n\t\t\treturn\n\t\t}\n\t\tino = fi.Inode()\n\t}\n\n\tpaths := j.vfs.Meta.GetPaths(ctx, ino)\n\tif len(paths) == 0 {\n\t\tret = p\n\t\treturn\n\t}\n\n\tif len(paths) == 1 {\n\t\tret = paths[0]\n\t\treturn\n\t}\n\n\tretCandidicate := paths[0]\n\n\tfor _, path := range paths {\n\t\tif p == path {\n\t\t\tret = path\n\t\t\treturn\n\t\t} else if strings.EqualFold(path, p) {\n\t\t\tretCandidicate = path\n\t\t}\n\t}\n\n\tret = retCandidicate\n\treturn\n}\n\nfunc getWinFspVersion() string {\n\tconst winfspKey = `SOFTWARE\\WOW6432Node\\WinFsp`\n\tconst sxsDirValue = \"SxsDir\"\n\tconst dllName = \"winfsp-x64.dll\"\n\n\t// Get SxsDir from registry\n\tk, err := registry.OpenKey(registry.LOCAL_MACHINE, winfspKey, registry.QUERY_VALUE)\n\tif err != nil {\n\t\tlogger.Errorf(\"Failed to open registry key %s: %v\", winfspKey, err)\n\t\treturn \"\"\n\t}\n\tdefer k.Close()\n\n\tsxsDir, _, err := k.GetStringValue(sxsDirValue)\n\tif err != nil {\n\t\tlogger.Errorf(\"Failed to get value %s from registry key %s: %v\", sxsDirValue, winfspKey, err)\n\t\treturn \"\"\n\t}\n\n\tif sxsDir == \"\" {\n\t\tlogger.Errorf(\"SxsDir value is empty in registry key %s\", winfspKey)\n\t\treturn \"\"\n\t}\n\n\tdllPath := filepath.Join(sxsDir, \"bin\", dllName)\n\tif _, err := os.Stat(dllPath); os.IsNotExist(err) {\n\t\tlogger.Errorf(\"WinFsp DLL not found at %s\", dllPath)\n\t\treturn \"\"\n\t}\n\n\t// Get version info from DLL using PowerShell\n\tcmd := exec.Command(\"powershell\", \"-NoProfile\", \"-Command\",\n\t\tfmt.Sprintf(`(Get-Item '%s').VersionInfo.FileVersion`, dllPath))\n\toutput, err := cmd.Output()\n\tif err != nil {\n\t\tlogger.Errorf(\"Failed to get version info from %s: %v\", dllPath, err)\n\t\treturn \"\"\n\t}\n\n\treturn strings.TrimSpace(string(output))\n}\n\nfunc compareWinFspVersion(v1, v2 string) int {\n\tparseVersion := func(v string) []int {\n\t\tparts := strings.Split(v, \".\")\n\t\tresult := make([]int, 3)\n\t\tfor i := 0; i < len(parts) && i < 3; i++ {\n\t\t\tresult[i], _ = strconv.Atoi(parts[i])\n\t\t}\n\t\treturn result\n\t}\n\n\tp1 := parseVersion(v1)\n\tp2 := parseVersion(v2)\n\n\tfor i := 0; i < 3; i++ {\n\t\tif p1[i] < p2[i] {\n\t\t\treturn -1\n\t\t}\n\t\tif p1[i] > p2[i] {\n\t\t\treturn 1\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc Serve(v *vfs.VFS, fuseOpt string, asRoot bool, delayCloseSec int, showDotFiles bool, threadsCount int, caseSensitive bool, enabledGetPath bool, c *cli.Context) error {\n\tvar jfs juice\n\tconf := v.Conf\n\tjfs.readdirBatchSize = c.Int(\"readdir-batch-size\")\n\tif jfs.readdirBatchSize <= 0 {\n\t\tjfs.readdirBatchSize = 1000\n\t}\n\tlogger.Debugf(\"Readdir batch size: %d\", jfs.readdirBatchSize)\n\n\tvolAlias := c.String(\"alias\")\n\tif volAlias == \"\" {\n\t\tvolAlias = conf.Format.Name\n\t} else {\n\t\t// alias maybe juicefs-alias\\alias when mounting by the net use command, we need the last part\n\t\tparts := strings.Split(volAlias, `\\`)\n\t\tif len(parts) > 1 {\n\t\t\tvolAlias = parts[len(parts)-1]\n\t\t}\n\t}\n\n\tjfs.attrCacheTimeout = v.Conf.AttrTimeout\n\tjfs.conf = conf\n\tjfs.vfs = v\n\tjfs.enabledGetPath = enabledGetPath\n\tjfs.adminAsRoot = c.Bool(\"admin-as-root\")\n\n\tfuseAccessLog := c.String(\"fuse-access-log\")\n\tif fuseAccessLog != \"\" {\n\t\tf, err := os.OpenFile(fuseAccessLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"open fuse access log %s: %s\", fuseAccessLog, err)\n\t\t} else {\n\t\t\tlogger.Infof(\"fuse access log: %s\", fuseAccessLog)\n\t\t\t_ = os.Chmod(fuseAccessLog, 0666)\n\t\t\tjfs.logBuffer = make(chan string, 1024)\n\t\t\trotateCount := c.Int(\"fuse-access-log-rotate-count\")\n\t\t\tif rotateCount <= 0 {\n\t\t\t\trotateCount = 7\n\t\t\t}\n\t\t\tgo jfs.flushLog(f, fuseAccessLog, rotateCount)\n\t\t}\n\t}\n\n\tvar err error\n\tjfs.fs, err = fs.NewFileSystem(conf, v.Meta, v.Store, nil)\n\tif err != nil {\n\t\tlogger.Fatalf(\"Initialize FileSystem failed: %s\", err)\n\t}\n\tjfs.disableSymlink = os.Getenv(\"JUICEFS_ENABLE_SYMLINK\") != \"1\"\n\tjfs.asRoot = asRoot\n\tjfs.delayClose = delayCloseSec\n\thost := fuse.NewFileSystemHost(&jfs)\n\tjfs.host = host\n\tvar options = \"volname=\" + volAlias\n\tsvrName := fmt.Sprintf(\"juicefs-%s\", volAlias)\n\toptions += fmt.Sprintf(\",ExactFileSystemName=%s,ThreadCount=%d\", svrName, threadsCount)\n\toptions += fmt.Sprintf(\",DirInfoTimeout=%d,VolumeInfoTimeout=1000,KeepFileCache\", int(conf.DirEntryTimeout.Seconds()*1000))\n\toptions += fmt.Sprintf(\",FileInfoTimeout=%d\", int(conf.EntryTimeout.Seconds()*1000))\n\n\tmountAsNetworkDrive := !c.Bool(\"as-local-volume\")\n\tif mountAsNetworkDrive {\n\t\t// when mounting as network drive, the second part of volume prefix should be the volume alias or the display won't be correct\n\t\toptions += fmt.Sprintf(\",VolumePrefix=/%s/%s\", svrName, volAlias)\n\t}\n\n\tcreatePerms := c.String(\"create-perm\")\n\tif createPerms != \"\" {\n\t\tif p, err := strconv.ParseUint(createPerms, 8, 32); err == nil {\n\t\t\toptions += fmt.Sprintf(\",create_umask=%03o\", 0o0777&^p)\n\t\t} else {\n\t\t\tlogger.Warningf(\"Invalid create-perm value: %s\", createPerms)\n\t\t}\n\t}\n\n\tif asRoot {\n\t\toptions += \",uid=-1,gid=-1\"\n\t}\n\tif fuseOpt != \"\" {\n\t\toptions += \",\" + fuseOpt\n\t}\n\tif !showDotFiles {\n\t\toptions += \",dothidden\"\n\t}\n\n\twinfspDbgLog := c.String(\"winfsp-dbg-log\")\n\tif winfspDbgLog != \"\" {\n\t\tlogger.Infof(\"WinFsp Debug Log Path: %s\", winfspDbgLog)\n\t\toptions += \",debug,DebugLog=\" + winfspDbgLog\n\t}\n\tflushOnCleanup := c.Bool(\"flush-on-cleanup\")\n\tif flushOnCleanup {\n\t\twinFSPVersion := getWinFspVersion()\n\t\tif winFSPVersion == \"\" {\n\t\t\tlogger.Warningf(\"Failed to detect WinFsp version, disabling flush-on-cleanup\")\n\t\t\tflushOnCleanup = false\n\t\t} else {\n\t\t\tconst minVersion = \"2.1.25156\"\n\t\t\tif compareWinFspVersion(winFSPVersion, minVersion) <= 0 {\n\t\t\t\tlogger.Warningf(\"Winfsp version %s <= %s, flush-on-cleanup disabled\", winFSPVersion, minVersion)\n\t\t\t\tflushOnCleanup = false\n\t\t\t} else {\n\t\t\t\tlogger.Debugf(\"Winfsp version %s > %s, flush-on-cleanup enabled\", winFSPVersion, minVersion)\n\t\t\t}\n\t\t}\n\t}\n\tif flushOnCleanup {\n\t\toptions += \",FlushOnCleanup=1\"\n\t}\n\n\thost.SetCapCaseInsensitive(!caseSensitive)\n\thost.SetCapReaddirPlus(true)\n\n\tmountVolumeName := filepath.VolumeName(conf.Mountpoint)\n\tmountPointIsDrive := isDriveByVolumeName(conf.Mountpoint)\n\tif mountPointIsDrive {\n\t\tconf.Mountpoint = mountVolumeName\n\t}\n\n\tif !mountPointIsDrive && mountAsNetworkDrive {\n\t\treturn fmt.Errorf(\"Cannot mount to a local directory when --as-local-volume is not set\")\n\t}\n\n\tif !mountPointIsDrive {\n\t\tif _, err := os.Stat(conf.Mountpoint); err == nil {\n\t\t\treturn fmt.Errorf(\"Mount point %s cannot be an existing folder\", conf.Mountpoint)\n\t\t}\n\n\t\t// the parent directory of the mount point must exist\n\t\tparentDir := filepath.Dir(conf.Mountpoint)\n\t\tif _, err := os.Stat(parentDir); os.IsNotExist(err) {\n\t\t\treturn fmt.Errorf(\"Parent directory %s of mount point %s does not exist\", parentDir, conf.Mountpoint)\n\t\t}\n\t}\n\n\tlogger.Debugf(\"mount point: %s, mountPointIsDrive: %v, options: %s\", conf.Mountpoint, mountPointIsDrive, options)\n\texitOk := host.Mount(conf.Mountpoint, []string{\"-o\", options})\n\tif exitOk {\n\t\treturn nil\n\t}\n\n\treturn fmt.Errorf(\"juicefs mount command exit with error, please check the log for details\")\n}\n\nconst winfspSecurityDescriptor = \"D:P(A;;RPWPLC;;;WD)\"\n\nfunc updateWinFspRegService(winfspServiceName string, cmdLine string, alias string, logPath string, asNetworkDrive bool) error {\n\tregKeyPath := \"SOFTWARE\\\\WOW6432Node\\\\WinFsp\\\\Services\\\\\" + winfspServiceName\n\tk, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.ALL_ACCESS)\n\tif err != nil {\n\t\tif err == syscall.ERROR_FILE_NOT_FOUND || err == syscall.ERROR_PATH_NOT_FOUND {\n\t\t\tlogger.Info(\"WinFsp service registry key not found, creating it.\")\n\t\t\tk, _, err = registry.CreateKey(registry.LOCAL_MACHINE, regKeyPath, registry.ALL_ACCESS)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Failed to create registry key: %s\", err)\n\t\t\t}\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"Failed to open registry key: %s\", err)\n\t\t}\n\t}\n\tdefer k.Close()\n\n\terr = k.SetStringValue(\"CommandLine\", cmdLine)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Failed to set registry key: %s\", err)\n\t}\n\n\tsecurityDescriptor := winfspSecurityDescriptor\n\terr = k.SetStringValue(\"Security\", securityDescriptor)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Failed to set registry key: %s\", err)\n\t}\n\n\tfilePath, err := os.Executable()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Failed to get current file path: %s\", err)\n\t}\n\n\terr = k.SetStringValue(\"Executable\", filePath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Failed to set registry key: %s\", err)\n\t}\n\n\terr = k.SetDWordValue(\"JobControl\", 1)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Failed to set registry key: %s\", err)\n\t}\n\n\tif logPath != \"\" {\n\t\terr = k.SetStringValue(\"Stderr\", logPath)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Failed to set registry key: %s\", err)\n\t\t}\n\t} else {\n\t\terr = k.DeleteValue(\"Stderr\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Failed to delete registry key: %s\", err)\n\t\t}\n\t}\n\n\t// RunAs NetworkService/LocalSystem\n\tif !asNetworkDrive {\n\t\terr = k.SetStringValue(\"RunAs\", \"LocalSystem\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Failed to set RunAs value: %s\", err)\n\t\t}\n\t} else {\n\t\tk.DeleteValue(\"RunAs\")\n\t}\n\n\t// SET \"HKLM\\\\SOFTWARE\\\\WOW6432Node\\\\WinFsp\\\\MountBroadcastDriveChange\" to 1\n\tk2, err := registry.OpenKey(registry.LOCAL_MACHINE, \"SOFTWARE\\\\WOW6432Node\\\\WinFsp\", registry.ALL_ACCESS)\n\tif err != nil {\n\t\tlogger.Warningf(\"Failed to open registry key for MountBroadcastDriveChange: %s\", err)\n\t} else {\n\t\tdefer k2.Close()\n\t\terr = k2.SetDWordValue(\"MountBroadcastDriveChange\", 1)\n\t\tif err != nil {\n\t\t\tlogger.Warningf(\"Failed to set MountBroadcastDriveChange value: %s\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc isDriveByVolumeName(s string) bool {\n\t// remove prefix \"\\\\.\\\" if exists\n\tif strings.HasPrefix(s, `\\\\.\\`) {\n\t\ts = s[4:]\n\t}\n\n\tvol := filepath.VolumeName(s)\n\tif len(vol) < 2 {\n\t\treturn false\n\t}\n\tif !unicode.IsLetter(rune(vol[0])) || vol[1] != ':' {\n\t\treturn false\n\t}\n\tif s == vol {\n\t\treturn true\n\t}\n\tif len(s) == len(vol)+1 && (s[len(vol)] == '\\\\' || s[len(vol)] == '/') {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc getWinFspBinPath() string {\n\t// read InstallDir in Computer\\HKEY_LOCAL_MACHINE\\SOFTWARE\\WOW6432Node\\WinFsp\n\n\tconst winfspKey = `SOFTWARE\\WOW6432Node\\WinFsp`\n\tconst installDirValue = \"InstallDir\"\n\tvar installDir string\n\tk, err := registry.OpenKey(registry.LOCAL_MACHINE, winfspKey, registry.QUERY_VALUE)\n\tif err != nil {\n\t\tlogger.Errorf(\"Failed to open registry key %s: %v\", winfspKey, err)\n\t\treturn \"\"\n\t}\n\tdefer k.Close()\n\tinstallDir, _, err = k.GetStringValue(installDirValue)\n\tif err != nil {\n\t\tlogger.Errorf(\"Failed to get value %s from registry key %s: %v\", installDirValue, winfspKey, err)\n\t\treturn \"\"\n\t}\n\n\t// check if the path exists\n\tif installDir == \"\" {\n\t\tlogger.Errorf(\"InstallDir value is empty in registry key %s\", winfspKey)\n\t\treturn \"\"\n\t}\n\n\treturn filepath.Join(installDir, \"bin\")\n}\n\nfunc checkIfMountProcessReady(mountpoint string, timeoutSec int) bool {\n\t// check if the mountpoint is ready\n\tstart := time.Now()\n\tlastPrint := start\n\tfor {\n\t\ttime.Sleep(time.Second)\n\t\t_, err := os.Stat(mountpoint)\n\t\tif err == nil {\n\t\t\treturn true\n\t\t}\n\t\tif time.Since(lastPrint) >= 5*time.Second {\n\t\t\tlogger.Infof(\"Waiting for the mount point %s to be ready...\", mountpoint)\n\t\t\tlastPrint = time.Now()\n\t\t}\n\t\tif time.Since(start) > time.Duration(timeoutSec)*time.Second {\n\t\t\treturn false\n\t\t}\n\t}\n}\n\nfunc RunAsSystemService(name string, mountpoint string, logPath string, defaultCacheDir string, ctx *cli.Context) error {\n\t// https://winfsp.dev/doc/WinFsp-Service-Architecture/\n\tlogger.Info(\"Running as Windows system service.\")\n\n\taddr := ctx.Args().Get(0)\n\tvar cmds []string = []string{\"mount\", addr, \"%2\"}\n\n\thasCacheDir := false\n\n\talias := ctx.String(\"alias\")\n\tif alias == \"\" {\n\t\talias = name\n\t}\n\n\tasNetworkDrive := !ctx.Bool(\"as-local-volume\")\n\n\tlogger.Infof(\"Mounting juicefs as Windows system service. This may require elevated privileges. (Network drive: %v)\", asNetworkDrive)\n\n\t// reconstruct command line from flags\n\tfor _, flag := range ctx.Command.Flags {\n\t\tfor _, v := range flag.Names() {\n\t\t\tif !ctx.IsSet(v) {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif v == \"cache-dir\" {\n\t\t\t\thasCacheDir = true\n\t\t\t}\n\t\t\tif v == \"d\" || v == \"background\" {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif v == \"alias\" {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif len(v) == 1 {\n\t\t\t\tcmds = append(cmds, \"-\"+v)\n\t\t\t} else {\n\t\t\t\tcmds = append(cmds, \"--\"+v)\n\t\t\t}\n\n\t\t\tval := ctx.Value(v)\n\t\t\tswitch val := val.(type) {\n\t\t\tcase bool:\n\t\t\t\tcmds[len(cmds)-1] = fmt.Sprintf(\"%s=%t\", cmds[len(cmds)-1], val)\n\t\t\tcase string:\n\t\t\t\tcmds = append(cmds, fmt.Sprintf(\"\\\"%s\\\"\", val))\n\t\t\tdefault:\n\t\t\t\tcmds = append(cmds, fmt.Sprintf(\"%v\", val))\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// check global flags\n\tfor _, flag := range ctx.App.Flags {\n\t\tfor _, v := range flag.Names() {\n\t\t\tif !ctx.IsSet(v) {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif len(v) == 1 {\n\t\t\t\tcmds = append(cmds, \"-\"+v)\n\t\t\t} else {\n\t\t\t\tcmds = append(cmds, \"--\"+v)\n\t\t\t}\n\n\t\t\tval := ctx.Value(v)\n\t\t\tswitch val := val.(type) {\n\t\t\tcase bool:\n\t\t\t\tcmds[len(cmds)-1] = fmt.Sprintf(\"%s=%t\", cmds[len(cmds)-1], val)\n\t\t\tcase string:\n\t\t\t\tcmds = append(cmds, fmt.Sprintf(\"\\\"%s\\\"\", val))\n\t\t\tdefault:\n\t\t\t\tcmds = append(cmds, fmt.Sprintf(\"%v\", val))\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\n\tcmds = append(cmds, \"--alias\", \"\\\"%1\\\"\") // We put %1 here since it will be replaced by WinFsp with the alias\n\n\tif !hasCacheDir && defaultCacheDir != \"\" {\n\t\tcmds = append(cmds, \"--cache-dir\", \"\\\"\"+defaultCacheDir+\"\\\"\")\n\t}\n\n\tlogger.Debug(\"Command line for juicefs service: \", strings.Join(cmds, \" \"))\n\n\tcmdLine := strings.Join(cmds, \" \")\n\n\twinfspServiceName := \"juicefs-\" + alias\n\tif err := updateWinFspRegService(winfspServiceName, cmdLine, alias, logPath, asNetworkDrive); err != nil {\n\t\treturn fmt.Errorf(\"Failed to update WinFsp service registry: %s\", err)\n\t}\n\n\t// We need to use the \"net use\" for some users who have enabled the 'net use /persistent:yes' option for\n\t// auto-reconnecting after reboot.\n\twinFspBinPath := getWinFspBinPath()\n\tmountByNetUse := os.Getenv(\"JFS_WIN_MOUNT_VIA\") != \"winfsp-launchctl\"\n\tif !asNetworkDrive {\n\t\tmountByNetUse = false\n\t}\n\n\tif winFspBinPath == \"\" && !mountByNetUse {\n\t\treturn fmt.Errorf(`Cannot find WinFsp installation path from registry, please make sure WinFsp is installed correctly.`)\n\t}\n\n\tif !mountByNetUse {\n\t\twinfspLauncher := \"launchctl-x64.exe\"\n\t\tlogger.Debugf(\"WinFsp Bin Path: %s\", winFspBinPath)\n\t\tif winFspBinPath != \"\" {\n\t\t\twinfspLauncher = filepath.Join(winFspBinPath, winfspLauncher)\n\t\t}\n\n\t\t// the second param of start subcommand must be the same as the third param\n\t\t// or the Explorer will not be able to disconnect the volume.\n\t\tcmd := exec.Command(winfspLauncher, \"start\", winfspServiceName, alias, alias, mountpoint)\n\t\tcmd.Dir = winFspBinPath\n\t\tlogger.Debugf(\"Mounting command(using launchctl): %s\", cmd.String())\n\n\t\tout, err := cmd.CombinedOutput()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Failed to mount juicefs as system service: %s, output: %s\", err, string(out))\n\t\t}\n\n\t\tif !checkIfMountProcessReady(mountpoint, 25) {\n\t\t\treturn fmt.Errorf(\"Mount command succeeded, but the mountpoint %s did not become ready in %d seconds, please check the juicefs logs for more information.\", mountpoint, 25)\n\t\t}\n\t} else {\n\t\tlogger.Debugf(\"Trying to start juicefs service by 'net use' command.\")\n\t\tcmd := exec.Command(\"net\", \"use\", mountpoint, fmt.Sprintf(\"\\\\\\\\%s\\\\%s\", winfspServiceName, alias), \"/Y\")\n\t\tout, err := cmd.CombinedOutput()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Failed to start juicefs service by 'net use': %s, output: %s\", err, string(out))\n\t\t}\n\t}\n\n\tlogger.Info(\"Juicefs mount process started successfully.\")\n\n\treturn nil\n}\n" }, { "path": "rfcs/1-dir-used-statistics.md", "content": "# Count space and inodes usage for each directory\n\n## Background\n\nCurrently, we have several counters to globally count the used space and inodes, which can be used to show information or set quota. However, we do not have efficient ways to show used information of or set quota for each directory.\n\n## Proposal\n\nThis document give a proposal to efficiently and almost immediately collect used space and inodes for each directory. The \"efficiently\" means this operation cannot affect the performance of normal IO operations like `mknod`, `write` .etc. And the \"almost immediately\" means this operation cannot be lazy or scheduled, we must update the used space and inodes actively, but there may be a little latency (between several seconds and 1 minute).\n\n## Implementation\n\n### Storage\n\nThe counters should be stored in meta engines, in this section we introduce how to store them in three kinds of meta engines.\n\n#### Redis\n\nRedis engine stores the counters in hashes.\n\n```go\nfunc (m *redisMeta) dirUsedSpaceKey() string {\n return m.prefix + \"dirUsedSpace\"\n}\n \nfunc (m *redisMeta) dirUsedInodesKey() string {\n return m.prefix + \"dirUsedInodes\"\n}\n```\n\n#### SQL\n\nSQL engine stores the counters in a table.\n\n```go\ntype dirUsage struct {\n Inode Ino `xorm:\"pk\"`\n UsedSpace uint64 `xorm:\"notnull\"`\n UsedInodes uint64 `xorm:\"notnull\"`\n}\n```\n\n#### TKV\n\nTKV engine stores each counter in one key.\n\n```go\nfunc (m *kvMeta) dirUsageKey(inode Ino) []byte {\n return m.fmtKey(\"U\", inode)\n}\n```\n\n### Usage\n\nIn this section we represent how and when to update and read the counters.\n\n#### Update\n\nThe are several file types among the children, we should clarify how to deal with each kinds of files first.\n\n| Type | Used space | Used inodes |\n| ------------- | --------------- | ----------- |\n| Normal file | `align4K(size)` | 1 |\n| Directory | 4KiB | 1 |\n| Symlink | 4KiB | 1 |\n| FIFO | 4KiB | 1 |\n| Block device | 4KiB | 1 |\n| Char device | 4KiB | 1 |\n| Socket | 4KiB | 1 |\n\nEach meta engine should implement `doUpdateDirUsage`.\n\n```go\ntype engine interface {\n ...\n doUpdateDirUsage(ctx Context, ino Ino, space int64, inodes int64)\n}\n```\n\nRelevant IO operations should call `doUpdateDirUsage` asynchronously.\n\n```go\nfunc (m *baseMeta) Mknod(ctx Context, parent Ino, ...) syscall.Errno {\n ...\n err := m.en.doMknod(ctx, m.checkRoot(parent), ...)\n ...\n go m.en.doUpdateDirUsage(ctx, parent, 1<<12, 1)\n return err\n}\n\nfunc (m *baseMeta) Unlink(ctx Context, parent Ino, name string) syscall.Errno {\n ...\n err := m.en.doUnlink(ctx, m.checkRoot(parent), name)\n ...\n go m.en.doUpdateDirUsage(ctx, parent, -align4K(attr.size), -1)\n return err\n}\n```\n\n#### Read\n\nEach meta engine should implement `doGetDirUsage`.\n\n```go\ntype engine interface {\n ...\n doGetDirUsage(ctx Context, ino Ino) (space, inodes uint64, err syscall.Errno)\n}\n```\n\nNow we can fasly recursively calculate the space and inodes usage in a directory by `doGetDirUsage`.\n\n```go\n// walk all directories in root\nfunc (m *baseMeta) fastWalkDir(ctx Context, inode Ino, walkDir func(Context, Ino)) syscall.Errno {\n walkDir(ctx, inode)\n var entries []*Entry\n st := m.en.doReaddir(ctx, inode, 0, &entries, -1) // disable plus\n ...\n for _, entry := range entries {\n \tif ent.Attr.Typ != TypeDirectory {\n continue\n \t}\n \tm.fastWalkDir(ctx, entry.Inode, walkFn)\n ...\n }\n return 0\n}\nfunc (m *baseMeta) getDirUsage(ctx Context, root Ino) (space, inodes uint64, err syscall.Errno) {\n m.fastWalkDir(ctx, root, func(_ Context, ino Ino) {\n s, i, err := m.doGetDirUsage(ctx, ino)\n ...\n space += s\n inodes += i\n })\n return\n}\n```\n\n\n" }, { "path": "sdk/java/.gitignore", "content": "*.dll\n*.dylib\n*.so\n.classpath\n.project\n.settings/\ndependency-reduced-pom.xml\ntarget/\n" }, { "path": "sdk/java/Makefile", "content": "GOROOT=$(shell go env GOROOT)\n\nall: package\n\nceph: libjfs-ceph\n\tmvn package -B -Dmaven.test.skip=true\n\nlibjfs-ceph: ../../pkg/*/*.go libjfs/*.go\n\tmake -C libjfs ceph\n\nlibjfs/libjfs: ../../pkg/*/*.go libjfs/*.go\n\tmake -C libjfs\n\ncompile:\n\tmvn compile -B --quiet\ntest: libjfs\n\tmvn test -B --quiet\npackage: libjfs/libjfs\n\tmvn package -B -Dmaven.test.skip=true\n\nwin: win-package package\n\nwin-package: ../../pkg/*/*.go libjfs/*.go\n\tmake -C libjfs win\n\npackage-all: libjfs-all\n\tmvn clean package -B -Dmaven.test.skip=true\n\nlibjfs-all: libjfs.so\n\tdocker run --rm \\\n\t\t-v ~/go/pkg/mod:/go/pkg/mod \\\n\t\t-v ~/work/juicefs/juicefs:/go/src/github.com/juicedata/juicefs \\\n\t\t-v /var/run/docker.sock:/var/run/docker.sock \\\n\t\t-w /go/src/github.com/juicedata/juicefs/sdk/java/libjfs \\\n\t\t--entrypoint=/bin/bash \\\n\t\tjuicedata/golang-cross:latest \\\n\t\t-c 'make mac win linux-arm64 mac-arm64'\n\nlibjfs.so:\n\tdocker run --rm \\\n\t\t-v ~/go/pkg/mod:/go/pkg/mod \\\n\t\t-v $(GOROOT):/go \\\n -v ~/work/juicefs/juicefs:/go/src/github.com/juicedata/juicefs \\\n -v /var/run/docker.sock:/var/run/docker.sock \\\n -w /go/src/github.com/juicedata/juicefs/sdk/java/libjfs \\\n juicedata/sdk-builder \\\n /bin/bash -c 'make'\n" }, { "path": "sdk/java/conf/contract/juicefs.xml", "content": "\n\t\n\t\tfs.contract.test.fs.jfs\n\t\tjfs:///\n\t\n\t\n\t\tfs.jfs.impl\n\t\tio.juicefs.JuiceFileSystem\n\t\n\t\n\t\tjuicefs.no-usage-report\n\t\ttrue\n\t\t\n\t\n\t\tjuicefs.names\n\t\ta.local,b.local,c.local,d.local,e.local\n\t\n\t\n\t\tjuicefs.hosts\n\t\t127.0.0.2,127.0.0.3,127.0.0.4,127.0.0.5,127.0.0.6\n\t\n\t\n\t\tfs.contract.test.root-tests-enabled\n\t\ttrue\n\t\n\t\n\t\tfs.contract.is-case-sensitive\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-append\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-atomic-directory-delete\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-block-locality\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-atomic-rename\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-settimes\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-getfilestatus\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-concat\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-seek\n\t\ttrue\n\t\n\t\n\t\tfs.contract.rejects-seek-past-eof\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-strict-exceptions\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-unix-permissions\n\t\ttrue\n\t\n\t\n\t\tfs.contract.rename-returns-false-if-dest-exists\n\t\ttrue\n\t\n\t\n\t\tfs.contract.supports-file-reference\n\t\ttrue\n\t\n\t\n\t\tfs.contract.rename-returns-false-if-source-missing\n\t\ttrue\n\t\n\n" }, { "path": "sdk/java/conf/core-site.xml", "content": "\n\n\n\t\n\t\tfs.defaultFS\n\t\tjfs://dev/\n\t\n\t\n\t\tfs.jfs.impl\n\t\tio.juicefs.JuiceFileSystem\n\t\n\t\n\t\tjuicefs.no-usage-report\n\t\ttrue\n\t\n\t\n\t\tjuicefs.file.checksum\n\t\ttrue\n\t\n\t\n\t\tjuicefs.access-log\n\t\t/tmp/juicefs-access.log\n\t\t\n\t\n\t\tjuicefs.dev.meta\n\t\t127.0.0.1\n\t\n\t\n\t\tjuicefs.names\n\t\ta.local,b.local,c.local,d.local,e.local\n\t\n\t\n\t\tjuicefs.hosts\n\t\t127.0.0.2,127.0.0.3,127.0.0.4,127.0.0.5,127.0.0.6\n\t\n\n" }, { "path": "sdk/java/conf/log4j.properties", "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n" }, { "path": "sdk/java/kerberos.sh", "content": "#!/bin/sh\n\n# JuiceFS, Copyright 2026 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -e\n\nKERBEROS_REALM=\"EXAMPLE.COM\"\nKERBEROS_PRINCIPLE=\"administrator\"\nKERBEROS_PASSWORD=\"password1234\"\n\nsudo tee /etc/krb5.conf << EOF\n[libdefaults]\n default_realm = $KERBEROS_REALM\n dns_lookup_realm = false\n dns_lookup_kdc = false\n[realms]\n $KERBEROS_REALM = {\n kdc = localhost\n admin_server = localhost\n }\n[logging]\n default = FILE:/var/log/krb5libs.log\n kdc = FILE:/var/log/krb5kdc.log\n admin_server = FILE:/var/log/kadmind.log\n[domain_realm]\n .localhost = $KERBEROS_REALM\n localhost = $KERBEROS_REALM\nEOF\n\nsudo mkdir /etc/krb5kdc\nsudo printf '*/*@%s\\t*' \"$KERBEROS_REALM\" | sudo tee /etc/krb5kdc/kadm5.acl\n\nsudo apt-get update\nsudo apt-get install -y krb5-kdc krb5-admin-server\n\nprintf \"$KERBEROS_PASSWORD\\n$KERBEROS_PASSWORD\" | sudo kdb5_util -r \"$KERBEROS_REALM\" create -s -W\nfor p in client server tom jerry; do\n sudo kadmin.local -q \"addprinc -randkey $p/localhost@$KERBEROS_REALM\"\n sudo kadmin.local -q \"xst -k /tmp/$p.keytab $p/localhost@$KERBEROS_REALM\"\n sudo chmod +rx /tmp/$p.keytab\ndone\n\necho \"Restarting krb services...\"\nsudo service krb5-kdc restart\nsudo service krb5-admin-server restart" }, { "path": "sdk/java/libjfs/Makefile", "content": "export GO111MODULE=on\nLDFLAGS = -s -w\n\nREVISION := $(shell git rev-parse --short HEAD 2>/dev/null)\nREVISIONDATE := $(shell git log -1 --pretty=format:'%cd' --date short 2>/dev/null)\nPKG := github.com/juicedata/juicefs/pkg/version\nLDFLAGS = -s -w\nLDFLAGS += -X $(PKG).revision=$(REVISION) \\\n\t\t-X $(PKG).revisionDate=$(REVISIONDATE)\nGOROOT=$(shell go env GOROOT)\n\nifeq ($(OS),Windows_NT)\n uname_S := Windows\nelse\n uname_S := $(shell uname -s)\n uname_m := $(shell uname -m)\nendif\n\nARCHNAME := amd64\n\nifeq ($(uname_m), aarch64)\n ARCHNAME = arm64\nendif\nifeq ($(uname_m), arm64)\n ARCHNAME = arm64\nendif\n\nLIBFILE := libjfs-$(ARCHNAME).so\nifeq ($(uname_S), Windows)\n LIBFILE = libjfs-$(ARCHNAME).dll\n CC = /usr/bin/musl-gcc\n export CC\nendif\nifeq ($(uname_S), Darwin)\n LIBFILE = libjfs-$(ARCHNAME).dylib\nendif\n\nall: default\n\ndefault: libjfs\n\tmkdir -p target\n\tgzip -c $(LIBFILE) > target/$(LIBFILE).gz\n\nceph: libjfs-ceph\n\tmkdir -p target\n\tgzip -c $(LIBFILE) > target/$(LIBFILE).gz\n\nlibjfs-ceph: *.go ../../../pkg/*/*.go\n\tgo build -tags \"ceph nogspt\" -buildmode=c-shared -ldflags=\"$(LDFLAGS)\" -o $(LIBFILE) .\n\nlibjfs: *.go ../../../pkg/*/*.go\n\tgo build -tags nogspt -buildmode=c-shared -ldflags=\"$(LDFLAGS)\" -o $(LIBFILE) .\n\nlinux-arm64: libjfs-arm64.so\n\tmkdir -p target\n\tgzip -c libjfs-arm64.so > target/libjfs-arm64.so.gz\n\nlibjfs-arm64.so: *.go ../../../pkg/*/*.go\n\tGOARCH=arm64 CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc go build -tags nogspt -buildmode=c-shared -ldflags=\"$(LDFLAGS)\" -o libjfs-arm64.so .\n\nmac: libjfs.dylib\n\tmkdir -p target\n\tgzip -c libjfs-amd64.dylib > target/libjfs-amd64.dylib.gz\n\nlibjfs.dylib: *.go ../../../pkg/*/*.go\n\tGOOS=darwin CGO_ENABLED=1 CC=o64-clang go build -o libjfs-amd64.dylib \\\n\t-tags nogspt -buildmode=c-shared -ldflags=\"$(LDFLAGS)\"\n\nmac-arm64: libjfs-arm64.dylib\n\tmkdir -p target\n\tgzip -c libjfs-arm64.dylib > target/libjfs-arm64.dylib.gz\n\nlibjfs-arm64.dylib: *.go ../../../pkg/*/*.go\n\tGOOS=darwin GOARCH=arm64 CGO_ENABLED=1 CC=o64-clang go build -o libjfs-arm64.dylib \\\n\t-tags nogspt -buildmode=c-shared -ldflags=\"$(LDFLAGS)\"\n\n/usr/local/include/winfsp:\n\tmkdir -p /usr/local/include/winfsp\n\tcp ../../../hack/winfsp_headers/* /usr/local/include/winfsp\n\nwin: libjfs.dll\n\tmkdir -p target\n\tgzip -c libjfs-amd64.dll > target/libjfs-amd64.dll.gz\n\nlibjfs.dll: /usr/local/include/winfsp *.go ../../../pkg/*/*.go\n\tGOOS=windows CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc go build -o libjfs-amd64.dll \\\n\t-tags nogspt -buildmode=c-shared -ldflags=\"$(LDFLAGS)\"\n" }, { "path": "sdk/java/libjfs/bridge.go", "content": "// Copyright 2016 The Prometheus Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package graphite provides a bridge to push Prometheus metrics to a Graphite\n// server.\n\n//nolint\npackage main\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"sort\"\n\t\"time\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tdto \"github.com/prometheus/client_model/go\"\n\t\"github.com/prometheus/common/expfmt\"\n\t\"github.com/prometheus/common/model\"\n\t\"google.golang.org/protobuf/proto\"\n)\n\nconst (\n\tdefaultInterval = 15 * time.Second\n\tmillisecondsPerSecond = 1000\n)\n\n// HandlerErrorHandling defines how a Handler serving metrics will handle\n// errors.\ntype HandlerErrorHandling int\n\n// These constants cause handlers serving metrics to behave as described if\n// errors are encountered.\nconst (\n\t// Ignore errors and try to push as many metrics to Graphite as possible.\n\tContinueOnError HandlerErrorHandling = iota\n\n\t// Abort the push to Graphite upon the first error encountered.\n\tAbortOnError\n)\n\n// Config defines the Graphite bridge config.\ntype Config struct {\n\t// Whether to use Graphite tags or not. Defaults to false.\n\tUseTags bool\n\n\t// The url to push data to. Required.\n\tURL string\n\n\t// The prefix for the pushed Graphite metrics. Defaults to empty string.\n\tPrefix string\n\n\t// The interval to use for pushing data to Graphite. Defaults to 15 seconds.\n\tInterval time.Duration\n\n\t// The timeout for pushing metrics to Graphite. Defaults to 15 seconds.\n\tTimeout time.Duration\n\n\t// The Gatherer to use for metrics. Defaults to prometheus.DefaultGatherer.\n\tGatherer prometheus.Gatherer\n\n\t// The logger that messages are written to. Defaults to no logging.\n\tLogger Logger\n\n\t// ErrorHandling defines how errors are handled. Note that errors are\n\t// logged regardless of the configured ErrorHandling provided Logger\n\t// is not nil.\n\tErrorHandling HandlerErrorHandling\n\n\tCommonLabels map[string]string\n}\n\n// Bridge pushes metrics to the configured Graphite server.\ntype Bridge struct {\n\tuseTags bool\n\turl string\n\tprefix string\n\tinterval time.Duration\n\ttimeout time.Duration\n\n\terrorHandling HandlerErrorHandling\n\tlogger Logger\n\n\tg prometheus.Gatherer\n\tcommonLabels map[string]string\n}\n\n// Logger is the minimal interface Bridge needs for logging. Note that\n// log.Logger from the standard library implements this interface, and it is\n// easy to implement by custom loggers, if they don't do so already anyway.\ntype Logger interface {\n\tPrintln(v ...interface{})\n}\n\n// NewBridge returns a pointer to a new Bridge struct.\nfunc NewBridge(c *Config) (*Bridge, error) {\n\tb := &Bridge{}\n\n\tb.useTags = c.UseTags\n\n\tif c.URL == \"\" {\n\t\treturn nil, errors.New(\"missing URL\")\n\t}\n\tb.url = c.URL\n\n\tif c.Gatherer == nil {\n\t\tb.g = prometheus.DefaultGatherer\n\t} else {\n\t\tb.g = c.Gatherer\n\t}\n\n\tif c.Logger != nil {\n\t\tb.logger = c.Logger\n\t}\n\n\tif c.Prefix != \"\" {\n\t\tb.prefix = c.Prefix\n\t}\n\n\tvar z time.Duration\n\tif c.Interval == z {\n\t\tb.interval = defaultInterval\n\t} else {\n\t\tb.interval = c.Interval\n\t}\n\n\tif c.Timeout == z {\n\t\tb.timeout = defaultInterval\n\t} else {\n\t\tb.timeout = c.Timeout\n\t}\n\n\tb.errorHandling = c.ErrorHandling\n\n\tb.commonLabels = c.CommonLabels\n\treturn b, nil\n}\n\n// Run starts the event loop that pushes Prometheus metrics to Graphite at the\n// configured interval.\nfunc (b *Bridge) Run(ctx context.Context) {\n\tticker := time.NewTicker(b.interval)\n\tdefer ticker.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-ticker.C:\n\t\t\tif err := b.Push(); err != nil && b.logger != nil {\n\t\t\t\tb.logger.Println(\"error pushing to Graphite:\", err)\n\t\t\t}\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Push pushes Prometheus metrics to the configured Graphite server.\nfunc (b *Bridge) Push() error {\n\tmfs, err := b.g.Gather()\n\tif b.commonLabels != nil {\n\t\tfor _, mf := range mfs {\n\t\t\tfor _, metric := range mf.Metric {\n\t\t\t\tfor k, v := range b.commonLabels {\n\t\t\t\t\tmetric.Label = append(metric.Label, &dto.LabelPair{\n\t\t\t\t\t\tName: proto.String(k),\n\t\t\t\t\t\tValue: proto.String(v),\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif err != nil || len(mfs) == 0 {\n\t\tswitch b.errorHandling {\n\t\tcase AbortOnError:\n\t\t\treturn err\n\t\tcase ContinueOnError:\n\t\t\tif b.logger != nil {\n\t\t\t\tb.logger.Println(\"continue on error:\", err)\n\t\t\t}\n\t\tdefault:\n\t\t\tpanic(\"unrecognized error handling value\")\n\t\t}\n\t}\n\n\tconn, err := net.DialTimeout(\"tcp\", b.url, b.timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer conn.Close()\n\n\treturn writeMetrics(conn, mfs, b.useTags, b.prefix, model.Now())\n}\n\nfunc writeMetrics(w io.Writer, mfs []*dto.MetricFamily, useTags bool, prefix string, now model.Time) error {\n\tvec, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{\n\t\tTimestamp: now,\n\t}, mfs...)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbuf := bufio.NewWriter(w)\n\tfor _, s := range vec {\n\t\tif prefix != \"\" {\n\t\t\tfor _, c := range prefix {\n\t\t\t\tif _, err := buf.WriteRune(c); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err := buf.WriteByte('.'); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif err := writeMetric(buf, s.Metric, useTags); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := fmt.Fprintf(buf, \" %g %d\\n\", s.Value, int64(s.Timestamp)/millisecondsPerSecond); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := buf.Flush(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc writeMetric(buf *bufio.Writer, m model.Metric, useTags bool) error {\n\tmetricName, hasName := m[model.MetricNameLabel]\n\tnumLabels := len(m) - 1\n\tif !hasName {\n\t\tnumLabels = len(m)\n\t}\n\n\tvar err error\n\tswitch numLabels {\n\tcase 0:\n\t\tif hasName {\n\t\t\treturn writeSanitized(buf, string(metricName))\n\t\t}\n\tdefault:\n\t\tif err = writeSanitized(buf, string(metricName)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif useTags {\n\t\t\treturn writeTags(buf, m)\n\t\t} else {\n\t\t\treturn writeLabels(buf, m, numLabels)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc writeTags(buf *bufio.Writer, m model.Metric) error {\n\tfor label, value := range m {\n\t\tif label != model.MetricNameLabel {\n\t\t\t_, _ = buf.WriteRune(';')\n\t\t\tif _, err := buf.WriteString(string(label)); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t_, _ = buf.WriteRune('=')\n\t\t\tif _, err := buf.WriteString(string(value)); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc writeLabels(buf *bufio.Writer, m model.Metric, numLabels int) error {\n\tlabelStrings := make([]string, 0, numLabels)\n\tfor label, value := range m {\n\t\tif label != model.MetricNameLabel {\n\t\t\tlabelString := string(label) + \" \" + string(value)\n\t\t\tlabelStrings = append(labelStrings, labelString)\n\t\t}\n\t}\n\tsort.Strings(labelStrings)\n\tfor _, s := range labelStrings {\n\t\tif err := buf.WriteByte('.'); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writeSanitized(buf, s); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc writeSanitized(buf *bufio.Writer, s string) error {\n\tprevUnderscore := false\n\n\tfor _, c := range s {\n\t\tc = replaceInvalidRune(c)\n\t\tif c == '_' {\n\t\t\tif prevUnderscore {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tprevUnderscore = true\n\t\t} else {\n\t\t\tprevUnderscore = false\n\t\t}\n\t\tif _, err := buf.WriteRune(c); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc replaceInvalidRune(c rune) rune {\n\tif c == ' ' {\n\t\treturn '.'\n\t}\n\tif !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == ':' || c == '-' || (c >= '0' && c <= '9')) {\n\t\treturn '_'\n\t}\n\treturn c\n}\n" }, { "path": "sdk/java/libjfs/bridge_test.go", "content": "// Copyright 2018 The Prometheus Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage main\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"net\"\n\t\"os\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/common/model\"\n)\n\nfunc TestSanitize(t *testing.T) {\n\ttestCases := []struct {\n\t\tin, out string\n\t}{\n\t\t{in: \"hello\", out: \"hello\"},\n\t\t{in: \"hE/l1o\", out: \"hE_l1o\"},\n\t\t{in: \"he,*ll(.o\", out: \"he_ll_o\"},\n\t\t{in: \"hello_there%^&\", out: \"hello_there_\"},\n\t\t{in: \"hell-.o\", out: \"hell-_o\"},\n\t}\n\n\tvar buf bytes.Buffer\n\tw := bufio.NewWriter(&buf)\n\n\tfor i, tc := range testCases {\n\t\tif err := writeSanitized(w, tc.in); err != nil {\n\t\t\tt.Fatalf(\"write failed: %v\", err)\n\t\t}\n\t\tif err := w.Flush(); err != nil {\n\t\t\tt.Fatalf(\"flush failed: %v\", err)\n\t\t}\n\n\t\tif want, got := tc.out, buf.String(); want != got {\n\t\t\tt.Fatalf(\"test case index %d: got sanitized string %s, want %s\", i, got, want)\n\t\t}\n\n\t\tbuf.Reset()\n\t}\n}\n\nfunc TestWriteSummary(t *testing.T) {\n\ttestWriteSummary(t, false)\n\ttestWriteSummary(t, true)\n}\n\nfunc testWriteSummary(t *testing.T, useTags bool) {\n\tsumVec := prometheus.NewSummaryVec(\n\t\tprometheus.SummaryOpts{\n\t\t\tName: \"name\",\n\t\t\tHelp: \"docstring\",\n\t\t\tConstLabels: prometheus.Labels{\"constname\": \"constvalue\"},\n\t\t\tObjectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},\n\t\t},\n\t\t[]string{\"labelname\"},\n\t)\n\n\tsumVec.WithLabelValues(\"val1\").Observe(float64(10))\n\tsumVec.WithLabelValues(\"val1\").Observe(float64(20))\n\tsumVec.WithLabelValues(\"val1\").Observe(float64(30))\n\tsumVec.WithLabelValues(\"val2\").Observe(float64(20))\n\tsumVec.WithLabelValues(\"val2\").Observe(float64(30))\n\tsumVec.WithLabelValues(\"val2\").Observe(float64(40))\n\n\treg := prometheus.NewRegistry()\n\treg.MustRegister(sumVec)\n\n\tmfs, err := reg.Gather()\n\tif err != nil {\n\t\tt.Fatalf(\"error: %v\", err)\n\t}\n\n\ttestCases := []struct {\n\t\tprefix string\n\t}{\n\t\t{prefix: \"prefix\"},\n\t\t{prefix: \"pre/fix\"},\n\t\t{prefix: \"pre.fix\"},\n\t\t{prefix: \"\"},\n\t}\n\n\tvar (\n\t\twant = `%s.name.constname.constvalue.labelname.val1.quantile.0_5 20 1477043\n%s.name.constname.constvalue.labelname.val1.quantile.0_9 30 1477043\n%s.name.constname.constvalue.labelname.val1.quantile.0_99 30 1477043\n%s.name_sum.constname.constvalue.labelname.val1 60 1477043\n%s.name_count.constname.constvalue.labelname.val1 3 1477043\n%s.name.constname.constvalue.labelname.val2.quantile.0_5 30 1477043\n%s.name.constname.constvalue.labelname.val2.quantile.0_9 40 1477043\n%s.name.constname.constvalue.labelname.val2.quantile.0_99 40 1477043\n%s.name_sum.constname.constvalue.labelname.val2 90 1477043\n%s.name_count.constname.constvalue.labelname.val2 3 1477043\n`\n\t\twantTagged = `%s.name;constname=constvalue;labelname=val1;quantile=0.5 20 1477043\n%s.name;constname=constvalue;labelname=val1;quantile=0.9 30 1477043\n%s.name;constname=constvalue;labelname=val1;quantile=0.99 30 1477043\n%s.name_sum;constname=constvalue;labelname=val1 60 1477043\n%s.name_count;constname=constvalue;labelname=val1 3 1477043\n%s.name;constname=constvalue;labelname=val2;quantile=0.5 30 1477043\n%s.name;constname=constvalue;labelname=val2;quantile=0.9 40 1477043\n%s.name;constname=constvalue;labelname=val2;quantile=0.99 40 1477043\n%s.name_sum;constname=constvalue;labelname=val2 90 1477043\n%s.name_count;constname=constvalue;labelname=val2 3 1477043\n`\n\t)\n\n\tif useTags {\n\t\twant = wantTagged\n\t}\n\n\tfor i, tc := range testCases {\n\n\t\tnow := model.Time(1477043083)\n\t\tvar buf bytes.Buffer\n\t\terr = writeMetrics(&buf, mfs, useTags, tc.prefix, now)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error: %v\", err)\n\t\t}\n\n\t\tvar wantWithPrefix string\n\t\tif tc.prefix == \"\" {\n\t\t\twantWithPrefix = strings.ReplaceAll(want, \"%s.\", \"\")\n\t\t} else {\n\t\t\twantWithPrefix = fmt.Sprintf(want,\n\t\t\t\ttc.prefix, tc.prefix, tc.prefix, tc.prefix, tc.prefix,\n\t\t\t\ttc.prefix, tc.prefix, tc.prefix, tc.prefix, tc.prefix,\n\t\t\t)\n\t\t}\n\n\t\tgot := buf.String()\n\n\t\tif err := checkLinesAreEqual(wantWithPrefix, got, useTags); err != nil {\n\t\t\tt.Fatalf(\"test case index %d:\\n%s\", i, err.Error())\n\t\t}\n\t}\n}\n\nfunc TestWriteHistogram(t *testing.T) {\n\ttestWriteHistogram(t, false)\n\ttestWriteHistogram(t, true)\n}\n\nfunc testWriteHistogram(t *testing.T, useTags bool) {\n\thistVec := prometheus.NewHistogramVec(\n\t\tprometheus.HistogramOpts{\n\t\t\tName: \"name\",\n\t\t\tHelp: \"docstring\",\n\t\t\tConstLabels: prometheus.Labels{\"constname\": \"constvalue\"},\n\t\t\tBuckets: []float64{0.01, 0.02, 0.05, 0.1},\n\t\t},\n\t\t[]string{\"labelname\"},\n\t)\n\n\thistVec.WithLabelValues(\"val1\").Observe(float64(10))\n\thistVec.WithLabelValues(\"val1\").Observe(float64(20))\n\thistVec.WithLabelValues(\"val1\").Observe(float64(30))\n\thistVec.WithLabelValues(\"val2\").Observe(float64(20))\n\thistVec.WithLabelValues(\"val2\").Observe(float64(30))\n\thistVec.WithLabelValues(\"val2\").Observe(float64(40))\n\n\treg := prometheus.NewRegistry()\n\treg.MustRegister(histVec)\n\n\tmfs, err := reg.Gather()\n\tif err != nil {\n\t\tt.Fatalf(\"error: %v\", err)\n\t}\n\n\tnow := model.Time(1477043083)\n\tvar buf bytes.Buffer\n\terr = writeMetrics(&buf, mfs, useTags, \"prefix\", now)\n\tif err != nil {\n\t\tt.Fatalf(\"error: %v\", err)\n\t}\n\n\tvar (\n\t\twant = `prefix.name_bucket.constname.constvalue.labelname.val1.le.0_01 0 1477043\nprefix.name_bucket.constname.constvalue.labelname.val1.le.0_02 0 1477043\nprefix.name_bucket.constname.constvalue.labelname.val1.le.0_05 0 1477043\nprefix.name_bucket.constname.constvalue.labelname.val1.le.0_1 0 1477043\nprefix.name_sum.constname.constvalue.labelname.val1 60 1477043\nprefix.name_count.constname.constvalue.labelname.val1 3 1477043\nprefix.name_bucket.constname.constvalue.labelname.val1.le._Inf 3 1477043\nprefix.name_bucket.constname.constvalue.labelname.val2.le.0_01 0 1477043\nprefix.name_bucket.constname.constvalue.labelname.val2.le.0_02 0 1477043\nprefix.name_bucket.constname.constvalue.labelname.val2.le.0_05 0 1477043\nprefix.name_bucket.constname.constvalue.labelname.val2.le.0_1 0 1477043\nprefix.name_sum.constname.constvalue.labelname.val2 90 1477043\nprefix.name_count.constname.constvalue.labelname.val2 3 1477043\nprefix.name_bucket.constname.constvalue.labelname.val2.le._Inf 3 1477043\n`\n\t\twantTagged = `prefix.name_bucket;constname=constvalue;labelname=val1;le=0.01 0 1477043\nprefix.name_bucket;constname=constvalue;labelname=val1;le=0.02 0 1477043\nprefix.name_bucket;constname=constvalue;labelname=val1;le=0.05 0 1477043\nprefix.name_bucket;constname=constvalue;labelname=val1;le=0.1 0 1477043\nprefix.name_sum;constname=constvalue;labelname=val1 60 1477043\nprefix.name_count;constname=constvalue;labelname=val1 3 1477043\nprefix.name_bucket;constname=constvalue;labelname=val1;le=+Inf 3 1477043\nprefix.name_bucket;constname=constvalue;labelname=val2;le=0.01 0 1477043\nprefix.name_bucket;constname=constvalue;labelname=val2;le=0.02 0 1477043\nprefix.name_bucket;constname=constvalue;labelname=val2;le=0.05 0 1477043\nprefix.name_bucket;constname=constvalue;labelname=val2;le=0.1 0 1477043\nprefix.name_sum;constname=constvalue;labelname=val2 90 1477043\nprefix.name_count;constname=constvalue;labelname=val2 3 1477043\nprefix.name_bucket;constname=constvalue;labelname=val2;le=+Inf 3 1477043\n`\n\t)\n\n\tif useTags {\n\t\twant = wantTagged\n\t}\n\n\tgot := buf.String()\n\n\tif err := checkLinesAreEqual(want, got, useTags); err != nil {\n\t\tt.Fatalf(err.Error())\n\t}\n}\n\nfunc TestToReader(t *testing.T) {\n\ttestToReader(t, false)\n\ttestToReader(t, true)\n}\n\nfunc testToReader(t *testing.T, useTags bool) {\n\tcntVec := prometheus.NewCounterVec(\n\t\tprometheus.CounterOpts{\n\t\t\tName: \"name\",\n\t\t\tHelp: \"docstring\",\n\t\t\tConstLabels: prometheus.Labels{\"constname\": \"constvalue\"},\n\t\t},\n\t\t[]string{\"labelname\"},\n\t)\n\tcntVec.WithLabelValues(\"val1\").Inc()\n\tcntVec.WithLabelValues(\"val2\").Inc()\n\n\treg := prometheus.NewRegistry()\n\treg.MustRegister(cntVec)\n\n\tvar (\n\t\twant = `prefix.name.constname.constvalue.labelname.val1 1 1477043\nprefix.name.constname.constvalue.labelname.val2 1 1477043\n`\n\t\twantTagged = `prefix.name;constname=constvalue;labelname=val1 1 1477043\nprefix.name;constname=constvalue;labelname=val2 1 1477043\n`\n\t)\n\n\tif useTags {\n\t\twant = wantTagged\n\t}\n\n\tmfs, err := reg.Gather()\n\tif err != nil {\n\t\tt.Fatalf(\"error: %v\", err)\n\t}\n\n\tnow := model.Time(1477043083)\n\tvar buf bytes.Buffer\n\terr = writeMetrics(&buf, mfs, useTags, \"prefix\", now)\n\tif err != nil {\n\t\tt.Fatalf(\"error: %v\", err)\n\t}\n\n\tgot := buf.String()\n\n\tif err := checkLinesAreEqual(want, got, useTags); err != nil {\n\t\tt.Fatalf(err.Error())\n\t}\n}\n\nfunc checkLinesAreEqual(w, g string, useTags bool) error {\n\tif useTags {\n\t\ttaggedLineRegexp := regexp.MustCompile(`;| `)\n\n\t\twantLines, err := stringToLines(w)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tgotLines, err := stringToLines(g)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor lineInd := range gotLines {\n\t\t\tvar log string\n\t\t\t// Tagged metric, order of tags doesn't matter\n\t\t\t// m1 := \"prefix.name;tag1=val1;tag2=val2 3 1477043\"\n\t\t\t// m2 := \"prefix.name;tag2=val2;tag1=val1 3 1477043\"\n\t\t\t// m1 should be equal to m2\n\t\t\twantSplit := taggedLineRegexp.Split(wantLines[lineInd], -1)\n\t\t\tgotSplit := taggedLineRegexp.Split(gotLines[lineInd], -1)\n\t\t\tsort.Strings(wantSplit)\n\t\t\tsort.Strings(gotSplit)\n\n\t\t\tlog += fmt.Sprintf(\"want: %v\\ngot: %v\\n\\n\", wantSplit, gotSplit)\n\n\t\t\tif !reflect.DeepEqual(wantSplit, gotSplit) {\n\t\t\t\treturn fmt.Errorf(log)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\tif w != g {\n\t\treturn fmt.Errorf(\"wanted:\\n\\n%s\\ngot:\\n\\n%s\", w, g)\n\t}\n\n\treturn nil\n}\n\nfunc stringToLines(s string) (lines []string, err error) {\n\tscanner := bufio.NewScanner(strings.NewReader(s))\n\tfor scanner.Scan() {\n\t\tlines = append(lines, scanner.Text())\n\t}\n\terr = scanner.Err()\n\treturn\n}\n\nfunc TestPush(t *testing.T) {\n\treg := prometheus.NewRegistry()\n\tcntVec := prometheus.NewCounterVec(\n\t\tprometheus.CounterOpts{\n\t\t\tName: \"name\",\n\t\t\tHelp: \"docstring\",\n\t\t\tConstLabels: prometheus.Labels{\"constname\": \"constvalue\"},\n\t\t},\n\t\t[]string{\"labelname\"},\n\t)\n\tcntVec.WithLabelValues(\"val1\").Inc()\n\tcntVec.WithLabelValues(\"val2\").Inc()\n\treg.MustRegister(cntVec)\n\n\thost := \"localhost\"\n\tport := \":56789\"\n\tb, err := NewBridge(&Config{\n\t\tURL: host + port,\n\t\tGatherer: reg,\n\t\tPrefix: \"prefix\",\n\t\tUseTags: true,\n\t\tCommonLabels: map[string]string{\"a\": \"b\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"error creating bridge: %v\", err)\n\t}\n\n\tnmg, err := newMockGraphite(port)\n\tif err != nil {\n\t\tt.Fatalf(\"error creating mock graphite: %v\", err)\n\t}\n\tdefer nmg.Close()\n\n\terr = b.Push()\n\tif err != nil {\n\t\tt.Fatalf(\"error pushing: %v\", err)\n\t}\n\n\twants := []string{\n\t\t\"prefix.name.constname.constvalue.labelname.val1 1\",\n\t\t\"prefix.name.constname.constvalue.labelname.val2 1\",\n\t}\n\n\tselect {\n\tcase got := <-nmg.readc:\n\t\tfor _, want := range wants {\n\t\t\tmatched, err := regexp.MatchString(want, got)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"error pushing: %v\", err)\n\t\t\t}\n\t\t\tif !matched {\n\t\t\t\tt.Fatalf(\"missing metric:\\nno match for %s received by server:\\n%s\", want, got)\n\t\t\t}\n\t\t}\n\t\treturn\n\tcase err := <-nmg.errc:\n\t\tt.Fatalf(\"error reading push: %v\", err)\n\tcase <-time.After(50 * time.Millisecond):\n\t\tt.Fatalf(\"no result from graphite server\")\n\t}\n}\n\nfunc newMockGraphite(port string) (*mockGraphite, error) {\n\treadc := make(chan string)\n\terrc := make(chan error)\n\tln, err := net.Listen(\"tcp\", port)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tgo func() {\n\t\tconn, err := ln.Accept()\n\t\tif err != nil {\n\t\t\terrc <- err\n\t\t}\n\t\tvar b bytes.Buffer\n\t\tio.Copy(&b, conn)\n\t\treadc <- b.String()\n\t}()\n\n\treturn &mockGraphite{\n\t\treadc: readc,\n\t\terrc: errc,\n\t\tListener: ln,\n\t}, nil\n}\n\ntype mockGraphite struct {\n\treadc chan string\n\terrc chan error\n\n\tnet.Listener\n}\n\nfunc ExampleBridge() {\n\tb, err := NewBridge(&Config{\n\t\tURL: \"graphite.example.org:3099\",\n\t\tGatherer: prometheus.DefaultGatherer,\n\t\tPrefix: \"prefix\",\n\t\tInterval: 15 * time.Second,\n\t\tTimeout: 10 * time.Second,\n\t\tErrorHandling: AbortOnError,\n\t\tLogger: log.New(os.Stdout, \"graphite bridge: \", log.Lshortfile),\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tgo func() {\n\t\t// Start something in a goroutine that uses metrics.\n\t}()\n\n\t// Push initial metrics to Graphite. Fail fast if the push fails.\n\tif err := b.Push(); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Create a Context to control stopping the Run() loop that pushes\n\t// metrics to Graphite.\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\t// Start pushing metrics to Graphite in the Run() loop.\n\tb.Run(ctx)\n}\n" }, { "path": "sdk/java/libjfs/callback.c", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include \n\nstatic void (*log_callback)(const char *msg);\n\ntypedef void LogCallBack(const char *msg);\n\nvoid jfs_set_logger(void*p);\n\nvoid jfs_set_callback(LogCallBack *callback)\n{\n log_callback = callback;\n jfs_set_logger(callback);\n}\n\nvoid jfs_callback(const char *msg)\n{\n if (log_callback != NULL) {\n (*log_callback)(msg);\n } else {\n fprintf(stderr, \"%s\", msg);\n }\n}\n" }, { "path": "sdk/java/libjfs/guid.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage main\n\nimport (\n\t\"crypto/md5\"\n\t\"encoding/binary\"\n\t\"os/user\"\n\t\"strconv\"\n\t\"sync\"\n)\n\ntype pwent struct {\n\tid uint32\n\tname string\n}\n\ntype mapping struct {\n\tsync.Mutex\n\tsalt string\n\tlocal bool\n\tmask uint32\n\tusernames map[string]uint32\n\tuserIDs map[uint32]string\n\tgroups map[string]uint32\n\tgroupIDs map[uint32]string\n}\n\nfunc newMapping(salt string) *mapping {\n\tm := &mapping{\n\t\tsalt: salt,\n\t\tusernames: make(map[string]uint32),\n\t\tuserIDs: make(map[uint32]string),\n\t\tgroups: make(map[string]uint32),\n\t\tgroupIDs: make(map[uint32]string),\n\t}\n\tm.update(genAllUids(), genAllGids(), true)\n\treturn m\n}\n\nfunc (m *mapping) genGuid(name string) uint32 {\n\tdigest := md5.Sum([]byte(m.salt + name + m.salt))\n\ta := binary.LittleEndian.Uint64(digest[0:8])\n\tb := binary.LittleEndian.Uint64(digest[8:16])\n\tid := uint32(a ^ b)\n\tif m.mask > 0 {\n\t\tid &= m.mask\n\t}\n\treturn id\n}\n\nfunc (m *mapping) lookupUser(name string) uint32 {\n\tm.Lock()\n\tdefer m.Unlock()\n\tvar id uint32\n\tif id, ok := m.usernames[name]; ok {\n\t\treturn id\n\t}\n\tif !m.local {\n\t\tid := m.genGuid(name)\n\t\tm.updateUser(name, id)\n\t\treturn id\n\t}\n\tif name == \"root\" { // root in hdfs sdk is a normal user\n\t\tid = m.genGuid(name)\n\t} else {\n\t\tu, _ := user.Lookup(name)\n\t\tif u != nil {\n\t\t\tid_, _ := strconv.ParseUint(u.Uid, 10, 32)\n\t\t\tid = uint32(id_)\n\t\t} else {\n\t\t\tid = m.genGuid(name)\n\t\t}\n\t}\n\tlogger.Debugf(\"update user to %s:%d by lookup user\", name, id)\n\tm.updateUser(name, id)\n\treturn id\n}\n\nfunc (m *mapping) lookupGroup(name string) uint32 {\n\tm.Lock()\n\tdefer m.Unlock()\n\tvar id uint32\n\tif id, ok := m.groups[name]; ok {\n\t\treturn id\n\t}\n\tif !m.local {\n\t\tid := m.genGuid(name)\n\t\tm.updateGroup(name, id)\n\t\treturn id\n\t}\n\tif name == \"root\" {\n\t\tid = m.genGuid(name)\n\t} else {\n\t\tg, _ := user.LookupGroup(name)\n\t\tif g == nil {\n\t\t\tid = m.genGuid(name)\n\t\t} else {\n\t\t\tid_, _ := strconv.ParseUint(g.Gid, 10, 32)\n\t\t\tid = uint32(id_)\n\t\t}\n\t}\n\tlogger.Debugf(\"update group to %s:%d by lookup group\", name, id)\n\tm.updateGroup(name, id)\n\treturn id\n}\n\nfunc (m *mapping) lookupUserID(id uint32) string {\n\tm.Lock()\n\tdefer m.Unlock()\n\tif name, ok := m.userIDs[id]; ok {\n\t\treturn name\n\t}\n\tif !m.local {\n\t\treturn strconv.Itoa(int(id))\n\t}\n\tu, _ := user.LookupId(strconv.Itoa(int(id)))\n\tif u == nil {\n\t\tu = &user.User{Username: strconv.Itoa(int(id))}\n\t}\n\tname := u.Username\n\tif len(name) > 49 {\n\t\tname = name[:49]\n\t}\n\tlogger.Debugf(\"update user to %s:%d by lookup user id\", name, id)\n\tm.updateUser(name, id)\n\treturn name\n}\n\nfunc (m *mapping) lookupGroupID(id uint32) string {\n\tm.Lock()\n\tdefer m.Unlock()\n\tif name, ok := m.groupIDs[id]; ok {\n\t\treturn name\n\t}\n\tif !m.local {\n\t\treturn strconv.Itoa(int(id))\n\t}\n\tg, _ := user.LookupGroupId(strconv.Itoa(int(id)))\n\tif g == nil {\n\t\tg = &user.Group{Name: strconv.Itoa(int(id))}\n\t}\n\tname := g.Name\n\tif len(name) > 49 {\n\t\tname = name[:49]\n\t}\n\tlogger.Debugf(\"update group to %s:%d by lookup group id\", name, id)\n\tm.updateGroup(name, id)\n\treturn name\n}\n\nfunc (m *mapping) update(uids []pwent, gids []pwent, local bool) {\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.local = local\n\tfor _, u := range uids {\n\t\tm.updateUser(u.name, u.id)\n\t}\n\tfor _, g := range gids {\n\t\tm.updateGroup(g.name, g.id)\n\t}\n\tlogger.Debugf(\"users:\\n%+v\", m.usernames)\n\tlogger.Debugf(\"userids:\\n%+v\", m.userIDs)\n\tlogger.Debugf(\"groups:\\n%+v\", m.groups)\n\tlogger.Debugf(\"gorupids:\\n%+v\", m.groupIDs)\n}\n\nfunc (m *mapping) updateUser(name string, id uint32) {\n\toldId := m.usernames[name]\n\toldName := m.userIDs[id]\n\tdelete(m.userIDs, oldId)\n\tdelete(m.usernames, oldName)\n\tm.usernames[name] = id\n\tm.userIDs[id] = name\n}\n\nfunc (m *mapping) updateGroup(name string, id uint32) {\n\toldId := m.groups[name]\n\toldName := m.groupIDs[id]\n\tdelete(m.groupIDs, oldId)\n\tdelete(m.groups, oldName)\n\tm.groups[name] = id\n\tm.groupIDs[id] = name\n}\n" }, { "path": "sdk/java/libjfs/guid_unix.go", "content": "//go:build !windows\n// +build !windows\n\n/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage main\n\n// #include \n// #include \nimport \"C\"\nimport (\n\t\"sync\"\n)\n\n// protect getpwent and getgrent\nvar cgoMutex sync.Mutex\n\nfunc genAllUids() []pwent {\n\tcgoMutex.Lock()\n\tdefer cgoMutex.Unlock()\n\tC.setpwent()\n\tdefer C.endpwent()\n\tvar uids []pwent\n\tfor {\n\t\tp := C.getpwent()\n\t\tif p == nil {\n\t\t\tbreak\n\t\t}\n\t\tname := C.GoString(p.pw_name)\n\t\tif name != \"root\" {\n\t\t\tuids = append(uids, pwent{uint32(p.pw_uid), name})\n\t\t}\n\t}\n\treturn uids\n}\n\nfunc genAllGids() []pwent {\n\tcgoMutex.Lock()\n\tdefer cgoMutex.Unlock()\n\tC.setgrent()\n\tdefer C.endgrent()\n\tvar gids []pwent\n\tfor {\n\t\tp := C.getgrent()\n\t\tif p == nil {\n\t\t\tbreak\n\t\t}\n\t\tname := C.GoString(p.gr_name)\n\t\tif name != \"root\" {\n\t\t\tgids = append(gids, pwent{uint32(p.gr_gid), name})\n\t\t}\n\t}\n\treturn gids\n}\n" }, { "path": "sdk/java/libjfs/guid_windows.go", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage main\n\nimport (\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n)\n\nfunc genAllUids() []pwent {\n\tout, err := exec.Command(\"wmic\", \"useraccount\", \"list\", \"brief\").Output()\n\tif err != nil {\n\t\tlogger.Errorf(\"cmd : %s\", err)\n\t\treturn nil\n\t}\n\tlines := strings.Split(string(out), \"\\r\\n\")\n\tif len(lines) < 2 {\n\t\tlogger.Errorf(\"no uids: %s\", string(out))\n\t\treturn nil\n\t}\n\tvar uids []pwent\n\tfor _, line := range lines[1 : len(lines)-1] {\n\t\tfields := strings.Fields(line)\n\t\tif len(fields) < 5 {\n\t\t\tcontinue\n\t\t}\n\t\tname := fields[len(fields)-2]\n\t\tsid := fields[len(fields)-1]\n\t\tps := strings.Split(sid, \"-\")\n\t\tauth, _ := strconv.ParseUint(ps[2], 10, 32)\n\t\tcount := len(ps) - 3\n\t\tvar subAuth uint64\n\t\tif count > 0 {\n\t\t\tsubAuth, _ = strconv.ParseUint(ps[3], 10, 32)\n\t\t}\n\t\trid, _ := strconv.ParseUint(ps[len(ps)-1], 10, 32)\n\t\tvar uid uint64\n\t\tif auth == 5 {\n\t\t\tif count == 1 {\n\t\t\t\t// \"SYSTEM\" S-1-5-18 <=> uid/gid: 18\n\t\t\t\tuid = rid\n\t\t\t} else if count == 2 && subAuth == 32 {\n\t\t\t\t// \"Users\" S-1-5-32-545 <=> uid/gid: 545\n\t\t\t\tuid = rid\n\t\t\t} else if count >= 2 && subAuth == 5 {\n\t\t\t\t// not supported\n\t\t\t} else if count >= 5 && subAuth == 21 {\n\t\t\t\t// S-1-5-21-X-Y-Z-RID <=> uid/gid: 0x30000 + RID\n\t\t\t\t// S-1-5-21-X-Y-Z-RID <=> uid/gid: 0x100000 + RID\n\t\t\t\tuid = 0x30000 + rid\n\t\t\t} else if count == 2 {\n\t\t\t\t// S-1-5-X-RID <=> uid/gid: 0x1000 * X + RID\n\t\t\t\tuid = 0x1000*subAuth + rid\n\t\t\t}\n\t\t} else if auth == 16 {\n\t\t\t// S-1-16-RID <=> uid/gid: 0x60000 + RID\n\t\t\tuid = 0x60000*subAuth + rid\n\t\t}\n\t\tif uid > 0 {\n\t\t\tuids = append(uids, pwent{uint32(uid), name})\n\t\t\tlogger.Tracef(\"found account %s -> %d (%s)\", name, uid, sid)\n\t\t}\n\t}\n\treturn uids\n}\n\nfunc genAllGids() []pwent {\n\tout, err := exec.Command(\"wmic\", \"group\", \"list\", \"brief\").Output()\n\tif err != nil {\n\t\tlogger.Errorf(\"cmd : %s\", err)\n\t\treturn nil\n\t}\n\tlines := strings.Split(string(out), \"\\r\\n\")\n\tif len(lines) < 2 {\n\t\tlogger.Errorf(\"no gids: %s\", string(out))\n\t\treturn nil\n\t}\n\ttitle := lines[0]\n\tnameIndex := strings.Index(title, \"Name\")\n\tsidIndex := strings.Index(title, \"SID\")\n\tvar gids []pwent\n\tfor _, line := range lines[1 : len(lines)-1] {\n\t\tif len(line) < sidIndex {\n\t\t\tcontinue\n\t\t}\n\t\tname := strings.TrimSpace(line[nameIndex : sidIndex-1])\n\t\tsid := strings.TrimSpace(line[sidIndex:])\n\t\tps := strings.Split(sid, \"-\")\n\t\tauth, _ := strconv.ParseUint(ps[2], 10, 32)\n\t\tcount := len(ps) - 3\n\t\tvar subAuth uint64\n\t\tif count > 0 {\n\t\t\tsubAuth, _ = strconv.ParseUint(ps[3], 10, 32)\n\t\t}\n\t\trid, _ := strconv.ParseUint(ps[len(ps)-1], 10, 32)\n\t\tvar gid uint64\n\t\tif auth == 5 {\n\t\t\tif count == 1 {\n\t\t\t\t// \"SYSTEM\" S-1-5-18 <=> uid/gid: 18\n\t\t\t\tgid = rid\n\t\t\t} else if count == 2 && subAuth == 32 {\n\t\t\t\t// \"Users\" S-1-5-32-545 <=> uid/gid: 545\n\t\t\t\tgid = rid\n\t\t\t} else if count >= 2 && subAuth == 5 {\n\t\t\t\t// not supported\n\t\t\t} else if count >= 5 && subAuth == 21 {\n\t\t\t\t// S-1-5-21-X-Y-Z-RID <=> uid/gid: 0x30000 + RID\n\t\t\t\t// S-1-5-21-X-Y-Z-RID <=> uid/gid: 0x100000 + RID\n\t\t\t\tgid = 0x30000 + rid\n\t\t\t} else if count == 2 {\n\t\t\t\t// S-1-5-X-RID <=> uid/gid: 0x1000 * X + RID\n\t\t\t\tgid = 0x1000*subAuth + rid\n\t\t\t}\n\t\t} else if auth == 16 {\n\t\t\t// S-1-16-RID <=> uid/gid: 0x60000 + RID\n\t\t\tgid = 0x60000*subAuth + rid\n\t\t}\n\t\tif gid > 0 {\n\t\t\tgids = append(gids, pwent{uint32(gid), name})\n\t\t\tlogger.Tracef(\"found group %s -> %d (%s)\", name, gid, sid)\n\t\t}\n\t}\n\treturn gids\n}\n" }, { "path": "sdk/java/libjfs/kerberos.go", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage main\n\nimport (\n\t\"bufio\"\n\t\"crypto/rand\"\n\t\"encoding/base64\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/jcmturner/gokrb5/v8/keytab\"\n\t\"github.com/jcmturner/gokrb5/v8/service\"\n\t\"github.com/jcmturner/gokrb5/v8/spnego\"\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n)\n\nconst (\n\tdefaultLife = 3600 * 24 * 7\n\tdefaultRenew = 3600 * 24\n)\n\nconst (\n\tmechanismHadoop = \"hadoop\"\n\tmechanismMIT = \"mit\"\n)\n\nvar (\n\tnamePattern = regexp.MustCompile(`([^/@]+)(/([^/@]+))?(@([^/@]+))?`)\n\tparamPattern = regexp.MustCompile(`[^$]*(\\$\\d)`)\n\truleParser = regexp.MustCompile(`(\\[(\\d+):([^\\]]+)\\](\\(([^\\)]+)\\))?(s/([^/]+)/([^/]*)/(g)?)?)/?(L)?`)\n\tnoSimplePattern = regexp.MustCompile(`[/@]`)\n)\n\ntype kRule struct {\n\tisDefault bool\n\tcomps int\n\tformat string\n\tmatch *regexp.Regexp\n\tfromPattern *regexp.Regexp\n\ttoPattern string\n\trepeat bool\n\tlower bool\n}\n\nfunc (r *kRule) String() string {\n\tif r.isDefault {\n\t\treturn \"DEFAULT\"\n\t}\n\ts := fmt.Sprintf(\"RULE:[%d:%s]\", r.comps, r.format)\n\tif r.match != nil {\n\t\ts += fmt.Sprintf(\"(%s)\", r.match)\n\t}\n\tif r.fromPattern != nil {\n\t\ts += fmt.Sprintf(\"s/%s/%s/\", r.fromPattern, r.toPattern)\n\t\tif r.repeat {\n\t\t\ts += \"g\"\n\t\t}\n\t}\n\tif r.lower {\n\t\ts += \"/L\"\n\t}\n\treturn s\n}\n\nfunc (r *kRule) replaceParameters(params []string) string {\n\treturn paramPattern.ReplaceAllStringFunc(r.format, func(s string) string {\n\t\tm := paramPattern.FindStringSubmatchIndex(s)\n\t\ti, _ := strconv.Atoi(s[m[2]+1:])\n\t\tif i >= len(params) {\n\t\t\tlogger.Warnf(\"invalid param %s\", s)\n\t\t\treturn s\n\t\t}\n\t\treturn s[:m[2]] + params[i]\n\t})\n}\n\nfunc (r *kRule) replaceSubs(base string) string {\n\tif r.fromPattern == nil {\n\t\treturn base\n\t}\n\tif r.repeat {\n\t\treturn r.fromPattern.ReplaceAllString(base, r.toPattern)\n\t}\n\tm := r.fromPattern.FindStringIndex(base)\n\tif m != nil {\n\t\treturn base[:m[0]] + r.toPattern + base[m[1]:]\n\t}\n\treturn base\n}\n\nfunc (r *kRule) apply(param []string, mechanism string, realm string) string {\n\tvar result string\n\tif r.isDefault {\n\t\tif realm == \"\" || param[0] == realm {\n\t\t\tresult = param[1]\n\t\t}\n\t} else if r.comps+1 == len(param) {\n\t\tbase := r.replaceParameters(param)\n\t\tif r.match == nil || r.match.MatchString(base) {\n\t\t\tresult = r.replaceSubs(base)\n\t\t}\n\t}\n\tif mechanism == mechanismHadoop && noSimplePattern.FindString(result) != \"\" {\n\t\treturn \"\"\n\t}\n\tif r.lower {\n\t\tresult = strings.ToLower(result)\n\t}\n\treturn result\n}\n\nfunc parseRule(rule string) *kRule {\n\trule = strings.TrimSpace(rule)\n\tif rule == \"DEFAULT\" {\n\t\treturn &kRule{isDefault: true}\n\t}\n\tvar r kRule\n\tm := ruleParser.FindStringSubmatch(rule)\n\tif m == nil {\n\t\treturn nil\n\t}\n\tr.comps, _ = strconv.Atoi(m[2])\n\tr.format = m[3]\n\tvar err error\n\tr.match, err = regexp.Compile(m[5])\n\tif err != nil {\n\t\tlogger.Warnf(\"compile %s: %s\", m[5], err)\n\t\treturn nil\n\t}\n\tr.fromPattern, err = regexp.Compile(m[7])\n\tif err != nil {\n\t\tlogger.Warnf(\"compile %s: %s\", m[7], err)\n\t\treturn nil\n\t}\n\tr.toPattern = m[8]\n\tr.repeat = m[9] == \"g\"\n\tr.lower = m[10] == \"L\"\n\treturn &r\n}\n\ntype kerberosRules struct {\n\tmechanism string\n\trealm string\n\trules []*kRule\n}\n\nfunc newkerberosRules(mechanism string, realm string, rules []string) *kerberosRules {\n\tif mechanism == \"\" {\n\t\tmechanism = mechanismHadoop\n\t}\n\tvar rs []*kRule\n\tfor _, rule := range rules {\n\t\trs = append(rs, parseRule(rule))\n\t}\n\treturn &kerberosRules{mechanism, realm, rs}\n}\n\nfunc (r *kerberosRules) getShortName(full string) string {\n\tservice, host, realm := parseFullName(full)\n\tvar param []string\n\tif host == \"\" {\n\t\tif realm == \"\" {\n\t\t\treturn service\n\t\t}\n\t\tparam = []string{realm, service}\n\t} else {\n\t\tparam = []string{realm, service, host}\n\t}\n\tif r.rules == nil {\n\t\tr.rules = append(r.rules, &kRule{isDefault: true})\n\t}\n\tfor _, rule := range r.rules {\n\t\tshort := rule.apply(param, r.mechanism, r.realm)\n\t\tif short != \"\" {\n\t\t\treturn short\n\t\t}\n\t}\n\tif r.mechanism == mechanismHadoop {\n\t\treturn \"\"\n\t}\n\treturn full\n}\n\nfunc parseFullName(full string) (string, string, string) {\n\tm := namePattern.FindStringSubmatch(full)\n\tif m == nil || m[0] != full {\n\t\treturn \"\", \"\", \"\"\n\t}\n\treturn m[1], m[3], m[5]\n}\n\ntype token struct {\n\tUser string\n\tRenewer string\n\tPassword string\n\tIssued int64\n\tExpire int64\n}\n\ntype hostParam struct {\n\tallAllowed bool\n\tcidr []*net.IPNet\n\taddrs map[string]bool\n}\ntype proxyParam struct {\n\tusers []string\n\tgroups []string\n\thosts *hostParam\n}\n\ntype volParams struct {\n\tm meta.Meta\n\tkeytab []byte\n\trenew int64\n\tlife int64\n\tsuperuser string\n\tsupergroup string\n\trules *kerberosRules\n\tproxies map[string]*proxyParam\n}\n\nfunc (vol *volParams) parse(kind, key, value string) {\n\tif vol.rules == nil {\n\t\tvol.rules = newkerberosRules(mechanismHadoop, \"\", nil)\n\t}\n\tswitch kind {\n\tcase \"keytab\":\n\t\tkt, err := base64.StdEncoding.DecodeString(value)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"decode keytab failed: %s\", err)\n\t\t} else {\n\t\t\tvol.keytab = kt\n\t\t}\n\tcase \"life\":\n\t\tperiod, err := strconv.ParseInt(value, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"can not parse %s as int: %s\", value, err)\n\t\t} else {\n\t\t\tvol.life = period\n\t\t}\n\tcase \"renew\":\n\t\tperiod, err := strconv.ParseInt(value, 10, 64)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"can not parse %s as int: %s\", value, err)\n\t\t} else {\n\t\t\tvol.renew = period\n\t\t}\n\tcase \"superuser\":\n\t\tvol.superuser = value\n\tcase \"supergroup\":\n\t\tvol.supergroup = value\n\tcase \"mechanism\":\n\t\tvalue = strings.ToLower(value)\n\t\tif value != mechanismHadoop && value != mechanismMIT {\n\t\t\tlogger.Errorf(\"invalid mechanism: %s\", value)\n\t\t} else {\n\t\t\tvol.rules.mechanism = value\n\t\t}\n\tcase \"realm\":\n\t\tvol.rules.realm = value\n\tcase \"rule\":\n\t\trule := parseRule(value)\n\t\tif rule != nil {\n\t\t\tvol.rules.rules = append(vol.rules.rules, rule)\n\t\t} else {\n\t\t\tlogger.Errorf(\"invalid kerberos rule: %s\", value)\n\t\t}\n\tdefault:\n\t\tsplit := strings.Split(key, \".\")\n\t\tif len(split) < 4 || split[1] != \"proxy\" {\n\t\t\tlogger.Warnf(\"invalid key: %s\", key)\n\t\t\treturn\n\t\t}\n\t\tuser := split[2]\n\t\tproxy := vol.proxies[user]\n\t\tif proxy == nil {\n\t\t\tproxy = &proxyParam{hosts: &hostParam{}}\n\t\t\tvol.proxies[user] = proxy\n\t\t}\n\t\tswitch kind {\n\t\tcase \"users\":\n\t\t\tproxy.users = strings.Split(value, \",\")\n\t\t\tfor i := range proxy.users {\n\t\t\t\tproxy.users[i] = strings.TrimSpace(proxy.users[i])\n\t\t\t}\n\t\tcase \"groups\":\n\t\t\tproxy.groups = strings.Split(value, \",\")\n\t\t\tfor i := range proxy.groups {\n\t\t\t\tproxy.groups[i] = strings.TrimSpace(proxy.groups[i])\n\t\t\t}\n\t\tcase \"hosts\":\n\t\t\tm := proxy.hosts\n\t\t\tif strings.Contains(value, \"*\") {\n\t\t\t\tm.allAllowed = true\n\t\t\t} else {\n\t\t\t\tm.addrs = make(map[string]bool)\n\t\t\t\tfor _, v := range strings.Split(value, \",\") {\n\t\t\t\t\tif strings.Contains(v, \"/\") {\n\t\t\t\t\t\t// ip range\n\t\t\t\t\t\t_, ipnet, err := net.ParseCIDR(v)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tlogger.Errorf(\"wrong ip range %s: %s\", v, err)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tm.cidr = append(m.cidr, ipnet)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tm.addrs[v] = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tlogger.Errorf(\"invalid key: %s\", key)\n\t\t}\n\t}\n}\n\nfunc (vol *volParams) canProxy(realUser, user, group, ips, hostname string) bool {\n\tif realUser == \"\" || realUser == user {\n\t\treturn true\n\t}\n\tif !vol.isUserGroupAllowed(realUser, user, group) {\n\t\tlogger.Errorf(\"user: %s is not allowed to impersonate %s\", realUser, user)\n\t\treturn false\n\t}\n\tif !vol.isHostAllowed(realUser, ips, hostname) {\n\t\tlogger.Errorf(\"user: %s is not allowed to impersonate %s on %s\", realUser, user, hostname)\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc (vol *volParams) isUserGroupAllowed(realUser, user, groups string) bool {\n\tproxy := vol.proxies[realUser]\n\tif proxy == nil {\n\t\treturn false\n\t}\n\tfor _, u := range proxy.users {\n\t\tif u == \"*\" || u == user {\n\t\t\treturn true\n\t\t}\n\t}\n\tfor _, group := range strings.Split(groups, \",\") {\n\t\tfor _, ag := range proxy.groups {\n\t\t\tif ag == \"*\" || ag == group {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (vol *volParams) isHostAllowed(realUser, ips, hostname string) bool {\n\tproxy := vol.proxies[realUser]\n\tif proxy == nil {\n\t\treturn false\n\t}\n\tm := proxy.hosts\n\tif m.allAllowed {\n\t\treturn true\n\t}\n\tif m.addrs[hostname] {\n\t\treturn true\n\t}\n\tfor _, ip := range strings.Split(ips, \",\") {\n\t\tif m.addrs[ip] {\n\t\t\treturn true\n\t\t}\n\t\tfor _, ipNet := range m.cidr {\n\t\t\tif net.ParseIP(ip) != nil && ipNet.Contains(net.ParseIP(ip)) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\ntype kerberos struct {\n\tvols map[string]*volParams\n\tmu sync.Mutex\n}\n\nfunc (k *kerberos) getVol(volname string) *volParams {\n\tk.mu.Lock()\n\tdefer k.mu.Unlock()\n\treturn k.vols[volname]\n}\n\nfunc (k *kerberos) auth(volname, user, realUser, group, ips, hostname string, reqBytes []byte) syscall.Errno {\n\tkrb5Token := spnego.KRB5Token{}\n\terr := krb5Token.Unmarshal(reqBytes)\n\treq := krb5Token.APReq\n\tif err != nil {\n\t\tlogger.Errorf(\"invalid AP_REQ: %s\", err)\n\t\treturn syscall.EINVAL\n\t}\n\tvol := k.getVol(volname)\n\tif vol == nil || vol.keytab == nil {\n\t\tlogger.Errorf(\"server keytab for %s not setted\", volname)\n\t\treturn syscall.ENODATA\n\t}\n\tkt := new(keytab.Keytab)\n\terr = kt.Unmarshal(vol.keytab)\n\tif err != nil {\n\t\tlogger.Errorf(\"unmarshal keytab: %s\", err)\n\t\treturn syscall.EINVAL\n\t}\n\ts := service.NewSettings(kt, service.DecodePAC(false))\n\tok, creds, err := service.VerifyAPREQ(&req, s)\n\tif err != nil {\n\t\tlogger.Errorf(\"verify: %s\", err)\n\t\treturn syscall.EINVAL\n\t} else if !ok {\n\t\treturn syscall.EACCES\n\t}\n\n\tprincipal := fmt.Sprintf(\"%s@%s\", creds.UserName(), creds.Realm())\n\tauthedUser := vol.rules.getShortName(principal)\n\tif authedUser == \"\" {\n\t\tlogger.Warnf(\"no rule for principal %s\", principal)\n\t\treturn syscall.EINVAL\n\t}\n\n\tif realUser == \"\" {\n\t\tif user == authedUser {\n\t\t\treturn 0\n\t\t}\n\t} else {\n\t\tif realUser == authedUser && vol.canProxy(realUser, user, group, ips, hostname) {\n\t\t\treturn 0\n\t\t}\n\t}\n\tlogger.Warnf(\"auth failed, principal: %s, authedUser: %s, user: %s, realUser: %s\", principal, authedUser, user, realUser)\n\treturn syscall.EACCES\n}\n\nfunc (k *kerberos) issue(ctx meta.Context, m meta.Meta, volname, user, renewer string) (uint32, *token, syscall.Errno) {\n\tvol := k.getVol(volname)\n\tif vol == nil {\n\t\treturn 0, nil, syscall.EINVAL\n\t}\n\tnow := time.Now()\n\tt := &token{\n\t\tUser: user,\n\t\tRenewer: renewer,\n\t\tIssued: now.Unix(),\n\t\tExpire: now.Unix() + vol.renew,\n\t}\n\tpasswd := make([]byte, 20)\n\t_, _ = io.ReadFull(rand.Reader, passwd)\n\tt.Password = hex.EncodeToString(passwd)\n\tid, eno := k.storeToken(ctx, m, t)\n\tif eno != 0 {\n\t\treturn 0, nil, eno\n\t}\n\treturn id, t, 0\n}\n\nfunc (k *kerberos) check(ctx meta.Context, m meta.Meta, volname, user string, id uint32, password string) syscall.Errno {\n\tt, eno := k.loadToken(ctx, m, id)\n\tif eno != 0 {\n\t\treturn eno\n\t}\n\tnow := time.Now().Unix()\n\tif now > t.Expire {\n\t\tlogger.Warnf(\"token %d expired\", id)\n\t\treturn syscall.EINVAL\n\t}\n\tif password != t.Password || user != t.User {\n\t\tlogger.Warnf(\"token %d invalid user or password\", id)\n\t\treturn syscall.EACCES\n\t}\n\treturn 0\n}\n\nfunc (k *kerberos) renew(ctx meta.Context, m meta.Meta, volname, renewer string, id uint32, password string) (int64, syscall.Errno) {\n\tt, eno := k.loadToken(ctx, m, id)\n\tif eno != 0 {\n\t\treturn 0, eno\n\t}\n\tif password != t.Password || renewer != t.Renewer {\n\t\treturn 0, syscall.EACCES\n\t}\n\tnow := time.Now().Unix()\n\tif now > t.Expire {\n\t\tlogger.Warnf(\"token %d expired for renew\", id)\n\t\treturn 0, syscall.EINVAL\n\t}\n\tvol := k.getVol(volname)\n\tt.Expire = min(t.Issued+vol.life, t.Expire+vol.renew)\n\teno = k.updateToken(ctx, m, id, t)\n\tif eno != 0 {\n\t\treturn 0, eno\n\t}\n\treturn t.Expire, 0\n}\n\nfunc (k *kerberos) storeToken(ctx meta.Context, m meta.Meta, t *token) (id uint32, st syscall.Errno) {\n\tmarshal, err := json.Marshal(t)\n\tif err != nil {\n\t\tlogger.Errorf(\"marshal token: %s\", err)\n\t\treturn 0, syscall.EINVAL\n\t}\n\treturn m.StoreToken(ctx, marshal)\n}\n\nfunc (k *kerberos) updateToken(ctx meta.Context, m meta.Meta, id uint32, t *token) syscall.Errno {\n\tmarshal, err := json.Marshal(t)\n\tif err != nil {\n\t\tlogger.Errorf(\"marshal token: %s\", err)\n\t\treturn syscall.EINVAL\n\t}\n\treturn m.UpdateToken(ctx, id, marshal)\n}\n\nfunc (k *kerberos) loadToken(ctx meta.Context, m meta.Meta, id uint32) (*token, syscall.Errno) {\n\ttb, errno := m.LoadToken(ctx, id)\n\tif errno != 0 {\n\t\treturn nil, errno\n\t}\n\tt := &token{}\n\terr := json.Unmarshal(tb, t)\n\tif err != nil {\n\t\tlogger.Errorf(\"unmarshal token %d: %s\", id, err)\n\t\treturn nil, syscall.EINVAL\n\t}\n\treturn t, 0\n}\n\nfunc (k *kerberos) cancelToken(ctx meta.Context, m meta.Meta, user string, id uint32, password string) syscall.Errno {\n\tt, eno := k.loadToken(ctx, m, id)\n\tif eno != 0 {\n\t\treturn eno\n\t}\n\tif password != t.Password || user != t.Renewer && user != t.User {\n\t\treturn syscall.EACCES\n\t}\n\treturn m.DeleteTokens(ctx, []uint32{id})\n}\n\nfunc (k *kerberos) cleanupTokens() {\n\tvar metas []meta.Meta\n\tk.mu.Lock()\n\tfor _, vol := range k.vols {\n\t\tmetas = append(metas, vol.m)\n\t}\n\tk.mu.Unlock()\n\tfor _, m := range metas {\n\t\tctx := meta.Background()\n\t\ttokens, eno := m.ListTokens(ctx)\n\t\tif eno != 0 {\n\t\t\tlogger.Errorf(\"list tokens: %s\", eno)\n\t\t\treturn\n\t\t}\n\t\tvar todelete []uint32\n\t\tnow := time.Now().Unix()\n\t\tfor id, data := range tokens {\n\t\t\tt := &token{}\n\t\t\terr := json.Unmarshal(data, t)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warnf(\"unmarshal token %d: %s\", id, err)\n\t\t\t}\n\t\t\tif t.Expire <= now {\n\t\t\t\ttodelete = append(todelete, id)\n\t\t\t}\n\t\t}\n\t\tif len(todelete) == 0 {\n\t\t\treturn\n\t\t}\n\t\tlogger.Infof(\"cleaning up %d expired tokens\", len(todelete))\n\t\teno = m.DeleteTokens(ctx, todelete)\n\t\tif eno != 0 {\n\t\t\tlogger.Errorf(\"delete tokens: %s\", eno)\n\t\t}\n\t}\n}\n\nfunc (k *kerberos) loadConf(name, content string, jfs *fs.FileSystem) {\n\tvol := &volParams{\n\t\tm: jfs.Meta(),\n\t\tlife: defaultLife,\n\t\trenew: defaultRenew,\n\t\tproxies: make(map[string]*proxyParam),\n\t}\n\tscanner := bufio.NewScanner(strings.NewReader(content))\n\tfor scanner.Scan() {\n\t\tline := scanner.Text()\n\t\tidx := strings.Index(line, \"#\")\n\t\tif idx >= 0 {\n\t\t\tline = line[:idx]\n\t\t}\n\t\tline = strings.TrimSpace(line)\n\t\tif len(line) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tfields := strings.SplitN(line, \"=\", 2)\n\t\tif len(fields) != 2 {\n\t\t\tlogger.Warningf(\"bad line: %s\", line)\n\t\t\tcontinue\n\t\t}\n\t\tkey := strings.TrimSpace(fields[0])\n\t\tvalue := strings.TrimSpace(fields[1])\n\t\tsplit := strings.Split(key, \".\")\n\t\tif len(split) < 2 {\n\t\t\tlogger.Warningf(\"bad line: %s\", line)\n\t\t\tcontinue\n\t\t}\n\t\tkeySuffix := split[len(split)-1]\n\t\tvolName := split[0]\n\t\tif volName != name {\n\t\t\tcontinue\n\t\t}\n\t\tvol.parse(keySuffix, key, value)\n\t}\n\tjfs.Superuser = vol.superuser\n\tjfs.Supergroup = vol.supergroup\n\tk.mu.Lock()\n\tk.vols[name] = vol\n\tk.mu.Unlock()\n}\n\nfunc (k *kerberos) init() int {\n\tk.vols = make(map[string]*volParams)\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(10 * time.Minute)\n\t\t\tk.cleanupTokens()\n\t\t}\n\t}()\n\treturn 0\n}\n\nvar kerb = kerberos{}\n" }, { "path": "sdk/java/libjfs/main.go", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage main\n\n// #cgo linux LDFLAGS: -ldl\n// #cgo linux CFLAGS: -Wno-discarded-qualifiers -D_GNU_SOURCE\n// #include \n// #include \n// #include \n// #include \n// #include \n// #include \n// #include \n// void jfs_callback(const char *msg);\n/*\n#include \n\ntypedef struct {\n\tuint64_t inode;\n\tuint32_t mode;\n\tuint32_t uid;\n\tuint32_t gid;\n\tuint32_t atime;\n\tuint32_t mtime;\n\tuint32_t ctime;\n\tuint32_t nlink;\n\tuint64_t length;\n} fileInfo;\n*/\nimport \"C\"\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t_ \"net/http/pprof\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime/debug\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\t\"unsafe\"\n\n\t\"github.com/juicedata/juicefs/cmd\"\n\t\"github.com/juicedata/juicefs/pkg/acl\"\n\t\"github.com/juicedata/juicefs/pkg/chunk\"\n\t\"github.com/juicedata/juicefs/pkg/fs\"\n\t\"github.com/juicedata/juicefs/pkg/meta\"\n\t\"github.com/juicedata/juicefs/pkg/metric\"\n\t\"github.com/juicedata/juicefs/pkg/object\"\n\t\"github.com/juicedata/juicefs/pkg/usage\"\n\t\"github.com/juicedata/juicefs/pkg/utils\"\n\t\"github.com/juicedata/juicefs/pkg/version\"\n\t\"github.com/juicedata/juicefs/pkg/vfs\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/collectors\"\n\t\"github.com/prometheus/client_golang/prometheus/push\"\n\t\"github.com/sirupsen/logrus\"\n)\n\nvar (\n\tfilesLock sync.Mutex\n\topenFiles = make(map[int32]*fwrapper)\n\tnextHandle = int32(1)\n\n\tfslock sync.Mutex\n\thandlers = make(map[int64]*wrapper)\n\tnextFsHandle int64 = 0\n\tactivefs = make(map[fsKey][]*wrapper)\n\tlogger = utils.GetLogger(\"juicefs\")\n\tbOnce sync.Once\n\tbridges []*Bridge\n\tpOnce sync.Once\n\tpushers []*push.Pusher\n\trOnce sync.Once\n\tremoteWriters []*RemoteWriter\n\n\tuserGroupCache = make(map[string]map[string][]string) // name -> (user -> groups)\n\n\tformats = make(map[string]*meta.Format)\n\n\tkerbOnce = sync.Once{}\n\tsuperuserChangedCb = make(map[string]struct{})\n\n\tMaxDeletes = meta.RmrDefaultThreads\n\tcaller = CALLER_JAVA\n)\n\nconst (\n\tCALLER_JAVA = iota\n\tCALLER_PYTHON\n)\n\nconst (\n\tBEHAVIOR_HADOOP = \"Hadoop\"\n)\n\nconst (\n\tEPERM = -0x01\n\tENOENT = -0x02\n\tEINTR = -0x04\n\tEIO = -0x05\n\tEACCES = -0x0d\n\tEEXIST = -0x11\n\tENOTDIR = -0x14\n\tEISDIR = -0x15\n\tEINVAL = -0x16\n\tENOSPC = -0x1c\n\tEDQUOT = -0x45\n\tEROFS = -0x1e\n\tENOTEMPTY = -0x27\n\tENODATA = -0x3d\n\tENOTSUP = -0x5f\n)\n\nfunc errno(err error) int32 {\n\tif err == nil {\n\t\treturn 0\n\t}\n\teno, ok := err.(syscall.Errno)\n\tif !ok {\n\t\treturn EIO\n\t}\n\tif eno == 0 {\n\t\treturn 0\n\t}\n\t// Use the errno in Linux for all the OS\n\tswitch eno {\n\tcase syscall.EPERM:\n\t\treturn EPERM\n\tcase syscall.ENOENT:\n\t\treturn ENOENT\n\tcase syscall.EINTR:\n\t\treturn EINTR\n\tcase syscall.EIO:\n\t\treturn EIO\n\tcase syscall.EACCES:\n\t\treturn EACCES\n\tcase syscall.EEXIST:\n\t\treturn EEXIST\n\tcase syscall.ENOTDIR:\n\t\treturn ENOTDIR\n\tcase syscall.EISDIR:\n\t\treturn EISDIR\n\tcase syscall.EINVAL:\n\t\treturn EINVAL\n\tcase syscall.ENOSPC:\n\t\treturn ENOSPC\n\tcase syscall.EDQUOT:\n\t\treturn EDQUOT\n\tcase syscall.EROFS:\n\t\treturn EROFS\n\tcase syscall.ENOTEMPTY:\n\t\treturn ENOTEMPTY\n\tcase syscall.ENODATA:\n\t\treturn ENODATA\n\tcase syscall.ENOTSUP:\n\t\treturn ENOTSUP\n\tdefault:\n\t\tlogger.Warnf(\"unknown errno %d: %s\", eno, err)\n\t\treturn -int32(eno)\n\t}\n}\n\ntype fsKey struct {\n\tname string\n\tconf javaConf\n}\n\ntype wrapper struct {\n\t*fs.FileSystem\n\tvolname string\n\tctx meta.Context\n\tm *mapping\n\tuser string\n\tsuperuser string\n\tsupergroup string\n\tconf javaConf\n}\n\ntype logWriter struct {\n\tbuf chan string\n}\n\nfunc (w *logWriter) Write(p []byte) (int, error) {\n\tselect {\n\tcase w.buf <- string(p):\n\t\t_, _ = os.Stderr.Write(p)\n\t\treturn len(p), nil\n\tdefault:\n\t\treturn os.Stderr.Write(p)\n\t}\n}\n\nfunc newLogWriter() *logWriter {\n\tw := &logWriter{\n\t\tbuf: make(chan string, 10),\n\t}\n\tgo func() {\n\t\tfor l := range w.buf {\n\t\t\tcmsg := C.CString(l)\n\t\t\tC.jfs_callback(cmsg)\n\t\t\tC.free(unsafe.Pointer(cmsg))\n\t\t}\n\t}()\n\treturn w\n}\n\n//export jfs_set_logger\nfunc jfs_set_logger(cb unsafe.Pointer) {\n\tutils.DisableLogColor()\n\tif cb != nil {\n\t\tutils.SetOutput(newLogWriter())\n\t} else {\n\t\tutils.SetOutput(os.Stderr)\n\t}\n}\n\nfunc (w *wrapper) withPid(pid int64) meta.Context {\n\t// mapping Java Thread ID to global one\n\tctx := meta.NewContext(w.ctx.Pid()*1000+uint32(pid), w.ctx.Uid(), w.ctx.Gids())\n\tif caller == CALLER_JAVA {\n\t\tctx = ctx.WithValue(meta.CtxKey(\"behavior\"), BEHAVIOR_HADOOP)\n\t}\n\treturn ctx\n}\n\nfunc (w *wrapper) getSuperUser() string {\n\tif w.Superuser != \"\" {\n\t\treturn w.Superuser\n\t}\n\treturn w.superuser\n}\n\nfunc (w *wrapper) getSuperGroup() string {\n\tif w.Supergroup != \"\" {\n\t\treturn w.Supergroup\n\t}\n\treturn w.supergroup\n}\n\nfunc (w *wrapper) isSuperuser(name string, groups []string) bool {\n\tif name == w.getSuperUser() || w.conf.SuperFS {\n\t\treturn true\n\t}\n\tsg := w.getSuperGroup()\n\tfor _, g := range groups {\n\t\tif g == sg {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (w *wrapper) lookupUid(name string) uint32 {\n\tif name == w.superuser {\n\t\treturn 0\n\t}\n\treturn uint32(w.m.lookupUser(name))\n}\n\nfunc (w *wrapper) lookupGid(group string) uint32 {\n\tif group == w.supergroup {\n\t\treturn 0\n\t}\n\treturn uint32(w.m.lookupGroup(group))\n}\n\nfunc (w *wrapper) lookupGids(groups []string) []uint32 {\n\tvar gids []uint32\n\tfor _, g := range groups {\n\t\tgids = append(gids, w.lookupGid(g))\n\t}\n\treturn gids\n}\n\nfunc (w *wrapper) uid2name(uid uint32) string {\n\tname := w.superuser\n\tif uid > 0 {\n\t\tname = w.m.lookupUserID(uid)\n\t}\n\treturn name\n}\n\nfunc (w *wrapper) gid2name(gid uint32) string {\n\tgroup := w.supergroup\n\tif gid > 0 {\n\t\tgroup = w.m.lookupGroupID(gid)\n\t}\n\treturn group\n}\n\ntype fwrapper struct {\n\t*fs.File\n\tw *wrapper\n}\n\nfunc nextFileHandle(f *fs.File, w *wrapper) int32 {\n\tfilesLock.Lock()\n\tdefer filesLock.Unlock()\n\tfor i := nextHandle; ; i++ {\n\t\tif _, ok := openFiles[i]; !ok {\n\t\t\topenFiles[i] = &fwrapper{f, w}\n\t\t\tnextHandle = i + 1\n\t\t\treturn i\n\t\t}\n\t}\n}\n\nfunc freeHandle(fd int32) {\n\tfilesLock.Lock()\n\tdefer filesLock.Unlock()\n\tf := openFiles[fd]\n\tif f != nil {\n\t\tdelete(openFiles, fd)\n\t}\n}\n\ntype javaConf struct {\n\tMetaURL string `json:\"meta\"`\n\tBucket string `json:\"bucket\"`\n\tStorageClass string `json:\"storageClass\"`\n\tReadOnly bool `json:\"readOnly\"`\n\tNoSession bool `json:\"noSession\"`\n\tNoBGJob bool `json:\"noBGJob\"`\n\tOpenCache string `json:\"openCache\"`\n\tBackupMeta string `json:\"backupMeta\"`\n\tBackupSkipTrash bool `json:\"backupSkipTrash\"`\n\tHeartbeat string `json:\"heartbeat\"`\n\tCacheDir string `json:\"cacheDir\"`\n\tCacheSize string `json:\"cacheSize\"`\n\tCacheItems int64 `json:\"cacheItems\"`\n\tFreeSpace string `json:\"freeSpace\"`\n\tAutoCreate bool `json:\"autoCreate\"`\n\tCacheFullBlock bool `json:\"cacheFullBlock\"`\n\tCacheChecksum string `json:\"cacheChecksum\"`\n\tCacheEviction string `json:\"cacheEviction\"`\n\tCacheScanInterval string `json:\"cacheScanInterval\"`\n\tCacheExpire string `json:\"cacheExpire\"`\n\tWriteback bool `json:\"writeback\"`\n\tMemorySize string `json:\"memorySize\"`\n\tPrefetch int `json:\"prefetch\"`\n\tReadahead string `json:\"readahead\"`\n\tUploadLimit string `json:\"uploadLimit\"`\n\tDownloadLimit string `json:\"downloadLimit\"`\n\tMaxUploads int `json:\"maxUploads\"`\n\tMaxDownloads int `json:\"maxDownloads\"`\n\tMaxDeletes int `json:\"maxDeletes\"`\n\tSkipDirNlink int `json:\"skipDirNlink\"`\n\tSkipDirMtime string `json:\"skipDirMtime\"`\n\tIORetries int `json:\"ioRetries\"`\n\tGetTimeout string `json:\"getTimeout\"`\n\tPutTimeout string `json:\"putTimeout\"`\n\tFastResolve bool `json:\"fastResolve\"`\n\tAttrTimeout string `json:\"attrTimeout\"`\n\tEntryTimeout string `json:\"entryTimeout\"`\n\tDirEntryTimeout string `json:\"dirEntryTimeout\"`\n\tDebug bool `json:\"debug\"`\n\tNoUsageReport bool `json:\"noUsageReport\"`\n\tAccessLog string `json:\"accessLog\"`\n\tPushGateway string `json:\"pushGateway\"`\n\tPushInterval string `json:\"pushInterval\"`\n\tPushAuth string `json:\"pushAuth\"`\n\tPushLabels string `json:\"pushLabels\"`\n\tPushGraphite string `json:\"pushGraphite\"`\n\tPushRemoteWrite string `json:\"pushRemoteWrite\"`\n\tPushRemoteWriteAuth string `json:\"pushRemoteWriteAuth\"`\n\tCaller int `json:\"caller\"`\n\tSubdir string `json:\"subdir\"`\n\n\tAuthMethod string `json:\"authMethod,omitempty\"`\n\tRealUser string `json:\"realUser,omitempty\"`\n\n\tSuperFS bool `json:\"superFs,omitempty\"`\n}\n\nfunc cleanConf(conf javaConf) javaConf {\n\tconf.AuthMethod = \"\"\n\tconf.RealUser = \"\"\n\tconf.SuperFS = false\n\treturn conf\n}\n\nfunc getOrCreate(name, user, groups, superuser, supergroup string, conf javaConf, f func() *fs.FileSystem) int64 {\n\tfslock.Lock()\n\tdefer fslock.Unlock()\n\tkey := fsKey{name: name, conf: cleanConf(conf)}\n\tws := activefs[key]\n\tvar jfs *fs.FileSystem\n\tvar m *mapping\n\tif len(ws) > 0 {\n\t\tjfs = ws[0].FileSystem\n\t\tm = ws[0].m\n\t} else {\n\t\tm = newMapping(name)\n\t\tjfs = f()\n\t\tif jfs == nil {\n\t\t\treturn 0\n\t\t}\n\t\tswitch jfs.Meta().Name() {\n\t\tcase \"mysql\", \"postgres\", \"sqlite3\":\n\t\t\tm.mask = 0x7FFFFFFF // limit generated uid to int32\n\t\t}\n\t\tlogger.Infof(\"JuiceFileSystem created for user:%s groups:%s\", user, groups)\n\t}\n\tw := &wrapper{jfs, name, nil, m, user, superuser, supergroup, conf}\n\tif formats[name] != nil && formats[name].KerbConf != \"\" {\n\t\tif _, ok := superuserChangedCb[name]; !ok {\n\t\t\tjfs.Meta().OnReload(func(format *meta.Format) {\n\t\t\t\tkerb.loadConf(name, format.KerbConf, jfs)\n\t\t\t\tupdateAllCtx(name, user, groups)\n\t\t\t})\n\t\t\tsuperuserChangedCb[name] = struct{}{}\n\t\t}\n\t}\n\tactivefs[key] = append(ws, w)\n\tupdateAllCtx(name, user, groups)\n\tnextFsHandle = nextFsHandle + 1\n\thandlers[nextFsHandle] = w\n\treturn nextFsHandle\n}\n\nfunc updateAllCtx(name string, user, groups string) {\n\tvar ws []*wrapper\n\tfor k, v := range activefs {\n\t\tif k.name == name {\n\t\t\tws = append(ws, v...)\n\t\t}\n\t}\n\tif len(ws) > 0 {\n\t\tfor _, w := range ws {\n\t\t\tvar gs []string\n\t\t\tif userGroupCache[name] != nil {\n\t\t\t\tgs = userGroupCache[name][user]\n\t\t\t}\n\t\t\tif gs == nil {\n\t\t\t\tgs = strings.Split(groups, \",\")\n\t\t\t}\n\t\t\tlogger.Debugf(\"update groups of %s to %s\", user, strings.Join(gs, \",\"))\n\t\t\tupdateCtx(w, gs)\n\t\t}\n\t}\n}\n\nfunc push2Gateway(pushGatewayAddr, pushAuth string, pushInterVal time.Duration, registry *prometheus.Registry, commonLabels map[string]string) {\n\tpusher := push.New(pushGatewayAddr, \"juicefs\").Gatherer(registry)\n\tfor k, v := range commonLabels {\n\t\tpusher.Grouping(k, v)\n\t}\n\tif pushAuth != \"\" {\n\t\tif strings.Contains(pushAuth, \":\") {\n\t\t\tparts := strings.Split(pushAuth, \":\")\n\t\t\tpusher.BasicAuth(parts[0], parts[1])\n\t\t}\n\t}\n\tpusher.Client(&http.Client{Timeout: 2 * time.Second})\n\tpushers = append(pushers, pusher)\n\n\tpOnce.Do(func() {\n\t\tgo func() {\n\t\t\tfor range time.NewTicker(pushInterVal).C {\n\t\t\t\tfor _, pusher := range pushers {\n\t\t\t\t\tif err := pusher.Push(); err != nil {\n\t\t\t\t\t\tlogger.Warnf(\"error pushing to PushGateway: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t})\n}\n\nfunc push2RemoteWrite(remoteWrite string, pushRemoteWriteAuth string, pushInterVal time.Duration, registry *prometheus.Registry, commonLabels map[string]string) {\n\twriter, err := NewRemoteWriter(&RemoteWriteConfig{\n\t\tURL: remoteWrite,\n\t\tGatherer: registry,\n\t\tAuth: pushRemoteWriteAuth,\n\t\tInterval: pushInterVal,\n\t\tTimeout: 2 * time.Second,\n\t\tErrorHandling: ContinueOnError,\n\t\tLogger: logger,\n\t\tCommonLabels: commonLabels,\n\t})\n\tif err != nil {\n\t\tlogger.Warnf(\"NewRemoteWriter error: %s\", err)\n\t\treturn\n\t}\n\tremoteWriters = append(remoteWriters, writer)\n\n\trOnce.Do(func() {\n\t\tgo func() {\n\t\t\tfor range time.NewTicker(pushInterVal).C {\n\t\t\t\tfor _, writer := range remoteWriters {\n\t\t\t\t\tif err := writer.Push(); err != nil {\n\t\t\t\t\t\tlogger.Warnf(\"error pushing to remote write: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t})\n}\n\nfunc push2Graphite(graphite string, pushInterVal time.Duration, registry *prometheus.Registry, commonLabels map[string]string) {\n\tif bridge, err := NewBridge(&Config{\n\t\tURL: graphite,\n\t\tGatherer: registry,\n\t\tUseTags: true,\n\t\tTimeout: 2 * time.Second,\n\t\tErrorHandling: ContinueOnError,\n\t\tLogger: logger,\n\t\tCommonLabels: commonLabels,\n\t}); err != nil {\n\t\tlogger.Warnf(\"NewBridge error:%s\", err)\n\t} else {\n\t\tbridges = append(bridges, bridge)\n\t}\n\n\tbOnce.Do(func() {\n\t\tgo func() {\n\t\t\tfor range time.NewTicker(pushInterVal).C {\n\t\t\t\tfor _, brg := range bridges {\n\t\t\t\t\tif err := brg.Push(); err != nil {\n\t\t\t\t\t\tlogger.Warnf(\"error pushing to Graphite: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t})\n}\n\n//export jfs_init\nfunc jfs_init(credentialPtr uintptr, count int32, cname, cjsonConf, cuser, group, superuser, supergroup *C.char) int64 {\n\tname := C.GoString(cname)\n\tuser := C.GoString(cuser)\n\tdebug.SetGCPercent(50)\n\tobject.UserAgent = \"JuiceFS-SDK \" + version.Version()\n\tvar jConf javaConf\n\terr := json.Unmarshal([]byte(C.GoString(cjsonConf)), &jConf)\n\tif err != nil {\n\t\tif os.Getenv(\"JUICEFS_DEBUG\") != \"\" {\n\t\t\tlogger.Fatalf(\"invalid json: %s\", C.GoString(cjsonConf))\n\t\t} else {\n\t\t\tlogger.Fatalf(\"invalid json\")\n\t\t}\n\t}\n\treturn getOrCreate(name, user, C.GoString(group), C.GoString(superuser), C.GoString(supergroup), jConf, func() *fs.FileSystem {\n\t\tif jConf.Debug || os.Getenv(\"JUICEFS_DEBUG\") != \"\" {\n\t\t\tutils.SetLogLevel(logrus.DebugLevel)\n\t\t\tgo func() {\n\t\t\t\tfor port := 6060; port < 6100; port++ {\n\t\t\t\t\tlogger.Debugf(\"listen at 127.0.0.1:%d\", port)\n\t\t\t\t\t_ = http.ListenAndServe(fmt.Sprintf(\"127.0.0.1:%d\", port), nil)\n\t\t\t\t}\n\t\t\t}()\n\t\t} else if os.Getenv(\"JUICEFS_LOGLEVEL\") != \"\" {\n\t\t\tlevel, err := logrus.ParseLevel(os.Getenv(\"JUICEFS_LOGLEVEL\"))\n\t\t\tif err == nil {\n\t\t\t\tutils.SetLogLevel(level)\n\t\t\t} else {\n\t\t\t\tutils.SetLogLevel(logrus.WarnLevel)\n\t\t\t\tlogger.Errorf(\"JUICEFS_LOGLEVEL: %s\", err)\n\t\t\t}\n\t\t} else {\n\t\t\tutils.SetLogLevel(logrus.WarnLevel)\n\t\t}\n\n\t\tcaller = jConf.Caller\n\t\tif jConf.MaxDeletes > 0 {\n\t\t\tMaxDeletes = jConf.MaxDeletes\n\t\t}\n\n\t\tmetaConf := meta.DefaultConf()\n\t\tmetaConf.Retries = jConf.IORetries\n\t\tmetaConf.MaxDeletes = jConf.MaxDeletes\n\t\tmetaConf.SkipDirNlink = jConf.SkipDirNlink\n\t\tmetaConf.SkipDirMtime = utils.Duration(jConf.SkipDirMtime)\n\t\tmetaConf.ReadOnly = jConf.ReadOnly\n\t\tmetaConf.NoBGJob = jConf.NoBGJob || jConf.NoSession\n\t\tmetaConf.OpenCache = utils.Duration(jConf.OpenCache)\n\t\tmetaConf.Heartbeat = utils.Duration(jConf.Heartbeat)\n\t\tm := meta.NewClient(jConf.MetaURL, metaConf)\n\t\tformat, err := m.Load(true)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"load setting: %s\", err)\n\t\t\treturn nil\n\t\t}\n\t\tformats[name] = format\n\t\tvar registerer prometheus.Registerer\n\t\tvar registry *prometheus.Registry\n\t\tif jConf.PushGateway != \"\" || jConf.PushGraphite != \"\" || jConf.PushRemoteWrite != \"\" || jConf.Caller == CALLER_PYTHON {\n\t\t\tcommonLabels := prometheus.Labels{\"vol_name\": name, \"mp\": \"sdk-\" + strconv.Itoa(os.Getpid())}\n\t\t\tif h, err := os.Hostname(); err == nil {\n\t\t\t\tcommonLabels[\"instance\"] = h\n\t\t\t} else {\n\t\t\t\tlogger.Warnf(\"cannot get hostname: %s\", err)\n\t\t\t}\n\t\t\tif jConf.PushLabels != \"\" {\n\t\t\t\tfor _, kv := range strings.Split(jConf.PushLabels, \";\") {\n\t\t\t\t\tvar splited = strings.Split(kv, \":\")\n\t\t\t\t\tif len(splited) != 2 {\n\t\t\t\t\t\tlogger.Errorf(\"invalid label format: %s\", kv)\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\tif utils.StringContains([]string{\"mp\", \"vol_name\", \"instance\"}, splited[0]) {\n\t\t\t\t\t\tlogger.Warnf(\"overriding reserved label: %s\", splited[0])\n\t\t\t\t\t}\n\t\t\t\t\tcommonLabels[splited[0]] = splited[1]\n\t\t\t\t}\n\t\t\t}\n\t\t\tregistry = prometheus.NewRegistry()\n\t\t\tregisterer = prometheus.WrapRegistererWithPrefix(\"juicefs_\", registry)\n\t\t\tregisterer.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))\n\t\t\tregisterer.MustRegister(collectors.NewGoCollector())\n\n\t\t\tvar interval = utils.Duration(jConf.PushInterval)\n\t\t\tif jConf.PushGraphite != \"\" {\n\t\t\t\tpush2Graphite(jConf.PushGraphite, interval, registry, commonLabels)\n\t\t\t}\n\t\t\tif jConf.PushGateway != \"\" {\n\t\t\t\tpush2Gateway(jConf.PushGateway, jConf.PushAuth, interval, registry, commonLabels)\n\t\t\t}\n\t\t\tif jConf.PushRemoteWrite != \"\" {\n\t\t\t\tpush2RemoteWrite(jConf.PushRemoteWrite, jConf.PushRemoteWriteAuth, interval, registry, commonLabels)\n\t\t\t}\n\t\t\tm.InitMetrics(registerer)\n\t\t\tvfs.InitMetrics(registerer)\n\t\t\tgo metric.UpdateMetrics(registerer)\n\t\t}\n\n\t\tblob, err := cmd.NewReloadableStorage(format, m, func(f *meta.Format) {\n\t\t\tif jConf.Bucket != \"\" {\n\t\t\t\tformat.Bucket = jConf.Bucket\n\t\t\t}\n\t\t\tif jConf.StorageClass != \"\" {\n\t\t\t\tformat.StorageClass = jConf.StorageClass\n\t\t\t}\n\t\t})\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"object storage: %s\", err)\n\t\t\treturn nil\n\t\t}\n\t\tlogger.Infof(\"Data use %s\", blob)\n\n\t\tvar freeSpaceRatio = 0.1\n\t\tif jConf.FreeSpace != \"\" {\n\t\t\tfreeSpaceRatio, _ = strconv.ParseFloat(jConf.FreeSpace, 64)\n\t\t}\n\t\tchunkConf := chunk.Config{\n\t\t\tBlockSize: format.BlockSize * 1024,\n\t\t\tCompress: format.Compression,\n\t\t\tCacheDir: jConf.CacheDir,\n\t\t\tCacheMode: 0644, // all user can read cache\n\t\t\tCacheSize: utils.ParseBytesStr(\"cache-size\", jConf.CacheSize, 'M'),\n\t\t\tCacheItems: jConf.CacheItems,\n\t\t\tFreeSpace: float32(freeSpaceRatio),\n\t\t\tAutoCreate: jConf.AutoCreate,\n\t\t\tCacheFullBlock: jConf.CacheFullBlock,\n\t\t\tCacheChecksum: jConf.CacheChecksum,\n\t\t\tCacheEviction: jConf.CacheEviction,\n\t\t\tCacheScanInterval: utils.Duration(jConf.CacheScanInterval),\n\t\t\tCacheExpire: utils.Duration(jConf.CacheExpire),\n\t\t\tOSCache: true,\n\t\t\tMaxUpload: jConf.MaxUploads,\n\t\t\tMaxDownload: jConf.MaxDownloads,\n\t\t\tMaxRetries: jConf.IORetries,\n\t\t\tUploadLimit: utils.ParseMbpsStr(\"upload-limit\", jConf.UploadLimit) * 1e6 / 8,\n\t\t\tDownloadLimit: utils.ParseMbpsStr(\"download-limit\", jConf.DownloadLimit) * 1e6 / 8,\n\t\t\tPrefetch: jConf.Prefetch,\n\t\t\tWriteback: jConf.Writeback,\n\t\t\tHashPrefix: format.HashPrefix,\n\t\t\tGetTimeout: utils.Duration(jConf.GetTimeout),\n\t\t\tPutTimeout: utils.Duration(jConf.PutTimeout),\n\t\t\tBufferSize: utils.ParseBytesStr(\"memory-size\", jConf.MemorySize, 'M'),\n\t\t\tReadahead: int(utils.ParseBytesStr(\"max-readahead\", jConf.Readahead, 'M')),\n\t\t}\n\t\tif chunkConf.UploadLimit == 0 {\n\t\t\tchunkConf.UploadLimit = format.UploadLimit * 1e6 / 8\n\t\t}\n\t\tif chunkConf.DownloadLimit == 0 {\n\t\t\tchunkConf.DownloadLimit = format.DownloadLimit * 1e6 / 8\n\t\t}\n\t\tchunkConf.SelfCheck(format.UUID)\n\t\tstore := chunk.NewCachedStore(blob, chunkConf, registerer)\n\t\tm.OnMsg(meta.DeleteSlice, func(args ...interface{}) error {\n\t\t\tid := args[0].(uint64)\n\t\t\tlength := args[1].(uint32)\n\t\t\treturn store.Remove(id, int(length))\n\t\t})\n\t\tm.OnMsg(meta.CompactChunk, func(args ...interface{}) error {\n\t\t\tslices := args[0].([]meta.Slice)\n\t\t\tid := args[1].(uint64)\n\t\t\treturn vfs.Compact(chunkConf, store, slices, id)\n\t\t})\n\t\terr = m.NewSession(!jConf.NoSession)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"new session: %s\", err)\n\t\t\treturn nil\n\t\t}\n\t\tm.OnReload(func(fmt *meta.Format) {\n\t\t\tif chunkConf.UploadLimit > 0 {\n\t\t\t\tfmt.UploadLimit = chunkConf.UploadLimit\n\t\t\t}\n\t\t\tif chunkConf.DownloadLimit > 0 {\n\t\t\t\tfmt.DownloadLimit = chunkConf.DownloadLimit\n\t\t\t}\n\t\t\tstore.UpdateLimit(fmt.UploadLimit, fmt.DownloadLimit)\n\t\t})\n\n\t\tconf := &vfs.Config{\n\t\t\tMeta: metaConf,\n\t\t\tFormat: *format,\n\t\t\tChunk: &chunkConf,\n\t\t\tAttrTimeout: utils.Duration(jConf.AttrTimeout),\n\t\t\tEntryTimeout: utils.Duration(jConf.EntryTimeout),\n\t\t\tDirEntryTimeout: utils.Duration(jConf.DirEntryTimeout),\n\t\t\tAccessLog: jConf.AccessLog,\n\t\t\tFastResolve: jConf.FastResolve,\n\t\t\tSubdir: jConf.Subdir,\n\t\t\tBackupMeta: utils.Duration(jConf.BackupMeta),\n\t\t\tBackupSkipTrash: jConf.BackupSkipTrash,\n\t\t}\n\t\tif !jConf.ReadOnly && !jConf.NoSession && !jConf.NoBGJob && conf.BackupMeta > 0 {\n\t\t\tgo vfs.Backup(m, blob, conf.BackupMeta, conf.BackupSkipTrash)\n\t\t}\n\t\tif !jConf.NoUsageReport && !jConf.NoSession {\n\t\t\tgo usage.ReportUsage(m, \"java-sdk \"+version.Version())\n\t\t}\n\t\tjfs, err := fs.NewFileSystem(conf, m, store, registry)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"Initialize failed: %s\", err)\n\t\t\treturn nil\n\t\t}\n\t\tjfs.InitMetrics(registerer)\n\t\tif format.KerbConf != \"\" {\n\t\t\tkerbOnce.Do(func() {\n\t\t\t\tkerb.init()\n\t\t\t})\n\t\t\tkerb.loadConf(name, format.KerbConf, jfs)\n\t\t\tvar credential []byte\n\t\t\tif credentialPtr == 0 {\n\t\t\t\tlogger.Errorf(\"kerberos credential is needed\")\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tcredential = toBuf(credentialPtr, count)\n\t\t\thostname, _ := os.Hostname()\n\t\t\tip := resolve(hostname)\n\t\t\tif ip == \"\" {\n\t\t\t\tip, _ = findLocalIP(\"\", \"\")\n\t\t\t\tlogger.Infof(\"use local ip %s for %s\", ip, hostname)\n\t\t\t}\n\t\t\tvar eno syscall.Errno\n\t\t\tif jConf.AuthMethod == \"kerberos\" {\n\t\t\t\teno = kerb.auth(name, user, jConf.RealUser, C.GoString(group), ip, hostname, credential)\n\t\t\t} else {\n\t\t\t\ttbuf := utils.FromBuffer(credential)\n\t\t\t\tid := tbuf.Get32()\n\t\t\t\tpassword := tbuf.Get(int(tbuf.Get32()))\n\t\t\t\teno = kerb.check(meta.Background(), jfs.Meta(), name, user, id, string(password))\n\t\t\t}\n\t\t\tif eno != 0 {\n\t\t\t\tlogger.Errorf(\"%s auth failed for vol:%s(%s:%s): %s\", jConf.AuthMethod, name, user, jConf.RealUser, eno)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn jfs\n\t})\n}\n\nfunc F(p int64) *wrapper {\n\tfslock.Lock()\n\tdefer fslock.Unlock()\n\treturn handlers[p]\n}\n\n//export jfs_update_uid_grouping\nfunc jfs_update_uid_grouping(cname, uidstr *C.char, grouping *C.char) {\n\tname := C.GoString(cname)\n\tvar uids []pwent\n\tif uidstr != nil {\n\t\tfor _, line := range strings.Split(C.GoString(uidstr), \"\\n\") {\n\t\t\tfields := strings.Split(line, \":\")\n\t\t\tif len(fields) < 2 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tusername := strings.TrimSpace(fields[0])\n\t\t\tuid, _ := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 32)\n\t\t\tuids = append(uids, pwent{uint32(uid), username})\n\t\t}\n\n\t\tvar buffer bytes.Buffer\n\t\tfor _, u := range uids {\n\t\t\tbuffer.WriteString(fmt.Sprintf(\"\\t%v:%v\\n\", u.name, u.id))\n\t\t}\n\t\tlogger.Debugf(\"Update uids mapping\\n %s\", buffer.String())\n\t}\n\n\tvar userGroups = make(map[string][]string) // user -> groups\n\n\tvar gids []pwent\n\tif grouping != nil {\n\t\tfor _, line := range strings.Split(C.GoString(grouping), \"\\n\") {\n\t\t\tfields := strings.Split(line, \":\")\n\t\t\tif len(fields) < 2 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tgname := strings.TrimSpace(fields[0])\n\t\t\tgid, _ := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 32)\n\t\t\tgids = append(gids, pwent{uint32(gid), gname})\n\t\t\tif len(fields) > 2 {\n\t\t\t\tfor _, user := range strings.Split(fields[len(fields)-1], \",\") {\n\t\t\t\t\tuserGroups[user] = append(userGroups[user], gname)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tvar buffer bytes.Buffer\n\t\tfor _, g := range gids {\n\t\t\tbuffer.WriteString(fmt.Sprintf(\"\\t%v:%v\\n\", g.name, g.id))\n\t\t}\n\t\tlogger.Debugf(\"Update gids mapping\\n %s\", buffer.String())\n\t}\n\n\tfslock.Lock()\n\tdefer fslock.Unlock()\n\tuserGroupCache[name] = userGroups\n\tvar ws []*wrapper\n\tfor k, wrappers := range activefs {\n\t\tif k.name == name {\n\t\t\tws = append(ws, wrappers...)\n\t\t}\n\t}\n\tif len(ws) > 0 {\n\t\tfor _, w := range ws {\n\t\t\tw.m.update(uids, gids, false)\n\t\t\tlogger.Debugf(\"Update groups of %s to %s\", w.user, strings.Join(userGroups[w.user], \",\"))\n\t\t\tupdateCtx(w, userGroups[w.user])\n\t\t}\n\t}\n}\n\nfunc updateCtx(w *wrapper, groups []string) {\n\tif w.isSuperuser(w.user, groups) {\n\t\tw.ctx = meta.NewContext(uint32(os.Getpid()), 0, []uint32{0})\n\t} else {\n\t\tvar gids []uint32\n\t\tif w.ctx != nil {\n\t\t\tgids = w.ctx.Gids()\n\t\t}\n\t\tif len(groups) > 0 {\n\t\t\tgids = w.lookupGids(groups)\n\t\t}\n\t\tw.ctx = meta.NewContext(uint32(os.Getpid()), w.lookupUid(w.user), gids)\n\t}\n}\n\n//export jfs_getGroups\nfunc jfs_getGroups(cname, cuser *C.char, buf uintptr, count int32) int32 {\n\tname := C.GoString(cname)\n\tuser := C.GoString(cuser)\n\tfslock.Lock()\n\tuserGroups := userGroupCache[name]\n\tfslock.Unlock()\n\tvar gStr string\n\tif userGroups != nil {\n\t\tgs := userGroups[user]\n\t\tif gs != nil {\n\t\t\tgStr = strings.Join(gs, \",\")\n\t\t}\n\t}\n\tcopy(toBuf(buf, count), gStr)\n\treturn int32(len(gStr))\n}\n\n//export jfs_is_superuser\nfunc jfs_is_superuser(h int64, user *C.char, groups *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tif w.isSuperuser(C.GoString(user), strings.Split(C.GoString(groups), \",\")) {\n\t\treturn 1\n\t} else {\n\t\treturn 0\n\t}\n}\n\n//export jfs_term\nfunc jfs_term(pid int64, h int64) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn 0\n\t}\n\tctx := w.withPid(pid)\n\t// sync all open files\n\tfilesLock.Lock()\n\tvar m sync.WaitGroup\n\tvar toClose []int32\n\tfor fd, f := range openFiles {\n\t\tif f.w == w {\n\t\t\tm.Add(1)\n\t\t\tgo func(f *fs.File) {\n\t\t\t\tdefer m.Done()\n\t\t\t\t_ = f.Close(ctx)\n\t\t\t}(f.File)\n\t\t\ttoClose = append(toClose, fd)\n\t\t}\n\t}\n\tfor _, fd := range toClose {\n\t\tdelete(openFiles, fd)\n\t}\n\tfilesLock.Unlock()\n\tm.Wait()\n\n\tfslock.Lock()\n\tdefer fslock.Unlock()\n\tdelete(handlers, h)\n\tfor k, ws := range activefs {\n\t\tfor i := range ws {\n\t\t\tif ws[i] == w {\n\t\t\t\tif len(ws) > 1 {\n\t\t\t\t\tws[i] = ws[len(ws)-1]\n\t\t\t\t\tactivefs[k] = ws[:len(ws)-1]\n\t\t\t\t} else {\n\t\t\t\t\t_ = w.Flush()\n\t\t\t\t\t// don't close the filesystem, so it can be re-used later\n\t\t\t\t\t// w.Close()\n\t\t\t\t\t// delete(activefs, name)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tfor _, bridge := range bridges {\n\t\tif err := bridge.Push(); err != nil {\n\t\t\tlogger.Warnf(\"error pushing to Graphite: %s\", err)\n\t\t}\n\t}\n\tfor _, pusher := range pushers {\n\t\tif err := pusher.Push(); err != nil {\n\t\t\tlogger.Warnf(\"error pushing to PushGatway: %s\", err)\n\t\t}\n\t}\n\tfor _, remoteWriter := range remoteWriters {\n\t\tif err := remoteWriter.Push(); err != nil {\n\t\t\tlogger.Warnf(\"error pushing to RemoteWrite: %s\", err)\n\t\t}\n\t}\n\treturn 0\n}\n\n//export jfs_open\nfunc jfs_open(pid int64, h int64, cpath *C.char, lenPtr uintptr, flags int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tpath := C.GoString(cpath)\n\tf, err := w.Open(w.withPid(pid), path, uint32(flags))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tst, _ := f.Stat()\n\tif st.IsDir() {\n\t\treturn ENOENT\n\t}\n\tif lenPtr != 0 {\n\t\tbuf := toBuf(lenPtr, 8)\n\t\twb := utils.NewNativeBuffer(buf)\n\t\twb.Put64(uint64(st.Size()))\n\t}\n\treturn nextFileHandle(f, w)\n}\n\n//export jfs_open_posix\nfunc jfs_open_posix(pid int64, h int64, cpath *C.char, lenPtr uintptr, flags int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tpath := C.GoString(cpath)\n\tf, err := w.Open(w.withPid(pid), path, uint32(flags))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tst, _ := f.Stat()\n\tif st.IsDir() {\n\t\treturn EISDIR\n\t}\n\tif lenPtr != 0 {\n\t\tbuf := toBuf(lenPtr, 8)\n\t\twb := utils.NewNativeBuffer(buf)\n\t\twb.Put64(uint64(st.Size()))\n\t}\n\treturn nextFileHandle(f, w)\n}\n\n//export jfs_access\nfunc jfs_access(pid int64, h int64, cpath *C.char, flags int64) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Access(w.withPid(pid), C.GoString(cpath), int(flags)))\n}\n\n//export jfs_create\nfunc jfs_create(pid int64, h int64, cpath *C.char, mode uint16, umask uint16) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tpath := C.GoString(cpath)\n\tf, err := w.Create(w.withPid(pid), path, mode, umask)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tif w.ctx.Uid() == 0 && w.user != w.superuser {\n\t\t// belongs to supergroup\n\t\t_ = setOwner(w, w.withPid(pid), C.GoString(cpath), w.user, \"\")\n\t}\n\treturn nextFileHandle(f, w)\n}\n\n//export jfs_mkdir\nfunc jfs_mkdir(pid int64, h int64, cpath *C.char, mode uint16, umask uint16) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\terr := errno(w.Mkdir(w.withPid(pid), C.GoString(cpath), mode, umask))\n\tif err == 0 && w.ctx.Uid() == 0 && w.user != w.superuser {\n\t\t// belongs to supergroup\n\t\t_ = setOwner(w, w.withPid(pid), C.GoString(cpath), w.user, \"\")\n\t}\n\treturn err\n}\n\n//export jfs_mkdirAll\nfunc jfs_mkdirAll(pid int64, h int64, cpath *C.char, mode, umask uint16, existOK bool) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tpath := C.GoString(cpath)\n\terr := errno(w.MkdirAll0(w.withPid(pid), path, mode, umask, existOK))\n\tif err == 0 && w.ctx.Uid() == 0 && w.user != w.superuser {\n\t\t// belongs to supergroup\n\t\tif err := setOwner(w, w.withPid(pid), path, w.user, \"\"); err != 0 {\n\t\t\tlogger.Errorf(\"change owner of %s to %s: %d\", path, w.user, err)\n\t\t}\n\t}\n\treturn err\n}\n\n//export jfs_delete\nfunc jfs_delete(pid int64, h int64, cpath *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Delete(w.withPid(pid), C.GoString(cpath)))\n}\n\n//export jfs_unlink\nfunc jfs_unlink(pid int64, h int64, cpath *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Unlink(w.withPid(pid), C.GoString(cpath)))\n}\n\n//export jfs_rmdir\nfunc jfs_rmdir(pid int64, h int64, cpath *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Rmdir(w.withPid(pid), C.GoString(cpath)))\n}\n\n//export jfs_rmr\nfunc jfs_rmr(pid int64, h int64, cpath *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Rmr(w.withPid(pid), C.GoString(cpath), false, MaxDeletes))\n}\n\n//export jfs_rename\nfunc jfs_rename(pid int64, h int64, oldpath *C.char, newpath *C.char) int32 {\n\treturn jfs_rename0(pid, h, oldpath, newpath, meta.RenameNoReplace)\n}\n\n//export jfs_rename0\nfunc jfs_rename0(pid int64, h int64, oldpath *C.char, newpath *C.char, flags uint32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Rename(w.withPid(pid), C.GoString(oldpath), C.GoString(newpath), flags))\n}\n\n//export jfs_truncate\nfunc jfs_truncate(pid int64, h int64, path *C.char, length uint64) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Truncate(w.withPid(pid), C.GoString(path), length))\n}\n\n//export jfs_setXattr\nfunc jfs_setXattr(pid int64, h int64, path *C.char, name *C.char, value uintptr, vlen, mode int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tvar flags uint32\n\tswitch mode {\n\tcase 1:\n\t\tflags = meta.XattrCreate\n\tcase 2:\n\t\tflags = meta.XattrReplace\n\t}\n\treturn errno(w.SetXattr(w.withPid(pid), C.GoString(path), C.GoString(name), toBuf(value, vlen), flags))\n}\n\n//export jfs_setXattr2\nfunc jfs_setXattr2(pid int64, h int64, path *C.char, name *C.char, value *C.char, mode int64) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tvar flags uint32\n\tswitch mode {\n\tcase 1:\n\t\tflags = meta.XattrCreate\n\tcase 2:\n\t\tflags = meta.XattrReplace\n\t}\n\treturn errno(w.SetXattr(w.withPid(pid), C.GoString(path), C.GoString(name), []byte(C.GoString(value)), flags))\n}\n\n//export jfs_getXattr\nfunc jfs_getXattr(pid int64, h int64, path *C.char, name *C.char, buf uintptr, bufsize int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tbuff, err := w.GetXattr(w.withPid(pid), C.GoString(path), C.GoString(name))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tif int32(len(buff)) >= bufsize {\n\t\treturn bufsize\n\t}\n\tcopy(toBuf(buf, bufsize), buff)\n\treturn int32(len(buff))\n}\n\n//export jfs_getXattr2\nfunc jfs_getXattr2(pid int64, h int64, path *C.char, name *C.char, value **C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tt, err := w.GetXattr(w.withPid(pid), C.GoString(path), C.GoString(name))\n\tif err == 0 {\n\t\t*value = C.CString(string(t))\n\t}\n\treturn errno(err)\n}\n\n//export jfs_listXattr\nfunc jfs_listXattr(pid int64, h int64, path *C.char, buf uintptr, bufsize int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tbuff, err := w.ListXattr(w.withPid(pid), C.GoString(path))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tif int32(len(buff)) >= bufsize {\n\t\treturn bufsize\n\t}\n\tcopy(toBuf(buf, bufsize), buff)\n\treturn int32(len(buff))\n}\n\n//export jfs_listXattr2\nfunc jfs_listXattr2(pid int64, h int64, path *C.char, value **C.char, size *int) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tt, err := w.ListXattr(w.withPid(pid), C.GoString(path))\n\tif err == 0 {\n\t\t*value = C.CString(string(t))\n\t\t*size = len(t)\n\t}\n\treturn errno(err)\n}\n\n//export jfs_removeXattr\nfunc jfs_removeXattr(pid int64, h int64, path *C.char, name *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.RemoveXattr(w.withPid(pid), C.GoString(path), C.GoString(name)))\n}\n\n//export jfs_getfacl\nfunc jfs_getfacl(pid int64, h int64, path *C.char, acltype int32, buf uintptr, blen int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\trule := acl.EmptyRule()\n\terr := w.GetFacl(w.withPid(pid), C.GoString(path), uint8(acltype), rule)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\twb := utils.NewNativeBuffer(toBuf(buf, blen))\n\twb.Put16(rule.Owner)\n\twb.Put16(rule.Group)\n\twb.Put16(rule.Other)\n\twb.Put16(rule.Mask)\n\twb.Put16(uint16(len(rule.NamedUsers)))\n\twb.Put16(uint16(len(rule.NamedGroups)))\n\tvar off uintptr = 12\n\tfor i, entry := range append(rule.NamedUsers, rule.NamedGroups...) {\n\t\tvar name string\n\t\tif i < len(rule.NamedUsers) {\n\t\t\tname = w.uid2name(entry.Id)\n\t\t} else {\n\t\t\tname = w.gid2name(entry.Id)\n\t\t}\n\t\tif wb.Left() < len(name)+1+2 {\n\t\t\treturn -100\n\t\t}\n\t\twb.Put([]byte(name))\n\t\twb.Put8(0)\n\t\twb.Put16(entry.Perm)\n\t}\n\treturn int32(off)\n}\n\n//export jfs_setfacl\nfunc jfs_setfacl(pid int64, h int64, path *C.char, acltype int32, buf uintptr, alen int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\trule := acl.EmptyRule()\n\tr := utils.NewNativeBuffer(toBuf(buf, alen))\n\trule.Owner = r.Get16()\n\trule.Group = r.Get16()\n\trule.Other = r.Get16()\n\trule.Mask = r.Get16()\n\tnamedusers := r.Get16()\n\tnamedgroups := r.Get16()\n\tfor i := uint16(0); i < namedusers+namedgroups; i++ {\n\t\tname := string(r.Get(int(r.Get8())))\n\t\tvar entry acl.Entry\n\t\tentry.Perm = uint16(r.Get8())\n\t\tif i < namedusers {\n\t\t\tentry.Id = w.lookupUid(name)\n\t\t\trule.NamedUsers = append(rule.NamedUsers, entry)\n\t\t} else {\n\t\t\tentry.Id = w.lookupGid(name)\n\t\t\trule.NamedGroups = append(rule.NamedGroups, entry)\n\t\t}\n\t}\n\treturn errno(w.SetFacl(w.withPid(pid), C.GoString(path), uint8(acltype), rule))\n}\n\n//export jfs_link\nfunc jfs_link(pid int64, h int64, src *C.char, dst *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(w.Link(w.withPid(pid), C.GoString(src), C.GoString(dst)))\n}\n\n//export jfs_symlink\nfunc jfs_symlink(pid int64, h int64, target_ *C.char, link_ *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\ttarget := C.GoString(target_)\n\tlink := C.GoString(link_)\n\tdir := path.Dir(strings.TrimRight(link, \"/\"))\n\trel, e := filepath.Rel(dir, target)\n\tif e != nil {\n\t\t// external link\n\t\trel = target\n\t}\n\treturn errno(w.Symlink(w.withPid(pid), rel, link))\n}\n\n//export jfs_readlink\nfunc jfs_readlink(pid int64, h int64, link *C.char, buf uintptr, bufsize int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\ttarget, err := w.Readlink(w.withPid(pid), C.GoString(link))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tif int32(len(target)+1) >= bufsize {\n\t\ttarget = target[:bufsize-1]\n\t}\n\twb := utils.NewNativeBuffer(toBuf(buf, bufsize))\n\twb.Put(target)\n\twb.Put8(0)\n\treturn int32(len(target))\n}\n\n// mode:4 length:8 mtime:8 atime:8 user:50 group:50\nfunc fill_stat(w *wrapper, wb *utils.Buffer, st *fs.FileStat) int32 {\n\twb.Put32(uint32(st.Mode()))\n\twb.Put64(uint64(st.Size()))\n\twb.Put64(uint64(st.Mtime()))\n\twb.Put64(uint64(st.Atime()))\n\tuser := w.uid2name(uint32(st.Uid()))\n\twb.Put([]byte(user))\n\twb.Put8(0)\n\tgroup := w.gid2name(uint32(st.Gid()))\n\twb.Put([]byte(group))\n\twb.Put8(0)\n\treturn 30 + int32(len(user)) + int32(len(group))\n}\n\n//export jfs_stat1\nfunc jfs_stat1(pid int64, h int64, cpath *C.char, buf uintptr) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tinfo, err := w.Stat(w.withPid(pid), C.GoString(cpath))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\treturn fill_stat(w, utils.NewNativeBuffer(toBuf(buf, 130)), info)\n}\n\n//export jfs_lstat1\nfunc jfs_lstat1(pid int64, h int64, cpath *C.char, buf uintptr) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tfi, err := w.Lstat(w.withPid(pid), C.GoString(cpath))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\treturn fill_stat(w, utils.NewNativeBuffer(toBuf(buf, 130)), fi)\n}\n\nfunc attrToInfo(fi *fs.FileStat, info *C.fileInfo) {\n\tattr := fi.Sys().(*meta.Attr)\n\tinfo.mode = C.uint32_t(attr.SMode())\n\tinfo.uid = C.uint32_t(attr.Uid)\n\tinfo.gid = C.uint32_t(attr.Gid)\n\tinfo.atime = C.uint32_t(attr.Atime)\n\tinfo.mtime = C.uint32_t(attr.Mtime)\n\tinfo.ctime = C.uint32_t(attr.Ctime)\n\tinfo.nlink = C.uint32_t(attr.Nlink)\n\tinfo.length = C.uint64_t(attr.Length)\n}\n\n//export jfs_stat\nfunc jfs_stat(pid int64, h int64, cpath *C.char, info *C.fileInfo) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tfi, err := w.Stat(w.withPid(pid), C.GoString(cpath))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tinfo.inode = C.uint64_t(fi.Inode())\n\tattrToInfo(fi, info)\n\treturn 0\n}\n\n//export jfs_lstat\nfunc jfs_lstat(pid int64, h int64, cpath *C.char, info *C.fileInfo) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tfi, err := w.Lstat(w.withPid(pid), C.GoString(cpath))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tinfo.inode = C.uint64_t(fi.Inode())\n\tattrToInfo(fi, info)\n\treturn 0\n}\n\n//export jfs_summary\nfunc jfs_summary(pid int64, h int64, cpath *C.char, buf uintptr) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tctx := w.withPid(pid)\n\tf, err := w.Open(ctx, C.GoString(cpath), 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tdefer f.Close(ctx)\n\tsummary, err := f.Summary(ctx, true, true)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\twb := utils.NewNativeBuffer(toBuf(buf, 40))\n\twb.Put64(summary.Length)\n\twb.Put64(summary.Files)\n\twb.Put64(summary.Dirs)\n\n\t// quota\n\tquota, _ := f.GetQuota(ctx)\n\tif quota != nil {\n\t\twb.Put64(uint64(quota.MaxInodes))\n\t\twb.Put64(uint64(quota.MaxSpace))\n\t} else {\n\t\twb.Put64(0)\n\t\twb.Put64(0)\n\t}\n\treturn 40\n}\n\n//export jfs_info\nfunc jfs_info(pid int64, h int64, cpath *C.char, p_buf **byte, recursive, strict bool) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tctx := w.withPid(pid)\n\tf, err := w.Open(ctx, C.GoString(cpath), 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tdefer f.Close(ctx)\n\tinfo, err := f.Summary(ctx, recursive, strict)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tres, err2 := json.Marshal(info)\n\tif err2 != nil {\n\t\treturn EINVAL\n\t}\n\tif *p_buf != nil {\n\t\treturn EINVAL\n\t}\n\n\t*p_buf = (*byte)(C.malloc(C.size_t(len(res))))\n\n\tbuf := unsafe.Slice(*p_buf, len(res))\n\treturn int32(copy(buf, res))\n}\n\n//export jfs_gettreesummary\nfunc jfs_gettreesummary(pid, h int64, cpath *C.char, depth, entries uint8, p_buf **byte) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tctx := w.withPid(pid)\n\tf, err := w.Open(ctx, C.GoString(cpath), 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tsummary, err := f.GetTreeSummary(ctx, depth, entries, true)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tres, err2 := json.Marshal(summary)\n\tif err2 != nil {\n\t\treturn EINVAL\n\t}\n\tif *p_buf != nil {\n\t\treturn EINVAL\n\t}\n\n\t*p_buf = (*byte)(C.malloc(C.size_t(len(res))))\n\n\tbuf := unsafe.Slice(*p_buf, len(res))\n\treturn int32(copy(buf, res))\n}\n\n//export jfs_quota\nfunc jfs_quota(pid int64, h int64, cpath *C.char, cmd uint8, cap, inodes uint64, strict, repair, create bool, p_buf **byte) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tif *p_buf != nil {\n\t\treturn EINVAL\n\t}\n\n\tqs, err := w.HandleQuota(w.withPid(pid), C.GoString(cpath), cmd, cap, inodes, strict, repair, create)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tres, err2 := json.Marshal(qs)\n\tif err2 != nil {\n\t\treturn EINVAL\n\t}\n\n\t*p_buf = (*byte)(C.malloc(C.size_t(len(res))))\n\tbuf := unsafe.Slice(*p_buf, len(res))\n\treturn int32(copy(buf, res))\n}\n\n//export jfs_statvfs\nfunc jfs_statvfs(pid int64, h int64, buf uintptr) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\ttotal, avail := w.StatFS(w.withPid(pid))\n\twb := utils.NewNativeBuffer(toBuf(buf, 16))\n\twb.Put64(total)\n\twb.Put64(avail)\n\treturn 0\n}\n\n//export jfs_chmod\nfunc jfs_chmod(pid int64, h int64, cpath *C.char, mode C.mode_t) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tf, err := w.Open(w.withPid(pid), C.GoString(cpath), 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tdefer f.Close(w.withPid(pid))\n\treturn errno(f.Chmod(w.withPid(pid), uint16(mode)))\n}\n\n//export jfs_chown\nfunc jfs_chown(pid int64, h int64, cpath *C.char, uid uint32, gid uint32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tf, err := w.Open(w.withPid(pid), C.GoString(cpath), 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\treturn errno(f.Chown(w.withPid(pid), uid, gid))\n}\n\n//export jfs_utime\nfunc jfs_utime(pid int64, h int64, cpath *C.char, mtime, atime int64) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tf, err := w.Open(w.withPid(pid), C.GoString(cpath), 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tdefer f.Close(w.withPid(pid))\n\treturn errno(f.Utime(w.withPid(pid), atime, mtime))\n}\n\n//export jfs_setOwner\nfunc jfs_setOwner(pid int64, h int64, cpath *C.char, owner *C.char, group *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn setOwner(w, w.withPid(pid), C.GoString(cpath), C.GoString(owner), C.GoString(group))\n}\n\nfunc setOwner(w *wrapper, ctx meta.Context, path string, owner, group string) int32 {\n\tf, err := w.Open(ctx, path, 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tdefer f.Close(ctx)\n\tst, _ := f.Stat()\n\tuid := uint32(st.(*fs.FileStat).Uid())\n\tgid := uint32(st.(*fs.FileStat).Gid())\n\tif owner != \"\" {\n\t\tuid = w.lookupUid(owner)\n\t}\n\tif group != \"\" {\n\t\tgid = w.lookupGid(group)\n\t}\n\treturn errno(f.Chown(ctx, uid, gid))\n}\n\n//export jfs_listdir\nfunc jfs_listdir(pid int64, h int64, cpath *C.char, offset int64, buf uintptr, bufsize int32) int32 {\n\tvar ctx meta.Context\n\tvar f *fs.File\n\tvar w *wrapper\n\tif offset > 0 {\n\t\tfilesLock.Lock()\n\t\tfw := openFiles[int32(h)]\n\t\tfilesLock.Unlock()\n\t\tif fw == nil {\n\t\t\treturn EINVAL\n\t\t}\n\t\tfreeHandle(int32(h))\n\t\tw = fw.w\n\t\tf = fw.File\n\t\tctx = w.withPid(pid)\n\t} else {\n\t\tw = F(h)\n\t\tif w == nil {\n\t\t\treturn EINVAL\n\t\t}\n\t\tvar err syscall.Errno\n\t\tctx = w.withPid(pid)\n\t\tf, err = w.Open(ctx, C.GoString(cpath), 0)\n\t\tif err != 0 {\n\t\t\treturn errno(err)\n\t\t}\n\t\tst, _ := f.Stat()\n\t\tif !st.IsDir() {\n\t\t\treturn ENOTDIR\n\t\t}\n\t}\n\n\tes, err := f.ReaddirPlus(ctx, int(offset))\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\n\twb := utils.NewNativeBuffer(toBuf(buf, bufsize))\n\tfor i, d := range es {\n\t\tif wb.Left() < 1+len(d.Name)+1+130+8 {\n\t\t\twb.Put32(uint32(len(es) - i))\n\t\t\twb.Put32(uint32(nextFileHandle(f, w)))\n\t\t\treturn bufsize - int32(wb.Left()) - 8\n\t\t}\n\t\twb.Put8(byte(len(d.Name)))\n\t\twb.Put(d.Name)\n\t\theader := wb.Get(1)\n\t\theader[0] = uint8(fill_stat(w, wb, fs.AttrToFileInfo(d.Inode, d.Attr)))\n\t}\n\twb.Put32(0)\n\treturn bufsize - int32(wb.Left()) - 4\n}\n\n//export jfs_listdir2\nfunc jfs_listdir2(pid int64, h int64, cpath *C.char, plus bool, buf **byte, size *int64) int32 {\n\tvar ctx meta.Context\n\tvar f *fs.File\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tvar err syscall.Errno\n\tctx = w.withPid(pid)\n\tf, err = w.Open(ctx, C.GoString(cpath), 0)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tst, _ := f.Stat()\n\tif !st.IsDir() {\n\t\treturn ENOTDIR\n\t}\n\n\t*size = 0\n\tif plus {\n\t\tes, err := f.ReaddirPlus(ctx, 0)\n\t\tif err != 0 {\n\t\t\treturn errno(err)\n\t\t}\n\t\tfor _, e := range es {\n\t\t\t*size += 2 + int64(len(e.Name)) + 4*11\n\t\t}\n\t\t*buf = (*byte)(C.malloc(C.size_t(*size)))\n\t\tout := utils.FromBuffer(unsafe.Slice(*buf, *size))\n\t\tfor _, e := range es {\n\t\t\tout.Put16(uint16(len(e.Name)))\n\t\t\tout.Put([]byte(e.Name))\n\t\t\tout.Put32(e.Attr.SMode())\n\t\t\tout.Put64(uint64(e.Inode))\n\t\t\tout.Put32(e.Attr.Nlink)\n\t\t\tout.Put32(e.Attr.Uid)\n\t\t\tout.Put32(e.Attr.Gid)\n\t\t\tout.Put64(e.Attr.Length)\n\t\t\tout.Put32(uint32(e.Attr.Atime))\n\t\t\tout.Put32(uint32(e.Attr.Mtime))\n\t\t\tout.Put32(uint32(e.Attr.Ctime))\n\t\t}\n\t} else {\n\t\tes, err := f.Readdir(ctx, 0)\n\t\tif err != 0 {\n\t\t\treturn errno(err)\n\t\t}\n\t\tfor _, e := range es {\n\t\t\t*size += 2 + int64(len(e.Name()))\n\t\t}\n\t\t*buf = (*byte)(C.malloc(C.size_t(*size)))\n\t\tout := utils.FromBuffer(unsafe.Slice(*buf, *size))\n\t\tfor _, e := range es {\n\t\t\tout.Put16(uint16(len(e.Name())))\n\t\t\tout.Put([]byte(e.Name()))\n\t\t}\n\t}\n\treturn 0\n}\n\nfunc toBuf(s uintptr, sz int32) []byte {\n\treturn (*[1 << 30]byte)(unsafe.Pointer(s))[:sz:sz]\n}\n\n//export jfs_concat\nfunc jfs_concat(pid int64, h int64, _dst *C.char, buf uintptr, bufsize int32) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tdst := C.GoString(_dst)\n\tctx := w.withPid(pid)\n\tdf, err := w.Open(ctx, dst, vfs.MODE_MASK_W)\n\tif err != 0 {\n\t\treturn errno(err)\n\t}\n\tdefer df.Close(ctx)\n\tsrcs := strings.Split(string(toBuf(buf, bufsize-1)), \"\\000\")\n\tvar tmp string\n\tif len(srcs) > 1 {\n\t\ttmp = filepath.Join(filepath.Dir(dst), \".\"+filepath.Base(dst)+\".merging\")\n\t\tfi, err := w.Create(ctx, tmp, 0666, 022)\n\t\tif err != 0 {\n\t\t\treturn errno(err)\n\t\t}\n\t\tdefer func() { _ = w.Delete(ctx, tmp) }()\n\t\tdefer fi.Close(ctx)\n\t\tvar off uint64\n\t\tfor _, src := range srcs {\n\t\t\tcopied, err := w.CopyFileRange(ctx, src, 0, tmp, off, 1<<63)\n\t\t\tif err != 0 {\n\t\t\t\treturn errno(err)\n\t\t\t}\n\t\t\toff += copied\n\t\t}\n\t} else {\n\t\ttmp = srcs[0]\n\t}\n\n\tdfi, _ := df.Stat()\n\t_, err = w.CopyFileRange(ctx, tmp, 0, dst, uint64(dfi.Size()), 1<<63)\n\tr := errno(err)\n\tif r == 0 {\n\t\tvar wg sync.WaitGroup\n\t\tvar limit = make(chan bool, 100)\n\t\tfor _, src := range srcs {\n\t\t\tlimit <- true\n\t\t\twg.Add(1)\n\t\t\tgo func(p string) {\n\t\t\t\tdefer func() { <-limit }()\n\t\t\t\tdefer wg.Done()\n\t\t\t\tif r := w.Delete(ctx, p); r != 0 {\n\t\t\t\t\tlogger.Errorf(\"delete source %s: %s\", p, r)\n\t\t\t\t}\n\t\t\t}(src)\n\t\t}\n\t\twg.Wait()\n\t}\n\treturn r\n}\n\n//export jfs_clone\nfunc jfs_clone(pid int64, h int64, _src *C.char, _dst *C.char, preserve bool) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tsrc := C.GoString(_src)\n\tdst := C.GoString(_dst)\n\tctx := w.withPid(pid)\n\terr := w.Clone(ctx, src, dst, preserve)\n\treturn errno(err)\n}\n\n//export jfs_status\nfunc jfs_status(pid int64, h int64, trash bool, session uint64, p_buf **byte) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tctx := w.withPid(pid)\n\n\tvar err error\n\tvar output []byte\n\tif session != 0 {\n\t\ts, err := w.Meta().GetSession(session, true)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"get session %d: %s\", session, err)\n\t\t\treturn errno(syscall.EIO)\n\t\t}\n\t\toutput, err = json.Marshal(s)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"marshal session: %v\", err)\n\t\t\treturn errno(syscall.EIO)\n\t\t}\n\t} else {\n\t\tsections := &meta.Sections{}\n\t\terr = meta.Status(ctx, w.Meta(), trash, sections)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"get status: %s\", err)\n\t\t\treturn errno(syscall.EIO)\n\t\t}\n\t\toutput, err = json.Marshal(sections)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"marshal sessions: %v\", err)\n\t\t\treturn errno(syscall.EIO)\n\t\t}\n\t}\n\n\t*p_buf = (*byte)(C.malloc(C.size_t(len(output))))\n\tbuf := unsafe.Slice(*p_buf, len(output))\n\treturn int32(copy(buf, output))\n}\n\n//export jfs_lseek\nfunc jfs_lseek(pid int64, fd int32, offset int64, whence int64) int64 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tif ok {\n\t\tfilesLock.Unlock()\n\t\toff, _ := f.Seek(f.w.withPid(pid), offset, int(whence))\n\t\treturn off\n\t}\n\tfilesLock.Unlock()\n\treturn int64(EINVAL)\n}\n\n//export jfs_read\nfunc jfs_read(pid int64, fd int32, cbuf uintptr, count int32) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tif !ok {\n\t\tfilesLock.Unlock()\n\t\treturn EINVAL\n\t}\n\tfilesLock.Unlock()\n\n\tn, err := f.Read(f.w.withPid(pid), toBuf(cbuf, int32(count)))\n\tif err != nil && err != io.EOF {\n\t\tlogger.Errorf(\"read %s: %s\", f.Name(), err)\n\t\treturn errno(err)\n\t}\n\treturn int32(n)\n}\n\n//export jfs_pread\nfunc jfs_pread(pid int64, fd int32, cbuf uintptr, count int32, offset int64) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tif !ok {\n\t\tfilesLock.Unlock()\n\t\treturn EINVAL\n\t}\n\tfilesLock.Unlock()\n\n\tif count > (1 << 30) {\n\t\tcount = 1 << 30\n\t}\n\tn, err := f.Pread(f.w.withPid(pid), toBuf(cbuf, count), offset)\n\tif err != nil && err != io.EOF {\n\t\tlogger.Errorf(\"read %s: %s\", f.Name(), err)\n\t\treturn errno(err)\n\t}\n\treturn int32(n)\n}\n\n//export jfs_write\nfunc jfs_write(pid int64, fd int32, cbuf uintptr, count int32) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tif !ok {\n\t\tfilesLock.Unlock()\n\t\treturn EINVAL\n\t}\n\tfilesLock.Unlock()\n\n\tbuf := toBuf(cbuf, count)\n\tn, err := f.Write(f.w.withPid(pid), buf)\n\tif err != 0 {\n\t\tlogger.Errorf(\"write %s: %s\", f.Name(), err)\n\t\treturn errno(err)\n\t}\n\treturn int32(n)\n}\n\n//export jfs_pwrite\nfunc jfs_pwrite(pid int64, fd int32, cbuf uintptr, count int32, offset int64) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tif !ok {\n\t\tfilesLock.Unlock()\n\t\treturn EINVAL\n\t}\n\tfilesLock.Unlock()\n\n\tbuf := toBuf(cbuf, count)\n\tn, err := f.Pwrite(f.w.withPid(pid), buf, int64(offset))\n\tif err != 0 {\n\t\tlogger.Errorf(\"pwrite %s: %s\", f.Name(), err)\n\t\treturn errno(err)\n\t}\n\treturn int32(n)\n}\n\n//export jfs_ftruncate\nfunc jfs_ftruncate(pid int64, fd int32, size uint64) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tfilesLock.Unlock()\n\tif !ok {\n\t\treturn EINVAL\n\t}\n\treturn errno(f.Truncate(f.w.withPid(pid), size))\n}\n\n//export jfs_flush\nfunc jfs_flush(pid int64, fd int32) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tif !ok {\n\t\tfilesLock.Unlock()\n\t\treturn EINVAL\n\t}\n\tfilesLock.Unlock()\n\n\treturn errno(f.Flush(f.w.withPid(pid)))\n}\n\n//export jfs_fsync\nfunc jfs_fsync(pid int64, fd int32) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tif !ok {\n\t\tfilesLock.Unlock()\n\t\treturn EINVAL\n\t}\n\tfilesLock.Unlock()\n\n\treturn errno(f.Fsync(f.w.withPid(pid)))\n}\n\n//export jfs_ranger_cfg\nfunc jfs_ranger_cfg(cname *C.char, buf uintptr, count int32) int32 {\n\tname := C.GoString(cname)\n\tfslock.Lock()\n\tformat := formats[name]\n\tfslock.Unlock()\n\tvar cfg string\n\tif format != nil {\n\t\turl := format.RangerRestUrl\n\t\tname := format.RangerService\n\t\tif url != \"\" && name != \"\" {\n\t\t\tcfg = fmt.Sprintf(\"%s?name=%s\", url, name)\n\t\t}\n\t}\n\tcopy(toBuf(buf, count), cfg)\n\treturn int32(len(cfg))\n}\n\n//export jfs_close\nfunc jfs_close(pid int64, fd int32) int32 {\n\tfilesLock.Lock()\n\tf, ok := openFiles[fd]\n\tfilesLock.Unlock()\n\tif !ok {\n\t\treturn 0\n\t}\n\tfreeHandle(fd)\n\treturn errno(f.Close(f.w.withPid(pid)))\n}\n\n//export jfs_warmup\nfunc jfs_warmup(pid int64, h int64, _paths *C.char, numthreads int32, background, isEvict, isCheck bool, p_buf **byte) int32 {\n\tresp := &vfs.CacheResponse{Locations: make(map[string]uint64)}\n\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tctx := w.withPid(pid)\n\n\tvar paths []string\n\terr := json.Unmarshal([]byte(C.GoString(_paths)), &paths)\n\tif err != nil {\n\t\tlogger.Errorf(\"invalid json: %s\", C.GoString(_paths))\n\t\treturn EINVAL\n\t}\n\tw.Warmup(ctx, paths, int(numthreads), background, isEvict, isCheck, resp)\n\tres, err := json.Marshal(resp)\n\tif err != nil {\n\t\tlogger.Fatalf(\"json: %s\", err)\n\t}\n\n\t*p_buf = (*byte)(C.malloc(C.size_t(len(res))))\n\tbuf := unsafe.Slice(*p_buf, len(res))\n\n\treturn int32(copy(buf, res))\n}\n\nfunc resolve(hostname string) string {\n\tif hostname == \"\" {\n\t\treturn \"\"\n\t}\n\tstart := time.Now()\n\tips, err := net.DefaultResolver.LookupIP(context.Background(), \"ip4\", hostname)\n\tif err != nil {\n\t\tlogger.Warningf(\"Fail to resolve host %s: %s\", hostname, err)\n\t\treturn \"\"\n\t}\n\tvar ipStr []string\n\tfor _, ip := range ips {\n\t\tipStr = append(ipStr, ip.To4().String())\n\t}\n\tlogger.Debugf(\"resolve %s to %s in %s\", hostname, strings.Join(ipStr, \",\"), time.Since(start))\n\treturn strings.Join(ipStr, \",\")\n}\n\nfunc findLocalIP(mask string, iname string) (string, error) {\n\tfor strings.HasSuffix(mask, \".0\") {\n\t\tmask = mask[:len(mask)-2]\n\t}\n\tifaces, err := net.Interfaces()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tfor _, iface := range ifaces {\n\t\tif iface.Flags&net.FlagUp == 0 && iname == \"\" && mask == \"\" {\n\t\t\tcontinue // interface down\n\t\t}\n\t\tif iface.Flags&net.FlagLoopback != 0 {\n\t\t\tcontinue // loopback interface\n\t\t}\n\t\tif iname != \"\" && iface.Name != iname && !strings.HasPrefix(iface.Name, iname+\".\") {\n\t\t\tcontinue\n\t\t}\n\t\taddrs, err := iface.Addrs()\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tfor _, addr := range addrs {\n\t\t\tvar ip net.IP\n\t\t\tswitch v := addr.(type) {\n\t\t\tcase *net.IPNet:\n\t\t\t\tip = v.IP\n\t\t\tcase *net.IPAddr:\n\t\t\t\tip = v.IP\n\t\t\t}\n\t\t\tif ip == nil || ip.IsLoopback() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tip = ip.To4()\n\t\t\tif ip == nil {\n\t\t\t\tcontinue // not an ipv4 address\n\t\t\t}\n\t\t\tif !strings.HasPrefix(ip.String(), mask) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn ip.String(), nil\n\t\t}\n\t}\n\treturn \"\", errors.New(\"are you connected to the network?\")\n}\n\n//export jfs_get_token\nfunc jfs_get_token(h int64, cname *C.char, buf uintptr, count int32, renewer *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\tid, t, eno := kerb.issue(w.ctx, w.Meta(), C.GoString(cname), w.user, C.GoString(renewer))\n\tif eno != 0 {\n\t\tlogger.Errorf(\"get token for %s: %s\", w.volname, eno)\n\t\treturn errno(eno)\n\t}\n\twb := utils.NewNativeBuffer(toBuf(buf, count))\n\twb.Put32(id)\n\twb.Put64(uint64(t.Issued))\n\twb.Put64(uint64(t.Expire))\n\twb.Put([]byte(t.Password))\n\treturn int32(wb.Offset())\n}\n\n//export jfs_renew_token\nfunc jfs_renew_token(h int64, id uint32, password *C.char) int64 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\texpire, eno := kerb.renew(w.ctx, w.Meta(), w.volname, w.user, id, C.GoString(password))\n\tif eno != 0 {\n\t\tlogger.Errorf(\"renew token %d for %s: %s\", id, w.volname, eno)\n\t\treturn int64(errno(eno))\n\t}\n\treturn expire\n}\n\n//export jfs_cancel_token\nfunc jfs_cancel_token(h int64, id uint32, password *C.char) int32 {\n\tw := F(h)\n\tif w == nil {\n\t\treturn EINVAL\n\t}\n\treturn errno(kerb.cancelToken(w.ctx, w.Meta(), w.user, id, C.GoString(password)))\n}\n\nfunc main() {\n}\n" }, { "path": "sdk/java/libjfs/remote_write.go", "content": "// Copyright 2025 JuiceFS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package main provides remote write functionality for pushing Prometheus metrics\n// to remote write endpoints.\n\npackage main\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/golang/snappy\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tdto \"github.com/prometheus/client_model/go\"\n\t\"github.com/prometheus/common/expfmt\"\n\t\"github.com/prometheus/common/model\"\n\t\"github.com/prometheus/prometheus/prompb\"\n\t\"google.golang.org/protobuf/proto\"\n)\n\nconst (\n\tdefaultRemoteWriteTimeout = 15 * time.Second\n)\n\n// RemoteWriteConfig defines the remote write configuration.\ntype RemoteWriteConfig struct {\n\t// The URL to push metrics to. Required.\n\tURL string\n\n\t// Basic authentication string in format \"username:password\". Optional.\n\tAuth string\n\n\t// The interval to use for pushing data. Defaults to 15 seconds.\n\tInterval time.Duration\n\n\t// The timeout for pushing metrics. Defaults to 15 seconds.\n\tTimeout time.Duration\n\n\t// The Gatherer to use for metrics. Defaults to prometheus.DefaultGatherer.\n\tGatherer prometheus.Gatherer\n\n\t// Common labels to add to all metrics. Optional.\n\tCommonLabels map[string]string\n\n\t// The logger that messages are written to. Defaults to no logging.\n\tLogger Logger\n\n\t// ErrorHandling defines how errors are handled.\n\tErrorHandling HandlerErrorHandling\n}\n\n// RemoteWriter pushes metrics to the configured remote write endpoint.\ntype RemoteWriter struct {\n\turl string\n\tgatherer prometheus.Gatherer\n\tauth string\n\tinterval time.Duration\n\ttimeout time.Duration\n\terrorHandling HandlerErrorHandling\n\tlogger Logger\n\tcommonLabels map[string]string\n\tclient *http.Client\n}\n\n// NewRemoteWriter returns a pointer to a new RemoteWriter struct.\nfunc NewRemoteWriter(c *RemoteWriteConfig) (*RemoteWriter, error) {\n\trw := &RemoteWriter{}\n\n\tif c.URL == \"\" {\n\t\treturn nil, errors.New(\"missing URL\")\n\t}\n\trw.url = c.URL\n\n\trw.auth = c.Auth\n\n\tvar z time.Duration\n\tif c.Interval == z {\n\t\trw.interval = defaultRemoteWriteTimeout\n\t} else {\n\t\trw.interval = c.Interval\n\t}\n\n\tif c.Timeout == z {\n\t\trw.timeout = defaultRemoteWriteTimeout\n\t} else {\n\t\trw.timeout = c.Timeout\n\t}\n\n\tif c.Gatherer == nil {\n\t\trw.gatherer = prometheus.DefaultGatherer\n\t} else {\n\t\trw.gatherer = c.Gatherer\n\t}\n\n\trw.commonLabels = c.CommonLabels\n\trw.logger = c.Logger\n\trw.errorHandling = c.ErrorHandling\n\n\trw.client = &http.Client{\n\t\tTimeout: rw.timeout,\n\t}\n\n\treturn rw, nil\n}\n\n// Push pushes Prometheus metrics to the configured remote write endpoint.\nfunc (rw *RemoteWriter) Push() error {\n\t// Gather metrics from registry\n\tmfs, err := rw.gatherer.Gather()\n\tif err == nil && rw.commonLabels != nil {\n\t\tfor _, mf := range mfs {\n\t\t\tfor _, metric := range mf.Metric {\n\t\t\t\tfor k, v := range rw.commonLabels {\n\t\t\t\t\tmetric.Label = append(metric.Label, &dto.LabelPair{\n\t\t\t\t\t\tName: proto.String(k),\n\t\t\t\t\t\tValue: proto.String(v),\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif err != nil || len(mfs) == 0 {\n\t\tswitch rw.errorHandling {\n\t\tcase AbortOnError:\n\t\t\treturn err\n\t\tcase ContinueOnError:\n\t\t\tif rw.logger != nil {\n\t\t\t\trw.logger.Println(\"continue on error:\", err)\n\t\t\t}\n\t\tdefault:\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Convert metrics to TimeSeries\n\ttsList, err := rw.ConvertMetricsToTimeSeries(mfs)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"convert metrics: %w\", err)\n\t}\n\n\tif len(tsList) == 0 {\n\t\treturn nil // No samples to push\n\t}\n\n\t// Send to remote write endpoint\n\twr := &prompb.WriteRequest{Timeseries: tsList}\n\tdata, err := wr.Marshal()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"marshal protobuf: %w\", err)\n\t}\n\n\tcompressed := snappy.Encode(nil, data)\n\treq, err := http.NewRequest(\"POST\", rw.url, bytes.NewReader(compressed))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"create request: %w\", err)\n\t}\n\n\treq.Header.Set(\"Content-Encoding\", \"snappy\")\n\treq.Header.Set(\"Content-Type\", \"application/x-protobuf\")\n\treq.Header.Set(\"X-Prometheus-Remote-Write-Version\", \"0.1.0\")\n\n\tif rw.auth != \"\" {\n\t\tif strings.Contains(rw.auth, \":\") {\n\t\t\tparts := strings.Split(rw.auth, \":\")\n\t\t\treq.SetBasicAuth(parts[0], parts[1])\n\t\t}\n\t}\n\n\tresp, err := rw.client.Do(req)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"send request: %w\", err)\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode/100 != 2 {\n\t\treturn fmt.Errorf(\"remote_write failed: %s\", resp.Status)\n\t}\n\n\treturn nil\n}\n\n// ConvertMetricsToTimeSeries converts Prometheus metric families to TimeSeries.\nfunc (rw *RemoteWriter) ConvertMetricsToTimeSeries(mfs []*dto.MetricFamily) ([]prompb.TimeSeries, error) {\n\tnow := model.Time(time.Now().UnixMilli())\n\tsamples, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{\n\t\tTimestamp: now,\n\t}, mfs...)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"extract samples: %w\", err)\n\t}\n\n\tvar tsList []prompb.TimeSeries\n\tfor _, sample := range samples {\n\t\t// Convert model.Metric to prompb.Label slice\n\t\tlabels := make([]prompb.Label, 0, len(sample.Metric))\n\t\tfor name, value := range sample.Metric {\n\t\t\tlabels = append(labels, prompb.Label{\n\t\t\t\tName: string(name),\n\t\t\t\tValue: string(value),\n\t\t\t})\n\t\t}\n\n\t\ttsList = append(tsList, prompb.TimeSeries{\n\t\t\tLabels: labels,\n\t\t\tSamples: []prompb.Sample{{\n\t\t\t\tValue: float64(sample.Value),\n\t\t\t\tTimestamp: int64(sample.Timestamp),\n\t\t\t}},\n\t\t})\n\t}\n\n\treturn tsList, nil\n}\n" }, { "path": "sdk/java/libjfs/remote_write_test.go", "content": "// Copyright 2025 JuiceFS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage main\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/golang/snappy\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tdto \"github.com/prometheus/client_model/go\"\n\t\"github.com/prometheus/prometheus/prompb\"\n\t\"google.golang.org/protobuf/proto\"\n)\n\n// mockLogger implements the Logger interface for testing.\ntype mockLogger struct {\n\tmessages []string\n}\n\nfunc (m *mockLogger) Println(v ...interface{}) {\n\tm.messages = append(m.messages, fmt.Sprint(v...))\n}\n\nfunc (m *mockLogger) Warnf(format string, args ...interface{}) {\n\tm.messages = append(m.messages, fmt.Sprintf(format, args...))\n}\n\nfunc TestNewRemoteWriter(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tconfig *RemoteWriteConfig\n\t\twantErr bool\n\t\terrMsg string\n\t}{\n\t\t{\n\t\t\tname: \"valid config\",\n\t\t\tconfig: &RemoteWriteConfig{\n\t\t\t\tURL: \"http://localhost:9090/api/v1/write\",\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"missing URL\",\n\t\t\tconfig: &RemoteWriteConfig{\n\t\t\t\tAuth: \"user:pass\",\n\t\t\t},\n\t\t\twantErr: true,\n\t\t\terrMsg: \"missing URL\",\n\t\t},\n\t\t{\n\t\t\tname: \"with all options\",\n\t\t\tconfig: &RemoteWriteConfig{\n\t\t\t\tURL: \"http://localhost:9090/api/v1/write\",\n\t\t\t\tAuth: \"user:pass\",\n\t\t\t\tInterval: 5 * time.Second,\n\t\t\t\tTimeout: 10 * time.Second,\n\t\t\t\tCommonLabels: map[string]string{\"job\": \"test\"},\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\trw, err := NewRemoteWriter(tt.config)\n\t\t\tif tt.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"NewRemoteWriter() expected error but got none\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif !strings.Contains(err.Error(), tt.errMsg) {\n\t\t\t\t\tt.Errorf(\"NewRemoteWriter() error = %v, want error containing %v\", err, tt.errMsg)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"NewRemoteWriter() unexpected error = %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif rw == nil {\n\t\t\t\tt.Errorf(\"NewRemoteWriter() returned nil\")\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Check defaults\n\t\t\tif rw.url != tt.config.URL {\n\t\t\t\tt.Errorf(\"NewRemoteWriter() url = %v, want %v\", rw.url, tt.config.URL)\n\t\t\t}\n\t\t\tif tt.config.Timeout == 0 && rw.timeout != defaultRemoteWriteTimeout {\n\t\t\t\tt.Errorf(\"NewRemoteWriter() timeout = %v, want %v\", rw.timeout, defaultRemoteWriteTimeout)\n\t\t\t}\n\t\t\tif tt.config.Gatherer == nil && rw.gatherer != prometheus.DefaultGatherer {\n\t\t\t\tt.Errorf(\"NewRemoteWriter() gatherer should be DefaultGatherer\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRemoteWriter_convertMetricsToTimeSeries(t *testing.T) {\n\t// Create test registry with various metric types\n\tregistry := prometheus.NewRegistry()\n\n\t// Counter\n\tcounter := prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"test_counter\",\n\t\tHelp: \"A test counter\",\n\t})\n\tcounter.Add(5)\n\tregistry.MustRegister(counter)\n\n\t// Gauge\n\tgauge := prometheus.NewGauge(prometheus.GaugeOpts{\n\t\tName: \"test_gauge\",\n\t\tHelp: \"A test gauge\",\n\t})\n\tgauge.Set(10)\n\tregistry.MustRegister(gauge)\n\n\t// Histogram\n\thistogram := prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"test_histogram\",\n\t\tHelp: \"A test histogram\",\n\t\tBuckets: []float64{0.1, 0.5, 1.0, 5.0},\n\t})\n\thistogram.Observe(0.3)\n\thistogram.Observe(0.8)\n\thistogram.Observe(2.0)\n\tregistry.MustRegister(histogram)\n\n\t// Summary\n\tsummary := prometheus.NewSummary(prometheus.SummaryOpts{\n\t\tName: \"test_summary\",\n\t\tHelp: \"A test summary\",\n\t\tObjectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},\n\t})\n\tsummary.Observe(0.2)\n\tsummary.Observe(0.6)\n\tsummary.Observe(1.5)\n\tregistry.MustRegister(summary)\n\n\trw := &RemoteWriter{\n\t\tcommonLabels: map[string]string{\"job\": \"test\"},\n\t}\n\n\tmfs, err := registry.Gather()\n\tif err == nil && rw.commonLabels != nil {\n\t\tfor _, mf := range mfs {\n\t\t\tfor _, metric := range mf.Metric {\n\t\t\t\tfor k, v := range rw.commonLabels {\n\t\t\t\t\tmetric.Label = append(metric.Label, &dto.LabelPair{\n\t\t\t\t\t\tName: proto.String(k),\n\t\t\t\t\t\tValue: proto.String(v),\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to gather metrics: %v\", err)\n\t}\n\n\ttsList, err := rw.ConvertMetricsToTimeSeries(mfs)\n\tif err != nil {\n\t\tt.Fatalf(\"convertMetricsToTimeSeries() error = %v\", err)\n\t}\n\n\tif len(tsList) == 0 {\n\t\tt.Fatalf(\"convertMetricsToTimeSeries() returned empty time series\")\n\t}\n\n\t// Check that we have the expected metrics\n\tmetricNames := make(map[string]bool)\n\tfor _, ts := range tsList {\n\t\tfor _, label := range ts.Labels {\n\t\t\tif label.Name == \"__name__\" {\n\t\t\t\tmetricNames[label.Value] = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\texpectedMetrics := []string{\"test_counter\", \"test_gauge\", \"test_histogram_bucket\", \"test_histogram_sum\", \"test_histogram_count\", \"test_summary\", \"test_summary_sum\", \"test_summary_count\"}\n\tfor _, expected := range expectedMetrics {\n\t\tif !metricNames[expected] {\n\t\t\tt.Errorf(\"Expected metric %s not found in time series\", expected)\n\t\t}\n\t}\n\n\t// Check that common labels are added\n\tfor _, ts := range tsList {\n\t\thasJobLabel := false\n\t\tfor _, label := range ts.Labels {\n\t\t\tif label.Name == \"job\" && label.Value == \"test\" {\n\t\t\t\thasJobLabel = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !hasJobLabel {\n\t\t\tt.Errorf(\"Common label 'job=test' not found in time series\")\n\t\t}\n\t}\n}\n\nfunc TestRemoteWriter_Push(t *testing.T) {\n\t// Create a test server\n\tvar receivedData []byte\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.Method != \"POST\" {\n\t\t\tt.Errorf(\"Expected POST request, got %s\", r.Method)\n\t\t}\n\t\tif r.Header.Get(\"Content-Encoding\") != \"snappy\" {\n\t\t\tt.Errorf(\"Expected snappy encoding\")\n\t\t}\n\t\tif r.Header.Get(\"Content-Type\") != \"application/x-protobuf\" {\n\t\t\tt.Errorf(\"Expected protobuf content type\")\n\t\t}\n\n\t\t// Read and decompress the body\n\t\tbuf := make([]byte, r.ContentLength)\n\t\tr.Body.Read(buf)\n\t\treceivedData, _ = snappy.Decode(nil, buf)\n\n\t\tw.WriteHeader(http.StatusOK)\n\t}))\n\tdefer server.Close()\n\n\t// Create test registry\n\tregistry := prometheus.NewRegistry()\n\t// Counter\n\tcounter := prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"test_counter\",\n\t\tHelp: \"A test counter\",\n\t})\n\tcounter.Add(5)\n\tregistry.MustRegister(counter)\n\n\t// Gauge\n\tgauge := prometheus.NewGauge(prometheus.GaugeOpts{\n\t\tName: \"test_gauge\",\n\t\tHelp: \"A test gauge\",\n\t})\n\tgauge.Set(10)\n\tregistry.MustRegister(gauge)\n\n\t// Histogram\n\thistogram := prometheus.NewHistogram(prometheus.HistogramOpts{\n\t\tName: \"test_histogram\",\n\t\tHelp: \"A test histogram\",\n\t\tBuckets: []float64{0.1, 0.5, 1.0, 5.0},\n\t})\n\thistogram.Observe(0.3)\n\thistogram.Observe(0.8)\n\thistogram.Observe(2.0)\n\tregistry.MustRegister(histogram)\n\n\t// Summary\n\tsummary := prometheus.NewSummary(prometheus.SummaryOpts{\n\t\tName: \"test_summary\",\n\t\tHelp: \"A test summary\",\n\t\tObjectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},\n\t})\n\tsummary.Observe(0.2)\n\tsummary.Observe(0.6)\n\tsummary.Observe(1.5)\n\tregistry.MustRegister(summary)\n\n\tlogger := &mockLogger{}\n\trw, err := NewRemoteWriter(&RemoteWriteConfig{\n\t\tURL: server.URL,\n\t\tGatherer: registry,\n\t\tLogger: logger,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"NewRemoteWriter() error = %v\", err)\n\t}\n\n\terr = rw.Push()\n\tif err != nil {\n\t\tt.Errorf(\"Push() error = %v\", err)\n\t}\n\n\tif len(receivedData) == 0 {\n\t\tt.Errorf(\"No data received by server\")\n\t}\n\n\t// Verify the received data can be unmarshaled\n\tvar wr prompb.WriteRequest\n\tif err := wr.Unmarshal(receivedData); err != nil {\n\t\tt.Errorf(\"Failed to unmarshal received data: %v\", err)\n\t}\n\n\tif len(wr.Timeseries) == 0 {\n\t\tt.Errorf(\"No time series in received data\")\n\t}\n}\n\nfunc TestRemoteWriter_PushWithAuth(t *testing.T) {\n\tvar authHeader string\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tauthHeader = r.Header.Get(\"Authorization\")\n\t\tw.WriteHeader(http.StatusOK)\n\t}))\n\tdefer server.Close()\n\n\tregistry := prometheus.NewRegistry()\n\tcounter := prometheus.NewCounter(prometheus.CounterOpts{\n\t\tName: \"test_metric\",\n\t\tHelp: \"A test metric\",\n\t})\n\tcounter.Add(1)\n\tregistry.MustRegister(counter)\n\n\trw, err := NewRemoteWriter(&RemoteWriteConfig{\n\t\tURL: server.URL,\n\t\tAuth: \"testuser:testpass\",\n\t\tGatherer: registry,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"NewRemoteWriter() error = %v\", err)\n\t}\n\n\terr = rw.Push()\n\tif err != nil {\n\t\tt.Errorf(\"Push() error = %v\", err)\n\t}\n\n\tif !strings.Contains(authHeader, \"Basic\") {\n\t\tt.Errorf(\"Expected Basic auth header, got: %s\", authHeader)\n\t}\n}\n" }, { "path": "sdk/java/pom.xml", "content": "\n\t4.0.0\n\tio.juicefs\n\tjuicefs-hadoop\n\thttps://github.com/juicedata/juicefs\n\tHadoop FileSystem implementation for JuiceFS\n\tjuicefs-hadoop\n\t1.4-dev\n\tjar\n\t\n\t\t3.1.4\n\t\t1.16.3\n\t\t-Djdk.net.URLClassPath.disableClassPathURLCheck=true\n\t\t\t-Djava.library.path=${project.basedir}/../mount/libjfs:${java.library.path}\n\t\t\t-Djdk.attach.allowAttachSelf=true\n\t\n\n\t\n\t\t\n\t\t\tJuicedata\n\t\t\tteam@juicedata.io\n\t\t\n\t\n\n\t\n\t\t\n\t\t\tThe Apache Software License, Version 2.0\n\t\t\thttps://www.apache.org/licenses/LICENSE-2.0.txt\n\t\t\trepo\n\t\t\n\t\n\n\t\n\t\thttps://github.com/juicedata/juicefs\n\t\thttps://github.com/juicedata/juicefs\n\t\tscm:git:https://github.com/juicedata/juicefs\n\t\n\n\t\n\t\t\n\t\t\t\n\t\t\t\tmaven-surefire-plugin\n\t\t\t\t2.19.1\n\t\t\t\t\n\t\t\t\t\t${argLine}\n\t\t\t\t\tfalse\n\t\t\t\t\t\n\t\t\t\t\t\t${project.build.directory}/test-classes\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.apache.maven.plugins\n\t\t\t\tmaven-shade-plugin\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tpackage\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tshade\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t${artifactId}-${version}\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\torg.objectweb.asm\n\t\t\t\t\t\t\tio.juicefs.shaded.org.objectweb.asm\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tcom.beust\n\t\t\t\t\t\t\tio.juicefs.shaded.com.beust\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\torg.json\n\t\t\t\t\t\t\tio.juicefs.shaded.org.json\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\torg.javassist\n\t\t\t\t\t\t\tio.juicefs.shaded.org.javassist\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tcom.google.common\n\t\t\t\t\t\t\tio.juicefs.shaded.com.google.common\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\torg.apache.commons.lang\n\t\t\t\t\t\t\tio.juicefs.shaded.org.apache.commons.lang\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tcom.kstruct.gethostname4j\n\t\t\t\t\t\t\tio.juicefs.shaded.com.kstruct.gethostname4j\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.apache.maven.plugins\n\t\t\t\tmaven-compiler-plugin\n\t\t\t\t\n\t\t\t\t\t8\n\t\t\t\t\t8\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.sonatype.central\n\t\t\t\tcentral-publishing-maven-plugin\n\t\t\t\t0.8.0\n\t\t\t\ttrue\n\t\t\t\t\n\t\t\t\t\tcentral\n\t\t\t\t\tfalse\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.apache.maven.plugins\n\t\t\t\tmaven-javadoc-plugin\n\t\t\t\t2.9.1\n\t\t\t\t\n\t\t\t\t\t8\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tattach-javadocs\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tjar\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\n\t\t\t\n\t\t\t\torg.apache.maven.plugins\n\t\t\t\tmaven-source-plugin\n\t\t\t\t2.2.1\n\t\t\t\t\n\t\t\t\t\ttrue\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tattach-sources\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tjar-no-fork\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.apache.maven.plugins\n\t\t\t\tmaven-gpg-plugin\n\t\t\t\t1.6\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tsign-artifacts\n\t\t\t\t\t\tverify\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tsign\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t--pinentry-mode\n\t\t\t\t\t\tloopback\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.apache.maven.plugins\n\t\t\t\tmaven-jar-plugin\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\ttrue\n\t\t\t\t\t\t\tio.juicefs.Main\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.jacoco\n\t\t\t\tjacoco-maven-plugin\n\t\t\t\t0.8.7\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tprepare-agent\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\treport\n\t\t\t\t\t\ttest\n\t\t\t\t\t\t\n\t\t\t\t\t\t\treport\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\torg.pitest\n\t\t\t\tpitest-maven\n\t\t\t\t1.9.11\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tio.juicefs.JuiceFileSystemImpl*\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tio.juicefs.JuiceFileSystemTest\n\t\t\t\t\t\n\t\t\t\t\t1000\n\t\t\t\t\t\n\t\t\t\t\t\torg.apache.log4j\n\t\t\t\t\t\torg.slf4j\n\t\t\t\t\t\torg.apache.commons.logging\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\tio.github.git-commit-id\n\t\t\t\tgit-commit-id-maven-plugin\n\t\t\t\t4.9.9\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tget-the-git-infos\n\t\t\t\t\t\t\n\t\t\t\t\t\t\trevision\n\t\t\t\t\t\t\n\t\t\t\t\t\tinitialize\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\ttrue\n\t\t\t\t\t\n\t\t\t\t\t\t${project.build.outputDirectory}/juicefs-ver.properties\n\t\t\t\t\t\n\t\t\t\t\t\t^git.build.(time|version)$\n\t\t\t\t\t\t^git.commit.id.(abbrev|full)$\n\t\t\t\t\t\n\t\t\t\t\tfull\n\t\t\t\t\n\t\t\t\n\t\t\n\t\t\n\t\t\t\n\t\t\t\tlibjfs/target\n\t\t\t\n\t\t\t\n\t\t\t\tsrc/main/resources\n\t\t\t\n\t\t\n\t\t\n\t\t\t\n\t\t\t\tconf\n\t\t\t\n\t\t\t\n\t\t\t\tsrc/test/resources\n\t\t\t\n\t\t\n\t\n\t\n\t\t\n\t\t\tcom.github.jnr\n\t\t\tjnr-ffi\n\t\t\t2.2.12\n\t\t\n\t\t\n\t\t\torg.apache.hadoop\n\t\t\thadoop-common\n\t\t\t${hadoop.version}\n\t\t\tprovided\n\t\t\n\t\t\n\t\t\torg.apache.hadoop\n\t\t\thadoop-common\n\t\t\t${hadoop.version}\n\t\t\ttest\n\t\t\ttest-jar\n\t\t\n\t\t\n\t\t\torg.apache.hadoop\n\t\t\thadoop-client\n\t\t\t${hadoop.version}\n\t\t\tprovided\n\t\t\n\t\t\n\t\t\tjunit\n\t\t\tjunit\n\t\t\t4.13.1\n\t\t\ttest\n\t\t\n\t\t\n\t\t\torg.json\n\t\t\tjson\n\t\t\t20180813\n\t\t\n\t\t\n\t\t\torg.javassist\n\t\t\tjavassist\n\t\t\t3.25.0-GA\n\t\t\n\t\t\n\t\t\tcom.beust\n\t\t\tjcommander\n\t\t\t1.81\n\t\t\n\t\t\n\t\t\torg.apache.hive\n\t\t\thive-metastore\n\t\t\t\n\t\t\t\t\n\t\t\t\t\torg.apache.hadoop\n\t\t\t\t\thadoop-annotations\n\t\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\tprovided\n\t\t\t\n\t\t\t1.2.1\n\t\t\n\t\t\n\t\t\torg.apache.flink\n\t\t\tflink-hadoop-fs\n\t\t\t${flink.version}\n\t\t\tprovided\n\t\t\n\t\t\n\t\t\torg.apache.flink\n\t\t\tflink-core\n\t\t\t${flink.version}\n\t\t\tprovided\n\t\t\n\t\t\n\t\t\tcom.google.guava\n\t\t\tguava\n\t\t\t32.0.1-jre\n\t\t\n\t\t\n\t\t\torg.apache.hadoop\n\t\t\thadoop-minicluster\n\t\t\t${hadoop.version}\n\t\t\ttest\n\t\t\n\t\t\n\t\t\torg.apache.flink\n\t\t\tflink-streaming-java\n\t\t\t${flink.version}\n\t\t\ttest\n\t\t\n\t\t\n\t\t\torg.kitesdk\n\t\t\tkite-data-core\n\t\t\t1.1.0\n\t\t\tprovided\n\t\t\t\n\t\t\t\t\n\t\t\t\t\t*\n\t\t\t\t\t*\n\t\t\t\t\n\t\t\t\n\t\t\n\t\t\n\t\t\tcommons-lang\n\t\t\tcommons-lang\n\t\t\t2.6\n\t\t\n\t\t\n\t\t\torg.apache.ranger\n\t\t\tranger-plugins-common\n\t\t\t2.3.0\n\t\t\t\n\t\t\t\t\n\t\t\t\t\t*\n\t\t\t\t\t*\n\t\t\t\t\n\t\t\t\n\t\t\n\t\t\n\t\t\tcom.kstruct\n\t\t\tgethostname4j\n\t\t\t0.0.2\n\t\t\n\t\t\n\t\t\tcom.sun.jersey\n\t\t\tjersey-bundle\n\t\t\t1.19.3\n\t\t\n\t\t\n\t\t\torg.codehaus.jackson\n\t\t\tjackson-jaxrs\n\t\t\t1.9.13\n\t\t\n\t\t\n\t\t\torg.apache.ranger\n\t\t\tranger-plugins-audit\n\t\t\t2.3.0\n\t\t\t\n\t\t\t\t\n\t\t\t\t\t*\n\t\t\t\t\t*\n\t\t\t\t\n\t\t\t\n\t\t\n\t\n\n" }, { "path": "sdk/java/src/main/java/io/juicefs/FlinkFileSystemFactory.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs;\n\nimport org.apache.flink.configuration.Configuration;\nimport org.apache.flink.core.fs.FileSystem;\nimport org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.IOException;\nimport java.net.URI;\n\npublic class FlinkFileSystemFactory implements org.apache.flink.core.fs.FileSystemFactory {\n private static final Logger LOG = LoggerFactory.getLogger(FlinkFileSystemFactory.class);\n private org.apache.hadoop.conf.Configuration conf;\n\n private static final String[] FLINK_CONFIG_PREFIXES = {\"fs.\", \"juicefs.\"};\n private String scheme;\n\n @Override\n public void configure(Configuration config) {\n conf = new org.apache.hadoop.conf.Configuration();\n if (config != null) {\n for (String key : config.keySet()) {\n for (String prefix : FLINK_CONFIG_PREFIXES) {\n if (key.startsWith(prefix)) {\n String value = config.getString(key, null);\n if (value != null) {\n if (\"io.juicefs.JuiceFileSystem\".equals(value.trim())) {\n this.scheme = key.split(\"\\\\.\")[1];\n }\n conf.set(key, value);\n }\n }\n }\n }\n }\n }\n\n @Override\n public String getScheme() {\n if (scheme == null) {\n return \"jfs\";\n }\n return scheme;\n }\n\n @Override\n public FileSystem create(URI fsUri) throws IOException {\n JuiceFileSystem fs = new JuiceFileSystem();\n fs.initialize(fsUri, conf);\n return new HadoopFileSystem(fs);\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/JuiceFS.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.DelegateToFileSystem;\nimport org.apache.hadoop.fs.FileSystem;\n\nimport java.io.IOException;\nimport java.net.URI;\nimport java.net.URISyntaxException;\n\npublic class JuiceFS extends DelegateToFileSystem {\n JuiceFS(final URI uri, final Configuration conf) throws IOException, URISyntaxException {\n super(uri, FileSystem.get(uri, conf), conf, uri.getScheme(), false);\n }\n\n @Override\n public int getUriDefaultPort() {\n return -1;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/JuiceFileSystem.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs;\n\nimport io.juicefs.utils.BgTaskUtil;\nimport io.juicefs.utils.PatchUtil;\nimport org.apache.hadoop.classification.InterfaceAudience;\nimport org.apache.hadoop.classification.InterfaceStability;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.*;\nimport org.apache.hadoop.fs.permission.FsPermission;\nimport org.apache.hadoop.security.UserGroupInformation;\nimport org.apache.hadoop.security.token.Token;\nimport org.apache.hadoop.util.Progressable;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.IOException;\nimport java.net.URI;\nimport java.security.PrivilegedExceptionAction;\nimport java.util.concurrent.TimeUnit;\n\n/****************************************************************\n * Implement the FileSystem API for JuiceFS\n *****************************************************************/\n@InterfaceAudience.Public\n@InterfaceStability.Stable\npublic class JuiceFileSystem extends FilterFileSystem {\n private static final Logger LOG = LoggerFactory.getLogger(JuiceFileSystem.class);\n\n private static boolean fileChecksumEnabled = false;\n private static boolean distcpPatched = false;\n\n static {\n PatchUtil.patchBefore(\"org.apache.flink.runtime.fs.hdfs.HadoopRecoverableFsDataOutputStream\",\n \"waitUntilLeaseIsRevoked\",\n new String[]{\"org.apache.hadoop.fs.FileSystem\", \"org.apache.hadoop.fs.Path\"},\n \"if (fs instanceof io.juicefs.JuiceFileSystem) {\\n\" +\n \" return ((io.juicefs.JuiceFileSystem)fs).isFileClosed(path);\\n\" +\n \" }\");\n }\n\n private synchronized static void patchDistCpChecksum() {\n if (distcpPatched)\n return;\n PatchUtil.patchBefore(\"org.apache.hadoop.tools.mapred.RetriableFileCopyCommand\",\n \"compareCheckSums\",\n null,\n \"if (sourceFS.getFileStatus(source).getBlockSize() != targetFS.getFileStatus(target).getBlockSize()) {return ;}\");\n distcpPatched = true;\n }\n\n @Override\n public void initialize(URI uri, Configuration conf) throws IOException {\n super.initialize(uri, conf);\n fileChecksumEnabled = Boolean.parseBoolean(getConf(conf, \"file.checksum\", \"false\"));\n boolean asBgTask = conf.getBoolean(\"juicefs.internal-bg-task\", false);\n if (!asBgTask && !Boolean.parseBoolean(getConf(conf, \"disable-trash-emptier\", \"false\"))) {\n BgTaskUtil.startTrashEmptier(uri.getHost(), () -> {\n runTrashEmptier(uri, conf);\n }, 10, TimeUnit.MINUTES);\n }\n }\n\n private void runTrashEmptier(URI uri, final Configuration conf) {\n try {\n Configuration newConf = new Configuration(conf);\n newConf.setBoolean(\"juicefs.internal-bg-task\", true);\n UserGroupInformation superUser = UserGroupInformation.createRemoteUser(getConf(conf, \"superuser\", \"hdfs\"));\n FileSystem emptierFs = superUser.doAs((PrivilegedExceptionAction) () -> {\n JuiceFileSystemImpl fs = new JuiceFileSystemImpl();\n fs.initialize(uri, newConf);\n return fs;\n });\n new Trash(emptierFs, newConf).getEmptier().run();\n } catch (Exception e) {\n LOG.warn(\"run trash emptier for {} failed\", uri.getHost(), e);\n }\n }\n\n private String getConf(Configuration conf, String key, String value) {\n String name = fs.getUri().getHost();\n String v = conf.get(\"juicefs.\" + key, value);\n if (name != null && !name.equals(\"\")) {\n v = conf.get(\"juicefs.\" + name + \".\" + key, v);\n }\n if (v != null)\n v = v.trim();\n return v;\n }\n\n public JuiceFileSystem() {\n super(new JuiceFileSystemImpl());\n }\n\n @Override\n public String getScheme() {\n StackTraceElement[] elements = Thread.currentThread().getStackTrace();\n if (elements[2].getClassName().equals(\"org.apache.flink.runtime.fs.hdfs.HadoopRecoverableWriter\") &&\n (elements[2].getMethodName().equals(\"\") || elements[2].getMethodName().equals(\"checkSupportedFSSchemes\"))) {\n return \"hdfs\";\n }\n return fs.getScheme();\n }\n\n public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {\n return fs.create(f, FsPermission.getFileDefault(), overwrite, bufferSize, replication, blockSize, progress);\n }\n\n public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {\n return fs.createNonRecursive(f, FsPermission.getFileDefault(), overwrite, bufferSize, replication, blockSize, progress);\n }\n\n @Override\n public ContentSummary getContentSummary(Path f) throws IOException {\n return fs.getContentSummary(f);\n }\n\n public boolean isFileClosed(final Path src) throws IOException {\n FileStatus st = fs.getFileStatus(src);\n return st.getLen() > 0;\n }\n\n @Override\n public FileChecksum getFileChecksum(Path f, long length) throws IOException {\n if (!fileChecksumEnabled)\n return null;\n patchDistCpChecksum();\n return super.getFileChecksum(f, length);\n }\n\n @Override\n public FileChecksum getFileChecksum(Path f) throws IOException {\n if (!fileChecksumEnabled)\n return null;\n patchDistCpChecksum();\n return super.getFileChecksum(f);\n }\n\n public Token getDelegationToken(String renewer) throws IOException {\n return fs.getDelegationToken(renewer);\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs;\n\nimport com.google.common.collect.Lists;\nimport com.kenai.jffi.internal.StubLoader;\nimport io.juicefs.exception.QuotaExceededException;\nimport io.juicefs.kerberos.AuthCredential;\nimport io.juicefs.kerberos.JuiceFSDelegationTokenIdentifier;\nimport io.juicefs.kerberos.KerberosUtil;\nimport io.juicefs.metrics.JuiceFSInstrumentation;\nimport io.juicefs.permission.RangerConfig;\nimport io.juicefs.permission.RangerPermissionChecker;\nimport io.juicefs.utils.*;\nimport jnr.ffi.LibraryLoader;\nimport jnr.ffi.Memory;\nimport jnr.ffi.Pointer;\nimport jnr.ffi.Runtime;\nimport jnr.ffi.annotations.Delegate;\nimport jnr.ffi.annotations.In;\nimport jnr.ffi.annotations.Out;\nimport org.apache.hadoop.HadoopIllegalArgumentException;\nimport org.apache.hadoop.classification.InterfaceAudience;\nimport org.apache.hadoop.classification.InterfaceStability;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.*;\nimport org.apache.hadoop.fs.permission.*;\nimport org.apache.hadoop.io.DataOutputBuffer;\nimport org.apache.hadoop.io.MD5Hash;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.security.AccessControlException;\nimport org.apache.hadoop.security.HadoopKerberosName;\nimport org.apache.hadoop.security.SecurityUtil;\nimport org.apache.hadoop.security.UserGroupInformation;\nimport org.apache.hadoop.security.token.Token;\nimport org.apache.hadoop.security.token.TokenIdentifier;\nimport org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;\nimport org.apache.hadoop.util.DataChecksum;\nimport org.apache.hadoop.util.DirectBufferPool;\nimport org.apache.hadoop.util.Progressable;\nimport org.apache.hadoop.util.VersionInfo;\nimport org.json.JSONObject;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.*;\nimport java.lang.reflect.Constructor;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.net.*;\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.util.*;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.jar.JarFile;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport java.util.stream.Collectors;\nimport java.util.zip.GZIPInputStream;\nimport java.util.zip.ZipEntry;\n\n/****************************************************************\n * Implement the FileSystem API for JuiceFS\n *****************************************************************/\n@InterfaceAudience.Public\n@InterfaceStability.Stable\npublic class JuiceFileSystemImpl extends FileSystem {\n\n public static final Logger LOG = LoggerFactory.getLogger(JuiceFileSystemImpl.class);\n public static final String gitVer = loadVersion();\n\n static String loadVersion() {\n try (InputStream in = JuiceFileSystemImpl.class.getClassLoader().getResourceAsStream(\"juicefs-ver.properties\")) {\n Properties prop = new Properties();\n prop.load(in);\n return prop.getProperty(\"git.commit.id.abbrev\");\n } catch (IOException e) {\n LOG.warn(\"Failed to load juicefs-ver.properties\", e);\n return \"unknown\";\n }\n }\n\n private Path workingDir;\n private String name;\n private String user;\n private String superuser;\n private String supergroup;\n private URI uri;\n private long blocksize;\n private int minBufferSize;\n private int cacheReplica;\n private boolean fileChecksumEnabled;\n private final boolean isSuperGroupFileSystem;\n private boolean isBackGroundTask = false;\n\n private JuiceFileSystemImpl superGroupFileSystem;\n private RangerPermissionChecker rangerPermissionChecker;\n private boolean dtEnabled; // whether delegation token was enabled\n private static Libjfs lib = loadLibrary();\n\n private long handle;\n private UserGroupInformation ugi;\n private String homeDirPrefix = \"/user\";\n private String discoverNodesUrl;\n private static final Map> cachedHostsForName = new ConcurrentHashMap<>(); // (name -> (ip -> hostname))\n private static final Map> hashForName = new ConcurrentHashMap<>(); // (name -> consistentHash)\n private static final Map lastFileStatus = new ConcurrentHashMap<>();\n\n private FsPermission uMask;\n private String hflushMethod;\n\n private static final DirectBufferPool directBufferPool = new DirectBufferPool();\n\n private boolean metricsEnable = false;\n\n /*\n * hadoop compatibility\n */\n private boolean withStreamCapability;\n private Constructor fileStatusConstructor;\n\n // constructor for BufferedFSOutputStreamWithStreamCapabilities\n private Constructor constructor;\n private Method setStorageIds;\n private String[] storageIds;\n private Random random = new Random();\n\n private static final String USERNAME_UID_PATTERN = \"[a-zA-Z0-9_-]+:[0-9]+\";\n private static final String GROUPNAME_GID_USERNAMES_PATTERN = \"[a-zA-Z0-9_-]+:[0-9]+:[,a-zA-Z0-9_-]+\";\n\n /*\n go call back\n */\n private static Libjfs.LogCallBack callBack;\n\n public static interface Libjfs {\n long jfs_init(Pointer credential, int size, String name, String jsonConf, String user, String group, String superuser, String supergroup);\n\n void jfs_update_uid_grouping(String name, String uidstr, String grouping);\n\n int jfs_term(long pid, long h);\n\n int jfs_open(long pid, long h, String path, @Out ByteBuffer fileLen, int flags);\n\n int jfs_access(long pid, long h, String path, int flags);\n\n long jfs_lseek(long pid, int fd, long pos, int whence);\n\n int jfs_pread(long pid, int fd, @Out ByteBuffer b, int len, long offset);\n\n int jfs_write(long pid, int fd, @In ByteBuffer b, int len);\n\n int jfs_flush(long pid, int fd);\n\n int jfs_fsync(long pid, int fd);\n\n int jfs_close(long pid, int fd);\n\n int jfs_create(long pid, long h, String path, short mode, short umask);\n\n int jfs_truncate(long pid, long h, String path, long length);\n\n int jfs_delete(long pid, long h, String path);\n\n int jfs_rmr(long pid, long h, String path);\n\n int jfs_mkdir(long pid, long h, String path, short mode, short umask);\n\n int jfs_rename(long pid, long h, String src, String dst);\n\n int jfs_stat1(long pid, long h, String path, Pointer buf);\n\n int jfs_lstat1(long pid, long h, String path, Pointer buf);\n\n int jfs_summary(long pid, long h, String path, Pointer buf);\n\n int jfs_statvfs(long pid, long h, Pointer buf);\n\n int jfs_chmod(long pid, long h, String path, int mode);\n\n int jfs_setOwner(long pid, long h, String path, String user, String group);\n\n int jfs_utime(long pid, long h, String path, long mtime, long atime);\n\n int jfs_listdir(long pid, long h, String path, int offset, Pointer buf, int size);\n\n int jfs_concat(long pid, long h, String path, Pointer buf, int bufsize);\n\n int jfs_setXattr(long pid, long h, String path, String name, Pointer value, int vlen, int mode);\n\n int jfs_getXattr(long pid, long h, String path, String name, Pointer buf, int size);\n\n int jfs_listXattr(long pid, long h, String path, Pointer buf, int size);\n\n int jfs_removeXattr(long pid, long h, String path, String name);\n\n int jfs_getfacl(long pid, long h, String path, int acltype, Pointer b, int len);\n\n int jfs_setfacl(long pid, long h, String path, int acltype, Pointer b, int len);\n\n int jfs_getGroups(String volName, String user, Pointer buf, int len);\n\n int jfs_ranger_cfg(String volName, Pointer buf, int size);\n\n int jfs_is_superuser(long h, String user, String group);\n\n void jfs_set_callback(LogCallBack callBack);\n\n int jfs_get_token(long h, String name, Pointer buf, int bufSize, String renewer);\n\n long jfs_renew_token(long h, int id, String password);\n\n int jfs_cancel_token(long h, int id, String password);\n\n interface LogCallBack {\n @Delegate\n void call(String msg);\n }\n }\n\n static class LogCallBackImpl implements Libjfs.LogCallBack {\n Libjfs lib;\n\n public LogCallBackImpl(Libjfs lib) {\n this.lib = lib;\n }\n\n @Override\n public void call(String msg){\n try {\n // 2022/12/20 14:48:30.808303 juicefs[80976] : error msg [main.go:357]\n msg = msg.trim();\n String[] items = msg.split(\"\\\\s+\", 5);\n if (items.length > 4) {\n switch (items[3]) {\n case \":\":\n LOG.debug(msg);\n break;\n case \":\":\n LOG.info(msg);\n break;\n case \":\":\n LOG.warn(msg);\n break;\n case \":\":\n LOG.error(msg);\n break;\n }\n }\n } catch (Throwable ignored){}\n }\n\n @Override\n protected void finalize() throws Throwable {\n lib.jfs_set_callback(null);\n }\n }\n\n static int EPERM = -0x01;\n static int ENOENT = -0x02;\n static int EINTR = -0x04;\n static int EIO = -0x05;\n static int EACCESS = -0xd;\n static int EEXIST = -0x11;\n static int ENOTDIR = -0x14;\n static int EINVAL = -0x16;\n static int ENOSPACE = -0x1c;\n static int EDQUOT = -0x45;\n static int EROFS = -0x1e;\n static int ENOTEMPTY = -0x27;\n static int ENODATA = -0x3d;\n static int ENOATTR = -0x5d;\n static int ENOTSUP = -0x5f;\n\n static int MODE_MASK_R = 4;\n static int MODE_MASK_W = 2;\n static int MODE_MASK_X = 1;\n\n private IOException error(int errno, Path p) {\n String pStr = p == null ? \"\" : p.toString();\n if (errno == EPERM) {\n return new PathPermissionException(pStr);\n } else if (errno == ENOTDIR) {\n return new ParentNotDirectoryException();\n } else if (errno == ENOENT) {\n return new FileNotFoundException(pStr+ \": not found\");\n } else if (errno == EACCESS) {\n try {\n FileStatus stat = getFileStatusInternalNoException(p);\n if (stat != null) {\n FsPermission perm = stat.getPermission();\n return new AccessControlException(String.format(\"Permission denied: user=%s, path=\\\"%s\\\":%s:%s:%s%s\", user, p,\n stat.getOwner(), stat.getGroup(), stat.isDirectory() ? \"d\" : \"-\", perm));\n }\n } catch (Exception e) {\n LOG.warn(\"fail to generate better error message\", e);\n }\n return new AccessControlException(\"Permission denied: \" + pStr);\n } else if (errno == EEXIST) {\n return new FileAlreadyExistsException();\n } else if (errno == EINVAL) {\n return new InvalidRequestException(\"Invalid parameter\");\n } else if (errno == ENOTEMPTY) {\n return new PathIsNotEmptyDirectoryException(pStr);\n } else if (errno == EINTR) {\n return new InterruptedIOException();\n } else if (errno == ENOTSUP) {\n return new PathOperationException(pStr);\n } else if (errno == ENOSPACE) {\n return new IOException(\"No space\");\n } else if (errno == EDQUOT) {\n return new QuotaExceededException(\"Quota exceeded\");\n } else if (errno == EROFS) {\n return new IOException(\"Read-only Filesystem\");\n } else if (errno == EIO) {\n return new IOException(pStr);\n } else {\n return new IOException(\"errno: \" + errno + \" \" + pStr);\n }\n }\n\n public JuiceFileSystemImpl() {\n this.isSuperGroupFileSystem = false;\n }\n\n @Override\n public long getDefaultBlockSize() {\n return blocksize;\n }\n\n private String normalizePath(Path path) {\n return makeQualified(path).toUri().getPath();\n }\n\n @Override\n public String getScheme() {\n return uri.getScheme();\n }\n\n @Override\n public String toString() {\n return uri.toString();\n }\n\n @Override\n public URI getUri() {\n return uri;\n }\n\n private String getConf(Configuration conf, String key, String value) {\n String v = conf.get(\"juicefs.\" + key, value);\n if (name != null && !name.equals(\"\")) {\n v = conf.get(\"juicefs.\" + name + \".\" + key, v);\n }\n if (v != null)\n v = v.trim();\n return v;\n }\n\n @Override\n public void initialize(URI uri, Configuration conf) throws IOException {\n super.initialize(uri, conf);\n setConf(conf);\n\n this.uri = uri;\n name = conf.get(\"juicefs.name\", uri.getHost());\n if (null == name) {\n throw new IOException(\"name is required\");\n }\n\n blocksize = conf.getLongBytes(\"juicefs.block.size\", conf.getLongBytes(\"dfs.blocksize\", 128 << 20));\n minBufferSize = conf.getInt(\"juicefs.min-buffer-size\", 128 << 10);\n cacheReplica = Integer.parseInt(getConf(conf, \"cache-replica\", \"1\"));\n fileChecksumEnabled = Boolean.parseBoolean(getConf(conf, \"file.checksum\", \"false\"));\n\n this.ugi = UserGroupInformation.getCurrentUser();\n user = ugi.getShortUserName();\n String groupStr = \"nogroup\";\n if (ugi.getGroupNames().length > 0) {\n groupStr = String.join(\",\", ugi.getGroupNames());\n }\n superuser = getConf(conf, \"superuser\", \"hdfs\");\n supergroup = getConf(conf, \"supergroup\", conf.get(\"dfs.permissions.superusergroup\", \"supergroup\"));\n isBackGroundTask = conf.getBoolean(\"juicefs.internal-bg-task\", false);\n boolean asSuperFs = isSuperGroupFileSystem || isBackGroundTask;\n\n synchronized (JuiceFileSystemImpl.class) {\n if (callBack == null) {\n callBack = new LogCallBackImpl(lib);\n lib.jfs_set_callback(callBack);\n }\n }\n\n JSONObject obj = new JSONObject();\n String spn = SecurityUtil.getServerPrincipal(getConf(conf, \"server-principal\", \"\"), name);\n if (spn.contains(\"@\")) {\n spn = spn.split(\"@\")[0];\n }\n AuthCredential authCredential = buildAuthCredential(spn);\n Pointer credential = null;\n int crdSize = 0;\n if (authCredential != null) {\n crdSize = authCredential.getCredential().length;\n credential = Memory.allocate(Runtime.getRuntime(lib), crdSize);\n credential.put(0, authCredential.getCredential(), 0, crdSize);\n }\n\n if (authCredential != null) {\n obj.put(\"authMethod\", authCredential.getMethod());\n }\n if (ugi.getRealUser() != null) {\n obj.put(\"realUser\", ugi.getRealUser().getShortUserName());\n }\n\n String[] keys = new String[]{\"meta\",};\n for (String key : keys) {\n obj.put(key, getConf(conf, key, \"\"));\n }\n String[] bkeys = new String[]{\"debug\", \"writeback\"};\n for (String key : bkeys) {\n obj.put(key, Boolean.valueOf(getConf(conf, key, \"false\")));\n }\n String subdir = getConf(conf, \"subdir\", \"\");\n if (!subdir.isEmpty()) {\n // Support multiple subdirs separated by comma\n String[] subdirs = subdir.split(\",\");\n List normalizedSubdirs = new ArrayList<>();\n for (String sd : subdirs) {\n sd = sd.trim();\n if (sd.isEmpty() || sd.equals(\"/\")) {\n continue; // skip empty string or root\n }\n if (!sd.startsWith(\"/\")) {\n sd = \"/\" + sd;\n }\n sd = sd.replaceAll(\"/+$\", \"\");\n normalizedSubdirs.add(sd);\n }\n if (normalizedSubdirs.isEmpty()) {\n subdir = \"\";\n } else {\n subdir = String.join(\",\", normalizedSubdirs);\n LOG.debug(\"subdir {} is enabled\", subdir);\n }\n }\n obj.put(\"bucket\", getConf(conf, \"bucket\", \"\"));\n obj.put(\"storageClass\", getConf(conf, \"storage-class\", \"\"));\n obj.put(\"readOnly\", Boolean.valueOf(getConf(conf, \"read-only\", \"false\")));\n obj.put(\"noSession\", Boolean.valueOf(getConf(conf, \"no-session\", \"false\")));\n obj.put(\"noBGJob\", Boolean.valueOf(getConf(conf, \"no-bgjob\", \"false\")));\n obj.put(\"cacheDir\", getConf(conf, \"cache-dir\", \"memory\"));\n obj.put(\"cacheSize\", getConf(conf, \"cache-size\", \"100\"));\n obj.put(\"cacheItems\", Integer.valueOf(getConf(conf, \"cache-items\", \"0\")));\n obj.put(\"openCache\", getConf(conf, \"open-cache\", \"0.0\"));\n obj.put(\"backupMeta\", getConf(conf, \"backup-meta\", \"3600\"));\n obj.put(\"backupSkipTrash\", Boolean.valueOf(getConf(conf, \"backup-skip-trash\", \"false\")));\n obj.put(\"heartbeat\", getConf(conf, \"heartbeat\", \"12\"));\n obj.put(\"attrTimeout\", getConf(conf, \"attr-cache\", \"0.0\"));\n obj.put(\"entryTimeout\", getConf(conf, \"entry-cache\", \"0.0\"));\n obj.put(\"dirEntryTimeout\", getConf(conf, \"dir-entry-cache\", \"0.0\"));\n obj.put(\"cacheFullBlock\", Boolean.valueOf(getConf(conf, \"cache-full-block\", \"true\")));\n obj.put(\"cacheChecksum\", getConf(conf, \"verify-cache-checksum\", \"extend\"));\n obj.put(\"cacheEviction\", getConf(conf, \"cache-eviction\", \"2-random\"));\n obj.put(\"cacheScanInterval\", getConf(conf, \"cache-scan-interval\", \"300\"));\n obj.put(\"cacheExpire\", getConf(conf, \"cache-expire\", \"0\"));\n obj.put(\"autoCreate\", Boolean.valueOf(getConf(conf, \"auto-create-cache-dir\", \"true\")));\n obj.put(\"maxUploads\", Integer.valueOf(getConf(conf, \"max-uploads\", \"20\")));\n obj.put(\"maxDownloads\", Integer.valueOf(getConf(conf, \"max-downloads\", \"200\")));\n obj.put(\"maxDeletes\", Integer.valueOf(getConf(conf, \"max-deletes\", \"10\")));\n obj.put(\"skipDirNlink\", Integer.valueOf(getConf(conf, \"skip-dir-nlink\", \"20\")));\n obj.put(\"skipDirMtime\", getConf(conf, \"skip-dir-mtime\", \"100ms\"));\n obj.put(\"uploadLimit\", getConf(conf, \"upload-limit\", \"0\"));\n obj.put(\"downloadLimit\", getConf(conf, \"download-limit\", \"0\"));\n obj.put(\"ioRetries\", Integer.valueOf(getConf(conf, \"io-retries\", \"10\")));\n obj.put(\"getTimeout\", getConf(conf, \"get-timeout\", getConf(conf, \"object-timeout\", \"5\")));\n obj.put(\"putTimeout\", getConf(conf, \"put-timeout\", getConf(conf, \"object-timeout\", \"60\")));\n obj.put(\"memorySize\", getConf(conf, \"memory-size\", \"300\"));\n obj.put(\"prefetch\", Integer.valueOf(getConf(conf, \"prefetch\", \"1\")));\n obj.put(\"readahead\", getConf(conf, \"max-readahead\", \"0\"));\n obj.put(\"pushGateway\", getConf(conf, \"push-gateway\", \"\"));\n obj.put(\"pushInterval\", getConf(conf, \"push-interval\", \"10\"));\n obj.put(\"pushAuth\", getConf(conf, \"push-auth\", \"\"));\n obj.put(\"pushLabels\", getConf(conf, \"push-labels\", \"\"));\n obj.put(\"pushGraphite\", getConf(conf, \"push-graphite\", \"\"));\n obj.put(\"pushRemoteWrite\", getConf(conf, \"push-remote-write\", \"\"));\n obj.put(\"pushRemoteWriteAuth\", getConf(conf, \"push-remote-write-auth\", \"\"));\n obj.put(\"fastResolve\", Boolean.valueOf(getConf(conf, \"fast-resolve\", \"true\")));\n obj.put(\"noUsageReport\", Boolean.valueOf(getConf(conf, \"no-usage-report\", \"false\")));\n obj.put(\"freeSpace\", getConf(conf, \"free-space\", \"0.1\"));\n obj.put(\"accessLog\", getConf(conf, \"access-log\", \"\"));\n obj.put(\"superFs\", asSuperFs);\n obj.put(\"subdir\", subdir);\n String jsonConf = obj.toString(2);\n handle = lib.jfs_init(credential, crdSize, name, jsonConf, user, groupStr, superuser, supergroup);\n if (handle <= 0) {\n throw new IOException(\"JuiceFS initialized failed for jfs://\" + name);\n }\n if (isBackGroundTask) {\n LOG.debug(\"background fs {}|({})\", name, handle);\n } else {\n BgTaskUtil.register(name, handle);\n }\n discoverNodesUrl = getConf(conf, \"discover-nodes-url\", null);\n homeDirPrefix = conf.get(\"dfs.user.home.dir.prefix\", \"/user\");\n this.workingDir = getHomeDirectory();\n\n // hadoop29 and above check\n try {\n Class.forName(\"org.apache.hadoop.fs.StreamCapabilities\");\n withStreamCapability = true;\n } catch (ClassNotFoundException e) {\n withStreamCapability = false;\n }\n if (withStreamCapability) {\n try {\n constructor = Class.forName(\"io.juicefs.JuiceFileSystemImpl$BufferedFSOutputStreamWithStreamCapabilities\")\n .getConstructor(OutputStream.class, Integer.TYPE, String.class);\n } catch (ClassNotFoundException | NoSuchMethodException e) {\n throw new RuntimeException(e);\n }\n }\n // for hadoop compatibility\n boolean hasAclMtd = ReflectionUtil.hasMethod(FileStatus.class.getName(), \"hasAcl\", (String[]) null);\n if (hasAclMtd) {\n fileStatusConstructor = ReflectionUtil.getConstructor(FileStatus.class,\n long.class, boolean.class, int.class, long.class, long.class,\n long.class, FsPermission.class, String.class, String.class, Path.class,\n Path.class, boolean.class, boolean.class, boolean.class);\n if (fileStatusConstructor == null) {\n throw new IOException(\"incompatible hadoop version\");\n }\n }\n\n String umaskStr = getConf(conf, \"umask\", null);\n if (!isEmpty(umaskStr)) {\n conf.set(\"fs.permissions.umask-mode\", umaskStr);\n LOG.debug(\"override fs.permissions.umask-mode to {}\", umaskStr);\n }\n uMask = FsPermission.getUMask(conf);\n\n hflushMethod = getConf(conf, \"hflush\", \"writeback\");\n initializeStorageIds(conf);\n\n if (\"true\".equalsIgnoreCase(getConf(conf, \"enable-metrics\", \"false\"))) {\n metricsEnable = true;\n JuiceFSInstrumentation.init(this, statistics);\n }\n\n RangerConfig rangerConfig = checkAndGetRangerParams(conf);\n if (rangerConfig != null && !isSuperGroupFileSystem && !isBackGroundTask) {\n Configuration superConf = new Configuration(conf);\n superConf.set(\"juicefs.internal-bg-task\", \"true\");\n superGroupFileSystem = new JuiceFileSystemImpl(true);\n superGroupFileSystem.initialize(uri, superConf);\n rangerPermissionChecker = RangerPermissionChecker.acquire(name, handle, superGroupFileSystem, rangerConfig);\n }\n\n if (!isBackGroundTask && !isSuperGroupFileSystem) {\n // use juicefs.users and juicefs.groups for global mapping\n String uidFile = getConf(conf, \"users\", null);\n String groupFile = getConf(conf, \"groups\", null);\n if (!isEmpty(uidFile) || !isEmpty(groupFile)) {\n BgTaskUtil.putTask(name, \"Refresh guid\", () -> {\n updateUidAndGrouping(uidFile, groupFile);\n }, 1, 1, TimeUnit.MINUTES);\n }\n }\n }\n\n public RangerConfig checkAndGetRangerParams(Configuration conf) throws IOException {\n if (System.getenv(\"JUICEFS_RANGER_TEST\") != null) {\n RangerConfig config = new RangerConfig(\"http://localhost:6080\", \"ranger_test\", 30000);\n config.setImpl(\"io.juicefs.permission.RangerAdminClientImpl\");\n return config;\n }\n int size = 0, r = 1 << 10;\n Pointer buf = null;\n while (r > size) {\n size = r;\n buf = Memory.allocate(Runtime.getRuntime(lib), size);\n r = lib.jfs_ranger_cfg(name, buf, size);\n }\n if (r == 0) {\n return null;\n }\n byte[] rBuf = new byte[r];\n buf.get(0, rBuf, 0, r);\n String cfgStr = new String(rBuf);\n // http://localhost:6080?name=service_name\n String[] split = cfgStr.split(\"\\\\?\", -1);\n if (split.length != 2) {\n throw new IOException(String.format(\"wrong ranger config: %s\", cfgStr));\n }\n String url = split[0];\n String serviceName = split[1].substring(5);\n if (!url.startsWith(\"http\")) {\n throw new IOException(\"illegal value for parameter 'ranger-rest-url': \" + url);\n }\n if (serviceName.isEmpty()) {\n throw new IOException(\"illegal value for parameter 'ranger-service': \" + serviceName);\n }\n String pollIntervalMs = getConf(conf, \"ranger-poll-interval-ms\", \"30000\");\n return new RangerConfig(url, serviceName, Long.parseLong(pollIntervalMs));\n }\n\n public JuiceFileSystemImpl(boolean isSuperGroupFileSystem) {\n this.isSuperGroupFileSystem = isSuperGroupFileSystem;\n }\n\n private Set getGroups() {\n String groupsFile = getConf(getConf(), \"groups\", null);\n if (isEmpty(groupsFile)) {\n return new HashSet<>(ugi.getGroups());\n }\n\n int size = 0, r = 1 << 10;\n Pointer buf = null;\n while (r > size) {\n size = r;\n buf = Memory.allocate(Runtime.getRuntime(lib), size);\n r = lib.jfs_getGroups(name, user, buf, size);\n }\n if (r == 0) {\n return new HashSet<>(ugi.getGroups());\n }\n byte[] rBuf = new byte[r];\n buf.get(0, rBuf, 0, r);\n\n return new HashSet<>(Arrays.asList(new String(rBuf).split(\",\")));\n }\n\n private boolean isSuperUser() throws IOException {\n int r = lib.jfs_is_superuser(handle, user, String.join(\",\", getGroups()));\n if (r < 0) {\n throw new InvalidRequestException(\"Invalid parameter\");\n }\n return r == 1;\n }\n\n private boolean needCheckPermission() throws IOException {\n return rangerPermissionChecker != null && !isSuperGroupFileSystem && !isBackGroundTask && !isSuperUser() ;\n }\n\n private boolean checkPathAccess(Path path, FsAction action, String operation) throws IOException {\n return rangerPermissionChecker.checkPermission(path, false, null, null, action, operation, user, getGroups());\n }\n\n private boolean checkParentPathAccess(Path path, FsAction action, String operation) throws IOException {\n return rangerPermissionChecker.checkPermission(path, false, null, action, null, operation, user, getGroups());\n }\n\n private boolean checkAncestorAccess(Path path, FsAction action, String operation) throws IOException {\n return rangerPermissionChecker.checkPermission(path, false, action, null, null, operation, user, getGroups());\n }\n\n private boolean checkOwner(Path path, String operation) throws IOException {\n return rangerPermissionChecker.checkPermission(path, true, null, null, null, operation, user, getGroups());\n }\n\n private boolean isEmpty(String str) {\n return str == null || str.trim().isEmpty();\n }\n\n private String readFile(String file) throws IOException {\n Path path = new Path(file);\n FileStatus lastStatus = lastFileStatus.get(file);\n Configuration newConf = new Configuration(getConf());\n newConf.setBoolean(\"juicefs.internal-bg-task\", true);\n try (FileSystem fs = FileSystem.newInstance(path.toUri(), newConf)) {\n FileStatus status = fs.getFileStatus(path);\n if (lastStatus != null && status.getModificationTime() == lastStatus.getModificationTime()\n && status.getLen() == lastStatus.getLen()) {\n return null;\n }\n try (FSDataInputStream in = fs.open(path)) {\n String res = new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining(\"\\n\"));\n lastFileStatus.put(file, status);\n return res;\n }\n }\n }\n\n private String parseUidAndGrouping(String pattern, String input) {\n String result = null;\n if (input == null || \"\".equals(input.trim())) {\n return result;\n }\n List matched = new ArrayList<>();\n Matcher matcher = Pattern.compile(pattern).matcher(input);\n while (matcher.find()) {\n matched.add(matcher.group());\n }\n if (matched.size() > 0) {\n result = String.join(\"\\n\", matched);\n }\n return result;\n }\n\n private void updateUidAndGrouping(String uidFile, String groupFile) throws IOException {\n String uidstr = parseUidAndGrouping(USERNAME_UID_PATTERN, uidFile);\n if (uidstr == null && uidFile != null && !\"\".equals(uidFile.trim())) {\n uidstr = readFile(uidFile);\n }\n String grouping = parseUidAndGrouping(GROUPNAME_GID_USERNAMES_PATTERN, groupFile);\n if (grouping == null && groupFile != null && !\"\".equals(groupFile.trim())) {\n grouping = readFile(groupFile);\n }\n\n lib.jfs_update_uid_grouping(name, uidstr, grouping);\n }\n\n private void initializeStorageIds(Configuration conf) throws IOException {\n try {\n Class clazz = Class.forName(\"org.apache.hadoop.fs.BlockLocation\");\n setStorageIds = clazz.getMethod(\"setStorageIds\", String[].class);\n } catch (ClassNotFoundException e) {\n throw new IllegalStateException(\n \"Hadoop version was incompatible, current hadoop version is:\\t\" + VersionInfo.getVersion());\n } catch (NoSuchMethodException e) {\n setStorageIds = null;\n }\n int vdiskPerCpu = Integer.parseInt(getConf(conf, \"vdisk-per-cpu\", \"4\"));\n storageIds = new String[java.lang.Runtime.getRuntime().availableProcessors() * vdiskPerCpu];\n for (int i = 0; i < storageIds.length; i++) {\n storageIds[i] = \"vd\" + i;\n }\n }\n\n @Override\n public Path getHomeDirectory() {\n return makeQualified(new Path(homeDirPrefix + \"/\" + user));\n }\n\n private static void initStubLoader() {\n int loadMaxTime = 30;\n long start = System.currentTimeMillis();\n Class clazz = null;\n // first try\n try {\n clazz = Class.forName(\"com.kenai.jffi.internal.StubLoader\");\n } catch (ClassNotFoundException e) {\n }\n\n // try try try ...\n while (StubLoader.getFailureCause() != null && (System.currentTimeMillis() - start) < loadMaxTime * 1000) {\n LOG.warn(\"StubLoader load failed, it'll be retried!\");\n try {\n Thread.interrupted();\n Method load = clazz.getDeclaredMethod(\"load\");\n load.setAccessible(true);\n load.invoke(null);\n\n Field loaded = clazz.getDeclaredField(\"loaded\");\n loaded.setAccessible(true);\n loaded.set(null, true);\n\n Field failureCause = clazz.getDeclaredField(\"failureCause\");\n failureCause.setAccessible(true);\n failureCause.set(null, null);\n } catch (Throwable e) {\n }\n }\n\n if (StubLoader.getFailureCause() != null) {\n throw new RuntimeException(\"StubLoader load failed\", StubLoader.getFailureCause());\n }\n }\n\n public static Libjfs loadLibrary() {\n initStubLoader();\n\n LibraryLoader libjfsLibraryLoader = LibraryLoader.create(Libjfs.class);\n libjfsLibraryLoader.failImmediately();\n\n String osId = \"so\";\n String archId = \"amd64\";\n String resourceFormat = \"libjfs-%s.%s.gz\";\n String nameFormat = \"libjfs-%s.%s.%s\";\n\n File dir = new File(\"/tmp\");\n String os = System.getProperty(\"os.name\");\n String arch = System.getProperty(\"os.arch\");\n if (arch.contains(\"aarch64\")) {\n archId = \"arm64\";\n }\n if (os.toLowerCase().contains(\"windows\")) {\n osId = \"dll\";\n dir = new File(System.getProperty(\"java.io.tmpdir\"));\n } else if (os.toLowerCase().contains(\"mac\")) {\n osId = \"dylib\";\n }\n\n String resource = String.format(resourceFormat, archId, osId);\n String name = String.format(nameFormat, archId, gitVer, osId);\n\n File libFile = new File(dir, name);\n\n InputStream ins;\n long soTime;\n URL location = JuiceFileSystemImpl.class.getProtectionDomain().getCodeSource().getLocation();\n if (location == null) {\n // jar may changed\n return loadExistLib(libjfsLibraryLoader, dir, name, libFile);\n }\n URLConnection con;\n try {\n try {\n con = location.openConnection();\n } catch (FileNotFoundException e) {\n // jar may changed\n return loadExistLib(libjfsLibraryLoader, dir, name, libFile);\n }\n if (location.getProtocol().equals(\"jar\") && (con instanceof JarURLConnection)) {\n LOG.debug(\"juicefs-hadoop.jar is a nested jar\");\n JarURLConnection connection = (JarURLConnection) con;\n JarFile jfsJar = connection.getJarFile();\n ZipEntry entry = jfsJar.getJarEntry(resource);\n soTime = entry.getLastModifiedTime().toMillis();\n ins = jfsJar.getInputStream(entry);\n } else {\n URI locationUri;\n try {\n locationUri = location.toURI();\n } catch (URISyntaxException e) {\n return loadExistLib(libjfsLibraryLoader, dir, name, libFile);\n }\n if (Files.isDirectory(Paths.get(locationUri))) { // for debug: sdk/java/target/classes\n soTime = con.getLastModified();\n ins = JuiceFileSystemImpl.class.getClassLoader().getResourceAsStream(resource);\n } else {\n JarFile jfsJar;\n try {\n jfsJar = new JarFile(locationUri.getPath());\n } catch (FileNotFoundException fne) {\n return loadExistLib(libjfsLibraryLoader, dir, name, libFile);\n }\n ZipEntry entry = jfsJar.getJarEntry(resource);\n soTime = entry.getLastModifiedTime().toMillis();\n ins = jfsJar.getInputStream(entry);\n }\n }\n\n synchronized (JuiceFileSystemImpl.class) {\n if (!libFile.exists() || libFile.lastModified() < soTime) {\n // try the name for current user\n libFile = new File(dir, System.getProperty(\"user.name\") + \"-\" + name);\n if (!libFile.exists() || libFile.lastModified() < soTime) {\n InputStream reader = new GZIPInputStream(ins);\n File tmp = File.createTempFile(name, null, dir);\n FileOutputStream writer = new FileOutputStream(tmp);\n byte[] buffer = new byte[128 << 10];\n int bytesRead = 0;\n while ((bytesRead = reader.read(buffer)) != -1) {\n writer.write(buffer, 0, bytesRead);\n }\n writer.close();\n reader.close();\n tmp.setLastModified(soTime);\n tmp.setReadable(true, false);\n try {\n File org = new File(dir, name);\n Files.move(tmp.toPath(), org.toPath(), StandardCopyOption.ATOMIC_MOVE);\n libFile = org;\n } catch (Exception ade) {\n Files.move(tmp.toPath(), libFile.toPath(), StandardCopyOption.ATOMIC_MOVE);\n }\n }\n }\n }\n ins.close();\n } catch (Exception e) {\n throw new RuntimeException(\"Init libjfs failed\", e);\n }\n return libjfsLibraryLoader.load(libFile.getAbsolutePath());\n }\n\n private static Libjfs loadExistLib(LibraryLoader libjfsLibraryLoader, File dir, String name, File libFile) {\n File currentUserLib = new File(dir, System.getProperty(\"user.name\") + \"-\" + name);\n if (currentUserLib.exists()) {\n return libjfsLibraryLoader.load(currentUserLib.getAbsolutePath());\n } else {\n return libjfsLibraryLoader.load(libFile.getAbsolutePath());\n }\n }\n\n private void initCache() {\n try {\n List newNodes = discoverNodes(discoverNodesUrl);\n Map newCachedHosts = new HashMap<>();\n for (String newNode : newNodes) {\n try {\n newCachedHosts.put(InetAddress.getByName(newNode).getHostAddress(), newNode);\n } catch (UnknownHostException e) {\n LOG.warn(\"unknown host: \" + newNode);\n }\n }\n\n // if newCachedHosts are not changed, skip\n if (!newCachedHosts.equals(cachedHostsForName.get(name))) {\n List ips = new ArrayList<>(newCachedHosts.keySet());\n LOG.debug(\"update nodes to: \" + String.join(\",\", ips));\n hashForName.put(name, new ConsistentHash<>(100, ips));\n cachedHostsForName.put(name, newCachedHosts);\n }\n } catch (Throwable e) {\n LOG.warn(\"failed to discover nodes\", e);\n }\n }\n\n private List discoverNodes(String urls) {\n LOG.debug(\"fetching nodes from {}\", urls);\n Configuration newConf = new Configuration(getConf());\n newConf.setBoolean(\"juicefs.internal-bg-task\", true);\n NodesFetcher fetcher = NodesFetcherBuilder.buildFetcher(urls, name, newConf);\n List fetched = fetcher.fetchNodes(urls);\n if (fetched == null) {\n fetched = new ArrayList<>();\n }\n LOG.debug(\"fetched nodes: {}\", fetched);\n return fetched;\n }\n\n private BlockLocation makeLocation(long code, long start, long len) {\n long index = (start + len / 2) / blocksize / 4;\n BlockLocation blockLocation;\n String[] ns = new String[cacheReplica];\n String[] hs = new String[cacheReplica];\n\n Map cachedHosts = cachedHostsForName.get(name);\n ConsistentHash hash = hashForName.get(name);\n for (int i = 0; i < cacheReplica; i++) {\n String h = \"localhost\";\n if (cachedHosts != null && hash != null) {\n h = cachedHosts.getOrDefault(hash.get(code + \"-\" + (index + i)), \"localhost\");\n }\n ns[i] = h + \":50010\";\n hs[i] = h;\n }\n blockLocation = new BlockLocation(ns, hs, null, null, start, len, false);\n if (setStorageIds != null) {\n try {\n setStorageIds.invoke(blockLocation, (Object) getStorageIds());\n } catch (IllegalAccessException | InvocationTargetException e) {\n throw new RuntimeException(e);\n }\n }\n return blockLocation;\n }\n\n private String[] getStorageIds() {\n String[] res = new String[cacheReplica];\n for (int i = 0; i < cacheReplica; i++) {\n res[i] = storageIds[random.nextInt(storageIds.length)];\n }\n return res;\n }\n\n private void setStorageId(BlockLocation bl) {\n if (setStorageIds != null) {\n try {\n setStorageIds.invoke(bl, (Object) getStorageIds());\n } catch (IllegalAccessException | InvocationTargetException e) {\n throw new RuntimeException(e);\n }\n }\n }\n\n @Override\n public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {\n if (needCheckPermission() && !checkPathAccess(file.getPath(), FsAction.READ, \"getFileBlockLocations\")) {\n return superGroupFileSystem.getFileBlockLocations(file, start, len);\n }\n\n if (isEmpty(discoverNodesUrl) || cacheReplica <= 0) {\n BlockLocation[] bls = super.getFileBlockLocations(file, start, len);\n if (bls != null) {\n for (BlockLocation bl : bls) {\n setStorageId(bl);\n }\n }\n return bls;\n }\n\n if (file == null) {\n return null;\n }\n\n if (start < 0 || len < 0) {\n throw new IllegalArgumentException(\"Invalid start or len parameter\");\n }\n if (file.getLen() <= start) {\n return new BlockLocation[0];\n }\n if (cacheReplica <= 0) {\n String[] name = new String[]{\"localhost:50010\"};\n String[] host = new String[]{\"localhost\"};\n return new BlockLocation[]{new BlockLocation(name, host, 0L, file.getLen())};\n }\n BgTaskUtil.putTask(name, \"Node fetcher\", this::initCache, 10, 10, TimeUnit.MINUTES);\n if (file.getLen() <= start + len) {\n len = file.getLen() - start;\n }\n long code = normalizePath(file.getPath()).hashCode();\n BlockLocation[] locs = new BlockLocation[(int) (len / blocksize) + 2];\n int indx = 0;\n while (len > 0) {\n long blen = len < blocksize ? len : blocksize - start % blocksize;\n locs[indx] = makeLocation(code, start, blen);\n start += blen;\n len -= blen;\n indx++;\n }\n // merge the last block\n if (indx > 1 && locs[indx - 1].getLength() < blocksize / 10) {\n locs[indx - 2].setLength(locs[indx - 2].getLength() + locs[indx - 1].getLength());\n indx--;\n }\n // merge the first block\n if (indx > 1 && locs[0].getLength() < blocksize / 10) {\n locs[1].setOffset(locs[0].getOffset());\n locs[1].setLength(locs[0].getLength() + locs[1].getLength());\n locs = Arrays.copyOfRange(locs, 1, indx);\n indx--;\n }\n return Arrays.copyOfRange(locs, 0, indx);\n }\n\n /*******************************************************\n * For open()'s FSInputStream.\n *******************************************************/\n class FileInputStream extends FSInputStream implements ByteBufferReadable {\n private int fd;\n private final Path path;\n\n private ByteBuffer buf;\n private long position;\n private long fileLen;\n\n public FileInputStream(Path f, int fd, int size, long fileLen) throws IOException {\n path = f;\n this.fd = fd;\n buf = directBufferPool.getBuffer(size);\n buf.limit(0);\n position = 0;\n this.fileLen = fileLen;\n }\n\n @Override\n public synchronized long getPos() throws IOException {\n if (buf == null)\n throw new IOException(\"stream was closed\");\n return position - buf.remaining();\n }\n\n @Override\n public boolean seekToNewSource(long targetPos) throws IOException {\n return false;\n }\n\n @Override\n public synchronized int available() throws IOException {\n if (buf == null)\n throw new IOException(\"stream was closed\");\n long remaining = fileLen - position + buf.remaining();\n if (remaining > Integer.MAX_VALUE) {\n return Integer.MAX_VALUE;\n }\n return (int)remaining;\n }\n\n @Override\n public boolean markSupported() {\n return false;\n }\n\n @Override\n public synchronized int read() throws IOException {\n if (buf == null)\n throw new IOException(\"stream was closed\");\n if (!buf.hasRemaining() && !refill())\n return -1; // EOF\n assert buf.hasRemaining();\n statistics.incrementBytesRead(1);\n return buf.get() & 0xFF;\n }\n\n @Override\n public synchronized int read(byte[] b, int off, int len) throws IOException {\n if (off < 0 || len < 0 || b.length - off < len)\n throw new IndexOutOfBoundsException();\n return read(ByteBuffer.wrap(b, off, len));\n }\n\n private boolean refill() throws IOException {\n buf.clear();\n int read = read(position, buf);\n if (read <= 0) {\n buf.limit(0);\n return false; // EOF\n }\n buf.position(0);\n buf.limit(read);\n position += read;\n return true;\n }\n\n @Override\n public synchronized int read(long pos, byte[] b, int off, int len) throws IOException {\n if (b == null || off < 0 || len < 0 || b.length - off < len) {\n throw new IllegalArgumentException(\"arguments: \" + off + \" \" + len);\n }\n int got = read(pos, ByteBuffer.wrap(b, off, len));\n statistics.incrementBytesRead(got);\n return got;\n }\n\n @Override\n public synchronized int read(ByteBuffer b) throws IOException {\n if (!b.hasRemaining())\n return 0;\n if (buf == null)\n throw new IOException(\"stream was closed\");\n if (!buf.hasRemaining() && b.remaining() <= buf.capacity() && !refill()) {\n return -1;\n }\n ByteBuffer srcBuf = buf.duplicate();\n int got = Math.min(b.remaining(), srcBuf.remaining());\n if (got > 0) {\n srcBuf.limit(srcBuf.position() + got);\n b.put(srcBuf);\n buf.position(srcBuf.position());\n statistics.incrementBytesRead(got);\n }\n int more = read(position, b);\n if (more <= 0)\n return got > 0 ? got : -1;\n position += more;\n statistics.incrementBytesRead(more);\n buf.position(0);\n buf.limit(0);\n return got + more;\n }\n\n private synchronized int read(long pos, ByteBuffer b) throws IOException {\n if (pos < 0)\n throw new EOFException(\"position is negative\");\n if (!b.hasRemaining())\n return 0;\n int got;\n int startPos = b.position();\n got = lib.jfs_pread(Thread.currentThread().getId(), fd, b, b.remaining(), pos);\n if (got == EINVAL)\n throw new IOException(\"stream was closed\");\n if (got < 0)\n throw error(got, path);\n if (got == 0)\n return -1;\n b.position(startPos + got);\n return got;\n }\n\n @Override\n public synchronized void seek(long p) throws IOException {\n if (p < 0) {\n throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);\n }\n if (buf == null)\n throw new IOException(\"stream was closed\");\n if (p < position && p >= position - buf.limit()) {\n buf.position((int) (p - (position - buf.limit())));\n } else {\n buf.position(0);\n buf.limit(0);\n position = p;\n }\n }\n\n public synchronized void skipNBytes(long n) throws IOException {\n if (buf == null) {\n throw new IOException(\"stream was closed\");\n }\n\n if (n <= 0) {\n return;\n }\n\n long np = position + n;\n if (np > fileLen) {\n throw new EOFException(String.format(\"Unable to skip %s bytes (position=%s, fileSize=%s): %s\", n, position, fileLen, np));\n }\n position = np;\n }\n @Override\n public synchronized long skip(long n) throws IOException {\n if (n < 0)\n return -1;\n if (buf == null)\n throw new IOException(\"stream was closed\");\n long pos = getPos();\n if (pos + n > fileLen) {\n n = fileLen - pos;\n }\n seek(pos + n);\n return n;\n }\n\n @Override\n public synchronized void close() throws IOException {\n if (buf == null) {\n return; // already closed\n }\n directBufferPool.returnBuffer(buf);\n buf = null;\n int r = lib.jfs_close(Thread.currentThread().getId(), fd);\n fd = 0;\n if (r < 0)\n throw error(r, path);\n }\n }\n\n @Override\n public FSDataInputStream open(Path f, int bufferSize) throws IOException {\n if (needCheckPermission() && !checkPathAccess(f, FsAction.READ, \"open\")) {\n return superGroupFileSystem.open(f, bufferSize);\n }\n statistics.incrementReadOps(1);\n ByteBuffer fileLen = ByteBuffer.allocate(8);\n fileLen.order(ByteOrder.nativeOrder());\n int fd = lib.jfs_open(Thread.currentThread().getId(), handle, normalizePath(f), fileLen, MODE_MASK_R);\n if (fd < 0) {\n throw error(fd, f);\n }\n long len = fileLen.getLong();\n return new FSDataInputStream(new FileInputStream(f, fd, checkBufferSize(bufferSize), len));\n }\n\n @Override\n public void access(Path path, FsAction mode) throws IOException {\n if (needCheckPermission() && !checkPathAccess(path, mode, \"access\")) {\n superGroupFileSystem.access(path, mode);\n return;\n }\n int r = lib.jfs_access(Thread.currentThread().getId(), handle, normalizePath(path), mode.ordinal());\n if (r < 0)\n throw error(r, path);\n }\n\n /*********************************************************\n * For create()'s FSOutputStream.\n *********************************************************/\n class FSOutputStream extends OutputStream {\n private int fd;\n private Path path;\n\n private FSOutputStream(int fd, Path p) throws IOException {\n this.fd = fd;\n this.path = p;\n }\n\n @Override\n public void close() throws IOException {\n int r = lib.jfs_close(Thread.currentThread().getId(), fd);\n if (r < 0)\n throw error(r, path);\n }\n\n @Override\n public void flush() throws IOException {\n }\n\n public void hflush() throws IOException {\n int r = lib.jfs_flush(Thread.currentThread().getId(), fd);\n if (r == EINVAL)\n throw new IOException(\"stream was closed\");\n if (r < 0)\n throw error(r, path);\n }\n\n public void fsync() throws IOException {\n int r = lib.jfs_fsync(Thread.currentThread().getId(), fd);\n if (r == EINVAL)\n throw new IOException(\"stream was closed\");\n if (r < 0)\n throw error(r, path);\n }\n\n @Override\n public void write(byte[] b, int off, int len) throws IOException {\n if (b.length - off < len) {\n throw new IndexOutOfBoundsException();\n }\n int done = lib.jfs_write(Thread.currentThread().getId(), fd, ByteBuffer.wrap(b, off, len), len);\n if (done == EINVAL)\n throw new IOException(\"stream was closed\");\n if (done < 0)\n throw error(done, path);\n if (done < len) {\n throw new IOException(\"write\");\n }\n }\n\n @Override\n public void write(int b) throws IOException {\n int done = lib.jfs_write(Thread.currentThread().getId(), fd, ByteBuffer.wrap(new byte[]{(byte) b}), 1);\n if (done == EINVAL)\n throw new IOException(\"stream was closed\");\n if (done < 0)\n throw error(done, path);\n if (done < 1)\n throw new IOException(\"write\");\n }\n }\n\n static class BufferedFSOutputStream extends BufferedOutputStream implements Syncable {\n private String hflushMethod;\n private boolean closed;\n\n public BufferedFSOutputStream(OutputStream out) {\n super(out);\n hflushMethod = \"writeback\";\n }\n\n public BufferedFSOutputStream(OutputStream out, int size, String hflushMethod) {\n super(out, size);\n this.hflushMethod = hflushMethod;\n }\n\n public void sync() throws IOException {\n hflush();\n }\n\n @Override\n public synchronized void write(int b) throws IOException {\n if (closed) {\n throw new IOException(\"stream was closed\");\n }\n super.write(b);\n }\n\n @Override\n public synchronized void write(byte[] b, int off, int len) throws IOException {\n if (closed) {\n throw new IOException(\"stream was closed\");\n }\n super.write(b, off, len);\n }\n\n @Override\n public synchronized void flush() throws IOException {\n if (closed) {\n throw new IOException(\"stream was closed\");\n }\n super.flush();\n }\n\n @Override\n public synchronized void hflush() throws IOException {\n if (closed) {\n throw new IOException(\"stream was closed\");\n }\n flush();\n if (hflushMethod.equals(\"writeback\")) {\n ((FSOutputStream) out).hflush();\n } else if (hflushMethod.equals(\"sync\") || hflushMethod.equals(\"fsync\")) {\n ((FSOutputStream) out).fsync();\n } else {\n // nothing\n }\n }\n\n @Override\n public synchronized void hsync() throws IOException {\n if (closed) {\n throw new IOException(\"stream was closed\");\n }\n flush();\n ((FSOutputStream) out).fsync();\n }\n\n @Override\n public synchronized void close() throws IOException {\n if (closed) {\n return;\n }\n super.close();\n closed = true;\n }\n\n public OutputStream getOutputStream() {\n return out;\n }\n }\n\n static class BufferedFSOutputStreamWithStreamCapabilities extends BufferedFSOutputStream\n implements StreamCapabilities {\n public BufferedFSOutputStreamWithStreamCapabilities(OutputStream out) {\n super(out);\n }\n\n public BufferedFSOutputStreamWithStreamCapabilities(OutputStream out, int size, String hflushMethod) {\n super(out, size, hflushMethod);\n }\n\n @Override\n public boolean hasCapability(String capability) {\n return capability.equalsIgnoreCase(\"hsync\") || capability.equalsIgnoreCase((\"hflush\"));\n }\n }\n\n @Override\n public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException {\n if (needCheckPermission() && !checkPathAccess(f, FsAction.WRITE, \"append\")) {\n return superGroupFileSystem.append(f, bufferSize, progress);\n }\n statistics.incrementWriteOps(1);\n int fd = lib.jfs_open(Thread.currentThread().getId(), handle, normalizePath(f), null, MODE_MASK_W);\n if (fd < 0)\n throw error(fd, f);\n long r = lib.jfs_lseek(Thread.currentThread().getId(), fd, 0, 2);\n if (r < 0)\n throw error((int) r, f);\n return createFsDataOutputStream(f, bufferSize, fd, r);\n }\n\n @Override\n public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize,\n short replication, long blockSize, Progressable progress) throws IOException {\n if (needCheckPermission() && !checkAncestorAccess(f, FsAction.WRITE, \"create\")) {\n if (!overwrite || !superGroupFileSystem.exists(f)) {\n return superGroupFileSystem.create(f, permission, overwrite, bufferSize, replication, blockSize, progress);\n } else if (!checkPathAccess(f, FsAction.WRITE, \"create\")) {\n return superGroupFileSystem.create(f, permission, overwrite, bufferSize, replication, blockSize, progress);\n }\n }\n statistics.incrementWriteOps(1);\n while (true) {\n int fd = lib.jfs_create(Thread.currentThread().getId(), handle, normalizePath(f), permission.toShort(), uMask.toShort());\n if (fd == ENOENT) {\n Path parent = makeQualified(f).getParent();\n try {\n mkdirs(parent, FsPermission.getDirDefault());\n } catch (FileAlreadyExistsException e) {\n }\n continue;\n }\n if (fd == EEXIST) {\n if (!overwrite || isDirectory(f)) {\n throw new FileAlreadyExistsException(\"Path already exists: \" + f);\n }\n delete(f, false);\n continue;\n }\n if (fd < 0) {\n throw error(fd, makeQualified(f).getParent());\n }\n return createFsDataOutputStream(f, bufferSize, fd, 0L);\n }\n }\n\n private int checkBufferSize(int size) {\n if (size < minBufferSize) {\n size = minBufferSize;\n }\n return size;\n }\n\n @Override\n public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet flag,\n int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {\n if (needCheckPermission() && !checkAncestorAccess(f, FsAction.WRITE, \"createNonRecursive\")) {\n if (!flag.contains(CreateFlag.OVERWRITE) || !superGroupFileSystem.exists(f)) {\n return superGroupFileSystem.createNonRecursive(f, permission, flag, bufferSize, replication, blockSize, progress);\n } else if (!checkPathAccess(f, FsAction.WRITE, \"createNonRecursive\")) {\n return superGroupFileSystem.createNonRecursive(f, permission, flag, bufferSize, replication, blockSize, progress);\n }\n }\n statistics.incrementWriteOps(1);\n int fd = lib.jfs_create(Thread.currentThread().getId(), handle, normalizePath(f), permission.toShort(), uMask.toShort());\n while (fd == EEXIST) {\n if (!flag.contains(CreateFlag.OVERWRITE) || isDirectory(f)) {\n throw new FileAlreadyExistsException(\"File already exists: \" + f);\n }\n delete(f, false);\n fd = lib.jfs_create(Thread.currentThread().getId(), handle, normalizePath(f), permission.toShort(), uMask.toShort());\n }\n if (fd < 0) {\n throw error(fd, makeQualified(f).getParent());\n }\n return createFsDataOutputStream(f, bufferSize, fd, 0L);\n }\n\n private FSDataOutputStream createFsDataOutputStream(Path f, int bufferSize, int fd, long startPosition) throws IOException {\n FSOutputStream out = new FSOutputStream(fd, f);\n if (withStreamCapability) {\n try {\n return new FSDataOutputStream(\n (OutputStream) constructor.newInstance(out, checkBufferSize(bufferSize), hflushMethod), statistics, startPosition);\n } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {\n throw new RuntimeException(e);\n }\n } else {\n return new FSDataOutputStream(new BufferedFSOutputStream(out, checkBufferSize(bufferSize), hflushMethod),\n statistics, startPosition);\n }\n }\n\n @Override\n public FileChecksum getFileChecksum(Path f, long length) throws IOException {\n if (needCheckPermission() && !checkPathAccess(f, FsAction.READ, \"getFileChecksum\")) {\n return superGroupFileSystem.getFileChecksum(f, length);\n }\n statistics.incrementReadOps(1);\n if (!fileChecksumEnabled)\n return null;\n String combineMode = getConf().get(\"dfs.checksum.combine.mode\", \"MD5MD5CRC\");\n if (!combineMode.equals(\"MD5MD5CRC\"))\n return null;\n DataChecksum.Type ctype = DataChecksum.Type.valueOf(getConf().get(\"dfs.checksum.type\", \"CRC32C\"));\n if (ctype.size != 4)\n return null;\n\n int bytesPerCrc = getConf().getInt(\"io.bytes.per.checksum\", 512);\n DataChecksum summer = DataChecksum.newDataChecksum(ctype, bytesPerCrc);\n\n DataOutputBuffer checksumBuf = new DataOutputBuffer();\n DataOutputBuffer crcBuf = new DataOutputBuffer();\n byte[] buf = new byte[bytesPerCrc];\n FSDataInputStream in = open(f, 1 << 20);\n boolean eof = false;\n long got = 0;\n while (got < length && !eof) {\n for (int i = 0; i < blocksize / bytesPerCrc && got < length; i++) {\n int n;\n if (length < bytesPerCrc) {\n n = in.read(buf, 0, (int) length);\n } else {\n n = in.read(buf);\n }\n if (n <= 0) {\n eof = true;\n break;\n } else {\n summer.update(buf, 0, n);\n summer.writeValue(crcBuf, true);\n got += n;\n }\n }\n if (crcBuf.getLength() > 0) {\n MD5Hash blockMd5 = MD5Hash.digest(crcBuf.getData(), 0, crcBuf.getLength());\n blockMd5.write(checksumBuf);\n crcBuf.reset();\n }\n }\n in.close();\n if (checksumBuf.getLength() == 0) { // empty file\n return new MD5MD5CRC32GzipFileChecksum(0, 0, MD5Hash.digest(new byte[32]));\n }\n MD5Hash md5 = MD5Hash.digest(checksumBuf.getData());\n long crcPerBlock = 0;\n if (got > blocksize) { // more than one block\n crcPerBlock = blocksize / bytesPerCrc;\n }\n if (ctype == DataChecksum.Type.CRC32C) {\n return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCrc, crcPerBlock, md5);\n } else {\n return new MD5MD5CRC32GzipFileChecksum(bytesPerCrc, crcPerBlock, md5);\n }\n }\n\n @Override\n public void concat(final Path dst, final Path[] srcs) throws IOException {\n if (needCheckPermission()) {\n access(dst.getParent(), FsAction.WRITE);\n access(dst, FsAction.WRITE);\n for (Path src : srcs) {\n access(src, FsAction.READ);\n }\n superGroupFileSystem.concat(dst, srcs);\n return;\n }\n statistics.incrementWriteOps(1);\n if (srcs.length == 0) {\n throw new IllegalArgumentException(\"No sources given\");\n }\n Path dp = makeQualified(dst).getParent();\n for (Path src : srcs) {\n if (!makeQualified(src).getParent().equals(dp)) {\n throw new HadoopIllegalArgumentException(\"Source file \" + normalizePath(src)\n + \" is not in the same directory with the target \"\n + normalizePath(dst));\n }\n }\n byte[][] srcbytes = new byte[srcs.length][];\n int bufsize = 0;\n for (int i = 0; i < srcs.length; i++) {\n srcbytes[i] = normalizePath(srcs[i]).getBytes(\"UTF-8\");\n bufsize += srcbytes[i].length + 1;\n }\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), bufsize);\n long offset = 0;\n for (int i = 0; i < srcs.length; i++) {\n buf.put(offset, srcbytes[i], 0, srcbytes[i].length);\n buf.putByte(offset + srcbytes[i].length, (byte) 0);\n offset += srcbytes[i].length + 1;\n }\n int r = lib.jfs_concat(Thread.currentThread().getId(), handle, normalizePath(dst), buf, bufsize);\n if (r < 0) {\n if (r == ENOENT) {\n if (!exists(dst)) {\n throw error(r, dst);\n } else {\n throw new FileNotFoundException(\"one of srcs is missing\");\n }\n }\n throw error(r, dst);\n }\n }\n\n @Override\n public boolean rename(Path src, Path dst) throws IOException {\n if (needCheckPermission()) {\n if (!superGroupFileSystem.exists(src)) {\n return false;\n }\n access(src.getParent(), FsAction.WRITE);\n Path dstAncestor = rangerPermissionChecker.getAncestor(dst).getPath();\n access(dstAncestor, FsAction.WRITE);\n return superGroupFileSystem.rename(src, dst);\n }\n statistics.incrementWriteOps(1);\n String srcStr = makeQualified(src).toUri().getPath();\n String dstStr = makeQualified(dst).toUri().getPath();\n if (src.equals(dst)) {\n FileStatus st = getFileStatus(src);\n return st.isFile();\n }\n if (dstStr.startsWith(srcStr) && (dstStr.charAt(srcStr.length()) == Path.SEPARATOR_CHAR)) {\n return false;\n }\n int r = lib.jfs_rename(Thread.currentThread().getId(), handle, normalizePath(src), normalizePath(dst));\n if (r == EEXIST) {\n try {\n FileStatus st = getFileStatus(dst);\n if (st.isDirectory()) {\n dst = new Path(dst, src.getName());\n r = lib.jfs_rename(Thread.currentThread().getId(), handle, normalizePath(src), normalizePath(dst));\n } else {\n return false;\n }\n } catch (FileNotFoundException ignored) {\n }\n }\n if (r == ENOENT || r == EEXIST)\n return false;\n if (r == EACCESS) {\n this.access(makeQualified(src).getParent(), FsAction.WRITE.or(FsAction.EXECUTE));\n this.access(makeQualified(dst).getParent(), FsAction.WRITE.or(FsAction.EXECUTE));\n }\n if (r < 0)\n throw error(r, src);\n return true;\n }\n\n @Override\n public boolean truncate(Path f, long newLength) throws IOException {\n if (needCheckPermission() && !checkPathAccess(f, FsAction.WRITE, \"truncate\")) {\n return superGroupFileSystem.truncate(f, newLength);\n }\n int r = lib.jfs_truncate(Thread.currentThread().getId(), handle, normalizePath(f), newLength);\n if (r < 0)\n throw error(r, f);\n return true;\n }\n\n private boolean rmr(Path p) throws IOException {\n int r = lib.jfs_rmr(Thread.currentThread().getId(), handle, normalizePath(p));\n if (r == ENOENT) {\n return false;\n }\n if (r < 0) {\n throw error(r, p);\n }\n return true;\n }\n\n @Override\n public boolean delete(Path p, boolean recursive) throws IOException {\n if (needCheckPermission()) {\n try {\n if (!checkParentPathAccess(p, FsAction.WRITE_EXECUTE, \"delete\")) {\n return superGroupFileSystem.delete(p, recursive);\n }\n } catch (Exception e) {\n if (!checkPathAccess(p, FsAction.WRITE_EXECUTE, \"delete\")) {\n return superGroupFileSystem.delete(p, recursive);\n }\n }\n }\n statistics.incrementWriteOps(1);\n if (recursive)\n return rmr(p);\n int r = lib.jfs_delete(Thread.currentThread().getId(), handle, normalizePath(p));\n if (r == ENOENT) {\n return false;\n }\n if (r < 0) {\n throw error(r, p);\n }\n return true;\n }\n\n @Override\n public ContentSummary getContentSummary(Path f) throws IOException {\n if (needCheckPermission() && !checkPathAccess(f, FsAction.READ_EXECUTE, \"getContentSummary\")) {\n return superGroupFileSystem.getContentSummary(f);\n }\n statistics.incrementReadOps(1);\n String path = normalizePath(f);\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 40);\n int r = lib.jfs_summary(Thread.currentThread().getId(), handle, path, buf);\n if (r < 0) {\n throw error(r, f);\n }\n long size = buf.getLongLong(0);\n long files = buf.getLongLong(8);\n long dirs = buf.getLongLong(16);\n long quota = buf.getLongLong(24);\n long spaceQuota = buf.getLongLong(32);\n quota = quota == 0 ? -1L : quota;\n spaceQuota = spaceQuota == 0 ? -1L : spaceQuota;\n return new ContentSummary(size, files, dirs, quota, size, spaceQuota);\n }\n\n private FileStatus newFileStatus(Path p, Pointer buf, int size, boolean readlink) throws IOException {\n int mode = buf.getInt(0);\n boolean isdir = ((mode >>> 31) & 1) == 1; // Go\n int stickybit = (mode >>> 20) & 1;\n boolean hasAcl = (mode >> 18 & 1) == 1;\n FsPermission perm = new FsPermission((short) ((mode & 0777) | (stickybit << 9)));\n perm = new FsPermissionExtension(perm, hasAcl, false);\n long length = buf.getLongLong(4);\n long mtime = buf.getLongLong(12);\n long atime = buf.getLongLong(20);\n String user = buf.getString(28);\n String group = buf.getString(28 + user.length() + 1);\n assert (30 + user.length() + group.length() == size);\n\n if (fileStatusConstructor == null) {\n return new FileStatus(length, isdir, 1, blocksize, mtime, atime, perm, user, group, p);\n } else {\n try {\n return fileStatusConstructor.newInstance(length, isdir, 1, blocksize, mtime, atime, perm, user, group, null, p, hasAcl, false, false);\n } catch (Exception e) {\n throw new IOException(\"construct fileStatus failed\", e);\n }\n }\n }\n\n @Override\n public FileStatus[] listStatus(Path f) throws IOException {\n if (needCheckPermission() && !checkPathAccess(f, FsAction.READ_EXECUTE, \"listStatus\")) {\n return superGroupFileSystem.listStatus(f);\n }\n statistics.incrementReadOps(1);\n int bufsize = 32 << 10;\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), bufsize); // TODO: smaller buff\n String path = normalizePath(f);\n int r = lib.jfs_listdir(Thread.currentThread().getId(), handle, path, 0, buf, bufsize);\n if (r == ENOENT) {\n throw new FileNotFoundException(f.toString());\n }\n if (r == ENOTDIR) {\n return new FileStatus[]{getFileStatus(f)};\n }\n\n FileStatus[] results;\n results = new FileStatus[1024];\n int j = 0;\n while (r > 0) {\n long offset = 0;\n while (offset < r) {\n int len = buf.getByte(offset) & 0xff;\n byte[] name = new byte[len];\n buf.get(offset + 1, name, 0, len);\n offset += 1 + len;\n int size = buf.getByte(offset) & 0xff;\n if (j == results.length) {\n FileStatus[] rs = new FileStatus[results.length * 2];\n System.arraycopy(results, 0, rs, 0, j);\n results = rs;\n }\n Path p = makeQualified(new Path(f, new String(name)));\n FileStatus st = newFileStatus(p, buf.slice(offset + 1), size, false);\n results[j] = st;\n offset += 1 + size;\n j++;\n }\n int left = buf.getInt(offset);\n if (left == 0)\n break;\n int fd = buf.getInt(offset + 4);\n r = lib.jfs_listdir(Thread.currentThread().getId(), fd, path, j, buf, bufsize);\n }\n if (r < 0) {\n throw error(r, f);\n }\n statistics.incrementReadOps(j);\n\n FileStatus[] sorted = Arrays.copyOf(results, j);\n Arrays.sort(sorted, (p1, p2) -> p1.getPath().compareTo(p2.getPath()));\n return sorted;\n }\n\n @Override\n public void setWorkingDirectory(Path newDir) {\n workingDir = fixRelativePart(newDir);\n checkPath(workingDir);\n }\n\n @Override\n public Path getWorkingDirectory() {\n return workingDir;\n }\n\n @Override\n public boolean mkdirs(Path f, FsPermission permission) throws IOException {\n if (needCheckPermission() && !checkAncestorAccess(f, FsAction.WRITE, \"mkdirs\")) {\n return superGroupFileSystem.mkdirs(f, permission);\n }\n statistics.incrementWriteOps(1);\n if (f == null) {\n throw new IllegalArgumentException(\"mkdirs path arg is null\");\n }\n String path = normalizePath(f);\n if (\"/\".equals(path))\n return true;\n int r = lib.jfs_mkdir(Thread.currentThread().getId(), handle, path, permission.toShort(), uMask.toShort());\n if (r == 0 || r == EEXIST && !isFile(f)) {\n return true;\n } else if (r == ENOENT) {\n Path parent = makeQualified(f).getParent();\n if (parent != null) {\n return mkdirs(parent, permission) && mkdirs(f, permission);\n }\n }\n throw error(r, makeQualified(f).getParent());\n }\n\n @Override\n public FileStatus getFileStatus(Path f) throws IOException {\n if (needCheckPermission() && !checkParentPathAccess(f, FsAction.EXECUTE, \"getFileStatus\")) {\n return superGroupFileSystem.getFileStatus(f);\n }\n statistics.incrementReadOps(1);\n try {\n return getFileStatusInternal(f, true);\n } catch (ParentNotDirectoryException e) {\n throw new FileNotFoundException(f.toString());\n }\n }\n\n private FileStatus getFileStatusInternal(final Path f, boolean dereference) throws IOException {\n String path = normalizePath(f);\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 130);\n int r;\n if (dereference) {\n r = lib.jfs_stat1(Thread.currentThread().getId(), handle, path, buf);\n } else {\n r = lib.jfs_lstat1(Thread.currentThread().getId(), handle, path, buf);\n }\n if (r < 0) {\n throw error(r, f);\n }\n return newFileStatus(makeQualified(f), buf, r, !dereference);\n }\n\n private FileStatus getFileStatusInternalNoException(final Path f) throws IOException {\n String path = normalizePath(f);\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 130);\n int r = lib.jfs_lstat1(Thread.currentThread().getId(), handle, path, buf);\n if (r < 0) {\n return null;\n }\n return newFileStatus(makeQualified(f), buf, r, false);\n }\n\n @Override\n public boolean supportsSymlinks() {\n return false;\n }\n\n @Override\n public FsStatus getStatus(Path p) throws IOException {\n if (needCheckPermission() && !checkParentPathAccess(p, FsAction.EXECUTE, \"getStatus\")) {\n return superGroupFileSystem.getStatus(p);\n }\n statistics.incrementReadOps(1);\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 16);\n int r = lib.jfs_statvfs(Thread.currentThread().getId(), handle, buf);\n if (r != 0)\n throw error(r, p);\n long capacity = buf.getLongLong(0);\n long remaining = buf.getLongLong(8);\n return new FsStatus(capacity, capacity - remaining, remaining);\n }\n\n @Override\n public void setPermission(Path p, FsPermission permission) throws IOException {\n if (needCheckPermission() && !checkOwner(p, \"setPermission\")) {\n superGroupFileSystem.setPermission(p, permission);\n return;\n }\n statistics.incrementWriteOps(1);\n int r = lib.jfs_chmod(Thread.currentThread().getId(), handle, normalizePath(p), permission.toShort());\n if (r != 0)\n throw error(r, p);\n }\n\n @Override\n public void setOwner(Path p, String username, String groupname) throws IOException {\n if (needCheckPermission()) {\n if (username == null) {\n throw new AccessControlException(\n \"User can not be null\");\n }\n if (!superuser.equals(username)) {\n throw new AccessControlException(\n \"Only SuperUser can do setOwner Action, the current user is \" + username);\n }\n superGroupFileSystem.setOwner(p, username, groupname);\n return;\n }\n statistics.incrementWriteOps(1);\n int r = lib.jfs_setOwner(Thread.currentThread().getId(), handle, normalizePath(p), username, groupname);\n if (r != 0)\n throw error(r, p);\n }\n\n @Override\n public void setTimes(Path p, long mtime, long atime) throws IOException {\n if (needCheckPermission() && !checkPathAccess(p, FsAction.WRITE, \"setTimes\")) {\n superGroupFileSystem.setTimes(p, mtime, atime);\n return;\n }\n statistics.incrementWriteOps(1);\n int r = lib.jfs_utime(Thread.currentThread().getId(), handle, normalizePath(p), mtime >= 0 ? mtime : -1,\n atime >= 0 ? atime : -1);\n if (r != 0)\n throw error(r, p);\n }\n\n @Override\n public void close() throws IOException {\n super.close();\n RangerPermissionChecker.release(name, handle);\n BgTaskUtil.unregister(name, handle, () -> {\n cachedHostsForName.clear();\n hashForName.clear();\n lastFileStatus.clear();\n });\n LOG.debug(\"close {}({})\", name, handle);\n lib.jfs_term(Thread.currentThread().getId(), handle);\n if (metricsEnable) {\n JuiceFSInstrumentation.close();\n }\n }\n\n @Override\n public void setXAttr(Path path, String name, byte[] value, EnumSet flag) throws IOException {\n if (needCheckPermission() && !checkPathAccess(path, FsAction.WRITE, \"setXAttr\")) {\n superGroupFileSystem.setXAttr(path, name, value, flag);\n return;\n }\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), value.length);\n buf.put(0, value, 0, value.length);\n int mode = 0; // create or replace\n if (flag.contains(XAttrSetFlag.CREATE) && flag.contains(XAttrSetFlag.REPLACE)) {\n mode = 0;\n } else if (flag.contains(XAttrSetFlag.CREATE)) {\n mode = 1;\n } else if (flag.contains(XAttrSetFlag.REPLACE)) {\n mode = 2;\n }\n int r = lib.jfs_setXattr(Thread.currentThread().getId(), handle, normalizePath(path), name, buf, value.length,\n mode);\n if (r < 0)\n throw error(r, path);\n }\n\n @Override\n public byte[] getXAttr(Path path, String name) throws IOException {\n if (needCheckPermission() && !checkPathAccess(path, FsAction.READ, \"getXAttr\")) {\n return superGroupFileSystem.getXAttr(path, name);\n }\n Pointer buf;\n int bufsize = 16 << 10;\n int r;\n do {\n bufsize *= 2;\n buf = Memory.allocate(Runtime.getRuntime(lib), bufsize);\n r = lib.jfs_getXattr(Thread.currentThread().getId(), handle, normalizePath(path), name, buf, bufsize);\n } while (r == bufsize);\n if (r == ENOATTR || r == ENODATA)\n return null; // attr not found\n if (r < 0)\n throw error(r, path);\n byte[] value = new byte[r];\n buf.get(0, value, 0, r);\n return value;\n }\n\n @Override\n public Map getXAttrs(Path path) throws IOException {\n return getXAttrs(path, listXAttrs(path));\n }\n\n @Override\n public Map getXAttrs(Path path, List names) throws IOException {\n if (needCheckPermission() && !checkPathAccess(path, FsAction.READ, \"getXAttrs\")) {\n return superGroupFileSystem.getXAttrs(path, names);\n }\n Map result = new HashMap();\n for (String n : names) {\n byte[] value = getXAttr(path, n);\n if (value != null) {\n result.put(n, value);\n }\n }\n return result;\n }\n\n @Override\n public List listXAttrs(Path path) throws IOException {\n if (needCheckPermission() && !checkPathAccess(path, FsAction.READ, \"listXAttrs\")) {\n return superGroupFileSystem.listXAttrs(path);\n }\n Pointer buf;\n int bufsize = 1024;\n int r;\n do {\n bufsize *= 2;\n buf = Memory.allocate(Runtime.getRuntime(lib), bufsize);\n r = lib.jfs_listXattr(Thread.currentThread().getId(), handle, normalizePath(path), buf, bufsize);\n } while (r == bufsize);\n if (r < 0)\n throw error(r, path);\n List result = new ArrayList();\n int off = 0, last = 0;\n while (off < r) {\n if (buf.getByte(off) == 0) {\n byte[] arr = new byte[off - last];\n buf.get(last, arr, 0, arr.length);\n result.add(new String(arr));\n last = off + 1;\n }\n off++;\n }\n return result;\n }\n\n @Override\n public void removeXAttr(Path path, String name) throws IOException {\n if (needCheckPermission() && !checkPathAccess(path, FsAction.WRITE, \"removeXAttr\")) {\n superGroupFileSystem.removeXAttr(path, name);\n return;\n }\n int r = lib.jfs_removeXattr(Thread.currentThread().getId(), handle, normalizePath(path), name);\n if (r == ENOATTR || r == ENODATA) {\n throw new IOException(\"No matching attributes found for remove operation\");\n }\n if (r < 0)\n throw error(r, path);\n }\n\n @Override\n public void modifyAclEntries(Path path, List aclSpec) throws IOException {\n if (needCheckPermission() && !checkOwner(path, \"modifyAclEntries\")) {\n superGroupFileSystem.modifyAclEntries(path, aclSpec);\n return;\n }\n List existingEntries = getAllAclEntries(path);\n List newAcl = AclTransformation.mergeAclEntries(existingEntries, aclSpec);\n setAclInternal(path, newAcl);\n }\n\n @Override\n public void removeAclEntries(Path path, List aclSpec) throws IOException {\n if (needCheckPermission() && !checkOwner(path, \"removeAclEntries\")) {\n superGroupFileSystem.removeAclEntries(path, aclSpec);\n return;\n }\n List existingEntries = getAllAclEntries(path);\n List newAcl = AclTransformation.filterAclEntriesByAclSpec(existingEntries, aclSpec);\n setAclInternal(path, newAcl);\n }\n\n @Override\n public void setAcl(Path path, List aclSpec) throws IOException {\n if (needCheckPermission() && !checkOwner(path, \"setAcl\")) {\n superGroupFileSystem.setAcl(path, aclSpec);\n return;\n }\n List existingEntries = getAllAclEntries(path);\n List newAcl = AclTransformation.replaceAclEntries(existingEntries, aclSpec);\n setAclInternal(path, newAcl);\n }\n\n private void setAclInternal(Path path, List aclSpec) throws IOException {\n List aclEntries = AclTransformation.buildAndValidateAcl(Lists.newArrayList(aclSpec));\n ScopedAclEntries scoped = new ScopedAclEntries(aclEntries);\n setAclInternal(path, AclEntryScope.ACCESS, scoped.getAccessEntries());\n setAclInternal(path, AclEntryScope.DEFAULT, scoped.getDefaultEntries());\n }\n\n private void removeAclInternal(Path path, AclEntryScope scope) throws IOException {\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 6 * 2);\n buf.putShort(0, (short) -1);\n buf.putShort(2, (short) -1);\n buf.putShort(4, (short) -1);\n buf.putShort(6, (short) -1);\n buf.putShort(8, (short) 0);\n buf.putShort(10, (short) 0);\n int r = lib.jfs_setfacl(Thread.currentThread().getId(), handle, normalizePath(path), scope.ordinal() + 1, buf,\n 6 * 2);\n if (r == ENOATTR || r == ENODATA)\n return;\n if (r < 0)\n throw error(r, path);\n }\n\n @Override\n public void removeDefaultAcl(Path path) throws IOException {\n if (needCheckPermission() && !checkOwner(path, \"removeDefaultAcl\")) {\n superGroupFileSystem.removeDefaultAcl(path);\n return;\n }\n removeAclInternal(path, AclEntryScope.DEFAULT);\n }\n\n @Override\n public void removeAcl(Path path) throws IOException {\n if (needCheckPermission() && !checkOwner(path, \"removeAcl\")) {\n superGroupFileSystem.removeAcl(path);\n return;\n }\n removeAclInternal(path, AclEntryScope.ACCESS);\n removeAclInternal(path, AclEntryScope.DEFAULT);\n }\n\n private void setAclInternal(Path path, AclEntryScope scope, List aclSpec) throws IOException {\n if (aclSpec.size() == 0)\n return;\n short userperm = -1, groupperm = -1, otherperm = -1, maskperm = -1;\n short namedusers = 0, namedgroups = 0;\n int namedaclsize = 0;\n for (AclEntry e : aclSpec) {\n if (e.getName() != null) {\n if (e.getType() == AclEntryType.USER) {\n namedusers++;\n } else {\n namedgroups++;\n }\n namedaclsize += e.getName().getBytes(\"utf8\").length + 2;\n } else {\n short perm = (short) e.getPermission().ordinal();\n switch (e.getType()) {\n case USER:\n userperm = perm;\n break;\n case GROUP:\n groupperm = perm;\n break;\n case OTHER:\n otherperm = perm;\n break;\n case MASK:\n maskperm = perm;\n break;\n }\n }\n }\n Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 12 + namedaclsize);\n buf.putShort(0, userperm);\n buf.putShort(2, groupperm);\n buf.putShort(4, otherperm);\n buf.putShort(6, maskperm);\n buf.putShort(8, namedusers);\n buf.putShort(10, namedgroups);\n int off = 12;\n for (AclEntry e : aclSpec) {\n String name = e.getName();\n if (name != null && e.getType() == AclEntryType.USER) {\n byte[] nb = name.getBytes(\"utf8\");\n buf.putByte(off, (byte) nb.length);\n buf.put(off + 1, nb, 0, nb.length);\n off += 1 + nb.length;\n buf.putByte(off, (byte) e.getPermission().ordinal());\n off += 1;\n }\n }\n for (AclEntry e : aclSpec) {\n String name = e.getName();\n if (name != null && e.getType() == AclEntryType.GROUP) {\n byte[] nb = name.getBytes(\"utf8\");\n buf.putByte(off, (byte) nb.length);\n buf.put(off + 1, nb, 0, nb.length);\n off += 1 + nb.length;\n buf.putByte(off, (byte) e.getPermission().ordinal());\n off += 1;\n }\n }\n int r = lib.jfs_setfacl(Thread.currentThread().getId(), handle, normalizePath(path), scope.ordinal() + 1, buf,\n 12 + namedaclsize);\n if (r == ENOTSUP) {\n throw new IOException(\"Invalid ACL: only directories may have a default ACL\");\n }\n if (r < 0)\n throw error(r, path);\n }\n\n private List getAclEntries(Path path, AclEntryScope scope) throws IOException {\n int bufsize = 1024;\n int r;\n Pointer buf;\n do {\n bufsize *= 2;\n buf = Memory.allocate(Runtime.getRuntime(lib), bufsize);\n r = lib.jfs_getfacl(Thread.currentThread().getId(), handle, normalizePath(path), scope.ordinal() + 1, buf,\n bufsize);\n } while (r == -100);\n if (r == ENOATTR || r == ENODATA) {\n return Lists.newArrayList();\n }\n if (r < 0)\n throw error(r, path);\n\n int off = 0;\n short userperm = buf.getShort(0);\n short groupperm = buf.getShort(2);\n short otherperm = buf.getShort(4);\n short maskperm = buf.getShort(6);\n short namedusers = buf.getShort(8);\n short namedgroups = buf.getShort(10);\n off += 12;\n\n List entries = new ArrayList<>();\n AclEntry.Builder builder = new AclEntry.Builder().setScope(scope);\n if (userperm != -1) {\n entries.add(builder.setType(AclEntryType.USER).setPermission(FsAction.values()[userperm]).build());\n }\n if (groupperm != -1) {\n entries.add(builder.setType(AclEntryType.GROUP).setPermission(FsAction.values()[groupperm]).build());\n }\n if (otherperm != -1) {\n entries.add(builder.setType(AclEntryType.OTHER).setPermission(FsAction.values()[otherperm]).build());\n }\n if (maskperm != -1) {\n entries.add(builder.setType(AclEntryType.MASK).setPermission(FsAction.values()[maskperm]).build());\n }\n\n for (int i = 0; i < namedusers + namedgroups; i++) {\n String name = buf.getString(off);\n off += name.length() + 1;\n short perm = buf.getShort(off);\n off += 2;\n entries.add(builder.setType(i < namedusers ? AclEntryType.USER : AclEntryType.GROUP).setName(name)\n .setPermission(FsAction.values()[perm]).build());\n }\n Collections.sort(entries, AclTransformation.ACL_ENTRY_COMPARATOR);\n return entries;\n }\n\n /**\n * include acl entries from permission\n */\n private List getAllAclEntries(Path path) throws IOException {\n List entries = getAclEntries(path, AclEntryScope.ACCESS);\n if (entries.size() == 0) {\n FsPermission perm = getFileStatus(path).getPermission();\n entries = AclUtil.getAclFromPermAndEntries(perm, entries);\n }\n entries.addAll(getAclEntries(path, AclEntryScope.DEFAULT));\n return entries;\n }\n\n /**\n * exclude acl entries from permission\n */\n private List getAclEntries(Path path) throws IOException {\n List res = new ArrayList<>();\n List accessEntries = getAclEntries(path, AclEntryScope.ACCESS);\n // minimal 3 acls for ugo\n if (accessEntries.size() != 0 && accessEntries.size() != 3) {\n res.addAll(accessEntries.subList(1, accessEntries.size() - 2));\n }\n res.addAll(getAclEntries(path, AclEntryScope.DEFAULT));\n return res;\n }\n\n @Override\n public AclStatus getAclStatus(Path path) throws IOException {\n if (needCheckPermission() && !checkOwner(path, \"getAclStatus\")) {\n return superGroupFileSystem.getAclStatus(path);\n }\n FileStatus st = getFileStatus(path);\n List entries = getAclEntries(path);\n AclStatus.Builder builder = new AclStatus.Builder().owner(st.getOwner()).group(st.getGroup())\n .stickyBit(st.getPermission().getStickyBit()).addEntries(entries);\n try {\n Class ab = AclStatus.Builder.class;\n Method abm = ab.getDeclaredMethod(\"setPermission\", FsPermission.class);\n abm.setAccessible(true);\n abm.invoke(builder, st.getPermission());\n } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {\n }\n return builder.build();\n }\n\n public AuthCredential buildAuthCredential(String spn) throws IOException {\n // auth use kerberos\n if (UserGroupInformation.getLoginUser().hasKerberosCredentials()) {\n dtEnabled = true;\n byte[] cred;\n try {\n cred = KerberosUtil.genApReq(spn);\n } catch (InterruptedException e) {\n throw new IOException(\"generate kerberos AP-REQ failed\", e);\n }\n return new AuthCredential(\"kerberos\", cred);\n }\n\n // auth use delegation token\n for (Token token : ugi.getCredentials().getAllTokens()) {\n if (token.getKind().equals(JuiceFSDelegationTokenIdentifier.TOKEN_KIND) &&\n buildServiceName().equals(token.getService().toString())) {\n dtEnabled = true;\n\n AbstractDelegationTokenIdentifier identifier = (AbstractDelegationTokenIdentifier) token.decodeIdentifier();\n int id = identifier.getMasterKeyId();\n byte[] password = token.getPassword();\n ByteBuffer buf = ByteBuffer.allocate(8 + password.length);\n buf.putInt(id);\n buf.putInt(password.length);\n buf.put(password);\n\n return new AuthCredential(\"token\", buf.array());\n }\n }\n\n return null;\n }\n\n private String buildServiceName() {\n return getScheme() + \"://\" + (name == null ? \"/\" : name);\n }\n\n @Override\n public String getCanonicalServiceName() {\n return dtEnabled ? buildServiceName() : null;\n }\n\n @Override\n public Token getDelegationToken(String renewer) throws IOException {\n if (!dtEnabled) {\n return null;\n }\n String owner = ugi.getShortUserName();\n String realUser = ugi.getRealUser() != null ? ugi.getRealUser().getShortUserName() : null;\n int tokenSize = 0, r = 8<<10;\n Pointer tokenBuf = null;\n while (r > tokenSize) {\n tokenSize = r;\n tokenBuf = Memory.allocate(Runtime.getRuntime(lib), tokenSize);\n r = lib.jfs_get_token(handle, name, tokenBuf, tokenSize, (new HadoopKerberosName(renewer)).getShortName());\n }\n if (r < 0) {\n throw new IOException(String.format(\"get delegation token failed, return code %d\", r));\n }\n int id = tokenBuf.getInt(0);\n long issueDate = tokenBuf.getLongLong(4);\n long maxDate = tokenBuf.getLongLong(12);\n int pwdLen = r - 20;\n byte[] pwd = new byte[pwdLen];\n tokenBuf.get(20, pwd, 0, pwdLen);\n\n JuiceFSDelegationTokenIdentifier identifier =\n new JuiceFSDelegationTokenIdentifier(\n owner,\n renewer,\n realUser);\n identifier.setIssueDate(issueDate);\n identifier.setMaxDate(maxDate);\n identifier.setMasterKeyId(id);\n\n return new Token<>(\n identifier.getBytes(),\n pwd,\n identifier.getKind(),\n new Text(getCanonicalServiceName()));\n }\n\n public long renewToken(Token token) throws IOException {\n AbstractDelegationTokenIdentifier identifier = (AbstractDelegationTokenIdentifier) token.decodeIdentifier();\n int id = identifier.getMasterKeyId();\n String pwd = new String(token.getPassword(), StandardCharsets.UTF_8);\n long r = lib.jfs_renew_token(handle, id, pwd);\n if (r == EACCESS) {\n throw new IOException(\"permission denied\");\n }\n if (r < 0) {\n throw new IOException(String.format(\"renew token failed, return code %d\", r));\n }\n return r * 1000;\n }\n\n public void cancelToken(Token token) throws IOException {\n AbstractDelegationTokenIdentifier identifier = (AbstractDelegationTokenIdentifier) token.decodeIdentifier();\n int id = identifier.getMasterKeyId();\n String pwd = new String(token.getPassword(), StandardCharsets.UTF_8);\n int r = lib.jfs_cancel_token(handle, id, pwd);\n if (r == EACCESS) {\n throw new IOException(\"permission denied\");\n }\n if (r < 0) {\n throw new IOException(String.format(\"cancel token failed, return code %d\", r));\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/KiteDataLoader.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.kitesdk.data.DatasetIOException;\nimport org.kitesdk.data.DatasetOperationException;\nimport org.kitesdk.data.spi.*;\nimport org.kitesdk.data.spi.filesystem.FileSystemDatasetRepository;\n\nimport java.io.IOException;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.util.Map;\n\npublic class KiteDataLoader implements Loadable {\n private static class URIBuilder implements OptionBuilder {\n\n @Override\n public DatasetRepository getFromOptions(Map match) {\n String path = match.get(\"path\");\n final Path root = (path == null || path.isEmpty()) ?\n new Path(\"/\") : new Path(\"/\", path);\n\n Configuration conf = DefaultConfiguration.get();\n FileSystem fs;\n try {\n fs = FileSystem.get(fileSystemURI(match), conf);\n } catch (IOException e) {\n throw new DatasetIOException(\"Could not get a FileSystem\", e);\n }\n return new FileSystemDatasetRepository.Builder()\n .configuration(new Configuration(conf)) // make a modifiable copy\n .rootDirectory(fs.makeQualified(root))\n .build();\n }\n }\n\n @Override\n public void load() {\n try {\n // load hdfs-site.xml by loading HdfsConfiguration\n FileSystem.getLocal(DefaultConfiguration.get());\n } catch (IOException e) {\n throw new DatasetIOException(\"Cannot load default config\", e);\n }\n\n OptionBuilder builder = new URIBuilder();\n Registration.register(\n new URIPattern(\"jfs:/*path\"),\n new URIPattern(\"jfs:/*path/:namespace/:dataset\"),\n builder);\n }\n\n private static URI fileSystemURI(Map match) {\n try {\n return new URI(match.get(URIPattern.SCHEME), null,\n match.get(URIPattern.HOST), -1, \"/\", null, null);\n } catch (URISyntaxException ex) {\n throw new DatasetOperationException(\"[BUG] Could not build FS URI\", ex);\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/Main.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs;\n\nimport com.beust.jcommander.JCommander;\nimport com.beust.jcommander.Parameter;\nimport io.juicefs.bench.NNBench;\nimport io.juicefs.bench.TestDFSIO;\nimport io.juicefs.tools.RangerDownloader;\nimport org.apache.commons.cli.ParseException;\n\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class Main {\n private static final Map COMMAND = new HashMap<>();\n\n @Parameter(names = {\"--help\", \"-h\", \"-help\"}, help = true)\n private boolean help = false;\n\n public abstract static class Command implements Closeable {\n @Parameter(names = {\"--help\", \"-h\", \"-help\"}, help = true)\n public boolean help;\n\n public Command() {\n COMMAND.put(getCommand(), this);\n }\n\n public abstract void init() throws IOException;\n\n public abstract void run() throws IOException;\n\n public abstract String getCommand();\n\n }\n\n public static void main(String[] args) throws ParseException, IOException {\n Main main = new Main();\n Command dfsio = new TestDFSIO();\n Command nnbench = new NNBench();\n Command ranger = new RangerDownloader();\n JCommander jc = JCommander.newBuilder()\n .addObject(main)\n .addCommand(dfsio.getCommand(), dfsio)\n .addCommand(nnbench.getCommand(), nnbench)\n .addCommand(ranger.getCommand(), ranger)\n .build();\n jc.parse(args);\n\n if (main.help) {\n jc.usage();\n return;\n }\n\n Command command = COMMAND.get(jc.getParsedCommand());\n if (command.help) {\n jc.getCommands().get(jc.getParsedCommand()).usage();\n return;\n }\n command.init();\n command.run();\n command.close();\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/bench/AccumulatingReducer.java", "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *

\n * http://www.apache.org/licenses/LICENSE-2.0\n *

\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.bench;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.mapred.MapReduceBase;\nimport org.apache.hadoop.mapred.OutputCollector;\nimport org.apache.hadoop.mapred.Reducer;\nimport org.apache.hadoop.mapred.Reporter;\n\nimport java.io.IOException;\nimport java.util.Iterator;\n\n/**\n * Reducer that accumulates values based on their type.\n *

\n * The type is specified in the key part of the key-value pair\n * as a prefix to the key in the following way\n *

\n * type:key\n *

\n * The values are accumulated according to the types:\n *

    \n *
  • s: - string, concatenate
    • \n *
  • f: - float, summ
    • \n *
  • l: - long, summ
    • \n *
\n */\n@SuppressWarnings(\"deprecation\")\npublic class AccumulatingReducer extends MapReduceBase\n implements Reducer {\n static final String VALUE_TYPE_LONG = \"l:\";\n static final String VALUE_TYPE_FLOAT = \"f:\";\n static final String VALUE_TYPE_STRING = \"s:\";\n private static final Log LOG = LogFactory.getLog(AccumulatingReducer.class);\n\n protected String hostName;\n\n public AccumulatingReducer() {\n try {\n hostName = java.net.InetAddress.getLocalHost().getHostName();\n } catch (Exception e) {\n hostName = \"localhost\";\n }\n LOG.info(\"Starting AccumulatingReducer on \" + hostName);\n }\n\n @Override\n public void reduce(Text key,\n Iterator values,\n OutputCollector output,\n Reporter reporter\n ) throws IOException {\n String field = key.toString();\n\n reporter.setStatus(\"starting \" + field + \" ::host = \" + hostName);\n\n // concatenate strings\n if (field.startsWith(VALUE_TYPE_STRING)) {\n StringBuffer sSum = new StringBuffer();\n while (values.hasNext())\n sSum.append(values.next().toString()).append(\";\");\n output.collect(key, new Text(sSum.toString()));\n reporter.setStatus(\"finished \" + field + \" ::host = \" + hostName);\n return;\n }\n // sum long values\n if (field.startsWith(VALUE_TYPE_FLOAT)) {\n float fSum = 0;\n while (values.hasNext())\n fSum += Float.parseFloat(values.next().toString());\n output.collect(key, new Text(String.valueOf(fSum)));\n reporter.setStatus(\"finished \" + field + \" ::host = \" + hostName);\n return;\n }\n // sum long values\n if (field.startsWith(VALUE_TYPE_LONG)) {\n long lSum = 0;\n while (values.hasNext()) {\n lSum += Long.parseLong(values.next().toString());\n }\n output.collect(key, new Text(String.valueOf(lSum)));\n }\n reporter.setStatus(\"finished \" + field + \" ::host = \" + hostName);\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/bench/IOMapperBase.java", "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *

\n * http://www.apache.org/licenses/LICENSE-2.0\n *

\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.bench;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.hadoop.conf.Configured;\nimport org.apache.hadoop.io.LongWritable;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.mapred.JobConf;\nimport org.apache.hadoop.mapred.Mapper;\nimport org.apache.hadoop.mapred.OutputCollector;\nimport org.apache.hadoop.mapred.Reporter;\n\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic abstract class IOMapperBase extends Configured\n implements Mapper {\n private static final Log LOG = LogFactory.getLog(IOMapperBase.class);\n\n protected String hostName;\n protected Closeable stream;\n protected int threadsPerMap;\n protected int filesPerThread;\n protected ExecutorService pool;\n\n public IOMapperBase() {\n }\n\n @Override\n public void configure(JobConf conf) {\n setConf(conf);\n\n try {\n hostName = InetAddress.getLocalHost().getHostName();\n } catch (Exception e) {\n hostName = \"localhost\";\n }\n threadsPerMap = conf.getInt(\"test.threadsPerMap\", 1);\n filesPerThread = conf.getInt(\"test.filesPerThread\", 1);\n pool = Executors.newFixedThreadPool(threadsPerMap, r -> {\n Thread t = new Thread(r);\n t.setDaemon(true);\n return t;\n });\n }\n\n @Override\n public void close() throws IOException {\n pool.shutdown();\n }\n\n abstract Long doIO(Reporter reporter,\n String name,\n long value, Closeable stream) throws IOException;\n\n\n public Closeable getIOStream(String name) throws IOException {\n return null;\n }\n\n abstract void collectStats(OutputCollector output,\n String name,\n long execTime,\n Long doIOReturnValue) throws IOException;\n\n @Override\n public void map(Text key,\n LongWritable value,\n OutputCollector output,\n Reporter reporter) throws IOException {\n String name = key.toString();\n long longValue = value.get();\n\n reporter.setStatus(\"starting \" + name + \" ::host = \" + hostName);\n AtomicLong execTime = new AtomicLong(0L);\n List> futures = new ArrayList<>(threadsPerMap);\n for (int i = 0; i < threadsPerMap; i++) {\n int id = i;\n Future future = pool.submit(() -> {\n long res = 0;\n for (int j = 0; j < filesPerThread; j++) {\n String filePath = String.format(\"%s/thread-%s/file-%s\", name, id, j);\n try (Closeable stream = getIOStream(filePath)) {\n long tStart = System.currentTimeMillis();\n res += doIO(reporter, name, longValue, stream);\n long tEnd = System.currentTimeMillis();\n execTime.addAndGet(tEnd - tStart);\n } catch (IOException e) {\n throw new RuntimeException(e);\n }\n }\n return res;\n });\n futures.add(future);\n }\n\n Long result = 0L;\n try {\n for (Future future : futures) {\n result += future.get();\n }\n } catch (InterruptedException | ExecutionException e) {\n throw new RuntimeException(e);\n }\n\n collectStats(output, name, execTime.get(), result);\n\n reporter.setStatus(\"finished \" + name + \" ::host = \" + hostName);\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/bench/NNBench.java", "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *

\n * http://www.apache.org/licenses/LICENSE-2.0\n *

\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.bench;\n\nimport com.beust.jcommander.Parameter;\nimport com.beust.jcommander.Parameters;\nimport io.juicefs.Main;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.conf.Configured;\nimport org.apache.hadoop.fs.FSDataInputStream;\nimport org.apache.hadoop.fs.FSDataOutputStream;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.io.LongWritable;\nimport org.apache.hadoop.io.SequenceFile;\nimport org.apache.hadoop.io.SequenceFile.CompressionType;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.mapred.*;\n\nimport java.io.BufferedReader;\nimport java.io.DataInputStream;\nimport java.io.IOException;\nimport java.io.InputStreamReader;\nimport java.text.SimpleDateFormat;\nimport java.util.*;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\n\n@Parameters(commandDescription = \"Distributed create/open/rename/delete meta benchmark\")\npublic class NNBench extends Main.Command {\n private static final Log LOG = LogFactory.getLog(\n NNBench.class);\n\n protected static String CONTROL_DIR_NAME = \"control\";\n protected static String OUTPUT_DIR_NAME = \"output\";\n protected static String DATA_DIR_NAME = \"data\";\n\n @Parameter(description = \"[create | open | rename | delete]\", required = true)\n public static String operation;\n @Parameter(names = {\"-maps\"}, description = \"number of maps\")\n public long numberOfMaps = 1l; // default is 1\n @Parameter(names = {\"-files\"}, description = \"number of files per thread\")\n public long numberOfFiles = 1l; // default is 1\n @Parameter(names = {\"-threads\"}, description = \"threads per map\")\n public int threadsPerMap = 1;\n public long numberOfReduces = 1l; // default is 1\n @Parameter(names = {\"-baseDir\"}, description = \"full path of dir on FileSystem\", required = true)\n public String baseDir = \"/benchmarks/NNBench\"; // default\n @Parameter(names = {\"-deleteBeforeRename\"}, description = \"delete files before or after rename operation\")\n public static boolean deleteBeforeRename;\n @Parameter(names = {\"-local\"}, description = \"run in local single process\")\n private boolean local;\n @Parameter(names = {\"-startTime\"}, description = \"start time in milliseconds\")\n public long startTime = System.currentTimeMillis() + (30 * 1000);\n\n // Supported operations\n private static final String OP_CREATE = \"create\";\n private static final String OP_OPEN = \"open\";\n private static final String OP_RENAME = \"rename\";\n private static final String OP_DELETE = \"delete\";\n\n // To display in the format that matches the NN and DN log format\n // Example: 2007-10-26 00:01:19,853\n static SimpleDateFormat sdf =\n new SimpleDateFormat(\"yyyy-MM-dd' 'HH:mm:ss','S\");\n\n private static Configuration config = new Configuration();\n\n /**\n * Clean up the files before a test run\n *\n * @throws IOException on error\n */\n private void cleanupBeforeTestrun() throws IOException {\n FileSystem tempFS = new Path(baseDir).getFileSystem(config);\n\n // Delete the data directory only if it is the create/write operation\n if (operation.equals(OP_CREATE)) {\n LOG.info(\"Deleting data directory\");\n tempFS.delete(new Path(baseDir, DATA_DIR_NAME), true);\n }\n tempFS.delete(new Path(baseDir, CONTROL_DIR_NAME), true);\n tempFS.delete(new Path(baseDir, OUTPUT_DIR_NAME), true);\n }\n\n /**\n * Create control files before a test run.\n * Number of files created is equal to the number of maps specified\n *\n * @throws IOException on error\n */\n private void createControlFiles() throws IOException {\n FileSystem tempFS = new Path(baseDir).getFileSystem(config);\n LOG.info(\"Creating \" + numberOfMaps + \" control files\");\n\n for (int i = 0; i < numberOfMaps; i++) {\n String strFileName = \"NNBench_Controlfile_\" + i;\n Path filePath = new Path(new Path(baseDir, CONTROL_DIR_NAME),\n strFileName);\n\n SequenceFile.Writer writer = null;\n try {\n writer = SequenceFile.createWriter(tempFS, config, filePath, Text.class,\n LongWritable.class, CompressionType.NONE);\n writer.append(new Text(strFileName), new LongWritable(i));\n } finally {\n if (writer != null) {\n writer.close();\n }\n }\n }\n }\n\n /**\n * Analyze the results\n *\n * @throws IOException on error\n */\n private void analyzeResults() throws IOException {\n final FileSystem fs = new Path(baseDir).getFileSystem(config);\n Path reduceFile = new Path(new Path(baseDir, OUTPUT_DIR_NAME),\n \"part-00000\");\n\n DataInputStream in;\n in = new DataInputStream(fs.open(reduceFile));\n\n BufferedReader lines;\n lines = new BufferedReader(new InputStreamReader(in));\n\n long totalTime = 0l;\n long lateMaps = 0l;\n long numOfExceptions = 0l;\n long successfulFileOps = 0l;\n\n long mapStartTimeTPmS = 0l;\n long mapEndTimeTPmS = 0l;\n\n String resultTPSLine1 = null;\n String resultALLine1 = null;\n\n String line;\n while ((line = lines.readLine()) != null) {\n StringTokenizer tokens = new StringTokenizer(line, \" \\t\\n\\r\\f%;\");\n String attr = tokens.nextToken();\n if (attr.endsWith(\":totalTime\")) {\n totalTime = Long.parseLong(tokens.nextToken());\n } else if (attr.endsWith(\":latemaps\")) {\n lateMaps = Long.parseLong(tokens.nextToken());\n } else if (attr.endsWith(\":numOfExceptions\")) {\n numOfExceptions = Long.parseLong(tokens.nextToken());\n } else if (attr.endsWith(\":successfulFileOps\")) {\n successfulFileOps = Long.parseLong(tokens.nextToken());\n } else if (attr.endsWith(\":mapStartTimeTPmS\")) {\n mapStartTimeTPmS = Long.parseLong(tokens.nextToken());\n } else if (attr.endsWith(\":mapEndTimeTPmS\")) {\n mapEndTimeTPmS = Long.parseLong(tokens.nextToken());\n }\n }\n\n // Average latency is the average time to perform 'n' number of\n // operations, n being the number of files\n double avgLatency = (double) totalTime / successfulFileOps;\n\n double totalTimeTPS =\n (double) (1000 * successfulFileOps) / (mapEndTimeTPmS - mapStartTimeTPmS);\n\n if (operation.equals(OP_CREATE)) {\n resultTPSLine1 = \" TPS: Create: \" +\n (int) (totalTimeTPS);\n resultALLine1 = \" Avg Lat (ms): Create: \" + avgLatency;\n } else if (operation.equals(OP_OPEN)) {\n resultTPSLine1 = \" TPS: Open: \" +\n (int) totalTimeTPS;\n resultALLine1 = \" Avg Lat (ms): Open: \" + avgLatency;\n } else if (operation.equals(OP_RENAME)) {\n resultTPSLine1 = \" TPS: Rename: \" +\n (int) totalTimeTPS;\n resultALLine1 = \" Avg Lat (ms): Rename: \" + avgLatency;\n } else if (operation.equals(OP_DELETE)) {\n resultTPSLine1 = \" TPS: Delete: \" +\n (int) totalTimeTPS;\n resultALLine1 = \" Avg Lat (ms): Delete: \" + avgLatency;\n }\n\n String resultLines[] = {\n \"-------------- NNBench -------------- : \",\n \" Date & time: \" + sdf.format(new Date(\n System.currentTimeMillis())),\n \"\",\n \" Test Operation: \" + operation,\n \" Start time: \" +\n sdf.format(new Date(startTime)),\n \" Maps to run: \" + numberOfMaps,\n \" Threads per map: \" + threadsPerMap,\n \" Files per thread: \" + numberOfFiles,\n \" Successful file operations: \" + successfulFileOps,\n \"\",\n \" # maps that missed the barrier: \" + lateMaps,\n \" # exceptions: \" + numOfExceptions,\n \"\",\n resultTPSLine1,\n resultALLine1,\n \"\",\n \" RAW DATA: TPS Total (ms): \" + totalTime,\n \" RAW DATA: Job Duration (ms): \" + (mapEndTimeTPmS - mapStartTimeTPmS),\n \" RAW DATA: Late maps: \" + lateMaps,\n \" RAW DATA: # of exceptions: \" + numOfExceptions,\n \"\"};\n\n // Write to a file and also dump to log\n for (int i = 0; i < resultLines.length; i++) {\n LOG.info(resultLines[i]);\n }\n }\n\n /**\n * Run the test\n *\n * @throws IOException on error\n */\n public void runTests() throws IOException {\n\n JobConf job = new JobConf(config, NNBench.class);\n\n job.setJobName(\"NNBench-\" + operation);\n FileInputFormat.setInputPaths(job, new Path(baseDir, CONTROL_DIR_NAME));\n job.setInputFormat(SequenceFileInputFormat.class);\n\n // Explicitly set number of max map attempts to 1.\n job.setMaxMapAttempts(1);\n\n // Explicitly turn off speculative execution\n job.setSpeculativeExecution(false);\n\n job.setMapperClass(NNBenchMapper.class);\n job.setReducerClass(NNBenchReducer.class);\n\n FileOutputFormat.setOutputPath(job, new Path(baseDir, OUTPUT_DIR_NAME));\n job.setOutputKeyClass(Text.class);\n job.setOutputValueClass(Text.class);\n job.setNumReduceTasks((int) numberOfReduces);\n JobClient.runJob(job);\n }\n\n /**\n * Validate the inputs\n */\n public void validateInputs() {\n // If it is not one of the four operations, then fail\n if (!operation.equals(OP_CREATE) &&\n !operation.equals(OP_OPEN) &&\n !operation.equals(OP_RENAME) &&\n !operation.equals(OP_DELETE)) {\n System.err.println(\"Error: Unknown operation: \" + operation);\n System.exit(-1);\n }\n\n // If number of maps is a negative number, then fail\n // Hadoop allows the number of maps to be 0\n if (numberOfMaps < 0) {\n System.err.println(\"Error: Number of maps must be a positive number\");\n System.exit(-1);\n }\n\n // If number of reduces is a negative number or 0, then fail\n if (numberOfReduces <= 0) {\n System.err.println(\"Error: Number of reduces must be a positive number\");\n System.exit(-1);\n }\n\n // If number of files is a negative number, then fail\n if (numberOfFiles < 0) {\n System.err.println(\"Error: Number of files must be a positive number\");\n System.exit(-1);\n }\n }\n\n @Override\n public void init() throws IOException {\n LOG.info(\"Test Inputs: \");\n LOG.info(\" Test Operation: \" + operation);\n LOG.info(\" Start time: \" + sdf.format(new Date(startTime)));\n if (!local) {\n LOG.info(\" Number of maps: \" + numberOfMaps);\n }\n LOG.info(\"Number of threads per map: \" + threadsPerMap);\n LOG.info(\" Number of files: \" + numberOfFiles);\n LOG.info(\" Base dir: \" + baseDir);\n\n // Set user-defined parameters, so the map method can access the values\n config.set(\"test.nnbench.operation\", operation);\n config.setLong(\"test.nnbench.maps\", numberOfMaps);\n config.setLong(\"test.nnbench.reduces\", numberOfReduces);\n config.setLong(\"test.nnbench.starttime\", startTime);\n config.setLong(\"test.nnbench.numberoffiles\", numberOfFiles);\n config.set(\"test.nnbench.basedir\", baseDir);\n config.setInt(\"test.nnbench.threadsPerMap\", threadsPerMap);\n config.setBoolean(\"test.nnbench.deleteBeforeRename\", deleteBeforeRename);\n config.setBoolean(\"test.nnbench.local\", local);\n\n config.set(\"test.nnbench.datadir.name\", DATA_DIR_NAME);\n config.set(\"test.nnbench.outputdir.name\", OUTPUT_DIR_NAME);\n config.set(\"test.nnbench.controldir.name\", CONTROL_DIR_NAME);\n }\n\n @Override\n public void run() throws IOException {\n validateInputs();\n cleanupBeforeTestrun();\n if (local) {\n localRun();\n return;\n }\n createControlFiles();\n runTests();\n analyzeResults();\n }\n\n private void localRun() {\n NNBenchMapper mapper = new NNBenchMapper();\n mapper.configure(new JobConf(config));\n\n ExecutorService pool = Executors.newFixedThreadPool(threadsPerMap, r -> {\n Thread t = new Thread(r);\n t.setDaemon(true);\n return t;\n });\n\n long start = System.currentTimeMillis();\n for (int i = 0; i < threadsPerMap; i++) {\n int threadNum = i;\n pool.submit(() -> {\n try {\n mapper.doMap(Collections.synchronizedList(new ArrayList<>()), 0, threadNum);\n } catch (IOException e) {\n e.printStackTrace();\n System.exit(1);\n throw new RuntimeException(e);\n }\n });\n }\n pool.shutdown();\n try {\n pool.awaitTermination(1, TimeUnit.DAYS);\n } catch (InterruptedException ignored) {\n }\n long end = System.currentTimeMillis();\n double totalTimeTPS =\n (double) (1000 * threadsPerMap * numberOfFiles) / (end - start);\n String[] resultLines = {\n \"-------------- NNBench -------------- : \",\n \" Date & time: \" + sdf.format(new Date(\n System.currentTimeMillis())),\n \"\",\n \" Test Operation: \" + operation,\n \" Start time: \" +\n sdf.format(new Date(startTime)),\n \" Threads: \" + threadsPerMap,\n \" Files per thread: \" + numberOfFiles,\n \" Successful file operations: \" + threadsPerMap * numberOfFiles,\n \"\",\n \" TPS: \" + (int) (totalTimeTPS),\n \" Avg Lat (ms): \" + String.format(\"%.2f\", (double) (end - start) / (threadsPerMap * numberOfFiles)),\n \"\",\n \" RAW DATA: Job Duration (ms): \" + (end - start),\n \"\"};\n\n for (int i = 0; i < resultLines.length; i++) {\n LOG.info(resultLines[i]);\n }\n }\n\n @Override\n public String getCommand() {\n return \"nnbench\";\n }\n\n @Override\n public void close() throws IOException {\n\n }\n\n /**\n * Mapper class\n */\n static class NNBenchMapper extends Configured\n implements Mapper {\n FileSystem filesystem = null;\n\n long numberOfFiles = 1l;\n boolean beforeRename = false;\n String baseDir = null;\n String dataDirName = null;\n String op = null;\n final int MAX_OPERATION_EXCEPTIONS = 1000;\n int threadsPerMap = 1;\n boolean local;\n\n ExecutorService executorService;\n\n // Data to collect from the operation\n\n /**\n * Constructor\n */\n public NNBenchMapper() {\n }\n\n\n /**\n * Mapper base implementation\n */\n public void configure(JobConf conf) {\n setConf(conf);\n local = conf.getBoolean(\"test.nnbench.local\", false);\n try {\n baseDir = conf.get(\"test.nnbench.basedir\");\n filesystem = new Path(baseDir).getFileSystem(conf);\n } catch (Exception e) {\n throw new RuntimeException(\"Cannot get file system.\", e);\n }\n\n numberOfFiles = conf.getLong(\"test.nnbench.numberoffiles\", 1l);\n dataDirName = conf.get(\"test.nnbench.datadir.name\");\n op = conf.get(\"test.nnbench.operation\");\n beforeRename = conf.getBoolean(\"test.nnbench.deleteBeforeRename\", false);\n\n threadsPerMap = conf.getInt(\"test.nnbench.threadsPerMap\", 1);\n executorService = Executors.newFixedThreadPool(threadsPerMap, r -> {\n Thread t = new Thread(r);\n t.setDaemon(true);\n return t;\n });\n }\n\n /**\n * Mapper base implementation\n */\n public void close() throws IOException {\n }\n\n /**\n * Returns when the current number of seconds from the epoch equals\n * the command line argument given by -startTime.\n * This allows multiple instances of this program, running on clock\n * synchronized nodes, to start at roughly the same time.\n *\n * @return true if the method was able to sleep for -startTime\n * without interruption; false otherwise\n */\n private boolean barrier() {\n if (local) {\n return true;\n }\n long startTime = getConf().getLong(\"test.nnbench.starttime\", 0l);\n long currentTime = System.currentTimeMillis();\n long sleepTime = startTime - currentTime;\n boolean retVal = false;\n\n // If the sleep time is greater than 0, then sleep and return\n if (sleepTime > 0) {\n LOG.info(\"Waiting in barrier for: \" + sleepTime + \" ms\");\n\n try {\n Thread.sleep(sleepTime);\n retVal = true;\n } catch (Exception e) {\n retVal = false;\n }\n }\n\n return retVal;\n }\n\n /**\n * Map method\n */\n public void map(Text key,\n LongWritable value,\n OutputCollector output,\n Reporter reporter) throws IOException {\n\n\n List res = Collections.synchronizedList(new ArrayList<>());\n\n for (int i = 0; i < threadsPerMap; i++) {\n int threadNum = i;\n executorService.submit(() -> {\n try {\n doMap(res, value.get(), threadNum);\n } catch (IOException e) {\n throw new RuntimeException(e);\n }\n });\n }\n\n executorService.shutdown();\n try {\n executorService.awaitTermination(1, TimeUnit.DAYS);\n } catch (InterruptedException e) {\n throw new RuntimeException(e);\n }\n\n long successOps = 0L;\n for (Entry entry : res) {\n if (entry.key.toString().contains(\"successfulFileOps\")) {\n successOps += Long.parseLong(entry.value.toString());\n }\n output.collect(entry.key, entry.value);\n }\n reporter.setStatus(\"Finish \" + successOps + \" files\");\n }\n\n static class Entry {\n Text key;\n Text value;\n\n Entry(Text key, Text value) {\n this.key = key;\n this.value = value;\n }\n }\n\n private void doMap(List res, long mapId, int threadNum) throws IOException {\n long startTimeTPmS = 0l;\n long endTimeTPms = 0l;\n\n AtomicLong successfulFileOps = new AtomicLong(0L);\n AtomicInteger numOfExceptions = new AtomicInteger(0);\n AtomicLong totalTime = new AtomicLong(0L);\n\n if (barrier()) {\n startTimeTPmS = System.currentTimeMillis();\n if (op.equals(OP_CREATE)) {\n doCreate(mapId, successfulFileOps, numOfExceptions, totalTime, threadNum);\n } else if (op.equals(OP_OPEN)) {\n doOpen(mapId, successfulFileOps, numOfExceptions, totalTime, threadNum);\n } else if (op.equals(OP_RENAME)) {\n doRenameOp(mapId, successfulFileOps, numOfExceptions, totalTime, threadNum);\n } else if (op.equals(OP_DELETE)) {\n doDeleteOp(mapId, successfulFileOps, numOfExceptions, totalTime, threadNum);\n }\n\n endTimeTPms = System.currentTimeMillis();\n } else {\n res.add(new Entry(new Text(\"l:latemaps\"), new Text(\"1\")));\n }\n\n // collect after the map end time is measured\n res.add(new Entry(new Text(\"l:totalTime\"),\n new Text(String.valueOf(totalTime.get()))));\n res.add(new Entry(new Text(\"l:numOfExceptions\"),\n new Text(String.valueOf(numOfExceptions.get()))));\n res.add(new Entry(new Text(\"l:successfulFileOps\"),\n new Text(String.valueOf(successfulFileOps.get()))));\n res.add(new Entry(new Text(\"min:mapStartTimeTPmS\"),\n new Text(String.valueOf(startTimeTPmS))));\n res.add(new Entry(new Text(\"max:mapEndTimeTPmS\"),\n new Text(String.valueOf(endTimeTPms))));\n }\n\n /**\n * Create operation.\n */\n private void doCreate(long mapId,\n AtomicLong successfulFileOps, AtomicInteger numOfExceptions, AtomicLong totalTime, int threadNum) throws IOException {\n FSDataOutputStream out;\n\n for (long l = 0L; l < numberOfFiles; l++) {\n Path filePath = new Path(new Path(baseDir, dataDirName),\n new Path(String.valueOf(mapId), new Path(String.valueOf(threadNum), \"file_\" + l)));\n boolean successfulOp = false;\n while (!successfulOp && numOfExceptions.get() < MAX_OPERATION_EXCEPTIONS) {\n try {\n // Set up timer for measuring AL (transaction #1)\n long startTime = System.currentTimeMillis();\n // Create the file\n out = filesystem.create(filePath, false);\n out.close();\n totalTime.addAndGet(System.currentTimeMillis() - startTime);\n successfulFileOps.getAndIncrement();\n successfulOp = true;\n } catch (IOException e) {\n LOG.info(\"Exception recorded in op: \" +\n \"Create\", e);\n numOfExceptions.getAndIncrement();\n throw e;\n }\n }\n }\n }\n\n /**\n * Open operation\n */\n private void doOpen(long mapId,\n AtomicLong successfulFileOps, AtomicInteger numOfExceptions, AtomicLong totalTime, int threadNum) throws IOException {\n FSDataInputStream input;\n\n for (long l = 0L; l < numberOfFiles; l++) {\n Path filePath = new Path(new Path(baseDir, dataDirName),\n new Path(String.valueOf(mapId), new Path(String.valueOf(threadNum), \"file_\" + l)));\n\n boolean successfulOp = false;\n while (!successfulOp && numOfExceptions.get() < MAX_OPERATION_EXCEPTIONS) {\n try {\n // Set up timer for measuring AL\n long startTime = System.currentTimeMillis();\n input = filesystem.open(filePath);\n input.close();\n totalTime.addAndGet(System.currentTimeMillis() - startTime);\n successfulFileOps.getAndIncrement();\n successfulOp = true;\n } catch (IOException e) {\n LOG.info(\"Exception recorded in op: OpenRead \" + e);\n numOfExceptions.getAndIncrement();\n throw e;\n }\n }\n }\n }\n\n /**\n * Rename operation\n */\n private void doRenameOp(long mapId,\n AtomicLong successfulFileOps, AtomicInteger numOfExceptions, AtomicLong totalTime, int threadNum) throws IOException {\n for (long l = 0L; l < numberOfFiles; l++) {\n Path filePath = new Path(new Path(baseDir, dataDirName),\n new Path(String.valueOf(mapId), new Path(String.valueOf(threadNum), \"file_\" + l)));\n Path filePathR = new Path(new Path(baseDir, dataDirName),\n new Path(String.valueOf(mapId), new Path(String.valueOf(threadNum), \"file_r_\" + l)));\n\n boolean successfulOp = false;\n while (!successfulOp && numOfExceptions.get() < MAX_OPERATION_EXCEPTIONS) {\n try {\n // Set up timer for measuring AL\n long startTime = System.currentTimeMillis();\n filesystem.rename(filePath, filePathR);\n totalTime.addAndGet(System.currentTimeMillis() - startTime);\n successfulFileOps.getAndIncrement();\n successfulOp = true;\n } catch (IOException e) {\n LOG.info(\"Exception recorded in op: Rename\");\n numOfExceptions.getAndIncrement();\n throw e;\n }\n }\n }\n }\n\n /**\n * Delete operation\n */\n private void doDeleteOp(long mapId,\n AtomicLong successfulFileOps, AtomicInteger numOfExceptions, AtomicLong totalTime, int threadNum) throws IOException {\n for (long l = 0L; l < numberOfFiles; l++) {\n Path filePath;\n if (beforeRename) {\n filePath = new Path(new Path(baseDir, dataDirName),\n new Path(String.valueOf(mapId), new Path(String.valueOf(threadNum), \"file_\" + l)));\n } else {\n filePath = new Path(new Path(baseDir, dataDirName),\n new Path(String.valueOf(mapId), new Path(String.valueOf(threadNum), \"file_r_\" + l)));\n }\n\n boolean successfulOp = false;\n while (!successfulOp && numOfExceptions.get() < MAX_OPERATION_EXCEPTIONS) {\n try {\n // Set up timer for measuring AL\n long startTime = System.currentTimeMillis();\n filesystem.delete(filePath, false);\n totalTime.addAndGet(System.currentTimeMillis() - startTime);\n successfulFileOps.getAndIncrement();\n successfulOp = true;\n } catch (IOException e) {\n LOG.info(\"Exception in recorded op: Delete\");\n numOfExceptions.getAndIncrement();\n throw e;\n }\n }\n }\n }\n }\n\n /**\n * Reducer class\n */\n static class NNBenchReducer extends MapReduceBase\n implements Reducer {\n\n protected String hostName;\n\n public NNBenchReducer() {\n LOG.info(\"Starting NNBenchReducer !!!\");\n try {\n hostName = java.net.InetAddress.getLocalHost().getHostName();\n } catch (Exception e) {\n hostName = \"localhost\";\n }\n LOG.info(\"Starting NNBenchReducer on \" + hostName);\n }\n\n /**\n * Reduce method\n */\n public void reduce(Text key,\n Iterator values,\n OutputCollector output,\n Reporter reporter\n ) throws IOException {\n String field = key.toString();\n\n reporter.setStatus(\"starting \" + field + \" ::host = \" + hostName);\n\n // sum long values\n if (field.startsWith(\"l:\")) {\n long lSum = 0;\n while (values.hasNext()) {\n lSum += Long.parseLong(values.next().toString());\n }\n output.collect(key, new Text(String.valueOf(lSum)));\n }\n\n if (field.startsWith(\"min:\")) {\n long minVal = -1;\n while (values.hasNext()) {\n long value = Long.parseLong(values.next().toString());\n\n if (minVal == -1) {\n minVal = value;\n } else {\n if (value != 0 && value < minVal) {\n minVal = value;\n }\n }\n }\n output.collect(key, new Text(String.valueOf(minVal)));\n }\n\n if (field.startsWith(\"max:\")) {\n long maxVal = -1;\n while (values.hasNext()) {\n long value = Long.parseLong(values.next().toString());\n\n if (maxVal == -1) {\n maxVal = value;\n } else {\n if (value > maxVal) {\n maxVal = value;\n }\n }\n }\n output.collect(key, new Text(String.valueOf(maxVal)));\n }\n\n reporter.setStatus(\"finished \" + field + \" ::host = \" + hostName);\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/bench/TestDFSIO.java", "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *

\n * http://www.apache.org/licenses/LICENSE-2.0\n *

\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.bench;\n\nimport com.beust.jcommander.Parameter;\nimport com.beust.jcommander.Parameters;\nimport io.juicefs.Main;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.*;\nimport org.apache.hadoop.io.LongWritable;\nimport org.apache.hadoop.io.SequenceFile;\nimport org.apache.hadoop.io.SequenceFile.CompressionType;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.io.compress.CompressionCodec;\nimport org.apache.hadoop.mapred.*;\nimport org.apache.hadoop.util.ReflectionUtils;\n\nimport java.io.*;\nimport java.text.DecimalFormat;\nimport java.util.Date;\nimport java.util.Locale;\nimport java.util.StringTokenizer;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\n\n\n@Parameters(commandDescription = \"Distributed i/o benchmark\")\npublic class TestDFSIO extends Main.Command {\n // Constants\n private static final Log LOG = LogFactory.getLog(TestDFSIO.class);\n private static final String BASE_FILE_NAME = \"test_io_\";\n private static final long MEGA = ByteMultiple.MB.value();\n\n @Parameter(description = \"[-read | -write]\", required = true)\n private String testType;\n @Parameter(names = {\"-random\"}, description = \"random read\")\n private boolean random;\n @Parameter(names = {\"-backward\"}, description = \"backward read\")\n private boolean backward;\n @Parameter(names = {\"-skip\"}, description = \"skip read\")\n private boolean skip;\n @Parameter(names = {\"-local\"}, description = \"run in local single process\")\n private boolean local;\n\n @Parameter(names = {\"-baseDir\"}, description = \"full path of dir on FileSystem\", required = true)\n private String baseDir = \"/benchmarks/DFSIO\";\n\n @Parameter(names = {\"-bufferSize\"}, description = \"bufferSize[B|KB|MB|GB|TB]\")\n private String bufferSize = \"1MB\";\n @Parameter(names = {\"-size\"}, description = \"per file size[B|KB|MB|GB|TB]\")\n private String size = \"1GB\";\n @Parameter(names = {\"-maps\"}, description = \"number of maps\")\n private int maps = 1;\n @Parameter(names = {\"-threads\"}, description = \"threads per map\")\n private int threadsPerMap = 1;\n @Parameter(names = {\"-files\"}, description = \"number of files per thread\")\n private int filesPerThread = 1;\n @Parameter(names = {\"-skipSize\"}, description = \"skipSize[B|KB|MB|GB|TB]\")\n private String skipSize;\n @Parameter(names = {\"-compression\"}, description = \"codecClassName\")\n String compression = null;\n @Parameter(names = {\"-randomBytes\"}, description = \"generate randomBytes\")\n boolean randomBytes = false;\n\n private FileSystem fs;\n private TestType type;\n private Configuration config;\n\n @Override\n public void close() throws IOException {\n this.fs.close();\n }\n\n private enum TestType {\n TEST_TYPE_READ(\"read\"),\n TEST_TYPE_WRITE(\"write\"),\n TEST_TYPE_CLEANUP(\"cleanup\"),\n TEST_TYPE_APPEND(\"append\"),\n TEST_TYPE_READ_RANDOM(\"random read\"),\n TEST_TYPE_READ_BACKWARD(\"backward read\"),\n TEST_TYPE_READ_SKIP(\"skip read\"),\n TEST_TYPE_TRUNCATE(\"truncate\");\n\n private String type;\n\n TestType(String t) {\n type = t;\n }\n\n @Override // String\n public String toString() {\n return type;\n }\n }\n\n static enum ByteMultiple {\n B(1L),\n KB(0x400L),\n MB(0x100000L),\n GB(0x40000000L),\n TB(0x10000000000L);\n\n private long multiplier;\n\n private ByteMultiple(long mult) {\n multiplier = mult;\n }\n\n long value() {\n return multiplier;\n }\n\n static ByteMultiple parseString(String sMultiple) {\n if (sMultiple == null || sMultiple.isEmpty()) // MB by default\n {\n return MB;\n }\n String sMU = sMultiple.toUpperCase(Locale.ENGLISH);\n if (B.name().toUpperCase(Locale.ENGLISH).endsWith(sMU)) {\n return B;\n }\n if (KB.name().toUpperCase(Locale.ENGLISH).endsWith(sMU)) {\n return KB;\n }\n if (MB.name().toUpperCase(Locale.ENGLISH).endsWith(sMU)) {\n return MB;\n }\n if (GB.name().toUpperCase(Locale.ENGLISH).endsWith(sMU)) {\n return GB;\n }\n if (TB.name().toUpperCase(Locale.ENGLISH).endsWith(sMU)) {\n return TB;\n }\n throw new IllegalArgumentException(\"Unsupported ByteMultiple \" + sMultiple);\n }\n }\n\n public TestDFSIO() {\n this.config = new Configuration();\n }\n\n @Override\n public void init() throws IOException {\n this.config = new Configuration();\n config.setBoolean(\"dfs.support.append\", true);\n this.fs = new Path(baseDir).getFileSystem(config);\n\n checkArgs();\n switch (testType) {\n case \"-read\":\n type = TestType.TEST_TYPE_READ;\n break;\n case \"-write\":\n type = TestType.TEST_TYPE_WRITE;\n break;\n case \"-append\":\n type = TestType.TEST_TYPE_APPEND;\n break;\n case \"-truncate\":\n type = TestType.TEST_TYPE_TRUNCATE;\n break;\n case \"-clean\":\n type = TestType.TEST_TYPE_CLEANUP;\n break;\n default:\n throw new IllegalArgumentException(\"wrong type\");\n }\n if (random) {\n type = TestType.TEST_TYPE_READ_RANDOM;\n } else if (backward) {\n type = TestType.TEST_TYPE_READ_BACKWARD;\n } else if (skip) {\n type = TestType.TEST_TYPE_READ_SKIP;\n }\n int bufferSizeBytes = (int) parseSize(bufferSize);\n long sizeInBytes = parseSize(size);\n long skipSizeInBytes = skipSize == null ? 0 : parseSize(skipSize);\n if (type == TestType.TEST_TYPE_READ_BACKWARD) {\n skipSizeInBytes = -bufferSizeBytes;\n } else if (type == TestType.TEST_TYPE_READ_SKIP && skipSizeInBytes == 0) {\n skipSizeInBytes = bufferSizeBytes;\n }\n\n config.setInt(\"test.io.file.buffer.size\", bufferSizeBytes);\n config.setLong(\"test.io.skip.size\", skipSizeInBytes);\n config.setBoolean(\"dfs.support.append\", true);\n config.setInt(\"test.threadsPerMap\", threadsPerMap);\n config.setInt(\"test.filesPerThread\", filesPerThread);\n config.set(\"test.basedir\", baseDir);\n config.setBoolean(\"test.randomBytes\", randomBytes);\n\n LOG.info(\"type = \" + type);\n if (!local) {\n LOG.info(\"maps = \" + maps);\n }\n LOG.info(\"threads = \" + threadsPerMap);\n LOG.info(\"files = \" + filesPerThread);\n LOG.info(\"randomBytes = \" + randomBytes);\n LOG.info(\"fileSize (MB) = \" + TestDFSIO.toMB(sizeInBytes));\n LOG.info(\"bufferSize = \" + bufferSize);\n if (skipSizeInBytes > 0)\n LOG.info(\"skipSize = \" + skipSize);\n LOG.info(\"baseDir = \" + baseDir);\n\n createControlFile(fs, sizeInBytes, maps);\n if (compression != null) {\n LOG.info(\"compressionClass = \" + compression);\n }\n }\n\n private void checkArgs() {\n if (!testType.equals(\"-read\")) {\n if (random || backward || skip) {\n throw new IllegalArgumentException(\"random, backward, skip are only valid under read\");\n }\n } else {\n boolean[] conds = {random, backward, skip};\n int trueCount = 0;\n for (boolean cond : conds) {\n if (cond) {\n trueCount++;\n if (trueCount > 1) {\n throw new IllegalArgumentException(\"random, backward, skip are mutually exclusive\");\n }\n }\n }\n }\n }\n\n private void localRun(TestType testType) throws IOException {\n IOStatMapper ioer;\n switch (testType) {\n case TEST_TYPE_READ:\n ioer = new ReadMapper();\n break;\n case TEST_TYPE_WRITE:\n ioer = new WriteMapper();\n fs.delete(getDataDir(config), true);\n break;\n case TEST_TYPE_APPEND:\n ioer = new AppendMapper();\n break;\n case TEST_TYPE_READ_RANDOM:\n case TEST_TYPE_READ_BACKWARD:\n case TEST_TYPE_READ_SKIP:\n ioer = new RandomReadMapper();\n break;\n case TEST_TYPE_TRUNCATE:\n ioer = new TruncateMapper();\n break;\n default:\n return;\n }\n ExecutorService pool = Executors.newFixedThreadPool(threadsPerMap, r -> {\n Thread t = new Thread(r);\n t.setDaemon(true);\n return t;\n });\n\n ioer.configure(new JobConf(config));\n AtomicLong sizeProcessed = new AtomicLong();\n long start = System.currentTimeMillis();\n for (int i = 0; i < threadsPerMap; i++) {\n int id = i;\n pool.execute(() -> {\n for (int j = 0; j < filesPerThread; j++) {\n String name = String.format(\"%s/thread-%s/file-%s\", getFileName(0), id, j);\n try {\n Long res = ioer.doIO(Reporter.NULL, name, parseSize(size), ioer.getIOStream(name));\n sizeProcessed.addAndGet(res);\n } catch (IOException e) {\n e.printStackTrace();\n System.exit(1);\n }\n }\n });\n\n }\n pool.shutdown();\n try {\n pool.awaitTermination(1, TimeUnit.DAYS);\n } catch (InterruptedException ignored) {\n }\n long end = System.currentTimeMillis();\n\n DecimalFormat df = new DecimalFormat(\"#.##\");\n String resultLines[] = {\n \"----- TestClient ----- : \" + testType,\n \" Date & time: \" + new Date(System.currentTimeMillis()),\n \" Number of threads: \" + threadsPerMap,\n \"Number files per thread: \" + filesPerThread,\n \" Total files: \" + threadsPerMap * filesPerThread,\n \" Total MBytes processed: \" + df.format(toMB(sizeProcessed.get())),\n \"Total Throughput MB/sec: \" + df.format(toMB(sizeProcessed.get()) / msToSecs(end - start)),\n \" Test exec time sec: \" + df.format(msToSecs(end - start)),\n \"\"};\n\n for (String resultLine : resultLines) {\n LOG.info(resultLine);\n }\n }\n\n @Override\n public void run() throws IOException {\n if (type == TestType.TEST_TYPE_CLEANUP) {\n cleanup(fs);\n return;\n }\n if (local) {\n localRun(type);\n return;\n }\n long tStart = System.currentTimeMillis();\n switch (type) {\n case TEST_TYPE_WRITE:\n writeTest(fs);\n break;\n case TEST_TYPE_READ:\n readTest(fs);\n break;\n case TEST_TYPE_APPEND:\n appendTest(fs);\n break;\n case TEST_TYPE_READ_RANDOM:\n case TEST_TYPE_READ_BACKWARD:\n case TEST_TYPE_READ_SKIP:\n randomReadTest(fs);\n break;\n case TEST_TYPE_TRUNCATE:\n truncateTest(fs);\n break;\n default:\n }\n long execTime = System.currentTimeMillis() - tStart;\n\n analyzeResult(fs, type, execTime);\n }\n\n @Override\n public String getCommand() {\n return \"dfsio\";\n }\n\n private String getBaseDir(Configuration conf) {\n return baseDir;\n }\n\n private Path getControlDir(Configuration conf) {\n return new Path(getBaseDir(conf), \"io_control\");\n }\n\n private Path getWriteDir(Configuration conf) {\n return new Path(getBaseDir(conf), \"io_write\");\n }\n\n private Path getReadDir(Configuration conf) {\n return new Path(getBaseDir(conf), \"io_read\");\n }\n\n private Path getAppendDir(Configuration conf) {\n return new Path(getBaseDir(conf), \"io_append\");\n }\n\n private Path getRandomReadDir(Configuration conf) {\n return new Path(getBaseDir(conf), \"io_random_read\");\n }\n\n private Path getTruncateDir(Configuration conf) {\n return new Path(getBaseDir(conf), \"io_truncate\");\n }\n\n private Path getDataDir(Configuration conf) {\n return new Path(getBaseDir(conf), \"io_data\");\n }\n\n\n @SuppressWarnings(\"deprecation\")\n private void createControlFile(FileSystem fs,\n long nrBytes, // in bytes\n int maps\n ) throws IOException {\n LOG.info(\"creating control file: \" + nrBytes + \" bytes, \" + maps + \" files\");\n final int maxDirItems = config.getInt(\"dfs.namenode.fs-limits.max-directory-items\", 1024 * 1024);\n Path controlDir = getControlDir(config);\n\n if (maps > maxDirItems) {\n final String message = \"The directory item limit of \" + controlDir +\n \" is exceeded: limit=\" + maxDirItems + \" items=\" + maps;\n throw new IOException(message);\n }\n\n fs.delete(controlDir, true);\n\n for (int i = 0; i < maps; i++) {\n String name = getFileName(i);\n Path controlFile = new Path(controlDir, \"in_file_\" + name);\n SequenceFile.Writer writer = null;\n try {\n writer = SequenceFile.createWriter(fs, config, controlFile,\n Text.class, LongWritable.class,\n CompressionType.NONE);\n writer.append(new Text(name), new LongWritable(nrBytes));\n } catch (Exception e) {\n throw new IOException(e.getLocalizedMessage());\n } finally {\n if (writer != null) {\n writer.close();\n }\n }\n }\n LOG.info(\"created control files for: \" + maps + \" files\");\n }\n\n private static String getFileName(int fIdx) {\n return BASE_FILE_NAME + fIdx;\n }\n\n /**\n * Write/Read mapper base class.\n *

\n * Collects the following statistics per task:\n *

    \n *
  • number of tasks completed
    • \n *
  • number of bytes written/read
    • \n *
  • execution time
    • \n *
  • i/o rate
    • \n *
  • i/o rate squared
    • \n *
\n */\n private abstract static class IOStatMapper extends IOMapperBase {\n protected CompressionCodec compressionCodec;\n private static final ThreadLocalRandom random = ThreadLocalRandom.current();\n private boolean randomBytes;\n protected FileSystem fs;\n protected String baseDir;\n protected ThreadLocal buffer;\n protected int bufferSize;\n\n IOStatMapper() {\n }\n\n public byte[] getBuffer() {\n if (randomBytes) {\n random.nextBytes(buffer.get());\n }\n return buffer.get();\n }\n\n @Override // Mapper\n public void configure(JobConf conf) {\n super.configure(conf);\n bufferSize = conf.getInt(\"test.io.file.buffer.size\", 4096);\n buffer = ThreadLocal.withInitial(() -> new byte[bufferSize]);\n try {\n baseDir = conf.get(\"test.basedir\");\n fs = new Path(baseDir).getFileSystem(conf);\n } catch (IOException e) {\n throw new RuntimeException(\"Cannot create file system.\", e);\n }\n randomBytes = conf.getBoolean(\"test.randomBytes\", false);\n\n // grab compression\n String compression = getConf().get(\"test.io.compression.class\", null);\n Class codec;\n\n // try to initialize codec\n try {\n codec = (compression == null) ? null :\n Class.forName(compression).asSubclass(CompressionCodec.class);\n } catch (Exception e) {\n throw new RuntimeException(\"Compression codec not found: \", e);\n }\n\n if (codec != null) {\n compressionCodec = (CompressionCodec)\n ReflectionUtils.newInstance(codec, getConf());\n }\n\n }\n\n Path getDataDir() {\n return new Path(baseDir, \"io_data\");\n }\n\n @Override\n // IOMapperBase\n void collectStats(OutputCollector output,\n String name,\n long execTime,\n Long objSize) throws IOException {\n long totalSize = objSize;\n float ioRateMbSec = (float) totalSize * 1000 / (execTime * MEGA);\n LOG.info(\"Number of bytes processed = \" + totalSize);\n LOG.info(\"Exec time = \" + execTime);\n LOG.info(\"IO rate = \" + ioRateMbSec);\n\n output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + \"tasks\"),\n new Text(String.valueOf(threadsPerMap * filesPerThread)));\n output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + \"size\"),\n new Text(String.valueOf(totalSize)));\n output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + \"time\"),\n new Text(String.valueOf(execTime)));\n output.collect(new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + \"rate\"),\n new Text(String.valueOf(ioRateMbSec * 1000 * threadsPerMap)));\n output.collect(new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + \"sqrate\"),\n new Text(String.valueOf(ioRateMbSec * ioRateMbSec * 1000 * threadsPerMap)));\n }\n }\n\n /**\n * Write mapper class.\n */\n public static class WriteMapper extends IOStatMapper {\n\n public WriteMapper() {\n }\n\n @Override // IOMapperBase\n public Closeable getIOStream(String name) throws IOException {\n // create file\n Path f = new Path(getDataDir(), name);\n fs.mkdirs(f.getParent());\n OutputStream out =\n fs.create(f, false, bufferSize);\n if (compressionCodec != null) {\n out = compressionCodec.createOutputStream(out);\n }\n LOG.info(\"out = \" + out.getClass().getName());\n return out;\n }\n\n @Override // IOMapperBase\n public Long doIO(Reporter reporter,\n String name,\n long totalSize, // in bytes\n Closeable stream) throws IOException {\n OutputStream out = (OutputStream) stream;\n\n // write to the file\n long nrRemaining;\n for (nrRemaining = totalSize; nrRemaining > 0; nrRemaining -= bufferSize) {\n int curSize = (bufferSize < nrRemaining) ? bufferSize : (int) nrRemaining;\n out.write(getBuffer(), 0, curSize);\n reporter.setStatus(\"writing \" + name + \"@\" +\n (totalSize - nrRemaining) + \"/\" + totalSize\n + \" ::host = \" + hostName);\n }\n return Long.valueOf(totalSize);\n }\n }\n\n private long writeTest(FileSystem fs) throws IOException {\n Path writeDir = getWriteDir(config);\n fs.delete(getDataDir(config), true);\n fs.delete(writeDir, true);\n long tStart = System.currentTimeMillis();\n runIOTest(WriteMapper.class, writeDir);\n long execTime = System.currentTimeMillis() - tStart;\n return execTime;\n }\n\n private void runIOTest(\n Class> mapperClass,\n Path outputDir) throws IOException {\n JobConf job = new JobConf(config, TestDFSIO.class);\n job.setBoolean(\"mapreduce.output.fileoutputformat.compress\", false);\n\n FileInputFormat.setInputPaths(job, getControlDir(config));\n job.setInputFormat(SequenceFileInputFormat.class);\n\n job.setMapperClass(mapperClass);\n job.setReducerClass(AccumulatingReducer.class);\n\n FileOutputFormat.setOutputPath(job, outputDir);\n job.setOutputKeyClass(Text.class);\n job.setOutputValueClass(Text.class);\n job.setNumReduceTasks(1);\n JobClient.runJob(job);\n }\n\n /**\n * Append mapper class.\n */\n public static class AppendMapper extends IOStatMapper {\n\n public AppendMapper() {\n }\n\n @Override // IOMapperBase\n public Closeable getIOStream(String name) throws IOException {\n // open file for append\n OutputStream out =\n fs.append(new Path(getDataDir(), name), bufferSize);\n if (compressionCodec != null)\n out = compressionCodec.createOutputStream(out);\n LOG.info(\"out = \" + out.getClass().getName());\n return out;\n }\n\n @Override // IOMapperBase\n public Long doIO(Reporter reporter,\n String name,\n long totalSize, // in bytes\n Closeable stream) throws IOException {\n OutputStream out = (OutputStream) stream;\n // write to the file\n long nrRemaining;\n for (nrRemaining = totalSize; nrRemaining > 0; nrRemaining -= bufferSize) {\n int curSize = (bufferSize < nrRemaining) ? bufferSize : (int) nrRemaining;\n out.write(getBuffer(), 0, curSize);\n reporter.setStatus(\"writing \" + name + \"@\" +\n (totalSize - nrRemaining) + \"/\" + totalSize\n + \" ::host = \" + hostName);\n }\n return totalSize;\n }\n\n\n }\n\n private long appendTest(FileSystem fs) throws IOException {\n Path appendDir = getAppendDir(config);\n fs.delete(appendDir, true);\n long tStart = System.currentTimeMillis();\n runIOTest(AppendMapper.class, appendDir);\n return System.currentTimeMillis() - tStart;\n }\n\n /**\n * Read mapper class.\n */\n public static class ReadMapper extends IOStatMapper {\n\n public ReadMapper() {\n }\n\n @Override // IOMapperBase\n public Closeable getIOStream(String name) throws IOException {\n // open file\n InputStream in = fs.open(new Path(getDataDir(), name));\n if (compressionCodec != null) {\n in = compressionCodec.createInputStream(in);\n }\n LOG.info(\"in = \" + in.getClass().getName());\n return in;\n }\n\n @Override // IOMapperBase\n public Long doIO(Reporter reporter,\n String name,\n long totalSize, // in bytes\n Closeable stream) throws IOException {\n InputStream in = (InputStream) stream;\n long actualSize = 0;\n while (actualSize < totalSize) {\n int curSize = in.read(buffer.get(), 0, bufferSize);\n if (curSize < 0) {\n break;\n }\n actualSize += curSize;\n reporter.setStatus(\"reading \" + name + \"@\" +\n actualSize + \"/\" + totalSize\n + \" ::host = \" + hostName);\n }\n return actualSize;\n }\n }\n\n private long readTest(FileSystem fs) throws IOException {\n Path readDir = getReadDir(config);\n fs.delete(readDir, true);\n long tStart = System.currentTimeMillis();\n runIOTest(ReadMapper.class, readDir);\n return System.currentTimeMillis() - tStart;\n }\n\n public static class RandomReadMapper extends IOStatMapper {\n private ThreadLocalRandom rnd;\n private long fileSize;\n private long skipSize;\n\n @Override // Mapper\n public void configure(JobConf conf) {\n super.configure(conf);\n skipSize = conf.getLong(\"test.io.skip.size\", 0);\n }\n\n public RandomReadMapper() {\n rnd = ThreadLocalRandom.current();\n }\n\n @Override // IOMapperBase\n public Closeable getIOStream(String name) throws IOException {\n Path filePath = new Path(getDataDir(), name);\n this.fileSize = fs.getFileStatus(filePath).getLen();\n InputStream in = fs.open(filePath);\n if (compressionCodec != null)\n in = new FSDataInputStream(compressionCodec.createInputStream(in));\n LOG.info(\"in = \" + in.getClass().getName());\n LOG.info(\"skipSize = \" + skipSize);\n return in;\n }\n\n @Override // IOMapperBase\n public Long doIO(Reporter reporter,\n String name,\n long totalSize, // in bytes\n Closeable stream) throws IOException {\n PositionedReadable in = (PositionedReadable) stream;\n long actualSize = 0;\n for (long pos = nextOffset(-1);\n actualSize < totalSize; pos = nextOffset(pos)) {\n int curSize = in.read(pos, buffer.get(), 0, bufferSize);\n if (curSize < 0) break;\n actualSize += curSize;\n reporter.setStatus(\"reading \" + name + \"@\" +\n actualSize + \"/\" + totalSize\n + \" ::host = \" + hostName);\n }\n return actualSize;\n }\n\n /**\n * Get next offset for reading.\n * If current < 0 then choose initial offset according to the read type.\n *\n * @param current offset\n * @return\n */\n private long nextOffset(long current) {\n if (skipSize == 0)\n return rnd.nextLong(fileSize);\n if (skipSize > 0)\n return (current < 0) ? 0 : (current + bufferSize + skipSize);\n // skipSize < 0\n return (current < 0) ? Math.max(0, fileSize - bufferSize) :\n Math.max(0, current + skipSize);\n }\n }\n\n private long randomReadTest(FileSystem fs) throws IOException {\n Path readDir = getRandomReadDir(config);\n fs.delete(readDir, true);\n long tStart = System.currentTimeMillis();\n runIOTest(RandomReadMapper.class, readDir);\n return System.currentTimeMillis() - tStart;\n }\n\n /**\n * Truncate mapper class.\n * The mapper truncates given file to the newLength, specified by -size.\n */\n public static class TruncateMapper extends IOStatMapper {\n private static final long DELAY = 100L;\n\n private Path filePath;\n private long fileSize;\n\n @Override // IOMapperBase\n public Closeable getIOStream(String name) throws IOException {\n filePath = new Path(getDataDir(), name);\n fileSize = fs.getFileStatus(filePath).getLen();\n return null;\n }\n\n @Override // IOMapperBase\n public Long doIO(Reporter reporter,\n String name,\n long newLength, // in bytes\n Closeable stream) throws IOException {\n boolean isClosed = fs.truncate(filePath, newLength);\n reporter.setStatus(\"truncating \" + name + \" to newLength \" +\n newLength + \" ::host = \" + hostName);\n for (int i = 0; !isClosed; i++) {\n try {\n Thread.sleep(DELAY);\n } catch (InterruptedException ignored) {\n }\n FileStatus status = fs.getFileStatus(filePath);\n assert status != null : \"status is null\";\n isClosed = (status.getLen() == newLength);\n reporter.setStatus(\"truncate recover for \" + name + \" to newLength \" +\n newLength + \" attempt \" + i + \" ::host = \" + hostName);\n }\n return fileSize - newLength;\n }\n }\n\n private long truncateTest(FileSystem fs) throws IOException {\n Path TruncateDir = getTruncateDir(config);\n fs.delete(TruncateDir, true);\n long tStart = System.currentTimeMillis();\n runIOTest(TruncateMapper.class, TruncateDir);\n return System.currentTimeMillis() - tStart;\n }\n\n /**\n * Returns size in bytes.\n *\n * @param arg = {d}[B|KB|MB|GB|TB]\n * @return\n */\n static long parseSize(String arg) {\n String[] args = arg.split(\"\\\\D\", 2); // get digits\n assert args.length <= 2;\n long nrBytes = Long.parseLong(args[0]);\n String bytesMult = arg.substring(args[0].length()); // get byte multiple\n return nrBytes * ByteMultiple.parseString(bytesMult).value();\n }\n\n static float toMB(long bytes) {\n return ((float) bytes) / MEGA;\n }\n\n static float msToSecs(long timeMillis) {\n return timeMillis / 1000.0f;\n }\n\n private void analyzeResult(FileSystem fs,\n TestType testType,\n long execTime\n ) throws IOException {\n Path reduceFile = getReduceFilePath(testType);\n long tasks = 0;\n long size = 0;\n long time = 0;\n float rate = 0;\n float sqrate = 0;\n DataInputStream in = null;\n BufferedReader lines = null;\n try {\n in = new DataInputStream(fs.open(reduceFile));\n lines = new BufferedReader(new InputStreamReader(in));\n String line;\n while ((line = lines.readLine()) != null) {\n StringTokenizer tokens = new StringTokenizer(line, \" \\t\\n\\r\\f%\");\n String attr = tokens.nextToken();\n if (attr.endsWith(\":tasks\"))\n tasks = Long.parseLong(tokens.nextToken());\n else if (attr.endsWith(\":size\"))\n size = Long.parseLong(tokens.nextToken());\n else if (attr.endsWith(\":time\"))\n time = Long.parseLong(tokens.nextToken());\n else if (attr.endsWith(\":rate\"))\n rate = Float.parseFloat(tokens.nextToken());\n else if (attr.endsWith(\":sqrate\"))\n sqrate = Float.parseFloat(tokens.nextToken());\n }\n } finally {\n if (in != null) in.close();\n if (lines != null) lines.close();\n }\n\n double med = rate / 1000 / tasks;\n double stdDev = Math.sqrt(Math.abs(sqrate / 1000 / tasks - med * med));\n DecimalFormat df = new DecimalFormat(\"#.##\");\n String resultLines[] = {\n \"----- TestDFSIO ----- : \" + testType,\n \" Date & time: \" + new Date(System.currentTimeMillis()),\n \" Number of files: \" + tasks,\n \" Total MBytes processed: \" + df.format(toMB(size)),\n \"Total Throughput MB/sec: \" + df.format(toMB(size) / msToSecs(time) * tasks),\n \" Average IO rate MB/sec: \" + df.format(med),\n \" IO rate std deviation: \" + df.format(stdDev),\n \" Test exec time sec: \" + df.format(msToSecs(execTime)),\n \"\"};\n for (String resultLine : resultLines) {\n LOG.info(resultLine);\n }\n }\n\n private Path getReduceFilePath(TestType testType) {\n switch (testType) {\n case TEST_TYPE_WRITE:\n return new Path(getWriteDir(config), \"part-00000\");\n case TEST_TYPE_APPEND:\n return new Path(getAppendDir(config), \"part-00000\");\n case TEST_TYPE_READ:\n return new Path(getReadDir(config), \"part-00000\");\n case TEST_TYPE_READ_RANDOM:\n case TEST_TYPE_READ_BACKWARD:\n case TEST_TYPE_READ_SKIP:\n return new Path(getRandomReadDir(config), \"part-00000\");\n case TEST_TYPE_TRUNCATE:\n return new Path(getTruncateDir(config), \"part-00000\");\n default:\n }\n return null;\n }\n\n private void cleanup(FileSystem fs)\n throws IOException {\n LOG.info(\"Cleaning up test files\");\n fs.delete(new Path(getBaseDir(config)), true);\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/exception/QuotaExceededException.java", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.exception;\n\nimport java.io.IOException;\n\npublic class QuotaExceededException extends IOException {\n protected static final long serialVersionUID = 1L;\n\n public QuotaExceededException() {\n }\n\n public QuotaExceededException(String msg) {\n super(msg);\n }\n\n @Override\n public String getMessage() {\n return super.getMessage();\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/kerberos/AuthCredential.java", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.kerberos;\n\npublic class AuthCredential {\n private String method;\n private byte[] credential;\n\n public AuthCredential(String method, byte[] credential) {\n this.method = method;\n this.credential = credential;\n }\n\n public String getMethod() {\n return method;\n }\n\n public void setMethod(String method) {\n this.method = method;\n }\n\n public byte[] getCredential() {\n return credential;\n }\n\n public void setCredential(byte[] credential) {\n this.credential = credential;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/kerberos/JuiceFSDelegationTokenIdentifier.java", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.kerberos;\n\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;\n\npublic class JuiceFSDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier {\n public static final Text TOKEN_KIND = new Text(\"JUICEFS_DELEGATION_TOKEN\");\n\n public JuiceFSDelegationTokenIdentifier() {\n }\n\n public JuiceFSDelegationTokenIdentifier(String owner, String renewer, String realUser) {\n super(new Text(owner), new Text(renewer), realUser == null ? null : new Text(realUser));\n }\n\n @Override\n public Text getKind() {\n return TOKEN_KIND;\n }\n\n @Override\n public String toString() {\n return \"token for \" + getUser().getShortUserName() +\n \": \" + super.toString();\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/kerberos/JuiceFSTokenRenewer.java", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.kerberos;\n\nimport io.juicefs.JuiceFileSystem;\nimport io.juicefs.JuiceFileSystemImpl;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.FilterFileSystem;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.security.token.Token;\nimport org.apache.hadoop.security.token.TokenRenewer;\n\nimport java.io.IOException;\nimport java.net.URI;\n\npublic class JuiceFSTokenRenewer extends TokenRenewer {\n\n @Override\n public boolean handleKind(Text kind) {\n return JuiceFSDelegationTokenIdentifier.TOKEN_KIND.equals(kind);\n }\n\n @Override\n public boolean isManaged(Token token) throws IOException {\n return true;\n }\n\n @Override\n public long renew(Token token, Configuration configuration) throws IOException, InterruptedException {\n String service = token.getService().toString();\n FileSystem fs = FileSystem.get(URI.create(service), configuration);\n if (fs instanceof JuiceFileSystem) {\n return ((JuiceFileSystemImpl) ((FilterFileSystem) fs).getRawFileSystem()).renewToken(token);\n }\n throw new IOException(\"renew token failed\");\n }\n\n @Override\n public void cancel(Token token, Configuration configuration) throws IOException, InterruptedException {\n String service = token.getService().toString();\n FileSystem fs = FileSystem.get(URI.create(service), configuration);\n if (fs instanceof JuiceFileSystem) {\n ((JuiceFileSystemImpl) ((FilterFileSystem) fs).getRawFileSystem()).cancelToken(token);\n return;\n }\n throw new IOException(\"cancel token failed\");\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/kerberos/KerberosUtil.java", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.kerberos;\n\nimport org.apache.hadoop.security.UserGroupInformation;\nimport org.ietf.jgss.GSSContext;\nimport org.ietf.jgss.GSSManager;\nimport org.ietf.jgss.GSSName;\n\nimport java.io.IOException;\nimport java.security.PrivilegedExceptionAction;\n\npublic class KerberosUtil {\n public static byte[] genApReq(String spn) throws IOException, InterruptedException {\n UserGroupInformation loginUser = UserGroupInformation.getLoginUser();\n if (UserGroupInformation.isLoginKeytabBased()) {\n loginUser.checkTGTAndReloginFromKeytab();\n } else if (UserGroupInformation.isLoginTicketBased()) {\n loginUser.reloginFromTicketCache();\n }\n return loginUser.doAs((PrivilegedExceptionAction) () -> {\n GSSManager manager = GSSManager.getInstance();\n GSSName serverName = manager.createName(spn, GSSName.NT_USER_NAME, org.apache.hadoop.security.authentication.util.KerberosUtil.GSS_KRB5_MECH_OID);\n GSSContext context = manager.createContext(serverName, org.apache.hadoop.security.authentication.util.KerberosUtil.GSS_KRB5_MECH_OID, null, GSSContext.DEFAULT_LIFETIME);\n byte[] token = new byte[0];\n return context.initSecContext(token, 0, token.length);\n });\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/metrics/JuiceFSInstrumentation.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.metrics;\n\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.metrics2.MetricsSystem;\nimport org.apache.hadoop.metrics2.annotation.Metric;\nimport org.apache.hadoop.metrics2.annotation.Metrics;\nimport org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;\n\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.Map;\n\n\n@Metrics(context = \"JuiceFileSystem\", name = \"client\")\npublic final class JuiceFSInstrumentation {\n private static MetricsSystem system;\n private static final String METRIC_NAME = \"JuiceFSMetrics\";\n\n private static int numFileSystems;\n\n private final Map valueState = new HashMap<>();\n private final Map timeState = new HashMap<>();\n\n static {\n system = DefaultMetricsSystem.initialize(\"juicefs\");\n }\n\n private final FileSystem fs;\n private final FileSystem.Statistics statistics;\n\n @Metric(\"number of bytes read from JuiceFS\")\n public long getBytesRead() {\n return statistics.getBytesRead();\n }\n\n @Metric(\"number of bytes write to JuiceFS\")\n public double getBytesWrite() {\n return statistics.getBytesWritten();\n }\n\n @Metric(\"write speed\")\n public synchronized double getBytesWritePerSec() {\n return getSpeedPerSec(\"writeSpeed\", statistics.getBytesWritten());\n }\n\n\n @Metric(\"read speed\")\n public synchronized double getBytesReadPerSec() {\n return getSpeedPerSec(\"readSpeed\", statistics.getBytesRead());\n }\n\n @Metric(\"JuiceFS client num\")\n public synchronized int getNumFileSystems() {\n return 1;\n }\n\n @Metric(\"JuiceFS used size\")\n public synchronized long getUsedSize() {\n try {\n return fs.getStatus(new Path(\"/\")).getUsed();\n } catch (IOException e) {\n return 0;\n }\n }\n\n @Metric(\"JuiceFS files\")\n public synchronized long getFiles() {\n try {\n return fs.getContentSummary(new Path(\"/\")).getFileCount();\n } catch (IOException e) {\n return 0;\n }\n }\n\n @Metric(\"JuiceFS dirs\")\n public synchronized long getDirs() {\n try {\n return fs.getContentSummary(new Path(\"/\")).getDirectoryCount();\n } catch (IOException e) {\n return 0;\n }\n }\n\n public double getSpeedPerSec(String name, long currentValue) {\n double speed = 0;\n long current = System.currentTimeMillis();\n long delta = current - timeState.getOrDefault(name, current);\n if (delta > 0) {\n speed = (currentValue - valueState.getOrDefault(name, currentValue)) / (delta / 1000.0);\n }\n valueState.put(name, currentValue);\n timeState.put(name, current);\n return speed;\n }\n\n public static synchronized void init(FileSystem fs, FileSystem.Statistics statistics) {\n if (numFileSystems == 0) {\n DefaultMetricsSystem.instance().register(METRIC_NAME, \"JuiceFS client metrics\",\n new JuiceFSInstrumentation(fs, statistics));\n }\n numFileSystems++;\n }\n\n private JuiceFSInstrumentation(FileSystem fs, FileSystem.Statistics statistics) {\n this.fs = fs;\n this.statistics = statistics;\n }\n\n public static synchronized void close() throws IOException {\n if (numFileSystems == 1) {\n system.publishMetricsNow();\n system.unregisterSource(METRIC_NAME);\n }\n numFileSystems--;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerAdminRefresher.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.permission;\n\nimport com.google.gson.Gson;\nimport com.google.gson.GsonBuilder;\nimport org.apache.hadoop.fs.*;\nimport org.apache.ranger.admin.client.RangerAdminClient;\nimport org.apache.ranger.plugin.contextenricher.RangerTagEnricher;\nimport org.apache.ranger.plugin.service.RangerBasePlugin;\nimport org.apache.ranger.plugin.util.RangerRoles;\nimport org.apache.ranger.plugin.util.RangerServiceNotFoundException;\nimport org.apache.ranger.plugin.util.ServicePolicies;\nimport org.apache.ranger.plugin.util.ServiceTags;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.net.URI;\nimport java.util.Arrays;\nimport java.util.Comparator;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\n\npublic class RangerAdminRefresher {\n private static final Logger LOG = LoggerFactory.getLogger(RangerAdminRefresher.class);\n\n private static final String JFS_RANGER_DIR = \"/.sys/ranger\";\n\n private RangerBasePlugin plugIn;\n private Path rangerDir;\n private Path rangerRulePath;\n private long lastMtime;\n private final long pollingIntervalMs;\n\n private final RangerAdminClient rangerAdmin;\n private final Gson gson = new GsonBuilder().setDateFormat(\"yyyyMMdd-HH:mm:ss.SSS-Z\").create();\n private long lastKnownPolicyVersion = -1L;\n private long lastPolicyActivationTimeInMillis;\n private long lastKnownRoleVersion = -1L;\n private long lastRoleActivationTimeInMillis;\n private long lastKnownTagVersion = -1L;\n private long lastTagActivationTimeInMillis;\n\n private final FileSystem fs;\n private final ScheduledExecutorService refreshThread;\n\n public RangerAdminRefresher(RangerBasePlugin plugIn, RangerAdminClient rangerAdmin, FileSystem fs, String rangerUrl, long pollingIntervalMs) {\n\n this.plugIn = plugIn;\n this.rangerAdmin = rangerAdmin;\n this.fs = fs;\n String serviceName = plugIn.getServiceName();\n URI uri = URI.create(rangerUrl);\n String rangerDirName = uri.getHost().replace(\".\", \"_\") + \"_\" + uri.getPort() + \"_\" + serviceName;\n this.rangerDir = new Path(JFS_RANGER_DIR, rangerDirName);\n this.rangerRulePath = new Path(rangerDir, \"rules\");\n this.refreshThread = Executors.newScheduledThreadPool(1, r -> {\n Thread t = new Thread(r, \"JuiceFS Ranger Refresher\");\n t.setDaemon(true);\n return t;\n });\n this.pollingIntervalMs = pollingIntervalMs;\n }\n\n public void start() {\n loadRangerItem();\n refreshThread.scheduleAtFixedRate(this::loadRangerItem, pollingIntervalMs, pollingIntervalMs, TimeUnit.MILLISECONDS);\n }\n\n /**\n * 1. read rules from jfs\n * 2. choose one client to check ranger admin, if updated, download and save rules to jfs\n */\n public void loadRangerItem() {\n RangerRules rangerRules = null;\n // try to load rules from jfs\n try {\n rangerRules = loadRangerRules();\n } catch (IOException e) {\n LOG.debug(\"Load ranger rules failed\", e);\n }\n\n if (rangerRules != null) {\n if (updateRules(rangerRules.getPolicies(), rangerRules.getTags(), rangerRules.getRoles())) {\n LOG.info(\"Ranger rules has been updated, use new rules from juicefs\");\n }\n }\n\n boolean checkUpdate = checkUpdate(pollingIntervalMs);\n // load rules from ranger admin\n if (rangerRules == null || checkUpdate) {\n ServicePolicies policiesFromRanger = null;\n ServiceTags tagsFromRanger = null;\n RangerRoles rolesFromRanger = null;\n try {\n policiesFromRanger = rangerAdmin.getServicePoliciesIfUpdated(lastKnownPolicyVersion, lastPolicyActivationTimeInMillis);\n tagsFromRanger = rangerAdmin.getServiceTagsIfUpdated(lastKnownTagVersion, lastTagActivationTimeInMillis);\n rolesFromRanger = rangerAdmin.getRolesIfUpdated(lastKnownRoleVersion, lastRoleActivationTimeInMillis);\n } catch (RangerServiceNotFoundException e) {\n LOG.warn(\"Ranger service not found\", e);\n } catch (Exception e) {\n LOG.warn(\"Load policies from ranger failed\", e);\n }\n if (updateRules(policiesFromRanger, tagsFromRanger, rolesFromRanger)) {\n if (checkUpdate) {\n try {\n ServicePolicies p = rangerRules != null ? rangerRules.getPolicies() : null;\n ServiceTags t = rangerRules != null ? rangerRules.getTags() : null;\n RangerRoles r = rangerRules != null ? rangerRules.getRoles() : null;\n if (policiesFromRanger != null) {\n LOG.info(\"ServicePolicies updated from Ranger Admin\");\n p = policiesFromRanger;\n }\n if (tagsFromRanger != null) {\n LOG.info(\"ServiceTags updated from Ranger Admin\");\n t = tagsFromRanger;\n }\n if (rolesFromRanger != null) {\n LOG.info(\"RangerRoles updated from Ranger Admin\");\n r = rolesFromRanger;\n }\n saveRangerRules(new RangerRules(p, t, r));\n } catch (IOException e) {\n LOG.warn(\"Save rules to juicefs failed\", e);\n }\n }\n }\n }\n }\n\n private boolean checkUpdate(long pollingIntervalMs) {\n try {\n boolean exists = fs.exists(rangerDir);\n if (!exists) {\n fs.mkdirs(rangerDir);\n }\n FileStatus[] lockFiles = fs.listStatus(rangerDir, path -> {\n String name = path.getName();\n return name.endsWith(\".lock\");\n });\n String prefix = String.valueOf((System.currentTimeMillis() / pollingIntervalMs) * pollingIntervalMs);\n Path lockPath = new Path(rangerDir, prefix + \".lock\");\n if (lockFiles == null || lockFiles.length == 0) {\n try (FSDataOutputStream ignore = fs.create(lockPath, false)) {\n return true;\n }\n } else {\n if (lockFiles.length > 1) {\n Arrays.sort(lockFiles, Comparator.comparing(o -> o.getPath().getName()));\n }\n if (lockFiles[lockFiles.length - 1].getPath().getName().compareTo(lockPath.getName()) >= 0) {\n return false;\n }\n try (FSDataOutputStream ignore = fs.create(lockPath, false)) {\n for (FileStatus lockFile : lockFiles) {\n fs.delete(lockFile.getPath(), false);\n }\n return true;\n }\n }\n } catch (FileAlreadyExistsException ignored) {\n return false;\n }\n catch (IOException e) {\n LOG.warn(\"Check update failed\", e);\n return false;\n }\n }\n\n private void saveRangerRules(RangerRules rules) throws IOException {\n String rulesJson = gson.toJson(rules, RangerRules.class);\n byte[] bytes = rulesJson.getBytes();\n try (FSDataOutputStream out = fs.create(rangerRulePath)) {\n out.write(bytes);\n } catch (FileNotFoundException e) {\n fs.mkdirs(rangerRulePath.getParent());\n try (FSDataOutputStream out = fs.create(rangerRulePath)) {\n out.write(bytes);\n }\n }\n }\n\n private RangerRules loadRangerRules() throws IOException {\n FileStatus fileStatus = fs.getFileStatus(rangerRulePath);\n long mtime = fileStatus.getModificationTime();\n if (lastMtime == mtime) {\n return null;\n }\n try (FSDataInputStream in = fs.open(rangerRulePath)) {\n byte[] bytes = new byte[(int) fileStatus.getLen()];\n in.readFully(bytes);\n String rulesJson = new String(bytes);\n RangerRules rangerRules = gson.fromJson(rulesJson, RangerRules.class);\n lastMtime = mtime;\n return rangerRules;\n }\n }\n\n private boolean updateRules(ServicePolicies newSvcPolicies, ServiceTags newTags, RangerRoles newRangerRoles) {\n boolean updated = false;\n if (newSvcPolicies != null) {\n long policyVersion = newSvcPolicies.getPolicyVersion() == null ? -1 : newSvcPolicies.getPolicyVersion();\n if (lastKnownPolicyVersion != policyVersion) {\n plugIn.setPolicies(newSvcPolicies);\n lastKnownPolicyVersion = policyVersion;\n lastPolicyActivationTimeInMillis = System.currentTimeMillis();\n updated = true;\n }\n }\n if (newTags != null) {\n long tagVersion = newTags.getTagVersion() == null ? -1 : newTags.getTagVersion();\n if (lastKnownTagVersion != tagVersion) {\n RangerTagEnricher tagEnricher = plugIn.getTagEnricher();\n if (tagEnricher != null) {\n tagEnricher.setServiceTags(newTags);\n }\n lastKnownTagVersion = tagVersion;\n lastTagActivationTimeInMillis = System.currentTimeMillis();\n updated = true;\n }\n }\n if (newRangerRoles != null) {\n long roleVersion = newRangerRoles.getRoleVersion() == null ? -1 : newRangerRoles.getRoleVersion();\n if (lastKnownRoleVersion != roleVersion) {\n plugIn.setRoles(newRangerRoles);\n lastKnownRoleVersion = roleVersion;\n lastRoleActivationTimeInMillis = System.currentTimeMillis();\n updated = true;\n }\n }\n return updated;\n }\n\n public void stop() {\n refreshThread.shutdownNow();\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerConfig.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.permission;\n\npublic class RangerConfig {\n\n public RangerConfig(String rangerRestUrl, String serviceName, long pollIntervalMs) {\n this.rangerRestUrl = rangerRestUrl;\n this.serviceName = serviceName;\n this.pollIntervalMs = pollIntervalMs;\n }\n\n private String rangerRestUrl;\n\n private String serviceName;\n\n private long pollIntervalMs;\n\n private String impl;\n\n public String getRangerRestUrl() {\n return rangerRestUrl;\n }\n\n public void setRangerRestUrl(String rangerRestUrl) {\n this.rangerRestUrl = rangerRestUrl;\n }\n\n public String getServiceName() {\n return serviceName;\n }\n\n public void setServiceName(String serviceName) {\n this.serviceName = serviceName;\n }\n\n public long getPollIntervalMs() {\n return pollIntervalMs;\n }\n\n public void setPollIntervalMs(long pollIntervalMs) {\n this.pollIntervalMs = pollIntervalMs;\n }\n\n public void setImpl(String impl) {\n this.impl = impl;\n }\n\n public String getImpl() {\n return impl;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerJfsAccessRequest.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.permission;\n\nimport org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;\n\nimport java.util.Date;\nimport java.util.Set;\n\nclass RangerJfsAccessRequest extends RangerAccessRequestImpl {\n\n RangerJfsAccessRequest(String path, String pathOwner, String accessType, String action, String user,\n Set groups) {\n setResource(new RangerJfsResource(path, pathOwner));\n setAccessType(accessType);\n setUser(user);\n setUserGroups(groups);\n setAccessTime(new Date());\n setAction(action);\n setForwardedAddresses(null);\n }\n\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerJfsPlugin.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.permission;\n\nimport io.juicefs.utils.ReflectionUtil;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.ranger.admin.client.RangerAdminClient;\nimport org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;\nimport org.apache.ranger.plugin.service.RangerBasePlugin;\nimport org.apache.ranger.plugin.service.RangerChainedPlugin;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.util.List;\n\npublic class RangerJfsPlugin extends RangerBasePlugin {\n private static final Logger LOG = LoggerFactory.getLogger(RangerJfsPlugin.class);\n\n private FileSystem fs;\n private String rangerUrl;\n private RangerAdminRefresher refresher;\n private long pollingIntervalMs;\n\n public RangerJfsPlugin(FileSystem fs, String serviceName, String rangerUrl, long pollingIntervalMs) {\n super(new RangerPluginCfg(\"hdfs\", serviceName, \"jfs\", null, null, null));\n this.fs = fs;\n this.rangerUrl = rangerUrl;\n RangerPluginConfig config = getConfig();\n config.addResource(fs.getConf());\n this.pollingIntervalMs = pollingIntervalMs;\n }\n\n @Override\n public void init() {\n cleanup();\n RangerAdminClient admin = createAdminClient(getConfig());\n refresher = new RangerAdminRefresher(this, admin, fs, rangerUrl, pollingIntervalMs);\n refresher.start();\n List chainedPlugins = null;\n try {\n chainedPlugins = (List) ReflectionUtil.getField(RangerBasePlugin.class.getName(), \"chainedPlugins\", this);\n } catch (Exception e) {\n LOG.warn(\"Get field \\\"chainedPlugins\\\" failed\", e);\n }\n if (chainedPlugins != null) {\n for (RangerChainedPlugin plugin : chainedPlugins) {\n plugin.init();\n }\n }\n }\n\n @Override\n public void cleanup() {\n super.cleanup();\n if (refresher != null) {\n refresher.stop();\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerJfsResource.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.permission;\n\nimport org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;\n\nclass RangerJfsResource extends RangerAccessResourceImpl {\n RangerJfsResource(String path, String owner) {\n setValue(\"path\", path);\n setOwnerUser(owner);\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerPermissionChecker.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.permission;\n\nimport com.google.common.collect.Sets;\nimport org.apache.commons.lang.StringUtils;\nimport org.apache.hadoop.fs.FileStatus;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.fs.permission.FsAction;\nimport org.apache.hadoop.security.AccessControlException;\nimport org.apache.ranger.plugin.policyengine.RangerAccessResult;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\n\n/**\n * for auth checker\n *\n * @author ming.li2\n **/\npublic class RangerPermissionChecker {\n\n private static final Logger LOG = LoggerFactory.getLogger(RangerPermissionChecker.class);\n\n private static final Map pcs = new ConcurrentHashMap<>();\n private static final Map> runningInstance = new HashMap<>();\n\n private final HashMap> fsAction2ActionMapper = new HashMap>() {\n {\n put(FsAction.NONE, new HashSet<>());\n put(FsAction.ALL, Sets.newHashSet(\"read\", \"write\", \"execute\"));\n put(FsAction.READ, Sets.newHashSet(\"read\"));\n put(FsAction.READ_WRITE, Sets.newHashSet(\"read\", \"write\"));\n put(FsAction.READ_EXECUTE, Sets.newHashSet(\"read\", \"execute\"));\n put(FsAction.WRITE, Sets.newHashSet(\"write\"));\n put(FsAction.WRITE_EXECUTE, Sets.newHashSet(\"write\", \"execute\"));\n put(FsAction.EXECUTE, Sets.newHashSet(\"execute\"));\n }\n };\n\n private final FileSystem superGroupFileSystem;\n private final RangerJfsPlugin rangerPlugin;\n\n public RangerPermissionChecker(FileSystem superGroupFileSystem, RangerConfig config) {\n this.superGroupFileSystem = superGroupFileSystem;\n rangerPlugin = new RangerJfsPlugin(superGroupFileSystem, config.getServiceName(), config.getRangerRestUrl(), config.getPollIntervalMs());\n rangerPlugin.getConfig().set(\"ranger.plugin.hdfs.service.name\", config.getServiceName());\n rangerPlugin.getConfig().set(\"ranger.plugin.hdfs.policy.rest.url\", config.getRangerRestUrl());\n // for test use\n if (config.getImpl() != null) {\n rangerPlugin.getConfig().set(\"ranger.plugin.hdfs.policy.source.impl\", config.getImpl());\n }\n rangerPlugin.getConfig().setIsFallbackSupported(true);\n rangerPlugin.init();\n }\n\n public static RangerPermissionChecker acquire(String volName, long handle, FileSystem superGroupFileSystem, RangerConfig config) throws IOException {\n synchronized (runningInstance) {\n if (!runningInstance.containsKey(volName)) {\n if (pcs.containsKey(volName)) {\n throw new IOException(\"RangerPermissionChecker for volume: \" + volName + \" is already created, but no running instance found.\");\n }\n RangerPermissionChecker pc = new RangerPermissionChecker(superGroupFileSystem, config);\n pcs.put(volName, pc);\n Set handles = new HashSet<>();\n handles.add(handle);\n runningInstance.put(volName, handles);\n return pc;\n } else {\n RangerPermissionChecker pc = pcs.get(volName);\n if (pc == null) {\n throw new IOException(\"RangerPermissionChecker for volume: \" + volName + \" is already created, but no instance found.\");\n }\n runningInstance.get(volName).add(handle);\n return pc;\n }\n }\n }\n\n public static void release(String volName, long handle) {\n if (handle <= 0) {\n return;\n }\n synchronized (runningInstance) {\n if (!runningInstance.containsKey(volName)) {\n return;\n }\n Set handles = runningInstance.get(volName);\n boolean removed = handles.remove(handle);\n if (!removed) {\n return;\n }\n if (handles.size() == 0) {\n RangerPermissionChecker pc = pcs.remove(volName);\n pc.cleanUp();\n runningInstance.remove(volName);\n }\n }\n }\n\n public boolean checkPermission(Path path, boolean checkOwner, FsAction ancestorAccess, FsAction parentAccess,\n FsAction access, String operationName, String user, Set groups) throws IOException {\n RangerPermissionContext context = new RangerPermissionContext(user, groups, operationName);\n PathObj obj = path2Obj(path);\n\n boolean fallback = true;\n AuthzStatus authzStatus = AuthzStatus.ALLOW;\n\n if (access != null && parentAccess != null\n && parentAccess.implies(FsAction.WRITE) && obj.parent != null && obj.current != null && obj.parent.getPermission().getStickyBit()) {\n if (!StringUtils.equals(obj.parent.getOwner(), user) && !StringUtils.equals(obj.current.getOwner(), user)) {\n authzStatus = AuthzStatus.NOT_DETERMINED;\n }\n }\n\n if (authzStatus == AuthzStatus.ALLOW && ancestorAccess != null && obj.ancestor != null) {\n authzStatus = isAccessAllowed(obj.ancestor, ancestorAccess, context);\n if (checkResult(authzStatus, user, ancestorAccess.toString(), toPathString(obj.ancestor.getPath()))) {\n return fallback;\n }\n }\n\n if (authzStatus == AuthzStatus.ALLOW && parentAccess != null && obj.parent != null) {\n authzStatus = isAccessAllowed(obj.parent, parentAccess, context);\n if (checkResult(authzStatus, user, parentAccess.toString(), toPathString(obj.parent.getPath()))) {\n return fallback;\n }\n }\n\n if (authzStatus == AuthzStatus.ALLOW && access != null && obj.current != null) {\n authzStatus = isAccessAllowed(obj.current, access, context);\n if (checkResult(authzStatus, user, access.toString(), toPathString(obj.current.getPath()))) {\n return fallback;\n }\n }\n\n if (checkOwner) {\n String owner = null;\n if (obj.current != null) {\n owner = obj.current.getOwner();\n }\n if (!user.equals(owner)) {\n throw new AccessControlException(\n assembleExceptionMessage(user, getFirstNonNullAccess(ancestorAccess, parentAccess, access),\n toPathString(obj.current.getPath())));\n }\n }\n // check access by ranger success\n return !fallback;\n }\n\n public void cleanUp() {\n try {\n rangerPlugin.cleanup();\n } catch (Exception e) {\n LOG.warn(\"Error when clean up ranger plugin threads.\", e);\n }\n try {\n superGroupFileSystem.close();\n } catch (Exception e) {\n LOG.warn(\"Error when close super group file system.\", e);\n }\n }\n\n private static boolean checkResult(AuthzStatus authzStatus, String user, String action, String path) throws AccessControlException {\n if (authzStatus == AuthzStatus.DENY) {\n throw new AccessControlException(assembleExceptionMessage(user, action, path));\n } else {\n return authzStatus == AuthzStatus.NOT_DETERMINED;\n }\n }\n\n private static String assembleExceptionMessage(String user, String action, String path) {\n return \"Permission denied: user=\" + user + \", access=\" + action + \", path=\\\"\" + path + \"\\\"\";\n }\n\n private static String getFirstNonNullAccess(FsAction ancestorAccess, FsAction parentAccess, FsAction access) {\n if (access != null) {\n return access.toString();\n }\n if (parentAccess != null) {\n return parentAccess.toString();\n }\n if (ancestorAccess != null) {\n return ancestorAccess.toString();\n }\n return FsAction.EXECUTE.toString();\n }\n\n private AuthzStatus isAccessAllowed(FileStatus file, FsAction access, RangerPermissionContext context) {\n String path = toPathString(file.getPath());\n Set accessTypes = fsAction2ActionMapper.getOrDefault(access, new HashSet<>());\n String pathOwner = file.getOwner();\n AuthzStatus authzStatus = null;\n for (String accessType : accessTypes) {\n RangerJfsAccessRequest request = new RangerJfsAccessRequest(path, pathOwner, accessType, context.operationName, context.user, context.userGroups);\n LOG.debug(request.toString());\n RangerAccessResult result = rangerPlugin.isAccessAllowed(request);\n if (result != null) {\n LOG.debug(result.toString());\n }\n if (result == null || !result.getIsAccessDetermined()) {\n authzStatus = AuthzStatus.NOT_DETERMINED;\n } else if (!result.getIsAllowed()) {\n authzStatus = AuthzStatus.DENY;\n break;\n } else {\n if (!AuthzStatus.NOT_DETERMINED.equals(authzStatus)) {\n authzStatus = AuthzStatus.ALLOW;\n }\n }\n\n }\n if (authzStatus == null) {\n authzStatus = AuthzStatus.NOT_DETERMINED;\n }\n return authzStatus;\n }\n\n private enum AuthzStatus {ALLOW, DENY, NOT_DETERMINED}\n\n ;\n\n private static String toPathString(Path path) {\n return path.toUri().getPath();\n }\n\n private PathObj path2Obj(Path path) throws IOException {\n\n FileStatus current = getIfExist(path);\n FileStatus parent = getIfExist(path.getParent());\n FileStatus ancestor = getAncestor(path);\n\n return new PathObj(ancestor, parent, current);\n }\n\n private FileStatus getIfExist(Path path) throws IOException {\n try {\n if (path != null) {\n return superGroupFileSystem.getFileStatus(path);\n }\n } catch (FileNotFoundException ignored) {\n }\n return null;\n }\n\n public FileStatus getAncestor(Path path) throws IOException {\n if (path.getParent() != null) {\n return getIfExist(path.getParent());\n }\n path = path.getParent();\n FileStatus tmp = null;\n while (path != null && tmp == null) {\n tmp = getIfExist(path);\n path = path.getParent();\n }\n return tmp;\n }\n\n public static class PathObj {\n\n FileStatus ancestor = null;\n\n FileStatus parent = null;\n\n FileStatus current = null;\n\n public PathObj(FileStatus ancestor, FileStatus parent, FileStatus current) {\n this.ancestor = ancestor;\n this.parent = parent;\n this.current = current;\n }\n }\n\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerPermissionContext.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.permission;\n\nimport java.util.Set;\n\npublic class RangerPermissionContext {\n\n public final String user;\n\n public final Set userGroups;\n\n public final String operationName;\n\n public RangerPermissionContext(String user, Set groups, String operationName) {\n this.user = user;\n this.userGroups = groups;\n this.operationName = operationName;\n }\n\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerPluginCfg.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.permission;\n\nimport org.apache.ranger.authorization.hadoop.config.RangerConfiguration;\nimport org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;\nimport org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.net.MalformedURLException;\nimport java.net.URL;\n\npublic class RangerPluginCfg extends RangerPluginConfig {\n private static final Logger LOG = LoggerFactory.getLogger(RangerPluginCfg.class);\n\n @Override\n public boolean addResourceIfReadable(String aResourceName) {\n URL fUrl = this.getFileLocation(aResourceName);\n if (fUrl != null) {\n try {\n this.addResource(fUrl);\n } catch (Exception e) {\n LOG.error(\"Unable to load the resource name [\" + aResourceName + \"]. Ignoring the resource:\" + fUrl);\n }\n }\n return true;\n }\n\n public static boolean isEmpty(String str) {\n return str == null || str.length() == 0;\n }\n\n private URL getFileLocation(String fileName) {\n URL lurl = null;\n if (!isEmpty(fileName)) {\n lurl = RangerConfiguration.class.getClassLoader().getResource(fileName);\n\n if (lurl == null ) {\n lurl = RangerConfiguration.class.getClassLoader().getResource(\"/\" + fileName);\n }\n\n if (lurl == null ) {\n File f = new File(fileName);\n if (f.exists()) {\n try {\n lurl=f.toURI().toURL();\n } catch (MalformedURLException e) {\n LOG.error(\"Unable to load the resource name [\" + fileName + \"]. Ignoring the resource:\" + f.getPath());\n }\n } else {\n if(LOG.isDebugEnabled()) {\n LOG.debug(\"Conf file path \" + fileName + \" does not exists\");\n }\n }\n }\n }\n return lurl;\n }\n\n public RangerPluginCfg(String serviceType, String serviceName, String appId, String clusterName, String clusterType, RangerPolicyEngineOptions policyEngineOptions) {\n super(serviceType, serviceName, appId, clusterName, clusterType, policyEngineOptions);\n }\n}" }, { "path": "sdk/java/src/main/java/io/juicefs/permission/RangerRules.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.permission;\n\nimport org.apache.ranger.plugin.util.RangerRoles;\nimport org.apache.ranger.plugin.util.ServicePolicies;\nimport org.apache.ranger.plugin.util.ServiceTags;\n\nimport java.io.Serializable;\n\npublic class RangerRules implements Serializable {\n private ServicePolicies policies;\n private ServiceTags tags;\n private RangerRoles roles;\n\n public RangerRules() {\n }\n\n public RangerRules(ServicePolicies policies, ServiceTags tags, RangerRoles roles) {\n this.policies = policies;\n this.tags = tags;\n this.roles = roles;\n }\n\n public ServicePolicies getPolicies() {\n return policies;\n }\n\n public void setPolicies(ServicePolicies policies) {\n this.policies = policies;\n }\n\n public ServiceTags getTags() {\n return tags;\n }\n\n public void setTags(ServiceTags tags) {\n this.tags = tags;\n }\n\n public RangerRoles getRoles() {\n return roles;\n }\n\n public void setRoles(RangerRoles roles) {\n this.roles = roles;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/tools/RangerDownloader.java", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.tools;\n\nimport com.beust.jcommander.Parameter;\nimport com.beust.jcommander.Parameters;\nimport io.juicefs.JuiceFileSystemImpl;\nimport io.juicefs.Main;\nimport io.juicefs.permission.RangerPermissionChecker;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.security.UserGroupInformation;\n\nimport java.io.IOException;\nimport java.net.URI;\n\n@Parameters(commandDescription = \"Download policies from ranger and save to JuiceFS\")\npublic class RangerDownloader extends Main.Command {\n\n @Parameter(names = {\"--fs\"}, description = \"JuiceFileSystem: jfs://{JFS_VOL_NAME}\", required = true)\n private String fs;\n\n @Parameter(names = {\"--keytab\"}, description = \"local keytab file location\")\n private String keytab;\n\n @Parameter(names = {\"--principal\"}, description = \"principal allowed access ranger admin\")\n private String principal;\n\n @Override\n public void init() throws IOException {\n\n }\n\n @Override\n public void run() throws IOException {\n UserGroupInformation ugi = UserGroupInformation.getCurrentUser();\n if (!ugi.hasKerberosCredentials() && (keytab == null || principal == null)) {\n throw new IllegalArgumentException(\"No kerberos credential was found! Parameter \\\"--keytab\\\" and \\\"--principal\\\" must be provided.\");\n }\n if (keytab != null) {\n UserGroupInformation.loginUserFromKeytab(principal, keytab);\n }\n Configuration cfg = new Configuration();\n JuiceFileSystemImpl jfs = new JuiceFileSystemImpl(true);\n jfs.initialize(URI.create(fs), cfg);\n new RangerPermissionChecker(jfs, jfs.checkAndGetRangerParams(cfg));\n }\n\n @Override\n public void close() throws IOException {\n\n }\n\n @Override\n public String getCommand() {\n return \"ranger\";\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/AclTransformation.java", "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *

\n * http://www.apache.org/licenses/LICENSE-2.0\n *

\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.utils;\n\nimport com.google.common.collect.ComparisonChain;\nimport com.google.common.collect.Lists;\nimport com.google.common.collect.Maps;\nimport com.google.common.collect.Ordering;\nimport org.apache.hadoop.fs.permission.*;\n\nimport java.io.IOException;\nimport java.util.*;\n\nimport static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;\nimport static org.apache.hadoop.fs.permission.AclEntryScope.DEFAULT;\nimport static org.apache.hadoop.fs.permission.AclEntryType.*;\n\n/**\n * AclTransformation defines the operations that can modify an ACL. All ACL\n * modifications take as input an existing ACL and apply logic to add new\n * entries, modify existing entries or remove old entries. Some operations also\n * accept an ACL spec: a list of entries that further describes the requested\n * change. Different operations interpret the ACL spec differently. In the\n * case of adding an ACL to an inode that previously did not have one, the\n * existing ACL can be a \"minimal ACL\" containing exactly 3 entries for owner,\n * group and other, all derived from the {@link FsPermission} bits.\n *

\n * The algorithms implemented here require sorted lists of ACL entries. For any\n * existing ACL, it is assumed that the entries are sorted. This is because all\n * ACL creation and modification is intended to go through these methods, and\n * they all guarantee correct sort order in their outputs. However, an ACL spec\n * is considered untrusted user input, so all operations pre-sort the ACL spec as\n * the first step.\n */\npublic final class AclTransformation {\n private static final int MAX_ENTRIES = 32;\n\n public static List filterAclEntriesByAclSpec(List existingAcl, List inAclSpec) throws AclException {\n ValidatedAclSpec aclSpec = new ValidatedAclSpec(inAclSpec);\n ArrayList aclBuilder = Lists.newArrayListWithCapacity(MAX_ENTRIES);\n EnumMap providedMask = Maps.newEnumMap(AclEntryScope.class);\n EnumSet maskDirty = EnumSet.noneOf(AclEntryScope.class);\n EnumSet scopeDirty = EnumSet.noneOf(AclEntryScope.class);\n for (AclEntry existingEntry : existingAcl) {\n if (aclSpec.containsKey(existingEntry)) {\n scopeDirty.add(existingEntry.getScope());\n if (existingEntry.getType() == MASK) {\n maskDirty.add(existingEntry.getScope());\n }\n } else {\n if (existingEntry.getType() == MASK) {\n providedMask.put(existingEntry.getScope(), existingEntry);\n } else {\n aclBuilder.add(existingEntry);\n }\n }\n }\n copyDefaultsIfNeeded(aclBuilder);\n calculateMasks(aclBuilder, providedMask, maskDirty, scopeDirty);\n return buildAndValidateAcl(aclBuilder);\n }\n\n public static List mergeAclEntries(List existingAcl, List inAclSpec) throws AclException {\n ValidatedAclSpec aclSpec = new ValidatedAclSpec(inAclSpec);\n ArrayList aclBuilder = Lists.newArrayListWithCapacity(MAX_ENTRIES);\n List foundAclSpecEntries = Lists.newArrayListWithCapacity(MAX_ENTRIES);\n EnumMap providedMask = Maps.newEnumMap(AclEntryScope.class);\n EnumSet maskDirty = EnumSet.noneOf(AclEntryScope.class);\n EnumSet scopeDirty = EnumSet.noneOf(AclEntryScope.class);\n for (AclEntry existingEntry : existingAcl) {\n AclEntry aclSpecEntry = aclSpec.findByKey(existingEntry);\n if (aclSpecEntry != null) {\n foundAclSpecEntries.add(aclSpecEntry);\n scopeDirty.add(aclSpecEntry.getScope());\n if (aclSpecEntry.getType() == MASK) {\n providedMask.put(aclSpecEntry.getScope(), aclSpecEntry);\n maskDirty.add(aclSpecEntry.getScope());\n } else {\n aclBuilder.add(aclSpecEntry);\n }\n } else {\n if (existingEntry.getType() == MASK) {\n providedMask.put(existingEntry.getScope(), existingEntry);\n } else {\n aclBuilder.add(existingEntry);\n }\n }\n }\n // ACL spec entries that were not replacements are new additions.\n for (AclEntry newEntry : aclSpec) {\n if (Collections.binarySearch(foundAclSpecEntries, newEntry, ACL_ENTRY_COMPARATOR) < 0) {\n scopeDirty.add(newEntry.getScope());\n if (newEntry.getType() == MASK) {\n providedMask.put(newEntry.getScope(), newEntry);\n maskDirty.add(newEntry.getScope());\n } else {\n aclBuilder.add(newEntry);\n }\n }\n }\n copyDefaultsIfNeeded(aclBuilder);\n calculateMasks(aclBuilder, providedMask, maskDirty, scopeDirty);\n return buildAndValidateAcl(aclBuilder);\n }\n\n public static List replaceAclEntries(List existingAcl, List inAclSpec) throws AclException {\n ValidatedAclSpec aclSpec = new ValidatedAclSpec(inAclSpec);\n ArrayList aclBuilder = Lists.newArrayListWithCapacity(MAX_ENTRIES);\n // Replacement is done separately for each scope: access and default.\n EnumMap providedMask = Maps.newEnumMap(AclEntryScope.class);\n EnumSet maskDirty = EnumSet.noneOf(AclEntryScope.class);\n EnumSet scopeDirty = EnumSet.noneOf(AclEntryScope.class);\n for (AclEntry aclSpecEntry : aclSpec) {\n scopeDirty.add(aclSpecEntry.getScope());\n if (aclSpecEntry.getType() == MASK) {\n providedMask.put(aclSpecEntry.getScope(), aclSpecEntry);\n maskDirty.add(aclSpecEntry.getScope());\n } else {\n aclBuilder.add(aclSpecEntry);\n }\n }\n // Copy existing entries if the scope was not replaced.\n for (AclEntry existingEntry : existingAcl) {\n if (!scopeDirty.contains(existingEntry.getScope())) {\n if (existingEntry.getType() == MASK) {\n providedMask.put(existingEntry.getScope(), existingEntry);\n } else {\n aclBuilder.add(existingEntry);\n }\n }\n }\n copyDefaultsIfNeeded(aclBuilder);\n calculateMasks(aclBuilder, providedMask, maskDirty, scopeDirty);\n return buildAndValidateAcl(aclBuilder);\n }\n\n private AclTransformation() {\n }\n\n public static final Comparator ACL_ENTRY_COMPARATOR = new Comparator() {\n @Override\n public int compare(AclEntry entry1, AclEntry entry2) {\n return ComparisonChain.start().compare(entry1.getScope(), entry2.getScope(), Ordering.explicit(ACCESS, DEFAULT)).compare(entry1.getType(), entry2.getType(), Ordering.explicit(USER, GROUP, MASK, OTHER)).compare(entry1.getName(), entry2.getName(), Ordering.natural().nullsFirst()).result();\n }\n };\n\n public static List buildAndValidateAcl(ArrayList aclBuilder) throws AclException {\n aclBuilder.trimToSize();\n Collections.sort(aclBuilder, ACL_ENTRY_COMPARATOR);\n // Full iteration to check for duplicates and invalid named entries.\n AclEntry prevEntry = null;\n for (AclEntry entry : aclBuilder) {\n if (prevEntry != null && ACL_ENTRY_COMPARATOR.compare(prevEntry, entry) == 0) {\n throw new AclException(\"Invalid ACL: multiple entries with same scope, type and name.\");\n }\n if (entry.getName() != null && (entry.getType() == MASK || entry.getType() == OTHER)) {\n throw new AclException(\"Invalid ACL: this entry type must not have a name: \" + entry + \".\");\n }\n prevEntry = entry;\n }\n\n ScopedAclEntries scopedEntries = new ScopedAclEntries(aclBuilder);\n checkMaxEntries(scopedEntries);\n\n // Search for the required base access entries. If there is a default ACL,\n // then do the same check on the default entries.\n for (AclEntryType type : EnumSet.of(USER, GROUP, OTHER)) {\n AclEntry accessEntryKey = new AclEntry.Builder().setScope(ACCESS).setType(type).build();\n if (Collections.binarySearch(scopedEntries.getAccessEntries(), accessEntryKey, ACL_ENTRY_COMPARATOR) < 0) {\n throw new AclException(\"Invalid ACL: the user, group and other entries are required.\");\n }\n if (!scopedEntries.getDefaultEntries().isEmpty()) {\n AclEntry defaultEntryKey = new AclEntry.Builder().setScope(DEFAULT).setType(type).build();\n if (Collections.binarySearch(scopedEntries.getDefaultEntries(), defaultEntryKey, ACL_ENTRY_COMPARATOR) < 0) {\n throw new AclException(\"Invalid default ACL: the user, group and other entries are required.\");\n }\n }\n }\n return Collections.unmodifiableList(aclBuilder);\n }\n\n private static void checkMaxEntries(ScopedAclEntries scopedEntries) throws AclException {\n List accessEntries = scopedEntries.getAccessEntries();\n List defaultEntries = scopedEntries.getDefaultEntries();\n if (accessEntries.size() > MAX_ENTRIES) {\n throw new AclException(\"Invalid ACL: ACL has \" + accessEntries.size() + \" access entries, which exceeds maximum of \" + MAX_ENTRIES + \".\");\n }\n if (defaultEntries.size() > MAX_ENTRIES) {\n throw new AclException(\"Invalid ACL: ACL has \" + defaultEntries.size() + \" default entries, which exceeds maximum of \" + MAX_ENTRIES + \".\");\n }\n }\n\n private static void calculateMasks(List aclBuilder, EnumMap providedMask, EnumSet maskDirty, EnumSet scopeDirty) throws AclException {\n EnumSet scopeFound = EnumSet.noneOf(AclEntryScope.class);\n EnumMap unionPerms = Maps.newEnumMap(AclEntryScope.class);\n EnumSet maskNeeded = EnumSet.noneOf(AclEntryScope.class);\n // Determine which scopes are present, which scopes need a mask, and the\n // union of group class permissions in each scope.\n for (AclEntry entry : aclBuilder) {\n scopeFound.add(entry.getScope());\n if (entry.getType() == GROUP || entry.getName() != null) {\n FsAction scopeUnionPerms = unionPerms.get(entry.getScope());\n if (scopeUnionPerms == null) {\n scopeUnionPerms = FsAction.NONE;\n }\n unionPerms.put(entry.getScope(), scopeUnionPerms.or(entry.getPermission()));\n }\n if (entry.getName() != null) {\n maskNeeded.add(entry.getScope());\n }\n }\n // Add mask entry if needed in each scope.\n for (AclEntryScope scope : scopeFound) {\n if (!providedMask.containsKey(scope) && maskNeeded.contains(scope) && maskDirty.contains(scope)) {\n // Caller explicitly removed mask entry, but it's required.\n throw new AclException(\"Invalid ACL: mask is required and cannot be deleted.\");\n } else if (providedMask.containsKey(scope) && (!scopeDirty.contains(scope) || maskDirty.contains(scope))) {\n // Caller explicitly provided new mask, or we are preserving the existing\n // mask in an unchanged scope.\n aclBuilder.add(providedMask.get(scope));\n } else if (maskNeeded.contains(scope) || providedMask.containsKey(scope)) {\n // Otherwise, if there are maskable entries present, or the ACL\n // previously had a mask, then recalculate a mask automatically.\n aclBuilder.add(new AclEntry.Builder().setScope(scope).setType(MASK).setPermission(unionPerms.get(scope)).build());\n }\n }\n }\n\n private static void copyDefaultsIfNeeded(List aclBuilder) {\n Collections.sort(aclBuilder, ACL_ENTRY_COMPARATOR);\n ScopedAclEntries scopedEntries = new ScopedAclEntries(aclBuilder);\n if (!scopedEntries.getDefaultEntries().isEmpty()) {\n List accessEntries = scopedEntries.getAccessEntries();\n List defaultEntries = scopedEntries.getDefaultEntries();\n List copiedEntries = Lists.newArrayListWithCapacity(3);\n for (AclEntryType type : EnumSet.of(USER, GROUP, OTHER)) {\n AclEntry defaultEntryKey = new AclEntry.Builder().setScope(DEFAULT).setType(type).build();\n int defaultEntryIndex = Collections.binarySearch(defaultEntries, defaultEntryKey, ACL_ENTRY_COMPARATOR);\n if (defaultEntryIndex < 0) {\n AclEntry accessEntryKey = new AclEntry.Builder().setScope(ACCESS).setType(type).build();\n int accessEntryIndex = Collections.binarySearch(accessEntries, accessEntryKey, ACL_ENTRY_COMPARATOR);\n if (accessEntryIndex >= 0) {\n copiedEntries.add(new AclEntry.Builder().setScope(DEFAULT).setType(type).setPermission(accessEntries.get(accessEntryIndex).getPermission()).build());\n }\n }\n }\n // Add all copied entries when done to prevent potential issues with binary\n // search on a modified aclBulider during the main loop.\n aclBuilder.addAll(copiedEntries);\n }\n }\n\n private static final class ValidatedAclSpec implements Iterable {\n private final List aclSpec;\n\n /**\n * Creates a ValidatedAclSpec by pre-validating and sorting the given ACL\n * entries. Pre-validation checks that it does not exceed the maximum\n * entries. This check is performed before modifying the ACL, and it's\n * actually insufficient for enforcing the maximum number of entries.\n * Transformation logic can create additional entries automatically,such as\n * the mask and some of the default entries, so we also need additional\n * checks during transformation. The up-front check is still valuable here\n * so that we don't run a lot of expensive transformation logic while\n * holding the namesystem lock for an attacker who intentionally sent a huge\n * ACL spec.\n *\n * @param aclSpec List containing unvalidated input ACL spec\n * @throws AclException if validation fails\n */\n public ValidatedAclSpec(List aclSpec) throws AclException {\n Collections.sort(aclSpec, ACL_ENTRY_COMPARATOR);\n checkMaxEntries(new ScopedAclEntries(aclSpec));\n this.aclSpec = aclSpec;\n }\n\n /**\n * Returns true if this contains an entry matching the given key. An ACL\n * entry's key consists of scope, type and name (but not permission).\n *\n * @param key AclEntry search key\n * @return boolean true if found\n */\n public boolean containsKey(AclEntry key) {\n return Collections.binarySearch(aclSpec, key, ACL_ENTRY_COMPARATOR) >= 0;\n }\n\n /**\n * Returns the entry matching the given key or null if not found. An ACL\n * entry's key consists of scope, type and name (but not permission).\n *\n * @param key AclEntry search key\n * @return AclEntry entry matching the given key or null if not found\n */\n public AclEntry findByKey(AclEntry key) {\n int index = Collections.binarySearch(aclSpec, key, ACL_ENTRY_COMPARATOR);\n if (index >= 0) {\n return aclSpec.get(index);\n }\n return null;\n }\n\n @Override\n public Iterator iterator() {\n return aclSpec.iterator();\n }\n }\n\n public static class AclException extends IOException {\n private static final long serialVersionUID = 1L;\n\n /**\n * Creates a new AclException.\n *\n * @param message String message\n */\n public AclException(String message) {\n super(message);\n }\n\n /**\n * Creates a new AclException.\n *\n * @param message String message\n * @param cause The cause of the exception\n */\n public AclException(String message, Throwable cause) {\n super(message, cause);\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/BgTaskUtil.java", "content": "/*\n * JuiceFS, Copyright 2023 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.utils;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\npublic class BgTaskUtil {\n private static final Logger LOG = LoggerFactory.getLogger(BgTaskUtil.class);\n\n private static final Map bgThreadForName = new HashMap<>(); // volName -> threadpool\n private static final Map tasks = new HashMap<>(); // volName|taskName -> running\n private static final Map> runningInstance = new HashMap<>();\n\n public static void reset() {\n bgThreadForName.clear();\n tasks.clear();\n runningInstance.clear();\n }\n\n public static Map getBgThreadForName() {\n return bgThreadForName;\n }\n\n public static Map> getRunningInstance() {\n return runningInstance;\n }\n\n public static void register(String volName, long handle) {\n if (handle <= 0) {\n return;\n }\n synchronized (runningInstance) {\n LOG.debug(\"register instance for {}({})\", volName, handle);\n if (!runningInstance.containsKey(volName)) {\n Set handles = new HashSet<>();\n handles.add(handle);\n runningInstance.put(volName, handles);\n } else {\n runningInstance.get(volName).add(handle);\n }\n }\n }\n\n public static void unregister(String volName, long handle, Runnable cleanupTask) {\n if (handle <= 0) {\n return;\n }\n synchronized (runningInstance) {\n if (!runningInstance.containsKey(volName)) {\n return;\n }\n Set handles = runningInstance.get(volName);\n boolean removed = handles.remove(handle);\n if (!removed) {\n return;\n }\n LOG.debug(\"unregister instance for {}({})\", volName, handle);\n if (handles.size() == 0) {\n LOG.debug(\"clean resources for {}\", volName);\n ScheduledExecutorService pool = bgThreadForName.remove(volName);\n if (pool != null) {\n pool.shutdownNow();\n }\n stopTrashEmptier(volName);\n tasks.entrySet().removeIf(e -> e.getKey().startsWith(volName + \"|\"));\n cleanupTask.run();\n runningInstance.remove(volName);\n }\n }\n }\n\n public interface Task {\n void run() throws IOException;\n }\n\n\n public static void putTask(String volName, String taskName, Task task, long delay, long period, TimeUnit unit) throws IOException {\n synchronized (tasks) {\n String key = volName + \"|\" + taskName;\n if (!tasks.containsKey(key)) {\n LOG.debug(\"start task {}\", key);\n task.run();\n // build background task thread for volume name\n ScheduledExecutorService pool = bgThreadForName.computeIfAbsent(volName,\n n -> Executors.newScheduledThreadPool(1, r -> {\n Thread thread = new Thread(r, \"JuiceFS Background Task\");\n thread.setDaemon(true);\n return thread;\n })\n );\n pool.scheduleAtFixedRate(()->{\n try {\n task.run();\n } catch (IOException e) {\n LOG.warn(\"run {} failed\", key, e);\n }\n }, delay, period, unit);\n tasks.put(key, new Object());\n }\n }\n }\n\n public static void startTrashEmptier(String name, Runnable emptierTask, long delay, TimeUnit unit) {\n synchronized (tasks) {\n String key = name + \"|\" + \"Trash emptier\";\n if (!tasks.containsKey(key)) {\n LOG.debug(\"run trash emptier for {}\", name);\n ScheduledExecutorService thread = Executors.newScheduledThreadPool(1, r -> {\n Thread t = new Thread(r, \"JuiceFS Trash Emptier\");\n t.setDaemon(true);\n return t;\n });\n thread.schedule(emptierTask, delay, unit);\n tasks.put(key, thread);\n }\n }\n }\n\n private static void stopTrashEmptier(String name) {\n synchronized (tasks) {\n String key = name + \"|\" + \"Trash emptier\";\n Object v = tasks.remove(key);\n if (v instanceof ScheduledExecutorService) {\n LOG.debug(\"close trash emptier for {}\", name);\n ((ScheduledExecutorService) v).shutdownNow();\n }\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/BufferPool.java", "content": "package io.juicefs.utils;\n\nimport java.lang.ref.WeakReference;\nimport java.nio.ByteBuffer;\nimport java.util.Queue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentLinkedQueue;\nimport java.util.concurrent.ConcurrentMap;\n\n/**\n * thread safe\n */\npublic class BufferPool {\n\n private static final ConcurrentMap>> buffersBySize = new ConcurrentHashMap<>();\n\n public static ByteBuffer getBuffer(int size) {\n Queue> list = buffersBySize.get(size);\n if (list == null) {\n return ByteBuffer.allocate(size);\n }\n\n WeakReference ref;\n while ((ref = list.poll()) != null) {\n ByteBuffer b = ref.get();\n if (b != null) {\n return b;\n }\n }\n\n return ByteBuffer.allocate(size);\n }\n\n public static void returnBuffer(ByteBuffer buf) {\n buf.clear();\n int size = buf.capacity();\n Queue> list = buffersBySize.get(size);\n if (list == null) {\n list = new ConcurrentLinkedQueue<>();\n Queue> prev = buffersBySize.putIfAbsent(size, list);\n // someone else put a queue in the map before we did\n if (prev != null) {\n list = prev;\n }\n }\n list.add(new WeakReference<>(buf));\n }\n}" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/CallerContextUtil.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport org.apache.hadoop.ipc.CallerContext;\n\n\npublic class CallerContextUtil {\n\n public static void setContext(String context) throws Exception {\n CallerContext current = CallerContext.getCurrent();\n CallerContext.Builder builder;\n if (current == null || !current.isContextValid()) {\n builder = new CallerContext.Builder(context);\n CallerContext.setCurrent(builder.build());\n } else if (current.getSignature() == null && !current.getContext().endsWith(\"_\" + context)) {\n builder = new CallerContext.Builder(current.getContext() + \"_\" + context);\n CallerContext.setCurrent(builder.build());\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/ConsistentHash.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.utils;\n\nimport com.google.common.hash.HashFunction;\nimport com.google.common.hash.Hashing;\n\nimport java.util.List;\nimport java.util.SortedMap;\nimport java.util.concurrent.ConcurrentSkipListMap;\n\npublic class ConsistentHash {\n\n private final int numberOfVirtualNodeReplicas;\n private final SortedMap circle = new ConcurrentSkipListMap<>();\n private final HashFunction nodeHash = Hashing.murmur3_32();\n private final HashFunction keyHash = Hashing.murmur3_32();\n\n public ConsistentHash(int numberOfVirtualNodeReplicas, List nodes) {\n this.numberOfVirtualNodeReplicas = numberOfVirtualNodeReplicas;\n addNode(nodes);\n }\n\n public void addNode(List nodes) {\n for (T node : nodes) {\n addNode(node);\n }\n }\n\n public void addNode(T node) {\n for (int i = 0; i < numberOfVirtualNodeReplicas; i++) {\n circle.put(getKetamaHash(i + \"\" + node), node);\n }\n }\n\n public void remove(List nodes) {\n for (T node : nodes) {\n remove(node);\n }\n }\n\n public void remove(T node) {\n for (int i = 0; i < numberOfVirtualNodeReplicas; i++) {\n circle.remove(getKetamaHash(i + \"\" + node));\n }\n }\n\n public T get(Object key) {\n if (circle.isEmpty()) {\n return null;\n }\n int hash = getKeyHash(key.toString());\n if (!circle.containsKey(hash)) {\n SortedMap tailMap = circle.tailMap(hash);\n hash = tailMap.isEmpty() ? circle.firstKey() : tailMap.firstKey();\n }\n return circle.get(hash);\n }\n\n private int getKeyHash(final String k) {\n return keyHash.hashBytes(k.getBytes()).asInt();\n }\n\n private int getKetamaHash(final String k) {\n return nodeHash.hashBytes(k.getBytes()).asInt();\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/FsNodesFetcher.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport io.juicefs.JuiceFileSystem;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FSDataInputStream;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.net.URI;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.stream.Collectors;\n\npublic class FsNodesFetcher extends NodesFetcher {\n private static final Logger LOG = LoggerFactory.getLogger(FsNodesFetcher.class);\n\n private Configuration conf;\n\n public FsNodesFetcher(String jfsName) {\n super(jfsName);\n }\n\n public void setConf(Configuration conf) {\n this.conf = conf;\n }\n\n @Override\n public List fetchNodes(String uri) {\n Path path = new Path(uri);\n try (FileSystem fs = FileSystem.newInstance(path.toUri(), conf);\n FSDataInputStream inputStream = fs.open(path)) {\n return new BufferedReader(new InputStreamReader(inputStream))\n .lines().filter(l->!l.isEmpty()).collect(Collectors.toList());\n } catch (Exception e) {\n LOG.warn(\"fetch nodes from {} failed\", uri, e);\n }\n return null;\n }\n\n @Override\n protected Set parseNodes(String response) throws Exception {\n return null;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/FsPermissionExtension.java", "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *

\n * http://www.apache.org/licenses/LICENSE-2.0\n *

\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.utils;\n\nimport org.apache.hadoop.classification.InterfaceAudience;\nimport org.apache.hadoop.fs.permission.FsPermission;\n\n/**\n * HDFS permission subclass used to indicate an ACL is present and/or that the\n * underlying file/dir is encrypted. The ACL/encrypted bits are not visible\n * directly to users of {@link FsPermission} serialization. This is\n * done for backwards compatibility in case any existing clients assume the\n * value of FsPermission is in a particular range.\n */\n@InterfaceAudience.Private\npublic class FsPermissionExtension extends FsPermission {\n private final static short ACL_BIT = 1 << 12;\n private final static short ENCRYPTED_BIT = 1 << 13;\n private final boolean aclBit;\n private final boolean encryptedBit;\n\n /**\n * Constructs a new FsPermissionExtension based on the given FsPermission.\n *\n * @param perm FsPermission containing permission bits\n */\n public FsPermissionExtension(FsPermission perm, boolean hasAcl,\n boolean isEncrypted) {\n super(perm.toShort());\n aclBit = hasAcl;\n encryptedBit = isEncrypted;\n }\n\n @Override\n public short toExtendedShort() {\n return (short) (toShort() |\n (aclBit ? ACL_BIT : 0) | (encryptedBit ? ENCRYPTED_BIT : 0));\n }\n\n public boolean getAclBit() {\n return aclBit;\n }\n\n @Override\n public boolean getEncryptedBit() {\n return encryptedBit;\n }\n}" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/NodesFetcher.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\n\nimport java.io.*;\nimport java.net.HttpURLConnection;\nimport java.net.URL;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.stream.Collectors;\n\n/**\n * fetch calculate nodes of the cluster\n */\npublic abstract class NodesFetcher {\n private static final Log LOG = LogFactory.getLog(NodesFetcher.class);\n\n protected File cacheFolder = new File(\"/tmp/.juicefs\");\n protected File cacheFile;\n private String jfsName;\n\n public NodesFetcher(String jfsName) {\n this.jfsName = jfsName;\n if (!cacheFolder.exists()) {\n cacheFolder.mkdirs();\n }\n cacheFile = new File(cacheFolder, jfsName + \".nodes\");\n cacheFolder.setWritable(true, false);\n cacheFolder.setReadable(true, false);\n cacheFolder.setExecutable(true, false);\n cacheFile.setWritable(true, false);\n cacheFile.setReadable(true, false);\n cacheFile.setExecutable(true, false);\n }\n\n public List fetchNodes(String urls) {\n List result = readCache();\n\n // refresh local disk cache every 10 mins\n long duration = System.currentTimeMillis() - cacheFile.lastModified();\n if (duration > 10 * 60 * 1000L || result == null) {\n Set nodes = getNodes(urls.split(\",\"));\n if (nodes == null) return result;\n result = new ArrayList<>(nodes);\n cache(result);\n }\n\n return result;\n }\n\n public List readCache() {\n try {\n if (!cacheFile.exists()) return null;\n return Files.readAllLines(cacheFile.toPath());\n } catch (IOException e) {\n LOG.warn(\"read cache failed due to: \", e);\n return null;\n }\n }\n\n public void cache(List hostnames) {\n File tmpFile = new File(cacheFolder, System.getProperty(\"user.name\") + \"-\" + jfsName + \".nodes.tmp\");\n try (RandomAccessFile writer = new RandomAccessFile(tmpFile, \"rws\")) {\n tmpFile.setWritable(true, false);\n tmpFile.setReadable(true, false);\n if (hostnames != null) {\n String content = String.join(\"\\n\", hostnames);\n writer.write(content.getBytes());\n }\n tmpFile.renameTo(cacheFile);\n } catch (IOException e) {\n LOG.warn(\"wirte cache failed due to: \", e);\n }\n }\n\n public Set getNodes(String[] urls) {\n if (urls == null) {\n return null;\n }\n for (String url : urls) {\n try {\n String response = doGet(url);\n if (response == null) {\n continue;\n }\n return parseNodes(response);\n } catch (Throwable e) {\n LOG.warn(\"fetch from:\" + url + \" failed, switch to another url\", e);\n }\n }\n return null;\n }\n\n protected abstract Set parseNodes(String response) throws Exception;\n\n protected String doGet(String url) {\n int timeout = 3; // seconds\n\n HttpURLConnection con = null;\n try {\n con = (HttpURLConnection) new URL(url).openConnection();\n con.setConnectTimeout(timeout * 1000);\n con.setReadTimeout(timeout * 1000);\n\n int status = con.getResponseCode();\n if (status != 200) return null;\n\n BufferedReader in = new BufferedReader(\n new InputStreamReader(con.getInputStream(), StandardCharsets.UTF_8));\n String content = in.lines().collect(Collectors.joining(\"\\n\"));\n in.close();\n return content;\n } catch (IOException e) {\n LOG.warn(e);\n return null;\n } finally {\n if (con != null) {\n con.disconnect();\n }\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/NodesFetcherBuilder.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport org.apache.hadoop.conf.Configuration;\n\npublic class NodesFetcherBuilder {\n public static NodesFetcher buildFetcher(String urls, String jfsName, Configuration conf) {\n NodesFetcher fetcher;\n if ((urls.startsWith(\"http\") && urls.contains(\"cluster/nodes\"))\n || \"yarn\".equals(urls.toLowerCase().trim())) {\n fetcher = new YarnNodesFetcher(jfsName);\n } else if (urls.startsWith(\"http\") && urls.contains(\"service/presto\")) {\n fetcher = new PrestoNodesFetcher(jfsName);\n } else if (urls.startsWith(\"http\") && urls.contains(\"/json\")) {\n fetcher = new SparkNodesFetcher(jfsName);\n } else if (urls.startsWith(\"http\") && urls.contains(\"api/v1/applications\")) {\n fetcher = new SparkThriftNodesFetcher(jfsName);\n } else {\n fetcher = new FsNodesFetcher(jfsName);\n ((FsNodesFetcher) fetcher).setConf(conf);\n }\n return fetcher;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/PatchUtil.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport javassist.ClassPool;\nimport javassist.CtClass;\nimport javassist.CtMethod;\nimport javassist.NotFoundException;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.lang.instrument.ClassDefinition;\n\npublic class PatchUtil {\n private static final Logger LOG = LoggerFactory.getLogger(PatchUtil.class);\n\n public enum PatchType {\n BODY, BEFORE, AFTER\n }\n\n public static class ClassMethod {\n private String method;\n private String[] params;\n private PatchType[] types;\n private String[] codes;\n\n public ClassMethod(String method, String[] params, String[] codes, PatchType[] types) {\n if (codes.length != types.length) {\n LOG.error(\"{} has {} codes, but only {} types\", method, codes.length, types.length);\n }\n this.method = method;\n this.params = params;\n this.codes = codes;\n this.types = types;\n }\n }\n\n public static synchronized void doPatch(String className, ClassMethod[] classMethods) {\n\n ClassPool classPool = ClassPool.getDefault();\n try {\n CtClass cls = classPool.get(className);\n\n for (ClassMethod classMethod : classMethods) {\n String method = classMethod.method;\n\n CtMethod mtd;\n String[] params = classMethod.params;\n if (params != null) {\n CtClass[] cts = new CtClass[params.length];\n for (int i = 0; i < params.length; i++) {\n cts[i] = classPool.get(params[i]);\n }\n mtd = cls.getDeclaredMethod(method, cts);\n } else {\n mtd = cls.getDeclaredMethod(method);\n }\n\n String[] codes = classMethod.codes;\n PatchType[] types = classMethod.types;\n for (int i = 0; i < codes.length; i++) {\n switch (types[i]) {\n case BODY:\n mtd.setBody(codes[0]);\n break;\n case AFTER:\n mtd.insertAfter(codes[0], true);\n break;\n case BEFORE:\n mtd.insertBefore(codes[0]);\n break;\n }\n }\n }\n\n RedefineClassAgent.redefineClasses(new ClassDefinition(Class.forName(className), cls.toBytecode()));\n cls.detach();\n } catch (NotFoundException | NoClassDefFoundError ignored) {\n } catch (Throwable e) {\n LOG.warn(String.format(\"patch %s failed\", className), e);\n }\n }\n\n public static void patchBody(String className, String method, String[] params, String code) {\n doPatch(className, new ClassMethod[]{new ClassMethod(method, params, new String[]{code}, new PatchType[]{PatchType.BODY})});\n }\n\n public static void patchBefore(String className, String method, String[] params, String code) {\n doPatch(className, new ClassMethod[]{new ClassMethod(method, params, new String[]{code}, new PatchType[]{PatchType.BEFORE})});\n }\n\n public static void patchAfter(String className, String method, String[] params, String code) {\n doPatch(className, new ClassMethod[]{new ClassMethod(method, params, new String[]{code}, new PatchType[]{PatchType.AFTER})});\n }\n\n public static void patchBeforeAndAfter(String className, String method, String[] params, String beforeCode, String afterCode) {\n doPatch(className, new ClassMethod[]{new ClassMethod(method, params, new String[]{beforeCode, afterCode}, new PatchType[]{PatchType.BEFORE, PatchType.AFTER})});\n }\n\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/PrestoNodesFetcher.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport org.json.JSONArray;\nimport org.json.JSONObject;\n\nimport java.net.URL;\nimport java.util.HashSet;\nimport java.util.Set;\n\npublic class PrestoNodesFetcher extends NodesFetcher {\n\n public PrestoNodesFetcher(String jfsName) {\n super(jfsName);\n }\n\n // url like \"http://hadoop01:8000/v1/service/presto\"\n @Override\n protected Set parseNodes(String response) throws Exception {\n Set result = new HashSet<>();\n JSONArray nodes = new JSONObject(response).getJSONArray(\"services\");\n for (Object node : nodes) {\n JSONObject nodeProperties = ((JSONObject) node).getJSONObject(\"properties\");\n if (nodeProperties.getString(\"coordinator\").equals(\"false\")) {\n String http = nodeProperties.getString(\"http\");\n result.add(new URL(http).getHost());\n }\n }\n return result;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/RedefineClassAgent.java", "content": "/*\nCopyright 2017 Turn Inc\nAll rights reserved.\nThe contents of this file are subject to the MIT License as provided\nbelow. Alternatively, the contents of this file may be used under\nthe terms of Mozilla Public License Version 1.1,\nthe terms of the GNU Lesser General Public License Version 2.1 or later,\nor the terms of the Apache License Version 2.0.\nLicense:\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n*/\n\npackage io.juicefs.utils;\n\n\nimport com.sun.tools.attach.VirtualMachine;\nimport javassist.CannotCompileException;\nimport javassist.ClassPool;\nimport javassist.CtClass;\nimport javassist.NotFoundException;\n\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.lang.instrument.ClassDefinition;\nimport java.lang.instrument.Instrumentation;\nimport java.lang.instrument.UnmodifiableClassException;\nimport java.lang.management.ManagementFactory;\nimport java.util.jar.Attributes;\nimport java.util.jar.JarEntry;\nimport java.util.jar.JarOutputStream;\nimport java.util.jar.Manifest;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n/**\n * Packages everything necessary to be able to redefine a class using {@link Instrumentation} as provided by\n * Java 1.6 or later. Class redefinition is the act of replacing a class' bytecode at runtime, after that class\n * has already been loaded.\n *

\n * The scheme employed by this class uses an agent (defined by this class) that, when loaded into the JVM, provides\n * an instance of {@link Instrumentation} which in turn provides a method to redefine classes.\n *

\n * Users of this class only need to call {@link #redefineClasses(ClassDefinition...)}. The agent stuff will be done\n * automatically (and lazily).\n *

\n * Note that classes cannot be arbitrarily redefined. The new version must retain the same schema; methods and fields\n * cannot be added or removed. In practice this means that method bodies can be changed.\n *

\n * Note that this is a replacement for javassist's {@code HotSwapper}. {@code HotSwapper} depends on the debug agent\n * to perform the hotswap. That agent is available since Java 1.3, but the JVM must be started with the agent enabled,\n * and the agent often fails to perform the swap if the machine is under heavy load. This class is both cleaner and more\n * reliable.\n *\n * @author Adam Lugowski\n * @see Instrumentation#redefineClasses(ClassDefinition...)\n */\npublic class RedefineClassAgent {\n /**\n * Use the Java logger to avoid any references to anything not supplied by the JVM. This avoids issues with\n * classpath when compiling/loading this class as an agent.\n */\n private static final Logger LOGGER = Logger.getLogger(RedefineClassAgent.class.getSimpleName());\n\n /**\n * Populated when this class is loaded into the JVM as an agent (via {@link #ensureAgentLoaded()}.\n */\n private static volatile Instrumentation instrumentation = null;\n\n /**\n * How long to wait for the agent to load before giving up and assuming the load failed.\n */\n private static final int AGENT_LOAD_WAIT_TIME_SEC = 3;\n\n /**\n * Agent entry point. Do not call this directly.\n *

\n * This method is called by the JVM when this class is loaded as an agent.\n *

\n * Sets {@link #instrumentation} to {@code inst}, provided {@code inst} supports class redefinition.\n *\n * @param agentArgs ignored.\n * @param inst This is the reason this class exists. {@link Instrumentation} has the\n * {@link Instrumentation#redefineClasses(ClassDefinition...)} method.\n */\n public static void agentmain(String agentArgs, Instrumentation inst) {\n if (!inst.isRedefineClassesSupported()) {\n LOGGER.severe(\"Class redefinition not supported. Aborting.\");\n return;\n }\n\n instrumentation = inst;\n }\n\n /**\n * Attempts to redefine class bytecode.\n *

\n * On first call this method will attempt to load an agent into the JVM to obtain an instance of\n * {@link Instrumentation}. This agent load can introduce a pause (in practice 1 to 2 seconds).\n *\n * @param definitions classes to redefine.\n * @throws UnmodifiableClassException as thrown by {@link Instrumentation#redefineClasses(ClassDefinition...)}\n * @throws ClassNotFoundException as thrown by {@link Instrumentation#redefineClasses(ClassDefinition...)}\n * @throws FailedToLoadAgentException if agent either failed to load or if the agent wasn't able to get an\n * instance of {@link Instrumentation} that allows class redefinitions.\n * @see Instrumentation#redefineClasses(ClassDefinition...)\n */\n public static void redefineClasses(ClassDefinition... definitions)\n throws UnmodifiableClassException, ClassNotFoundException, FailedToLoadAgentException {\n ensureAgentLoaded();\n instrumentation.redefineClasses(definitions);\n }\n\n /**\n * Lazy loads the agent that populates {@link #instrumentation}. OK to call multiple times.\n *\n * @throws FailedToLoadAgentException if agent either failed to load or if the agent wasn't able to get an\n * instance of {@link Instrumentation} that allows class redefinitions.\n */\n private static void ensureAgentLoaded() throws FailedToLoadAgentException {\n if (instrumentation != null) {\n // already loaded\n return;\n }\n\n // load the agent\n try {\n File agentJar = createAgentJarFile();\n\n // Loading an agent requires the PID of the JVM to load the agent to. Find out our PID.\n String nameOfRunningVM = ManagementFactory.getRuntimeMXBean().getName();\n String pid = nameOfRunningVM.substring(0, nameOfRunningVM.indexOf('@'));\n\n // load the agent\n VirtualMachine vm = VirtualMachine.attach(pid);\n vm.loadAgent(agentJar.getAbsolutePath(), \"\");\n vm.detach();\n } catch (Exception e) {\n throw new FailedToLoadAgentException(e);\n }\n\n // wait for the agent to load\n for (int sec = 0; sec < AGENT_LOAD_WAIT_TIME_SEC; sec++) {\n if (instrumentation != null) {\n // success!\n return;\n }\n\n try {\n LOGGER.info(\"Sleeping for 1 second while waiting for agent to load.\");\n Thread.sleep(1000);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n throw new FailedToLoadAgentException();\n }\n }\n\n // agent didn't load\n throw new FailedToLoadAgentException();\n }\n\n /**\n * An agent must be specified as a .jar where the manifest has an Agent-Class attribute. Additionally, in order\n * to be able to redefine classes, the Can-Redefine-Classes attribute must be true.\n *

\n * This method creates such an agent Jar as a temporary file. The Agent-Class is this class. If the returned Jar\n * is loaded as an agent then {@link #agentmain(String, Instrumentation)} will be called by the JVM.\n *\n * @return a temporary {@link File} that points at Jar that packages this class.\n * @throws IOException if agent Jar creation failed.\n */\n private static File createAgentJarFile() throws IOException {\n File jarFile = File.createTempFile(\"agent\", \".jar\");\n jarFile.deleteOnExit();\n\n // construct a manifest that allows class redefinition\n Manifest manifest = new Manifest();\n Attributes mainAttributes = manifest.getMainAttributes();\n mainAttributes.put(Attributes.Name.MANIFEST_VERSION, \"1.0\");\n mainAttributes.put(new Attributes.Name(\"Agent-Class\"), RedefineClassAgent.class.getName());\n mainAttributes.put(new Attributes.Name(\"Can-Retransform-Classes\"), \"true\");\n mainAttributes.put(new Attributes.Name(\"Can-Redefine-Classes\"), \"true\");\n\n try (JarOutputStream jos = new JarOutputStream(new FileOutputStream(jarFile), manifest)) {\n // add the agent .class into the .jar\n JarEntry agent = new JarEntry(RedefineClassAgent.class.getName().replace('.', '/') + \".class\");\n jos.putNextEntry(agent);\n\n // dump the class bytecode into the entry\n ClassPool pool = ClassPool.getDefault();\n CtClass ctClass = pool.get(RedefineClassAgent.class.getName());\n jos.write(ctClass.toBytecode());\n jos.closeEntry();\n } catch (CannotCompileException | NotFoundException e) {\n // Realistically this should never happen.\n LOGGER.log(Level.SEVERE, \"Exception while creating RedefineClassAgent jar.\", e);\n throw new IOException(e);\n }\n\n return jarFile;\n }\n\n /**\n * Marks a failure to load the agent and get an instance of {@link Instrumentation} that is able to redefine\n * classes.\n */\n public static class FailedToLoadAgentException extends Exception {\n public FailedToLoadAgentException() {\n super();\n }\n\n public FailedToLoadAgentException(Throwable cause) {\n super(cause);\n }\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/ReflectionUtil.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.utils;\n\nimport java.lang.reflect.Constructor;\nimport java.lang.reflect.Field;\n\npublic class ReflectionUtil {\n public static boolean hasMethod(String className, String method, String[] params) {\n try {\n Class[] classes = null;\n if (params != null) {\n classes = new Class[params.length];\n for (int i = 0; i < params.length; i++) {\n classes[i] = Class.forName(params[i], false, Thread.currentThread().getContextClassLoader());\n }\n }\n return hasMethod(className, method, classes);\n } catch (ClassNotFoundException e) {\n return false;\n }\n }\n\n public static boolean hasMethod(String className, String method, Class[] params) {\n try {\n Class clazz = Class.forName(className, false, Thread.currentThread().getContextClassLoader());\n clazz.getDeclaredMethod(method, params);\n } catch (ClassNotFoundException | NoSuchMethodException e) {\n return false;\n }\n return true;\n }\n\n public static Constructor getConstructor(Class clazz, Class... params) {\n try {\n return clazz.getConstructor(params);\n } catch (NoSuchMethodException e) {\n return null;\n }\n }\n\n public static Object getField(String className, String field, Object obj) throws ClassNotFoundException, NoSuchFieldException, IllegalAccessException {\n Class clazz = Class.forName(className);\n Field f = clazz.getDeclaredField(field);\n f.setAccessible(true);\n return f.get(obj);\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/SparkNodesFetcher.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport org.json.JSONArray;\nimport org.json.JSONObject;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\npublic class SparkNodesFetcher extends NodesFetcher {\n public SparkNodesFetcher(String jfsName) {\n super(jfsName);\n }\n\n // url like \"http://host:8888/json/\"\n @Override\n protected Set parseNodes(String response) throws Exception {\n Set result = new HashSet<>();\n JSONArray workers = new JSONObject(response).getJSONArray(\"workers\");\n for (Object worker : workers) {\n if (((JSONObject) worker).getString(\"state\").equals(\"ALIVE\")) {\n result.add(((JSONObject) worker).getString(\"host\"));\n }\n }\n return result;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/SparkThriftNodesFetcher.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.json.JSONArray;\nimport org.json.JSONObject;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\n// \"http://hadoop01:4040/api/v1/applications/\";\npublic class SparkThriftNodesFetcher extends NodesFetcher {\n private static final Log LOG = LogFactory.getLog(SparkThriftNodesFetcher.class);\n\n public SparkThriftNodesFetcher(String jfsName) {\n super(jfsName);\n }\n\n @Override\n public Set getNodes(String[] urls) {\n if (urls == null || urls.length == 0) {\n return null;\n }\n for (String url : urls) {\n try {\n JSONArray appArrays = new JSONArray(doGet(url));\n if (appArrays.length() > 0) {\n String id = appArrays.getJSONObject(0).getString(\"id\");\n url = url.endsWith(\"/\") ? url : url + \"/\";\n return parseNodes(doGet(url + id + \"/allexecutors\"));\n }\n } catch (Throwable e) {\n LOG.warn(\"fetch from spark thrift server failed!\", e);\n }\n }\n return null;\n }\n\n @Override\n protected Set parseNodes(String response) throws Exception {\n if (response == null) {\n return null;\n }\n Set res = new HashSet<>();\n for (Object item : new JSONArray(response)) {\n JSONObject obj = (JSONObject) item;\n String id = obj.getString(\"id\");\n boolean isActive = obj.getBoolean(\"isActive\");\n String hostPort = obj.getString(\"hostPort\");\n boolean isBlacklisted = obj.getBoolean(\"isBlacklisted\");\n String[] hAp = hostPort.split(\":\");\n if (hAp.length > 0 && !\"driver\".equals(id) && isActive && !isBlacklisted) {\n res.add(hAp[0]);\n }\n }\n return res;\n }\n}\n" }, { "path": "sdk/java/src/main/java/io/juicefs/utils/YarnNodesFetcher.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.hadoop.conf.Configuration;\nimport org.json.JSONArray;\nimport org.json.JSONObject;\n\nimport java.util.*;\n\npublic class YarnNodesFetcher extends NodesFetcher {\n private static final Log LOG = LogFactory.getLog(YarnNodesFetcher.class);\n\n public YarnNodesFetcher(String jfsName) {\n super(jfsName);\n }\n\n @Override\n public Set getNodes(String[] urls) {\n if (urls == null || urls.length == 0) {\n return null;\n }\n List yarnUrls = Arrays.asList(urls);\n for (String url : urls) {\n if (\"yarn\".equals(url.toLowerCase().trim())) {\n Configuration conf = new Configuration();\n Map props = conf.getValByRegex(\"yarn\\\\.resourcemanager\\\\.webapp\\\\.address.*\");\n if (props.size() == 0) {\n return null;\n }\n yarnUrls = new ArrayList<>();\n for (String v : props.values()) {\n yarnUrls.add(\"http://\" + v + \"/ws/v1/cluster/nodes/\");\n }\n break;\n }\n }\n return super.getNodes(yarnUrls.toArray(new String[0]));\n }\n\n @Override\n protected Set parseNodes(String response) {\n Set result = new HashSet<>();\n JSONArray allNodes = new JSONObject(response).getJSONObject(\"nodes\").getJSONArray(\"node\");\n for (Object obj : allNodes) {\n if (obj instanceof JSONObject) {\n JSONObject node = (JSONObject) obj;\n String state = node.getString(\"state\");\n String hostname = node.getString(\"nodeHostName\");\n if (\"RUNNING\".equals(state)) {\n result.add(hostname);\n }\n }\n }\n return result;\n }\n}\n" }, { "path": "sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory", "content": "# JuiceFS, Copyright 2020 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nio.juicefs.FlinkFileSystemFactory" }, { "path": "sdk/java/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier", "content": "# JuiceFS, Copyright 2025 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nio.juicefs.kerberos.JuiceFSDelegationTokenIdentifier\n" }, { "path": "sdk/java/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer", "content": "# JuiceFS, Copyright 2025 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nio.juicefs.kerberos.JuiceFSTokenRenewer\n" }, { "path": "sdk/java/src/main/resources/META-INF/services/org.kitesdk.data.spi.Loadable", "content": "# JuiceFS, Copyright 2021 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nio.juicefs.KiteDataLoader\n" }, { "path": "sdk/java/src/test/java/io/juicefs/JuiceFileSystemBgTaskTest.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs;\n\nimport io.juicefs.utils.BgTaskUtil;\nimport junit.framework.TestCase;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.net.URI;\nimport java.util.Map;\nimport java.util.concurrent.*;\n\nimport static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;\nimport static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;\n\npublic class JuiceFileSystemBgTaskTest extends TestCase {\n private static final Logger LOG = LoggerFactory.getLogger(JuiceFileSystemBgTaskTest.class);\n\n public void testJuiceFileSystemBgTask() throws Exception {\n FileSystem.closeAll();\n BgTaskUtil.reset();\n Configuration conf = new Configuration();\n conf.addResource(JuiceFileSystemTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n conf.set(FS_TRASH_INTERVAL_KEY, \"6\");\n conf.set(FS_TRASH_CHECKPOINT_INTERVAL_KEY, \"2\");\n conf.set(\"juicefs.users\", \"jfs://dev/users\");\n conf.set(\"juicefs.groups\", \"jfs://dev/groups\");\n conf.set(\"juicefs.discover-nodes-url\", \"jfs://dev/etc/nodes\");\n int threads = 100;\n int instances = 1000;\n CountDownLatch latch = new CountDownLatch(instances);\n ExecutorService pool = Executors.newFixedThreadPool(threads);\n for (int i = 0; i < instances; i++) {\n pool.submit(() -> {\n try (JuiceFileSystem jfs = new JuiceFileSystem()) {\n jfs.initialize(URI.create(\"jfs://dev/\"), conf);\n if (ThreadLocalRandom.current().nextInt(10) % 2 == 0) {\n jfs.getFileBlockLocations(jfs.getFileStatus(new Path(\"jfs://dev/users\")), 0, 1000);\n }\n } catch (Exception e) {\n LOG.error(\"unexpected exception\", e);\n } finally {\n latch.countDown();\n }\n });\n }\n latch.await();\n Map bgThreadForName = BgTaskUtil.getBgThreadForName();\n for (String s : bgThreadForName.keySet()) {\n System.out.println(s);\n }\n assertEquals(0, bgThreadForName.size());\n assertEquals(0, BgTaskUtil.getRunningInstance().size());\n pool.shutdown();\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs;\n\nimport com.google.common.collect.Lists;\nimport io.juicefs.utils.AclTransformation;\nimport junit.framework.TestCase;\nimport org.apache.commons.io.IOUtils;\nimport org.apache.flink.runtime.fs.hdfs.HadoopRecoverableWriter;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.*;\nimport org.apache.hadoop.fs.permission.*;\nimport org.apache.hadoop.io.MD5Hash;\nimport org.apache.hadoop.security.AccessControlException;\nimport org.apache.hadoop.security.UserGroupInformation;\n\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.lang.reflect.Method;\nimport java.net.InetAddress;\nimport java.net.URI;\nimport java.nio.ByteBuffer;\nimport java.security.PrivilegedExceptionAction;\nimport java.util.*;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;\nimport static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;\nimport static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;\nimport static org.apache.hadoop.fs.permission.AclEntryScope.DEFAULT;\nimport static org.apache.hadoop.fs.permission.AclEntryType.*;\nimport static org.apache.hadoop.fs.permission.FsAction.*;\nimport static org.junit.Assert.assertArrayEquals;\n\npublic class JuiceFileSystemTest extends TestCase {\n FsShell shell;\n FileSystem fs;\n Configuration cfg;\n\n public void setUp() throws Exception {\n cfg = new Configuration();\n cfg.addResource(JuiceFileSystemTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n cfg.set(FS_TRASH_INTERVAL_KEY, \"6\");\n cfg.set(FS_TRASH_CHECKPOINT_INTERVAL_KEY, \"2\");\n cfg.set(\"juicefs.access-log\", \"/tmp/jfs.access.log\");\n cfg.set(\"juicefs.discover-nodes-url\", \"jfs:///etc/nodes\");\n fs = FileSystem.newInstance(cfg);\n fs.delete(new Path(\"/hello\"));\n FSDataOutputStream out = fs.create(new Path(\"/hello\"), true);\n out.writeBytes(\"hello\\n\");\n out.close();\n\n cfg.setQuietMode(false);\n shell = new FsShell(cfg);\n }\n\n public void tearDown() throws Exception {\n fs.close();\n FileSystem.closeAll();\n }\n\n public void testFsStatus() throws IOException {\n FsStatus st = fs.getStatus();\n assertTrue(\"capacity\", st.getCapacity() > 0);\n assertTrue(\"remaining\", st.getRemaining() > 0);\n }\n\n public void testSummary() throws IOException {\n ContentSummary summary = fs.getContentSummary(new Path(\"/\"));\n assertTrue(\"length\", summary.getLength() > 0);\n assertTrue(\"fileCount\", summary.getFileCount() > 0);\n summary = fs.getContentSummary(new Path(\"/hello\"));\n assertEquals(6, summary.getLength());\n assertEquals(1, summary.getFileCount());\n assertEquals(0, summary.getDirectoryCount());\n assertEquals(-1L, summary.getQuota());\n assertEquals(-1L, summary.getSpaceQuota());\n }\n\n public void testLongName() throws IOException {\n Path p = new Path(\n \"/longname/very_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_name\");\n fs.mkdirs(p);\n FileStatus[] files = fs.listStatus(new Path(\"/longname\"));\n if (files.length != 1) {\n throw new IOException(\"expected one file but got \" + files.length);\n }\n if (!files[0].getPath().getName().equals(p.getName())) {\n throw new IOException(\"not equal\");\n }\n }\n\n public void testLocation() throws IOException {\n FileStatus f = new FileStatus(3L << 30, false, 1, 128L << 20, 0, new Path(\"/hello\"));\n BlockLocation[] locations = fs.getFileBlockLocations(f, 128L * 1024 * 1024 - 256, 5L * 64 * 1024 * 1024 - 512L);\n\n String[] names = locations[0].getNames();\n for (String name : names) {\n assertEquals(name.split(\":\").length, 2);\n }\n\n String[] storageIds = locations[0].getStorageIds();\n assertNotNull(storageIds);\n assertEquals(names.length, storageIds.length);\n\n assertEquals(InetAddress.getLocalHost().getHostName() + \":50010\", names[0]);\n }\n\n public void testReadWrite() throws Exception {\n long l = fs.getFileStatus(new Path(\"/hello\")).getLen();\n assertEquals(6, l);\n byte[] buf = new byte[(int) l];\n FSDataInputStream in = fs.open(new Path(\"/hello\"));\n in.readFully(buf);\n in.close();\n assertEquals(\"hello\\n\", new String(buf));\n assertEquals(0, shell.run(new String[]{\"-cat\", \"/hello\"}));\n\n fs.setPermission(new Path(\"/hello\"), new FsPermission((short) 0000));\n UserGroupInformation ugi =\n UserGroupInformation.createUserForTesting(\"nobody\", new String[]{\"nogroup\"});\n FileSystem fs2 = ugi.doAs(new PrivilegedExceptionAction() {\n @Override\n public FileSystem run() throws Exception {\n return FileSystem.get(new URI(\"jfs://dev\"), cfg);\n }\n });\n try {\n in = fs2.open(new Path(\"/hello\"));\n assertEquals(in, null);\n } catch (IOException e) {\n fs.setPermission(new Path(\"/hello\"), new FsPermission((short) 0644));\n }\n }\n\n public void testWrite() throws Exception {\n Path f = new Path(\"/testWriteFile\");\n FSDataOutputStream fou = fs.create(f);\n byte[] b = \"hello world\".getBytes();\n OutputStream ou = ((JuiceFileSystemImpl.BufferedFSOutputStream)fou.getWrappedStream()).getOutputStream();\n ou.write(b, 6, 5);\n ou.close();\n FSDataInputStream in = fs.open(f);\n String str = IOUtils.toString(in);\n assertEquals(\"world\", str);\n in.close();\n\n int fileLen = 1 << 20;\n byte[] contents = new byte[fileLen];\n Random random = new Random();\n random.nextBytes(contents);\n f = new Path(\"/tmp/writeFile\");\n FSDataOutputStream out = fs.create(f);\n int off = 0;\n int len = 256<<10;\n out.write(contents, off, len);\n out.close();\n\n byte[] readBytes = new byte[len];\n in = fs.open(f);\n in.read(readBytes);\n assertArrayEquals(Arrays.copyOfRange(contents, off, off + len), readBytes);\n in.close();\n\n out = fs.create(f);\n off = 0;\n len = fileLen;\n for (int i = off; i < len; i++) {\n out.write(contents[i]);\n }\n out.hflush();\n readBytes = new byte[len];\n in = fs.open(f);\n in.read(readBytes);\n assertArrayEquals(Arrays.copyOfRange(contents, off, off + len), readBytes);\n out.close();\n in.close();\n }\n\n public void testReadSkip() throws Exception {\n Path p = new Path(\"/test_readskip\");\n fs.create(p).close();\n String content = \"12345\";\n writeFile(fs, p, content);\n FSDataInputStream in = fs.open(p);\n long skip = in.skip(2);\n assertEquals(2, skip);\n\n byte[] bytes = new byte[content.length() - (int)skip];\n in.readFully(bytes);\n assertEquals(\"345\", new String(bytes));\n }\n\n public void testReadAfterClose() throws Exception {\n byte[] buf = new byte[6];\n FSDataInputStream in = fs.open(new Path(\"/hello\"));\n in.close();\n try {\n in.read(0, buf, 0, 5);\n } catch (IOException e) {\n if (!e.getMessage().contains(\"closed\")) {\n throw new IOException(\"message should be closed, but got \" + e.getMessage());\n }\n }\n FSDataInputStream in2 = fs.open(new Path(\"/hello\"));\n in.close(); // repeated close should not close other's fd\n in2.read(0, buf, 0, 5);\n in2.close();\n }\n\n public void testMkdirs() throws Exception {\n assertTrue(fs.mkdirs(new Path(\"/mkdirs\")));\n assertTrue(fs.mkdirs(new Path(\"/mkdirs/dir\")));\n assertTrue(fs.delete(new Path(\"/mkdirs\"), true));\n assertTrue(fs.mkdirs(new Path(\"/mkdirs/test\")));\n for (int i = 0; i < 50; i++) {\n fs.mkdirs(new Path(\"/mkdirs/d\" + i));\n }\n assertEquals(51, fs.listStatus(new Path(\"/mkdirs/\")).length);\n assertTrue(fs.delete(new Path(\"/mkdirs\"), true));\n assertTrue(fs.mkdirs(new Path(\"parent/dir\")));\n assertTrue(fs.exists(new Path(fs.getHomeDirectory(), \"parent\")));\n }\n\n public void testCreateWithoutPermission() throws Exception {\n assertTrue(fs.mkdirs(new Path(\"/noperm\")));\n fs.setPermission(new Path(\"/noperm\"), new FsPermission((short) 0555));\n UserGroupInformation ugi =\n UserGroupInformation.createUserForTesting(\"nobody\", new String[]{\"nogroup\"});\n FileSystem fs2 = ugi.doAs(new PrivilegedExceptionAction() {\n @Override\n public FileSystem run() throws Exception {\n return FileSystem.get(new URI(\"jfs://dev\"), cfg);\n }\n });\n try {\n fs2.create(new Path(\"/noperm/a/file\"));\n throw new Exception(\"create should fail\");\n } catch (IOException e) {\n }\n }\n\n public void testCreateNonRecursive() throws Exception {\n Path p = new Path(\"/NOT_EXIST_DIR\");\n p = new Path(p, \"file\");\n try (FSDataOutputStream ou = fs.createNonRecursive(p, false, 1 << 20, (short) 1, 128 << 20, null);) {\n fail(\"createNonRecursive in a not exit dir should fail\");\n } catch (IOException ignored) {\n }\n }\n\n public void testTruncate() throws Exception {\n Path p = new Path(\"/test_truncate\");\n fs.create(p).close();\n fs.truncate(p, 1 << 20);\n assertEquals(1 << 20, fs.getFileStatus(p).getLen());\n fs.truncate(p, 1 << 10);\n assertEquals(1 << 10, fs.getFileStatus(p).getLen());\n }\n\n public void testAccess() throws Exception {\n Path p1 = new Path(\"/test_access\");\n FileSystem newFs = createNewFs(cfg, \"user1\", new String[]{\"group1\"});\n newFs.create(p1).close();\n newFs.setPermission(p1, new FsPermission((short) 0444));\n newFs.access(p1, FsAction.READ);\n try {\n newFs.access(p1, FsAction.WRITE);\n fail(\"The access call should have failed.\");\n } catch (AccessControlException e) {\n }\n\n Path badPath = new Path(\"/bad/bad\");\n try {\n newFs.access(badPath, FsAction.READ);\n fail(\"The access call should have failed\");\n } catch (FileNotFoundException e) {\n }\n newFs.close();\n }\n\n public void testSetPermission() throws Exception {\n assertEquals(0, shell.run(new String[]{\"-chmod\", \"0777\", \"/hello\"}));\n assertEquals(0777, fs.getFileStatus(new Path(\"/hello\")).getPermission().toShort());\n assertEquals(0, shell.run(new String[]{\"-chmod\", \"0666\", \"/hello\"}));\n assertEquals(0666, fs.getFileStatus(new Path(\"/hello\")).getPermission().toShort());\n }\n\n public void testSetTimes() throws Exception {\n fs.setTimes(new Path(\"/hello\"), 1000, 2000);\n assertEquals(1000, fs.getFileStatus(new Path(\"/hello\")).getModificationTime());\n // assertEquals(2000, fs.getFileStatus(new Path(\"/hello\")).getAccessTime());\n\n Path p = new Path(\"/test-mtime\");\n fs.delete(p, true);\n FSDataOutputStream out = fs.create(p);\n Thread.sleep(1000);\n long mtime1 = fs.getFileStatus(p).getModificationTime();\n out.writeBytes(\"hello\\n\");\n out.close();\n long mtime2 = fs.getFileStatus(p).getModificationTime();\n if (mtime2 - mtime1 < 1000) {\n throw new IOException(\"stale mtime\");\n }\n Thread.sleep(1000);\n long mtime3 = fs.getFileStatus(p).getModificationTime();\n if (mtime3 != mtime2) {\n throw new IOException(\"mtime was updated\");\n }\n }\n\n public void testSetOwner() throws Exception {\n fs.create(new Path(\"/hello\"));\n FileStatus parent = fs.getFileStatus(new Path(\"/\"));\n FileStatus st = fs.getFileStatus(new Path(\"/hello\"));\n if (!parent.getGroup().equals(st.getGroup())) {\n throw new Exception(\n \"group of new created file should be \" + parent.getGroup() + \", but got \" + st.getGroup());\n }\n return; // only root can change the owner/group to others\n // fs.setOwner(new Path(\"/hello\"), null, \"nogroup\");\n // assertEquals(\"nogroup\", fs.getFileStatus(new Path(\"/hello\")).getGroup());\n }\n\n public void testCloseFileSystem() throws Exception {\n Configuration conf = new Configuration();\n conf.addResource(JuiceFileSystemTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n for (int i = 0; i < 5; i++) {\n FileSystem fs = FileSystem.get(conf);\n fs.getFileStatus(new Path(\"/hello\"));\n fs.close();\n }\n }\n\n public void testReadahead() throws Exception {\n FSDataOutputStream out = fs.create(new Path(\"/hello\"), true);\n for (int i = 0; i < 1000000; i++) {\n out.writeBytes(\"hello\\n\");\n }\n out.close();\n\n // simulate reading a parquet file\n int size = 1000000 * 6;\n byte[] buf = new byte[128000];\n FSDataInputStream in = fs.open(new Path(\"/hello\"));\n in.read(size - 8, buf, 0, 8);\n in.read(size - 5000, buf, 0, 3000);\n in.close();\n in = fs.open(new Path(\"/hello\"));\n in.read(size - 8, buf, 0, 8);\n in.read(size - 5000, buf, 0, 3000);\n in.close();\n in = fs.open(new Path(\"/hello\"));\n in.read(2000000, buf, 0, 128000);\n in.close();\n }\n\n public void testOutputStream() throws Exception {\n FSDataOutputStream out = fs.create(new Path(\"/haha\"));\n if (!(out instanceof Syncable)) {\n throw new RuntimeException(\"FSDataOutputStream should be syncable\");\n }\n if (!(out.getWrappedStream() instanceof Syncable)) {\n throw new RuntimeException(\"BufferedOutputStream should be syncable\");\n }\n out.hflush();\n out.hsync();\n }\n\n public void testInputStream() throws Exception {\n FSDataInputStream in = fs.open(new Path(\"/hello\"));\n if (!(in instanceof ByteBufferReadable)) {\n throw new RuntimeException(\"Inputstream should be bytebufferreadable\");\n }\n if (!(in.getWrappedStream() instanceof ByteBufferReadable)) {\n throw new RuntimeException(\"Inputstream should not be bytebufferreadable\");\n }\n\n FSDataOutputStream out = fs.create(new Path(\"/hello\"), true);\n for (int i = 0; i < 1000000; i++) {\n out.writeBytes(\"hello\\n\");\n }\n out.close();\n\n in = fs.open(new Path(\"/hello\"));\n ByteBuffer buf = ByteBuffer.allocateDirect(6 * 1000000);\n buf.put((byte) in.read());\n while (buf.hasRemaining()) {\n int readCount = in.read(buf);\n if (readCount == -1) {\n // this is probably a bug in the ParquetReader. We shouldn't have called\n // readFully with a buffer\n // that has more remaining than the amount of data in the stream.\n throw new IOException(\"Reached the end of stream. Still have: \" + buf.remaining() + \" bytes left\");\n }\n }\n\n Path directReadFile = new Path(\"/direct_file\");\n FSDataOutputStream ou = fs.create(directReadFile);\n ou.write(\"hello world\".getBytes());\n ou.close();\n FSDataInputStream dto = fs.open(directReadFile);\n ByteBuffer directBuf = ByteBuffer.allocateDirect(11);\n directBuf.put(\"hello \".getBytes());\n dto.seek(6);\n dto.read(directBuf);\n byte[] rest = new byte[11];\n directBuf.flip();\n directBuf.get(rest, 0, rest.length);\n assertEquals(\"hello world\", new String(rest));\n\n /*\n * FSDataOutputStream out = fs.create(new Path(\"/bigfile\"), true); byte[] arr =\n * new byte[1<<20]; for (int i=0; i<1024; i++) { out.write(arr); } out.close();\n *\n * long start = System.currentTimeMillis(); in = fs.open(new Path(\"/bigfile\"));\n * ByteBuffer buf = ByteBuffer.allocateDirect(1<<20); long total=0; while (true)\n * { int n = in.read(buf); total += n; if (n < buf.capacity()) { break; } } long\n * used = System.currentTimeMillis() - start;\n * System.out.printf(\"ByteBuffer read %d throughput %f MB/s\\n\", total,\n * total/1024.0/1024.0/used*1000);\n *\n * start = System.currentTimeMillis(); in = fs.open(new Path(\"/bigfile\"));\n * total=0; while (true) { int n = in.read(buf); total += n; if (n <\n * buf.capacity()) { break; } } used = System.currentTimeMillis() - start;\n * System.out.printf(\"ByteBuffer read %d throughput %f MB/s\\n\", total,\n * total/1024.0/1024.0/used*1000);\n *\n * start = System.currentTimeMillis(); in = fs.open(new Path(\"/bigfile\"));\n * total=0; while (true) { int n = in.read(arr); total += n; if (n <\n * buf.capacity()) { break; } } used = System.currentTimeMillis() - start;\n * System.out.printf(\"Array read %d throughput %f MB/s\\n\", total,\n * total/1024.0/1024.0/used*1000);\n */\n }\n\n public void testInputStreamSkipNBytes() throws Exception {\n Path f = new Path(\"/test-skipnbytes\");\n try (FSDataOutputStream out = fs.create(f)) {\n out.writeBytes(\"hello juicefs\");\n }\n Class inputStreamClass = JuiceFileSystemImpl.FileInputStream.class;\n Method skipNBytes = inputStreamClass.getMethod(\"skipNBytes\", long.class);\n try (FSDataInputStream in = fs.open(f)) {\n skipNBytes.invoke(in.getWrappedStream(), 6);\n String s = IOUtils.toString(in);\n assertEquals(\"juicefs\", s);\n }\n }\n\n public void testReadStats() throws IOException {\n FileSystem.Statistics statistics = FileSystem.getStatistics(fs.getScheme(),\n ((FilterFileSystem) fs).getRawFileSystem().getClass());\n statistics.reset();\n Path path = new Path(\"/hello\");\n FSDataOutputStream out = fs.create(path, true);\n for (int i = 0; i < 1 << 20; i++) {\n out.writeBytes(\"hello\\n\");\n }\n out.close();\n FSDataInputStream in = fs.open(path);\n\n int readSize = 512 << 10;\n\n ByteBuffer buf = ByteBuffer.allocateDirect(readSize);\n while (buf.hasRemaining()) {\n in.read(buf);\n }\n assertEquals(readSize, statistics.getBytesRead());\n\n in.seek(0);\n buf = ByteBuffer.allocate(readSize);\n while (buf.hasRemaining()) {\n in.read(buf);\n }\n assertEquals(readSize * 2, statistics.getBytesRead());\n\n in.read(0, new byte[3000], 0, 3000);\n assertEquals(readSize * 2 + 3000, statistics.getBytesRead());\n\n in.read(3000, new byte[6000], 0, 3000);\n assertEquals(readSize * 2 + 3000 + 3000, statistics.getBytesRead());\n\n in.read(new byte[3000], 0, 3000);\n assertEquals(readSize * 2 + 3000 + 3000 + 3000, statistics.getBytesRead());\n\n in.close();\n }\n\n public void testChecksum() throws IOException {\n Path f = new Path(\"/empty\");\n FSDataOutputStream out = fs.create(f, true);\n out.close();\n FileChecksum sum = fs.getFileChecksum(f);\n assertEquals(new MD5MD5CRC32GzipFileChecksum(0, 0, new MD5Hash(\"70bc8f4b72a86921468bf8e8441dce51\")), sum);\n\n f = new Path(\"/small\");\n out = fs.create(f, true);\n out.writeBytes(\"world\\n\");\n out.close();\n sum = fs.getFileChecksum(f);\n assertEquals(new MD5MD5CRC32CastagnoliFileChecksum(512, 0, new MD5Hash(\"a74dcf6d5ba98e50ae0182c9d5d886fe\")),\n sum);\n sum = fs.getFileChecksum(f, 5);\n assertEquals(new MD5MD5CRC32CastagnoliFileChecksum(512, 0, new MD5Hash(\"05a157db1cc7549c82ec6f31f63fdb46\")),\n sum);\n\n f = new Path(\"/medium\");\n out = fs.create(f, true);\n byte[] bytes = new byte[(128 << 20) - 1];\n out.write(bytes);\n out.close();\n sum = fs.getFileChecksum(f);\n assertEquals(\n new MD5MD5CRC32CastagnoliFileChecksum(512, 0, new MD5Hash(\"1cf326bae8274fd824ec69ece3e4082f\")),\n sum);\n\n f = new Path(\"/big\");\n out = fs.create(f, true);\n byte[] zeros = new byte[1024 * 1000];\n for (int i = 0; i < 150; i++) {\n out.write(zeros);\n }\n out.close();\n sum = fs.getFileChecksum(f);\n assertEquals(\n new MD5MD5CRC32CastagnoliFileChecksum(512, 262144, new MD5Hash(\"7d04ac8132ad64988f7ba4d819cbde62\")),\n sum);\n }\n\n public void testXattr() throws IOException {\n Path p = new Path(\"/test-xattr\");\n fs.delete(p, true);\n fs.create(p);\n assertEquals(null, fs.getXAttr(p, \"x1\"));\n fs.setXAttr(p, \"x1\", new byte[1]);\n fs.setXAttr(p, \"x2\", new byte[2]);\n List names = fs.listXAttrs(p);\n assertEquals(2, names.size());\n Map values = fs.getXAttrs(p);\n assertEquals(2, values.size());\n assertEquals(1, values.get(\"x1\").length);\n assertEquals(2, values.get(\"x2\").length);\n fs.removeXAttr(p, \"x2\");\n names = fs.listXAttrs(p);\n assertEquals(1, names.size());\n assertEquals(\"x1\", names.get(0));\n\n // stress\n for (int i = 0; i < 100; i++) {\n fs.setXAttr(p, \"test\" + i, new byte[4096]);\n }\n values = fs.getXAttrs(p);\n assertEquals(101, values.size());\n // xattr should be remove together with file\n fs.delete(p);\n fs.create(p);\n names = fs.listXAttrs(p);\n assertEquals(0, names.size());\n }\n\n public void testAppend() throws Exception {\n Path f = new Path(\"/tmp/testappend\");\n fs.delete(f);\n FSDataOutputStream out = fs.create(f);\n out.write(\"hello\".getBytes());\n out.close();\n FSDataOutputStream append = fs.append(f);\n assertEquals(5, append.getPos());\n }\n\n public void testFlinkHadoopRecoverableWriter() throws Exception {\n new HadoopRecoverableWriter(fs);\n }\n\n public void testConcat() throws Exception {\n Path trg = new Path(\"/tmp/concat\");\n Path src1 = new Path(\"/tmp/concat1\");\n Path src2 = new Path(\"/tmp/concat2\");\n FSDataOutputStream ou = fs.create(trg);\n ou.write(\"hello\".getBytes());\n ou.close();\n FSDataOutputStream sou1 = fs.create(src1);\n sou1.write(\"hello\".getBytes());\n sou1.close();\n FSDataOutputStream sou2 = fs.create(src2);\n sou2.write(\"hello\".getBytes());\n sou2.close();\n fs.concat(trg, new Path[]{src1, src2});\n FSDataInputStream in = fs.open(trg);\n assertEquals(\"hellohellohello\", IOUtils.toString(in));\n in.close();\n // src should be deleted after concat\n assertFalse(fs.exists(src1));\n assertFalse(fs.exists(src2));\n\n Path emptyFile = new Path(\"/tmp/concat_empty_file\");\n Path src = new Path(\"/tmp/concat_empty_file_src\");\n FSDataOutputStream srcOu = fs.create(src);\n srcOu.write(\"hello\".getBytes());\n srcOu.close();\n fs.create(emptyFile).close();\n fs.concat(emptyFile, new Path[]{src});\n in = fs.open(emptyFile);\n assertEquals(\"hello\", IOUtils.toString(in));\n in.close();\n }\n\n public void testList() throws Exception {\n Path p = new Path(\"/listsort\");\n String[] org = new String[]{\n \"/listsort/p4\",\n \"/listsort/p2\",\n \"/listsort/p1\",\n \"/listsort/p3\"\n };\n fs.mkdirs(p);\n for (String path : org) {\n fs.mkdirs(new Path(path));\n }\n FileStatus[] fss = fs.listStatus(p);\n String[] res = new String[fss.length];\n for (int i = 0; i < fss.length; i++) {\n res[i] = fss[i].getPath().toUri().getPath();\n }\n Arrays.sort(org);\n assertArrayEquals(org, res);\n }\n\n private void writeFile(FileSystem fs, Path p, String content) throws IOException {\n FSDataOutputStream ou = fs.create(p);\n ou.write(content.getBytes());\n ou.close();\n }\n\n public FileSystem createNewFs(Configuration conf, String user, String[] group) throws IOException, InterruptedException {\n if (user != null && group != null) {\n UserGroupInformation root = UserGroupInformation.createUserForTesting(user, group);\n return root.doAs((PrivilegedExceptionAction) () -> FileSystem.newInstance(FileSystem.getDefaultUri(conf), conf));\n }\n return FileSystem.newInstance(FileSystem.getDefaultUri(conf), conf);\n }\n\n public void testUsersAndGroups() throws Exception {\n Path users1 = new Path(\"/tmp/users1\");\n Path groups1 = new Path(\"/tmp/groups1\");\n Path users2 = new Path(\"/tmp/users2\");\n Path groups2 = new Path(\"/tmp/groups2\");\n\n writeFile(fs, users1, \"user1:2001\\n\");\n writeFile(fs, groups1, \"group1:3001:user1\\n\");\n writeFile(fs, users2, \"user2:2001\\n\");\n writeFile(fs, groups2, \"group2:3001:user2\\n\");\n fs.close();\n\n Configuration conf = new Configuration(cfg);\n conf.set(\"juicefs.users\", users1.toUri().getPath());\n conf.set(\"juicefs.groups\", groups1.toUri().getPath());\n conf.set(\"juicefs.superuser\", UserGroupInformation.getCurrentUser().getShortUserName());\n\n FileSystem newFs = createNewFs(conf, null, null);\n Path p = new Path(\"/test_user_group_file\");\n newFs.create(p).close();\n newFs.setOwner(p, \"user1\", \"group1\");\n newFs.close();\n\n conf.set(\"juicefs.users\", users2.toUri().getPath());\n conf.set(\"juicefs.groups\", groups2.toUri().getPath());\n newFs = createNewFs(conf, null, null);\n FileStatus fileStatus = newFs.getFileStatus(p);\n assertEquals(\"user2\", fileStatus.getOwner());\n assertEquals(\"group2\", fileStatus.getGroup());\n newFs.close();\n }\n\n public void testGroupPerm() throws Exception {\n Path testPath = new Path(\"/test_group_perm\");\n\n Configuration conf = new Configuration(cfg);\n conf.set(\"juicefs.supergroup\", \"hadoop\");\n conf.set(\"juicefs.superuser\", \"hadoop\");\n FileSystem uer1Fs = createNewFs(conf, \"user1\", new String[]{\"hadoop\"});\n uer1Fs.delete(testPath, true);\n uer1Fs.mkdirs(testPath);\n uer1Fs.setPermission(testPath, FsPermission.createImmutable((short) 0775));\n uer1Fs.close();\n\n FileSystem uer2Fs = createNewFs(conf, \"user2\", new String[]{\"hadoop\"});\n Path f = new Path(testPath, \"test_file\");\n uer2Fs.create(f).close();\n FileStatus fileStatus = uer2Fs.getFileStatus(f);\n assertEquals(\"user2\", fileStatus.getOwner());\n uer2Fs.close();\n }\n\n public void testUmask() throws Exception {\n Configuration conf = new Configuration(cfg);\n conf.set(\"juicefs.umask\", \"077\");\n UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();\n FileSystem newFs = createNewFs(conf, currentUser.getShortUserName(), currentUser.getGroupNames());\n newFs.delete(new Path(\"/test_umask\"), true);\n newFs.mkdirs(new Path(\"/test_umask/dir\"));\n newFs.create(new Path(\"/test_umask/dir/f\")).close();\n assertEquals(FsPermission.createImmutable((short) 0700), newFs.getFileStatus(new Path(\"/test_umask\")).getPermission());\n assertEquals(FsPermission.createImmutable((short) 0700), newFs.getFileStatus(new Path(\"/test_umask/dir\")).getPermission());\n assertEquals(FsPermission.createImmutable((short) 0600), newFs.getFileStatus(new Path(\"/test_umask/dir/f\")).getPermission());\n newFs.close();\n\n conf.set(\"juicefs.umask\", \"000\");\n newFs = createNewFs(conf, currentUser.getShortUserName(), currentUser.getGroupNames());\n newFs.delete(new Path(\"/test_umask\"), true);\n newFs.mkdirs(new Path(\"/test_umask/dir\"));\n newFs.create(new Path(\"/test_umask/dir/f\")).close();\n assertEquals(FsPermission.createImmutable((short) 0777), newFs.getFileStatus(new Path(\"/test_umask\")).getPermission());\n assertEquals(FsPermission.createImmutable((short) 0777), newFs.getFileStatus(new Path(\"/test_umask/dir\")).getPermission());\n assertEquals(FsPermission.createImmutable((short) 0666), newFs.getFileStatus(new Path(\"/test_umask/dir/f\")).getPermission());\n\n conf.set(\"juicefs.umask\", \"022\");\n conf.set(\"fs.permissions.umask-mode\", \"077\");\n Path p = new Path(\"/test_umask/u_parent/f\");\n newFs = createNewFs(conf, currentUser.getShortUserName(), currentUser.getGroupNames());\n newFs.delete(p.getParent());\n FSDataOutputStream out = newFs.create(p, true);\n out.close();\n assertEquals(FsPermission.createImmutable((short) 0755), fs.getFileStatus(p.getParent()).getPermission());\n assertEquals(FsPermission.createImmutable((short) 0644), fs.getFileStatus(p).getPermission());\n\n newFs.close();\n }\n\n public void testGuidMapping() throws Exception {\n Configuration newConf = new Configuration(cfg);\n\n FSDataOutputStream ou = fs.create(new Path(\"/etc/users\"));\n ou.write(\"foo:10000\\n\".getBytes());\n ou.close();\n newConf.set(\"juicefs.users\", \"/etc/users\");\n\n FileSystem fooFs = createNewFs(newConf, \"foo\", new String[]{\"nogrp\"});\n Path f = new Path(\"/test_foo\");\n fooFs.create(f).close();\n assertEquals(\"foo\", fooFs.getFileStatus(f).getOwner());\n fooFs.close();\n\n ou = fs.create(new Path(\"/etc/users\"));\n ou.write(\"foo:10001\\n\".getBytes());\n ou.close();\n fs.close();\n\n FileSystem newFS = FileSystem.newInstance(newConf);\n assertEquals(\"10000\", newFS.getFileStatus(f).getOwner());\n newFS.delete(f, false);\n newFS.close();\n }\n\n public void testGuidMappingFromString() throws Exception {\n fs.close();\n Configuration newConf = new Configuration(cfg);\n\n newConf.set(\"juicefs.users\", \"bar:10000;foo:20000;baz:30000\");\n newConf.set(\"juicefs.groups\", \"user:10000:foo,bar;admin:2000:baz\");\n newConf.set(\"juicefs.superuser\", UserGroupInformation.getCurrentUser().getShortUserName());\n\n FileSystem fooFs = createNewFs(newConf, \"foo\", new String[]{\"nogrp\"});\n Path f = new Path(\"/test_foo\");\n fooFs.create(f).close();\n fooFs.setOwner(f, \"foo\", \"user\");\n assertEquals(\"foo\", fooFs.getFileStatus(f).getOwner());\n assertEquals(\"user\", fooFs.getFileStatus(f).getGroup());\n fooFs.close();\n\n newConf.set(\"juicefs.users\", \"foo:20001\");\n newConf.set(\"juicefs.groups\", \"user:1001:foo,bar;admin:2001:baz\");\n FileSystem newFS = FileSystem.newInstance(newConf);\n assertEquals(\"20000\", newFS.getFileStatus(f).getOwner());\n assertEquals(\"10000\", newFS.getFileStatus(f).getGroup());\n\n newFS.delete(f, false);\n newFS.close();\n }\n\n public void testTrash() throws Exception {\n Trash trash = new Trash(fs, cfg);\n Path trashFile = new Path(\"/tmp/trashfile\");\n trash.expungeImmediately();\n fs.create(trashFile).close();\n Trash.moveToAppropriateTrash(fs, trashFile, cfg);\n trash.checkpoint();\n fs.create(trashFile).close();\n Trash.moveToAppropriateTrash(fs, trashFile, cfg);\n assertEquals(2, fs.listStatus(fs.getTrashRoot(trashFile)).length);\n trash.expungeImmediately();\n assertEquals(0, fs.listStatus(fs.getTrashRoot(trashFile)).length);\n }\n\n public void testBlockSize() throws Exception {\n Configuration newConf = new Configuration(cfg);\n newConf.set(\"dfs.blocksize\", \"256m\");\n FileSystem newFs = FileSystem.newInstance(newConf);\n assertEquals(256 << 20, newFs.getDefaultBlockSize(new Path(\"/\")));\n }\n\n public void testReadSpeed() throws Exception {\n int read = (128 << 10) ;\n Path speedFile = new Path(\"/tmp/speedFile\");\n fs.delete(speedFile, false);\n FSDataOutputStream ou = fs.create(speedFile);\n int fileSize = 128 << 20;\n ou.write(new byte[fileSize]);\n ou.close();\n FSDataInputStream open = fs.open(speedFile);\n AtomicLong counter = new AtomicLong(0L);\n AtomicBoolean finished = new AtomicBoolean(false);\n TimerTask timerTask = new TimerTask() {\n @Override\n public void run() {\n System.out.printf(\"read method calls: %d\\n\", counter.get());\n finished.set(true);\n }\n };\n Timer timer = new Timer();\n timer.schedule(timerTask, 1000);\n\n ByteBuffer readArray = ByteBuffer.allocateDirect(read);\n while (!finished.get()) {\n open.seek(0);\n readArray.position(0);\n readArray.limit(read);\n open.read(readArray);\n counter.getAndIncrement();\n }\n }\n\n private void createFileWithContents(FileSystem fs, Path f, byte[] contents) throws IOException {\n try (FSDataOutputStream out = fs.create(f)) {\n if (contents != null) {\n out.write(contents);\n }\n }\n }\n\n public void testIOClosed() throws Exception {\n Path f = new Path(\"/tmp/closedFile\");\n FSDataOutputStream ou = fs.create(f);\n ou.close();\n try {\n ou.write(new byte[1]);\n fail(\"should not work when write to a closed stream\");\n } catch (IOException ignored) {\n }\n FSDataInputStream in = fs.open(f);\n in.close();\n try {\n in.read(new byte[1]);\n fail(\"should not work when read a closed stream\");\n } catch (IOException ignored) {\n }\n\n ou = fs.create(f);\n ou.close();\n ou.close();\n }\n\n public void testRead() throws Exception {\n Path f = new Path(\"/tmp/posFile\");\n int fileLen = 1 << 20;\n byte[] contents = new byte[fileLen];\n Random random = new Random();\n random.nextBytes(contents);\n createFileWithContents(fs, f, contents);\n FSDataInputStream in = fs.open(f);\n\n byte[] readBytes = new byte[fileLen];\n int got = in.read(readBytes);\n assertFalse(in.markSupported());\n assertEquals(fileLen, got);\n assertEquals(fileLen, in.getPos());\n assertArrayEquals(Arrays.copyOfRange(contents, 0, fileLen), readBytes);\n in.close();\n\n in = fs.open(f);\n int b = 0;\n int count = 0;\n while ((b = in.read()) != -1) {\n assertEquals(contents[count]&0xFF, b);\n count++;\n }\n assertEquals(fileLen, count);\n in.close();\n\n int readSize = 100;\n in = fs.open(f);\n got = in.read(new byte[readSize]);\n assertEquals(readSize, got);\n assertEquals(readSize, in.getPos());\n assertEquals(fileLen - readSize, in.available());\n in.close();\n\n in = fs.open(f);\n readBytes = new byte[128<<10];\n int off = 100;\n int len = 100;\n int read = in.read(readBytes, off, len);\n assertEquals(len, read);\n assertArrayEquals(Arrays.copyOfRange(contents, 0, len), Arrays.copyOfRange(readBytes, off, off + len));\n in.close();\n\n try {\n in = fs.open(f);\n in.read(readBytes, off, readBytes.length - off + 1);\n fail(\"IndexOutOfBoundsException\");\n } catch (IndexOutOfBoundsException ignored) {\n } finally {\n in.close();\n }\n\n in = fs.open(f);\n in.seek(fileLen - 100);\n long skip = in.skip(100);\n assertEquals(100, skip);\n\n in.seek(fileLen - 100);\n skip = in.skip(fileLen - 100 + 1);\n assertEquals(100, skip);\n in.close();\n }\n\n public void testInnerSymlink() throws Exception {\n //echo \"hello juicefs\" > inner_sym_link\n FileStatus status = fs.getFileStatus(new Path(\"/inner_sym_link\"));\n assertEquals(\"inner_sym_link\", status.getPath().getName());\n assertEquals(14, status.getLen());\n }\n\n public void testUserWithMultiGroups() throws Exception {\n Path users = new Path(\"/etc/users\");\n Path groups = new Path(\"/etc/groups_multi\");\n\n writeFile(fs, users, \"tom:2001\\n\");\n writeFile(fs, groups, \"groupa:3001:tom\\ngroupb:3002:tom\");\n fs.close();\n\n Configuration conf = new Configuration(cfg);\n conf.set(\"juicefs.users\", users.toUri().getPath());\n conf.set(\"juicefs.groups\", groups.toUri().getPath());\n conf.set(\"juicefs.debug\", \"true\");\n\n FileSystem superFs = createNewFs(conf, \"hdfs\", new String[]{\"hadoop\"});\n Path testDir = new Path(\"/test_multi_group/d1\");\n superFs.mkdirs(testDir);\n superFs.setOwner(testDir.getParent(), \"hdfs\", \"groupb\");\n superFs.setOwner(testDir, \"hdfs\", \"groupb\");\n superFs.setPermission(testDir.getParent(), FsPermission.createImmutable((short) 0770));\n superFs.setPermission(testDir, FsPermission.createImmutable((short) 0770));\n\n FileSystem tomFs = createNewFs(conf, \"tom\", new String[]{\"randgroup\"});\n tomFs.listStatus(testDir);\n\n superFs.delete(testDir.getParent(), true);\n tomFs.close();\n superFs.close();\n }\n\n public void testConcurrentCreate() throws Exception {\n int threads = 100;\n ExecutorService pool = Executors.newFixedThreadPool(threads);\n for (int i = 0; i < threads; i++) {\n pool.submit(() -> {\n JuiceFileSystem jfs = new JuiceFileSystem();\n try {\n jfs.initialize(URI.create(\"jfs://dev/\"), cfg);\n jfs.listStatus(new Path(\"/\"));\n jfs.close();\n } catch (IOException e) {\n fail(\"concurrent create failed\");\n System.exit(1);\n }\n });\n }\n pool.shutdown();\n pool.awaitTermination(1, TimeUnit.MINUTES);\n }\n\n private boolean tryAccess(Path path, String user, String[] group, FsAction action) throws Exception {\n UserGroupInformation testUser = UserGroupInformation.createUserForTesting(user, group);\n FileSystem fs = testUser.doAs((PrivilegedExceptionAction) () -> {\n Configuration conf = new Configuration();\n conf.set(\"juicefs.grouping\", \"\");\n return FileSystem.get(conf);\n });\n\n boolean canAccess;\n try {\n fs.access(path, action);\n canAccess = true;\n } catch (AccessControlException e) {\n canAccess = false;\n }\n return canAccess;\n }\n static AclEntry aclEntry(AclEntryScope scope, AclEntryType type, FsAction permission) {\n return new AclEntry.Builder().setScope(scope).setType(type).setPermission(permission).build();\n }\n\n static AclEntry aclEntry(AclEntryScope scope, AclEntryType type, String name, FsAction permission) {\n return new AclEntry.Builder().setScope(scope).setType(type).setName(name).setPermission(permission).build();\n }\n\n public void testAcl() throws Exception {\n List acls = Lists.newArrayList(\n aclEntry(DEFAULT, USER, \"foo\", ALL)\n );\n Path p = new Path(\"/testacldir\");\n fs.delete(p, true);\n fs.mkdirs(p);\n fs.setAcl(p, acls);\n Path childFile = new Path(p, \"file\");\n fs.create(childFile).close();\n assertTrue(tryAccess(childFile, \"foo\", new String[]{\"nogrp\"}, WRITE));\n assertFalse(tryAccess(childFile, \"wrong\", new String[]{\"nogrp\"}, WRITE));\n assertEquals(fs.getFileStatus(childFile).getPermission().getGroupAction(), READ_WRITE);\n\n Path childDir = new Path(p, \"dir\");\n fs.mkdirs(childDir);\n assertEquals(fs.getFileStatus(childDir).getPermission().getGroupAction(), ALL);\n }\n\n public void testAclException() throws Exception {\n List acls = Lists.newArrayList(\n aclEntry(ACCESS, USER, \"foo\", ALL)\n );\n Path p = new Path(\"/test_acl_exception\");\n fs.delete(p, true);\n fs.mkdirs(p);\n try {\n fs.setAcl(p, acls);\n fail(\"Invalid ACL: the user, group and other entries are required.\");\n } catch (AclTransformation.AclException ignored) {\n }\n }\n\n public void testDefaultAclExistingDirFile() throws Exception {\n Path parent = new Path(\"/testDefaultAclExistingDirFile\");\n fs.delete(parent, true);\n fs.mkdirs(parent);\n // the old acls\n List acls1 = Lists.newArrayList(aclEntry(DEFAULT, USER, \"foo\", ALL));\n // the new acls\n List acls2 = Lists.newArrayList(aclEntry(DEFAULT, USER, \"foo\", READ_EXECUTE));\n // set parent to old acl\n fs.setAcl(parent, acls1);\n\n Path childDir = new Path(parent, \"childDir\");\n fs.mkdirs(childDir);\n // the sub directory should also have the old acl\n AclEntry[] childDirExpectedAcl = new AclEntry[] { aclEntry(ACCESS, USER, \"foo\", ALL),\n aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(DEFAULT, USER, ALL),\n aclEntry(DEFAULT, USER, \"foo\", ALL), aclEntry(DEFAULT, GROUP, READ_EXECUTE),\n aclEntry(DEFAULT, MASK, ALL), aclEntry(DEFAULT, OTHER, READ_EXECUTE) };\n AclStatus childDirAcl = fs.getAclStatus(childDir);\n assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());\n\n Path childFile = new Path(childDir, \"childFile\");\n // the sub file should also have the old acl\n fs.create(childFile).close();\n AclEntry[] childFileExpectedAcl = new AclEntry[] { aclEntry(ACCESS, USER, \"foo\", ALL),\n aclEntry(ACCESS, GROUP, READ_EXECUTE) };\n AclStatus childFileAcl = fs.getAclStatus(childFile);\n assertArrayEquals(childFileExpectedAcl, childFileAcl.getEntries().toArray());\n\n // now change parent to new acls\n fs.setAcl(parent, acls2);\n\n // sub directory and sub file should still have the old acls\n childDirAcl = fs.getAclStatus(childDir);\n assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());\n childFileAcl = fs.getAclStatus(childFile);\n assertArrayEquals(childFileExpectedAcl, childFileAcl.getEntries().toArray());\n\n // now remove the parent acls\n fs.removeAcl(parent);\n\n // sub directory and sub file should still have the old acls\n childDirAcl = fs.getAclStatus(childDir);\n assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());\n childFileAcl = fs.getAclStatus(childFile);\n assertArrayEquals(childFileExpectedAcl, childFileAcl.getEntries().toArray());\n\n // check changing the access mode of the file\n // mask out the access of group other for testing\n fs.setPermission(childFile, new FsPermission((short) 0640));\n boolean canAccess = tryAccess(childFile, \"other\", new String[] { \"other\" }, READ);\n assertFalse(canAccess);\n fs.delete(parent, true);\n }\n\n public void testAccessAclNotInherited() throws IOException {\n Path parent = new Path(\"/testAccessAclNotInherited\");\n fs.delete(parent, true);\n fs.mkdirs(parent);\n // parent have both access acl and default acl\n List acls = Lists.newArrayList(aclEntry(DEFAULT, USER, \"foo\", READ_EXECUTE),\n aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, GROUP, READ), aclEntry(ACCESS, OTHER, READ),\n aclEntry(ACCESS, USER, \"bar\", ALL));\n fs.setAcl(parent, acls);\n AclEntry[] expectedAcl = new AclEntry[] { aclEntry(ACCESS, USER, \"bar\", ALL), aclEntry(ACCESS, GROUP, READ),\n aclEntry(DEFAULT, USER, ALL), aclEntry(DEFAULT, USER, \"foo\", READ_EXECUTE),\n aclEntry(DEFAULT, GROUP, READ), aclEntry(DEFAULT, MASK, READ_EXECUTE), aclEntry(DEFAULT, OTHER, READ) };\n AclStatus dirAcl = fs.getAclStatus(parent);\n assertArrayEquals(expectedAcl, dirAcl.getEntries().toArray());\n\n Path childDir = new Path(parent, \"childDir\");\n fs.mkdirs(childDir);\n // subdirectory should only have the default acl inherited\n AclEntry[] childDirExpectedAcl = new AclEntry[] { aclEntry(ACCESS, USER, \"foo\", READ_EXECUTE),\n aclEntry(ACCESS, GROUP, READ), aclEntry(DEFAULT, USER, ALL),\n aclEntry(DEFAULT, USER, \"foo\", READ_EXECUTE), aclEntry(DEFAULT, GROUP, READ),\n aclEntry(DEFAULT, MASK, READ_EXECUTE), aclEntry(DEFAULT, OTHER, READ) };\n AclStatus childDirAcl = fs.getAclStatus(childDir);\n assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());\n\n Path childFile = new Path(parent, \"childFile\");\n fs.create(childFile).close();\n // sub file should only have the default acl inherited\n AclEntry[] childFileExpectedAcl = new AclEntry[] { aclEntry(ACCESS, USER, \"foo\", READ_EXECUTE),\n aclEntry(ACCESS, GROUP, READ) };\n AclStatus childFileAcl = fs.getAclStatus(childFile);\n assertArrayEquals(childFileExpectedAcl, childFileAcl.getEntries().toArray());\n\n fs.delete(parent, true);\n }\n\n public void testFileStatusWithAcl() throws Exception {\n List acls = Lists.newArrayList(\n aclEntry(ACCESS, USER, ALL),\n aclEntry(ACCESS, USER, \"foo\", ALL),\n aclEntry(ACCESS, OTHER, ALL),\n aclEntry(ACCESS, GROUP, ALL)\n );\n Path p = new Path(\"/test_acl_status\");\n fs.delete(p, true);\n fs.mkdirs(p);\n FileStatus pStatus = fs.getFileStatus(p);\n assertFalse(pStatus.hasAcl());\n\n Path f = new Path(p, \"f\");\n fs.create(f).close();\n fs.setAcl(f, acls);\n FileStatus[] fileStatuses = fs.listStatus(p);\n assertTrue(fileStatuses[0].getPermission().getAclBit());\n assertTrue(fileStatuses[0].hasAcl());\n }\n\n public void testRenameAccessControlException() throws Exception {\n Path d1 = new Path(\"/renameAccessControlExceptionDir1\");\n Path d2 = new Path(\"/renameAccessControlExceptionDir2\");\n Path p = new Path(d1, \"file\");\n FileSystem user1Fs = createNewFs(cfg, \"user1\", new String[]{\"group1\"});\n\n user1Fs.mkdirs(d1);\n user1Fs.mkdirs(d2);\n user1Fs.create(p).close();\n user1Fs.setPermission(d1, new FsPermission((short) 0000));\n user1Fs.setPermission(d2, new FsPermission((short) 0777));\n try {\n user1Fs.rename(p, d2);\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"renameAccessControlExceptionDir1\"));\n }\n\n user1Fs.setPermission(d1, new FsPermission((short) 0777));\n user1Fs.setPermission(d2, new FsPermission((short) 000));\n try {\n user1Fs.rename(p, d2);\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"renameAccessControlExceptionDir2\"));\n }\n\n // clean\n user1Fs.setPermission(d1, new FsPermission((short) 0777));\n user1Fs.setPermission(d2, new FsPermission((short) 0777));\n user1Fs.delete(d1, true);\n user1Fs.delete(d2, true);\n }\n\n public void testSubdir() throws IOException, InterruptedException {\n Configuration newConf = new Configuration(cfg);\n newConf.set(\"fs.defaultFS\", \"jfs://test/\");\n newConf.set(\"juicefs.name\", \"test\");\n newConf.set(\"juicefs.test.meta\", newConf.get(\"juicefs.dev.meta\"));\n\n // Test creating a new filesystem with a valid subdir\n Path subdir = new Path(\"/test_subdir\");\n fs.delete(subdir, true);\n fs.mkdirs(subdir);\n fs.setPermission(subdir, new FsPermission((short) 0777));\n newConf.set(\"juicefs.subdir\", \"/test_subdir\");\n FileSystem newFS = FileSystem.newInstance(newConf);\n\n // Test file operations within the subdir\n assertTrue(newFS.mkdirs(new Path(\"/test_subdir/dir\")));\n newFS.create(new Path(\"/test_subdir/dir/f\")).close();\n assertTrue(newFS.exists(new Path(\"/test_subdir/dir/f\")));\n\n // Test file operations not within the subdir\n Path nonexistent = new Path(\"/nonexistent\");\n try {\n newFS.exists(nonexistent);\n fail(\"exists should not work because the path is not under the subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n try {\n newFS.mkdirs(nonexistent);\n fail(\"mkdirs should not work because the path is not under the subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n try {\n newFS.create(nonexistent);\n fail(\"create should not work because the path is not under the subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n\n // Test creating a path with the same prefix but not under the subdir\n Path wrongPathWithSamePrefix = new Path(\"/test_subdir_wrong\");\n fs.mkdirs(wrongPathWithSamePrefix);\n try {\n newFS.listStatus(wrongPathWithSamePrefix);\n fail(\"listStatus should not work because the path is not under the subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n newFS.close();\n }\n\n public void testMultipleSubdirs() throws IOException, InterruptedException {\n Configuration newConf = new Configuration(cfg);\n newConf.set(\"fs.defaultFS\", \"jfs://test/\");\n newConf.set(\"juicefs.name\", \"test\");\n newConf.set(\"juicefs.test.meta\", newConf.get(\"juicefs.dev.meta\"));\n\n // Create multiple subdirs\n Path subdir1 = new Path(\"/subdir1\");\n Path subdir2 = new Path(\"/subdir2\");\n Path subdir3 = new Path(\"/subdir3\");\n \n fs.delete(subdir1, true);\n fs.delete(subdir2, true);\n fs.delete(subdir3, true);\n \n fs.mkdirs(subdir1);\n fs.mkdirs(subdir2);\n fs.mkdirs(subdir3);\n fs.setPermission(subdir1, new FsPermission((short) 0777));\n fs.setPermission(subdir2, new FsPermission((short) 0777));\n fs.setPermission(subdir3, new FsPermission((short) 0777));\n \n // Set multiple subdirs separated by comma\n newConf.set(\"juicefs.subdir\", \"/subdir1,/subdir2,/subdir3\");\n FileSystem newFS = FileSystem.newInstance(newConf);\n\n // Test file operations within subdir1\n assertTrue(newFS.mkdirs(new Path(\"/subdir1/dir1\")));\n newFS.create(new Path(\"/subdir1/dir1/f1\")).close();\n assertTrue(newFS.exists(new Path(\"/subdir1/dir1/f1\")));\n\n // Test file operations within subdir2\n assertTrue(newFS.mkdirs(new Path(\"/subdir2/dir2\")));\n newFS.create(new Path(\"/subdir2/dir2/f2\")).close();\n assertTrue(newFS.exists(new Path(\"/subdir2/dir2/f2\")));\n\n // Test file operations within subdir3\n assertTrue(newFS.mkdirs(new Path(\"/subdir3/dir3\")));\n newFS.create(new Path(\"/subdir3/dir3/f3\")).close();\n assertTrue(newFS.exists(new Path(\"/subdir3/dir3/f3\")));\n\n // Test file operations not within any subdir\n Path nonexistent = new Path(\"/nonexistent\");\n try {\n newFS.exists(nonexistent);\n fail(\"exists should not work because the path is not under any subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n try {\n newFS.mkdirs(nonexistent);\n fail(\"mkdirs should not work because the path is not under any subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n try {\n newFS.create(nonexistent);\n fail(\"create should not work because the path is not under any subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n\n // Test creating a path with the same prefix but not under any subdir\n Path wrongPathWithSamePrefix = new Path(\"/subdir1_wrong\");\n fs.mkdirs(wrongPathWithSamePrefix);\n try {\n newFS.listStatus(wrongPathWithSamePrefix);\n fail(\"listStatus should not work because the path is not under any subdir\");\n } catch (AccessControlException e) {\n assertTrue(e.getMessage().contains(\"Permission denied\"));\n }\n\n // Test that paths in different subdirs are accessible\n assertTrue(newFS.exists(new Path(\"/subdir1/dir1/f1\")));\n assertTrue(newFS.exists(new Path(\"/subdir2/dir2/f2\")));\n assertTrue(newFS.exists(new Path(\"/subdir3/dir3/f3\")));\n\n // Cleanup\n newFS.close();\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/acl/TestAclCLI.java", "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *

\n * http://www.apache.org/licenses/LICENSE-2.0\n *

\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.acl;\n\nimport org.apache.hadoop.cli.CLITestHelperDFS;\nimport org.apache.hadoop.cli.util.CLICommand;\nimport org.apache.hadoop.cli.util.CommandExecutor.Result;\nimport org.apache.hadoop.hdfs.DFSConfigKeys;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class TestAclCLI extends CLITestHelperDFS {\n private String vol = null;\n private String username = null;\n\n protected void initConf() {\n conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);\n conf.setBoolean(\n DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY, false);\n }\n\n @Before\n @Override\n public void setUp() throws Exception {\n super.setUp();\n initConf();\n vol = \"jfs://dev/\";\n username = System.getProperty(\"user.name\");\n }\n\n @After\n @Override\n public void tearDown() throws Exception {\n super.tearDown();\n }\n\n @Override\n protected String getTestFile() {\n return \"testAclCLI.xml\";\n }\n\n @Override\n protected String expandCommand(final String cmd) {\n String expCmd = cmd;\n expCmd = expCmd.replaceAll(\"NAMENODE\", vol);\n expCmd = expCmd.replaceAll(\"USERNAME\", username);\n expCmd = expCmd.replaceAll(\"#LF#\",\n System.getProperty(\"line.separator\"));\n expCmd = super.expandCommand(expCmd);\n return expCmd;\n }\n\n @Override\n protected Result execute(CLICommand cmd) throws Exception {\n return cmd.getExecutor(vol, conf).executeCommand(cmd.getCmd());\n }\n\n @Test\n @Override\n public void testAll() {\n super.testAll();\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/JuiceFSContract.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractBondedFSContract;\n\npublic class JuiceFSContract extends AbstractBondedFSContract {\n\n public JuiceFSContract(Configuration conf) {\n super(conf);\n addConfResource(\"contract/juicefs.xml\");\n }\n\n @Override\n public String getScheme() {\n return \"jfs\";\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestAppend.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.fs.contract.AbstractContractAppendTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\n\npublic class TestAppend extends AbstractContractAppendTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n\n @Override\n public void teardown() throws Exception {\n getFileSystem().delete(new Path(path(\"test\"), \"target\"));\n super.teardown();\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestConcat.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractConcatTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\npublic class TestConcat extends AbstractContractConcatTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestCreate.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractCreateTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\npublic class TestCreate extends AbstractContractCreateTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestDelete.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractDeleteTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\npublic class TestDelete extends AbstractContractDeleteTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestGetFileStatus.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\n\npublic class TestGetFileStatus extends AbstractContractGetFileStatusTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n\n @Override\n public void setup() throws Exception {\n super.setup();\n getFileSystem().delete(new Path(\"jfs:///test\"));\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestJuiceFileSystemContract.java", "content": "/*\n * JuiceFS, Copyright 2021 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport io.juicefs.JuiceFileSystemTest;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.*;\nimport org.apache.hadoop.fs.permission.FsPermission;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.io.IOException;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assume.assumeNotNull;\n\npublic class TestJuiceFileSystemContract extends FileSystemContractBaseTest {\n @Before\n public void setUp() throws Exception {\n Configuration cfg = new Configuration();\n cfg.addResource(JuiceFileSystemTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n fs = FileSystem.get(cfg);\n assumeNotNull(fs);\n }\n\n public FileSystem createNewFs(Configuration conf) throws IOException {\n return FileSystem.newInstance(FileSystem.getDefaultUri(conf), conf);\n }\n\n @Test\n public void testMkdirsWithUmask() throws Exception {\n Configuration conf = new Configuration(fs.getConf());\n conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, TEST_UMASK);\n FileSystem newFs = createNewFs(conf);\n try {\n final Path dir = path(\"newDir\");\n assertTrue(newFs.mkdirs(dir, new FsPermission((short) 0777)));\n FileStatus status = newFs.getFileStatus(dir);\n assertTrue(status.isDirectory());\n assertEquals((short) 0715, status.getPermission().toShort());\n } finally {\n newFs.close();\n }\n }\n}" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestMkdir.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractMkdirTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\npublic class TestMkdir extends AbstractContractMkdirTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestOpen.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractOpenTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\npublic class TestOpen extends AbstractContractOpenTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestRename.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractRenameTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\n\npublic class TestRename extends AbstractContractRenameTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestSeek.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractSeekTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\npublic class TestSeek extends AbstractContractSeekTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n\n @Override\n public void teardown() throws Exception {\n getFileSystem().delete(path(\"bigseekfile.txt\"));\n super.teardown();\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/contract/TestSetTimes.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.contract;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.contract.AbstractContractSetTimesTest;\nimport org.apache.hadoop.fs.contract.AbstractFSContract;\n\npublic class TestSetTimes extends AbstractContractSetTimesTest {\n @Override\n protected AbstractFSContract createContract(Configuration conf) {\n return new JuiceFSContract(conf);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/kerberos/KerberosTest.java", "content": "/*\n * JuiceFS, Copyright 2025 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.juicefs.kerberos;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileStatus;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.security.UserGroupInformation;\nimport org.apache.hadoop.security.token.Token;\nimport org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;\nimport org.junit.Test;\n\nimport java.io.IOException;\nimport java.security.PrivilegedExceptionAction;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.fail;\n\npublic class KerberosTest {\n private static final String clientPrincipal = \"client/localhost\";\n private static final String clientKeytab = \"/tmp/client.keytab\";\n private static final String tomPrincipal = \"tom/localhost\";\n private static final String tomKeytab = \"/tmp/tom.keytab\";\n\n private static final String jerryPrincipal = \"jerry/localhost\";\n private static final String jerryKeytab = \"/tmp/jerry.keytab\";\n private static final String serverPrincipal = \"server/localhost\";\n\n\n @Test\n public void testWithoutKrb() throws Exception {\n UserGroupInformation.reset();\n Configuration cfg = new Configuration();\n cfg.set(\"juicefs.memory-size\", \"99\"); // to new another jfs\n cfg.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n try (FileSystem fs = FileSystem.newInstance(cfg)) {\n fail(\"should not success without kerberos login\");\n } catch (IOException ignored) {\n }\n UserGroupInformation.reset();\n }\n\n @Test\n public void test() throws Exception {\n UserGroupInformation.reset();\n Configuration cfg = new Configuration();\n cfg.set(\"hadoop.security.authentication\", \"kerberos\");\n cfg.set(\"juicefs.server-principal\", serverPrincipal);\n cfg.set(\"juicefs.memory-size\", \"100\"); // to new another jfs\n UserGroupInformation.setConfiguration(cfg);\n UserGroupInformation.loginUserFromKeytab(clientPrincipal, clientKeytab);\n cfg.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n FileSystem fs = FileSystem.newInstance(cfg);\n fs.listStatus(new Path(\"/\"));\n UserGroupInformation.reset();\n fs.close();\n }\n\n @Test\n public void testToken() throws Exception {\n UserGroupInformation.reset();\n Configuration cfg = new Configuration();\n cfg.set(\"hadoop.security.authentication\", \"kerberos\");\n cfg.set(\"juicefs.server-principal\", serverPrincipal);\n cfg.set(\"juicefs.memory-size\", \"101\"); // to new another jfs\n UserGroupInformation.setConfiguration(cfg);\n UserGroupInformation.loginUserFromKeytab(clientPrincipal, clientKeytab);\n cfg.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n FileSystem fs = FileSystem.newInstance(cfg);\n long start = System.currentTimeMillis();\n Token t = fs.getDelegationToken(UserGroupInformation.getCurrentUser().getShortUserName());\n long end = System.currentTimeMillis();\n System.out.println(\"get token time: \" + (end - start) + \" ms\");\n\n // token renewer\n JuiceFSTokenRenewer renewer = new JuiceFSTokenRenewer();\n start = System.currentTimeMillis();\n System.out.println(renewer.renew(t, cfg));\n AbstractDelegationTokenIdentifier identifier = (AbstractDelegationTokenIdentifier) t.decodeIdentifier();\n System.out.println(\"token id: \" + identifier.getMasterKeyId());\n end = System.currentTimeMillis();\n System.out.println(\"renew token time: \" + (end - start) + \" ms\");\n start = System.currentTimeMillis();\n renewer.cancel(t, cfg);\n end = System.currentTimeMillis();\n System.out.println(\"cancel token time: \" + (end - start) + \" ms\");\n UserGroupInformation.reset();\n fs.close();\n }\n\n @Test\n public void testProxyUser() throws Exception {\n UserGroupInformation.reset();\n Configuration cfg = new Configuration();\n cfg.set(\"hadoop.security.authentication\", \"kerberos\");\n cfg.set(\"juicefs.server-principal\", serverPrincipal);\n UserGroupInformation.setConfiguration(cfg);\n UserGroupInformation.loginUserFromKeytab(clientPrincipal, clientKeytab);\n cfg.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n UserGroupInformation realUser = UserGroupInformation.getCurrentUser();\n UserGroupInformation foo = UserGroupInformation.createProxyUser(\"foo\", realUser);\n foo.doAs(new PrivilegedExceptionAction() {\n @Override\n public Object run() throws Exception {\n cfg.set(\"juicefs.memory-size\", \"102\"); // to new another jfs\n FileSystem fs = FileSystem.newInstance(cfg);\n fs.close();\n return null;\n }\n });\n\n UserGroupInformation bar = UserGroupInformation.createProxyUser(\"bar\", realUser);\n bar.doAs(new PrivilegedExceptionAction() {\n @Override\n public Object run() throws Exception {\n try {\n cfg.set(\"juicefs.memory-size\", \"103\"); // to new another jfs\n FileSystem fs = FileSystem.newInstance(cfg);\n fail(\"user bar should not proxyed\");\n } catch (Exception ignored){\n }\n return null;\n }\n });\n }\n\n @Test\n public void testSuperUser() throws Exception {\n UserGroupInformation.reset();\n Configuration cfg = new Configuration();\n cfg.set(\"hadoop.security.authentication\", \"kerberos\");\n cfg.set(\"juicefs.server-principal\", serverPrincipal);\n cfg.set(\"juicefs.memory-size\", \"104\"); // to new another jfs\n cfg.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n\n UserGroupInformation.setConfiguration(cfg);\n UserGroupInformation.loginUserFromKeytab(clientPrincipal, clientKeytab);\n FileSystem fs = FileSystem.newInstance(cfg);\n Path dir = new Path(\"/testsuperuser\");\n fs.delete(dir);\n fs.mkdirs(dir);\n fs.setOwner(dir, \"foo\", \"foo\"); // only superuser has permission\n }\n\n @Test\n public void testMapRule() throws Exception {\n UserGroupInformation.reset();\n Configuration cfg = new Configuration();\n cfg.set(\"hadoop.security.authentication\", \"kerberos\");\n cfg.set(\"juicefs.server-principal\", serverPrincipal);\n cfg.set(\"juicefs.memory-size\", \"105\"); // to new another jfs\n cfg.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n cfg.set(\"hadoop.security.auth_to_local\", \"RULE:[2:$1/$2@$0](jerry/.*@EXAMPLE\\\\.COM)s/.*/jerry_map/\\nDEFAULT\");\n\n UserGroupInformation.setConfiguration(cfg);\n UserGroupInformation.loginUserFromKeytab(jerryPrincipal, jerryKeytab);\n FileSystem fs = FileSystem.newInstance(cfg);\n Path dir = new Path(\"/testAuthToLocal\");\n fs.delete(dir);\n fs.mkdirs(dir);\n FileStatus[] statuses = fs.listStatus(new Path(\"/\"));\n assertEquals(\"jerry_map\", fs.getFileStatus(dir).getOwner());\n fs.close();\n }\n\n @Test\n public void testMapRuleWithProxyUser() throws Exception {\n // test for proxy user\n UserGroupInformation.reset();\n Configuration cfg = new Configuration();\n cfg.set(\"hadoop.security.authentication\", \"kerberos\");\n cfg.set(\"juicefs.server-principal\", serverPrincipal);\n cfg.set(\"juicefs.memory-size\", \"106\"); // to new another jfs\n cfg.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n // map tom to client\n cfg.set(\"hadoop.security.auth_to_local\", \"RULE:[2:$1/$2@$0](tom/.*@EXAMPLE\\\\.COM)s/.*/client/\\nDEFAULT\");\n UserGroupInformation.setConfiguration(cfg);\n UserGroupInformation.loginUserFromKeytab(tomPrincipal, tomKeytab);\n UserGroupInformation foo = UserGroupInformation.createProxyUser(\"foo\", UserGroupInformation.getCurrentUser());\n foo.doAs((PrivilegedExceptionAction) () -> {\n FileSystem fs = FileSystem.newInstance(cfg);\n Path dir = new Path(\"/testAuthToLocalWithProxyUser\");\n fs.delete(dir);\n fs.mkdirs(dir);\n FileStatus[] statuses = fs.listStatus(new Path(\"/\"));\n for (FileStatus status : statuses) {\n System.out.println(status.getPath().toString() + \" \" + status.getOwner() + \" \" + status.getGroup());\n }\n assertEquals(\"foo\", fs.getFileStatus(dir).getOwner());\n fs.close();\n return null;\n });\n\n // test for proxy user\n UserGroupInformation.reset();\n Configuration cfg2 = new Configuration();\n cfg2.set(\"hadoop.security.authentication\", \"kerberos\");\n cfg2.set(\"juicefs.server-principal\", serverPrincipal);\n cfg.set(\"juicefs.memory-size\", \"107\"); // to new another jfs\n cfg2.addResource(KerberosTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n // map tom to client\n cfg2.set(\"hadoop.security.auth_to_local\", \"RULE:[2:$1/$2@$0](tom/.*@EXAMPLE\\\\.COM)s/.*/client/\\nDEFAULT\");\n UserGroupInformation.setConfiguration(cfg2);\n UserGroupInformation.loginUserFromKeytab(tomPrincipal, tomKeytab);\n UserGroupInformation bar = UserGroupInformation.createProxyUser(\"bar\", UserGroupInformation.getCurrentUser());\n bar.doAs((PrivilegedExceptionAction) () -> {\n try {\n FileSystem fs = FileSystem.newInstance(cfg2);\n fs.close();\n fail(\"user client should not proxy bar\");\n } catch (Exception ignored) {\n }\n return null;\n });\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/permission/RangerAdminClientImpl.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.permission;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.ranger.admin.client.AbstractRangerAdminClient;\nimport org.apache.ranger.plugin.util.ServicePolicies;\nimport org.apache.ranger.plugin.util.ServiceTags;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.nio.file.FileSystems;\nimport java.nio.file.Files;\nimport java.util.List;\n\npublic class RangerAdminClientImpl extends AbstractRangerAdminClient {\n\n private static final Logger LOG = LoggerFactory.getLogger(RangerAdminClientImpl.class);\n\n private final static String cacheFilename = \"hdfs-policies.json\";\n private final static String tagFilename = \"hdfs-policies-tag.json\";\n public void init(String serviceName, String appId, String configPropertyPrefix, Configuration config) {\n super.init(serviceName, appId, configPropertyPrefix, config);\n }\n\n public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception {\n\n String basedir = System.getProperty(\"basedir\");\n if (basedir == null) {\n basedir = new File(\".\").getCanonicalPath();\n }\n final String relativePath = \"/src/test/resources/\";\n java.nio.file.Path cachePath = FileSystems.getDefault().getPath(basedir, relativePath + cacheFilename);\n byte[] cacheBytes = Files.readAllBytes(cachePath);\n return gson.fromJson(new String(cacheBytes), ServicePolicies.class);\n }\n\n public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception {\n String basedir = System.getProperty(\"basedir\");\n if (basedir == null) {\n basedir = new File(\".\").getCanonicalPath();\n }\n final String relativePath = \"/src/test/resources/\";\n java.nio.file.Path cachePath = FileSystems.getDefault().getPath(basedir, relativePath + tagFilename);\n byte[] cacheBytes = Files.readAllBytes(cachePath);\n return gson.fromJson(new String(cacheBytes), ServiceTags.class);\n }\n\n public List getTagTypes(String tagTypePattern) throws Exception {\n return null;\n }\n\n\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/permission/RangerPermissionCheckerTest.java", "content": "/*\n * JuiceFS, Copyright 2024 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\npackage io.juicefs.permission;\n\nimport io.juicefs.JuiceFileSystemTest;\nimport junit.framework.TestCase;\nimport org.apache.commons.io.IOUtils;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.*;\nimport org.apache.hadoop.fs.permission.FsAction;\nimport org.apache.hadoop.fs.permission.FsPermission;\nimport org.apache.hadoop.security.AccessControlException;\nimport org.apache.hadoop.security.UserGroupInformation;\nimport org.junit.Assert;\n\nimport java.io.ByteArrayOutputStream;\nimport java.security.PrivilegedExceptionAction;\n\npublic class RangerPermissionCheckerTest extends TestCase {\n\n private FileSystem fs;\n private Configuration cfg;\n\n public void setUp() throws Exception {\n cfg = new Configuration();\n cfg.addResource(JuiceFileSystemTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n // set superuser\n cfg.set(\"juicefs.superuser\", UserGroupInformation.getCurrentUser().getShortUserName());\n fs = FileSystem.newInstance(cfg);\n cfg.setQuietMode(false);\n }\n\n public void tearDown() throws Exception {\n fs.close();\n }\n\n public void testRangerCheckerInitFailed() throws Exception {\n Configuration cfg1 = new Configuration();\n cfg1.addResource(JuiceFileSystemTest.class.getClassLoader().getResourceAsStream(\"core-site.xml\"));\n cfg1.set(\"juicefs.superuser\", UserGroupInformation.getCurrentUser().getShortUserName());\n cfg1.setQuietMode(false);\n\n FileSystem fs1 = FileSystem.newInstance(cfg1);\n\n final Path file = new Path(\"/tmp/tmpdir/data-file2\");\n FSDataOutputStream out = fs1.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n\n fs1.setPermission(file, new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE));\n\n // Now try to read the file as unknown user \"bob\" - ranger should allow this user, but now should not be allowed\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"bob\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg1);\n try {\n fs.open(file);\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n\n fs.close();\n return null;\n }\n });\n\n fs1.delete(file);\n fs1.close();\n }\n\n public void testRead() throws Exception {\n HDFSReadTest(\"/tmp/tmpdir/data-file2\");\n }\n\n public void testWrite() throws Exception {\n\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n final Path file = new Path(\"/tmp/tmpdir2/data-file3\");\n FSDataOutputStream out = fs.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n\n fs.setPermission(file, new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, FsAction.NONE));\n\n // Now try to write to the file as \"bob\" - this should be allowed (by the policy - user)\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"bob\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Write to the file\n fs.append(file);\n fs.close();\n return null;\n }\n });\n\n // Now try to write to the file as \"alice\" - this should be allowed (by the policy - group)\n ugi = UserGroupInformation.createUserForTesting(\"alice\", new String[]{\"IT\"});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Write to the file\n fs.append(file);\n fs.close();\n return null;\n }\n });\n\n // Now try to read the file as unknown user \"eve\" - this should not be allowed\n ugi = UserGroupInformation.createUserForTesting(\"eve\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Write to the file\n try {\n fs.append(file);\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n // expected\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n fs.close();\n return null;\n }\n });\n\n fs.delete(file);\n }\n\n public void testExecute() throws Exception {\n\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n final Path file = new Path(\"/tmp/tmpdir3/data-file2\");\n FSDataOutputStream out = fs.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n\n fs.setPermission(file, new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE));\n\n Path parentDir = new Path(\"/tmp/tmpdir3\");\n\n fs.setPermission(parentDir, new FsPermission(FsAction.ALL, FsAction.READ_EXECUTE, FsAction.NONE));\n\n\n // Try to read the directory as \"bob\" - this should be allowed (by the policy - user)\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"bob\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n RemoteIterator iter = fs.listFiles(file.getParent(), false);\n Assert.assertTrue(iter.hasNext());\n\n fs.close();\n return null;\n }\n });\n // Try to read the directory as \"alice\" - this should be allowed (by the policy - group)\n ugi = UserGroupInformation.createUserForTesting(\"alice\", new String[]{\"IT\"});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n RemoteIterator iter = fs.listFiles(file.getParent(), false);\n Assert.assertTrue(iter.hasNext());\n fs.close();\n return null;\n }\n });\n\n // Now try to read the directory as unknown user \"eve\" - this should not be allowed\n ugi = UserGroupInformation.createUserForTesting(\"eve\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n try {\n RemoteIterator iter = fs.listFiles(file.getParent(), false);\n Assert.assertTrue(iter.hasNext());\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n\n fs.close();\n return null;\n }\n });\n\n fs.delete(file);\n fs.delete(parentDir);\n }\n\n public void testSetPermission() throws Exception {\n\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n final Path file = new Path(\"/tmp/tmpdir123/data-file3\");\n FSDataOutputStream out = fs.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n\n // Now try to read the file as unknown user \"eve\" - this will not find in ranger, and fallback check by origin Mask which should fail\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"eve\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Write to the file\n try {\n fs.setPermission(file, new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n // expected\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n fs.close();\n return null;\n }\n });\n\n fs.delete(file);\n }\n\n public void testSetOwner() throws Exception {\n\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n final Path file = new Path(\"/tmp/tmpdir123/data-file3\");\n FSDataOutputStream out = fs.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n\n // Now try to read the file as unknown user \"eve\" - this will not find in ranger, and fallback check by origin Mask which should fail\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"eve\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Write to the file\n try {\n fs.setOwner(file, \"eve\", \"eve\");\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n // expected\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n fs.close();\n return null;\n }\n });\n\n fs.delete(file);\n }\n\n public void testReadTestUsingTagPolicy() throws Exception {\n\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n final Path file = new Path(\"/tmp/tmpdir6/data-file2\");\n FSDataOutputStream out = fs.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n\n fs.setPermission(file, new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE));\n\n // Now try to read the file as \"bob\" - this should be allowed (by the policy - user)\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"bob\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Read the file\n FSDataInputStream in = fs.open(file);\n ByteArrayOutputStream output = new ByteArrayOutputStream();\n IOUtils.copy(in, output);\n String content = new String(output.toByteArray());\n Assert.assertTrue(content.startsWith(\"data0\"));\n fs.close();\n return null;\n }\n });\n\n // Now try to read the file as \"alice\" - this should be allowed (by the policy - group)\n ugi = UserGroupInformation.createUserForTesting(\"alice\", new String[]{\"IT\"});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Read the file\n FSDataInputStream in = fs.open(file);\n ByteArrayOutputStream output = new ByteArrayOutputStream();\n IOUtils.copy(in, output);\n String content = new String(output.toByteArray());\n Assert.assertTrue(content.startsWith(\"data0\"));\n\n fs.close();\n return null;\n }\n });\n\n // Now try to read the file as unknown user \"eve\" - this should not be allowed\n ugi = UserGroupInformation.createUserForTesting(\"eve\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Read the file\n try {\n fs.open(file);\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n // expected\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n fs.close();\n return null;\n }\n });\n\n // Now try to read the file as known user \"dave\" - this should not be allowed, as he doesn't have the correct permissions\n ugi = UserGroupInformation.createUserForTesting(\"dave\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n\n // Read the file\n try {\n fs.open(file);\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n // expected\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n\n fs.close();\n return null;\n }\n });\n\n fs.delete(file);\n }\n\n public void testHDFSContentSummary() throws Exception {\n HDFSGetContentSummary(\"/tmp/get-content-summary\");\n fs.delete(new Path(\"/tmp/get-content-summary\"), true);\n }\n\n void HDFSReadTest(String fileName) throws Exception {\n\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n final Path file = new Path(fileName);\n FSDataOutputStream out = fs.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n\n fs.setPermission(file, new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE));\n\n // Now try to read the file as \"bob\" - this should be allowed (by the policy - user)\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"bob\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n // Read the file\n FSDataInputStream in = fs.open(file);\n ByteArrayOutputStream output = new ByteArrayOutputStream();\n IOUtils.copy(in, output);\n String content = new String(output.toByteArray());\n Assert.assertTrue(content.startsWith(\"data0\"));\n\n fs.close();\n return null;\n }\n });\n\n // Now try to read the file as \"alice\" - this should be allowed (by the policy - group)\n ugi = UserGroupInformation.createUserForTesting(\"alice\", new String[]{\"IT\"});\n ugi.doAs(new PrivilegedExceptionAction() {\n\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n FSDataInputStream in = fs.open(file);\n ByteArrayOutputStream output = new ByteArrayOutputStream();\n IOUtils.copy(in, output);\n String content = new String(output.toByteArray());\n Assert.assertTrue(content.startsWith(\"data0\"));\n fs.close();\n return null;\n }\n });\n\n // Now try to read the file as unknown user \"eve\" - this should not be allowed\n ugi = UserGroupInformation.createUserForTesting(\"eve\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n try {\n fs.open(file);\n Assert.fail(\"Failure expected on an incorrect permission\");\n } catch (AccessControlException ex) {\n Assert.assertTrue(AccessControlException.class.getName().equals(ex.getClass().getName()));\n }\n\n fs.close();\n return null;\n }\n });\n\n fs.delete(file);\n }\n\n void HDFSGetContentSummary(final String dirName) throws Exception {\n\n String subdirName = dirName + \"/tmpdir\";\n\n createFile(subdirName, 1);\n createFile(subdirName, 2);\n\n fs.setPermission(new Path(dirName), new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE));\n\n UserGroupInformation ugi = UserGroupInformation.createUserForTesting(\"bob\", new String[]{});\n ugi.doAs(new PrivilegedExceptionAction() {\n\n public Void run() throws Exception {\n FileSystem fs = FileSystem.get(cfg);\n try {\n // GetContentSummary on the directory dirName\n ContentSummary contentSummary = fs.getContentSummary(new Path(dirName));\n\n long directoryCount = contentSummary.getDirectoryCount();\n Assert.assertTrue(\"Found unexpected number of directories; expected-count=3, actual-count=\" + directoryCount, directoryCount == 3);\n } catch (Exception e) {\n Assert.fail(\"Failed to getContentSummary, exception=\" + e);\n }\n fs.close();\n return null;\n }\n });\n\n deleteFile(subdirName, 1);\n deleteFile(subdirName, 2);\n }\n\n void createFile(String baseDir, Integer index) throws Exception {\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n String dirName = baseDir + (index != null ? String.valueOf(index) : \"\");\n String fileName = dirName + \"/dummy-data\";\n final Path file = new Path(fileName);\n FSDataOutputStream out = fs.create(file);\n for (int i = 0; i < 1024; ++i) {\n out.write((\"data\" + i + \"\\n\").getBytes(\"UTF-8\"));\n out.flush();\n }\n out.close();\n }\n\n void deleteFile(String baseDir, Integer index) throws Exception {\n // Write a file - the AccessControlEnforcer won't be invoked as we are the \"superuser\"\n String dirName = baseDir + (index != null ? String.valueOf(index) : \"\");\n String fileName = dirName + \"/dummy-data\";\n final Path file = new Path(fileName);\n fs.delete(file);\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/utils/BgTaskUtilTest.java", "content": "package io.juicefs.utils;\n\nimport junit.framework.TestCase;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.net.URI;\nimport java.util.concurrent.*;\n\npublic class BgTaskUtilTest extends TestCase {\n private static final Logger LOG = LoggerFactory.getLogger(BgTaskUtilTest.class);\n\n public void testBgTask() throws Exception {\n BgTaskUtil.reset();\n\n String[] volNames = new String[]{\"fs1\", \"fs2\", \"fs3\"};\n String[] taskNames = new String[]{\"task1\", \"task2\", \"task3\"};\n int threads = 20;\n ExecutorService pool = Executors.newFixedThreadPool(threads);\n\n int instances = 100;\n CountDownLatch latch = new CountDownLatch(instances);\n\n for (int i = 0; i < instances; i++) {\n int handle = i + 1;\n pool.submit(() -> {\n String volName = volNames[ThreadLocalRandom.current().nextInt(100) % volNames.length];\n try {\n BgTaskUtil.register(volName, handle);\n BgTaskUtil.startTrashEmptier(volName, () -> {\n LOG.info(\"tid {} running trash empiter for {}\", Thread.currentThread().getId(), volName);\n while (true) {\n try {\n Thread.sleep(100);\n } catch (InterruptedException e) {\n break;\n }\n }\n }, 0, TimeUnit.MINUTES);\n // put many tasks\n for (int j = 0; j < 10; j++) {\n String taskName = taskNames[ThreadLocalRandom.current().nextInt(100) % taskNames.length];\n BgTaskUtil.putTask(volName,\n taskName,\n () -> {\n LOG.info(\"running {}|{}\", volName, taskName);\n try {\n Thread.sleep(ThreadLocalRandom.current().nextInt(2000));\n } catch (InterruptedException e) {\n throw new RuntimeException(e);\n }\n },\n 0, 1, TimeUnit.MINUTES\n );\n }\n } catch (Exception e) {\n LOG.error(\"unexpected\", e);\n } finally {\n BgTaskUtil.unregister(volName, handle, () -> {\n LOG.info(\"clean {}\", volName);\n });\n latch.countDown();\n }\n });\n }\n latch.await();\n assertEquals(0, BgTaskUtil.getBgThreadForName().size());\n assertEquals(0, BgTaskUtil.getRunningInstance().size());\n }\n}\n" }, { "path": "sdk/java/src/test/java/io/juicefs/utils/HashTest.java", "content": "/*\n * JuiceFS, Copyright 2020 Juicedata, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * \n * http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.juicefs.utils;\n\nimport com.google.common.collect.Lists;\nimport junit.framework.TestCase;\nimport org.apache.commons.math3.stat.descriptive.SummaryStatistics;\n\nimport java.util.*;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\n\npublic class HashTest extends TestCase {\n private static List PATHS = new ArrayList() {\n {\n String prefix = \"jfs:///tmp/file\";\n for (int i = 0; i < 1_000; i++) {\n add(prefix + i);\n }\n }\n };\n\n public void testConsitentHashCompat() {\n ConsistentHash hash = new ConsistentHash<>(100, Lists.newArrayList());\n hash.addNode(\"192.168.1.1\");\n hash.addNode(\"192.168.2.2\");\n hash.addNode(\"192.168.3.3\");\n hash.addNode(\"192.168.4.4\");\n assertEquals(\"192.168.3.3\", hash.get(\"123-0\"));\n assertEquals(\"192.168.4.4\", hash.get(\"456-2\"));\n assertEquals(\"192.168.2.2\", hash.get(\"789-3\"));\n }\n\n public void testConsitentHash() {\n ConsistentHash hash = new ConsistentHash<>(100, getNodes());\n Map before = new HashMap<>();\n Map after = new HashMap<>();\n\n for (String path : PATHS) {\n before.put(path, hash.get(path));\n }\n\n hash.remove(\"Node4\");\n for (String path : PATHS) {\n after.put(path, hash.get(path));\n }\n System.out.println(\"====== stdev\");\n System.out.println(\"before:\\t\" + stdev(before));\n System.out.println(\"after:\\t\" + stdev(after));\n\n System.out.println(\"====== (max - min)/avg\");\n Map collect = after.values().stream()\n .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));\n Long max = Collections.max(collect.values());\n Long min = Collections.min(collect.values());\n long sum = collect.values().stream().mapToLong(i -> i).sum();\n System.out.println((double) (max - min) / ((double) sum / getNodes().size()));\n\n int count = 0; // total count of path that was moved\n for (Map.Entry entry : before.entrySet()) {\n String path = entry.getKey();\n String host = entry.getValue();\n if (!host.equals(after.get(path)))\n count++;\n }\n double moveRatio = (double) count / before.size();\n System.out.println(\"move ratio:\\t\" + moveRatio);\n\n assertTrue(moveRatio < (double) 2 / getNodes().size());\n }\n\n private static double stdev(Map after) {\n Map collect = after.values().stream()\n .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));\n SummaryStatistics statistics = new SummaryStatistics();\n for (Long value : collect.values()) {\n statistics.addValue(value);\n }\n double sum = statistics.getSum();\n statistics.clear();\n for (Long value : collect.values()) {\n statistics.addValue((double) value / sum);\n }\n\n return statistics.getStandardDeviation();\n }\n\n private List getNodes() {\n List nodes = Lists.newArrayList();\n for (int i = 0; i < 100; i++) {\n nodes.add(\"Node\" + i);\n }\n return nodes;\n }\n}\n" }, { "path": "sdk/java/src/test/resources/hdfs-policies-tag.json", "content": "{\n \"op\": \"add_or_update\",\n \"serviceName\": \"cl1_hadoop\",\n \"tagVersion\": 2,\n \"tagDefinitions\": {},\n \"tags\": {\n \"2\": {\n \"type\": \"TmpdirTag\",\n \"owner\": 0,\n \"attributes\": {},\n \"id\": 2,\n \"isEnabled\": true,\n \"version\": 1\n }\n },\n \"serviceResources\": [\n {\n \"resourceElements\": {\n \"path\": {\n \"values\": [\n \"/tmp/tmpdir6\"\n ],\n \"isExcludes\": false,\n \"isRecursive\": true\n }\n },\n \"id\": 2,\n \"isEnabled\": true,\n \"version\": 2\n }\n ],\n \"resourceToTagIds\": {\n \"2\": [\n 2\n ]\n }\n}" }, { "path": "sdk/java/src/test/resources/hdfs-policies.json", "content": "{\n \"serviceName\": \"cl1_hadoop\",\n \"serviceId\": 6,\n \"policyVersion\": 7,\n \"policyUpdateTime\": \"20170220-12:36:01.000-+0000\",\n \"policies\": [\n {\n \"service\": \"cl1_hadoop\",\n \"name\": \"/tmp/tmpdir\",\n \"policyType\": 0,\n \"policyPriority\": 0,\n \"description\": \"\",\n \"isAuditEnabled\": false,\n \"resources\": {\n \"path\": {\n \"values\": [\n \"/tmp/tmpdir/\"\n ],\n \"isExcludes\": false,\n \"isRecursive\": true\n }\n },\n \"policyItems\": [\n {\n \"accesses\": [\n {\n \"type\": \"read\",\n \"isAllowed\": true\n }\n ],\n \"users\": [],\n \"groups\": [\n \"IT\"\n ],\n \"roles\": [],\n \"conditions\": [],\n \"delegateAdmin\": false\n },\n {\n \"accesses\": [\n {\n \"type\": \"read\",\n \"isAllowed\": true\n }\n ],\n \"users\": [\n \"bob\"\n ],\n \"groups\": [],\n \"roles\": [],\n \"conditions\": [],\n \"delegateAdmin\": false\n }\n ],\n \"denyPolicyItems\": [],\n \"allowExceptions\": [],\n \"denyExceptions\": [],\n \"dataMaskPolicyItems\": [],\n \"rowFilterPolicyItems\": [],\n \"serviceType\": \"hdfs\",\n \"id\": 14,\n \"isEnabled\": true,\n \"version\": 4\n },\n {\n \"service\": \"cl1_hadoop\",\n \"name\": \"/tmp/tmpdir2\",\n \"policyType\": 0,\n \"description\": \"\",\n \"isAuditEnabled\": true,\n \"resources\": {\n \"path\": {\n \"values\": [\n \"/tmp/tmpdir2\"\n ],\n \"isExcludes\": false,\n \"isRecursive\": true\n }\n },\n \"policyItems\": [\n {\n \"accesses\": [\n {\n \"type\": \"write\",\n \"isAllowed\": true\n }\n ],\n \"users\": [],\n \"groups\": [\n \"IT\"\n ],\n \"conditions\": [],\n \"delegateAdmin\": false\n },\n {\n \"accesses\": [\n {\n \"type\": \"write\",\n \"isAllowed\": true\n }\n ],\n \"users\": [\n \"bob\"\n ],\n \"groups\": [],\n \"conditions\": [],\n \"delegateAdmin\": false\n }\n ],\n \"denyPolicyItems\": [],\n \"allowExceptions\": [],\n \"denyExceptions\": [],\n \"dataMaskPolicyItems\": [],\n \"rowFilterPolicyItems\": [],\n \"id\": 15,\n \"isEnabled\": true,\n \"version\": 1\n },\n {\n \"service\": \"cl1_hadoop\",\n \"name\": \"/tmp/tmpdir3\",\n \"policyType\": 0,\n \"description\": \"\",\n \"isAuditEnabled\": true,\n \"resources\": {\n \"path\": {\n \"values\": [\n \"/tmp/tmpdir3\"\n ],\n \"isExcludes\": false,\n \"isRecursive\": true\n }\n },\n \"policyItems\": [\n {\n \"accesses\": [\n {\n \"type\": \"read\",\n \"isAllowed\": true\n },\n {\n \"type\": \"execute\",\n \"isAllowed\": true\n }\n ],\n \"users\": [],\n \"groups\": [\n \"IT\"\n ],\n \"conditions\": [],\n \"delegateAdmin\": false\n },\n {\n \"accesses\": [\n {\n \"type\": \"read\",\n \"isAllowed\": true\n },\n {\n \"type\": \"execute\",\n \"isAllowed\": true\n }\n ],\n \"users\": [\n \"bob\"\n ],\n \"groups\": [],\n \"conditions\": [],\n \"delegateAdmin\": false\n }\n ],\n \"denyPolicyItems\": [],\n \"allowExceptions\": [],\n \"denyExceptions\": [],\n \"dataMaskPolicyItems\": [],\n \"rowFilterPolicyItems\": [],\n \"id\": 16,\n \"isEnabled\": true,\n \"version\": 1\n },\n {\n \"service\": \"cl1_hadoop\",\n \"name\": \"/tmp/get-content-summary\",\n \"policyType\": 0,\n \"description\": \"\",\n \"isAuditEnabled\": true,\n \"resources\": {\n \"path\": {\"values\": [\"/tmp/get-content-summary\", \"/tmp/get-content-summary/tmpdir1\", \"/tmp/get-content-summary/tmpdir2\"], \"isExcludes\": false, \"isRecursive\": false}\n },\n \"policyItems\": [\n {\n \"accesses\": [{\"type\": \"read\",\"isAllowed\": true}, {\"type\": \"execute\",\"isAllowed\": true}],\n \"users\": [\"bob\"],\n \"groups\": [\"IT\"],\n \"conditions\": [],\n \"delegateAdmin\": false\n }\n ],\n \"denyPolicyItems\": [],\n \"allowExceptions\": [],\n \"denyExceptions\": [],\n \"dataMaskPolicyItems\": [],\n \"rowFilterPolicyItems\": [],\n \"id\": 40,\n \"isEnabled\": true,\n \"version\": 1\n }\n ],\n \"serviceDef\": {\n \"name\": \"hdfs\",\n \"implClass\": \"org.apache.ranger.services.hdfs.RangerServiceHdfs\",\n \"label\": \"HDFS Repository\",\n \"description\": \"HDFS Repository\",\n \"options\": {},\n \"configs\": [\n {\n \"itemId\": 1,\n \"name\": \"username\",\n \"type\": \"string\",\n \"subType\": \"\",\n \"mandatory\": true,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"Username\"\n },\n {\n \"itemId\": 2,\n \"name\": \"password\",\n \"type\": \"password\",\n \"subType\": \"\",\n \"mandatory\": true,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"Password\"\n },\n {\n \"itemId\": 3,\n \"name\": \"fs.default.name\",\n \"type\": \"string\",\n \"subType\": \"\",\n \"mandatory\": true,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"Namenode URL\"\n },\n {\n \"itemId\": 4,\n \"name\": \"hadoop.security.authorization\",\n \"type\": \"bool\",\n \"subType\": \"YesTrue:NoFalse\",\n \"mandatory\": true,\n \"defaultValue\": \"false\",\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"Authorization Enabled\"\n },\n {\n \"itemId\": 5,\n \"name\": \"hadoop.security.authentication\",\n \"type\": \"enum\",\n \"subType\": \"authnType\",\n \"mandatory\": true,\n \"defaultValue\": \"simple\",\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"Authentication Type\"\n },\n {\n \"itemId\": 6,\n \"name\": \"hadoop.security.auth_to_local\",\n \"type\": \"string\",\n \"subType\": \"\",\n \"mandatory\": false,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\"\n },\n {\n \"itemId\": 7,\n \"name\": \"dfs.datanode.kerberos.principal\",\n \"type\": \"string\",\n \"subType\": \"\",\n \"mandatory\": false,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\"\n },\n {\n \"itemId\": 8,\n \"name\": \"dfs.namenode.kerberos.principal\",\n \"type\": \"string\",\n \"subType\": \"\",\n \"mandatory\": false,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\"\n },\n {\n \"itemId\": 9,\n \"name\": \"dfs.secondary.namenode.kerberos.principal\",\n \"type\": \"string\",\n \"subType\": \"\",\n \"mandatory\": false,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\"\n },\n {\n \"itemId\": 10,\n \"name\": \"hadoop.rpc.protection\",\n \"type\": \"enum\",\n \"subType\": \"rpcProtection\",\n \"mandatory\": false,\n \"defaultValue\": \"authentication\",\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"RPC Protection Type\"\n },\n {\n \"itemId\": 11,\n \"name\": \"commonNameForCertificate\",\n \"type\": \"string\",\n \"subType\": \"\",\n \"mandatory\": false,\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"Common Name for Certificate\"\n }\n ],\n \"resources\": [\n {\n \"itemId\": 1,\n \"name\": \"path\",\n \"type\": \"path\",\n \"level\": 10,\n \"mandatory\": true,\n \"lookupSupported\": true,\n \"recursiveSupported\": true,\n \"excludesSupported\": false,\n \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher\",\n \"matcherOptions\": {\n \"wildCard\": \"true\",\n \"ignoreCase\": \"false\"\n },\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"\",\n \"label\": \"Resource Path\",\n \"description\": \"HDFS file or directory path\"\n }\n ],\n \"accessTypes\": [\n {\n \"itemId\": 1,\n \"name\": \"read\",\n \"label\": \"Read\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 2,\n \"name\": \"write\",\n \"label\": \"Write\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3,\n \"name\": \"execute\",\n \"label\": \"Execute\",\n \"impliedGrants\": []\n }\n ],\n \"policyConditions\": [],\n \"contextEnrichers\": [],\n \"enums\": [\n {\n \"itemId\": 1,\n \"name\": \"authnType\",\n \"elements\": [\n {\n \"itemId\": 1,\n \"name\": \"simple\",\n \"label\": \"Simple\"\n },\n {\n \"itemId\": 2,\n \"name\": \"kerberos\",\n \"label\": \"Kerberos\"\n }\n ],\n \"defaultIndex\": 0\n },\n {\n \"itemId\": 2,\n \"name\": \"rpcProtection\",\n \"elements\": [\n {\n \"itemId\": 1,\n \"name\": \"authentication\",\n \"label\": \"Authentication\"\n },\n {\n \"itemId\": 2,\n \"name\": \"integrity\",\n \"label\": \"Integrity\"\n },\n {\n \"itemId\": 3,\n \"name\": \"privacy\",\n \"label\": \"Privacy\"\n }\n ],\n \"defaultIndex\": 0\n }\n ],\n \"dataMaskDef\": {\n \"maskTypes\": [],\n \"accessTypes\": [],\n \"resources\": []\n },\n \"rowFilterDef\": {\n \"accessTypes\": [],\n \"resources\": []\n },\n \"id\": 1,\n \"guid\": \"0d047247-bafe-4cf8-8e9b-d5d377284b2d\",\n \"isEnabled\": true,\n \"createTime\": \"20170217-11:41:31.000-+0000\",\n \"updateTime\": \"20170217-11:41:31.000-+0000\",\n \"version\": 1\n },\n \"auditMode\": \"audit-default\",\n \"tagPolicies\": {\n \"serviceName\": \"KafkaTagService\",\n \"serviceId\": 5,\n \"policyVersion\": 5,\n \"policyUpdateTime\": \"20170220-12:35:51.000-+0000\",\n \"policies\": [\n {\n \"service\": \"KafkaTagService\",\n \"name\": \"EXPIRES_ON\",\n \"policyType\": 0,\n \"description\": \"Policy for data with EXPIRES_ON tag\",\n \"isAuditEnabled\": true,\n \"resources\": {\n \"tag\": {\n \"values\": [\n \"EXPIRES_ON\"\n ],\n \"isExcludes\": false,\n \"isRecursive\": false\n }\n },\n \"policyItems\": [],\n \"denyPolicyItems\": [\n {\n \"accesses\": [\n {\n \"type\": \"hdfs:read\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hdfs:write\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hdfs:execute\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hbase:read\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hbase:write\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hbase:create\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hbase:admin\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:select\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:update\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:create\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:drop\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:alter\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:index\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:lock\",\n \"isAllowed\": true\n },\n {\n \"type\": \"hive:all\",\n \"isAllowed\": true\n },\n {\n \"type\": \"yarn:submit-app\",\n \"isAllowed\": true\n },\n {\n \"type\": \"yarn:admin-queue\",\n \"isAllowed\": true\n },\n {\n \"type\": \"knox:allow\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:submitTopology\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:fileUpload\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:fileDownload\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:killTopology\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:rebalance\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:activate\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:deactivate\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:getTopologyConf\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:getTopology\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:getUserTopology\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:getTopologyInfo\",\n \"isAllowed\": true\n },\n {\n \"type\": \"storm:uploadNewCredentials\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:create\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:delete\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:rollover\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:setkeymaterial\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:get\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:getkeys\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:getmetadata\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:generateeek\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kms:decrypteek\",\n \"isAllowed\": true\n },\n {\n \"type\": \"solr:query\",\n \"isAllowed\": true\n },\n {\n \"type\": \"solr:update\",\n \"isAllowed\": true\n },\n {\n \"type\": \"solr:others\",\n \"isAllowed\": true\n },\n {\n \"type\": \"solr:solr_admin\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:publish\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:consume\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:configure\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:describe\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:create\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:delete\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:kafka_admin\",\n \"isAllowed\": true\n },\n {\n \"type\": \"atlas:read\",\n \"isAllowed\": true\n },\n {\n \"type\": \"atlas:create\",\n \"isAllowed\": true\n },\n {\n \"type\": \"atlas:update\",\n \"isAllowed\": true\n },\n {\n \"type\": \"atlas:delete\",\n \"isAllowed\": true\n },\n {\n \"type\": \"atlas:all\",\n \"isAllowed\": true\n }\n ],\n \"users\": [],\n \"groups\": [\n \"public\"\n ],\n \"conditions\": [\n {\n \"type\": \"accessed-after-expiry\",\n \"values\": [\n \"yes\"\n ]\n }\n ],\n \"delegateAdmin\": false\n }\n ],\n \"allowExceptions\": [],\n \"denyExceptions\": [],\n \"dataMaskPolicyItems\": [],\n \"rowFilterPolicyItems\": [],\n \"id\": 10,\n \"isEnabled\": true,\n \"version\": 1\n },\n {\n \"service\": \"KafkaTagService\",\n \"name\": \"AtlasKafkaTagPolicy\",\n \"policyType\": 0,\n \"description\": \"\",\n \"isAuditEnabled\": true,\n \"resources\": {\n \"tag\": {\n \"values\": [\n \"KafkaTag\"\n ],\n \"isExcludes\": false,\n \"isRecursive\": false\n }\n },\n \"policyItems\": [\n {\n \"accesses\": [\n {\n \"type\": \"kafka:consume\",\n \"isAllowed\": true\n },\n {\n \"type\": \"kafka:describe\",\n \"isAllowed\": true\n }\n ],\n \"users\": [\n \"CN\\u003dClient,O\\u003dApache,L\\u003dDublin,ST\\u003dLeinster,C\\u003dIE\"\n ],\n \"groups\": [],\n \"conditions\": [],\n \"delegateAdmin\": false\n }\n ],\n \"denyPolicyItems\": [],\n \"allowExceptions\": [],\n \"denyExceptions\": [],\n \"dataMaskPolicyItems\": [],\n \"rowFilterPolicyItems\": [],\n \"id\": 11,\n \"isEnabled\": true,\n \"version\": 2\n },\n {\n \"service\": \"KafkaTagService\",\n \"name\": \"TmpdirTagPolicy\",\n \"policyType\": 0,\n \"description\": \"\",\n \"isAuditEnabled\": true,\n \"resources\": {\n \"tag\": {\n \"values\": [\n \"TmpdirTag\"\n ],\n \"isExcludes\": false,\n \"isRecursive\": false\n }\n },\n \"policyItems\": [\n {\n \"accesses\": [\n {\n \"type\": \"hdfs:read\",\n \"isAllowed\": true\n }\n ],\n \"users\": [],\n \"groups\": [\n \"IT\"\n ],\n \"conditions\": [],\n \"delegateAdmin\": false\n },\n {\n \"accesses\": [\n {\n \"type\": \"hdfs:read\",\n \"isAllowed\": true\n }\n ],\n \"users\": [\n \"bob\"\n ],\n \"groups\": [],\n \"conditions\": [],\n \"delegateAdmin\": false\n }\n ],\n \"denyPolicyItems\": [],\n \"allowExceptions\": [],\n \"denyExceptions\": [],\n \"dataMaskPolicyItems\": [],\n \"rowFilterPolicyItems\": [],\n \"id\": 17,\n \"isEnabled\": true,\n \"version\": 1\n }\n ],\n \"serviceDef\": {\n \"name\": \"tag\",\n \"implClass\": \"org.apache.ranger.services.tag.RangerServiceTag\",\n \"label\": \"TAG\",\n \"description\": \"TAG Service Definition\",\n \"options\": {\n \"ui.pages\": \"tag-based-policies\"\n },\n \"configs\": [],\n \"resources\": [\n {\n \"itemId\": 1,\n \"name\": \"tag\",\n \"type\": \"string\",\n \"level\": 1,\n \"mandatory\": true,\n \"lookupSupported\": true,\n \"recursiveSupported\": false,\n \"excludesSupported\": false,\n \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher\",\n \"matcherOptions\": {\n \"wildCard\": \"false\",\n \"ignoreCase\": \"false\"\n },\n \"validationRegEx\": \"\",\n \"validationMessage\": \"\",\n \"uiHint\": \"{ \\\"singleValue\\\":true }\",\n \"label\": \"TAG\",\n \"description\": \"TAG\"\n }\n ],\n \"accessTypes\": [\n {\n \"itemId\": 1002,\n \"name\": \"hdfs:read\",\n \"label\": \"Read\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 1003,\n \"name\": \"hdfs:write\",\n \"label\": \"Write\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 1004,\n \"name\": \"hdfs:execute\",\n \"label\": \"Execute\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 2003,\n \"name\": \"hbase:read\",\n \"label\": \"Read\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 2004,\n \"name\": \"hbase:write\",\n \"label\": \"Write\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 2005,\n \"name\": \"hbase:create\",\n \"label\": \"Create\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 2006,\n \"name\": \"hbase:admin\",\n \"label\": \"Admin\",\n \"impliedGrants\": [\n \"hbase:read\",\n \"hbase:write\",\n \"hbase:create\"\n ]\n },\n {\n \"itemId\": 3004,\n \"name\": \"hive:select\",\n \"label\": \"select\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3005,\n \"name\": \"hive:update\",\n \"label\": \"update\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3006,\n \"name\": \"hive:create\",\n \"label\": \"Create\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3007,\n \"name\": \"hive:drop\",\n \"label\": \"Drop\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3008,\n \"name\": \"hive:alter\",\n \"label\": \"Alter\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3009,\n \"name\": \"hive:index\",\n \"label\": \"Index\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3010,\n \"name\": \"hive:lock\",\n \"label\": \"Lock\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 3011,\n \"name\": \"hive:all\",\n \"label\": \"All\",\n \"impliedGrants\": [\n \"hive:select\",\n \"hive:update\",\n \"hive:create\",\n \"hive:drop\",\n \"hive:alter\",\n \"hive:index\",\n \"hive:lock\"\n ]\n },\n {\n \"itemId\": 4005,\n \"name\": \"yarn:submit-app\",\n \"label\": \"submit-app\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 4006,\n \"name\": \"yarn:admin-queue\",\n \"label\": \"admin-queue\",\n \"impliedGrants\": [\n \"yarn:submit-app\"\n ]\n },\n {\n \"itemId\": 5006,\n \"name\": \"knox:allow\",\n \"label\": \"Allow\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6007,\n \"name\": \"storm:submitTopology\",\n \"label\": \"Submit Topology\",\n \"impliedGrants\": [\n \"storm:fileUpload\",\n \"storm:fileDownload\"\n ]\n },\n {\n \"itemId\": 6008,\n \"name\": \"storm:fileUpload\",\n \"label\": \"File Upload\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6011,\n \"name\": \"storm:fileDownload\",\n \"label\": \"File Download\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6012,\n \"name\": \"storm:killTopology\",\n \"label\": \"Kill Topology\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6013,\n \"name\": \"storm:rebalance\",\n \"label\": \"Rebalance\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6014,\n \"name\": \"storm:activate\",\n \"label\": \"Activate\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6015,\n \"name\": \"storm:deactivate\",\n \"label\": \"Deactivate\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6016,\n \"name\": \"storm:getTopologyConf\",\n \"label\": \"Get Topology Conf\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6017,\n \"name\": \"storm:getTopology\",\n \"label\": \"Get Topology\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6018,\n \"name\": \"storm:getUserTopology\",\n \"label\": \"Get User Topology\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6019,\n \"name\": \"storm:getTopologyInfo\",\n \"label\": \"Get Topology Info\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 6020,\n \"name\": \"storm:uploadNewCredentials\",\n \"label\": \"Upload New Credential\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7008,\n \"name\": \"kms:create\",\n \"label\": \"Create\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7009,\n \"name\": \"kms:delete\",\n \"label\": \"Delete\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7010,\n \"name\": \"kms:rollover\",\n \"label\": \"Rollover\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7011,\n \"name\": \"kms:setkeymaterial\",\n \"label\": \"Set Key Material\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7012,\n \"name\": \"kms:get\",\n \"label\": \"Get\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7013,\n \"name\": \"kms:getkeys\",\n \"label\": \"Get Keys\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7014,\n \"name\": \"kms:getmetadata\",\n \"label\": \"Get Metadata\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7015,\n \"name\": \"kms:generateeek\",\n \"label\": \"Generate EEK\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 7016,\n \"name\": \"kms:decrypteek\",\n \"label\": \"Decrypt EEK\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 8108,\n \"name\": \"solr:query\",\n \"label\": \"Query\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 8208,\n \"name\": \"solr:update\",\n \"label\": \"Update\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 8308,\n \"name\": \"solr:others\",\n \"label\": \"Others\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 8908,\n \"name\": \"solr:solr_admin\",\n \"label\": \"Solr Admin\",\n \"impliedGrants\": [\n \"solr:query\",\n \"solr:update\",\n \"solr:others\"\n ]\n },\n {\n \"itemId\": 9010,\n \"name\": \"kafka:publish\",\n \"label\": \"Publish\",\n \"impliedGrants\": [\n \"kafka:describe\"\n ]\n },\n {\n \"itemId\": 9011,\n \"name\": \"kafka:consume\",\n \"label\": \"Consume\",\n \"impliedGrants\": [\n \"kafka:describe\"\n ]\n },\n {\n \"itemId\": 9014,\n \"name\": \"kafka:configure\",\n \"label\": \"Configure\",\n \"impliedGrants\": [\n \"kafka:describe\"\n ]\n },\n {\n \"itemId\": 9015,\n \"name\": \"kafka:describe\",\n \"label\": \"Describe\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 9017,\n \"name\": \"kafka:create\",\n \"label\": \"Create\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 9018,\n \"name\": \"kafka:delete\",\n \"label\": \"Delete\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 9016,\n \"name\": \"kafka:kafka_admin\",\n \"label\": \"Kafka Admin\",\n \"impliedGrants\": [\n \"kafka:publish\",\n \"kafka:consume\",\n \"kafka:configure\",\n \"kafka:describe\",\n \"kafka:create\",\n \"kafka:delete\"\n ]\n },\n {\n \"itemId\": 11012,\n \"name\": \"atlas:read\",\n \"label\": \"read\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 11013,\n \"name\": \"atlas:create\",\n \"label\": \"create\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 11014,\n \"name\": \"atlas:update\",\n \"label\": \"update\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 11015,\n \"name\": \"atlas:delete\",\n \"label\": \"delete\",\n \"impliedGrants\": []\n },\n {\n \"itemId\": 11016,\n \"name\": \"atlas:all\",\n \"label\": \"All\",\n \"impliedGrants\": [\n \"atlas:read\",\n \"atlas:create\",\n \"atlas:update\",\n \"atlas:delete\"\n ]\n }\n ],\n \"policyConditions\": [\n {\n \"itemId\": 1,\n \"name\": \"accessed-after-expiry\",\n \"evaluator\": \"org.apache.ranger.plugin.conditionevaluator.RangerScriptTemplateConditionEvaluator\",\n \"evaluatorOptions\": {\n \"scriptTemplate\": \"ctx.isAccessedAfter(\\u0027expiry_date\\u0027);\"\n },\n \"uiHint\": \"{ \\\"singleValue\\\":true }\",\n \"label\": \"Accessed after expiry_date (yes/no)?\",\n \"description\": \"Accessed after expiry_date? (yes/no)\"\n }\n ],\n \"contextEnrichers\": [\n {\n \"itemId\": 1,\n \"name\": \"TagEnricher\",\n \"enricher\": \"org.apache.ranger.plugin.contextenricher.RangerTagEnricher\",\n \"enricherOptions\": {\n \"tagRetrieverClassName\": \"org.apache.ranger.plugin.contextenricher.RangerAdminTagRetriever\",\n \"tagRefresherPollingInterval\": \"60000\"\n }\n }\n ],\n \"enums\": [],\n \"dataMaskDef\": {\n \"maskTypes\": [],\n \"accessTypes\": [],\n \"resources\": []\n },\n \"rowFilterDef\": {\n \"accessTypes\": [],\n \"resources\": []\n },\n \"id\": 100,\n \"guid\": \"0d047248-baff-4cf9-8e9e-d5d377284b2e\",\n \"isEnabled\": true,\n \"createTime\": \"20170217-11:41:33.000-+0000\",\n \"updateTime\": \"20170217-11:41:35.000-+0000\",\n \"version\": 11\n },\n \"auditMode\": \"audit-default\"\n }\n}" }, { "path": "sdk/java/src/test/resources/kerberos.cfg", "content": "# kerberos keytab\ndev.keytab={BASE64 KEYTAB}\n# delegation token\ndev.token.life=604800\ndev.token.renew=86400\n\n# superuser and supergroup\ndev.superuser=client\ndev.supergroup=supergroup\n\n# Mapping from Kerberos principals to OS user accounts\n# https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html#Mapping_from_Kerberos_principals_to_OS_user_accounts\ndev.mechanism=hadoop\ndev.rule=RULE:[2:$1/$2@$0](root/.*@example.com)s/.*/hdfs/\ndev.rule=RULE:[2:$1/$2@$0](jerry/.*@EXAMPLE\\.COM)s/.*/jerry_map/\ndev.rule=RULE:[2:$1/$2@$0](tom/.*@EXAMPLE\\.COM)s/.*/client/\ndev.rule=DEFAULT\n\n# proxy user settings\n# https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html#Proxy_user\n# users: user1,user2 or *\ndev.proxy.client.users=foo\n# groups: group1,group2 or *\ndev.proxy.client.groups=foogrp\n# hosts: host1,host2 or 192.168.1.1,192.168.1.2 or 192.168.1.1/32 or *\ndev.proxy.client.hosts=*\n" }, { "path": "sdk/java/src/test/resources/log4j.properties", "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppenderk\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n" }, { "path": "sdk/java/src/test/resources/testAclCLI.xml", "content": "\n\n\n\n\n\n \n test\n\n \n \n \n \n getfacl: basic permissions\n \n -fs NAMENODE -touchz /file1\n -fs NAMENODE -getfacl /file1\n \n \n -fs NAMENODE -rm /file1\n \n \n \n SubstringComparator\n # file: /file1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rw-\n \n \n SubstringComparator\n group::r--\n \n \n SubstringComparator\n other::r--\n \n \n \n \n getfacl: basic permissions for directory\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm /dir1\n \n \n \n SubstringComparator\n # file: /dir1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rwx\n \n \n SubstringComparator\n group::r-x\n \n \n SubstringComparator\n other::r-x\n \n \n \n \n setfacl : Add an ACL\n \n -fs NAMENODE -touchz /file1\n -fs NAMENODE -setfacl -m user:bob:r-- /file1\n -fs NAMENODE -getfacl /file1\n \n \n -fs NAMENODE -rm /file1\n \n \n \n SubstringComparator\n # file: /file1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rw-\n \n \n SubstringComparator\n user:bob:r--\n \n \n SubstringComparator\n group::r--\n \n \n SubstringComparator\n mask::r--\n \n \n SubstringComparator\n other::r--\n \n \n \n \n setfacl : Add multiple ACLs at once\n \n -fs NAMENODE -touchz /file1\n -fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /file1\n -fs NAMENODE -getfacl /file1\n \n \n -fs NAMENODE -rm /file1\n \n \n \n SubstringComparator\n # file: /file1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rw-\n \n \n SubstringComparator\n user:bob:r--\n \n \n SubstringComparator\n group::r--\n \n \n SubstringComparator\n group:users:r-x\n \n \n SubstringComparator\n mask::r-x\n \n \n SubstringComparator\n other::r--\n \n \n \n \n setfacl : Remove an ACL\n \n -fs NAMENODE -touchz /file1\n -fs NAMENODE -setfacl -m user:bob:r--,user:charlie:r-x /file1\n -fs NAMENODE -setfacl -x user:bob /file1\n -fs NAMENODE -getfacl /file1\n \n \n -fs NAMENODE -rm /file1\n \n \n \n SubstringComparator\n # file: /file1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rw-\n \n \n SubstringComparator\n user:charlie:r-x\n \n \n SubstringComparator\n group::r--\n \n \n SubstringComparator\n other::r--\n \n \n RegexpAcrossOutputComparator\n .*(?!bob)*\n \n \n \n \n setfacl : Add default ACL\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1\n -fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n SubstringComparator\n # file: /dir1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rwx\n \n \n SubstringComparator\n user:bob:r--\n \n \n SubstringComparator\n group::r-x\n \n \n SubstringComparator\n group:users:r-x\n \n \n SubstringComparator\n mask::r-x\n \n \n SubstringComparator\n other::r-x\n \n \n SubstringComparator\n default:user::rwx\n \n \n SubstringComparator\n default:user:charlie:r-x\n \n \n SubstringComparator\n default:group::r-x\n \n \n SubstringComparator\n default:group:admin:rwx\n \n \n SubstringComparator\n default:mask::rwx\n \n \n SubstringComparator\n default:other::r-x\n \n \n \n \n setfacl : Add minimal default ACL\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m default:user::rwx /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n SubstringComparator\n # file: /dir1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rwx\n \n \n SubstringComparator\n group::r-x\n \n \n SubstringComparator\n other::r-x\n \n \n SubstringComparator\n default:user::rwx\n \n \n SubstringComparator\n default:group::r-x\n \n \n SubstringComparator\n default:other::r-x\n \n \n RegexpAcrossOutputComparator\n .*(?!default\\:mask)*\n \n \n \n \n setfacl : try adding default ACL to file\n \n -fs NAMENODE -touchz /file1\n -fs NAMENODE -setfacl -m default:user:charlie:r-x /file1\n \n \n -fs NAMENODE -rm /file1\n \n \n \n SubstringComparator\n setfacl: Invalid ACL: only directories may have a default ACL\n \n \n \n \n setfacl : Remove one default ACL\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1\n -fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1\n -fs NAMENODE -setfacl -x default:user:charlie /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n SubstringComparator\n # file: /dir1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rwx\n \n \n SubstringComparator\n user:bob:r--\n \n \n SubstringComparator\n group::r-x\n \n \n SubstringComparator\n group:users:r-x\n \n \n SubstringComparator\n mask::r-x\n \n \n SubstringComparator\n other::r-x\n \n \n SubstringComparator\n default:user::rwx\n \n \n SubstringComparator\n default:group::r-x\n \n \n SubstringComparator\n default:group:admin:rwx\n \n \n SubstringComparator\n default:mask::rwx\n \n \n SubstringComparator\n default:other::r-x\n \n \n RegexpAcrossOutputComparator\n .*(?!default:user:charlie).*\n \n \n \n \n setfacl : Remove all default ACL\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1\n -fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1\n -fs NAMENODE -setfacl -k /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n SubstringComparator\n # file: /dir1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rwx\n \n \n SubstringComparator\n user:bob:r--\n \n \n SubstringComparator\n group::r-x\n \n \n SubstringComparator\n group:users:r-x\n \n \n SubstringComparator\n mask::r-x\n \n \n SubstringComparator\n other::r-x\n \n \n RegexpAcrossOutputComparator\n .*(?!default).*\n \n \n \n \n setfacl : Remove all but base ACLs for a directory\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m user:charlie:r-x,default:group:admin:rwx /dir1\n -fs NAMENODE -setfacl -b /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n SubstringComparator\n # file: /dir1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rwx\n \n \n SubstringComparator\n group::r-x\n \n \n SubstringComparator\n other::r-x\n \n \n RegexpAcrossOutputComparator\n .*(?!charlie).*\n \n \n RegexpAcrossOutputComparator\n .*(?!default).*\n \n \n RegexpAcrossOutputComparator\n .*(?!admin).*\n \n \n \n \n setfacl : Remove all but base ACLs for a file\n \n -fs NAMENODE -touchz /file1\n -fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /file1\n -fs NAMENODE -setfacl -b /file1\n -fs NAMENODE -getfacl /file1\n \n \n -fs NAMENODE -rm /file1\n \n \n \n SubstringComparator\n # file: /file1\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rw-\n \n \n SubstringComparator\n group::r--\n \n \n SubstringComparator\n other::r--\n \n \n RegexpAcrossOutputComparator\n .*(?!charlie).*\n \n \n RegexpAcrossOutputComparator\n .*(?!admin).*\n \n \n \n \n setfacl : check inherit default ACL to file\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1\n -fs NAMENODE -touchz /dir1/file\n -fs NAMENODE -getfacl /dir1/file\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n SubstringComparator\n # file: /dir1/file\n \n \n SubstringComparator\n # owner: USERNAME\n \n \n SubstringComparator\n # group: supergroup\n \n \n SubstringComparator\n user::rw-\n \n \n SubstringComparator\n user:charlie:r-x\n \n \n SubstringComparator\n group::r-x\n \n \n SubstringComparator\n group:admin:rwx\n \n \n SubstringComparator\n mask::rw-\n \n \n SubstringComparator\n other::r--\n \n \n RegexpAcrossOutputComparator\n .*(?!default).*\n \n \n \n \n setfacl : check inherit default ACL to dir\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1\n -fs NAMENODE -mkdir /dir1/dir2\n -fs NAMENODE -getfacl /dir1/dir2\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n ExactLineComparator\n # file: /dir1/dir2\n \n \n ExactLineComparator\n # owner: USERNAME\n \n \n ExactLineComparator\n # group: supergroup\n \n \n ExactLineComparator\n user::rwx\n \n \n ExactLineComparator\n user:charlie:r-x\n \n \n ExactLineComparator\n group::r-x\n \n \n RegexpComparator\n ^group:admin:rwx\\b.*\n \n \n ExactLineComparator\n mask::rwx\n \n \n ExactLineComparator\n default:user::rwx\n \n \n ExactLineComparator\n default:user:charlie:r-x\n \n \n ExactLineComparator\n default:group::r-x\n \n \n ExactLineComparator\n default:group:admin:rwx\n \n \n ExactLineComparator\n default:mask::rwx\n \n \n ExactLineComparator\n default:other::r-x\n \n \n ExactLineComparator\n other::r-x\n \n \n \n \n getfacl -R : recursive\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1\n -fs NAMENODE -mkdir /dir1/dir2\n -fs NAMENODE -setfacl -m user:user1:r-x,group:users:rwx /dir1/dir2\n -fs NAMENODE -getfacl -R /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n ExactComparator\n # file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:user1:r-x#LF#group::r-x#LF#group:users:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#\n \n \n \n \n setfacl -R : recursive\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -mkdir /dir1/dir2\n -fs NAMENODE -setfacl -R -m user:charlie:r-x,group:admin:rwx /dir1\n -fs NAMENODE -getfacl -R /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n ExactComparator\n # file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#\n \n \n \n \n setfacl --set : Set full set of ACLs\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1\n -fs NAMENODE -setfacl --set user::rw-,group::r--,other::r--,user:user1:r-x,group:users:rw- /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n ExactComparator\n # file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:user1:r-x#LF#group::r--#LF#group:users:rw-#LF#mask::rwx#LF#other::r--#LF##LF#\n \n \n \n \n setfacl -x mask : remove mask entry along with other ACL entries\n \n -fs NAMENODE -mkdir /dir1\n -fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1\n -fs NAMENODE -setfacl -x mask::,user:charlie,group:admin /dir1\n -fs NAMENODE -getfacl /dir1\n \n \n -fs NAMENODE -rm -R /dir1\n \n \n \n ExactComparator\n # file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#group::r-x#LF#other::r-x#LF##LF#\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n" }, { "path": "sdk/java/src/test/test-spark.sh", "content": "#!/usr/bin/env bash\n\nset -e\nset -o pipefail\n\nHADOOP_VERSION=\"2.7.7\"\nSPARK_VERSION=\"2.4.0\"\nEXAMPLES_JAR=\"spark-examples_2.11-2.4.0.jar\"\n\nSPARK_DIST=\"spark-${SPARK_VERSION}-bin-without-hadoop\"\nSPARK_HOME=\"/opt/${SPARK_DIST}\"\nHADOOP_DIST=\"hadoop-${HADOOP_VERSION}\"\nHADOOP_HOME=\"/opt/${HADOOP_DIST}\"\n\ncurl -o \"${HADOOP_HOME}.tar.gz\" \"https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_DIST}.tar.gz\"\ntar -xf \"${HADOOP_HOME}.tar.gz\" -C /opt\n\nexport _JAVA_OPTIONS=\"-Djava.library.path=$(pwd)/../mount/libjfs\"\nexport HADOOP_CLASSPATH=\"$(pwd)/target/juicefs-hadoop-0.1-SNAPSHOT.jar\"\n\"${HADOOP_HOME}/bin/hadoop\" --config \"$(pwd)/conf\" jar \"${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-${HADOOP_VERSION}.jar\" grep hello output 'dfs[a-z.]+'\n\ncurl -o \"${SPARK_HOME}.tgz\" \"https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_DIST}.tgz\"\ntar -xf \"${SPARK_HOME}.tgz\" -C /opt\n\necho \"export SPARK_DIST_CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath)\" > \"${SPARK_HOME}/conf/spark-env.sh\"\necho \"export HADOOP_CONF_DIR=$(pwd)/conf\" >> \"${SPARK_HOME}/conf/spark-env.sh\"\ncp \"${SPARK_HOME}/examples/jars/${EXAMPLES_JAR}\" /jfs/\n\n\"${SPARK_HOME}/bin/spark-submit\" --class org.apache.spark.examples.JavaWordCount --master \"local\" \"jfs:///${EXAMPLES_JAR}\" \"jfs:///hello\"\n" }, { "path": "sdk/python/.gitignore", "content": "dist\nbuild\n*.egg-info \n*.h\n*.so\n" }, { "path": "sdk/python/Dockerfile.builder", "content": "FROM centos/python-38-centos7\n\nUSER 0\n\nRUN curl -fsSL https://autoinstall.plesk.com/PSA_18.0.62/examiners/repository_check.sh | bash -s -- update >/dev/null && \\\n yum install -y make gcc && \\\n cd /tmp && \\\n curl -L https://static.juicefs.com/misc/go1.21.13.linux-amd64.tar.gz -o go1.21.13.linux-amd64.tar.gz && \\\n tar -C /usr/local -xzf go1.21.13.linux-amd64.tar.gz && \\\n rm go1.21.13.linux-amd64.tar.gz && \\\n ln -s /usr/local/go/bin/go /usr/bin/go && \\\n python3 -m pip install --upgrade pip && \\\n python3 -m pip install --upgrade setuptools && \\\n pip install wheel build \n" }, { "path": "sdk/python/Dockerfile.builder.arm", "content": "FROM golang:1.24\n\nRUN apt update && \\\n apt install -y --no-install-recommends \\\n git \\\n make \\\n gcc \\\n python3 \\\n python3-pip \\\n python3-setuptools \\\n python3-wheel \\\n python3-build \\\n python3-venv \\\n ca-certificates \\\n && \\\n apt clean && \\\n rm -rf /var/lib/apt/lists/*\n" }, { "path": "sdk/python/Makefile", "content": "LDFLAGS = -s -w\n\n.PHONY: libjfs.so juicefs\n\n# SET GOPROXY if WITH_PROXY is set\nCN_GOPROXY ?= 0\nifeq ($(CN_GOPROXY), 1)\n\tGOPROXY = https://proxy.golang.com.cn,direct\nendif\n\nVERSION_FILE := ../../pkg/version/version.go\n\nVERSION := $(shell awk '/major[[:space:]]*:[[:space:]]*/ {gsub(/[^0-9]/, \"\", $$2); major=$$2} \\\n /minor[[:space:]]*:[[:space:]]*/ {gsub(/[^0-9]/, \"\", $$2); minor=$$2} \\\n /patch[[:space:]]*:[[:space:]]*/ {gsub(/[^0-9]/, \"\", $$2); patch=$$2} \\\n END {print major \".\" minor \".\" patch}' $(VERSION_FILE))\n\nREVISION := $(shell git rev-parse --short HEAD 2>/dev/null)\nREVISIONDATE := $(shell git log -1 --pretty=format:'%cd' --date short 2>/dev/null)\nBUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')\nBUILD_DATE_SHORT := $(shell date -u +'%Y%m%d%H%M')\n\nPKG := github.com/juicedata/juicefs/pkg/version\nifneq ($(strip $(REVISION)),) # Use git clone\n\tLDFLAGS += -X $(PKG).revision=$(REVISION) \\\n\t\t -X $(PKG).revisionDate=$(REVISIONDATE)\nendif\n\n# libjfs is located in the sdk/java/libjfs\nlibjfs.so:\n\tgo build -buildmode c-shared -ldflags=\"$(LDFLAGS)\" -o juicefs/juicefs/libjfs.so ../java/libjfs\n\nbuilder: Dockerfile.builder\n\tdocker build -t sdkbuilder -f Dockerfile.builder .\n\narm-builder: Dockerfile.builder.arm\n\tdocker build -t sdkbuilder -f Dockerfile.builder.arm .\n\njuicefs:\n\tsudo rm -rf juicefs.egg-info\n\techo \"Building juicefs version $(VERSION).$(BUILD_DATE_SHORT)\"\n\tsed -i 's/^VERSION = .*/VERSION = \"$(VERSION).$(BUILD_DATE_SHORT)\"/' juicefs/setup.py\n\tsed -i 's/^BUILD_INFO = .*/BUILD_INFO = \"$(BUILD_DATE) $(REVISION)\"/' juicefs/setup.py\n\tdocker run --rm -i -v ${PWD}/../../:/opt/jfs -w /opt/jfs/sdk/python -e GOPROXY=${GOPROXY} sdkbuilder sh -c 'make libjfs.so && cd juicefs && python3 -m build -w'\n\nclean:\n $(clean)\n" }, { "path": "sdk/python/examples/ffrecord/dataloader.py", "content": "# encoding: utf-8\n# JuiceFS, Copyright 2025 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport numpy as np\nfrom typing import List, Iterator, Callable\nfrom multiprocessing import Pool\nfrom dataset import FFRecordDataset\nimport os\nimport torch\nimport time\n\nclass FFRecordDataLoader(torch.utils.data.DataLoader):\n def __init__(\n self,\n dataset: FFRecordDataset,\n batch_size=1,\n shuffle: bool = False,\n sampler=None,\n batch_sampler=None,\n num_workers: int = 0,\n collate_fn=None,\n pin_memory: bool = False,\n drop_last: bool = False,\n timeout: float = 0,\n worker_init_fn=None,\n generator=None,\n *,\n prefetch_factor: int = 2,\n persistent_workers: bool = False,\n skippable: bool = True):\n\n # use fork to create subprocesses\n if num_workers == 0:\n multiprocessing_context = None\n dataset.initialize()\n else:\n multiprocessing_context = 'fork'\n self.skippable = skippable\n\n super(FFRecordDataLoader,\n self).__init__(dataset=dataset,\n batch_size=batch_size,\n shuffle=shuffle,\n sampler=sampler,\n batch_sampler=batch_sampler,\n num_workers=num_workers,\n collate_fn=collate_fn,\n pin_memory=pin_memory,\n drop_last=drop_last,\n timeout=timeout,\n worker_init_fn=worker_init_fn,\n multiprocessing_context=multiprocessing_context,\n generator=generator,\n prefetch_factor=prefetch_factor,\n persistent_workers=persistent_workers)\n\nif __name__ == \"__main__\":\n fnames = [\"/demo.ffr\"]\n\n dataset = FFRecordDataset(fnames, check_data=True)\n\n def worker_init_fn(worker_id):\n worker_info = torch.utils.data.get_worker_info()\n print(f\"Worker initialized pid: {os.getpid()}, work_info: {worker_info}\")\n dataset = worker_info.dataset\n dataset.initialize(worker_id=worker_id)\n\n def collate_fn(batch):\n return batch\n\n begin_time = time.time()\n\n dataloader = FFRecordDataLoader(dataset, batch_size=1, shuffle=True, num_workers=10, worker_init_fn=worker_init_fn, prefetch_factor=None, collate_fn=collate_fn)\n\n i=0\n for batch in dataloader:\n # print(i, \": \", batch[0][\"index\"], \"----\", time.time()-begin_time)\n i+=1\n if i>1000:\n break\n end_time = time.time()\n print(f\"takes: {end_time-begin_time}\")\n" }, { "path": "sdk/python/examples/ffrecord/dataset.py", "content": "# encoding: utf-8\n# JuiceFS, Copyright 2025 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport numpy as np\nfrom typing import List, Union\nfrom filereader import FileReader\n# from filereader_dio import FileReader\nimport torch\nimport os\n\nclass FFRecordDataset(torch.utils.data.Dataset):\n def __init__(self, fnames: Union[str, List[str]], check_data: bool = True):\n if isinstance(fnames, str):\n fnames = [fnames]\n self.reader = FileReader(fnames, check_data=check_data)\n self.n = self.reader.n\n self.reader.close_fd()\n\n def initialize(self, worker_id=0, num_workers=1):\n self.reader.open_fd()\n self.n = self.reader.n\n\n def __len__(self) -> int:\n return self.n\n\n def __getitem__(self, index: Union[int, List[int]]) -> Union[np.array, List[np.array]]:\n if isinstance(index, int):\n return self.reader.read_one(index)\n elif isinstance(index, list):\n return self.reader.read_batch(index)\n else:\n raise TypeError(f\"Index must be int or list, got {type(index)}\")\n\n def close(self):\n self.reader.close_fd()\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n self.close()\n\n\nif __name__ == \"__main__\":\n fnames = [\"/demo.ffr\"]\n\n with FFRecordDataset(fnames, check_data=True) as dataset:\n sample = dataset[0]\n print(\"Sample 0:\", sample)\n\n batch = dataset[[1, 2, 3]]\n print(batch)\n print(\"Dataset length:\", len(dataset))\n" }, { "path": "sdk/python/examples/ffrecord/filereader.py", "content": "# encoding: utf-8\n# JuiceFS, Copyright 2025 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nsys.path.append('.')\nfrom sdk.python.juicefs.juicefs import juicefs\n# import juicefs\nimport zlib\nfrom typing import Union\nimport struct\nimport os\nimport struct\nimport zlib\nfrom typing import List, Tuple, Optional\nimport io\nimport pickle\nimport numpy as np\n\nMAX_SIZE = 512 * (1 << 20) # 512 MB\n\ndef ffcrc32(code: int, data: Union[bytes, bytearray], length: int) -> int:\n start = 0\n while start < length:\n chunk_size = min(MAX_SIZE, length - start)\n code = zlib.crc32(data[start:start + chunk_size], code)\n start += chunk_size\n return code\n\nclass FileHeader:\n def __init__(self, jfscli: juicefs.Client, fname: str, check_data: bool = True):\n self.fname = fname\n self.fd = jfscli.open(fname, mode='rb')\n\n self.fd.seek(0)\n self.checksum_meta = self._read_uint32()\n self.n = self._read_uint64()\n\n self.checksums = [self._read_uint32() for _ in range(self.n)]\n self.fd.seek(4+8+4*self.n)\n self.offsets = [self._read_uint64() for _ in range(self.n + 1)]\n\n self.offsets[self.n] = jfscli.stat(fname).st_size\n\n if check_data:\n self.validate()\n self.fd.close()\n self.fd = jfscli.open(fname, mode='rb', buffering=0)\n self.aiofd = self.fd\n\n\n def _read_uint32(self) -> int:\n return struct.unpack(' int:\n return struct.unpack(' Tuple[int, int, int, int]:\n fd = self.aiofd if use_aio else self.fd\n offset = self.offsets[index]\n length = self.offsets[index + 1] - self.offsets[index]\n checksum = self.checksums[index]\n return fd, offset, length, checksum\n\n\nclass FileReader:\n def __init__(self, fnames: List[str], check_data: bool = True):\n self.fnames = fnames\n self.check_data = check_data\n self.nfiles = len(fnames)\n self.n = 1000\n self.nsamples = [0]\n self.headers = []\n\n def close_fd(self):\n for header in self.headers:\n header.close_fd()\n self.headers = []\n self.n = 0\n self.nsamples = [0]\n return\n \n def open_fd(self):\n self.v = juicefs.Client(\"myjfs\", \"redis://localhost\", cache_dir=\"/tmp/data\", cache_size=\"0\", debug=False)\n\n for fname in self.fnames:\n header = FileHeader(self.v, fname, self.check_data)\n self.headers.append(header)\n self.n += header.n\n self.nsamples.append(self.n)\n\n def validate(self):\n for header in self.headers:\n header.validate()\n\n def validate_sample(self, index: int, buf: bytes, checksum: int):\n if self.check_data:\n checksum2 = ffcrc32(0, buf, len(buf))\n assert checksum2 == checksum, f\"Sample {index}: checksum mismatched!\"\n\n def read(self, indices: List[int]):\n return self.read_batch(indices)\n\n def read_batch(self, indices: List[int]):\n assert not any(index >= self.n for index in indices), \"Index out of range\"\n results = []\n\n for index in indices:\n results.append(self.read_one(index))\n\n return results\n\n def read_one(self, index: int):\n assert index < self.n, \"Index out of range\"\n\n fid = 0\n while index >= self.nsamples[fid + 1]:\n fid += 1\n\n header = self.headers[fid]\n fd, offset, length, checksum = header.access(index - self.nsamples[fid], use_aio=False)\n\n fd.seek(offset)\n buf = fd.read(length)\n self.validate_sample(index, buf, checksum)\n res = pickle.loads(buf)\n return res\n \n def close(self):\n self.close_fd()\n\nif __name__ == \"__main__\":\n fnames = [\"/demo.ffr\"]\n reader = FileReader(fnames, check_data=True)\n reader.open_fd()\n data = reader.read_one(0)\n print(data)\n data = pickle.loads(data)\n print(data[\"index\"])\n print(data[\"txt\"])\n" }, { "path": "sdk/python/examples/ffrecord/filereader_dio.py", "content": "# encoding: utf-8\n# JuiceFS, Copyright 2025 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport zlib\nimport os\nimport struct\nfrom typing import List, Tuple, Union\nimport numpy as np\n\nMAX_SIZE = 512 * (1 << 20) # 512 MB\nDIRECTIO_BLOCK_SIZE = 1 * (1 << 20) # 1 MB\n\ndef ffcrc32(code: int, data: Union[bytes, bytearray], length: int) -> int:\n start = 0\n while start < length:\n chunk_size = min(MAX_SIZE, length - start)\n code = zlib.crc32(data[start:start + chunk_size], code)\n start += chunk_size\n return code\n\nclass FileHeader:\n def __init__(self, fname: str, check_data: bool = True):\n print(f\"__init__ self: {hex(id(self))}\")\n print(f\"pid: {os.getpid()}\")\n self.fname = fname\n self.fd = os.open(fname, os.O_RDONLY | os.O_DIRECT)\n self.aiofd = self.fd \n\n self.file_obj = os.fdopen(self.fd, 'rb', buffering=0)\n\n self.checksum_meta = self._read_uint32()\n self.n = self._read_uint64()\n\n checksums_size = 4 * self.n\n offsets_size = 8 * (self.n + 1)\n combined_data = self.file_obj.read(checksums_size + offsets_size)\n self.checksums = list(struct.unpack(f'<{self.n}I', combined_data[:checksums_size]))\n self.offsets = list(struct.unpack(f'<{self.n + 1}Q', combined_data[checksums_size:checksums_size + offsets_size]))\n\n self.offsets[self.n] = os.path.getsize(fname)\n if check_data:\n self.validate()\n\n print(\"FileHeader initialized for:\", fname, \"fd:\", self.fd)\n\n def _read_uint32(self) -> int:\n return struct.unpack(' int:\n return struct.unpack(' Tuple[int, int, int, int]:\n fd = self.aiofd if use_aio else self.fd\n offset = self.offsets[index]\n length = self.offsets[index + 1] - self.offsets[index]\n checksum = self.checksums[index]\n return fd, offset, length, checksum\n\n\nclass FileReader:\n def __init__(self, fnames: List[str], check_data: bool = True):\n self.fnames = fnames\n self.check_data = check_data\n self.nfiles = len(fnames)\n self.n = 0\n self.nsamples = [0]\n self.headers = []\n\n for fname in fnames:\n header = FileHeader(fname, check_data)\n self.headers.append(header)\n self.n += header.n\n self.nsamples.append(self.n)\n\n def close_fd(self):\n for header in self.headers:\n header.close_fd()\n \n def open_fd(self):\n print(f\"open_fd address: {hex(id(self))} pid: {os.getpid()}\")\n for header in self.headers:\n header.open_fd()\n\n def validate(self):\n for header in self.headers:\n header.validate()\n\n def validate_sample(self, index: int, buf: bytes, checksum: int):\n if self.check_data:\n checksum2 = ffcrc32(0, buf, len(buf))\n assert checksum2 == checksum, f\"Sample {index}: checksum mismatched!\"\n\n def read_batch(self, indices: List[int]) -> List[np.array]:\n assert not any(index >= self.n for index in indices), \"Index out of range\"\n results = []\n\n for index in indices:\n results.append(self.read_one(index))\n\n return results\n\n def read_one(self, index: int) -> np.array:\n assert index < self.n, \"Index out of range\"\n\n fid = 0\n while index >= self.nsamples[fid + 1]:\n fid += 1\n\n header = self.headers[fid]\n fd, offset, length, checksum = header.access(index - self.nsamples[fid], use_aio=False)\n\n buf = bytearray(length)\n start = 0\n while start < length:\n chunk_size = min(DIRECTIO_BLOCK_SIZE, length - start)\n read_bytes = os.pread(fd, chunk_size, offset + start)\n buf[start:start + chunk_size] = read_bytes\n start += chunk_size\n\n self.validate_sample(index, buf, checksum)\n array = np.frombuffer(buf, dtype=np.uint8)\n\n return array\n\n\nif __name__ == \"__main__\":\n fnames = [\"/demo.ffr\"]\n reader = FileReader(fnames, check_data=True)\n data = reader.read_one(0)\n print(data)\n" }, { "path": "sdk/python/examples/ffrecord/main.py", "content": "# encoding: utf-8\n# JuiceFS, Copyright 2025 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nfrom pathlib import Path\nimport loguru\nimport random\nimport pickle\nfrom multiprocessing import Pool\nimport numpy as np\nfrom PIL import Image\nfrom faker import Faker\nimport io\nimport time\nfrom tqdm import tqdm\nfrom ffrecord import FileWriter\nfrom ffrecord.torch import Dataset, DataLoader\nfrom ffrecord import FileReader\n\nlogger = loguru.logger\nfake = Faker()\n\ndef serialize(sample):\n return pickle.dumps(sample)\n\ndef deserialize(sample):\n return pickle.loads(sample)\n\ndef generate_random_image_np(\n width=256,\n height=256,\n format=\"JPEG\", # JPEG, PNG, WEBP\n quality=90, # only for JPEG/WEBP\n):\n image_np = np.random.randint(0, 255, (height, width, 3), dtype=np.uint8)\n img = Image.fromarray(image_np)\n \n img_bytes = io.BytesIO()\n img.save(img_bytes, format=format, quality=quality)\n return img_bytes.getvalue()\n\ndef generate_data_entry(\n idx,\n text=None,\n avg_width=1024,\n avg_height=1024,\n variance=50,\n possible_formats=[\"PNG\"],\n # possible_formats=[\"JPEG\", \"PNG\", \"WEBP\"],\n):\n \"\"\"\n - avg_width/avg_height ± variance\n \"\"\"\n image_format = random.choice(possible_formats).lower()\n\n width = random.randint(avg_width - variance, avg_width + variance)\n height = random.randint(avg_height - variance, avg_height + variance)\n width, height = max(width, 32), max(height, 32) \n \n img_bytes = generate_random_image_np(\n width=width,\n height=height,\n format=image_format.upper(),\n )\n \n if text is None:\n text = fake.sentence()\n \n return {\n \"index\": idx,\n \"txt\": text,\n image_format: img_bytes,\n }\n\ndef write_ffrecord():\n ffr_output = Path(ffrecord_file)\n if ffr_output.exists():\n logger.warning(f\"Output {ffr_output} exists, removing\") \n logger.info(f\"Generating {num_samples} samples\")\n with Pool(num_proc) as pool:\n data_to_write = list(\n tqdm(\n pool.imap_unordered(generate_data_entry, range(num_samples), chunksize=10),\n total=num_samples,\n desc=\"Generating data\"\n )\n )\n begin_time = time.time()\n writer = FileWriter(ffr_output, len(data_to_write))\n for i, data in enumerate(data_to_write):\n writer.write_one(serialize(data))\n # writer.write_one(data)\n writer.close()\n end_time = time.time()\n lmdb_size = ffr_output.stat().st_size\n logger.info(f\"FFRecord size: {lmdb_size / 1024 ** 3:.2f} GB\")\n logger.info(f\"Time taken to write: {end_time - begin_time:.2f} seconds\")\n\ndef read_ffrecord(batch_size: int):\n reader = FileReader([ffrecord_file], check_data=True)\n\n sample_indices = list(range(num_samples))\n random.Random(0).shuffle(sample_indices)\n sample_batches = [sample_indices[i: i + batch_size] for i in range(0, len(sample_indices), batch_size)]\n logger.info(f'Number of samples to read: {reader.n}, batch_size = {batch_size}, num_batches = {len(sample_batches)}')\n read_indices = set()\n begin_time = time.time()\n index_iter = sample_batches\n index_iter = tqdm(index_iter, desc=\"Reading data in batches\", total=len(sample_batches))\n\n for indices in index_iter:\n all_data = reader.read(indices)\n for data in all_data:\n data = deserialize(data)\n read_indices.add(data[\"index\"])\n end_time = time.time()\n reader.close()\n assert read_indices == set(range(num_samples))\n logger.info(f\"Read {len(read_indices)} samples in {end_time - begin_time:.2f} s: {len(read_indices) / (end_time - begin_time):.2f} samples/s\")\n\n\nclass MyDataset(Dataset):\n def __init__(self, fnames, check_data=True):\n self.reader = FileReader(fnames, check_data=check_data)\n\n def __len__(self):\n return self.reader.n\n\n def __getitem__(self, indices):\n data = self.reader.read(indices)\n samples = []\n\n for bytes_ in data:\n item = pickle.loads(bytes_)\n samples.append(item)\n\n return samples\n\n\nffrecord_file=\"/tmp/jfs/demo.ffr\"\nnum_samples=1000\nnum_proc=4\n\nif __name__ == \"__main__\":\n if len(sys.argv) > 1:\n if sys.argv[1] == \"write\":\n write_ffrecord()\n elif sys.argv[1] == \"read\":\n read_ffrecord(batch_size=1)\n else:\n begin_time = time.time()\n dataset = MyDataset([ffrecord_file], check_data=True)\n dataloader = DataLoader(dataset, batch_size=1, shuffle=True, num_workers=10,prefetch_factor=None)\n\n i=0\n for batch in dataloader:\n i+=1\n if i>1000:\n break\n end_time = time.time()\n print(f\"takes: {end_time-begin_time}\")\n" }, { "path": "sdk/python/examples/ffrecord/readme.md", "content": "```bash\n# This is a ffrecord dataloader example.\n# Prepare\n# Install ffrecord here: https://github.com/HFAiLab/ffrecord\n# Mount JuiceFS\njuicefs mount redis://localhost /tmp/jfs -d\n\n# Generate dataset\npython3 sdk/python/examples/ffrecord/main.py write\n# Simple read dataset\npython3 sdk/python/examples/ffrecord/main.py read\n# Read dataset with dataloader: (takes 39.55s)\npython3 sdk/python/examples/ffrecord/main.py\n\n# Prepare python-sdk\nmake -C sdk/python libjfs.so\n# Read dataset with Juicefs-pythonsdk-dataloader: (takes 10.02s)\npython3 sdk/python/examples/ffrecord/dataloader.py\n```" }, { "path": "sdk/python/examples/fsspec/main.py", "content": "import fsspec\nimport ray\nimport sys\nsys.path.append('.')\nimport sdk.python.juicefs.juicefs.spec\n# from sdk.python.juicefs.juicefs.spec import JuiceFS\n\nfs = fsspec.filesystem('https')\nds = ray.data.read_csv(\n \"https://gender-pay-gap.service.gov.uk/viewing/download-data/2021\",\n filesystem=fs,\n partition_filter=None # Since the file doesn't end in .csv\n)\nds.count()\n\nprint(\"----++++----++++----\")\n\njfs = fsspec.filesystem(\"jfs\", auto_mkdir=True, name=\"myjfs\", meta=\"redis://localhost\")\ndsjfs = ray.data.read_csv('/ray_demo_data.csv', filesystem=jfs)\ndsjfs.count()\n\n" }, { "path": "sdk/python/examples/fsspec/readme.md", "content": "```bash\n# This example demonstrates how to use the fsspec library to read a CSV file.\njuicefs mount redis://localhost /tmp/jfs -d\n# Download the data file\nwget https://gender-pay-gap.service.gov.uk/viewing/download-data/2021 -O /tmp/jfs/ray_demo_data.csv\n\n# run the example\npython3 sdk/python/examples/fsspec/main.py\n```" }, { "path": "sdk/python/juicefs/juicefs/__init__.py", "content": "from .juicefs import Client\n" }, { "path": "sdk/python/juicefs/juicefs/juicefs.py", "content": "# encoding: utf-8\n# JuiceFS, Copyright 2024 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport codecs\nimport errno\nimport grp\nimport io\nimport json\nimport locale\nimport os\nimport pwd\nimport six\nimport struct\nimport threading\nimport time\nfrom ctypes import *\n\n# pkg/vfs/helpers.go\nMODE_WRITE = 2\nMODE_READ = 4\n\nXATTR_CREATE = 1\nXATTR_REPLACE = 2\n\ndef check_error(r, fn, args):\n if fn.__name__ == \"jfs_init\" and r == 0:\n name = args[0].decode()\n e = OSError(f'JuiceFS initialized failed for {name}')\n e.errno = 1\n raise e\n elif r < 0:\n formatted_args = []\n for arg in args[2:]:\n if isinstance(arg, (bytes, bytearray)) and len(arg) > 1024:\n formatted_args.append(f'bytes(len={len(arg)})')\n else:\n formatted_args.append(repr(arg))\n\n e = OSError(f'call {fn.__name__} failed: [Errno {-r}] {os.strerror(-r)}: {formatted_args}')\n e.errno = -r\n raise e\n return r\n\nclass FileInfo(Structure):\n _fields_ = [\n ('inode', c_uint64),\n ('mode', c_uint32),\n ('uid', c_uint32),\n ('gid', c_uint32),\n ('atime', c_uint32),\n ('mtime', c_uint32),\n ('ctime', c_uint32),\n ('nlink', c_uint32),\n ('length', c_uint64),\n ]\n\ndef _tid():\n return threading.current_thread().ident\n\ndef _bin(s):\n return six.ensure_binary(s)\n\ndef unpack(fmt, buf):\n if not fmt.startswith(\"!\"):\n fmt = \"!\" + fmt\n return struct.unpack(fmt, buf[: struct.calcsize(fmt)])\n\n\nclass JuiceFSLib(object):\n def __init__(self):\n self.lib = cdll.LoadLibrary(os.path.join(os.path.dirname(__file__), \"libjfs.so\"))\n\n def __getattr__(self, n):\n fn = getattr(self.lib, n)\n if n == \"jfs_init\" or n == \"jfs_lseek\":\n fn.restype = c_int64\n fn.errcheck = check_error\n elif n.startswith(\"jfs\"):\n fn.restype = c_int32\n fn.errcheck = check_error\n return fn\n\nclass Client(object):\n \"\"\"A JuiceFS client.\"\"\"\n def __init__(self, name, meta, *, bucket=\"\", storage_class=\"\", read_only=False, no_session=False, no_bgjob=True,\n open_cache=\"0\", backup_meta=\"3600\", backup_skip_trash=False, heartbeat=\"12\",\n cache_dir=\"memory\", cache_size=\"100M\", free_space_ratio=\"0.1\", cache_partial_only=False,\n verify_cache_checksum=\"extend\", cache_eviction=\"2-random\", cache_scan_interval=\"3600\", cache_expire=\"0\",\n writeback=False, buffer_size=\"300M\", prefetch=1, max_readahead=\"0\", upload_limit=\"0\",\n download_limit=\"0\", max_uploads=20, max_deletes=10, skip_dir_nlink=20, skip_dir_mtime=\"100ms\",\n io_retries=10, get_timeout=\"5\", put_timeout=\"60\", fast_resolve=False, attr_cache=\"1s\",\n entry_cache=\"0s\", dir_entry_cache=\"1s\", debug=False, no_usage_report=False, access_log=\"\",\n push_gateway=\"\", push_interval=\"10\", push_auth=\"\", push_labels=\"\", push_graphite=\"\", push_remote_write=\"\", \n push_remote_write_auth=\"\"):\n self.lib = JuiceFSLib()\n kwargs = {}\n kwargs[\"meta\"] = meta\n kwargs[\"bucket\"] = bucket\n kwargs[\"storageClass\"] = storage_class\n kwargs[\"readOnly\"] = read_only\n kwargs[\"noSession\"] = no_session\n kwargs[\"noBGJob\"] = no_bgjob\n kwargs[\"openCache\"] = open_cache\n kwargs[\"backupMeta\"] = backup_meta\n kwargs[\"backupSkipTrash\"] = backup_skip_trash\n kwargs[\"heartbeat\"] = heartbeat\n kwargs[\"cacheDir\"] = cache_dir\n kwargs[\"cacheSize\"] = cache_size\n kwargs[\"freeSpace\"] = free_space_ratio\n kwargs[\"autoCreate\"] = True\n kwargs[\"cacheFullBlock\"] = not cache_partial_only\n kwargs[\"cacheChecksum\"] = verify_cache_checksum\n kwargs[\"cacheEviction\"] = cache_eviction\n kwargs[\"cacheScanInterval\"] = cache_scan_interval\n kwargs[\"cacheExpire\"] = cache_expire\n kwargs[\"writeback\"] = writeback\n kwargs[\"memorySize\"] = buffer_size\n kwargs[\"prefetch\"] = prefetch\n kwargs[\"readahead\"] = max_readahead\n kwargs[\"uploadLimit\"] = upload_limit\n kwargs[\"downloadLimit\"] = download_limit\n kwargs[\"maxUploads\"] = max_uploads\n kwargs[\"maxDeletes\"] = max_deletes\n kwargs[\"skipDirNlink\"] = skip_dir_nlink\n kwargs[\"skipDirMtime\"] = skip_dir_mtime\n kwargs[\"ioRetries\"] = io_retries\n kwargs[\"getTimeout\"] = get_timeout\n kwargs[\"putTimeout\"] = put_timeout\n kwargs[\"fastResolve\"] = fast_resolve\n kwargs[\"attrTimeout\"] = attr_cache\n kwargs[\"entryTimeout\"] = entry_cache\n kwargs[\"dirEntryTimeout\"] = dir_entry_cache\n kwargs[\"debug\"] = debug\n kwargs[\"noUsageReport\"] = no_usage_report\n kwargs[\"accessLog\"] = access_log\n kwargs[\"pushGateway\"] = push_gateway\n kwargs[\"pushInterval\"] = push_interval\n kwargs[\"pushAuth\"] = push_auth\n kwargs[\"pushLabels\"] = push_labels\n kwargs[\"pushGraphite\"] = push_graphite\n kwargs[\"pushRemoteWrite\"] = push_remote_write\n kwargs[\"pushRemoteWriteAuth\"] = push_remote_write_auth\n kwargs[\"caller\"] = 1\n\n jsonConf = json.dumps(kwargs, sort_keys=True)\n self.umask = os.umask(0)\n os.umask(self.umask)\n user = pwd.getpwuid(os.geteuid())\n groups = [grp.getgrgid(gid).gr_name for gid in os.getgrouplist(user.pw_name, user.pw_gid)]\n superuser = pwd.getpwuid(0)\n supergroups = [grp.getgrgid(gid).gr_name for gid in os.getgrouplist(superuser.pw_name, superuser.pw_gid)]\n self.h = self.lib.jfs_init(0, 0, name.encode(), jsonConf.encode(), user.pw_name.encode(), ','.join(groups).encode(), superuser.pw_name.encode(), ''.join(supergroups).encode())\n\n def __del__(self):\n self.lib.jfs_term(c_int64(_tid()), c_int64(self.h))\n\n def stat(self, path):\n \"\"\"Get the status of a file or a directory.\"\"\"\n fi = FileInfo()\n self.lib.jfs_stat(c_int64(_tid()), c_int64(self.h), _bin(path), byref(fi))\n return os.stat_result((fi.mode, fi.inode, 0, fi.nlink, fi.uid, fi.gid, fi.length, fi.atime, fi.mtime, fi.ctime))\n\n def exists(self, path):\n \"\"\"Check if a file exists.\"\"\"\n try:\n self.stat(path)\n return True\n except OSError as e:\n return False\n\n def open(self, path, mode='r', buffering=-1, encoding=None, errors=None):\n \"\"\"Open a file, returns a filelike object.\"\"\"\n if len(mode) != len(set(mode)):\n raise ValueError(f'invalid mode: {mode}')\n flag = 0\n cnt = 0\n for c in mode:\n if c in 'rwxa':\n cnt += 1\n if c == 'r':\n flag |= MODE_READ\n else:\n flag |= MODE_WRITE\n elif c == '+':\n flag |= MODE_READ | MODE_WRITE\n elif c not in 'tb':\n raise ValueError(f'invalid mode: {mode}')\n if cnt != 1:\n raise ValueError('must have exactly one of create/read/write/append mode')\n if 'b' in mode:\n if 't' in mode:\n raise ValueError(\"can't have text and binary mode at once\")\n if encoding:\n raise ValueError(\"binary mode doesn't take an encoding argument\")\n if errors:\n raise ValueError(\"binary mode doesn't take an errors argument\")\n else:\n if not encoding:\n encoding = locale.getpreferredencoding(False).lower()\n if not errors:\n errors = 'strict'\n codecs.lookup(encoding)\n\n size = 0\n if 'x' in mode:\n fd = self.lib.jfs_create(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint16(0o666), c_uint16(self.umask))\n else:\n try:\n sz = c_uint64()\n fd = self.lib.jfs_open_posix(c_int64(_tid()), c_int64(self.h), _bin(path), byref(sz), c_int32(flag))\n if 'w' in mode:\n self.lib.jfs_ftruncate(c_int64(_tid()), fd, c_uint64(0))\n else:\n size = sz.value\n except OSError as e:\n if e.errno != errno.ENOENT:\n raise e\n if 'r' in mode:\n raise FileNotFoundError(e)\n fd = self.lib.jfs_create(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint16(0o666), c_uint16(self.umask))\n return File(self.lib, fd, path, mode, flag, size, buffering, encoding, errors)\n\n def truncate(self, path, size):\n \"\"\"Truncate a file to a specified size.\"\"\"\n self.lib.jfs_truncate(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint64(size))\n\n def remove(self, path):\n \"\"\"Remove a file.\"\"\"\n self.lib.jfs_delete(c_int64(_tid()), c_int64(self.h), _bin(path))\n\n def mkdir(self, path, mode=0o777):\n \"\"\"Create a directory.\"\"\"\n self.lib.jfs_mkdir(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint16(mode&0o777), c_uint16(self.umask))\n\n def makedirs(self, path, mode=0o777, exist_ok=False):\n \"\"\"Create a directory and all its parent components if they do not exist.\"\"\"\n self.lib.jfs_mkdirAll(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint16(mode&0o777), c_uint16(self.umask), c_bool(exist_ok))\n\n def rmdir(self, path):\n \"\"\"Remove a directory. The directory must be empty.\"\"\"\n self.lib.jfs_rmdir(c_int64(_tid()), c_int64(self.h), _bin(path))\n\n def rename(self, old, new):\n \"\"\"Rename the file or directory old to new.\"\"\"\n self.lib.jfs_rename0(c_int64(_tid()), c_int64(self.h), _bin(old), _bin(new), c_uint32(0))\n\n def listdir(self, path, detail=False):\n \"\"\"Return a list containing the names of the entries in the directory given by path.\"\"\"\n buf = c_void_p()\n size = c_int()\n # func jfs_listdir(pid int, h int64, cpath *C.char, offset int, buf uintptr, bufsize int) int {\n\n self.lib.jfs_listdir2(c_int64(_tid()), c_int64(self.h), _bin(path), bool(detail), byref(buf), byref(size))\n data = string_at(buf, size)\n infos = []\n pos = 0\n while pos < len(data):\n nlen, = unpack(\"H\", data[pos:pos+2])\n pos += 2\n name = six.ensure_str(data[pos : pos + nlen], errors='replace')\n pos += nlen\n if detail:\n mode, inode, nlink, uid, gid, length, atime, mtime, ctime = \\\n unpack(\"IQIIIQIII\", data[pos:pos+44])\n infos.append((name, os.stat_result((mode, inode, 0, nlink, uid, gid, length, atime, mtime, ctime))))\n pos += 44\n else:\n infos.append(name)\n self.lib.free(buf)\n return sorted(infos)\n\n def chmod(self, path, mode):\n \"\"\"Change the mode of a file.\"\"\"\n self.lib.jfs_chmod(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint16(mode))\n\n def chown(self, path, uid, gid):\n \"\"\"Change the owner and group id of a file.\"\"\"\n self.lib.jfs_chown(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint32(uid), c_uint32(gid))\n\n def link(self, src, dst):\n \"\"\"Create a hard link to a file.\"\"\"\n self.lib.jfs_link(c_int64(_tid()), c_int64(self.h), _bin(src), _bin(dst))\n\n def lstat(self, path):\n \"\"\"Like stat(), but do not follow symbolic links.\"\"\"\n info = FileInfo()\n self.lib.jfs_lstat(c_int64(_tid()), c_int64(self.h), _bin(path), byref(info))\n return os.stat_result((info.mode, info.inode, 0, info.nlink, info.uid, info.gid, info.length, info.atime, info.mtime, info.ctime))\n\n def readlink(self, path):\n \"\"\"Return a string representing the path to which the symbolic link points.\"\"\"\n buf = bytes(1<<16)\n n = self.lib.jfs_readlink(c_int64(_tid()), c_int64(self.h), _bin(path), buf, c_int32(len(buf)))\n return buf[:n].decode()\n\n def symlink(self, src, dst):\n \"\"\"Create a symbolic link.\"\"\"\n self.lib.jfs_symlink(c_int64(_tid()), c_int64(self.h), _bin(src), _bin(dst))\n\n def unlink(self, path):\n \"\"\"Remove a file.\"\"\"\n self.lib.jfs_unlink(c_int64(_tid()), c_int64(self.h), _bin(path))\n\n def rmr(self, path):\n \"\"\"Remove a directory and all its contents recursively.\"\"\"\n self.lib.jfs_rmr(c_int64(_tid()), c_int64(self.h), _bin(path))\n\n def utime(self, path, times=None):\n \"\"\"Set the access and modified times of a file.\"\"\"\n if not times:\n now = time.time()\n times = (now, now)\n self.lib.jfs_utime(c_int64(_tid()), c_int64(self.h), _bin(path), c_int64(int(times[1]*1000)), c_int64(int(times[0]*1000)))\n\n def walk(self, top, topdown=True, onerror=None, followlinks=False):\n raise NotImplementedError\n\n def getxattr(self, path, name):\n \"\"\"Get an extended attribute on a file.\"\"\"\n size = 64 << 10 # XattrSizeMax\n buf = bytes(size)\n size = self.lib.jfs_getXattr(c_int64(_tid()), c_int64(self.h), _bin(path), _bin(name), buf, c_int32(size))\n return buf[:size]\n\n def listxattr(self, path):\n \"\"\"List extended attributes on a file.\"\"\"\n buf = c_void_p()\n size = c_int()\n self.lib.jfs_listXattr2(c_int64(_tid()), c_int64(self.h), _bin(path), byref(buf), byref(size))\n data = string_at(buf, size).decode()\n self.lib.free(buf)\n if not data:\n return []\n return data.split('\\0')[:-1]\n\n def setxattr(self, path, name, value, flags=0):\n \"\"\"Set an extended attribute on a file.\"\"\"\n value = _bin(value)\n self.lib.jfs_setXattr(c_int64(_tid()), c_int64(self.h), _bin(path), _bin(name), value, c_int32(len(value)), c_int32(flags))\n\n def removexattr(self, path, name):\n \"\"\"Remove an extended attribute from a file.\"\"\"\n self.lib.jfs_removeXattr(c_int64(_tid()), c_int64(self.h), _bin(path), _bin(name))\n\n def clone(self, src, dst, preserve=False):\n \"\"\"Clone a file or directory.\"\"\"\n self.lib.jfs_clone(c_int64(_tid()), c_int64(self.h), _bin(src), _bin(dst), c_bool(preserve))\n\n def set_quota(self, path, capacity=0, inodes=0, create=False, strict=False):\n \"\"\"Set the quota of a directory.\"\"\"\n self._quota(0, path, capacity, inodes, create=create, strict=strict)\n\n def get_quota(self, path):\n \"\"\"Get the quota of a directory.\"\"\"\n return self._quota(1, path)\n\n def del_quota(self, path):\n \"\"\"Delete the quota of a directory.\"\"\"\n self._quota(2, path)\n\n def list_quota(self):\n \"\"\"List the quota of all directories.\"\"\"\n return self._quota(3)\n\n def check_quota(self, path, repair=False, strict=False):\n \"\"\"Check the quota of a directory.\"\"\"\n return self._quota(4, path, repair=repair, strict=strict)\n\n def _quota(self, cmd, path=\"\", capacity=0, inodes=0, create=False, repair=False, strict=False):\n \"\"\"Get the quota of a directory.\"\"\"\n buf = c_void_p()\n n = self.lib.jfs_quota(c_int64(_tid()), c_int64(self.h), _bin(path), c_uint8(cmd), c_uint64(capacity), c_uint64(inodes), c_bool(strict), c_bool(repair), c_bool(create), byref(buf))\n data = string_at(buf, n)\n res = json.loads(str(data, encoding='utf-8'))\n self.lib.free(buf)\n return res\n\n def info(self, path, recursive=False, strict=False):\n \"\"\"Get the information of a file or a directory.\"\"\"\n buf = c_void_p()\n n = self.lib.jfs_info(c_int64(_tid()), c_int64(self.h), _bin(path), byref(buf), c_bool(recursive), c_bool(strict))\n data = string_at(buf, n)\n res = json.loads(str(data, encoding='utf-8'))\n\n self.lib.free(buf)\n return res\n\n def summary(self, path, depth=0, entries=1):\n \"\"\"Get the summary of a directory.\"\"\"\n buf = c_void_p()\n\n n = self.lib.jfs_gettreesummary(_tid(), self.h, _bin(path), c_uint8(depth), c_uint32(entries), byref(buf))\n data = string_at(buf, n)\n res = json.loads(str(data, encoding='utf-8'))\n\n def parseSummary(entry, removefields):\n for f in removefields:\n entry.pop(f, None)\n\n if entry[\"Dirs\"] == 0:\n entry.pop(\"Children\", None)\n elif entry.get(\"Children\") is not None:\n for v in entry[\"Children\"]:\n parseSummary(v, removefields)\n\n parseSummary(res, [\"Inode\"])\n self.lib.free(buf)\n return res\n\n def warmup(self, paths, threads=10, evict=False, check=False, background=False, **kwargs):\n # numthreads=10, background=False, isEvict=False, isCheck=False,\n for k in kwargs:\n if k == 'numthreads':\n threads = kwargs[k]\n elif k == 'isEvict':\n evict = kwargs[k]\n elif k == 'isCheck':\n check = kwargs[k]\n else:\n raise TypeError(f\"warmup() got an unexpected keyword argument '{k}'\")\n\n \"\"\"Warm up a file or a directory.\"\"\"\n if type(paths) is not list:\n paths = [paths]\n\n buf = c_void_p()\n\n n = self.lib.jfs_warmup(c_int64(_tid()), c_int64(self.h), json.dumps(paths).encode(), c_int32(threads), c_bool(background), c_bool(evict), c_bool(check), byref(buf))\n res = json.loads(str(string_at(buf, n), encoding='utf-8'))\n self.lib.free(buf)\n return res\n\n def status(self, trash=False, session=0):\n \"\"\"Get the status of the volume and client sessions.\"\"\"\n buf = c_void_p()\n n = self.lib.jfs_status(c_int64(_tid()), c_int64(self.h), c_bool(trash), c_bool(session), byref(buf))\n res = json.loads(str(string_at(buf, n), encoding='utf-8'))\n self.lib.free(buf)\n return res\n\n\nclass _File(object):\n \"\"\"A JuiceFS file.\"\"\"\n def __init__(self, lib, fd, path, mode, flag, length):\n self.lib = lib\n self.fd = fd\n self.name = path\n self.flag = flag\n self.length = length\n self.closed = False\n self.append = 'a' in mode\n self.off = self.length if self.append else 0\n\n def __fspath__(self):\n return self.name\n\n def readable(self):\n return self.flag & MODE_READ != 0\n\n def writable(self):\n return self.flag & MODE_WRITE != 0\n\n def seekable(self):\n return True\n\n def fileno(self):\n return self.fd\n\n def isatty(self):\n return False\n\n def read(self, size=-1):\n \"\"\"Read at most size bytes, returned as a byes.\"\"\"\n self._check_closed()\n if self.flag & MODE_READ == 0:\n raise io.UnsupportedOperation('not readable')\n # read directly\n rs = []\n got = 0\n while size > 0 or size < 0:\n n = 4 << 20\n if size > 0 and size < n:\n n = size\n buf = bytes(n)\n n = self.lib.jfs_pread(c_int64(_tid()), c_int32(self.fd), buf, c_int32(n), c_int64(self.off+got))\n if n == 0:\n break\n if n < len(buf):\n buf = buf[:n]\n rs.append(buf)\n got += n\n if size > 0:\n size -= n\n if len(rs) == 1:\n buf = rs[0]\n else:\n buf = b''.join(rs)\n self.off += len(buf)\n return buf\n\n def readinto(self, buffer):\n data = self.read(len(buffer))\n if not data:\n return 0\n buffer[:len(data)] = data\n return len(data)\n\n def write(self, data):\n \"\"\"Write the string data to the file.\"\"\"\n self._check_closed()\n if isinstance(data, memoryview):\n data = data.tobytes()\n if not isinstance(data, six.binary_type):\n raise TypeError(f\"a bytes-like object is required, not '{type(data).__name__}'\")\n if not self.writable():\n raise io.UnsupportedOperation('not writable')\n\n if not data:\n return 0\n if self.append:\n self.off = self.length\n n = self.lib.jfs_pwrite(c_int64(_tid()), c_int32(self.fd), data, c_int32(len(data)), c_int64(self.off))\n self.off += n\n if self.off > self.length:\n self.length = self.off\n return n\n\n def seek(self, offset, whence=0):\n \"\"\"Set the stream position to the given byte offset.\n offset is interpreted relative to the position indicated by whence.\n The default value for whence is SEEK_SET.\"\"\"\n self._check_closed()\n if whence not in (os.SEEK_SET, os.SEEK_CUR, os.SEEK_END):\n raise ValueError(f'invalid whence ({whence}, should be {os.SEEK_SET}, {os.SEEK_CUR} or {os.SEEK_END})')\n if whence == os.SEEK_SET:\n self.off = offset\n elif whence == os.SEEK_CUR:\n self.off += offset\n else:\n self.off = self.length + offset\n return self.off\n\n def tell(self):\n \"\"\"Return the current stream position.\"\"\"\n self._check_closed()\n return self.off\n\n def truncate(self, size=None):\n \"\"\"Truncate the file to at most size bytes.\n Size defaults to the current file position, as returned by tell().\"\"\"\n self._check_closed()\n if not self.writable():\n raise io.UnsupportedOperation('File not open for writing')\n if size is None:\n size = self.tell()\n self.lib.jfs_ftruncate(c_int64(_tid()), c_int32(self.fd), c_uint64(size))\n self.length = size\n return size\n\n def flush(self):\n return\n\n def fsync(self):\n self.lib.jfs_fsync(c_int64(_tid()), c_int32(self.fd))\n\n def close(self):\n if self.closed:\n return\n self.lib.jfs_close(c_int64(_tid()), c_int32(self.fd))\n self.closed = True\n\n def __del__(self):\n self.close()\n\n def _check_closed(self):\n if self.closed:\n raise ValueError('I/O operation on closed file.')\n\n def readline(self): # TODO: add parameter `size=-1`\n \"\"\"Read until newline or EOF.\"\"\"\n ls = self.readlines(1)\n if ls:\n return ls[0]\n return b''\n\n def xreadlines(self):\n return self\n\n def readlines(self, hint=-1):\n \"\"\"Return a list of lines from the stream.\"\"\"\n self._check_closed()\n if hint == -1:\n data = self.read(-1)\n else:\n rs = []\n while hint > 0:\n r = self.read(1)\n if not r:\n break\n rs.append(r)\n if r[0] == b'\\n':\n hint -= 1\n data = b''.join(rs)\n return data.splitlines(True)\n\n def writelines(self, lines):\n \"\"\"Write a list of lines to the file.\"\"\"\n self._check_closed()\n self.write(b''.join(lines))\n self.flush()\n\nclass File(object):\n \"\"\"A JuiceFS file.\"\"\"\n def __init__(self, lib, fd, path, mode, flag, length, buffering, encoding=None, errors=None):\n self._file = _File(lib, fd, path, mode, flag, length)\n\n if buffering < 0:\n buffering = 128<<10\n\n if buffering == 0:\n self.raw_io = self._file\n elif self._file.readable():\n if self._file.writable():\n self.raw_io = io.BufferedRandom(self._file, buffer_size=buffering)\n else:\n self.raw_io = io.BufferedReader(self._file, buffer_size=buffering)\n else:\n self.raw_io = io.BufferedWriter(self._file, buffer_size=buffering)\n\n if encoding:\n self.io = io.TextIOWrapper(self.raw_io, encoding=encoding, errors=errors)\n else:\n self.io = self.raw_io\n\n def __getattr__(self, name):\n return getattr(self.io, name)\n\n def __fspath__(self):\n return self._file.name\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_value, traceback):\n self.close()\n\n def __iter__(self):\n return self\n\n def __next__(self):\n return self.next()\n\n def next(self):\n lines = self.readlines(1)\n if lines:\n return lines[0]\n raise StopIteration\n\n def fileno(self):\n return self._file.fd\n\n def isatty(self):\n return False\n\n def fsync(self):\n \"\"\"Force write file data to the backend storage.\"\"\"\n self.io.flush()\n return self._file.fsync()\n\n def close(self):\n \"\"\"Close the file. A closed file cannot be used for further I/O operations.\"\"\"\n self.io.close()\n self._file.close()\n\n\ndef test():\n volume = os.getenv(\"JFS_VOLUME\", \"test\")\n meta = os.getenv(\"JFS_META\", \"redis://localhost\")\n v = Client(volume, meta, access_log=\"/tmp/jfs.log\")\n with v.open(\"/.config\", \"r\") as f:\n print(f.read())\n with v.open(\"/.stats\", \"r\") as f:\n print(f.read())\n print(v.status())\n st = v.stat(\"/\")\n print(st)\n if v.exists(\"/d\"):\n v.rmr(\"/d\")\n v.makedirs(\"/d\")\n if v.exists(\"/d/file\"):\n v.remove(\"/d/file\")\n with v.open(\"/d/file\", \"w\") as f:\n f.write(\"hello\")\n with v.open(\"/d/file\", \"a+\") as f:\n f.write(\"world\")\n with v.open(\"/d/file\") as f:\n data = f.read()\n assert data == \"helloworld\"\n with v.open(\"/d/file\", \"w\") as f:\n f.write(\"hello\")\n with v.open(\"/d/file\", 'rb', 5) as f:\n data = f.readlines()\n assert data == [b\"hello\"]\n with v.open(\"/d/file\", 'rb', 0) as f:\n data = f.readlines()\n assert data == [b\"hello\"]\n print(list(v.open(\"/d/file\")))\n assert list(v.open(\"/d/file\")) == ['hello']\n try:\n v.open(\"/d/d/file\", \"w\")\n except OSError as e:\n if e.errno != errno.ENOENT:\n raise e\n else:\n raise AssertionError\n v.chmod(\"/d/file\", 0o777)\n # v.chown(\"/d/file\", 0, 0)\n v.symlink(\"/d/file\", \"/d/link\")\n assert v.readlink(\"/d/link\") == \"file\"\n v.unlink(\"/d/link\")\n v.link(\"/d/file\", \"/d/link\")\n v.rename(\"/d/link\", \"/d/link2\")\n names = sorted(v.listdir(\"/d\"))\n assert names == [\"file\", \"link2\"]\n v.setxattr(\"/d/file\", \"user.key\", b\"value\\0\")\n xx = v.getxattr(\"/d/file\", \"user.key\")\n assert xx == b\"value\\0\"\n print(v.listxattr(\"/d/file\"))\n assert v.listxattr(\"/d/file\") == [\"user.key\"]\n v.removexattr(\"/d/file\", \"user.key\")\n assert v.listxattr(\"/d/file\") == []\n with v.open(\"/d/file\", \"a\") as f:\n f.seek(0, 0)\n f.write(\"world\")\n assert f.truncate(2) == 2\n assert f.seek(0, 2) == 2\n assert v.open(\"/d/file\").read() == \"he\"\n k=1024\n start = time.time()\n size = 0\n with v.open(\"/bigfile\", mode=\"wb\") as f:\n for i in range(4000):\n f.write(b\"!\"*(k*k))\n size += k*k\n print(\"write time:\", time.time()-start, size>>20)\n start = time.time()\n size = 0\n with v.open(\"/bigfile\",mode='rb') as f:\n while True:\n t = f.read(4*k)\n if not t: break\n size += len(t)\n print(\"read time:\", time.time()-start, size>>20)\n v.remove(\"/bigfile\")\n v.rmr(\"/d\")\n\n\nif __name__ == '__main__':\n test()\n" }, { "path": "sdk/python/juicefs/juicefs/spec.py", "content": "# encoding: utf-8\n# JuiceFS, Copyright 2024 Juicedata, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport datetime\nimport logging\nimport uuid\nimport os\nfrom stat import S_ISDIR, S_ISLNK, S_ISREG\n\nfrom fsspec.spec import AbstractFileSystem, AbstractBufferedFile\n\nfrom .juicefs import Client\n\nlogger = logging.getLogger(\"fsspec.jfs\")\n\n\nclass JuiceFS(AbstractFileSystem):\n \"\"\"\n A JuiceFS file system.\n \"\"\"\n protocol = \"jfs\", \"juicefs\"\n def __init__(self, name, auto_mkdir=False, **kwargs):\n if self._cached:\n return\n super().__init__(**kwargs)\n self.auto_mkdir = auto_mkdir\n self.temppath = kwargs.pop(\"temppath\", \"/tmp\")\n self.fs = Client(name, **kwargs)\n\n @property\n def fsid(self):\n return \"jfs_\" + self.fs.name\n\n def makedirs(self, path, exist_ok=False, mode=511):\n if self.exists(path) and not exist_ok:\n raise FileExistsError(f\"File exists: {path}\")\n self.fs.makedirs(self._strip_protocol(path), mode, exist_ok=exist_ok)\n\n def mkdir(self, path, create_parents=True, mode=0o511):\n if self.exists(path):\n raise FileExistsError(f\"File exists: {path}\")\n if create_parents:\n self.fs.makedirs(self._strip_protocol(path), mode=mode)\n else:\n self.fs.mkdir(self._strip_protocol(path), mode)\n\n def rmdir(self, path):\n self.fs.rmdir(self._strip_protocol(path))\n\n def ls(self, path, detail=False, **kwargs):\n infos = self.fs.listdir(self._strip_protocol(path), detail)\n if not detail:\n return infos\n stats = []\n for name, st in infos:\n info = {\n \"name\": os.path.join(path, name),\n \"size\": st.st_size,\n \"type\": \"directory\" if S_ISDIR(st.st_mode) else \"link\" if S_ISLNK(st.st_mode) else \"file\",\n \"mode\": st.st_mode,\n \"ino\": st.st_ino,\n \"nlink\": st.st_nlink,\n \"uid\": st.st_uid,\n \"gid\": st.st_gid,\n \"created\": st.st_atime,\n \"mtime\": st.st_mtime,\n }\n if S_ISLNK(st.st_mode):\n info.update(**self.info(f\"{path}/{name}\"))\n stats.append(info)\n return stats\n\n def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):\n if total:\n info = self.info(path)\n return info[\"size\"]\n return super().du(path, total=total, maxdepth=maxdepth, withdirs=withdirs, **kwargs)\n\n def info(self, path):\n path = self._strip_protocol(path)\n try:\n st = self.fs.lstat(path)\n except OSError:\n raise FileNotFoundError(path)\n info = {\n \"name\": path,\n }\n if S_ISLNK(st.st_mode):\n info['destination'] = self.fs.readlink(path)\n st = self.fs.stat(path)\n info.update({\n \"type\": \"directory\" if S_ISDIR(st.st_mode) else \"file\" if S_ISREG(st.st_mode) else \"other\",\n \"size\": st.st_size,\n \"uid\": st.st_uid,\n \"gid\": st.st_gid,\n \"created\": st.st_atime,\n \"mtime\": st.st_mtime,\n })\n return info\n\n def lexists(self, path, **kwargs):\n try:\n self.fs.lstat(self._strip_protocol(path))\n return True\n except OSError:\n return False\n\n def cp_file(self, path1, path2, **kwargs):\n if self.isfile(path1):\n if self.auto_mkdir:\n self.makedirs(self._parent(path2), exist_ok=True)\n self.fs.clone(self._strip_protocol(path1), self._strip_protocol(path2))\n else:\n self.mkdirs(path2, exist_ok=True)\n\n def rm(self, path, recursive=False, maxdepth=None):\n if not isinstance(path, list):\n path = [path]\n for p in path:\n if recursive:\n self.fs.rmr(self._strip_protocol(p))\n else:\n self.fs.remove(self._strip_protocol(p))\n\n def _rm(self, path):\n self.fs.remove(self._strip_protocol(path))\n\n def mv(self, old, new, recursive=False, maxdepth=None, **kwargs):\n self.fs.rename(self._strip_protocol(old), self._strip_protocol(new))\n\n def link(self, src, dst, **kwargs):\n src = self._strip_protocol(src)\n dst = self._strip_protocol(dst)\n self.fs.link(src, dst, **kwargs)\n\n def symlink(self, src, dst, **kwargs):\n src = self._strip_protocol(src)\n dst = self._strip_protocol(dst)\n self.fs.symlink(src, dst, **kwargs)\n\n def islink(self, path) -> bool:\n try:\n self.fs.readlink(self._strip_protocol(path))\n return True\n except OSError:\n return False\n\n def _open(self, path, mode=\"rb\", block_size=None, autocommit=True, **kwargs):\n path = self._strip_protocol(path)\n if self.auto_mkdir and \"w\" in mode:\n self.makedirs(self._parent(path), exist_ok=True)\n return JuiceFile(self, path, mode, block_size, autocommit, **kwargs)\n\n def touch(self, path, truncate=True, **kwargs):\n path = self._strip_protocol(path)\n if self.auto_mkdir:\n self.makedirs(self._parent(path), exist_ok=True)\n if truncate or not self.exists(path):\n with self.open(path, \"wb\", **kwargs):\n pass\n else:\n self.fs.utime(self._strip_protocol(path))\n\n @classmethod\n def _parent(cls, path):\n path = cls._strip_protocol(path)\n if os.sep == \"/\":\n # posix native\n return path.rsplit(\"/\", 1)[0] or \"/\"\n else:\n # NT\n path_ = path.rsplit(\"/\", 1)[0]\n if len(path_) <= 3:\n if path_[1:2] == \":\":\n # nt root (something like c:/)\n return path_[0] + \":/\"\n # More cases may be required here\n return path_\n\n def created(self, path):\n return datetime.datetime.fromtimestamp(\n self.info(path)[\"created\"], tz=datetime.timezone.utc\n )\n\n def modified(self, path):\n return datetime.datetime.fromtimestamp(\n self.info(path)[\"mtime\"], tz=datetime.timezone.utc\n )\n\n def _isfilestore(self):\n # Inheriting from DaskFileSystem makes this False (S3, etc. were)\n # the original motivation. But we are a posix-like file system.\n # See https://github.com/dask/dask/issues/5526\n return True\n\n def chmod(self, path, mode):\n path = self._strip_protocol(path)\n return self.fs.chmod(path, mode)\n\n\nclass JuiceFile(AbstractBufferedFile):\n def __init__(self, fs, path, mode=\"rb\", block_size=None, autocommit=True, cache_options=None, **kwargs):\n super().__init__(fs, path, mode, block_size, autocommit, cache_options=cache_options, **kwargs)\n if autocommit:\n self.temp = path\n self.f = None\n self._open()\n\n def _open(self):\n if self.f is None or self.f.closed:\n if self.autocommit or \"w\" not in self.mode:\n self.f = self.fs.fs.open(self.path, self.mode, buffering=self.blocksize)\n else:\n self.temp = \"/\".join([self.fs.temppath, str(uuid.uuid4())])\n self.f = open(self.temp, self.mode, buffering=self.blocksize)\n if \"w\" not in self.mode:\n self.size = self.f.seek(0, 2)\n self.f.seek(0)\n\n def _fetch_range(self, start, end):\n # probably only used by cached FS\n if \"r\" not in self.mode:\n raise ValueError\n self._open()\n self.f.seek(start)\n return self.f.read(end - start)\n\n def __setstate__(self, state):\n self.f = None\n loc = state.pop(\"loc\", None)\n self.__dict__.update(state)\n if \"r\" in state[\"mode\"]:\n self.f = None\n self._open()\n self.f.seek(loc)\n\n def __getstate__(self):\n d = self.__dict__.copy()\n d.pop(\"f\")\n if \"r\" in self.mode:\n d[\"loc\"] = self.f.tell()\n else:\n if not self.f.closed:\n raise ValueError(\"Cannot serialise open write-mode local file\")\n return d\n\n def commit(self):\n if self.autocommit:\n raise RuntimeError(\"Can only commit if not already set to autocommit\")\n self.fs.fs.rename(self.temp, self.path)\n\n def discard(self):\n if self.autocommit:\n raise RuntimeError(\"Can only commit if not already set to autocommit\")\n self.fs.fs.remove(self.temp)\n\n def tell(self):\n return self.f.tell()\n\n def seek(self, loc, whence=0):\n return self.f.seek(loc, whence)\n\n def write(self, data):\n return self.f.write(data)\n\n def read(self, length=-1):\n return self.f.read(length)\n\n def flush(self, force=True):\n return self.f.flush()\n\n def truncate(self, size=None):\n return self.f.truncate(size)\n\n def close(self):\n super().close()\n if getattr(self, \"_unclosable\", False):\n return\n self.f.close()\n\n def __getattr__(self, item):\n return getattr(self.f, item)\n\n def __del__(self):\n pass\n\nfrom fsspec.registry import register_implementation\nregister_implementation(\"jfs\", JuiceFS, True)\nregister_implementation(\"juicefs\", JuiceFS, True)\n" }, { "path": "sdk/python/juicefs/setup.py", "content": "from setuptools import setup, find_packages\n\n# The following line will be replaced by the actual version number during the Make process\nVERSION = \"1.3.0\"\nBUILD_INFO = \"BUILDDATE+COMMIT HASH\"\n\n\nsetup(\n name='juicefs',\n version=VERSION,\n description=BUILD_INFO,\n package_data={'juicefs': ['*.so']},\n packages=find_packages(where=\".\"),\n include_package_data=True,\n install_requires=['six'],\n entry_points={\n 'fsspec.specs': [\n 'jfs = juicefs.JuiceFS',\n ],\n },\n)\n" }, { "path": "sdk/python/juicefs/tests/__init__.py", "content": "" }, { "path": "sdk/python/juicefs/tests/test.py", "content": "import pytest\n\nfrom fsspec import filesystem\nimport fsspec.tests.abstract as abstract\n\nfrom juicefs.spec import JuiceFS\nimport os\n\nclass JuiceFSFixtures(abstract.AbstractFixtures):\n @pytest.fixture(scope=\"class\")\n def fs(self):\n meta = os.getenv(\"JUICEFS_META\", \"redis://localhost\")\n m = filesystem(\"jfs\", auto_mkdir=True, name=\"test\", meta=meta)\n return m\n\n @pytest.fixture\n def fs_path(self, tmpdir):\n return str(tmpdir)\n\n\nclass TestJuiceFSGet(abstract.AbstractGetTests, JuiceFSFixtures):\n pass\n\n\nclass TestJuiceFSPut(abstract.AbstractPutTests, JuiceFSFixtures):\n pass\n\n\nclass TestJuiceFSCopy(abstract.AbstractCopyTests, JuiceFSFixtures):\n pass\n" } ]