Repository: duna-oss/flystorage Branch: main Commit: 29a09a1e4d0e Files: 128 Total size: 258.2 KB Directory structure: gitextract_ft0asde_/ ├── .github/ │ ├── actions/ │ │ └── install-dependencies/ │ │ └── action.yml │ ├── dependabot.yml │ └── workflows/ │ ├── automerge-dependabot.yml │ ├── compile.yml │ ├── qa-ubuntu-bun.yml │ ├── qa-ubuntu-node.yml │ ├── qa-windows-node.yml │ └── shared-tests.yml ├── .gitignore ├── bin/ │ └── watch.ts ├── docker/ │ └── sftp/ │ ├── id_rsa │ ├── id_rsa.pub │ ├── ssh_host_ed25519_key │ ├── ssh_host_ed25519_key.pub │ ├── ssh_host_rsa_key │ ├── ssh_host_rsa_key.pub │ ├── sshd_custom_configs.sh │ ├── unknown.key │ └── users.conf ├── docker-compose.yml ├── fixtures/ │ └── adapter.template.ts ├── package.json ├── packages/ │ ├── aws-s3/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── aws-s3-file-storage.test.ts │ │ │ ├── aws-s3-storage-adapter.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── azure-storage-blob/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── azure-storage-blob.test.ts │ │ │ ├── azure-storage-blob.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── chaos/ │ │ ├── .npmignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── changelog.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── index.test.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── dynamic-import/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── index.d.ts │ │ ├── index.js │ │ ├── index.test.ts │ │ ├── package.json │ │ └── src/ │ │ └── index.ts │ ├── file-storage/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── checksum-from-stream.ts │ │ │ ├── errors.ts │ │ │ ├── file-storage.test.ts │ │ │ ├── file-storage.ts │ │ │ ├── index.ts │ │ │ ├── path-normalizer.test.ts │ │ │ ├── path-normalizer.ts │ │ │ ├── path-prefixer.test.ts │ │ │ ├── path-prefixer.ts │ │ │ ├── portable-visibility.ts │ │ │ ├── readable-convertion.test.ts │ │ │ └── utilities.test.ts │ │ └── tsconfig.json │ ├── google-cloud-storage/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── google-cloud-storage.test.ts │ │ │ ├── google-cloud-storage.ts │ │ │ ├── index.ts │ │ │ └── visibility-handling.ts │ │ └── tsconfig.json │ ├── in-memory/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── in-memory-file-storage.test.ts │ │ │ ├── in-memory-file-storage.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── local-fs/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── index.ts │ │ │ ├── local-file-storage.test.ts │ │ │ ├── local-file-storage.ts │ │ │ └── unix-visibility.ts │ │ └── tsconfig.json │ ├── multer-storage/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── index.test.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ └── stream-mime-type/ │ ├── .npmignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── package.json │ ├── src/ │ │ ├── index.ts │ │ ├── stream-mime-type.test.ts │ │ └── stream-mime-type.ts │ └── tsconfig.json ├── packed/ │ ├── .gitignore │ ├── cjs/ │ │ ├── index.js │ │ ├── index.test.js │ │ └── package.json │ └── esm/ │ ├── index.js │ └── package.json ├── tsconfig.build.json ├── tsconfig.json └── vitest.config.ts ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/actions/install-dependencies/action.yml ================================================ name: 'Install NPM Dependencies' description: 'Install NPM Dependencies' inputs: cache-key: description: 'Cache key' required: true cache-restore-keys: description: 'Cache restore keys' required: true outputs: cache-hit: description: 'Whether or not the cache was hit' value: ${{ steps.npm-cache.outputs.cache-hit }} runs: using: 'composite' steps: - run: mkdir ~/.npm-cache shell: sh name: Create NPM cache directory - name: Load NPM cache id: npm-cache uses: actions/cache/restore@v3 with: path: | ~/.npm-cache node_modules key: ${{ inputs.cache-key }} restore-keys: ${{ inputs.cache-restore-keys }} - name: Install NPM dependencies if: steps.npm-cache.outputs.cache-hit != 'true' run: npm ci --prefer-offline --no-audit shell: sh - name: Save NPM cache if: steps.npm-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: key: ${{ inputs.cache-key }} path: | ~/.npm-cache node_modules ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: "npm" directory: "/" schedule: interval: "weekly" groups: dependencies: patterns: - "*" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" ================================================ FILE: .github/workflows/automerge-dependabot.yml ================================================ name: Auto-merge Dependabot PRs on: pull_request_target: paths: - package.json - package-lock.json branches: - main permissions: pull-requests: write contents: write jobs: merge-dependabot-pr: name: Merge dependabot PR runs-on: ubuntu-latest if: ${{ github.actor == 'dependabot[bot]' }} steps: - name: Fetch Dependabot metadata id: meta uses: dependabot/fetch-metadata@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Merge PR if: ${{ steps.meta.outputs.update-type != 'version-update:semver-major' }} run: gh pr merge --auto --squash "$PR_URL" env: PR_URL: ${{ github.event.pull_request.html_url }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/compile.yml ================================================ name: Compile libraries on: push: branches: - main pull_request: branches: - main jobs: run-tests: runs-on: ${{ matrix.platform }} strategy: matrix: platform: [ubuntu-latest] node: [22] # latest LTS version name: Compile libraries steps: - name: Checkout the repository uses: actions/checkout@v6 - name: Install Node.js uses: actions/setup-node@v6 with: node-version: ${{ matrix.node }} - uses: ./.github/actions/install-dependencies name: Install NPM dependencies with: cache-key: npm-${{ matrix.node }}-${{ hashFiles('package-lock.json') }} cache-restore-keys: npm-caches-${{ matrix.node }} - name: Run build run: npm run build:only ================================================ FILE: .github/workflows/qa-ubuntu-bun.yml ================================================ name: Ubuntu + Bun on: push: branches: - main pull_request: branches: - main concurrency: group: ${{ github.workflow }}-${{ github.ref }}-ubuntu-bun cancel-in-progress: true jobs: tests: uses: ./.github/workflows/shared-tests.yml secrets: inherit with: os: ubuntu-latest runtime: bun # as string for inputs workaround not accepting arrays versions: '["1.2"]' experimental: true ================================================ FILE: .github/workflows/qa-ubuntu-node.yml ================================================ name: Ubuntu + Node on: push: branches: - main pull_request: branches: - main concurrency: group: ${{ github.workflow }}-${{ github.ref }}-ubuntu-node cancel-in-progress: true jobs: tests: uses: ./.github/workflows/shared-tests.yml secrets: inherit with: os: ubuntu-latest runtime: node # as string for inputs workaround not accepting arrays versions: '[20, 21, 22, 23, 24]' ================================================ FILE: .github/workflows/qa-windows-node.yml ================================================ name: Ubuntu + Windows on: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-windows-node cancel-in-progress: true jobs: tests: uses: ./.github/workflows/shared-tests.yml secrets: inherit with: experimental: true os: windows-latest runtime: node # as string for inputs workaround not accepting arrays versions: '[20, 21, 22, 23, 24]' ================================================ FILE: .github/workflows/shared-tests.yml ================================================ name: Tests on: workflow_call: secrets: GCS_KEY_CONTENTS: required: true AWS_ACCESS_KEY_ID: required: true AWS_SECRET_ACCESS_KEY: required: true AWS_REGION: required: true AZURE_DSN: required: true inputs: os: required: true type: string runtime: required: true type: string versions: required: true type: string experimental: required: false type: boolean default: false #permissions: # id-token: write # This is required for requesting the JWT # contents: read # This is required for actions/checkout # pull-requests: read # Required for change detection jobs: run-tests: runs-on: ${{ inputs.os }} continue-on-error: ${{ inputs.experimental }} strategy: matrix: version: ${{ fromJSON(inputs.versions) }} name: ${{ inputs.os }} - ${{ inputs.runtime }} - ${{ matrix.version }} steps: - name: Checkout the repository uses: actions/checkout@v6 - name: Install Node.js if: inputs.runtime == 'node' uses: actions/setup-node@v6 with: node-version: ${{ matrix.version }} - uses: ./.github/actions/install-dependencies if: inputs.runtime == 'node' name: Install NPM dependencies with: cache-key: ${{inputs.os}}-${{inputs.runtime}}-npm-${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }} cache-restore-keys: ${{inputs.os}}-${{inputs.runtime}}-npm-${{ matrix.node }} - uses: oven-sh/setup-bun@v2 if: inputs.runtime == 'bun' with: bun-version: ${{ inputs.version }} - name: Install NPM dependencies using Bun if: inputs.runtime == 'bun' run: bun install - name: Configure GCP access run: | echo $CONTENTS > google-cloud-service-account.json env: CONTENTS: "${{ secrets.GCS_KEY_CONTENTS }}" - name: Run relevant tests run: "npm run test" if: inputs.runtime == 'node' env: AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" AWS_REGION: "${{ secrets.AWS_REGION }}" AZURE_DSN: "${{ secrets.AZURE_DSN }}" - name: Run relevant tests using Bun run: | npm run build:only bun test packages if: inputs.runtime == 'bun' env: AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" AWS_REGION: "${{ secrets.AWS_REGION }}" AZURE_DSN: "${{ secrets.AZURE_DSN }}" - name: Delete GCP access if: always() run: | node_modules/.bin/rimraf google-cloud-service-account.json ================================================ FILE: .gitignore ================================================ google-cloud-service-account.json dist .env **/*/dist node_modules bin/* !bin/watch.ts fixtures/test-files/ .node-version ================================================ FILE: bin/watch.ts ================================================ import {readdir} from 'node:fs/promises'; import path from 'node:path'; import {concurrently} from 'concurrently'; const directories = (await readdir(path.resolve(process.cwd(), 'packages'), { withFileTypes: true, })) .filter(item => item.isDirectory()) .map(dir => path.parse(dir.name).name); const commands = directories.map(directory => ({ name: directory, command: `npm run watch -w ./packages/${directory} --if-present` })); console.log('starting', commands); concurrently(commands); ================================================ FILE: docker/sftp/id_rsa ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEA0cx7jSu+FZQJFmz0+vlbo1I+awlaxvqZUseuPvX/YA32c3hE nr0OpEMhe/Pvs4jk/BiPp5CfITFe6ykxUvNlvxh/9EOsnUFaFrfyLhYn3TicW6lv zSJmJ7tydW/L0zpMPliItbjtXItxHOyoVcl20+DT+LZWlRRnnuSDTC1Oc6vvLosw b4lOTNVB2kV7W9urU6W+OilcERwjjQpiIM4mkwg2dX0aPWS5o0N9tYi3+k6gStqj DfcTw3L8tTWeDcWrblvFaknFfBzkeXx9dTyrAYw3ii3LZQ6hA5UK8tUgwUVThTh+ QBhzhmn14Ljfp1fa3vgljY9yn5Zw+Drhbtn4TwIDAQABAoIBACaZstnEhJK/y/Q+ U8yheITSKv3SmMsnbHJYnuyiojvwFbolFKsIKdt7JnwB48Zql4bylevEpiKbTNWD nLmgYsYIIfK1SNseHQ81BPAJz4faVJpg0FszywvgZyzIRv40KbcG3xBgV/vBBCzI NiiiiqRtJ1MJaWDAglgvvyCS7W5GjHue74o/wPugyvIi7zmMqMmZLg3zz/YF3zxU h3JGFnw/TlN3yDJzdAOFiLGpSRD/bMcBJyWc9Oa2LQj+Emkc7dojmB/IJ7wJ9iuo Ni2/EJIQoFtVYxK6r+nRVuvT6/siMi7Ok4EL7PANVNVA7s7e+rmZX3NjDiGnvh5V DyxdT5ECgYEA/UFm3qxLiSQwNGuQLpCO9WR+4eeZPthNI1ySkcAnGyrmxX70ws7M 5j1jOIYUhf8i1ALUu++sBfvd+OhD0yeRgFvqrCdeHdFnAdgOtQDUGfU9bh1YpEX0 4A7zIxK7TBi26hXb/fTu7YwDtv58W1BtOOfo9ZKOWPu1ZQbEG5fp9pMCgYEA1BKF MRWshqBOSqRhyrl34wXTr6EI1XcwIDNyAGJnQFlbxWcCuPYCkF+ENZOe8D6soXiZ tTW6zSkM+UmsNgEYjD5b0eWgd0yUdrOi/Z4JZ16SlroeLYfbqVYY6NPE6XEuzZ9p XEB5WedUGIdU2JZQ422m7BxpJBIRtlbhG1Oi8NUCgYEA9Y3MaGs2ciqccrc4fW28 r0JZpEAi3kRrxrWjh56ATF80kpmeSKSrFzK+WbfnfmT7KAX2rqKccNDdUNIjsUDU W1jEGVeyccbv0WHkIKxE+0ZF4daic+VAoV7dcExhPk9YS3AWdg5e/ASeNXhaq084 F80Em9cWHkEwiFwfGYIaX/ECgYEAnUqfPyi0LaX4a6RAY/vrz5Yiy8DErI8aQsfl ZiOWMUQVrPQaMNVGUY6GoLY8zDOwFpM8bgrL4h7wYHUkJWnqqxoVQDjwK4vBEclq unDcyK58Sw8AEwURBye0kft/sSUhcaEqpCGt3+CTnx3A8GOM2yIZDEaGNRqxyGvn yjzePYECgYBqr0OQJUDUOGcL+V5z8wCaLN2bbWK/bukaGe3ZU7W2kWhNvVL0SBNC dUcxLp5ELOj8wFHmTvZv1q2aszIY2CQhnc1qGYCh34UAK14DkTvSYie9XcGPhkWK UC9VLorkPbPAZipbBs9Dt079ub3HRyeXX4b0VHHKzu2XVGXVEOXO6A== -----END RSA PRIVATE KEY----- ================================================ FILE: docker/sftp/id_rsa.pub ================================================ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRzHuNK74VlAkWbPT6+VujUj5rCVrG+plSx64+9f9gDfZzeESevQ6kQyF78++ziOT8GI+nkJ8hMV7rKTFS82W/GH/0Q6ydQVoWt/IuFifdOJxbqW/NImYnu3J1b8vTOkw+WIi1uO1ci3Ec7KhVyXbT4NP4tlaVFGee5INMLU5zq+8uizBviU5M1UHaRXtb26tTpb46KVwRHCONCmIgziaTCDZ1fRo9ZLmjQ321iLf6TqBK2qMN9xPDcvy1NZ4NxatuW8VqScV8HOR5fH11PKsBjDeKLctlDqEDlQry1SDBRVOFOH5AGHOGafXguN+nV9re+CWNj3KflnD4OuFu2fhP your_email@example.com ================================================ FILE: docker/sftp/ssh_host_ed25519_key ================================================ -----BEGIN OPENSSH PRIVATE KEY----- b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW QyNTUxOQAAACCPTP3g9O8X9Ir5KwrSVkYRqgCZFFt4i7oHBKyy31HiCQAAALBMCYEjTAmB IwAAAAtzc2gtZWQyNTUxOQAAACCPTP3g9O8X9Ir5KwrSVkYRqgCZFFt4i7oHBKyy31HiCQ AAAEAsRUm7efqtLmLIUwhaHzPnKf+hUubVc+xr49XcILRW5Y9M/eD07xf0ivkrCtJWRhGq AJkUW3iLugcErLLfUeIJAAAAKmZyYW5rZGVqb25nZUBGcmFuay1kZS1Kb25nZXMtTUJQMj AxNS5sb2NhbAECAw== -----END OPENSSH PRIVATE KEY----- ================================================ FILE: docker/sftp/ssh_host_ed25519_key.pub ================================================ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII9M/eD07xf0ivkrCtJWRhGqAJkUW3iLugcErLLfUeIJ frankdejonge@Frank-de-Jonges-MBP2015.local ================================================ FILE: docker/sftp/ssh_host_rsa_key ================================================ -----BEGIN OPENSSH PRIVATE KEY----- b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn NhAAAAAwEAAQAAAgEA48TBtykiBjI65ZBWCpOHk86EZrIxBFxb5WpL0ujBSrp05UJ8eKeX moEefhR4Vqll9Oi+ejmKXUvvkH1M638N35SFq+S3HR8TsMvgn5MaWO5HH3F5zzzLzyZjtZ H/K6mroMhG1b/DZoG3s4A5jN1i2ByRJS9d65q+Txd9ItlEifiCs7YYDS/5CkMWp3I6+i64 CmARpMUvfVwpclFFIaxnlevymRIwSxzUoZxGuLYR2+QUx/4sbx41CIOLwEWrxf0oq9+DS3 eyp+av4itLo47ndLfNEFexUHSPE8IofGuKAIUkHrPMVwLoFT+FbIDVH6vnU9XawIpbHFev ju2Mopb3DS9CKLLziQhBSxwCgsJmFyTqzkPZ3YeSsaz22o+vW98HRnp6TTtv+iefArNRPr gikBdFEjK8jWwZWV6MyeU5u4fZngMghV3DOfBjTlhrONI50iicM1sNKEvUqzEqCL/fs+zX V04aeDyNw+O6Xy1uIT+2Ozo5bHMoeyqqM33Zb7CB+polZQckFrXZAS59KLRYy2EpNQVzNY UThywQDPk5fkJ1CczHwlAzLlNZdxnsG69+JYGWlqKu//dT4AJPuYNDVW7SxPec8+yYq7uu DZnsnp8in/0tiyr4AS4fnAOszIndqRwwzqm13BVIDNJbdwgJTRDC+3wBT6hjnPC0ktACtN sAAAdo+KBGYPigRmAAAAAHc3NoLXJzYQAAAgEA48TBtykiBjI65ZBWCpOHk86EZrIxBFxb 5WpL0ujBSrp05UJ8eKeXmoEefhR4Vqll9Oi+ejmKXUvvkH1M638N35SFq+S3HR8TsMvgn5 MaWO5HH3F5zzzLzyZjtZH/K6mroMhG1b/DZoG3s4A5jN1i2ByRJS9d65q+Txd9ItlEifiC s7YYDS/5CkMWp3I6+i64CmARpMUvfVwpclFFIaxnlevymRIwSxzUoZxGuLYR2+QUx/4sbx 41CIOLwEWrxf0oq9+DS3eyp+av4itLo47ndLfNEFexUHSPE8IofGuKAIUkHrPMVwLoFT+F bIDVH6vnU9XawIpbHFevju2Mopb3DS9CKLLziQhBSxwCgsJmFyTqzkPZ3YeSsaz22o+vW9 8HRnp6TTtv+iefArNRPrgikBdFEjK8jWwZWV6MyeU5u4fZngMghV3DOfBjTlhrONI50iic M1sNKEvUqzEqCL/fs+zXV04aeDyNw+O6Xy1uIT+2Ozo5bHMoeyqqM33Zb7CB+polZQckFr XZAS59KLRYy2EpNQVzNYUThywQDPk5fkJ1CczHwlAzLlNZdxnsG69+JYGWlqKu//dT4AJP uYNDVW7SxPec8+yYq7uuDZnsnp8in/0tiyr4AS4fnAOszIndqRwwzqm13BVIDNJbdwgJTR DC+3wBT6hjnPC0ktACtNsAAAADAQABAAACAQCESra9HK3/bVNaHNBsyi2YAv5R67OetcpG YMvzj28daVkWA9zp82WRvucoEdmndDKc4kYoFZ2w/LcDdFOmAKDdOJW/NlPJHVDBgllQNg +6kYNL1wwJ+2ThR4noXwkXoi/mbgz+f6gNtNAu+Q30LG4J2eXP9EgX3UQmCh2LjShK/sVj fiNQHYoHlNnmnel1gIcyt4Pn8QPZSxtjo6KEoW9025uHntHf/rnduDg3dsC+uCX91zqVu7 TP4h/cqFrR322tDmBjB/4DmXCU69K+B/WVjGAV2ulJMrobns0HHysDjFFjZ8kKzMxh8wga 8mVXRPBSeEbbSEENIDz+xijGEusga3EK3I4LMNaAN47oulrVM++oF1pHL74jiEAC1SGHTv 0ZOZdCDmdmPyxCC9ruvJmQb3IbvoVyROcrHd4AvYSKnRqE2ljzLnFHG+Qp13meoXNz8Bup NO9ra+HJYWu6QgqHnPvwrIP5NdmwwSRWunglOvVSO2c11X7Li/NBkLLLk9wX1w9awAPfTf nUGCFnKi+1ianXD94zEG0FyXImN1eRK2lgs8ul6wGr3qV5B3sv7llgyWF5ZKLLj9iQ79M5 OKret5evybFbEbIOWbRTTcREfmSLqcKv2g6ZJpBNI8z4bz0jgc4hEVcMJlyiH52v3yMePh f7tXvv62Ws9IHLNhnSkQAAAQEApXAZ5Ns3aUOuCVvK3mEUWuQEsh/aM+rdrthX+zrw7Ma7 kw2O6zh6VviP5BV+8qG7sx1KyK9b0u+/qGDy4pMHN8SjNq1W+pTZ4FHkA/1UKI0kTvTwq/ WQvO1pBxLbx77gl5FuBjysPljnvMh3+l9QOFX4iwjFoyep1FgammuD4hnSR1yWuF3H/2xc sb75LnXdaEDm5ALyauTwmzdufTCaQmq19y8hZ5Kbhxd/Pao160tm38n2gektN458AbGxri 8tHSwpj4+3PSOwFeFalxk/jO8daye4PluoqK+O60Z5NWqdDDaBdvxEMcj9sddUQaQ0BzmK Fci3Hl6vS9TzBUWNOQAAAQEA/hDQkIH3ZIdZYBWIsinTKoztovUZHfHZihkBxcR0IbKZkn 7HGgtNtv1QshSvpzMp7HsAT45SqzIbN4ldNh0aSRmoOb2/pVNVNeJrWR0B+aS9hya6bAOY AiZlfMMo+2cU+gJzGD50tlu7ge4hT3YkN3vmoDyJZ2p7DxOioQQ10D5Pno8ZGh8kZX3t5d F+5GwPnsYXsi/iMgw18ggPTBPwAP+TeUsdP8Ae93h33LgCOkUfXeq/HmQ0jTUT84lxd9tW 2N/w6P/2SYa85oFVa2Y+u0qRJOLf6zJKTOH/ERV9IJYGLRzyxybQuo4iqw9V+kP13KkC2w 6vh2RcmEwXmxXRyQAAAQEA5YCwEjBqYLlPzQ3lOf75SbaT5+CIiyNy0P0ZUvpBl+URJkGC m8/AA3naWylEP/oUJRUkv21qXB8EEL62404nhNVtfDGX1mpht8H4N9lcXAdNFg1SWmqnqb 58PTODzM4n9CVjqnqqlTGQi1ph77RLDVXc29UooopLtYFsttiFBvv8Cs/cKx3t1ECQRklU hjPSdijeRTtv5x604uXt6mObYIGMAC87OSP4iaY00Dz17lnJ9m9UK5BCX+44sIEf3IQ9Ad jfaeN5UoUTv+y/fB419baC+dlIR9+SaEFqNgMHQqpyZApWMhCf99oNQhJ9XDjVEtKtXnhc 5Lh4tYyNGMsDgwAAACpmcmFua2Rlam9uZ2VARnJhbmstZGUtSm9uZ2VzLU1CUDIwMTUubG 9jYWwBAgMEBQYH -----END OPENSSH PRIVATE KEY----- ================================================ FILE: docker/sftp/ssh_host_rsa_key.pub ================================================ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDjxMG3KSIGMjrlkFYKk4eTzoRmsjEEXFvlakvS6MFKunTlQnx4p5eagR5+FHhWqWX06L56OYpdS++QfUzrfw3flIWr5LcdHxOwy+CfkxpY7kcfcXnPPMvPJmO1kf8rqaugyEbVv8NmgbezgDmM3WLYHJElL13rmr5PF30i2USJ+IKzthgNL/kKQxancjr6LrgKYBGkxS99XClyUUUhrGeV6/KZEjBLHNShnEa4thHb5BTH/ixvHjUIg4vARavF/Sir34NLd7Kn5q/iK0ujjud0t80QV7FQdI8Twih8a4oAhSQes8xXAugVP4VsgNUfq+dT1drAilscV6+O7YyilvcNL0IosvOJCEFLHAKCwmYXJOrOQ9ndh5KxrPbaj69b3wdGenpNO2/6J58Cs1E+uCKQF0USMryNbBlZXozJ5Tm7h9meAyCFXcM58GNOWGs40jnSKJwzWw0oS9SrMSoIv9+z7NdXThp4PI3D47pfLW4hP7Y7Ojlscyh7KqozfdlvsIH6miVlByQWtdkBLn0otFjLYSk1BXM1hROHLBAM+Tl+QnUJzMfCUDMuU1l3Gewbr34lgZaWoq7/91PgAk+5g0NVbtLE95zz7Jiru64NmeyenyKf/S2LKvgBLh+cA6zMid2pHDDOqbXcFUgM0lt3CAlNEML7fAFPqGOc8LSS0AK02w== frankdejonge@Frank-de-Jonges-MBP2015.local ================================================ FILE: docker/sftp/sshd_custom_configs.sh ================================================ #!/bin/bash cat <<'EOF' >> /etc/ssh/sshd_config KexAlgorithms curve25519-sha256 Ciphers aes256-gcm@openssh.com MACs hmac-sha2-256-etm@openssh.com HostKeyAlgorithms ssh-ed25519 EOF ================================================ FILE: docker/sftp/unknown.key ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAxZQy1FnuqhoZsiqvFanN5tZRL6GVeCBeoLroGBEggJYVDLw2 nPaXCDOaDoVzs1PkLoKhttA6g4l4eHZgDL1b6QTxRnDbkdOxKJHGEapk+WZk23yq QmU4thDdmgj0clJfS3aTbF7hGWpxrv9zB5saWwgWB/fBeYXu+kXuuO1UJF+F0QGt 6TVVMc6o3NaI7Cq1qoMvt7XGN5LegwfNWRGxaKTL1yn7pkljhkPLOoGedBemieya aQhV+dwzOVeBAXGKSmtiUwto8OMYl/suptMdo+91cLMRJkgXNQ5gA+4niqja5HGW 2Dw8vXja2Spp4+gWWh5OpXwd2pNnJScNINL30wIDAQABAoIBAHzBD7s/sdAcPN9f zj+qgUVhS8/8gilgnv90JPqVTeWDXnU1HnLLzR+znXHP1/eCYBDyEPQi1N+bXMML U6iXpEIlCcfFmQ6iETmhmeQrqChF/CcOt17HFSD400PgpaDN3DgE/h8uZYmryW6L A3HpAKI8H9UWHkcCR5wlrg98Y2W3AacHMwRG8da7QxLcGd10+1lFLuPXTu7LwDf0 u/+Jq9IPwm8WemGiuuOEQ06r7y1s929pA7fWSsn8DlVBIFnPHCtGKpiDfAyEHmCh /LUU6Z5Cl9186sGohu5hWUA+qTMAHw9IR0TbWxF+tAEX8qL5z/qH/surRvzfeRCi BooQB4ECgYEA99xz7AWMUzu6J4FSm29M2AS989Yz0XAgGIYs69pxoaRR5azRffY+ F6wjBpeSI5BElu1DAublhv5hpCgdcw6X2EfMnKvP1jtSbv+DOdR0p80LVAnG7tb3 stSgNwMsGdBpTCghHKUvOUGDU3D3c9O8jCrrJd9Unu4gj2/xbVuC45MCgYEAzBER XeuKuwrN0HmLCZ7AJ1aq7A60UKJjMicdXD5n95ePwHQ32s+hmrcGfUOjcZoWc1Dp yZaZvqFDYu52ZJ1QbNZqjBQJ8sjERSxNlV8DZe1I3ohXvsRQ4I6g8/HRgSl0CVqg +LeJA7n18+SrhhCWhY+dggV1m4qCNcWI7FCQwsECgYEAw+V7xT35U0twbIq8lFba QB03WEGiwNRCub9KP7ptdtjdVY5KIKj/GEyXfj1LZko+u56YCPIe1Ju25jxCUk5l Wq4cnHL6mBJYq5vMxmcRMBJR8sCrdtd1++QrIG+kal6a6nMJAI/ZjAIoXkl5ehUN /yZopY0mX1pLZ7KM+OaLw3sCgYEAvaACllbA9Gvmspmu5IKbJjL34yEK137+VGVa eBQZgk5ZK0oTeQXFssHuisomf/Lid8exZzzFowmxV6YlZ/ty96ALJB2e3PdIwsqX UX0X6EgllXv2pXNBgFmpIOYNe0ts4yBPQq8x57+O2FMePBb/+B5rC55NGfsMYjEr ugRncEECgYB9Wrnp4/WpTwcSJTJXWB+FJaMmcXqbFLk/AI1HbRXJFmR/ssK1gugQ Y3vevvRMSJkYkd1Zmbp9x8ZAGKzR/yeT8lQ6gY/xYq1fYLKKKtzHdwTMCVIVDgSf /v9Y8mLylcgr7ZjEo8xMTnb98ozSSFPBn2jq8c31AsQxXaIMK0BE7g== -----END RSA PRIVATE KEY----- ================================================ FILE: docker/sftp/users.conf ================================================ foo:pass:1001:100:upload bar:pass:1001:100:upload ================================================ FILE: docker-compose.yml ================================================ --- version: "3" services: # sabredav: # image: php:8.1-alpine3.15 # restart: always # volumes: # - ./:/var/www/html/ # ports: # - "4040:4040" # command: php -S 0.0.0.0:4040 /var/www/html/src/WebDAV/resources/server.php # webdav: # image: bytemark/webdav # restart: always # ports: # - "4080:80" # environment: # AUTH_TYPE: Digest # USERNAME: alice # PASSWORD: secret1234 # ANONYMOUS_METHODS: 'GET,OPTIONS' sftp: container_name: sftp restart: always image: atmoz/sftp:alpine volumes: - ./docker/sftp/users.conf:/etc/sftp/users.conf - ./docker/sftp/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key - ./docker/sftp/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key - ./docker/sftp/id_rsa.pub:/home/bar/.ssh/keys/id_rsa.pub ports: - "2222:22" sftp_eddsa_only: container_name: sftp_eddsa_only restart: always image: atmoz/sftp:alpine volumes: - ./docker/sftp/users.conf:/etc/sftp/users.conf - ./docker/sftp/sshd_custom_configs.sh:/etc/sftp.d/sshd_custom_configs.sh - ./docker/sftp/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key ports: - "2223:22" # ftp: # container_name: ftp # restart: always # image: delfer/alpine-ftp-server # environment: # USERS: 'foo|pass|/home/foo/upload' # ADDRESS: 'localhost' # ports: # - "2121:21" # - "21000-21010:21000-21010" azurite: image: mcr.microsoft.com/azure-storage/azurite profiles: ["azure"] restart: unless-stopped healthcheck: test: wget --no-verbose --tries=1 --spider http://localhost:10000/ || exit 1 interval: 10s timeout: 10s retries: 3 start_period: 10s ports: - 10000:10000 - 10001:10001 - 10002:10002 ================================================ FILE: fixtures/adapter.template.ts ================================================ import {Readable} from "stream"; import { StorageAdapter, ChecksumOptions, CreateDirectoryOptions, FileContents, MimeTypeOptions, PublicUrlOptions, StatEntry, TemporaryUrlOptions, WriteOptions, PathPrefixer, CopyFileOptions, MoveFileOptions, } from "@flystorage/file-storage"; export type AdapterFileStorageOptions = { prefix?: string, } export class AdapterFileStorage implements StorageAdapter { private readonly prefixer: PathPrefixer; constructor( private readonly options: AdapterFileStorageOptions = {}, ) { this.prefixer = new PathPrefixer(options.prefix || ''); } async write(path: string, contents: Readable, options: WriteOptions): Promise { throw new Error('Not implemented'); } async read(path: string): Promise { throw new Error('Not implemented'); } async deleteFile(path: string): Promise { throw new Error('Not implemented'); } async createDirectory(path: string, options: CreateDirectoryOptions): Promise { throw new Error('Not implemented'); } async stat(path: string): Promise { throw new Error('Not implemented'); } list(path: string, options: {deep: boolean}): AsyncGenerator { throw new Error('Not implemented'); } async changeVisibility(path: string, visibility: string): Promise { throw new Error('Not implemented'); } async visibility(path: string): Promise { throw new Error('Not implemented'); } async deleteDirectory(path: string): Promise { throw new Error('Not implemented'); } async fileExists(path: string): Promise { throw new Error('Not implemented'); } async directoryExists(path: string): Promise { throw new Error('Not implemented'); } async publicUrl(path: string, options: PublicUrlOptions): Promise { throw new Error('Not implemented'); } async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { throw new Error('Not implemented'); } async checksum(path: string, options: ChecksumOptions): Promise { throw new Error('Not implemented'); } async mimeType(path: string, options: MimeTypeOptions): Promise { throw new Error('Not implemented'); } async lastModified(path: string): Promise { throw new Error('Not implemented'); } async fileSize(path: string): Promise { throw new Error('Not implemented'); } async copyFile(from: string, to: string, options: CopyFileOptions): Promise { throw new Error('Not implemented'); } async moveFile(from: string, to: string, options: MoveFileOptions): Promise { throw new Error('Not implemented'); } } ================================================ FILE: package.json ================================================ { "private": true, "type": "module", "scripts": { "build": "npm run compile -ws --if-present && npm run patch -ws --if-present", "build:only": "npm run compile -ws --if-present", "clean:build": "npm run clean && npm run build", "watch": "concurrently npm:watch:*", "watch:file-storage": "npm run watch -w ./packages/file-storage", "watch:stream-mime-type": "npm run watch -w ./packages/stream-mime-type", "watch:local": "npm run watch -w ./packages/local-fs", "watch:aws-s3": "npm run watch -w ./packages/aws-s3", "clean": "rm -rf ./packages/*/dist/", "ts": "node --import tsx/esm", "lint": "tsc --noEmit --incremental false", "test": "npm run lint && npm run vitest", "vitest": "vitest run" }, "workspaces": [ "./packages/stream-mime-type", "./packages/file-storage", "./packages/*" ], "devDependencies": { "@swc/core": "^1.15.18", "@types/express": "^5.0.6", "@types/mime-types": "^3.0.1", "@types/multer": "^2.1.0", "@types/node": "^25.3.5", "@vitest/ui": "^4.0.18", "concurrently": "^9.2.1", "cross-env": "^10.1.0", "express": "^5.2.1", "node-fetch": "^3.3.2", "rimraf": "^6.1.3", "tsx": "^4.21.0", "typescript": "^5.9.3", "vite": "^7.3.1", "vite-tsconfig-paths": "^6.1.1", "vitest": "^4.0.14" }, "dependencies": { "dotenv": "^17.3.1" } } ================================================ FILE: packages/aws-s3/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/aws-s3/CHANGELOG.md ================================================ # `@flystorage/aws-s3` ## 1.2.1 - Upgraded dependencies ## 1.2.0 ## Changes - Make tests run on Bun ## 1.1.1 ## Changes - Do not send a body when creating a directory to prevent hanging requests. ## 1.1.0 ## Added - AbortSignal Support ## 1.0.1 ### Fixes - Bug: fixed not being able to use write options ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/aws-s3/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/aws-s3/README.md ================================================ # Flystorage adapter for AWS S3 This package contains the Flystorage adapter for AWS S3 using the V3 SDK. ## Installation Install all the required packages ```bash npm install --save @flystorage/file-storage @flystorage/aws-s3 @aws-sdk/client-s3 ``` ## Usage ```typescript import {FileStorage} from '@flystorage/file-storage'; import {AwsS3StorageAdapter} from '@flystorage/aws-s3'; import {S3Client} from '@aws-sdk/client-s3'; const client = new S3Client(); const adapter = new AwsS3StorageAdapter(client, { bucket: '{your-bucket-name}', prefix: '{optional-path-prefix}', }); const storage = new FileStorage(adapter); ``` > ⚠️ Always use the FileStorage, it is essential for security and a good developer > experience. Do not use the adapter directly. ================================================ FILE: packages/aws-s3/package.json ================================================ { "name": "@flystorage/aws-s3", "type": "module", "version": "1.2.0", "dependencies": { "@aws-sdk/client-s3": "^3.1004.0", "@aws-sdk/lib-storage": "^3.1004.0", "@aws-sdk/s3-request-presigner": "^3.1004.0", "@flystorage/file-storage": "^1.1.0", "@flystorage/stream-mime-type": "^1.0.0", "file-type": "^21.3.1", "mime-types": "^3.0.2" }, "description": "", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "keywords": [ "s3", "file", "storage", "flystorage", "filesystem" ], "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/adapter/aws-s3/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/aws-s3" }, "license": "MIT" } ================================================ FILE: packages/aws-s3/src/aws-s3-file-storage.test.ts ================================================ import {S3Client} from '@aws-sdk/client-s3'; import { FileStorage, readableToString, Visibility, closeReadable, UploadRequestHeaders, UnableToWriteFile, UnableToReadFile, } from '@flystorage/file-storage'; import {BinaryToTextEncoding, createHash, randomBytes} from 'crypto'; import * as https from 'https'; import {AwsS3StorageAdapter} from './aws-s3-storage-adapter.js'; import {createReadStream} from "node:fs"; import * as path from "node:path"; import 'dotenv/config'; import {PassThrough} from 'node:stream'; let client: S3Client; let storage: FileStorage; let bucket = 'flysystem-check'; const testSegment = randomBytes(10).toString('hex'); describe('aws-s3 file storage', () => { beforeAll(() => { const [major] = process.versions.node.split('.').map(Number); const versionToBucketMapping = [[20, 'a'], [21, 'b'], [22, 'c'], [23, 'd']] as [number, string][]; const bucketSuffix = versionToBucketMapping.find(([version]) => version === major); if (bucketSuffix) { bucket = `flystorage-${bucketSuffix[1]}`; } client = new S3Client(); }); beforeEach(async () => { const secondSegment = randomBytes(10).toString('hex'); storage = new FileStorage(new AwsS3StorageAdapter(client, { bucket: bucket, prefix: `storage/${testSegment}/${secondSegment}`, })); }); afterAll(async () => { await new FileStorage(new AwsS3StorageAdapter(client, { bucket: bucket, prefix: 'storage', })).deleteDirectory(testSegment); client.destroy(); }); test('uploading using a prepared request', async () => { const request = await storage.prepareUpload('prepared/request-file.txt', { expiresAt: Date.now() + 60 * 1000, headers: { 'Content-Type': 'text/plain', } }); await naivelyMakeRequestFile( request.url, request.headers, request.method, 'this is the contents', ); const contents = await storage.readToString('prepared/request-file.txt'); expect(contents).toEqual('this is the contents'); }); test('writing and reading a file', async () => { await storage.write('path+name.txt', 'this is the contents'); expect(await storage.readToString('path+name.txt')).toEqual('this is the contents'); }); test('non deep and deep listing should have consistent and similar results', async () => { await storage.write('file_1.txt', 'contents'); await storage.write('file_2.txt', 'contents'); await storage.createDirectory('directory_1'); await storage.createDirectory('directory_2'); const non_deep_listing = await storage.list('/', {deep: false}).toArray(); const deep_listing = await storage.list('/', {deep: true}).toArray(); expect(non_deep_listing).toHaveLength(4); expect(deep_listing).toHaveLength(4); expect(non_deep_listing).toEqual(deep_listing); }); test('root listing should work without root-slash', async () => { const rootStorage = new FileStorage(new AwsS3StorageAdapter(client, { bucket: 'flystorage-root-check', })); await rootStorage.write('test-file.txt', 'contents'); await rootStorage.createDirectory('test-directory'); const non_deep_listing = await rootStorage.list('', {deep: false}).toArray(); const deep_listing = await rootStorage.list('', {deep: true}).toArray(); expect(non_deep_listing).toHaveLength(2); expect(deep_listing).toHaveLength(2); expect(non_deep_listing.map(item => item.path)).toEqual(deep_listing.map(item => item.path)); }); test('root listing should work with root-slash', async () => { const rootStorage = new FileStorage(new AwsS3StorageAdapter(client, { bucket: 'flystorage-root-check', })); await rootStorage.write('test-file.txt', 'contents'); await rootStorage.createDirectory('test-directory'); const non_deep_listing = await rootStorage.list('/', {deep: false}).toArray(); const deep_listing = await rootStorage.list('/', {deep: true}).toArray(); expect(non_deep_listing).toHaveLength(2); expect(deep_listing).toHaveLength(2); expect(non_deep_listing.map(item => item.path)).toEqual(deep_listing.map(item => item.path)); }); test('moving a file', async () => { await storage.write('from+here.txt', 'this'); await storage.moveFile('from+here.txt', 'to+there.txt'); expect(await storage.fileExists('from+here.txt')).toEqual(false); expect(await storage.readToString('to+there.txt')).toEqual('this'); }); test('copying a file', async () => { await storage.write('from+this.txt', 'this'); await storage.copyFile('from+this.txt', 'to+that.txt'); expect(await storage.fileExists('from+this.txt')).toEqual(true); expect(await storage.readToString('to+that.txt')).toEqual('this'); }); test('trying to copy a file that does not exist', async () => { await expect(storage.copyFile('404.txt', 'to.txt')).rejects.toThrow(); }); test('timing out when writing a file', async () => { const writeStream = new PassThrough(); writeStream.write('something'); setTimeout(() => { writeStream.end('this'); }, 150); await expect(storage.write('somewhere.txt', writeStream, { timeout: 10, })).rejects.toThrow(); }) test('trying to read a file that does not exist', async () => { let was404 = false; try { await storage.read('404.txt'); } catch (err) { if (err instanceof UnableToReadFile) { was404 = err.wasFileNotFound; } } expect(was404).toEqual(true); }); test('trying to move a file that does not exist', async () => { await expect(storage.moveFile('404.txt', 'to.txt')).rejects.toThrow(); }); test('you can download public files using a public URL', async () => { await storage.write('public+file.txt', 'contents of the public file', { visibility: Visibility.PUBLIC, }); const url = await storage.publicUrl('public+file.txt'); const contents = await naivelyDownloadFile(url); expect(contents).toEqual('contents of the public file'); }); test('private files can only be downloaded using a temporary URL', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(naivelyDownloadFile(await storage.publicUrl('private+file.txt'))).rejects.toThrow(); await expect(naivelyDownloadFile( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}) )).resolves.toEqual('contents of the private file'); }); test('getting a temporary url for a non-existing-file', async () => { await expect( storage.temporaryUrl('this-does-not-exist.txt', {expiresAt: Date.now() + 60 * 1000}) ).resolves.toContain('this-does-not-exist.txt'); }); describe('response headers', () => { test('fetches file with Content-Disposition header when specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Disposition': 'attachment; filename="private+file.txt"'} }), 'Content-Disposition' )).resolves.toEqual('attachment; filename="private+file.txt"'); }); test('fetches file without Content-Disposition header when not specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}), 'Content-Disposition' )).resolves.toBeUndefined(); }); test('fetches file with Cache-Control header when specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Cache-Control': 'none'} }), 'Cache-Control' )).resolves.toEqual('none'); }); test('fetches file without Cache-Control header when not specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}), 'Cache-Control' )).resolves.toBeUndefined(); }); test('fetches file with Content-Encoding header when specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Encoding': 'br'} }), 'Content-Encoding' )).resolves.toEqual('br'); }); test('fetches file without Content-Encoding header when not specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}), 'Content-Encoding' )).resolves.toBeUndefined(); }); test('fetches file with Content-Language header when specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Language': 'en-US'} }), 'Content-Language' )).resolves.toEqual('en-US'); }); test('fetches file without Content-Language header when not specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}), 'Content-Language' )).resolves.toBeUndefined(); }); test('fetches file with Content-Type header when specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Type': 'image/jpeg+special'} }), 'Content-Type' )).resolves.toEqual('image/jpeg+special'); }); test('fetches file with Expires header when specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Expires': 'Sat, 21 Oct 2023 07:28:00 GMT'} }), 'Expires' )).resolves.toEqual('Sat, 21 Oct 2023 07:28:00 GMT'); }); test('fetches file without Expires header when not specified in the options', async () => { await storage.write('private+file.txt', 'contents of the private file', { visibility: Visibility.PRIVATE, }); await expect(responseHeaderValue( await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}), 'Expires' )).resolves.toBeUndefined(); }); }); test('retrieving the size of a file', async () => { const contents = 'this is the contents of the file'; await storage.write('something+file.txt', contents); expect(await storage.fileSize('something+file.txt')).toEqual(contents.length); }); test('writing a png and fetching its mime-type', async () => { const handle = createReadStream(path.resolve(process.cwd(), 'fixtures/screenshot.png')); await storage.write('image.png', handle); closeReadable(handle); const mimeType = await storage.mimeType('image.png'); expect(mimeType).toEqual('image/png'); }); test('fetching the last modified', async () => { await storage.write('test.txt', 'contents'); const lastModified = await storage.lastModified('test.txt'); expect(lastModified).toBeGreaterThan(Date.now() - 5000); expect(lastModified).toBeLessThan(Date.now() + 5000); }); test('it can request checksums', async () => { function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string { return createHash(algo).update(input).digest(encoding); } const contents = 'this is for the checksum'; await storage.write('path.txt', contents); const expectedChecksum = hashString(contents, 'md5'); const checksum = await storage.checksum('path.txt', { algo: 'etag', }); expect(checksum).toEqual(expectedChecksum); }); test('it can request sha256 checksums', async () => { function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string { return createHash(algo).update(input).digest(encoding); } const contents = 'this is for the checksum'; await storage.write('path.txt', contents, { ChecksumAlgorithm: 'SHA256' }); const expectedChecksum = hashString(contents, 'sha256', 'base64'); const checksum = await storage.checksum('path.txt', { algo: 'sha256', }); expect(checksum).toEqual(expectedChecksum); }); test('uploads can be aborted', async () => { const reason = new Error('Because I say so'); const controller = new AbortController(); const options = { abortSignal: controller.signal, } as const; await expect((async () => { const promise = storage.write('cache.txt', 'some content', { abortSignal: controller.signal, }); controller.abort(reason); return promise; })()).rejects.toThrow(UnableToWriteFile.because( 'Because I say so', {cause: reason, context: {path: 'cache.txt', options}}, )); }) test('it handles custom Cache-Control header', async() => { await storage.write('cache.txt', 'some content', {cacheControl: "max-age=7200, public", visibility: Visibility.PUBLIC}); const url = await storage.publicUrl('cache.txt') const res = await fetch(url); expect(res.headers.get("Cache-Control")).toEqual("max-age=7200, public") }); }); function naivelyDownloadFile(url: string): Promise { return new Promise((resolve, reject) => { https.get(url, async res => { if (res.statusCode !== 200) { reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`)); } else { resolve(await readableToString(res)); } }); }); } function naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise { return new Promise((resolve, reject) => { const req = https.request(url, { method: method, headers: { ...headers, 'Content-Length': new Blob([data]).size, } }, async res => { let responseBody = ''; res.on('data', (chunk) => { responseBody += chunk; }); res.on('end', () => { const statusCode = res.statusCode ?? 500; if (statusCode <= 200 && statusCode >= 299) { reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`)); } else { resolve(); } }); }); req.on('error', (err) => { reject(err); }); req.write(data); req.end(); }); } function responseHeaderValue(url: string, header: string): Promise { return new Promise((resolve, reject) => { https.get(url, res => { if (res.statusCode !== 200) { reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`)); } else { resolve( res.headers[header]?.toString() ?? res.headers[header.toLowerCase()]?.toString() ?? undefined ); } }); }); } ================================================ FILE: packages/aws-s3/src/aws-s3-storage-adapter.ts ================================================ import { _Object, CommonPrefix, CopyObjectCommand, CopyObjectRequest, DeleteObjectCommand, DeleteObjectsCommand, GetObjectAclCommand, GetObjectAclOutput, GetObjectCommand, GetObjectCommandInput, GetObjectCommandOutput, HeadObjectCommand, ListObjectsV2Command, ListObjectsV2Output, ObjectCannedACL, PutObjectAclCommand, PutObjectCommand, PutObjectCommandInput, S3Client, S3ServiceException, } from '@aws-sdk/client-s3'; import {Configuration, Upload} from '@aws-sdk/lib-storage'; import {getSignedUrl} from '@aws-sdk/s3-request-presigner'; import {posix} from 'node:path'; import { AdapterListOptions, ChecksumIsNotAvailable, ChecksumOptions, closeReadable, CopyFileOptions, CreateDirectoryOptions, FileContents, FileWasNotFound, MimeTypeOptions, MiscellaneousOptions, MoveFileOptions, normalizeExpiryToMilliseconds, PathPrefixer, PublicUrlOptions, StatEntry, StorageAdapter, TemporaryUrlOptions, UploadRequest, UploadRequestHeaders, UploadRequestOptions, Visibility, WriteOptions, } from '@flystorage/file-storage'; import {resolveMimeType} from '@flystorage/stream-mime-type'; import {Readable} from 'stream'; import {lookup} from 'mime-types'; type PutObjectOptions = Omit; export type WriteOptionsForS3 = Omit; const possibleChecksumAlgos = ['SHA1', 'SHA256', 'CRC32', 'CRC32C', 'ETAG'] as const; type ChecksumAlgo = typeof possibleChecksumAlgos[number]; function isSupportedAlgo(algo: string): algo is ChecksumAlgo { return possibleChecksumAlgos.includes(algo as ChecksumAlgo); } export type AwsS3StorageAdapterOptions = Readonly<{ bucket: string, prefix?: string, region?: string, publicUrlOptions?: PublicUrlOptions, uploadRequestOptions?: UploadRequestOptions, putObjectOptions?: PutObjectOptions, uploadConfiguration?: Partial>, defaultChecksumAlgo?: ChecksumAlgo, }>; export type AwsPublicUrlOptions = PublicUrlOptions & { bucket: string, region?: string, forcePathStyle?: boolean, baseUrl?: string, } export type AwsPublicUrlGenerator = { publicUrl(path: string, options: AwsPublicUrlOptions): Promise; }; export class DefaultAwsPublicUrlGenerator implements AwsPublicUrlGenerator { async publicUrl(path: string, options: AwsPublicUrlOptions): Promise { const baseUrl = options.baseUrl ?? 'https://{subdomain}.amazonaws.com/{uri}'; const subdomain = options.forcePathStyle !== true ? `${options.bucket}.s3` : options.region === undefined ? 's3' : `s3-${options.region}`; const uri = options.forcePathStyle !== true ? encodePath(path) : `${options.bucket}/${encodePath(path)}`; return baseUrl.replace('{subdomain}', subdomain).replace('{uri}', uri); } } /** * BC extension */ export class HostStyleAwsPublicUrlGenerator extends DefaultAwsPublicUrlGenerator { } export type TimestampResolver = () => number; type AclOptions = Pick; /** * Some commands need URI encoded paths to work ¯\_(ツ)_/¯ */ function encodePath(path: string): string { return path.split('/').map(encodeURIComponent).join('/'); } function maybeAbort(signal?: AbortSignal) { if (signal?.aborted) { throw signal.reason; } } export class AwsS3StorageAdapter implements StorageAdapter { private readonly prefixer: PathPrefixer; constructor( private readonly client: S3Client, private readonly options: AwsS3StorageAdapterOptions, private readonly publicUrlGenerator: AwsPublicUrlGenerator = new DefaultAwsPublicUrlGenerator(), private readonly timestampResolver: TimestampResolver = () => Date.now(), ) { this.prefixer = new PathPrefixer(options.prefix ?? '', '/', (...paths) => { const path = posix.join(...paths); if (path === "." || path === "/") { // 1) https://nodejs.org/api/path.html#pathjoinpaths // Zero-length path segments are ignored. If the joined path string is a zero-length string then '.' will be // returned, representing the current working directory. // 2) In S3 we use delimiter:"/". In that case we need to remove the root-slash in order to list the // root-directory contents. return ""; } else { return path; } }); } async copyFile(from: string, to: string, options: CopyFileOptions): Promise { maybeAbort(options.abortSignal); let visibility: string | undefined = options.visibility; if (visibility === undefined && options.retainVisibility) { visibility = await this.visibility(from, options); maybeAbort(options.abortSignal); } let acl: AclOptions = (visibility !== undefined && options.useVisibility !== false) ? {ACL: this.visibilityToAcl(visibility)} : {}; await this.client.send(new CopyObjectCommand({ Bucket: this.options.bucket, CopySource: posix.join('/', this.options.bucket, encodePath(this.prefixer.prefixFilePath(from))), Key: this.prefixer.prefixFilePath(to), ...acl, }), {abortSignal: options.abortSignal}); } async moveFile(from: string, to: string, options: MoveFileOptions): Promise { await this.copyFile(from, to, options); await this.deleteFile(from, options); } async prepareUpload(path: string, options: UploadRequestOptions): Promise { maybeAbort(options.abortSignal); const expiry = normalizeExpiryToMilliseconds(options.expiresAt); const now = (this.timestampResolver)(); const putObjectParams: PutObjectCommandInput = { Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), }; const headers: UploadRequestHeaders = {}; const contentType = options['Content-Type'] ?? options.contentType; if (typeof contentType === 'string') { putObjectParams.ContentType = contentType; headers['Content-Type'] = contentType; } const url = await getSignedUrl(this.client, new PutObjectCommand(putObjectParams), { expiresIn: Math.floor((expiry - now) / 1000), }); return { url, method: 'PUT', provider: 'aws-s3', headers, }; } async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { maybeAbort(options.abortSignal); const expiry = normalizeExpiryToMilliseconds(options.expiresAt); const now = (this.timestampResolver)(); const getObjectParams: GetObjectCommandInput = { Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), }; if (options.responseHeaders) { if (options.responseHeaders['Cache-Control']) { getObjectParams.ResponseCacheControl = options.responseHeaders['Cache-Control']; } if (options.responseHeaders['Content-Disposition']) { getObjectParams.ResponseContentDisposition = options.responseHeaders['Content-Disposition']; } if (options.responseHeaders['Content-Encoding']) { getObjectParams.ResponseContentEncoding = options.responseHeaders['Content-Encoding']; } if (options.responseHeaders['Content-Language']) { getObjectParams.ResponseContentLanguage = options.responseHeaders['Content-Language']; } if (options.responseHeaders['Content-Type']) { getObjectParams.ResponseContentType = options.responseHeaders['Content-Type']; } if (options.responseHeaders['Expires']) { getObjectParams.ResponseExpires = new Date(options.responseHeaders['Expires']); } } return await getSignedUrl(this.client, new GetObjectCommand(getObjectParams), { expiresIn: Math.floor((expiry - now) / 1000), }); } async lastModified(path: string, options: MiscellaneousOptions): Promise { const stat = await this.stat(path, options); if (stat.lastModifiedMs === undefined) { throw new Error('Last modified is not available in stat'); } return stat.lastModifiedMs; } async fileSize(path: string, options: MiscellaneousOptions): Promise { const stat = await this.stat(path, options); if (stat.isFile === false) { throw new Error('Path is not a file'); } if (stat.size === undefined) { throw new Error('File size is not available in stat.') } return stat.size; } async mimeType(path: string, options: MimeTypeOptions): Promise { const response = await this.stat(path, options); if (!response.isFile) { throw new Error(`Path "${path} is not a file.`); } if (response.mimeType) { return response.mimeType; } if (options.disallowFallback) { throw new Error('Mime-type not available via HeadObject'); } maybeAbort(options.abortSignal); const method = options.fallbackMethod ?? 'path'; const mimeType = method === 'path' ? lookup(path) : await this.lookupMimeTypeFromStream(path, options); if (mimeType === undefined || mimeType === false) { throw new Error('Unable to resolve mime-type'); } return mimeType; } async visibility(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); const response: GetObjectAclOutput = await this.client.send(new GetObjectAclCommand({ Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), }), { abortSignal: options.abortSignal, }); const publicRead = response.Grants?.some(grant => grant.Grantee?.URI === 'http://acs.amazonaws.com/groups/global/AllUsers' && grant.Permission === 'READ' ) ?? false; return publicRead ? Visibility.PUBLIC : Visibility.PRIVATE; } async* list(path: string, options: AdapterListOptions): AsyncGenerator { const listing = this.listObjects(path, { deep: options.deep, includePrefixes: true, includeSelf: false, abortSignal: options.abortSignal, }); for await (const {type, item} of listing) { if (type === 'prefix') { yield { type: 'directory', isFile: false, isDirectory: true, path: this.prefixer.stripDirectoryPath(item.Prefix!), }; } else { const path = item.Key!; if (path.endsWith('/')) { yield { type: 'directory', isFile: false, isDirectory: true, path: this.prefixer.stripDirectoryPath(path), }; } else { yield { type: 'file', isFile: true, isDirectory: false, path: this.prefixer.stripFilePath(path), size: item.Size ?? 0, lastModifiedMs: item.LastModified?.getTime(), }; } } } } async * listObjects( path: string, options: { deep: boolean, includePrefixes: boolean, includeSelf: boolean, maxKeys?: number, abortSignal?: AbortSignal, }, ): AsyncGenerator<{ type: 'prefix', item: CommonPrefix } | { type: 'object', item: _Object }, any, unknown> { maybeAbort(options.abortSignal); const prefix = this.prefixer.prefixDirectoryPath(path); let collectedKeys = 0; let shouldContinue = true; let continuationToken: string | undefined = undefined; while (shouldContinue && (options.maxKeys === undefined || collectedKeys < options.maxKeys)) { maybeAbort(options.abortSignal); const response: ListObjectsV2Output = await this.client.send(new ListObjectsV2Command({ Bucket: this.options.bucket, Prefix: prefix, Delimiter: options.deep ? undefined : '/', ContinuationToken: continuationToken, MaxKeys: options.maxKeys, })); continuationToken = response.NextContinuationToken; shouldContinue = response.IsTruncated ?? false; const prefixes = options.includePrefixes ? response.CommonPrefixes ?? [] : []; for (const item of prefixes) { if ((!options.includeSelf && item.Prefix === prefix) || item.Prefix === undefined) { continue; } collectedKeys++; yield {type: 'prefix', item}; } for (const item of response.Contents ?? []) { if ((!options.includeSelf && item.Key === prefix) || item.Key === undefined) { // not interested in itself // not interested in empty prefixes continue; } collectedKeys++; yield {type: 'object', item}; } } } async read(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); let response: GetObjectCommandOutput; try { response = await this.client.send(new GetObjectCommand({ Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), }), { abortSignal: options.abortSignal, }); } catch (err) { if (err instanceof S3ServiceException && err.$metadata.httpStatusCode === 404) { throw FileWasNotFound.atLocation(path, { context: {path, options}, cause: err, }); } throw err; } if (response.Body instanceof Readable || response.Body instanceof ReadableStream) { return response.Body; } throw new Error('No response body was provided'); } async stat(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); const response = await this.client.send(new HeadObjectCommand({ Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), }), { abortSignal: options.abortSignal, }); return { path, type: 'file', isDirectory: false, isFile: true, size: response.ContentLength ?? 0, lastModifiedMs: response.LastModified?.getTime(), mimeType: response.ContentType, }; } async createDirectory(path: string, options: CreateDirectoryOptions): Promise { const key = this.prefixer.prefixDirectoryPath(path); const abortSignal = options.abortSignal; const abortController = new AbortController(); if (abortSignal) { if (abortSignal.aborted) { throw abortSignal.reason; } abortSignal.addEventListener('abort', () => { abortController.abort(abortSignal.reason); }); } const params = this.createPutObjectParams(key, '', { ContentLength: 0, ACL: options.directoryVisibility ? this.visibilityToAcl(options.directoryVisibility) : undefined, }); maybeAbort(abortSignal); await this.client.send(new PutObjectCommand(params), { abortSignal, }); } async deleteDirectory(path: string): Promise { // @ts-ignore because we know it will only be objects let itemsToDelete: AsyncGenerator<{ item: _Object }> = this.listObjects(path, { deep: true, includeSelf: true, includePrefixes: false, }); const flush = async (keys: { Key: string }[]) => this.client.send(new DeleteObjectsCommand({ Bucket: this.options.bucket, Delete: { Objects: keys, }, })); let bucket: { Key: string }[] = []; let promises: Promise[] = []; for await (const {item} of itemsToDelete) { bucket.push({Key: item.Key!}); if (bucket.length > 1000) { promises.push(flush(bucket)); bucket = []; } } if (bucket.length > 0) { promises.push(flush(bucket)); } await Promise.all(promises); } async write(path: string, contents: Readable, options: WriteOptions): Promise { let mimeType = options.mimeType; if (mimeType === undefined) { [mimeType, contents] = await resolveMimeType(path, contents); } const writeOptions: PutObjectOptions = { ACL: options.visibility ? this.visibilityToAcl(options.visibility) : undefined, ContentType: mimeType, ContentLength: options.size, CacheControl: options.cacheControl, } for (const option of Object.keys(options)) { if (isWriteOptionKey(option)) { const resolver = (writeOptionResolvers as any)[option]; const value = options[option]; if (resolver(value)) { (writeOptions as any)[option] = value; } } } const abortController = new AbortController(); if (options.abortSignal) { const abortSignal = options.abortSignal; if (abortSignal.aborted) { throw abortSignal.reason; } abortSignal.addEventListener('abort', () => { abortController.abort(abortSignal.reason); }); } const upload = new Upload({ client: this.client, params: this.createPutObjectParams( this.prefixer.prefixFilePath(path), contents, writeOptions, ), abortController, ...this.options.uploadConfiguration, }); await upload.done(); } private createPutObjectParams( key: string, contents: Readable | '', options: PutObjectOptions, ): PutObjectCommandInput { const params: PutObjectCommandInput = { Bucket: this.options.bucket, Key: key, ...Object.assign({}, this.options.putObjectOptions, options), }; if (contents !== '') { params.Body = contents; } return params; } async deleteFile(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); const key = this.prefixer.prefixFilePath(path); await this.client.send(new DeleteObjectCommand({ Bucket: this.options.bucket, Key: key, }), { abortSignal: options.abortSignal, }); } private visibilityToAcl(visibility: string): ObjectCannedACL { if (visibility === Visibility.PUBLIC) { return 'public-read'; } else if (visibility === Visibility.PRIVATE) { return 'private'; } throw new Error(`Unrecognized visibility provided; ${visibility}`); } async changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); await this.client.send(new PutObjectAclCommand({ Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), ACL: this.visibilityToAcl(visibility), }), { abortSignal: options.abortSignal, }); } async fileExists(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); try { await this.client.send(new HeadObjectCommand({ Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), }), { abortSignal: options.abortSignal, }); return true; } catch (e) { if (e instanceof S3ServiceException && e.$metadata.httpStatusCode === 404) { return false; } throw e; } } async directoryExists(path: string, options: MiscellaneousOptions): Promise { const listing = this.listObjects(path, { deep: true, includePrefixes: true, includeSelf: true, maxKeys: 1, abortSignal: options.abortSignal, }); for await (const _item of listing) { return true; } return false; } async publicUrl(path: string, options: PublicUrlOptions): Promise { maybeAbort(options.abortSignal); return this.publicUrlGenerator.publicUrl(this.prefixer.prefixFilePath(path), { bucket: this.options.bucket, ...options, ...this.options.publicUrlOptions, }); } async checksum(path: string, options: ChecksumOptions): Promise { maybeAbort(options.abortSignal); const algo = (options.algo || this.options.defaultChecksumAlgo || 'SHA256').toUpperCase(); if (!isSupportedAlgo(algo)) { throw ChecksumIsNotAvailable.checksumNotSupported(algo); } const responseKey = algo === 'ETAG' ? 'ETag' : `Checksum${algo}` as const; const response = await this.client.send(new HeadObjectCommand({ Bucket: this.options.bucket, Key: this.prefixer.prefixFilePath(path), ...algo === 'ETAG' ? {} : {ChecksumMode: 'ENABLED'}, }), { abortSignal: options.abortSignal, }); const checksum = response[responseKey]; if (checksum === undefined) { throw new Error(`Unable to retrieve checksum with algo ${algo}`); } return checksum.replace(/^"(.+)"$/, '$1'); } private async lookupMimeTypeFromStream(path: string, options: MimeTypeOptions) { const [mimetype, stream] = await resolveMimeType(path, Readable.from(await this.read(path, options))); await closeReadable(stream); return mimetype; } } /** * BC export * * @deprecated */ export class AwsS3FileStorage extends AwsS3StorageAdapter {} type ResolversForWriteOptions = { [K in keyof WriteOptionsForS3]-?: (value: any) => value is WriteOptionsForS3[K] } function isWriteOptionKey(key: string): key is string & keyof ResolversForWriteOptions { return Object.hasOwn(writeOptionResolvers, key); } export const writeOptionResolvers: ResolversForWriteOptions = { ChecksumCRC64NVME: function (value: any): value is PutObjectOptions['ChecksumCRC64NVME'] { return typeof value === 'string'; }, IfMatch: function (value: any): value is PutObjectOptions['IfMatch'] { return typeof value === 'string'; }, WriteOffsetBytes: function (value: any): value is PutObjectOptions['WriteOffsetBytes'] { return typeof value === 'string'; }, ChecksumSHA1: function (value: any): value is PutObjectOptions['ChecksumSHA1'] { return typeof value === 'string'; }, ChecksumSHA256: function (value: any): value is PutObjectOptions['ChecksumSHA256'] { return typeof value === 'string'; }, ChecksumCRC32: function (value: any): value is PutObjectOptions['ChecksumCRC32'] { return typeof value === 'string'; }, ChecksumCRC32C: function (value: any): value is PutObjectOptions['ChecksumCRC32C'] { return typeof value === 'string'; }, CacheControl: function (value: any): value is PutObjectOptions['CacheControl'] { return typeof value === 'string'; }, ContentDisposition: function (value: any): value is PutObjectOptions['ContentDisposition'] { return typeof value === 'string'; }, ContentEncoding: function (value: any): value is PutObjectOptions['ContentEncoding'] { return typeof value === 'string'; }, ContentLanguage: function (value: any): value is PutObjectOptions['ContentLanguage'] { return typeof value === 'string'; }, ContentMD5: function (value: any): value is PutObjectOptions['ContentMD5'] { return typeof value === 'string'; }, ContentType: function (value: any): value is PutObjectOptions['ContentType'] { return typeof value === 'string'; }, ChecksumAlgorithm: function (value: any): value is PutObjectOptions['ChecksumAlgorithm'] { return typeof value === 'string'; }, Expires: function (value: any): value is PutObjectOptions['Expires'] { return value instanceof Date; }, GrantFullControl: function (value: any): value is PutObjectOptions['GrantFullControl'] { return typeof value === 'string'; }, GrantRead: function (value: any): value is PutObjectOptions['GrantRead'] { return typeof value === 'string'; }, GrantReadACP: function (value: any): value is PutObjectOptions['GrantReadACP'] { return typeof value === 'string'; }, GrantWriteACP: function (value: any): value is PutObjectOptions['GrantWriteACP'] { return typeof value === 'string'; }, Metadata: function (value: any): value is PutObjectOptions['Metadata'] { return typeof value === 'object'; }, ServerSideEncryption: function (value: any): value is PutObjectOptions['ServerSideEncryption'] { return typeof value === 'string'; }, StorageClass: function (value: any): value is PutObjectOptions['StorageClass'] { return typeof value === 'string'; }, WebsiteRedirectLocation: function (value: any): value is PutObjectOptions['WebsiteRedirectLocation'] { return typeof value === 'string'; }, SSECustomerAlgorithm: function (value: any): value is PutObjectOptions['SSECustomerAlgorithm'] { return typeof value === 'string'; }, SSECustomerKey: function (value: any): value is PutObjectOptions['SSECustomerKey'] { return typeof value === 'string'; }, SSECustomerKeyMD5: function (value: any): value is PutObjectOptions['SSECustomerKeyMD5'] { return typeof value === 'string'; }, SSEKMSKeyId: function (value: any): value is PutObjectOptions['SSEKMSKeyId'] { return typeof value === 'string'; }, SSEKMSEncryptionContext: function (value: any): value is PutObjectOptions['SSEKMSEncryptionContext'] { return typeof value === 'string'; }, BucketKeyEnabled: function (value: any): value is PutObjectOptions['BucketKeyEnabled'] { return typeof value === 'string'; }, RequestPayer: function (value: any): value is PutObjectOptions['RequestPayer'] { return typeof value === 'string'; }, Tagging: function (value: any): value is PutObjectOptions['Tagging'] { return typeof value === 'string'; }, ObjectLockMode: function (value: any): value is PutObjectOptions['ObjectLockMode'] { return typeof value === 'string'; }, ObjectLockRetainUntilDate: function (value: any): value is PutObjectOptions['ObjectLockRetainUntilDate'] { return value instanceof Date; }, ObjectLockLegalHoldStatus: function (value: any): value is PutObjectOptions['ObjectLockLegalHoldStatus'] { return typeof value === 'string'; }, ExpectedBucketOwner: function (value: any): value is PutObjectOptions['ExpectedBucketOwner'] { return typeof value === 'string'; }, IfNoneMatch: function (value: any): value is WriteOptionsForS3['IfNoneMatch'] { return typeof value === 'string'; }, }; ================================================ FILE: packages/aws-s3/src/index.ts ================================================ export * from './aws-s3-storage-adapter.js'; ================================================ FILE: packages/aws-s3/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packages/azure-storage-blob/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/azure-storage-blob/CHANGELOG.md ================================================ # `@flystorage/azure-storage-blob` ## 1.2.0 ## Changes - Support Bun runtime - Updated dependencies - 404 detection for reads ## 1.1.0 ## Added - AbortSignal Support ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/azure-storage-blob/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/azure-storage-blob/README.md ================================================ # Flystorage adapter for Azure Storage Blob This package contains the Flystorage adapter for Azure Storage Blob ## Installation Install all the required packages ```bash npm install --save @flystorage/file-storage @flystorage/azure-storage-blob @azure/storage-blob ``` ## Usage ```typescript import {FileStorage} from '@flystorage/file-storage'; import {AzureStorageBlobStorageAdapter} from '@flystorage/azure-storage-blob'; const blobService = BlobServiceClient.fromConnectionString(process.env.AZURE_DSN!); const container = blobService.getContainerClient('flysystem'); const adapter = new AzureStorageBlobStorageAdapter(container); const storage = new FileStorage(adapter); ``` > ⚠️ Always use the FileStorage, it is essential for security and a good developer > experience. Do not use the adapter directly. ================================================ FILE: packages/azure-storage-blob/package.json ================================================ { "name": "@flystorage/azure-storage-blob", "type": "module", "version": "1.2.0", "dependencies": { "@azure/storage-blob": "^12.31.0", "@flystorage/file-storage": "^1.1.0", "@flystorage/stream-mime-type": "^1.0.0" }, "description": "", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "keywords": [ "s3", "file", "storage", "flystorage", "filesystem" ], "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/adapter/azure-storage-blob/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/aws-s3" }, "license": "MIT" } ================================================ FILE: packages/azure-storage-blob/src/azure-storage-blob.test.ts ================================================ import {BlobServiceClient} from "@azure/storage-blob"; import {AzureStorageBlobStorageAdapter} from "./azure-storage-blob.js"; import {randomBytes} from "crypto"; import { FileStorage, UploadRequestHeaders, Visibility, readableToString, UnableToReadFile, } from '@flystorage/file-storage'; import fetch from "node-fetch"; import { Readable } from "node:stream"; import https from 'https'; import 'dotenv/config'; const runSegment = process.env.AZURE_PREFIX ?? randomBytes(10).toString('hex'); describe('AzureStorageBlobStorageAdapter', () => { const blobService = BlobServiceClient.fromConnectionString( process.env.AZURE_DSN || "UseDevelopmentStorage=true;" ); const container = blobService.getContainerClient('flysystem'); let storage: FileStorage; beforeAll(async () => { await container.createIfNotExists({ access: "container" }); }); beforeEach(() => { const testSegment = randomBytes(10).toString('hex'); const adapter = new AzureStorageBlobStorageAdapter( container, { prefix: `flystorage/${runSegment}/${testSegment}`, } ); storage = new FileStorage(adapter); }); afterAll(async () => { const adapter = new AzureStorageBlobStorageAdapter(container); storage = new FileStorage(adapter); await storage.deleteDirectory(`flystorage/${runSegment}`); container }); test('non deep and deep listing should have consistent and similar results', async () => { await storage.write('file_1.txt', 'contents'); await storage.write('file_2.txt', 'contents'); await storage.write('directory_1/file.txt', 'contents'); await storage.write('directory_2/file.tx', 'contents'); const non_deep_listing = await storage.list('/', {deep: false}).toArray(); const deep_listing = await storage.list('/', {deep: true}).toArray(); expect(non_deep_listing).toHaveLength(4); expect(deep_listing).toHaveLength(6); }); test('it can request a checksum', async () => { const contents = 'this is for the checksum'; await storage.write('path.txt', contents); const checksum = await storage.checksum('path.txt', { algo: 'etag', }); expect(typeof checksum).toEqual('string'); expect(checksum.length).toBeGreaterThan(5); }); test('reading a file that was written', async () => { await storage.write('path.txt', 'content in azure'); const content = await storage.readToString('path.txt'); expect(content).toEqual('content in azure'); }); test('trying to read a file that does not exist', async () => { let was404 = false; try { await storage.readToString('404.txt'); } catch (err) { if (err instanceof UnableToReadFile) { was404 = err.wasFileNotFound; } } expect(was404).toEqual(true); }); test('trying to see if a non-existing file exists', async () => { expect(await storage.fileExists('404.txt')).toEqual(false); }); test('trying to see if an existing file exists', async () => { await storage.write('existing.txt', 'contents'); expect(await storage.fileExists('existing.txt')).toEqual(true); }); test('deleting an existing file', async () => { await storage.write('existing.txt', 'contents'); expect(await storage.fileExists('existing.txt')).toEqual(true); await storage.deleteFile('existing.txt'); expect(await storage.fileExists('existing.txt')).toEqual(false); }); test('deleting a non-existing file is OK', async () => { await storage.deleteFile('404.txt'); }); test('copying a file', async () => { await storage.write('file.txt', 'copied'); await storage.copyFile('file.txt', 'new-file.txt'); expect(await storage.fileExists('file.txt')).toEqual(true); expect(await storage.fileExists('new-file.txt')).toEqual(true); expect(await storage.readToString('new-file.txt')).toEqual('copied'); }); test('moving a file', async () => { await storage.write('file.txt', 'moved'); await storage.moveFile('file.txt', 'new-file.txt'); expect(await storage.fileExists('file.txt')).toEqual(false); expect(await storage.fileExists('new-file.txt')).toEqual(true); expect(await storage.readToString('new-file.txt')).toEqual('moved'); }); test('setting visibility always fails', async () => { await storage.write('existing.txt', 'yes'); await expect(storage.changeVisibility('existing.txt', Visibility.PRIVATE)).rejects.toThrow(); await expect(storage.changeVisibility('404.txt', Visibility.PUBLIC)).rejects.toThrow(); }); test('listing entries in a directory, shallow', async () => { await storage.write('outside/path.txt', 'test'); await storage.write('inside/a.txt', 'test'); await storage.write('inside/b.txt', 'test'); await storage.write('inside/c/a.txt', 'test'); const listing = await storage.list('inside').toArray(); expect(listing).toHaveLength(3); expect(listing[0].type).toEqual('file'); expect(listing[1].type).toEqual('file'); expect(listing[2].type).toEqual('directory'); expect(listing[0].path).toEqual('inside/a.txt'); expect(listing[1].path).toEqual('inside/b.txt'); expect(listing[2].path).toEqual('inside/c'); }); test('listing entries in a directory, deep', async () => { await storage.write('outside/path.txt', 'test'); await storage.write('inside/a.txt', 'test'); await storage.write('inside/b.txt', 'test'); await storage.write('inside/c/a.txt', 'test'); const listing = await storage.list('inside', {deep: true}).toArray(); expect(listing).toHaveLength(4); expect(listing[0].type).toEqual('file'); expect(listing[1].type).toEqual('file'); expect(listing[2].type).toEqual('directory'); expect(listing[3].type).toEqual('file'); expect(listing[0].path).toEqual('inside/a.txt'); expect(listing[1].path).toEqual('inside/b.txt'); expect(listing[2].path).toEqual('inside/c'); expect(listing[3].path).toEqual('inside/c/a.txt'); }); test('deleting a full directory', async () => { await storage.write('directory/a.txt', 'test'); await storage.write('directory/b.txt', 'test'); await storage.write('directory/c/a.txt', 'test'); await storage.deleteDirectory('directory'); const listing = await storage.list('directory', {deep: true}).toArray(); expect(listing).toEqual([]); }); test('checking if a directory exists', async () => { await storage.write('directory/a.txt', 'test'); await storage.write('directory/b.txt', 'test'); await storage.write('directory/c/a.txt', 'test'); expect(await storage.directoryExists('directory')).toEqual(true); expect(await storage.directoryExists('directory/c')).toEqual(true); expect(await storage.directoryExists('directory/a')).toEqual(false); }); test('accessing a file though public URL', async () => { await storage.write('something.txt', 'something'); const url = await storage.publicUrl('something.txt'); const contents = await naivelyDownloadFile(url); expect(contents).toEqual('something'); }); test('accessing a file though temporary URL', async () => { await storage.write('something.txt', 'something'); const url = await storage.temporaryUrl('something.txt', { expiresAt: Date.now() + 600000, }); const contents = await naivelyDownloadFile(url); expect(contents).toEqual('something'); }); test('uploading using a prepared request', async () => { const request = await storage.prepareUpload('prepared/request-file.txt', { expiresAt: Date.now() + 60 * 1000, headers: { 'Content-Type': 'text/plain', } }); await naivelyMakeRequestFile( request.url, request.headers, request.method, 'this is the contents', ); const contents = await storage.readToString('prepared/request-file.txt'); expect(contents).toEqual('this is the contents'); }); test('setting cache-control headers', async () => { await storage.write('cachecontrol.txt', 'something', { cacheControl: 'max-age=3200, public' }); const url = await storage.publicUrl('cachecontrol.txt'); const res = await fetch(url); expect(res.headers.get("Cache-Control")).toEqual('max-age=3200, public'); }); }); async function naivelyDownloadFile(url: string): Promise { const res = await fetch(url); if (res.status !== 200 || !res.body) { throw new Error(`Not able to download the file from ${url}, response status [${res.status}]`); } else { return await readableToString(Readable.from(res.body)); } } function naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise { return new Promise((resolve, reject) => { const req = https.request(url, { method: method, headers: { ...headers, 'Content-Length': new Blob([data]).size, } }, async res => { let responseBody = ''; res.on('data', (chunk) => { responseBody += chunk; }); res.on('end', () => { const statusCode = res.statusCode ?? 500; if (statusCode <= 200 && statusCode >= 299) { reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`)); } else { resolve(); } }); }); req.on('error', (err) => { reject(err); }); req.write(data); req.end(); }); } ================================================ FILE: packages/azure-storage-blob/src/azure-storage-blob.ts ================================================ import {Readable} from 'stream'; import { ChecksumIsNotAvailable, ChecksumOptions, CopyFileOptions, FileContents, FileWasNotFound, ListOptions, MimeTypeOptions, MiscellaneousOptions, MoveFileOptions, normalizeExpiryToDate, PathPrefixer, PublicUrlOptions, StatEntry, StorageAdapter, TemporaryUrlOptions, UploadRequest, UploadRequestHeaders, UploadRequestOptions, WriteOptions, } from '@flystorage/file-storage'; import { BlobDownloadResponseParsed, BlobGenerateSasUrlOptions, BlobGetPropertiesResponse, BlobProperties, BlobSASPermissions, ContainerClient, } from '@azure/storage-blob'; import {resolveMimeType} from '@flystorage/stream-mime-type'; import {dirname} from 'node:path'; export type AzureStorageBlobStorageAdapterOptions = { prefix?: string, uploadMaxConcurrency?: number, ignoreVisibility?: boolean, ignoredVisibilityResponse?: string, deleteDirBatchSize?: number, temporaryUrlOptions?: TemporaryUrlOptions, } function maybeAbort(signal?: AbortSignal) { if (signal?.aborted) { throw signal.reason; } } export class AzureStorageBlobStorageAdapter implements StorageAdapter { private readonly prefixer: PathPrefixer; constructor( private readonly container: ContainerClient, private readonly options: AzureStorageBlobStorageAdapterOptions = {}, ) { this.prefixer = new PathPrefixer(options.prefix || ''); } async copyFile(from: string, to: string, options: CopyFileOptions): Promise { const fromUrl = this.blockClient(from).url; maybeAbort(options.abortSignal); await this.blockClient(to).syncCopyFromURL(fromUrl, {abortSignal: options.abortSignal}); } async moveFile(from: string, to: string, options: MoveFileOptions): Promise { await this.copyFile(from, to, options); await this.deleteFile(from, options); } async write(path: string, contents: Readable, options: WriteOptions): Promise { let mimeType = options.mimeType; let stream = contents; maybeAbort(options.abortSignal); if (mimeType === undefined) { [mimeType, stream] = await this.resolveMimetype(path, contents, options); } maybeAbort(options.abortSignal); const blob = this.blockClient(path); await blob.uploadStream( stream, options.size, this.options.uploadMaxConcurrency, { abortSignal: options.abortSignal, blobHTTPHeaders: { blobContentType: mimeType, blobCacheControl: options.cacheControl }, }, ); } private blockClient(path: string) { return this.container.getBlockBlobClient(this.prefixer.prefixFilePath(path)); } async read(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); const blob = this.blockClient(path); let response: BlobDownloadResponseParsed; try { response = await blob.download(undefined, undefined, { abortSignal: options.abortSignal, }); } catch (err) { if ((err as any).statusCode === 404) { throw FileWasNotFound.atLocation(path, { context: {path, options}, cause: err, }) } throw err; } if (!response.readableStreamBody) { throw new Error('No readable stream body in response.'); } return response.readableStreamBody; } async deleteFile(path: string, options: MiscellaneousOptions): Promise { const blob = this.blockClient(path); maybeAbort(options.abortSignal); await blob.deleteIfExists({ abortSignal: options.abortSignal, }); } async createDirectory(): Promise { // no-op, directories do not exist. } async stat(path: string, options: {abortSignal?: AbortSignal} = {}): Promise { maybeAbort(options.abortSignal); const blob = this.blockClient(path); const properties = await blob.getProperties({ abortSignal: options.abortSignal, }); return this.mapToStatEntry(path, properties); } private mapToStatEntry(path: string, properties: BlobGetPropertiesResponse | BlobProperties): StatEntry { return { type: 'file', isFile: true, isDirectory: false, path, mimeType: properties.contentType, size: properties.contentLength, lastModifiedMs: properties.lastModified?.getTime(), }; } list(path: string, options: ListOptions): AsyncGenerator { return options.deep ? this.listDeep(path, options) : this.listShallow(path, options); } async *listDeep(path: string, options: ListOptions): AsyncGenerator { maybeAbort(options?.abortSignal); const directories = new Set(); const listing = this.container.listBlobsFlat({ prefix: this.prefixer.prefixDirectoryPath(path), abortSignal: options.abortSignal, }); const listedPath = path; for await (const item of listing) { maybeAbort(options?.abortSignal); const path = this.prefixer.stripFilePath(item.name); let parentDir = dirname(path); while(!['.', '', listedPath].includes(parentDir)) { if (directories.has(parentDir)) { break; } yield { type: 'directory', isFile: false, isDirectory: true, path: parentDir, }; directories.add(parentDir); parentDir = dirname(parentDir); } yield this.mapToStatEntry(path, item.properties); } } async *listShallow(path: string, options: ListOptions): AsyncGenerator { maybeAbort(options?.abortSignal); const listing = this.container.listBlobsByHierarchy('/', { prefix: this.prefixer.prefixDirectoryPath(path), abortSignal: options.abortSignal, }); for await (const item of listing) { maybeAbort(options?.abortSignal); if (item.kind === 'blob') { yield this.mapToStatEntry( this.prefixer.stripFilePath(item.name), item.properties ) } else { yield { path: this.prefixer.stripDirectoryPath(item.name), type: 'directory', isFile: false, isDirectory: true, } } } } async changeVisibility(path: string, visibility: string): Promise { if (this.options.ignoreVisibility !== true) { throw new Error('Not supported by this adapter'); } } async visibility(path: string): Promise { if (this.options.ignoreVisibility !== true) { throw new Error('Not implemented'); } // default to indicating it ss public because we cannot know if the default is private return this.options.ignoredVisibilityResponse ?? 'public'; } async deleteDirectory(path: string, options: MiscellaneousOptions): Promise { let deletes: Promise[] = []; const batchSize = this.options.deleteDirBatchSize ?? 10; for await (const item of this.list(path, {deep: true})) { if (item.isFile) { deletes.push(this.deleteFile(item.path, options)); } if (deletes.length >= batchSize) { await Promise.all(deletes); deletes = []; } } await Promise.all(deletes); } async fileExists(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); return await this.blockClient(path).exists({ abortSignal: options.abortSignal, }) } async directoryExists(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); const listing = this.container.listBlobsFlat({ prefix: this.prefixer.prefixDirectoryPath(path), abortSignal: options.abortSignal, }).byPage({ maxPageSize: 1, }); return (await listing.next()).value.segment.blobItems.length > 0; } async publicUrl(path: string, options?: PublicUrlOptions): Promise { return this.blockClient(path).url; } async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { return await this.blockClient(path).generateSasUrl({ expiresOn: normalizeExpiryToDate(options.expiresAt), permissions: BlobSASPermissions.parse('r'), ...(this.options.temporaryUrlOptions ?? {}), }); } async prepareUpload(path: string, options: UploadRequestOptions): Promise { const headers: UploadRequestHeaders = {}; headers['x-ms-blob-type'] = options['x-ms-blob-type'] ?? 'BlockBlob'; const config: BlobGenerateSasUrlOptions = { expiresOn: normalizeExpiryToDate(options.expiresAt), permissions: BlobSASPermissions.parse('w'), ...(this.options.temporaryUrlOptions ?? {}), }; const contentType = options['Content-Type'] ?? options.contentType; if (typeof contentType === 'string') { config.contentType = contentType; headers['Content-Type'] = contentType; } const url = await this.blockClient(path).generateSasUrl(config); return {method: 'PUT', provider: 'azure-storage-blob', url, headers}; } async checksum(path: string, options: ChecksumOptions): Promise { maybeAbort(options?.abortSignal); const algo = options.algo ?? 'etag'; if (algo !== 'etag') { throw ChecksumIsNotAvailable.checksumNotSupported(algo); } const blob = this.blockClient(path); const properties = await blob.getProperties({abortSignal: options.abortSignal}); const etag = properties.etag; if (etag === undefined) { throw new Error('Etag is not defined on blob properties.'); } return etag; } async mimeType(path: string, options: MimeTypeOptions): Promise { const stat = await this.stat(path, options); if (stat.isDirectory) { throw new Error('Path is not a file. No mimetype available.'); } if (stat.mimeType === undefined) { throw new Error('Mime-type not found for file.'); } return stat.mimeType; } async lastModified(path: string): Promise { const stat = await this.stat(path); if (stat.isDirectory) { throw new Error('Path is not a file. No last modified available.'); } if (stat.lastModifiedMs === undefined) { throw new Error('Last modified not found for file.'); } return stat.lastModifiedMs; } async fileSize(path: string): Promise { const stat = await this.stat(path); if (stat.isDirectory) { throw new Error('Path is not a file. No file size available.'); } if (stat.size === undefined) { throw new Error('File size not found for file.'); } return stat.size; } private async resolveMimetype(path: string, contents: Readable, options: WriteOptions): Promise<[string, Readable]> { if (options.mimeType) { return [options.mimeType, contents]; } const [mimeType, stream] = await resolveMimeType(path, contents); return [mimeType ?? 'application/octet-stream', stream]; } } /** * BC export * * @deprecated */ export class AzureStorageBlobFileStorage extends AzureStorageBlobStorageAdapter {} ================================================ FILE: packages/azure-storage-blob/src/index.ts ================================================ export * from './azure-storage-blob.js'; ================================================ FILE: packages/azure-storage-blob/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packages/chaos/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/chaos/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/chaos/README.md ================================================ # Flystorage adapter for chaos engineering This package contains an adapter decorator that causes instrumented failures. ## Installation Install all the required packages ```bash npm install --save @flystorage/file-storage @flystorage/chaos ``` ## Setup ```typescript import {FileStorage} from '@flystorage/file-storage'; import {ChaosStorageAdapterDecorator, TriggeredErrors} from '@flystorage/chaos'; const strategy = new TriggeredErrors(); const adapter = new ChaosStorageAdapterDecorator( createActualAdapter(), strategy, ); const storage = new FileStorage(adapter); ``` > ⚠️ Always use the FileStorage, it is essential for security and a good developer > experience. Do not use the adapter directly. ## Usage ```typescript import {TriggeredErrors} from '@flystorage/chaos'; const strategy = new TriggeredErrors(); // error on all write calls strategy.on('write', () => new Error()); // error on first 2 stat calls strategy.on('stat', () => new Error(), {times: 2}); // error after first 2 deleteFile calls strategy.on('deleteFile', () => new Error(), {after: 2}); // error on 2nd and 3rd call to any method strategy.on('*', () => new Error(), {after: 1, times: 2}); ``` ================================================ FILE: packages/chaos/changelog.md ================================================ # `@flystorage/chaos` ## 1.1.0 ### Changes - Added AbortSignal support ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/chaos/package.json ================================================ { "name": "@flystorage/chaos", "type": "module", "version": "1.1.0", "dependencies": { "@flystorage/file-storage": "^1.1.0" }, "description": "A storage adapter decorator with the ability to stage errors.", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "keywords": [ "testing", "chaos", "resillience", "flystorage", "filesystem" ], "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/tools/chaos/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/chaos" }, "license": "MIT" } ================================================ FILE: packages/chaos/src/index.test.ts ================================================ import { FileStorage } from "@flystorage/file-storage"; import {InMemoryStorageAdapter} from '@flystorage/in-memory'; import {AlwaysThrowError, ChaosStorageAdapterDecorator, NeverThrowError, TriggeredErrors} from './index.js'; describe('chaos adapter decorator', () => { describe('triggered strategy', () => { const strategy = new TriggeredErrors(); const adapter = new InMemoryStorageAdapter(); afterEach(() => { strategy.clearTriggers(); adapter.deleteEverything(); }); const storage = new FileStorage( new ChaosStorageAdapterDecorator( adapter, strategy, ), ); test('trigger immediately on write', async () => { strategy.on('write', () => new Error('Oh no...')); await expect(storage.write('path.txt', 'context')).rejects.toThrow(); }); test('trigger immediately after 3 writes', async () => { strategy.on('write', () => new Error('Oh no...'), {after: 3}); await storage.write('path.txt', 'context'); await storage.write('path.txt', 'context'); await storage.write('path.txt', 'context'); await expect(storage.write('path.txt', 'context')).rejects.toThrow(); }); test('trigger for 2 stat calls after 1 successful call', async () => { await storage.write('path.txt', 'contents'); strategy.on('stat', () => new Error('Oh no...'), {times: 2, after: 1}); await storage.stat('path.txt'); await expect(storage.stat('path.txt')).rejects.toThrow(); await expect(storage.stat('path.txt')).rejects.toThrow(); await storage.stat('path.txt'); }); test('triggering on any method call with: *', async () => { await storage.write('path.txt', 'contents'); strategy.on('*', () => new Error('Oh no...'), {times: 2, after: 1}); await storage.stat('path.txt'); await expect(storage.fileSize('path.txt')).rejects.toThrow(); await expect(storage.mimeType('path.txt')).rejects.toThrow(); await storage.deleteFile('path.txt'); }); }); describe('always throwing an error', () => { const strategy = new AlwaysThrowError(() => new Error('Oh no...')); const adapter = new InMemoryStorageAdapter(); afterEach(() => adapter.deleteEverything()); const storage = new FileStorage( new ChaosStorageAdapterDecorator( adapter, strategy, ), ); test('on any call', async () => { await expect(storage.write('path.txt', 'contents')).rejects.toThrow(); }); }); describe('never throwing an error', () => { const strategy = new NeverThrowError(); const adapter = new InMemoryStorageAdapter(); afterEach(() => adapter.deleteEverything()); const storage = new FileStorage( new ChaosStorageAdapterDecorator( adapter, strategy, ), ); test('on any call', async () => { await storage.write('path.txt', 'contents'); expect(await storage.readToString('path.txt')).toEqual('contents'); }); }); }); ================================================ FILE: packages/chaos/src/index.ts ================================================ import { ChecksumOptions, CopyFileOptions, CreateDirectoryOptions, FileContents, MimeTypeOptions, MiscellaneousOptions, MoveFileOptions, StatEntry, StorageAdapter, TemporaryUrlOptions, WriteOptions, } from '@flystorage/file-storage'; import {Readable} from 'stream'; export type AnyAdapterMethodName = keyof StorageAdapter; export interface ChaosStrategy { maybeGoNuts(method: AnyAdapterMethodName): void; } export class AlwaysThrowError implements ChaosStrategy { constructor(private readonly newError: () => Error) { } maybeGoNuts(method: keyof StorageAdapter): void { throw (this.newError)(); } } export class NeverThrowError implements ChaosStrategy { maybeGoNuts(method: keyof StorageAdapter): void { // do not do anything } } export class TriggeredErrors implements ChaosStrategy { private triggers: { [I in AnyAdapterMethodName | '*']?: { after: number, times: number, newError: () => unknown, } } = {}; on(method: AnyAdapterMethodName | '*', newError: () => unknown, options: {after?: number, times?: number} = {}) { this.triggers[method] = { after: options.after ?? 0, times: options.times ?? Number.MAX_SAFE_INTEGER, newError, }; } clearTriggers(): void { this.triggers = {}; } maybeGoNuts(method: keyof StorageAdapter): void { const trigger = this.triggers[method] ?? this.triggers['*']; if (trigger === undefined) { return; } if (trigger.after > 0) { trigger.after--; } else if (trigger.times > 0) { trigger.times--; throw (trigger.newError)(); } } } export class ChaosStorageAdapterDecorator implements StorageAdapter { constructor( private readonly storage: StorageAdapter, private readonly chaos: ChaosStrategy, ) { } write(path: string, contents: Readable, options: WriteOptions): Promise { this.chaos.maybeGoNuts('write'); return this.storage.write(path, contents, options); } read(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('read'); return this.storage.read(path, options); } deleteFile(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('deleteFile'); return this.storage.deleteFile(path, options); } createDirectory(path: string, options: CreateDirectoryOptions): Promise { this.chaos.maybeGoNuts('createDirectory'); return this.storage.createDirectory(path, options); } copyFile(from: string, to: string, options: CopyFileOptions): Promise { this.chaos.maybeGoNuts('copyFile'); return this.storage.copyFile(from, to, options); } moveFile(from: string, to: string, options: MoveFileOptions): Promise { this.chaos.maybeGoNuts('moveFile'); return this.storage.moveFile(from, to, options); } stat(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('stat'); return this.storage.stat(path, options); } list(path: string, options: { deep: boolean; }): AsyncGenerator { this.chaos.maybeGoNuts('list'); return this.storage.list(path, options); } changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('changeVisibility'); return this.storage.changeVisibility(path, visibility, options); } visibility(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('visibility'); return this.storage.visibility(path, options); } deleteDirectory(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('deleteDirectory'); return this.storage.deleteDirectory(path, options); } fileExists(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('fileExists'); return this.storage.fileExists(path, options); } directoryExists(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('directoryExists'); return this.storage.directoryExists(path, options); } publicUrl(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('publicUrl'); return this.storage.publicUrl(path, options); } temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { this.chaos.maybeGoNuts('publicUrl'); return this.storage.publicUrl(path, options); } checksum(path: string, options: ChecksumOptions): Promise { this.chaos.maybeGoNuts('checksum'); return this.storage.checksum(path, options); } mimeType(path: string, options: MimeTypeOptions): Promise { this.chaos.maybeGoNuts('mimeType'); return this.storage.mimeType(path, options); } lastModified(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('lastModified'); return this.storage.lastModified(path, options); } fileSize(path: string, options: MiscellaneousOptions): Promise { this.chaos.maybeGoNuts('fileSize'); return this.storage.fileSize(path, options); } } ================================================ FILE: packages/chaos/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packages/dynamic-import/.npmignore ================================================ index.test.ts tsconfig.json src/ ================================================ FILE: packages/dynamic-import/CHANGELOG.md ================================================ # `@flystorage/dynamic-import` ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/dynamic-import/README.md ================================================ # Utility for dynamically importing ESM code The TypeScript compiler turns dynamic import statements into require calls. This breaks importing ESM packages in CommonJS code. This utility helps to work around this by provided a JavaScript function to do the actual importing. ## Installation Install all the required packages ```bash npm install --save @flystorage/dynamic-import ``` ## Usage ```typescript const {dynamicallyImport} = require('@flystorage/dynamic-import'); async function useFileType() { const {fileTypeFromBuffer} = await dynamicallyImport('file-type'); } useFileType(); ``` ================================================ FILE: packages/dynamic-import/index.d.ts ================================================ export declare function dynamicallyImport(path: string): Promise; ================================================ FILE: packages/dynamic-import/index.js ================================================ async function dynamicallyImport(path) { return await import(path); }; exports.dynamicallyImport = dynamicallyImport; ================================================ FILE: packages/dynamic-import/index.test.ts ================================================ import {dynamicallyImport} from './index.js'; // @ts-ignore type FileType = typeof import('file-type'); describe('dynamic-import', () => { test('dynamic-import', async () => { const fileType = await dynamicallyImport('file-type'); expect(fileType.supportedExtensions.has('jpg')).toEqual(true); }); }) ================================================ FILE: packages/dynamic-import/package.json ================================================ { "name": "@flystorage/dynamic-import", "type": "commonjs", "version": "1.0.0", "description": "Utility to dynamically import ESM packages from CommonJS without tsc messing things up.", "main": "./index.js", "types": "./index.d.ts", "exports": { ".": { "import": { "types": "./index.d.ts", "default": "./index.js" }, "require": { "types": "./index.d.ts", "default": "./index.js" } } }, "scripts": {}, "author": "Frank de Jonge (https://frankdejonge.nl)", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/dynamic-import" }, "keywords": [], "license": "MIT" } ================================================ FILE: packages/dynamic-import/src/index.ts ================================================ // export * from '../index.js'; ================================================ FILE: packages/file-storage/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/file-storage/CHANGELOG.md ================================================ # `@flystorage/file-storage` ## 1.2.2 ### Fixes - Fixed typo in timeout options name ## 1.2.1 ### Fixes - Prevent unprefixed root paths to become '/' for directory listings. ## 1.2.0 ### Added - Ability to ignore visibility using configuration - Support Bun runtime ## 1.1.0 ### Added - AbortSignal Support - Normalised over stream errors for reads - Introduce way to detect failed reads because of missing files. ## 1.0.1 ### Fixes - Corrected error message for `UnableToGetChecksum` error. ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/file-storage/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/file-storage/README.md ================================================ # Flystorage Flystorage is a file storage abstraction for NodeJS and TypeScript. It is an 80/20 solution that is built around a set of goals: - Provide a straight-forward API that is easy to use. - Allow application code to be unaware WHERE files are stored. - Pragmatically smooth over underlying storage differences. - Expose an async/await based API, promises all the way. - Abstract over file permissions using "visibility". - Actually tested using real integrations, _mocks are not welcome_. - Stand on the shoulders of giants, use official vendor packages when possible. ### What is Flystorage NOT: Flystorage is meant to be used in cases for generic file storage use-cases. It's not an API for any specific filesystem. It's a generalised solution and will not implement feature only specific to one particular storage implementation. There will be use-cases that are not catered to, simply because they cannot be abstracted over in a reasonable manner. ## Capabilities - Write files using string | buffer | readable/stream - Read files as stream, string, or Uint8Array - List the contents of a directory/prefix, (shallow and deep). - Metadata, such as; mime-type, last modified timestamp, and file size. - Expose or calculate checksums for files. - Delete files without failing when they don't exist. - Set permissions using abstracted visibility - Delete directories (and any files it contains) - Generate public URLs. - Generate temporary (signed) URLs. - Secure direct uploads from the browser. - Moving files - Copying files ## Adapters - [Local Filesystem](https://www.npmjs.com/package/@flystorage/local-fs) - [AWS S3 (using the V3 SDK)](https://www.npmjs.com/package/@flystorage/aws-s3) - [Azure Blob Storage](https://www.npmjs.com/package/@flystorage/azure-storage-blob) - [Test implementation (in-memory)](https://www.npmjs.com/package/@flystorage/in-memory) - [Google Cloud Storage](https://www.npmjs.com/package/@flystorage/google-cloud-storage) - [Chaos adapter decorator](https://www.npmjs.com/package/@flystorage/chaos) ## Usage Install the main package and any adapters you might need: ```bash npm i -S @flystorage/file-storage # for using AWS S3 npm i -S @flystorage/aws-s3 # for using the local filesystem npm i -S @flystorage/local-fs ``` ## Local Usage ```typescript import {resolve} from 'node:path'; import {createReadStream} from 'node:fs'; import {FileStorage, Visibility} from '@flystorage/file-storage'; import {LocalStorageAdapter} from '@flystorage/local-fs'; /** * SETUP **/ const rootDirectory = resolve(process.cwd(), 'my-files'); const storage = new FileStorage(new LocalStorageAdapter(rootDirectory)); /** * USAGE **/ // Write using a string await storage.write('write-from-a-string.txt', 'file contents'); // Write using a stream const stream = createReadStream(resolve(process.cwd(), 'test-files/picture.png')); await storage.write('picture.png', stream); // Write with visibility (permissions). await storage.write('public.txt', 'debug', { visibility: Visibility.PUBLIC, // mode: 0o644 }); await storage.write('private.txt', 'debug', { visibility: Visibility.PRIVATE, // mode: 0o600 }); // List directory contents const contentsAsAsyncGenerator = storage.list('', {deep: true}); for await (const item of contentsAsAsyncGenerator) { console.log(item.path); if (item.isFile) { // do something with the file } else if (item.isDirectory) { // do something with the directory } } // Delete a file await storage.deleteFile('some-file.txt'); // Delete a directory (with all contents) await storage.deleteDirectory('some-directory'); ``` ## Author Flystorage is built by the maintainer of [Flysystem](https://flysystem.thephpleague.com), a filesystem abstraction for PHP. This brings along over a decade of filesystem abstraction experience. ================================================ FILE: packages/file-storage/package.json ================================================ { "name": "@flystorage/file-storage", "type": "module", "version": "1.2.2", "description": "File-storage abstraction: multiple filesystems, one API.", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "author": "Frank de Jonge (https://frankdejonge.nl)", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/file-storage" }, "homepage": "https://flystorage.dev/", "keywords": [ "fs", "file", "files", "filesystem", "filesystems", "storage" ], "license": "MIT" } ================================================ FILE: packages/file-storage/src/checksum-from-stream.ts ================================================ import {BinaryToTextEncoding, createHash} from 'crypto'; import {Readable} from 'node:stream'; import {TextEncoder} from 'util'; const encoder = new TextEncoder(); export async function checksumFromStream( stream: Readable, options: { algo?: string; encoding?: BinaryToTextEncoding } ): Promise { return new Promise((resolve, reject) => { const hash = createHash(options.algo ?? 'md5'); stream.on('error', err => reject(err)); stream.on('data', (chunk: Uint8Array | string | number) => { const type = typeof chunk; if (type === 'string') { chunk = encoder.encode(chunk as string); } else if (type === 'number') { chunk = new Uint8Array([chunk as number]); } hash.update(chunk as Uint8Array); }); stream.on('end', () => resolve(hash.digest(options.encoding ?? 'hex'))); }); } ================================================ FILE: packages/file-storage/src/errors.ts ================================================ export type ErrorContext = { [index: string]: any }; export type OperationError = Error & { readonly code: string, readonly context: ErrorContext, } export function errorToMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } export abstract class FlystorageError extends Error implements OperationError { readonly code: string = 'unknown_error'; constructor( message: string, public readonly context: ErrorContext = {}, cause: unknown = undefined, ) { const options = cause === undefined ? undefined : {cause}; // @ts-ignore TS2554 super(message, options); } } /** * Thrown when the checksum algo is not supported or not pre-computed. This error * is thrown with the intention of falling back to computing it based on a file read. */ export class ChecksumIsNotAvailable extends FlystorageError { public readonly code = 'flystorage.checksum_not_supported'; constructor( message: string, public readonly algo: string, context: ErrorContext = {}, cause: unknown = undefined, ) { super(message, context, cause); } static checksumNotSupported = (algo: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown } = {}) => new ChecksumIsNotAvailable( `Checksum algo "${algo}" is not supported`, algo, {...context, algo}, cause, ); static isErrorOfType(error: unknown): error is ChecksumIsNotAvailable { return (typeof error === 'object' && (error as any).code === 'flystorage.checksum_not_supported'); } } export class UnableToGetChecksum extends FlystorageError { public readonly code = 'flystorage.unable_to_get_checksum'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetChecksum( `Unable to get checksum for file. Reason: ${reason}`, context, cause, ); } export class UnableToGetMimeType extends FlystorageError { public readonly code = 'flystorage.unable_to_get_mimetype'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetMimeType( `Unable to get mime-type. Reason: ${reason}`, context, cause, ); } export class UnableToGetLastModified extends FlystorageError { public readonly code = 'flystorage.unable_to_get_last_modified'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetLastModified( `Unable to get last modified. Reason: ${reason}`, context, cause, ); } export class UnableToGetFileSize extends FlystorageError { public readonly code = 'flystorage.unable_to_get_file_size'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetFileSize( `Unable to get file size. Reason: ${reason}`, context, cause, ); } export class UnableToWriteFile extends FlystorageError { public readonly code = 'flystorage.unable_to_write_file'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToWriteFile( `Unable to write the file. Reason: ${reason}`, context, cause, ); } export class UnableToReadFile extends FlystorageError { public readonly code = 'flystorage.unable_to_read_file'; constructor( public readonly wasFileNotFound: boolean, message: string, public readonly context: ErrorContext = {}, cause: unknown = undefined, ) { super(message, context, cause); } static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToReadFile( false, `Unable to read the file. Reason: ${reason}`, context, cause, ); static becauseFileWasNotFound = (error: FileWasNotFound) => new UnableToReadFile( true, `Unable to read the file. Reason: ${error.message}`, error.context, error, ); } export class FileWasNotFound extends FlystorageError { public readonly code = 'flystorage.file_was_not_found'; static atLocation = (location: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new FileWasNotFound( `File was not found at location: ${location}`, context, cause, ); } export function isFileWasNotFound(error: unknown): error is FileWasNotFound { return (typeof error === 'object' && (error as any).code === 'flystorage.file_was_not_found'); } export class UnableToSetVisibility extends FlystorageError { public readonly code = 'flystorage.unable_to_set_visibility'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToSetVisibility( `Unable to set visibility. Reason: ${reason}`, context, cause, ); } export class UnableToGetVisibility extends FlystorageError { public readonly code = 'flystorage.unable_to_get_visibility'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetVisibility( `Unable to get visibility. Reason: ${reason}`, context, cause, ); } export class UnableToGetPublicUrl extends FlystorageError { public readonly code = 'flystorage.unable_to_get_public_url'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetPublicUrl( `Unable to get public URL. Reason: ${reason}`, context, cause, ); } export class UnableToGetTemporaryUrl extends FlystorageError { public readonly code = 'flystorage.unable_to_get_temporary_url'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetTemporaryUrl( `Unable to get temporary URL. Reason: ${reason}`, context, cause, ); } export class UnableToPrepareUploadRequest extends FlystorageError { public readonly code = 'flystorage.unable_to_prepare_upload_request'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetTemporaryUrl( `Unable to prepare upload request. Reason: ${reason}`, context, cause, ); } export class UnableToCopyFile extends FlystorageError { public readonly code = 'flystorage.unable_to_copy_file'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToCopyFile( `Unable to copy file. Reason: ${reason}`, context, cause, ); } export class UnableToMoveFile extends FlystorageError { public readonly code = 'flystorage.unable_to_move_file'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToMoveFile( `Unable to move file. Reason: ${reason}`, context, cause, ); } export class UnableToGetStat extends FlystorageError { public readonly code = 'flystorage.unable_to_get_stat'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetStat( `Unable to get stat. Reason: ${reason}`, context, cause, ); static noFileStatResolved = ({context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToGetStat( `Stat was not a file.`, context, cause, ); } export class UnableToCreateDirectory extends FlystorageError { public readonly code = 'flystorage.unable_to_create_directory'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToCreateDirectory( `Unable to create directory. Reason: ${reason}`, context, cause, ); } export class UnableToDeleteDirectory extends FlystorageError { public readonly code = 'flystorage.unable_to_delete_directory'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToDeleteDirectory( `Unable to delete directory. Reason: ${reason}`, context, cause, ); } export class UnableToDeleteFile extends FlystorageError { public readonly code = 'flystorage.unable_to_delete_file'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToDeleteFile( `Unable to delete file. Reason: ${reason}`, context, cause, ); } export class UnableToCheckFileExistence extends FlystorageError { public readonly code = 'flystorage.unable_to_check_file_existence'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToCheckFileExistence( `Unable to check file existence. Reason: ${reason}`, context, cause, ); } export class UnableToCheckDirectoryExistence extends FlystorageError { public readonly code = 'flystorage.unable_to_check_directory_existence'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToCheckDirectoryExistence( `Unable to check directory existence. Reason: ${reason}`, context, cause, ); } export class UnableToListDirectory extends FlystorageError { public readonly code = 'flystorage.unable_to_list_directory_contents'; static because = (reason: string, {context = {}, cause = undefined}: { context?: ErrorContext, cause?: unknown }) => new UnableToListDirectory( `Unable to list directory contents. Reason: ${reason}`, context, cause, ); } ================================================ FILE: packages/file-storage/src/file-storage.test.ts ================================================ import {FileStorage, UploadRequest, UploadRequestOptions} from './file-storage.js'; import {InMemoryStorageAdapter} from '@flystorage/in-memory'; import {createHash} from 'crypto'; describe('FileStorage', () => { test('calculating a checksum through a fallback', async () => { const hash = createHash('md5'); hash.update('contents'); const expectedChecksum = hash.digest('hex'); const storage = new FileStorage(new InMemoryStorageAdapter()); await storage.write('something.txt', 'contents'); const checksum = await storage.checksum('something.txt', {algo: 'md5'}); expect(checksum).toEqual(expectedChecksum); }); test('supplying a prepared upload strategy', async () => { const storage = new FileStorage( new InMemoryStorageAdapter(), undefined, { preparedUploadStrategy: { async prepareUpload(path: string, options: UploadRequestOptions): Promise { return { method: 'POST', url: `https://here.com/${path}`, headers: options.headers ?? {}, }; } } } ); const request = await storage.prepareUpload('here.txt', { expiresAt: 0, headers: { 'content-type': 'application/json', } }); expect(request).toEqual({ method: 'POST', url: 'https://here.com/here.txt', headers: { 'content-type': 'application/json', }, }); }); }); ================================================ FILE: packages/file-storage/src/file-storage.ts ================================================ import {BinaryToTextEncoding} from 'crypto'; import {Readable} from 'stream'; import {checksumFromStream} from './checksum-from-stream.js'; import {PathNormalizer, PathNormalizerV1} from './path-normalizer.js'; import {TextEncoder} from 'util'; import { ChecksumIsNotAvailable, errorToMessage, isFileWasNotFound, UnableToCheckDirectoryExistence, UnableToCheckFileExistence, UnableToCopyFile, UnableToCreateDirectory, UnableToDeleteDirectory, UnableToDeleteFile, UnableToGetChecksum, UnableToGetFileSize, UnableToGetLastModified, UnableToGetMimeType, UnableToGetPublicUrl, UnableToGetStat, UnableToGetTemporaryUrl, UnableToGetVisibility, UnableToListDirectory, UnableToMoveFile, UnableToPrepareUploadRequest, UnableToReadFile, UnableToSetVisibility, UnableToWriteFile, } from './errors.js'; import {PassThrough} from 'node:stream'; export type CommonStatInfo = Readonly<{ path: string, lastModifiedMs?: number, visibility?: string, }>; export type FileInfo = Readonly<{ type: 'file', size?: number, isFile: true, isDirectory: false, mimeType?: string, } & CommonStatInfo>; export type DirectoryInfo = Readonly<{ type: 'directory', isFile: false, isDirectory: true, } & CommonStatInfo>; export function isFile(stat: StatEntry): stat is FileInfo { return stat.isFile; } export function isDirectory(stat: StatEntry): stat is DirectoryInfo { return stat.isDirectory; } export type StatEntry = FileInfo | DirectoryInfo; export type AdapterListOptions = ListOptions & { deep: boolean }; export interface StorageAdapter { write(path: string, contents: Readable, options: WriteOptions): Promise; read(path: string, options: MiscellaneousOptions): Promise; deleteFile(path: string, options: MiscellaneousOptions): Promise; createDirectory(path: string, options: CreateDirectoryOptions): Promise; copyFile(from: string, to: string, options: CopyFileOptions): Promise; moveFile(from: string, to: string, options: MoveFileOptions): Promise; stat(path: string, options: MiscellaneousOptions): Promise; list(path: string, options: AdapterListOptions): AsyncGenerator; changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise; visibility(path: string, options: MiscellaneousOptions): Promise; deleteDirectory(path: string, options: MiscellaneousOptions): Promise; fileExists(path: string, options: MiscellaneousOptions): Promise; directoryExists(path: string, options: MiscellaneousOptions): Promise; publicUrl(path: string, options: PublicUrlOptions): Promise; temporaryUrl(path: string, options: TemporaryUrlOptions): Promise; prepareUpload?(path: string, options: UploadRequestOptions): Promise; checksum(path: string, options: ChecksumOptions): Promise; mimeType(path: string, options: MimeTypeOptions): Promise; lastModified(path: string, options: MiscellaneousOptions): Promise; fileSize(path: string, options: MiscellaneousOptions): Promise; } export class DirectoryListing implements AsyncIterable { constructor( private readonly listing: AsyncGenerator, private readonly path: string, private readonly deep: boolean, ) { } async toArray(sorted: boolean = true): Promise { const items = []; for await (const item of this.listing) { items.push(item); } return sorted ? items.sort((a, b) => naturalSorting.compare(a.path, b.path)) : items; } filter(filter: (entry: StatEntry) => boolean): DirectoryListing { const listing = this.listing; const filtered = (async function* () { for await (const entry of listing) { if (filter(entry)) { yield entry; } } })(); return new DirectoryListing(filtered, this.path, this.deep); } async* [Symbol.asyncIterator]() { try { for await (const item of this.listing) { yield item; } } catch (error) { throw UnableToListDirectory.because( errorToMessage(error), {cause: error, context: {path: this.path, deep: this.deep}}, ); } } } export type FileContents = Iterable | AsyncIterable | NodeJS.ReadableStream | Readable | string; export type TimeoutOptions = { timeout?: number }; export type MiscellaneousOptions = TimeoutOptions & { [option: string]: any, abortSignal?: AbortSignal, } export type MimeTypeOptions = MiscellaneousOptions & { disallowFallback?: boolean, fallbackMethod?: 'contents' | 'path', } export type VisibilityFallback = { strategy: 'ignore', stagedVisibilityResponse?: string, } | { strategy: 'error', errorMessage?: string, } export type VisibilityOptions = MiscellaneousOptions & { visibility?: string, directoryVisibility?: string, retainVisibility?: boolean, } & ({ useVisibility: true, } | { useVisibility: false, visibilityFallback: VisibilityFallback, } | {}); export type WriteOptions = VisibilityOptions & MiscellaneousOptions & { mimeType?: string, size?: number, cacheControl?: string, }; export type CreateDirectoryOptions = MiscellaneousOptions & Pick & {}; export type PublicUrlOptions = MiscellaneousOptions & {}; export type UploadRequestOptions = MiscellaneousOptions & { expiresAt: ExpiresAt, contentType?: string, headers?: UploadRequestHeaders, }; export type CopyFileOptions = MiscellaneousOptions & VisibilityOptions & { retainVisibility?: boolean, }; export type MoveFileOptions = MiscellaneousOptions & VisibilityOptions & { retainVisibility?: boolean, }; export type ListOptions = MiscellaneousOptions & { deep?: boolean }; export type TemporaryUrlOptions = MiscellaneousOptions & { expiresAt: ExpiresAt, responseHeaders?: { [header: string]: string }, }; export type ChecksumOptions = MiscellaneousOptions & { algo?: string, encoding?: BinaryToTextEncoding, } export type ConfigurationOptions = { visibility?: VisibilityOptions, writes?: WriteOptions, moves?: MoveFileOptions, copies?: CopyFileOptions, publicUrls?: PublicUrlOptions, temporaryUrls?: TemporaryUrlOptions, uploadRequest?: UploadRequestOptions, checksums?: ChecksumOptions, mimeTypes?: MimeTypeOptions, preparedUploadStrategy?: PreparedUploadStrategy, timeout?: TimeoutOptions, list?: ListOptions, } export function toReadable(contents: FileContents): Readable { if (contents instanceof Readable) { return contents; } return Readable.from(contents); } const naturalSorting = new Intl.Collator(undefined, { numeric: true, sensitivity: 'base', }); function instrumentAbortSignal(options: Options): Options { let abortSignal = options.abortSignal; if (options.timeout !== undefined) { const timeoutAbort = AbortSignal.timeout(options.timeout); if (options.abortSignal) { const originalAbortSignal = options.abortSignal; abortSignal = AbortSignal.any([ originalAbortSignal, timeoutAbort, ]); } else { abortSignal = timeoutAbort; } } if (abortSignal?.aborted) { throw abortSignal.reason; } return {...options, abortSignal}; } export class FileStorage { constructor( private readonly adapter: StorageAdapter, private readonly pathNormalizer: PathNormalizer = new PathNormalizerV1(), private readonly options: ConfigurationOptions = {}, ) { } public async write(path: string, contents: FileContents, options: WriteOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.writes, ...options}); try { const body = toReadable(contents); await this.adapter.write( this.pathNormalizer.normalizePath(path), body, options, ); await closeReadable(body); } catch (error) { throw UnableToWriteFile.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } public async read(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { const stream = Readable.from( await this.adapter.read(this.pathNormalizer.normalizePath(path), options), ); const streamOut = new PassThrough(); stream.on('error', (error) => { stream.unpipe(streamOut); streamOut.destroy( isFileWasNotFound(error) ? UnableToReadFile.becauseFileWasNotFound(error) : error, ); }); stream.pipe(streamOut); return streamOut; } catch (error) { if (isFileWasNotFound(error)) { throw UnableToReadFile.becauseFileWasNotFound(error); } throw UnableToReadFile.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public async readToString(path: string, options: MiscellaneousOptions = {}): Promise { return await readableToString(await this.read(path, options)); } public async readToUint8Array(path: string, options: MiscellaneousOptions = {}): Promise { return await readableToUint8Array(await this.read(path, options)); } public async readToBuffer(path: string, options: MiscellaneousOptions = {}): Promise { return Buffer.from(await this.readToUint8Array(path, options)); } public async deleteFile(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { await this.adapter.deleteFile( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToDeleteFile.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public async createDirectory(path: string, options: CreateDirectoryOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...options}); try { return await this.adapter.createDirectory( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToCreateDirectory.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } public async deleteDirectory(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { return await this.adapter.deleteDirectory(this.pathNormalizer.normalizePath(path), options); } catch (error) { throw UnableToDeleteDirectory.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public async stat(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { return await this.adapter.stat(this.pathNormalizer.normalizePath(path), options); } catch (error) { throw UnableToGetStat.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public async moveFile(from: string, to: string, options: MoveFileOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.moves, ...options}); try { await this.adapter.moveFile( this.pathNormalizer.normalizePath(from), this.pathNormalizer.normalizePath(to), options, ); } catch (error) { throw UnableToMoveFile.because( errorToMessage(error), {cause: error, context: {from, to}}, ); } } public async copyFile(from: string, to: string, options: CopyFileOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.copies, ...options}); try { await this.adapter.copyFile( this.pathNormalizer.normalizePath(from), this.pathNormalizer.normalizePath(to), options, ); } catch (error) { throw UnableToCopyFile.because( errorToMessage(error), {cause: error, context: {from, to}}, ); } } public async changeVisibility(path: string, visibility: string, options: VisibilityOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); if (options.useVisibility === false) { const fallback: VisibilityFallback = options.visibilityFallback; if (fallback.strategy === 'ignore') { return; } else if (fallback.strategy === 'error') { throw UnableToSetVisibility.because(fallback.errorMessage ?? 'Configured not to use visibility', { context: {path, visibility}, }); } } try { return await this.adapter.changeVisibility(this.pathNormalizer.normalizePath(path), visibility, options); } catch (error) { throw UnableToSetVisibility.because( errorToMessage(error), {cause: error, context: {path, visibility}}, ); } } public async visibility(path: string, options: VisibilityOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...options}); if (options.useVisibility === false) { const fallback: VisibilityFallback = options.visibilityFallback; if (fallback.strategy === 'ignore') { return fallback.stagedVisibilityResponse ?? 'unknown'; } else if (fallback.strategy === 'error') { throw UnableToGetVisibility.because(fallback.errorMessage ?? 'Configured not to use visibility', { context: {path}, }); } } try { return await this.adapter.visibility(this.pathNormalizer.normalizePath(path), options); } catch (error) { throw UnableToGetVisibility.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public async fileExists(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { return await this.adapter.fileExists(this.pathNormalizer.normalizePath(path), options); } catch (error) { throw UnableToCheckFileExistence.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public list(path: string, options: ListOptions = {}): DirectoryListing { options = instrumentAbortSignal({...this.options.timeout, ...this.options.list, ...options}); const adapterOptions: AdapterListOptions = { ...options, deep: options.deep ?? false, }; return new DirectoryListing( this.adapter.list(this.pathNormalizer.normalizePath(path), adapterOptions), path, adapterOptions.deep, ); } public async statFile(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); const stat = await this.stat(path, options); if (isFile(stat)) { return stat; } throw UnableToGetStat.noFileStatResolved({context: {path}}); } public async directoryExists(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { return await this.adapter.directoryExists(this.pathNormalizer.normalizePath(path), options); } catch (error) { throw UnableToCheckDirectoryExistence.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public async publicUrl(path: string, options: PublicUrlOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.publicUrls, ...options}); try { return await this.adapter.publicUrl( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToGetPublicUrl.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } public async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.temporaryUrls, ...options}); try { return await this.adapter.temporaryUrl( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToGetTemporaryUrl.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } public async prepareUpload(path: string, options: UploadRequestOptions): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.uploadRequest, ...options}); if (this.options.preparedUploadStrategy !== undefined) { try { return this.options.preparedUploadStrategy.prepareUpload(path, options); } catch (error) { throw UnableToPrepareUploadRequest.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } if (typeof this.adapter.prepareUpload !== 'function') { throw new Error('The used adapter does not support prepared uploads.'); } try { return await this.adapter.prepareUpload( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToPrepareUploadRequest.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } public async checksum(path: string, options: ChecksumOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.checksums, ...options}); try { return await this.adapter.checksum( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { if (ChecksumIsNotAvailable.isErrorOfType(error)) { return this.calculateChecksum(path, options); } throw UnableToGetChecksum.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } public async mimeType(path: string, options: MimeTypeOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...this.options.mimeTypes, ...options}); try { return await this.adapter.mimeType( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToGetMimeType.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } public async lastModified(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { return await this.adapter.lastModified( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToGetLastModified.because( errorToMessage(error), {cause: error, context: {path}}, ); } } public async fileSize(path: string, options: MiscellaneousOptions = {}): Promise { options = instrumentAbortSignal({...this.options.timeout, ...options}); try { return await this.adapter.fileSize( this.pathNormalizer.normalizePath(path), options, ); } catch (error) { throw UnableToGetFileSize.because( errorToMessage(error), {cause: error, context: {path}}, ); } } private async calculateChecksum(path: string, options: ChecksumOptions): Promise { try { return await checksumFromStream(await this.read(path, options), options); } catch (error) { throw UnableToGetChecksum.because( errorToMessage(error), {cause: error, context: {path, options}}, ); } } } export type TimestampMs = number; export type ExpiresAt = Date | TimestampMs; export function normalizeExpiryToDate(expiresAt: ExpiresAt): Date { return expiresAt instanceof Date ? expiresAt : new Date(expiresAt); } export function normalizeExpiryToMilliseconds(expiresAt: ExpiresAt): number { return expiresAt instanceof Date ? expiresAt.getTime() : expiresAt; } export async function closeReadable(body: Readable) { if (body.closed || body.destroyed) { return; } await new Promise((resolve, reject) => { body.on('error', reject); body.on('close', (err: any) => { err ? reject(err) : resolve(); }); body.destroy(); }); } const decoder = new TextDecoder(); export async function readableToString(stream: Readable): Promise { const contents = decoder.decode(await readableToUint8Array(stream)); await closeReadable(stream); return contents; } export async function readableToBuffer(stream: Readable): Promise { return new Promise((resolve, reject) => { const buffers: Buffer[] = []; stream.on('data', chunk => buffers.push(Buffer.from(chunk))); stream.on('end', () => resolve(Buffer.concat(buffers))); stream.on('finish', () => resolve(Buffer.concat(buffers))); stream.on('error', err => reject(err)); }); } const encoder = new TextEncoder(); export function readableToUint8Array(stream: Readable): Promise { return new Promise((resolve, reject) => { const parts: Uint8Array[] = []; stream.on('data', (chunk: Uint8Array | string | number) => { const type = typeof chunk; if (type === 'string') { chunk = encoder.encode(chunk as string); } else if (type === 'number') { chunk = new Uint8Array([chunk as number]); } parts.push(chunk as Uint8Array); }); stream.on('error', reject); stream.on('end', () => resolve(concatUint8Arrays(parts))); }); } function concatUint8Arrays(input: Uint8Array[]): Uint8Array { const length = input.reduce((l, a) => l + (a.byteLength), 0); const output = new Uint8Array(length); let position = 0; input.forEach(i => { output.set(i, position); position += i.byteLength; }); return output; } export type UploadRequestHeaders = Record>; export type UploadRequest = { url: string, provider?: string, method: 'PUT' | 'POST' headers: UploadRequestHeaders, } export interface PreparedUploadStrategy { prepareUpload(path: string, options: UploadRequestOptions): Promise; } export class PreparedUploadsAreNotSupported implements PreparedUploadStrategy { prepareUpload(): Promise { throw new Error('The used adapter does not support prepared uploads.'); } } ================================================ FILE: packages/file-storage/src/index.ts ================================================ export * from './checksum-from-stream.js'; export * from './file-storage.js'; export * from './errors.js'; export * from './path-normalizer.js'; export * from './path-prefixer.js' export * from './portable-visibility.js'; ================================================ FILE: packages/file-storage/src/path-normalizer.test.ts ================================================ import {CorruptedPathDetected, PathNormalizerV1, PathTraversalDetected} from './path-normalizer.js'; describe('PathNormalizerV1', () => { const normalizer = new PathNormalizerV1(); test.each([ ['.', ''], ['/path/to/dir/.', 'path/to/dir'], ['/dirname/', 'dirname'], ['dirname/..', ''], ['dirname/../', ''], ['dirname./', 'dirname.'], ['dirname/./', 'dirname'], ['dirname/.', 'dirname'], ['./dir/../././', ''], ['/something/deep/../../dirname', 'dirname'], ['00004869/files/other/10-75..stl', '00004869/files/other/10-75..stl'], ['/dirname//subdir///subsubdir', 'dirname/subdir/subsubdir'], ['example/path/..txt', 'example/path/..txt'], ['/example/../path.txt', 'path.txt'], ])('normalizing "%s" to "%s"', (path, expected) => { expect(normalizer.normalizePath(path)).toEqual(expected); }); test.each([ ['something/../../../hehe'], ['/something/../../..'], ['..'], ])('detecting path traversal on "%s"', (path) => { expect(() => normalizer.normalizePath(path)).toThrow(PathTraversalDetected.forPath(path)); }); test.each([ ['some\0/path.txt'], ['s\x09i.php'], ])('detecting funky whitespace on "%s"', (path) => { expect(() => normalizer.normalizePath(path)).toThrow(CorruptedPathDetected.unexpectedWhitespace(path)); }); }) ================================================ FILE: packages/file-storage/src/path-normalizer.ts ================================================ import {join} from 'node:path'; export interface PathNormalizer { normalizePath(path: string): string } const funkyWhiteSpaceRegex = new RegExp('\\p{C}+', 'u'); export class PathNormalizerV1 implements PathNormalizer { normalizePath(path: string): string { if (funkyWhiteSpaceRegex.test(path)) { throw CorruptedPathDetected.unexpectedWhitespace(path); } const normalized = join(...(path.split('/'))); if (normalized.indexOf('../') !== -1 || normalized == '..') { throw PathTraversalDetected.forPath(path); } return normalized === '.' ? '' : normalized; } } export class CorruptedPathDetected extends Error { static unexpectedWhitespace = (path: string) => new CorruptedPathDetected( `Corrupted path detected with unexpected whitespace: ${path}` ); } export class PathTraversalDetected extends Error { static forPath = (path: string) => new PathTraversalDetected( `Path traversal detected for: ${path}` ); } ================================================ FILE: packages/file-storage/src/path-prefixer.test.ts ================================================ import {PathPrefixer} from './path-prefixer.js'; describe('PathPrefixer', () => { describe.each([ ['with', '/prefix/'], ['without', '/prefix'], ])('prefixing %s a trailing slash prefix', (_name, prefix: string) => { const prefixer = new PathPrefixer(prefix); test('prefixing a file path with a leading slash', () => { expect(prefixer.prefixFilePath('/file.txt')).toEqual('/prefix/file.txt'); }); test('prefixing a file path without a leading slash', () => { expect(prefixer.prefixFilePath('file.txt')).toEqual('/prefix/file.txt'); }); test('prefixing a directory path with a leading slash', () => { expect(prefixer.prefixDirectoryPath('/dirname')).toEqual('/prefix/dirname/'); }); test('prefixing a directory path with a trailing slash', () => { expect(prefixer.prefixDirectoryPath('dirname/')).toEqual('/prefix/dirname/'); }); }); describe.each([ ['with', '/prefix/'], ['without', '/prefix'], ])('stripping %s a trailing slash prefix', (_name, prefix: string) => { const prefixer = new PathPrefixer(prefix); test('stripping a file path', () => { expect(prefixer.stripFilePath('/prefix/file.txt')).toEqual('file.txt'); }); test('stripping a directory path without a trailing slash', () => { expect(prefixer.stripDirectoryPath('/prefix/dirname')).toEqual('dirname'); }); test('prefixing a directory path with a trailing slash', () => { expect(prefixer.stripDirectoryPath('/prefix/dirname/')).toEqual('dirname'); }); }); describe('prefixing with an empty prefix', () => { const prefixer = new PathPrefixer(); test('prefixing with a file path with leading slash', () => { expect(prefixer.prefixFilePath('/file.txt')).toEqual('/file.txt'); }); test('prefixing with a directory path with leading slash', () => { expect(prefixer.prefixDirectoryPath('/directory')).toEqual('/directory/'); }); }) }); ================================================ FILE: packages/file-storage/src/path-prefixer.ts ================================================ import {type join, posix} from 'node:path'; export class PathPrefixer { private readonly prefix: string = ''; constructor( prefix: string = '', private readonly separator: string = '/', private readonly joinFunc: typeof join = posix.join, ) { if (prefix.length > 0) { this.prefix = this.joinFunc(prefix, this.separator); } } prefixFilePath(path: string): string { return this.prefix.length > 0 ? this.joinFunc(this.prefix, path): path; } prefixDirectoryPath(path: string): string { let fullPath = this.prefix.length > 0 ? this.joinFunc(this.prefix, path) : path; if (fullPath.length > 0 && !fullPath.endsWith(this.separator)) { fullPath = `${fullPath}${this.separator}`; } return fullPath; } stripFilePath(path: string): string { return path.substring(this.prefix.length); } stripDirectoryPath(path: string): string { return this.stripFilePath(path).replace(/\/+$/g, ''); } } ================================================ FILE: packages/file-storage/src/portable-visibility.ts ================================================ export enum Visibility { PUBLIC = 'public', PRIVATE = 'private', } ================================================ FILE: packages/file-storage/src/readable-convertion.test.ts ================================================ import {Readable} from "node:stream"; import {readableToString} from "./file-storage.js"; describe('converting readables', () => { test('converting a stream to a string', async () => { expect(true).toEqual(true); const readable = Readable.from('something'); const result = await readableToString(readable); expect(result).toEqual('something'); }); }); ================================================ FILE: packages/file-storage/src/utilities.test.ts ================================================ import {Readable} from 'stream'; import {readableToBuffer} from './file-storage.js'; describe('utilities', () => { describe('readableToBuffer', () => { test('converting to and from a buffer', async () => { const input = Buffer.from('buffer contents'); const inputReadable = Readable.from(input); const output = await readableToBuffer(inputReadable); expect(output.toString()).toEqual(input.toString()); }) }) }); ================================================ FILE: packages/file-storage/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packages/google-cloud-storage/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/google-cloud-storage/CHANGELOG.md ================================================ # `@flystorage/google-cloud-storage` ## 1.2.0 ### Changes - Upgraded dependencies - Remove not-used import ## 1.1.0 ### Changes - Improved stream errors - Added way to detect reads failed due to missing files ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/google-cloud-storage/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/google-cloud-storage/README.md ================================================ # Flystorage adapter for AWS S3 This package contains the Flystorage adapter for Google Cloud Storage ## Installation Install all the required packages ```bash npm install --save @flystorage/file-storage @flystorage/google-cloud-storage @google-cloud/storage ``` ## Usage ```typescript import {FileStorage} from '@flystorage/file-storage'; import {GoogleCloudStorageStorageAdapter} from '@flystorage/google-cloud-storage'; import {Storage} from '@google-cloud/storage'; const client = new Storage(); const bucket = client.bucket('{bucket-name}}', { userProject: '{user-project}}', }); const adapter = new GoogleCloudStorageStorageAdapter(bucket, { prefix: '{optional-path-prefix}', }); const storage = new FileStorage(adapter); ``` > ⚠️ Always use the FileStorage, it is essential for security and a good developer > experience. Do not use the adapter directly. ## Visibility Setting an retrieving visibility is only meaningful for legacy buckets. To use this functionality with Flystorage, pass the legacy visibility handling to the constructor: ```typescript import {GoogleCloudStorageStorageAdapter, LegacyVisibilityHandling} from '@flystorage/google-cloud-storage'; const adapter = new GoogleCloudStorageStorageAdapter(bucket, { prefix: '{optional-path-prefix}', }, new LegacyVisibilityHandling( 'allUsers', // acl entity, optional 'publicRead', // acl for Visibility.PUBLIC, optional, 'projectPrivate', // acl for Visibility.PRIVATE, optional, )); ``` ================================================ FILE: packages/google-cloud-storage/package.json ================================================ { "name": "@flystorage/google-cloud-storage", "type": "module", "version": "1.2.0", "dependencies": { "@flystorage/file-storage": "^1.1.0", "@flystorage/stream-mime-type": "^1.0.0", "@google-cloud/storage": "^7.19.0", "file-type": "^21.3.1", "mime-types": "^3.0.2" }, "description": "", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "keywords": [ "google", "file", "storage", "flystorage", "filesystem" ], "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/adapter/google-cloud-storage/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/google-cloud-storage" }, "license": "MIT" } ================================================ FILE: packages/google-cloud-storage/src/google-cloud-storage.test.ts ================================================ import { Storage } from '@google-cloud/storage'; import {GoogleCloudStorageAdapter} from './google-cloud-storage.js'; import { FileStorage, UploadRequestHeaders, readableToString, UnableToReadFile, Visibility, } from '@flystorage/file-storage'; import {randomBytes} from 'crypto'; import {resolve} from 'node:path'; import * as https from 'https'; import {LegacyVisibilityHandling} from './visibility-handling.js'; const testSegment = randomBytes(10).toString('hex'); let adapter: GoogleCloudStorageAdapter; let storage: FileStorage; let cleanupStorage: FileStorage; describe('GoogleCloudStorageAdapter', () => { const googleStorage = new Storage({keyFilename: resolve(process.cwd(), 'google-cloud-service-account.json')}); const bucket = googleStorage.bucket('no-acl-bucket-for-ci', { userProject: 'flysystem-testing', }); const legacyBucket = googleStorage.bucket('flysystem', { userProject: 'flysystem-testing', }); beforeAll(async () => { const cleanupAdapter = new GoogleCloudStorageAdapter(bucket, { prefix: `flystorage`, }); cleanupStorage = new FileStorage(cleanupAdapter); await cleanupStorage.deleteDirectory('non-existing-directory'); }) afterAll(async () => { await cleanupStorage.deleteDirectory(testSegment); }); describe('base API', () => { beforeEach(() => { const secondSegment = randomBytes(10).toString('hex'); adapter = new GoogleCloudStorageAdapter(bucket, { prefix: `flystorage/${testSegment}/${secondSegment}`, }); storage = new FileStorage(adapter); }); test('reading a file that was written', async () => { await storage.write('path.txt', 'content in azure'); const content = await storage.readToString('path.txt'); expect(content).toEqual('content in azure'); expect(await storage.fileExists('path.txt')).toEqual(true); }); test('deleting a directory', async () => { await storage.createDirectory('directory/a'); await storage.write('directory/b.txt', 'contents'); await storage.deleteDirectory('directory'); expect(await storage.fileExists('directory/b.txt')).toEqual(false); }); test('statting a file', async () => { await storage.write('directory/b.txt', 'contents'); const stat = await storage.statFile('directory/b.txt'); expect(stat.path).toEqual('directory/b.txt'); expect(typeof stat.lastModifiedMs).toEqual('number'); }); test('trying to read a file that does not exist', async () => { let was404 = false; try { await storage.readToString('404.txt'); } catch (err) { if (err instanceof UnableToReadFile) { was404 = err.wasFileNotFound; } else { throw err; } } expect(was404).toEqual(true); }); test('checking if a directory exists by prefix', async () => { await storage.write('directory/file.txt', 'contents'); expect(await storage.directoryExists('directory')).toEqual(true); }); test('getting a crc32c checksum', async () => { await storage.write('directory/file.txt', 'contents'); expect(await storage.checksum('directory/file.txt', { algo: 'crc32c', })).toEqual('E2fhmA=='); }); test('getting an md5 checksum', async () => { await storage.write('directory/file.txt', 'contents'); expect(await storage.checksum('directory/file.txt', { algo: 'md5', })).toEqual('mL99jBV4Two9YyBEQeHiqg=='); }); test('getting an sha1 checksum', async () => { await storage.write('directory/file.txt', 'contents'); expect(await storage.checksum('directory/file.txt', { algo: 'sha1', })).toEqual('4a756ca07e9487f482465a99e8286abc86ba4dc7'); }); test('checking if a created directory', async () => { await storage.createDirectory('directory/here'); expect(await storage.directoryExists('directory/here')).toEqual(true); expect(await storage.directoryExists('directory')).toEqual(true); }); test('non deep and deep listing should have consistent and similar results', async () => { await storage.write('file_1.txt', 'contents'); await storage.write('file_2.txt', 'contents'); await storage.createDirectory('directory_1'); await storage.createDirectory('directory_2'); const non_deep_listing = await storage.list('/', {deep: false}).toArray(); const deep_listing = await storage.list('/', {deep: true}).toArray(); expect(non_deep_listing).toHaveLength(4); expect(deep_listing).toHaveLength(4); expect(non_deep_listing).toEqual(deep_listing); }); }); describe('visibility for legacy buckets', () => { beforeEach(() => { const secondSegment = randomBytes(10).toString('hex'); adapter = new GoogleCloudStorageAdapter(legacyBucket, { prefix: `flystorage/${testSegment}/${secondSegment}`, }, new LegacyVisibilityHandling()); storage = new FileStorage(adapter); }); test('getting the visibility of a public file', async () => { await storage.write('path.txt', 'contents', { visibility: Visibility.PUBLIC, }); expect(await storage.visibility('path.txt')).toEqual('public'); }); test('getting the visibility of a private file', async () => { await storage.write('path.txt', 'contents', { visibility: Visibility.PRIVATE, }); expect(await storage.visibility('path.txt')).toEqual('private'); }); test('changing the visibility of a private file', async () => { await storage.write('path.txt', 'contents', { visibility: Visibility.PRIVATE, }); await storage.changeVisibility('path.txt', Visibility.PUBLIC) expect(await storage.visibility('path.txt')).toEqual('public'); }); test('using a public URL', async () => { await storage.write('path.txt', 'public contents', { visibility: Visibility.PUBLIC, }); const url = await storage.publicUrl('path.txt'); const contents = await naivelyDownloadFile(url); expect(contents).toEqual('public contents'); }); test('with a custom Cache-Control Header', async () => { await storage.write('cache.txt', 'contents', {visibility: Visibility.PUBLIC, cacheControl: "max-age=9999, public"}); const url = await storage.publicUrl('cache.txt'); const res = await fetch(url); expect (res.headers.get('Cache-Control')).toEqual('max-age=9999, public'); }) test('using a temporary URL', async () => { await storage.write('path.txt', 'private contents', { visibility: Visibility.PRIVATE, }); const url = await storage.temporaryUrl('path.txt', { expiresAt: Date.now() + 15 * 1000, }); const contents = await naivelyDownloadFile(url); expect(contents).toEqual('private contents'); }); test('uploading using a prepared request', async () => { const request = await storage.prepareUpload('prepared/request-file.txt', { expiresAt: Date.now() + 60 * 1000, headers: { 'Content-Type': 'text/plain', } }); await naivelyMakeRequestFile( request.url, request.headers, request.method, 'this is the contents', ); const contents = await storage.readToString('prepared/request-file.txt'); expect(contents).toEqual('this is the contents'); }); }); }); function naivelyDownloadFile(url: string): Promise { return new Promise((resolve, reject) => { https.get(url, async res => { if (res.statusCode !== 200) { reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`)); } else { resolve(await readableToString(res)); } }); }); } function naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise { return new Promise((resolve, reject) => { const req = https.request(url, { method: method, headers: { ...headers, 'Content-Length': new Blob([data]).size, } }, async res => { let responseBody = ''; res.on('data', (chunk) => { responseBody += chunk; }); res.on('end', () => { const statusCode = res.statusCode ?? 500; if (statusCode <= 200 && statusCode >= 299) { reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`)); } else { resolve(); } }); }); req.on('error', (err) => { reject(err); }); req.write(data); req.end(); }); } ================================================ FILE: packages/google-cloud-storage/src/google-cloud-storage.ts ================================================ import { ChecksumIsNotAvailable, ChecksumOptions, CopyFileOptions, CreateDirectoryOptions, FileContents, FileWasNotFound, MimeTypeOptions, MoveFileOptions, PathPrefixer, PublicUrlOptions, StatEntry, StorageAdapter, TemporaryUrlOptions, UploadRequest, UploadRequestHeaders, UploadRequestOptions, WriteOptions, } from '@flystorage/file-storage'; import {Readable} from 'stream'; import {Bucket, GetFilesOptions, File, GetSignedUrlConfig, ApiError} from '@google-cloud/storage'; import {resolveMimeType} from '@flystorage/stream-mime-type'; import {pipeline} from 'node:stream/promises'; import { UniformBucketLevelAccessVisibilityHandling, VisibilityHandlingForGoogleCloudStorage, } from './visibility-handling.js'; import {PassThrough} from 'node:stream'; export type GoogleCloudStorageAdapterOptions = { prefix?: string, } export class GoogleCloudStorageAdapter implements StorageAdapter { private readonly prefixer: PathPrefixer; constructor( private readonly bucket: Bucket, readonly options: GoogleCloudStorageAdapterOptions = {}, readonly visibilityHandling: VisibilityHandlingForGoogleCloudStorage = new UniformBucketLevelAccessVisibilityHandling() ) { this.prefixer = new PathPrefixer(options.prefix ?? ''); } async write(path: string, contents: Readable, options: WriteOptions): Promise { let mimeType = options.mimeType; if (mimeType === undefined) { [mimeType, contents] = await resolveMimeType(path, contents); } const writeStream = this.bucket.file(this.prefixer.prefixFilePath(path)) .createWriteStream({ contentType: mimeType, predefinedAcl: options.visibility ? this.visibilityHandling.visibilityToPredefinedAcl(options.visibility) : undefined, metadata: options.cacheControl? {cacheControl: options.cacheControl} : undefined, }); await pipeline(contents, writeStream); } async read(path: string): Promise { const readStream = this.bucket.file(this.prefixer.prefixFilePath(path)).createReadStream(); readStream.on('error', err => { readStream.unpipe(errorHandler); if (err instanceof ApiError && err.code === 404) { errorHandler.destroy(FileWasNotFound.atLocation(path, { context: {path}, cause: err, })) } else { errorHandler.destroy(err); } }); const errorHandler = new PassThrough(); readStream.pipe(errorHandler); return errorHandler; } async deleteFile(path: string): Promise { await this.bucket.file(this.prefixer.prefixFilePath(path)).delete({ ignoreNotFound: true, }); } async createDirectory(path: string, options: CreateDirectoryOptions): Promise { await this.bucket.file(this.prefixer.prefixDirectoryPath(path)).save(''); } async copyFile(from: string, to: string, options: CopyFileOptions): Promise { await this.bucket.file(this.prefixer.prefixFilePath(from)).copy( this.bucket.file(this.prefixer.prefixFilePath(to)), ); } async moveFile(from: string, to: string, options: MoveFileOptions): Promise { await this.copyFile(from, to, {}); await this.deleteFile(from); } async stat(path: string): Promise { const [metadata] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getMetadata(); return this.mapToStatEntry(metadata); } async* list(path: string, options: { deep: boolean; }): AsyncGenerator { let response: File[]; let query: GetFilesOptions | null = { autoPaginate: false, delimiter: options.deep ? undefined : '/', includeTrailingDelimiter: options.deep ? undefined : true, prefix: this.prefixer.prefixDirectoryPath(path), } while (query !== null) { [response, query] = await this.bucket.getFiles(query); for (const item of response) { yield this.mapToStatEntry(item.metadata); } } } private mapToStatEntry(file: File['metadata']): StatEntry { if (file.name!.endsWith('/')) { return { type: 'directory', isFile: false, isDirectory: true, path: this.prefixer.stripDirectoryPath(file.name!), }; } return { type: 'file', isFile: true, isDirectory: false, path: this.prefixer.stripFilePath(file.name!), lastModifiedMs: file.updated ? new Date(file.updated).getTime() : undefined, mimeType: file.contentType, }; } async changeVisibility(path: string, visibility: string): Promise { await this.visibilityHandling.changeVisibility( this.bucket.file(this.prefixer.prefixFilePath(path)), visibility, ); } async visibility(path: string): Promise { return await this.visibilityHandling.determineVisibility( this.bucket.file(this.prefixer.prefixFilePath(path)), ); } async deleteDirectory(path: string): Promise { const prefix = this.prefixer.prefixDirectoryPath(path); await this.bucket.deleteFiles({ prefix, }); await this.bucket.file(prefix).delete({ ignoreNotFound: true, }); } async fileExists(path: string): Promise { const [exists] = await this.bucket.file(this.prefixer.prefixFilePath(path)).exists(); return exists; } async directoryExists(path: string): Promise { const [exists] = await this.bucket.file(this.prefixer.prefixDirectoryPath(path)).exists(); if (exists) { return true; } const [response] = await this.bucket.getFiles({ autoPaginate: false, maxResults: 1, prefix: this.prefixer.prefixDirectoryPath(path), }); return response.length > 0; } async publicUrl(path: string, options: PublicUrlOptions): Promise { return this.bucket.file(this.prefixer.prefixFilePath(path)).publicUrl(); } async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { const [response] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getSignedUrl({ action: 'read', expires: options.expiresAt, }); return response; } async prepareUpload(path: string, options: UploadRequestOptions): Promise { const headers: UploadRequestHeaders = {}; const config: GetSignedUrlConfig = { action: 'write', expires: options.expiresAt, }; const contentType = options['Content-Type'] ?? options.contentType; if (typeof contentType === 'string') { config.contentType = contentType; headers['Content-Type'] = contentType; } const [url] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getSignedUrl(config); return { url, headers, method: 'PUT', provider: 'google-cloud-storage', }; } async checksum(path: string, options: ChecksumOptions): Promise { const algo = options.algo ?? 'md5'; if (algo !== 'md5' && algo !== 'crc32c') { throw ChecksumIsNotAvailable.checksumNotSupported(algo); } const [metadata] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getMetadata(); return algo === 'crc32c' ? metadata.crc32c! : metadata.md5Hash!; } async mimeType(path: string, options: MimeTypeOptions): Promise { const stat = await this.stat(path); if (stat.type !== 'file' || stat.mimeType === undefined) { throw new Error('Unable to resolve mime-type, not available in stat entry.'); } return stat.mimeType; } async lastModified(path: string): Promise { const stat = await this.stat(path); if (stat.type !== 'file' || stat.lastModifiedMs === undefined) { throw new Error('Unable to resolve last modified time, not available in stat entry.'); } return stat.lastModifiedMs; } async fileSize(path: string): Promise { const stat = await this.stat(path); if (stat.type !== 'file' || stat.size === undefined) { throw new Error('Unable to resolve file size, not available in stat entry.'); } return stat.size; } } /** * BC export * * @deprecated */ export class GoogleCloudFileStorage extends GoogleCloudStorageAdapter {} ================================================ FILE: packages/google-cloud-storage/src/index.ts ================================================ export * from './google-cloud-storage.js'; export * from './visibility-handling.js'; ================================================ FILE: packages/google-cloud-storage/src/visibility-handling.ts ================================================ import {Visibility} from '@flystorage/file-storage'; import {ApiError, File, PredefinedAcl} from '@google-cloud/storage'; export interface VisibilityHandlingForGoogleCloudStorage { changeVisibility(file: File, visibility: string): Promise; determineVisibility(file: File): Promise; visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefined, } export class UniformBucketLevelAccessVisibilityHandling implements VisibilityHandlingForGoogleCloudStorage { constructor( private readonly errorOnChange: boolean = false, private readonly errorOnDetermine: boolean = false, private readonly fakeVisibility: string = 'unknown', ) { } async changeVisibility(file: File, visibility: string): Promise { if (this.errorOnChange) { throw new Error('Unable to set visibility when using uniform bucket level access control.'); } // ignored, no-op } async determineVisibility(file: File): Promise { if (this.errorOnDetermine) { throw new Error('Unable to determine visibility when using uniform bucket level access control.'); } return 'unknown'; } visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefined { if (this.errorOnChange) { throw new Error('Unable to set visibility when using uniform bucket level access control.'); } return undefined; } } export class LegacyVisibilityHandling implements VisibilityHandlingForGoogleCloudStorage { constructor( private readonly entity: string = 'allUsers', private readonly publicAcl: PredefinedAcl = 'publicRead', private readonly privateAcl: PredefinedAcl = 'projectPrivate', ) { } async changeVisibility(file: File, visibility: string): Promise { if (visibility === Visibility.PRIVATE) { await file.acl.delete({ entity: this.entity, }) } else if (visibility === Visibility.PUBLIC) { await file.acl.update({ entity: this.entity, role: 'READER', }) } } async determineVisibility(file: File): Promise { try { const [, metadata] = await file.acl.get({entity: 'allUsers'}); return metadata.role === 'READER' ? Visibility.PUBLIC : Visibility.PRIVATE; } catch (error) { if (!(error instanceof ApiError) || error.response?.statusCode !== 404) { throw error; } return Visibility.PRIVATE; } } visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefined { if (visibility === Visibility.PUBLIC) { return this.publicAcl; } else if (visibility === Visibility.PRIVATE) { return this.privateAcl; } throw new Error(`Not able to set visibility ${visibility}, no mapping known.`); } } ================================================ FILE: packages/google-cloud-storage/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packages/in-memory/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/in-memory/CHANGELOG.md ================================================ # `@flystorage/in-memory` ## 1.1.0 ### Changes - Use a Buffer for internal storage ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/in-memory/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/in-memory/README.md ================================================ # Flystorage adapter using memory This package contains the Flystorage adapter that uses only memory ## Installation Install all the required packages ```bash npm install --save @flystorage/file-storage @flystorage/in-memory ``` ## Usage ```typescript import {FileStorage} from '@flystorage/file-storage'; import {InMemoryStorageAdapter} from '@flystorage/in-memory'; const adapter = new InMemoryStorageAdapter(); const storage = new FileStorage(adapter); ``` > ⚠️ Always use the FileStorage, it is essential for security and a good developer > experience. Do not use the adapter directly. ================================================ FILE: packages/in-memory/package.json ================================================ { "name": "@flystorage/in-memory", "type": "module", "version": "1.1.0", "dependencies": { "@flystorage/file-storage": "^1.1.0", "file-type": "^21.3.1", "mime-types": "^3.0.2" }, "description": "", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "keywords": [ "test double", "memory", "file", "storage", "flystorage", "filesystem" ], "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/adapter/in-memory/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/in-memory" }, "license": "MIT" } ================================================ FILE: packages/in-memory/src/in-memory-file-storage.test.ts ================================================ import {FileStorage, Visibility} from "@flystorage/file-storage"; import {InMemoryStorageAdapter} from "./in-memory-file-storage.js"; describe('InMemoryStorageAdapter', () => { const adapter = new InMemoryStorageAdapter(); const storage = new FileStorage(adapter); beforeEach(() => { adapter.deleteEverything(); }); test('non deep and deep listing should have consistent and similar results', async () => { await storage.write('file_1.txt', 'contents'); await storage.write('file_2.txt', 'contents'); await storage.createDirectory('directory_1'); await storage.createDirectory('directory_2'); const non_deep_listing = await storage.list('/', {deep: false}).toArray(); const deep_listing = await storage.list('/', {deep: true}).toArray(); expect(non_deep_listing).toHaveLength(4); expect(deep_listing).toHaveLength(4); expect(non_deep_listing).toEqual(deep_listing); }); test('reading a file that was written', async () => { await storage.write('path.txt', 'content in azure'); const content = await storage.readToString('path.txt'); expect(content).toEqual('content in azure'); }); test('trying to read a file that does not exist', async () => { await expect(storage.readToString('404.tx')).rejects.toThrow(); }); test('trying to see if a non-existing file exists', async () => { expect(await storage.fileExists('404.txt')).toEqual(false); }); test('trying to see if an existing file exists', async () => { await storage.write('existing.txt', 'contents'); expect(await storage.fileExists('existing.txt')).toEqual(true); }); test('deleting an existing file', async () => { await storage.write('existing.txt', 'contents'); expect(await storage.fileExists('existing.txt')).toEqual(true); await storage.deleteFile('existing.txt'); expect(await storage.fileExists('existing.txt')).toEqual(false); }); test('deleting a non-existing file is OK', async () => { await expect(storage.deleteFile('404.txt')); }); test('copying a file', async () => { await storage.write('file.txt', 'copied'); await storage.copyFile('file.txt', 'new-file.txt'); expect(await storage.fileExists('file.txt')).toEqual(true); expect(await storage.fileExists('new-file.txt')).toEqual(true); expect(await storage.readToString('new-file.txt')).toEqual('copied'); }); test('moving a file', async () => { await storage.write('file.txt', 'moved'); await storage.moveFile('file.txt', 'new-file.txt'); expect(await storage.fileExists('file.txt')).toEqual(false); expect(await storage.fileExists('new-file.txt')).toEqual(true); expect(await storage.readToString('new-file.txt')).toEqual('moved'); }); test('setting visibility always fails', async () => { await storage.write('exsiting.txt', 'yes'); await expect(storage.changeVisibility('existing.txt', Visibility.PUBLIC)).rejects.toThrow(); await expect(storage.changeVisibility('404.txt', Visibility.PRIVATE)).rejects.toThrow(); }); test('listing entries in a directory, shallow', async () => { await storage.write('outside/path.txt', 'test'); await storage.write('inside/a.txt', 'test'); await storage.write('inside/b.txt', 'test'); await storage.write('inside/c/a.txt', 'test'); const listing = await storage.list('inside').toArray(); expect(listing).toHaveLength(3); expect(listing[0].type).toEqual('file'); expect(listing[1].type).toEqual('file'); expect(listing[2].type).toEqual('directory'); expect(listing[0].path).toEqual('inside/a.txt'); expect(listing[1].path).toEqual('inside/b.txt'); expect(listing[2].path).toEqual('inside/c'); }); test('listing entries in a directory, deep', async () => { await storage.write('outside/path.txt', 'test'); await storage.write('inside/a.txt', 'test'); await storage.write('inside/b.txt', 'test'); await storage.write('inside/c/a.txt', 'test'); const listing = await storage.list('inside', {deep: true}).toArray(); expect(listing).toHaveLength(4); expect(listing[0].type).toEqual('file'); expect(listing[1].type).toEqual('file'); expect(listing[2].type).toEqual('directory'); expect(listing[3].type).toEqual('file'); expect(listing[0].path).toEqual('inside/a.txt'); expect(listing[1].path).toEqual('inside/b.txt'); expect(listing[2].path).toEqual('inside/c'); expect(listing[3].path).toEqual('inside/c/a.txt'); }); test('deleting a full directory', async () => { await storage.write('directory/a.txt', 'test'); await storage.write('directory/b.txt', 'test'); await storage.write('directory/c/a.txt', 'test'); await storage.deleteDirectory('directory'); const listing = await storage.list('directory', {deep: true}).toArray(); expect(listing).toEqual([]); }); test('checking if a directory exists', async () => { await storage.write('directory/a.txt', 'test'); await storage.write('directory/b.txt', 'test'); await storage.write('directory/c/a.txt', 'test'); expect(await storage.directoryExists('directory')).toEqual(true); expect(await storage.directoryExists('directory/c')).toEqual(true); expect(await storage.directoryExists('directory/a')).toEqual(false); }); test('getting a md5 checksum of a file', async () => { await storage.write('this.txt', 'test'); const checksum = await storage.checksum('this.txt', {algo: 'md5'}); expect(checksum).toEqual('098f6bcd4621d373cade4e832627b4f6'); }); }); ================================================ FILE: packages/in-memory/src/in-memory-file-storage.ts ================================================ import { ChecksumIsNotAvailable, ChecksumOptions, CopyFileOptions, CreateDirectoryOptions, FileContents, MimeTypeOptions, MoveFileOptions, PublicUrlOptions, StatEntry, StorageAdapter, TemporaryUrlOptions, VisibilityOptions, WriteOptions, readableToBuffer, readableToString, } from '@flystorage/file-storage'; import {Readable} from "node:stream"; import {dirname, parse} from 'node:path' import {lookup as mimeTimeForExt} from "mime-types"; type FileEntry = { type: 'file', path: string, contents: Buffer, lastModifiedMs: number, visibility?: string, } type DirectoryEntry = { type: 'directory', path: string, visibility?: string, } export type TimestampResolver = () => number; function cloneBuffer(input: Buffer): Buffer { const output = Buffer.alloc(input.length); input.copy(output); return output; } export class InMemoryStorageAdapter implements StorageAdapter { private entries: Map = new Map; constructor( private readonly timestampResolver: TimestampResolver = () => Date.now(), ) { } deleteEverything(): void { this.entries = new Map; } async write(path: string, contents: Readable, options: WriteOptions): Promise { this.ensureParentDirsExist(path, options); this.entries.set(path, { type: 'file', path, contents: await readableToBuffer(contents), lastModifiedMs: this.timestampResolver(), visibility: options.visibility, }) } private ensureParentDirsExist(path: string, options: VisibilityOptions) { let parentDir = dirname(path); while (!['.', ''].includes(parentDir) && !this.entries.has(parentDir)) { this.entries.set(parentDir, { path: parentDir, type: 'directory', visibility: options.directoryVisibility, }); parentDir = dirname(parentDir); } } async read(path: string): Promise { const file = this.entries.get(path); if (file?.type !== 'file') { throw new Error(`Path "${path}" is not a file`); } return Readable.from(cloneBuffer(file.contents)); } async deleteFile(path: string): Promise { this.entries.delete(path); } async createDirectory(path: string, options: CreateDirectoryOptions): Promise { path = path.replace(/\/+$/g, ''); this.entries.set(path, { path, type: 'directory', visibility: options.directoryVisibility, }); } async copyFile(from: string, to: string, options: CopyFileOptions): Promise { const source = this.entries.get(from); if (source?.type !== 'file') { throw new Error(`Source file ${from} does not exist`); } this.ensureParentDirsExist(to, options); this.entries.set(to, { ...source, path: to, }); } async moveFile(from: string, to: string, options: MoveFileOptions): Promise { await this.copyFile(from, to, options); await this.deleteFile(from); } async stat(path: string): Promise { const entry = this.entries.get(path); if (entry === undefined) { throw new Error(`No entry found for path "${path}"`); } return this.mapToStatEntry(entry); } private mapToStatEntry(entry: DirectoryEntry | FileEntry): StatEntry { if (entry.type === 'directory') { return { ...entry, isDirectory: true, isFile: false, }; } return { type: 'file', path: entry.path, visibility: entry.visibility, isFile: true, isDirectory: false, lastModifiedMs: entry.lastModifiedMs, } } async *list(path: string, options: { deep: boolean; }): AsyncGenerator { const entries = this.entries.values(); const prefix = path === '' || path === '/' ? '' : `${path.replace(/\/+$/g, '')}/`; for (const entry of entries) { if (!entry.path.startsWith(prefix)) { continue; } if (options.deep !== true && entry.path.indexOf('/', prefix.length) !== -1) { continue; } yield this.mapToStatEntry(entry); } } async changeVisibility(path: string, visibility: string): Promise { const entry = this.entries.get(path); if (entry?.type !== 'file') { throw new Error(`Path ${path} is not a file`); } this.entries.set(path, { ...entry, visibility, }); } async visibility(path: string): Promise { const entry = this.entries.get(path); if (entry === undefined) { throw new Error(`Path ${path} does not exist`); } return entry.visibility ?? 'public'; } async deleteDirectory(path: string): Promise { const entries = this.entries.values(); const prefix = `${path.replace(/\/+$/g, '')}/`; for (const entry of entries) { if (entry.path.startsWith(prefix) || entry.path === path) { this.entries.delete(entry.path); } } } async fileExists(path: string): Promise { return this.entries.get(path)?.type === 'file'; } async directoryExists(path: string): Promise { return this.entries.get(path)?.type === 'directory'; } async publicUrl(path: string, options: PublicUrlOptions): Promise { throw new Error("Method not implemented."); } async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { throw new Error("Method not implemented."); } async checksum(path: string, options: ChecksumOptions): Promise { if (this.entries.get(path)?.type !== 'file') { // throw new Error(`File ${path} does not exists`); } throw ChecksumIsNotAvailable.checksumNotSupported(options.algo ?? 'unknown', { context: {path}, }); } async mimeType(path: string, options: MimeTypeOptions): Promise { const entry = this.entries.get(path); if (entry?.type !== 'file') { throw new Error(`File ${path} does not exist`); } return await resolveMimeType(path, Buffer.from(entry.contents)); } async lastModified(path: string): Promise { const entry = this.entries.get(path); if (entry?.type !== 'file') { throw new Error(`File ${path} does not exist`); } return entry.lastModifiedMs; } async fileSize(path: string): Promise { const entry = this.entries.get(path); if (entry?.type !== 'file') { throw new Error(`File ${path} does not exist`); } return Buffer.byteLength(entry.contents); } } export async function resolveMimeType( filename: string, contents: Uint8Array, ): Promise { const {fileTypeFromBuffer} = await import('file-type'); const lookup = await fileTypeFromBuffer(contents); if (lookup) { return lookup.mime; } const {ext} = parse(filename); return mimeTimeForExt(ext) || 'application/octet-stream'; } /** * BC export * * @deprecated */ export class InMemoryFileStorage extends InMemoryStorageAdapter {} ================================================ FILE: packages/in-memory/src/index.ts ================================================ export * from './in-memory-file-storage.js'; ================================================ FILE: packages/in-memory/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packages/local-fs/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/local-fs/CHANGELOG.md ================================================ # `@flystorage/local-fs` ## 1.1.0 ## Added - AbortSignal Support ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/local-fs/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/local-fs/README.md ================================================ # Flystorage adapter for the local filesystem This package contains the Flystorage adapter for the local filesystem. ## Installation Install all the required packages ```bash npm install --save @flystorage/file-storage @flystorage/local-fs ``` ## Usage ```typescript import {FileStorage} from '@flystorage/file-storage'; import {LocalStorageAdapter} from '@flystorage/local-fs'; const rootDirectory = resolve(process.cwd(), 'my-files'); const adapter = new LocalStorageAdapter(rootDirectory); const storage = new FileStorage(adapter); ``` > ⚠️ Always use the FileStorage, it is essential for security and a good developer > experience. Do not use the adapter directly. ================================================ FILE: packages/local-fs/package.json ================================================ { "name": "@flystorage/local-fs", "type": "module", "version": "1.2.0", "dependencies": { "@flystorage/dynamic-import": "^1.0.0", "@flystorage/file-storage": "^1.1.0", "file-type": "^21.3.1", "mime-types": "^3.0.2" }, "description": "", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "patch": "gsed -i 's/dynamicallyImport(/import(/g' dist/esm/local-file-storage.js", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "keywords": [ "fs", "file", "storage", "flystorage", "filesystem" ], "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/adapter/local-fs/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/local-fs" }, "engines": { "node": ">=20.1.0" }, "license": "MIT" } ================================================ FILE: packages/local-fs/src/index.ts ================================================ export * from './local-file-storage.js'; export * from './unix-visibility.js'; ================================================ FILE: packages/local-fs/src/local-file-storage.test.ts ================================================ import { FileInfo, FileStorage, normalizeExpiryToMilliseconds, UnableToReadFile, UploadRequest, Visibility, } from '@flystorage/file-storage'; import {BinaryToTextEncoding, createHash, randomBytes} from 'crypto'; import * as path from 'node:path'; import {LocalStorageAdapter, LocalTemporaryUrlGenerator, LocalTemporaryUrlOptions} from './local-file-storage.js'; import {execSync} from 'child_process'; import {createReadStream} from 'node:fs'; import {closeReadable} from "@flystorage/file-storage"; const rootDirectory = path.resolve(process.cwd(), 'fixtures/test-files'); let storage: FileStorage; describe('LocalStorageAdapter', () => { beforeEach(() => { const testSegment = randomBytes(10).toString('hex'); storage = new FileStorage( new LocalStorageAdapter( path.join(rootDirectory, testSegment), { publicUrlOptions: { baseUrl: 'https://default.com/', } } ), ); }) beforeAll(() => { execSync(`rm -rf ${rootDirectory}/*`); }); test('preparing uploads is not supported by default', async () => { await expect(storage.prepareUpload('somewhere', { expiresAt: Date.now() + 10000, })).rejects.toThrowError(); }); test('you can use your own upload preparer to support preparing uploads', async () => { const storage = new FileStorage( new LocalStorageAdapter( rootDirectory, { publicUrlOptions: { baseUrl: 'https://default.com/', }, }, undefined, undefined, undefined, { async prepareUpload(path) { return { method: 'PUT', url: `https://fixates.url.com/${path}`, headers: {}, } }, }, ), ); const request = await storage.prepareUpload('somewhere/here.txt', { expiresAt: 0, }); expect(request).toEqual({ method: 'PUT', url: 'https://fixates.url.com/somewhere/here.txt', headers: {}, } satisfies UploadRequest); }); test('deleting a file', async () => { await storage.write('text.txt', 'contents'); expect(await storage.fileExists('text.txt')).toEqual(true); await storage.deleteFile('text.txt'); expect(await storage.fileExists('text.txt')).toEqual(false); }); test('deleting a directory', async () => { await storage.write('directory/text.txt', 'contents'); await storage.deleteDirectory('directory'); expect(await storage.fileExists('directory/text.txt')).toEqual(false); expect(await storage.directoryExists('directory')).toEqual(false); }); test('files written with private visibility have private visibility', async () => { await storage.write('test.txt', 'contents', { visibility: Visibility.PRIVATE, }); const stat = await storage.stat('test.txt'); expect(stat.visibility).toEqual(Visibility.PRIVATE); }); test('writing a png and fetching its mime-type', async () => { const handle = createReadStream(path.resolve(process.cwd(), 'fixtures/screenshot.png')); await storage.write('image.png', handle); closeReadable(handle); const mimeType = await storage.mimeType('image.png'); expect(mimeType).toEqual('image/png'); }); test('retrieving the size of a file', async () => { const contents = 'this is the contents of the file'; await storage.write('something.txt', contents); expect(await storage.fileSize('something.txt')).toEqual(contents.length); }); describe('stat for (public) files', () => { let fileInfo: FileInfo; beforeEach(async () => { await storage.write('test.txt', 'contents', { visibility: Visibility.PUBLIC, }); fileInfo = await storage.statFile('test.txt'); }); test('it exposes file size', async () => { expect(fileInfo.size).toEqual(8); }); test('it exposes a type', async () => { expect(fileInfo.type).toEqual('file'); }); test('it exposes (public) visibility', async () => { expect(fileInfo.visibility).toEqual(Visibility.PUBLIC); }); }); test('listing the contents of a directory, shallow', async () => { await storage.write('file-1.txt', 'contents'); await storage.write('file-2.txt', 'contents'); const listing = await storage.list('/').toArray(); expect(listing).toHaveLength(2); expect(listing.map(l => l.type)).toEqual(['file', 'file']); expect(listing.map(l => l.path).sort()).toEqual(['file-1.txt', 'file-2.txt']); }); test('file file visibility can be changes', async () => { await storage.write('file.txt', 'contents', { visibility: Visibility.PUBLIC, }); expect((await storage.stat('file.txt')).visibility).toEqual(Visibility.PUBLIC); await storage.changeVisibility('file.txt', Visibility.PRIVATE); expect((await storage.statFile('file.txt')).visibility).toEqual(Visibility.PRIVATE); }); test('checking if a non-existing file exists', async () => { expect(await storage.fileExists('non-existing-file.txt')).toEqual(false); }); test('checking if an existing file exists', async () => { await storage.write('existing-file.txt', 'contents'); expect(await storage.fileExists('existing-file.txt')).toEqual(true); }); test('checking if a directory is an existing file', async () => { await storage.createDirectory('existing-file.txt'); expect(await storage.fileExists('existing-file.txt')).toEqual(false); }); test('checking if a non-existing directory exists', async () => { expect(await storage.directoryExists('does-not-exist')).toEqual(false); }); test('checking if a file is a directory', async () => { await storage.write('location', 'nothing'); expect(await storage.directoryExists('location')).toEqual(false); }); test('checking if an existing directory exists', async () => { await storage.createDirectory('location'); expect(await storage.directoryExists('location')).toEqual(true); }); test('writing a file implicitly creates parent directories', async () => { await storage.write('deeply/nested/directory.txt', 'contents'); const listing = await storage.list('/', {deep: true}).toArray(); expect(listing).toHaveLength(3); expect(listing.map(l => l.type).sort()).toEqual(['directory', 'directory', 'file']); }); test('non deep and deep listing should have consistent and similar results', async () => { await storage.write('file_1.txt', 'contents'); await storage.write('file_2.txt', 'contents'); await storage.createDirectory('directory_1'); await storage.createDirectory('directory_2'); const non_deep_listing = await storage.list('/', {deep: false}).toArray(); const deep_listing = await storage.list('/', {deep: true}).toArray(); expect(non_deep_listing).toHaveLength(4); expect(deep_listing).toHaveLength(4); expect(non_deep_listing).toEqual(deep_listing); }); test('filtering out directories for a deep listing', async () => { await storage.write('deeply/nested/directory.txt', 'contents'); const listing = await storage.list('/', {deep: true}) .filter(entry => entry.isFile) .toArray(); expect(listing).toHaveLength(1); expect(listing.map(l => l.type).sort()).toEqual(['file']); }); test('generating a public urls works when the base URL is provided in the constructor', async () => { const url = await storage.publicUrl('some/path.txt'); expect(url).toEqual('https://default.com/some/path.txt'); }); test('generating a public urls works when the base URL is provided as an option', async () => { const url = await storage.publicUrl('/some/path.txt', { baseUrl: 'https://example.org/with-prefix/', }); expect(url).toEqual('https://example.org/with-prefix/some/path.txt'); }); test('moving a file', async () => { await storage.write('move-from.txt', 'this'); expect(await storage.readToString('move-from.txt')).toEqual('this'); await storage.moveFile('move-from.txt', 'move-to.txt'); expect(await storage.fileExists('move-from.txt')).toEqual(false); expect(await storage.readToString('move-to.txt')).toEqual('this'); }); test('copying a file', async () => { await storage.write('from.txt', 'this'); await storage.copyFile('from.txt', 'to.txt'); expect(await storage.fileExists('from.txt')).toEqual(true); expect(await storage.readToString('to.txt')).toEqual('this'); }); test('trying to copy a file that does not exist', async () => { await expect(storage.copyFile('from.txt', 'to.txt')).rejects.toThrow(); }); test('trying to read a file that does not exist', async () => { let was404 = false; try { await storage.readToString('404.txt'); } catch (err) { if (err instanceof UnableToReadFile) { was404 = err.wasFileNotFound; } } expect(was404).toEqual(true); }); test('trying to move a file that does not exist', async () => { await expect(storage.moveFile('from.txt', 'to.txt')).rejects.toThrow(); }); test('moving a file between directories', async () => { await storage.write('dir-a/here.txt', 'this'); await storage.moveFile('dir-a/here.txt', 'dir-b/there.txt'); expect(await storage.fileExists('dir-a/here.txt')).toEqual(false); expect(await storage.readToString('dir-b/there.txt')).toEqual('this'); }); test('copying a file between directories', async () => { await storage.write('dir-a/here.txt', 'this'); await storage.copyFile('dir-a/here.txt', 'dir-b/there.txt'); expect(await storage.fileExists('dir-a/here.txt')).toEqual(true); expect(await storage.readToString('dir-b/there.txt')).toEqual('this'); }); test('generating a public urls failed when the base URL is undefined', async () => { const url = storage.publicUrl('/some/path.txt', { baseUrl: undefined }); await expect(url).rejects.toThrow(); }); test('generating a temporary URL fails when no generator is configured', async () => { await expect(storage.temporaryUrl('path.txt', { expiresAt: new Date(), })).rejects.toThrow(); }); test('generating a temporary URL works when the generator is configured', async () => { const storage = new FileStorage( new LocalStorageAdapter( rootDirectory, { publicUrlOptions: { baseUrl: 'https://default.com/', }, temporaryUrlOptions: { baseUrl: 'https://secret.com/' } }, undefined, undefined, new FakeTemporaryUrlGenerator(), ), ); const now = Date.now(); await expect(storage.temporaryUrl('fake.txt', { expiresAt: now, })).resolves.toEqual(`https://secret.com/fake.txt?ts=${now}`); }); test('it can calculate checksums', async () => { function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string { return createHash(algo).update(input).digest(encoding); } const contents = 'this is for the checksum'; await storage.write('path.txt', contents); const expectedChecksum = hashString(contents, 'md5'); const checksum = await storage.checksum('path.txt', { algo: 'md5', }); expect(checksum).toEqual(expectedChecksum); }); }); class FakeTemporaryUrlGenerator implements LocalTemporaryUrlGenerator { async temporaryUrl(path: string, options: LocalTemporaryUrlOptions): Promise { return `${options.baseUrl}${path}?ts=${normalizeExpiryToMilliseconds(options.expiresAt)}`; } } ================================================ FILE: packages/local-fs/src/local-file-storage.ts ================================================ import { checksumFromStream, ChecksumOptions, CreateDirectoryOptions, PathPrefixer, PublicUrlOptions, StatEntry, StorageAdapter, TemporaryUrlOptions, WriteOptions, CopyFileOptions, MoveFileOptions, VisibilityOptions, MimeTypeOptions, UploadRequestOptions, UploadRequest, MiscellaneousOptions, FileWasNotFound, } from '@flystorage/file-storage'; import {lookup} from "mime-types"; import {createReadStream, createWriteStream, Dirent, Stats} from 'node:fs'; import {chmod, mkdir, opendir, rm, stat, rename, copyFile} from 'node:fs/promises'; import {join, extname, dirname, sep} from 'node:path'; import {Readable} from 'stream'; import {pipeline} from 'stream/promises'; import {PortableUnixVisibilityConversion, UnixVisibilityConversion} from './unix-visibility.js'; import {dynamicallyImport} from '@flystorage/dynamic-import'; import {PreparedUploadsAreNotSupported, PreparedUploadStrategy} from '@flystorage/file-storage'; import {PassThrough} from 'node:stream'; export type LocalStorageAdapterOptions = { rootDirectoryVisibility?: string, publicUrlOptions?: LocalPublicUrlOptions, temporaryUrlOptions?: Omit, }; export type LocalPublicUrlOptions = PublicUrlOptions & { baseUrl?: string, } export type LocalPublicUrlGenerator = { publicUrl(path: string, options: LocalPublicUrlOptions): Promise; } export class BaseUrlLocalPublicUrlGenerator implements LocalPublicUrlGenerator { async publicUrl(path: string, options: LocalPublicUrlOptions): Promise { if (options.baseUrl === undefined) { throw new Error('No base URL defined for public URL generation'); } const base = options.baseUrl.endsWith('/') ? options.baseUrl : `${options.baseUrl}/`; if (sep === '\\' && path.includes(sep)) { path = path.replace(sep, '/'); } return `${base}${path}`; } } export type LocalTemporaryUrlOptions = TemporaryUrlOptions & { baseUrl?: string, } export type LocalTemporaryUrlGenerator = { temporaryUrl(path: string, options: LocalTemporaryUrlOptions): Promise; } export class FailingLocalTemporaryUrlGenerator implements LocalTemporaryUrlGenerator { async temporaryUrl(): Promise { throw new Error('No temporary URL generator provided'); } } type FileTypePackage = typeof import('file-type'); let fileTypeImport: Promise | undefined; let fileTypes: FileTypePackage | undefined = undefined; function maybeAbort(signal?: AbortSignal) { if (signal?.aborted) { throw signal.reason; } } export class LocalStorageAdapter implements StorageAdapter { private prefixer: PathPrefixer; constructor( readonly rootDir: string, private readonly options: LocalStorageAdapterOptions = {}, private readonly visibilityConversion: UnixVisibilityConversion = new PortableUnixVisibilityConversion(), private readonly publicUrlGenerator: LocalPublicUrlGenerator = new BaseUrlLocalPublicUrlGenerator(), private readonly temporaryUrlGenerator: LocalTemporaryUrlGenerator = new FailingLocalTemporaryUrlGenerator(), private readonly uploadPreparer: PreparedUploadStrategy = new PreparedUploadsAreNotSupported(), ) { this.rootDir = join(this.rootDir, sep); this.prefixer = new PathPrefixer(this.rootDir, sep, join); } async copyFile(from: string, to: string, options: CopyFileOptions): Promise { maybeAbort(options.abortSignal); await this.ensureRootDirectoryExists(); maybeAbort(options.abortSignal); await this.ensureParentDirectoryExists(to, options); maybeAbort(options.abortSignal); await copyFile( this.prefixer.prefixFilePath(from), this.prefixer.prefixFilePath(to), ); } async moveFile(from: string, to: string, options: MoveFileOptions): Promise { maybeAbort(options.abortSignal); await this.ensureRootDirectoryExists(); maybeAbort(options.abortSignal); await this.ensureParentDirectoryExists(to, options); maybeAbort(options.abortSignal); await rename( this.prefixer.prefixFilePath(from), this.prefixer.prefixFilePath(to), ); } prepareUpload(path: string, options: UploadRequestOptions): Promise { maybeAbort(options.abortSignal); return this.uploadPreparer.prepareUpload(path, options); } temporaryUrl(path: string, options: TemporaryUrlOptions): Promise { maybeAbort(options.abortSignal); return this.temporaryUrlGenerator.temporaryUrl(path, {...this.options.temporaryUrlOptions, ...options}); } publicUrl(path: string, options: PublicUrlOptions): Promise { maybeAbort(options.abortSignal); return this.publicUrlGenerator.publicUrl(path, {...this.options.publicUrlOptions, ...options}); } async mimeType(path: string, options: MimeTypeOptions): Promise { maybeAbort(options.abortSignal); if (fileTypeImport === undefined) { fileTypeImport = dynamicallyImport('file-type'); } if (fileTypes === undefined) { fileTypes = await fileTypeImport; maybeAbort(options.abortSignal); } const {fileTypeFromFile, supportedExtensions} = fileTypes; const extension = extname(path); if (!supportedExtensions.has(extension as any)) { const mimetype = lookup(extension); if (mimetype === false) { throw new Error('Unable to resolve mime-type'); } return mimetype; } const location = this.prefixer.prefixFilePath(path); const result = await fileTypeFromFile(location); if (result === undefined) { throw new Error('Unable to resolve mime-type'); } return result.mime; } async fileSize(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); const stat = await this.doStat(path, 'file'); if (!stat.isFile) { throw new Error(`Path ${path} is not a file.`); } if (stat.size === undefined) { throw new Error('Stat unexpectedly did not return file size.'); } return stat.size; } async lastModified(path: string, options: MiscellaneousOptions): Promise { const stat = await this.doStat(path, 'file'); if (!stat.isFile) { throw new Error(`Path ${path} is not a file.`); } if (stat.lastModifiedMs === undefined) { throw new Error('Stat unexpectedly did not return last modified.'); } return stat.lastModifiedMs; } async* list(path: string, {deep}: { deep: boolean }): AsyncGenerator { let entries = await opendir(this.prefixer.prefixDirectoryPath(path), { recursive: deep, }); for await (const item of entries) { const itemPath = join(item.parentPath, item.name); yield this.mapStatToEntry( item, item.isFile() ? this.prefixer.stripFilePath(itemPath) : this.prefixer.stripDirectoryPath(itemPath) ); } } async read(path: string, options: MiscellaneousOptions): Promise { const readStream = createReadStream(this.prefixer.prefixFilePath(path)); const errorProxy = new PassThrough(); readStream.on('error', error => { readStream.unpipe(errorProxy); if ((error as any).message?.includes('ENOENT')) { errorProxy.destroy(FileWasNotFound.atLocation(path, { cause: error, context: {path, options}, })); } else { errorProxy.destroy(error); } }); readStream.pipe(errorProxy); return errorProxy; } async write(path: string, contents: Readable, options: WriteOptions): Promise { maybeAbort(options?.abortSignal); await this.ensureRootDirectoryExists(); maybeAbort(options?.abortSignal); await this.ensureParentDirectoryExists(path, options); maybeAbort(options?.abortSignal); const writeStream = createWriteStream( this.prefixer.prefixFilePath(path), { flags: 'w+', mode: options.visibility ? this.visibilityConversion.visibilityToFilePermissions(options.visibility) : undefined, }, ); if (options.abortSignal) { const signal = options.abortSignal; signal.addEventListener('abort', event => { contents.destroy(signal.reason); writeStream.destroy(signal.reason); }); } await pipeline(contents, writeStream); } async deleteFile(path: string): Promise { await rm(this.prefixer.prefixFilePath(path), { force: true, }); } async createDirectory(path: string, options: CreateDirectoryOptions): Promise { await mkdir(this.prefixer.prefixDirectoryPath(path), { recursive: true, mode: options.directoryVisibility ? this.visibilityConversion.visibilityToDirectoryPermissions(options.directoryVisibility) : undefined, }); } async stat(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); return this.doStat(path, 'file'); } private async doStat(path: string, type: 'file' | 'directory' = 'file'): Promise { return this.mapStatToEntry( await stat(type === 'file' ? this.prefixer.prefixFilePath(path) : this.prefixer.prefixDirectoryPath(path)), path, ); } async fileExists(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); try { const stat = await this.doStat(path, 'file'); return stat.isFile; } catch (e) { if (typeof e === 'object' && (e as any).code === 'ENOENT') { return false; } throw e; } } async deleteDirectory(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); await rm(this.prefixer.prefixDirectoryPath(path), { recursive: true, force: true, }); } private mapStatToEntry(info: Stats | Dirent, path: string): StatEntry { if (!info.isFile() && !info.isDirectory()) { throw new Error('Unsupported file entry encountered...'); } const isDirent = info instanceof Dirent; return info.isFile() ? { path, type: 'file', isFile: true, isDirectory: false, visibility: isDirent ? undefined : this.visibilityConversion.filePermissionsToVisibility(info.mode & 0o777), lastModifiedMs: isDirent ? undefined : info.mtimeMs, size: isDirent ? undefined : info.size, } : { path, type: 'directory', isFile: false, isDirectory: true, visibility: isDirent ? undefined : this.visibilityConversion.directoryPermissionsToVisibility(info.mode & 0o777), lastModifiedMs: isDirent ? undefined : info.mtimeMs, }; } async changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); await chmod( this.prefixer.prefixFilePath(path), this.visibilityConversion.visibilityToFilePermissions(visibility), ); } async visibility(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); const stat = await this.doStat(path, 'file'); if (!stat.visibility) { throw new Error('Unable to determine visibility'); } return stat.visibility; } async directoryExists(path: string, options: MiscellaneousOptions): Promise { maybeAbort(options.abortSignal); try { const stat = await this.doStat(path, 'directory'); return stat.isDirectory; } catch (e) { if (typeof e === 'object' && ['ENOTDIR', 'ENOENT'].includes((e as any).code)) { return false; } throw e; } } private rootDirectoryCreation: Promise | undefined = undefined; private async ensureRootDirectoryExists(): Promise { if (this.rootDirectoryCreation === undefined) { this.rootDirectoryCreation = this.createDirectory('', { directoryVisibility: this.options.rootDirectoryVisibility ?? this.visibilityConversion.defaultDirectoryVisibility, }); } return await this.rootDirectoryCreation; } private async ensureParentDirectoryExists(path: string, options: VisibilityOptions) { const directoryName = dirname(path); if (directoryName !== '.' && directoryName !== sep) { await this.createDirectory(directoryName, { directoryVisibility: options.directoryVisibility, }); } } async checksum(path: string, options: ChecksumOptions): Promise { maybeAbort(options.abortSignal); return checksumFromStream(await this.read(path, options), options); } } /** * BC export * * @deprecated */ export class LocalFileStorage extends LocalStorageAdapter {} ================================================ FILE: packages/local-fs/src/unix-visibility.ts ================================================ import {Visibility} from '@flystorage/file-storage'; export interface UnixVisibilityConversion { visibilityToFilePermissions(visibility: string): number; visibilityToDirectoryPermissions(visibility: string): number; filePermissionsToVisibility(permissions: number): string; directoryPermissionsToVisibility(permissions: number): string; defaultDirectoryPermissions: number; defaultDirectoryVisibility: string; } export class PortableUnixVisibilityConversion implements UnixVisibilityConversion { constructor( private readonly filePublic: number = 0o644, private readonly filePrivate: number = 0o600, private readonly directoryPublic: number = 0o755, private readonly directoryPrivate: number = 0o700, public readonly defaultDirectoryVisibility: Visibility = Visibility.PUBLIC, ) { } get defaultDirectoryPermissions(): number { return this.visibilityToDirectoryPermissions(this.defaultDirectoryVisibility); } directoryPermissionsToVisibility(permissions: number): string { if (permissions === this.directoryPrivate) { return Visibility.PRIVATE; } return Visibility.PUBLIC; } filePermissionsToVisibility(permissions: number): string { if (permissions === this.filePrivate) { return Visibility.PRIVATE; } return Visibility.PUBLIC; } visibilityToDirectoryPermissions(visibility: string): number { if (visibility === Visibility.PUBLIC) { return this.directoryPublic; } else if (visibility === Visibility.PRIVATE) { return this.directoryPrivate; } throw new Error(`Unsupported visibility was provided: ${visibility}`); } visibilityToFilePermissions(visibility: string): number { if (visibility === Visibility.PUBLIC) { return this.filePublic; } else if (visibility === Visibility.PRIVATE) { return this.filePrivate; } throw new Error(`Unsupported visibility was provided: ${visibility}`); } } ================================================ FILE: packages/local-fs/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "./src/**/*.ts" ] } ================================================ FILE: packages/multer-storage/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/multer-storage/CHANGELOG.md ================================================ # `@flystorage/multer-storage` ## 1.2.0 ### Changes - Force security upgrades ## 1.1.0 ### Changes - Unblock security fixes ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/multer-storage/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/multer-storage/README.md ================================================ # Flystorage for multer This package contains the Flystorage bindings for [multer](https://github.com/expressjs/multer). This allows multer to upload to any of the supported Flystorage filesystems. ## Installation Install all the required packages ```bash npm install --save @flystorage/file-storage @flystorage/multer-storage ``` ## Usage ```typescript import {FileStorage} from '@flystorage/file-storage'; import {FlystorageMulterStorageEngine} from '@flystorage/multer-storage'; import multer from 'multer'; const adapter = createYourAdapter(); const fileStorage = new FileStorage(adapter); const storage = new FlystorageMulterStorageEngine( uploadStorage, async (action, _req: express.Request, file: Express.Multer.File) => { if (action === 'handle') { return file.originalname; } else { return file.destination; } } ); const uploader = multer({storage}); ``` ================================================ FILE: packages/multer-storage/package.json ================================================ { "name": "@flystorage/multer-storage", "type": "module", "version": "1.2.0", "dependencies": { "@flystorage/file-storage": "^1.0.0", "@flystorage/stream-mime-type": "^1.0.0", "multer": "^2.1.1" }, "description": "", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "keywords": [ "s3", "file", "storage", "flystorage", "filesystem" ], "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/tools/multer/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/aws-s3" }, "license": "MIT", "devDependencies": { "body-parser": "^2.2.2", "form-data": "^4.0.5" } } ================================================ FILE: packages/multer-storage/src/index.test.ts ================================================ import express, {Response, Request} from 'express'; import {FlystorageMulterStorageEngine} from './index.js'; import {LocalStorageAdapter} from "@flystorage/local-fs"; import {FileStorage} from "@flystorage/file-storage"; import {resolve} from "node:path"; import FormData from "form-data"; import bodyParser from "body-parser"; import multer from 'multer'; import fetch from 'node-fetch'; import {IncomingMessage} from 'node:http'; describe('FlystorageMulterStorageEngine', () => { test('it can process uploaded files', async () => { const uploadStorage = new FileStorage( new LocalStorageAdapter( resolve(process.cwd(), 'fixtures/test-files'), ) ); const fixtureStorage = new FileStorage( new LocalStorageAdapter( resolve(process.cwd(), 'fixtures'), ) ); const app = express(); app.use(bodyParser.urlencoded({ extended: false })); const storage = new FlystorageMulterStorageEngine( uploadStorage, async (action, _req: express.Request, file: Express.Multer.File) => { if (action === 'handle') { return file.originalname; } else { return file.destination; } } ); const uploader = multer({storage}); app.post('/upload', uploader.single('image'), (_: Request, res: Response) => { res.status(200).json({ status: 'OK', }); }) const server = app.listen(5555); await new Promise(resolve => setTimeout(resolve, 100)); const formData = new FormData(); const stat = await fixtureStorage.statFile('screenshot.png'); formData.append('image', await fixtureStorage.read('screenshot.png'), { knownLength: stat.size, contentType: await fixtureStorage.mimeType('screenshot.png'), filename: 'screenshot-uploaded.png', }); const response = await new Promise((resolve, reject) => { formData.submit('http://localhost:5555/upload', (error, response) => { if (error) { reject(error); } else { resolve(response); } }) }); expect(response.statusCode).toEqual(200); server.close(); expect(await uploadStorage.fileExists('screenshot-uploaded.png')).toEqual(true); }) }) ================================================ FILE: packages/multer-storage/src/index.ts ================================================ import {StorageEngine} from 'multer'; import type {Request} from 'express'; import {FileStorage} from '@flystorage/file-storage'; export type DestinationResolver = (action: 'handle' | 'remove', req: Request, file: Express.Multer.File) => Promise; export class FlystorageMulterStorageEngine implements StorageEngine { constructor( private readonly storage: FileStorage, private readonly destinationResolver: DestinationResolver, ) { } _handleFile(req: Request, file: Express.Multer.File, callback: (error?: any, info?: Partial) => void): void { (async () => { const destination = await (this.destinationResolver)('handle', req, file); await this.storage.write(destination, file.stream, { size: file.size, mimeType: file.mimetype, }); return destination; })() .then(destination => callback(null, {destination})) .catch(error => callback(error)); } _removeFile(req: Request, file: Express.Multer.File, callback: (error: (Error | null)) => void): void { (async () => { const destination = await (this.destinationResolver)('remove', req, file); await this.storage.deleteFile(destination); })() .then(() => callback(null)) .catch(error => callback(error)); } } ================================================ FILE: packages/multer-storage/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packages/stream-mime-type/.npmignore ================================================ src !dist tsconfig.json ================================================ FILE: packages/stream-mime-type/CHANGELOG.md ================================================ # `@flystorage/stream-mime-type` ## 1.1.0 ### Minor Changes - Upgrade to file-type 20 ## 1.0.0 ### Major Changes - First 1.0.0 release ================================================ FILE: packages/stream-mime-type/LICENSE ================================================ Copyright (c) 2023-2024 Frank de Jonge Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: packages/stream-mime-type/package.json ================================================ { "name": "@flystorage/stream-mime-type", "type": "module", "version": "1.1.0", "dependencies": { "@flystorage/dynamic-import": "^1.0.0", "file-type": "^21.3.1", "mime-types": "^3.0.2" }, "description": "Get the mime-type of a readable stream, non-destructive", "main": "./dist/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { ".": { "import": { "types": "./dist/types/index.d.ts", "default": "./dist/esm/index.js" }, "require": { "types": "./dist/types/index.d.ts", "default": "./dist/cjs/index.js" } } }, "scripts": { "compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json", "patch": "gsed -i 's/dynamicallyImport(/import(/g' dist/esm/stream-mime-type.js", "compile:esm": "tsc --outDir ./dist/esm/ --declaration false", "compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node", "compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly", "watch": "tsc --watch" }, "author": "Frank de Jonge (https://frankdejonge.nl)", "homepage": "https://flystorage.dev/tools/stream-mime-type/", "repository": { "type": "git", "url": "git+https://github.com/duna-oss/flystorage.git", "directory": "packages/stream-mime-type" }, "license": "MIT" } ================================================ FILE: packages/stream-mime-type/src/index.ts ================================================ export * from './stream-mime-type.js'; ================================================ FILE: packages/stream-mime-type/src/stream-mime-type.test.ts ================================================ import * as fs from 'node:fs'; import path from 'node:path'; import {Readable} from 'stream'; import {resolveMimeType} from './stream-mime-type.js'; describe('resolveMimeType', () => { let stream: Readable; let mime: string | undefined; afterEach(() => { stream?.closed === false && stream.destroy(); mime = undefined; }); test('resolving the mime-type of a file', async () => { stream = fs.createReadStream(path.resolve(process.cwd(), 'fixtures/screenshot.png')); [mime, stream] = await resolveMimeType('screenshot.png', stream); expect(mime).toEqual('image/png'); stream.closed || stream.destroy(); }); test('when no mime-type can be resolved, use an extension based lookup', async () => { stream = Readable.from(Buffer.from('')); [mime, stream] = await resolveMimeType('screenshot.jpg', stream); expect(mime).toEqual('image/jpeg'); }); test('when no mime-type can be resolved, the fallback is used', async () => { stream = Readable.from(Buffer.from('')); [mime, stream] = await resolveMimeType('screenshot.unkown', stream, 'application/this-is-not-known'); expect(mime).toEqual('application/this-is-not-known'); stream.closed || stream.destroy(); }); }); ================================================ FILE: packages/stream-mime-type/src/stream-mime-type.ts ================================================ import {Readable} from 'node:stream'; import {parse} from 'node:path'; import {lookup as mimeTimeForExt} from 'mime-types'; import {PassThrough} from 'node:stream'; import {dynamicallyImport} from '@flystorage/dynamic-import'; function concatUint8Arrays(input: Uint8Array[]): Uint8Array { const length = input.reduce((l, a) => l + a.byteLength, 0); const output = new Uint8Array(length); let position = 0; input.forEach(i => { output.set(i, position); position += i.byteLength; }); return output; } export async function streamHead(stream: Readable, size: number): Promise<[Uint8Array, Readable]> { return new Promise((resolve, reject) => { const tunnel = new PassThrough(); const outputStream = new PassThrough(); let readBytes = 0; let buffers: Uint8Array[] = []; let resolved = false; tunnel.once('error', reject); tunnel.on('data', (chunk: Uint8Array) => { if (!resolved) { readBytes += chunk.byteLength; buffers.push(chunk); if (readBytes >= size) { resolved = true; const head = concatUint8Arrays(buffers); buffers = []; resolve([head, outputStream]); } } }); tunnel.once('end', () => { if (!resolved) { resolve([concatUint8Arrays(buffers), outputStream]); } }); stream.pipe(tunnel).pipe(outputStream); }); } type FileTypePackage = typeof import('file-type'); let fileTypeImport: Promise | undefined; let fileTypes: FileTypePackage | undefined = undefined; export async function resolveMimeType( filename: string, stream: Readable, fallback: string | undefined = undefined ): Promise<[string|undefined, Readable]> { const [head, readable] = await streamHead(stream, 4100); if (fileTypeImport === undefined) { fileTypeImport = dynamicallyImport('file-type'); } if (fileTypes === undefined) { fileTypes = await fileTypeImport; } const {fileTypeFromBuffer} = fileTypes; const lookup = await fileTypeFromBuffer(head); if (lookup) { return [lookup.mime, readable]; } const {ext} = parse(filename); return [mimeTimeForExt(ext) || fallback, readable]; } ================================================ FILE: packages/stream-mime-type/tsconfig.json ================================================ { "extends": "../../tsconfig.build.json", "compilerOptions": { "outDir": "./dist/" }, "include": [ "src/**/*.ts" ] } ================================================ FILE: packed/.gitignore ================================================ *.tgz */node_modules/ */package-lock.json */files/ ================================================ FILE: packed/cjs/index.js ================================================ const {FileStorage} = require('@flystorage/file-storage'); const {LocalStorageAdapter} = require('@flystorage/local-fs'); const {resolve} = require("path"); const storage = new FileStorage( new LocalStorageAdapter( resolve(__dirname, 'files'), ) ); (async () => { await storage.write('some/deep/file.txt', 'contents'); console.log(await storage.mimeType('path.svg')); const mimetype = await storage.mimeType('screenshot.png'); console.log(mimetype); console.log(await storage.list('', {deep: true}).toArray()); })(); ================================================ FILE: packed/cjs/index.test.js ================================================ const {FileStorage} = require('@flystorage/file-storage'); const {LocalStorageAdapter} = require('@flystorage/local-fs'); describe('running inside jest', () => { test('testing it', async () => { const fs = new FileStorage( new LocalStorageAdapter( `${__dirname}/files`, ), ); await fs.write('jest/file.txt', 'contents'); const listing = await fs.list('jest').toArray(); expect(listing.length).toBe(1); }) }) ================================================ FILE: packed/cjs/package.json ================================================ { "private": true, "type": "commonjs", "name": "packed-cjs", "version": "1.0.0", "description": "", "main": "index.js", "dependencies": { "@flystorage/file-storage": "file:~/Sites/filestorage/packed/flystorage-file-storage-0.1.7.tgz", "@flystorage/local-fs": "file:~/Sites/filestorage/packed/flystorage-local-fs-0.1.9.tgz", "@flystorage/stream-mime-type": "file:~/Sites/filestorage/packed/flystorage-stream-mime-type-0.1.6.tgz" }, "keywords": [], "author": "", "license": "ISC", "devDependencies": { "jest": "^30.0.0" } } ================================================ FILE: packed/esm/index.js ================================================ import {FileStorage} from '@flystorage/file-storage'; import {LocalStorageAdapter} from '@flystorage/local-fs'; import {join} from 'path'; const storage = new FileStorage( new LocalStorageAdapter( join(process.cwd(), 'files'), ) ); await storage.write('some/deep/file.txt', 'contents'); console.log(await storage.mimeType('path.svg')); const mimetype = await storage.mimeType('screenshot.png'); console.log(mimetype); console.log(await storage.list('', {deep: true}).toArray()); ================================================ FILE: packed/esm/package.json ================================================ { "private": true, "type": "module", "name": "packed-esm", "version": "1.0.0", "description": "", "main": "index.js", "dependencies": { "@flystorage/file-storage": "file:~/Sites/filestorage/packed/flystorage-file-storage-0.1.7.tgz", "@flystorage/local-fs": "file:~/Sites/filestorage/packed/flystorage-local-fs-0.1.10.tgz", "@flystorage/stream-mime-type": "file:~/Sites/filestorage/packed/flystorage-stream-mime-type-0.1.7.tgz" }, "keywords": [], "author": "", "license": "ISC" } ================================================ FILE: tsconfig.build.json ================================================ { "exclude": [ "**/*.test.ts", "node_modules", "dist" ], "compilerOptions": { "esModuleInterop": false, "declaration": true, "skipLibCheck": true, "lib": ["es2023"], "module": "node16", "target": "es2022", "strict": true, "forceConsistentCasingInFileNames": true, } } ================================================ FILE: tsconfig.json ================================================ { "exclude": [ "node_modules", "dist", "**/dist/**/*.js" ], "include": [ "./**/*.ts" ], "compilerOptions": { "declaration": true, "skipLibCheck": true, "lib": ["es2023"], "forceConsistentCasingInFileNames": true, "target": "ES2020", "module": "NodeNext", "strict": true, "moduleResolution": "NodeNext", "esModuleInterop": true, "allowSyntheticDefaultImports": true, "types": [ "node", "vitest/globals" ], "paths": { "@flystorage/*": ["./packages/*/src/index.ts"] } }, } ================================================ FILE: vitest.config.ts ================================================ import { defineConfig } from 'vitest/config'; import tsconfigPaths from 'vite-tsconfig-paths'; import path from 'node:path'; export default defineConfig({ plugins: [tsconfigPaths()], resolve: { alias: [ { find: /^@flystorage\/(.*)$/, replacement: path.join(new URL(import.meta.url).pathname, '../packages', '$1', 'src/index.ts'), } ] }, test: { testTimeout: 10_000, include: ['packages/**/*.{test,spec}.?(c|m)[jt]s?(x)'], globals: true, clearMocks: false, setupFiles: ['dotenv/config'], // profiling // pool: 'forks', // poolOptions: { // forks: { // execArgv: [ // '--cpu-prof', // '--cpu-prof-dir=test-runner-profile', // // '--heap-prof', // // '--heap-prof-dir=test-runner-profile' // ], // // // To generate a single profile // singleFork: true, // }, // }, }, });