Showing preview only (288K chars total). Download the full file or copy to clipboard to get everything.
Repository: duna-oss/flystorage
Branch: main
Commit: 29a09a1e4d0e
Files: 128
Total size: 258.2 KB
Directory structure:
gitextract_ft0asde_/
├── .github/
│ ├── actions/
│ │ └── install-dependencies/
│ │ └── action.yml
│ ├── dependabot.yml
│ └── workflows/
│ ├── automerge-dependabot.yml
│ ├── compile.yml
│ ├── qa-ubuntu-bun.yml
│ ├── qa-ubuntu-node.yml
│ ├── qa-windows-node.yml
│ └── shared-tests.yml
├── .gitignore
├── bin/
│ └── watch.ts
├── docker/
│ └── sftp/
│ ├── id_rsa
│ ├── id_rsa.pub
│ ├── ssh_host_ed25519_key
│ ├── ssh_host_ed25519_key.pub
│ ├── ssh_host_rsa_key
│ ├── ssh_host_rsa_key.pub
│ ├── sshd_custom_configs.sh
│ ├── unknown.key
│ └── users.conf
├── docker-compose.yml
├── fixtures/
│ └── adapter.template.ts
├── package.json
├── packages/
│ ├── aws-s3/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── aws-s3-file-storage.test.ts
│ │ │ ├── aws-s3-storage-adapter.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ ├── azure-storage-blob/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── azure-storage-blob.test.ts
│ │ │ ├── azure-storage-blob.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ ├── chaos/
│ │ ├── .npmignore
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── changelog.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── index.test.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ ├── dynamic-import/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── README.md
│ │ ├── index.d.ts
│ │ ├── index.js
│ │ ├── index.test.ts
│ │ ├── package.json
│ │ └── src/
│ │ └── index.ts
│ ├── file-storage/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── checksum-from-stream.ts
│ │ │ ├── errors.ts
│ │ │ ├── file-storage.test.ts
│ │ │ ├── file-storage.ts
│ │ │ ├── index.ts
│ │ │ ├── path-normalizer.test.ts
│ │ │ ├── path-normalizer.ts
│ │ │ ├── path-prefixer.test.ts
│ │ │ ├── path-prefixer.ts
│ │ │ ├── portable-visibility.ts
│ │ │ ├── readable-convertion.test.ts
│ │ │ └── utilities.test.ts
│ │ └── tsconfig.json
│ ├── google-cloud-storage/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── google-cloud-storage.test.ts
│ │ │ ├── google-cloud-storage.ts
│ │ │ ├── index.ts
│ │ │ └── visibility-handling.ts
│ │ └── tsconfig.json
│ ├── in-memory/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── in-memory-file-storage.test.ts
│ │ │ ├── in-memory-file-storage.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ ├── local-fs/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── index.ts
│ │ │ ├── local-file-storage.test.ts
│ │ │ ├── local-file-storage.ts
│ │ │ └── unix-visibility.ts
│ │ └── tsconfig.json
│ ├── multer-storage/
│ │ ├── .npmignore
│ │ ├── CHANGELOG.md
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── index.test.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ └── stream-mime-type/
│ ├── .npmignore
│ ├── CHANGELOG.md
│ ├── LICENSE
│ ├── package.json
│ ├── src/
│ │ ├── index.ts
│ │ ├── stream-mime-type.test.ts
│ │ └── stream-mime-type.ts
│ └── tsconfig.json
├── packed/
│ ├── .gitignore
│ ├── cjs/
│ │ ├── index.js
│ │ ├── index.test.js
│ │ └── package.json
│ └── esm/
│ ├── index.js
│ └── package.json
├── tsconfig.build.json
├── tsconfig.json
└── vitest.config.ts
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/actions/install-dependencies/action.yml
================================================
name: 'Install NPM Dependencies'
description: 'Install NPM Dependencies'
inputs:
cache-key:
description: 'Cache key'
required: true
cache-restore-keys:
description: 'Cache restore keys'
required: true
outputs:
cache-hit:
description: 'Whether or not the cache was hit'
value: ${{ steps.npm-cache.outputs.cache-hit }}
runs:
using: 'composite'
steps:
- run: mkdir ~/.npm-cache
shell: sh
name: Create NPM cache directory
- name: Load NPM cache
id: npm-cache
uses: actions/cache/restore@v3
with:
path: |
~/.npm-cache
node_modules
key: ${{ inputs.cache-key }}
restore-keys: ${{ inputs.cache-restore-keys }}
- name: Install NPM dependencies
if: steps.npm-cache.outputs.cache-hit != 'true'
run: npm ci --prefer-offline --no-audit
shell: sh
- name: Save NPM cache
if: steps.npm-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v3
with:
key: ${{ inputs.cache-key }}
path: |
~/.npm-cache
node_modules
================================================
FILE: .github/dependabot.yml
================================================
version: 2
updates:
- package-ecosystem: "npm"
directory: "/"
schedule:
interval: "weekly"
groups:
dependencies:
patterns:
- "*"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
================================================
FILE: .github/workflows/automerge-dependabot.yml
================================================
name: Auto-merge Dependabot PRs
on:
pull_request_target:
paths:
- package.json
- package-lock.json
branches:
- main
permissions:
pull-requests: write
contents: write
jobs:
merge-dependabot-pr:
name: Merge dependabot PR
runs-on: ubuntu-latest
if: ${{ github.actor == 'dependabot[bot]' }}
steps:
- name: Fetch Dependabot metadata
id: meta
uses: dependabot/fetch-metadata@v2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Merge PR
if: ${{ steps.meta.outputs.update-type != 'version-update:semver-major' }}
run: gh pr merge --auto --squash "$PR_URL"
env:
PR_URL: ${{ github.event.pull_request.html_url }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .github/workflows/compile.yml
================================================
name: Compile libraries
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
run-tests:
runs-on: ${{ matrix.platform }}
strategy:
matrix:
platform: [ubuntu-latest]
node: [22] # latest LTS version
name: Compile libraries
steps:
- name: Checkout the repository
uses: actions/checkout@v6
- name: Install Node.js
uses: actions/setup-node@v6
with:
node-version: ${{ matrix.node }}
- uses: ./.github/actions/install-dependencies
name: Install NPM dependencies
with:
cache-key: npm-${{ matrix.node }}-${{ hashFiles('package-lock.json') }}
cache-restore-keys: npm-caches-${{ matrix.node }}
- name: Run build
run: npm run build:only
================================================
FILE: .github/workflows/qa-ubuntu-bun.yml
================================================
name: Ubuntu + Bun
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-ubuntu-bun
cancel-in-progress: true
jobs:
tests:
uses: ./.github/workflows/shared-tests.yml
secrets: inherit
with:
os: ubuntu-latest
runtime: bun
# as string for inputs workaround not accepting arrays
versions: '["1.2"]'
experimental: true
================================================
FILE: .github/workflows/qa-ubuntu-node.yml
================================================
name: Ubuntu + Node
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-ubuntu-node
cancel-in-progress: true
jobs:
tests:
uses: ./.github/workflows/shared-tests.yml
secrets: inherit
with:
os: ubuntu-latest
runtime: node
# as string for inputs workaround not accepting arrays
versions: '[20, 21, 22, 23, 24]'
================================================
FILE: .github/workflows/qa-windows-node.yml
================================================
name: Ubuntu + Windows
on:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-windows-node
cancel-in-progress: true
jobs:
tests:
uses: ./.github/workflows/shared-tests.yml
secrets: inherit
with:
experimental: true
os: windows-latest
runtime: node
# as string for inputs workaround not accepting arrays
versions: '[20, 21, 22, 23, 24]'
================================================
FILE: .github/workflows/shared-tests.yml
================================================
name: Tests
on:
workflow_call:
secrets:
GCS_KEY_CONTENTS:
required: true
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
AWS_REGION:
required: true
AZURE_DSN:
required: true
inputs:
os:
required: true
type: string
runtime:
required: true
type: string
versions:
required: true
type: string
experimental:
required: false
type: boolean
default: false
#permissions:
# id-token: write # This is required for requesting the JWT
# contents: read # This is required for actions/checkout
# pull-requests: read # Required for change detection
jobs:
run-tests:
runs-on: ${{ inputs.os }}
continue-on-error: ${{ inputs.experimental }}
strategy:
matrix:
version: ${{ fromJSON(inputs.versions) }}
name: ${{ inputs.os }} - ${{ inputs.runtime }} - ${{ matrix.version }}
steps:
- name: Checkout the repository
uses: actions/checkout@v6
- name: Install Node.js
if: inputs.runtime == 'node'
uses: actions/setup-node@v6
with:
node-version: ${{ matrix.version }}
- uses: ./.github/actions/install-dependencies
if: inputs.runtime == 'node'
name: Install NPM dependencies
with:
cache-key: ${{inputs.os}}-${{inputs.runtime}}-npm-${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}
cache-restore-keys: ${{inputs.os}}-${{inputs.runtime}}-npm-${{ matrix.node }}
- uses: oven-sh/setup-bun@v2
if: inputs.runtime == 'bun'
with:
bun-version: ${{ inputs.version }}
- name: Install NPM dependencies using Bun
if: inputs.runtime == 'bun'
run: bun install
- name: Configure GCP access
run: |
echo $CONTENTS > google-cloud-service-account.json
env:
CONTENTS: "${{ secrets.GCS_KEY_CONTENTS }}"
- name: Run relevant tests
run: "npm run test"
if: inputs.runtime == 'node'
env:
AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
AWS_REGION: "${{ secrets.AWS_REGION }}"
AZURE_DSN: "${{ secrets.AZURE_DSN }}"
- name: Run relevant tests using Bun
run: |
npm run build:only
bun test packages
if: inputs.runtime == 'bun'
env:
AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
AWS_REGION: "${{ secrets.AWS_REGION }}"
AZURE_DSN: "${{ secrets.AZURE_DSN }}"
- name: Delete GCP access
if: always()
run: |
node_modules/.bin/rimraf google-cloud-service-account.json
================================================
FILE: .gitignore
================================================
google-cloud-service-account.json
dist
.env
**/*/dist
node_modules
bin/*
!bin/watch.ts
fixtures/test-files/
.node-version
================================================
FILE: bin/watch.ts
================================================
import {readdir} from 'node:fs/promises';
import path from 'node:path';
import {concurrently} from 'concurrently';
const directories = (await readdir(path.resolve(process.cwd(), 'packages'), {
withFileTypes: true,
}))
.filter(item => item.isDirectory())
.map(dir => path.parse(dir.name).name);
const commands = directories.map(directory => ({
name: directory,
command: `npm run watch -w ./packages/${directory} --if-present`
}));
console.log('starting', commands);
concurrently(commands);
================================================
FILE: docker/sftp/id_rsa
================================================
-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA0cx7jSu+FZQJFmz0+vlbo1I+awlaxvqZUseuPvX/YA32c3hE
nr0OpEMhe/Pvs4jk/BiPp5CfITFe6ykxUvNlvxh/9EOsnUFaFrfyLhYn3TicW6lv
zSJmJ7tydW/L0zpMPliItbjtXItxHOyoVcl20+DT+LZWlRRnnuSDTC1Oc6vvLosw
b4lOTNVB2kV7W9urU6W+OilcERwjjQpiIM4mkwg2dX0aPWS5o0N9tYi3+k6gStqj
DfcTw3L8tTWeDcWrblvFaknFfBzkeXx9dTyrAYw3ii3LZQ6hA5UK8tUgwUVThTh+
QBhzhmn14Ljfp1fa3vgljY9yn5Zw+Drhbtn4TwIDAQABAoIBACaZstnEhJK/y/Q+
U8yheITSKv3SmMsnbHJYnuyiojvwFbolFKsIKdt7JnwB48Zql4bylevEpiKbTNWD
nLmgYsYIIfK1SNseHQ81BPAJz4faVJpg0FszywvgZyzIRv40KbcG3xBgV/vBBCzI
NiiiiqRtJ1MJaWDAglgvvyCS7W5GjHue74o/wPugyvIi7zmMqMmZLg3zz/YF3zxU
h3JGFnw/TlN3yDJzdAOFiLGpSRD/bMcBJyWc9Oa2LQj+Emkc7dojmB/IJ7wJ9iuo
Ni2/EJIQoFtVYxK6r+nRVuvT6/siMi7Ok4EL7PANVNVA7s7e+rmZX3NjDiGnvh5V
DyxdT5ECgYEA/UFm3qxLiSQwNGuQLpCO9WR+4eeZPthNI1ySkcAnGyrmxX70ws7M
5j1jOIYUhf8i1ALUu++sBfvd+OhD0yeRgFvqrCdeHdFnAdgOtQDUGfU9bh1YpEX0
4A7zIxK7TBi26hXb/fTu7YwDtv58W1BtOOfo9ZKOWPu1ZQbEG5fp9pMCgYEA1BKF
MRWshqBOSqRhyrl34wXTr6EI1XcwIDNyAGJnQFlbxWcCuPYCkF+ENZOe8D6soXiZ
tTW6zSkM+UmsNgEYjD5b0eWgd0yUdrOi/Z4JZ16SlroeLYfbqVYY6NPE6XEuzZ9p
XEB5WedUGIdU2JZQ422m7BxpJBIRtlbhG1Oi8NUCgYEA9Y3MaGs2ciqccrc4fW28
r0JZpEAi3kRrxrWjh56ATF80kpmeSKSrFzK+WbfnfmT7KAX2rqKccNDdUNIjsUDU
W1jEGVeyccbv0WHkIKxE+0ZF4daic+VAoV7dcExhPk9YS3AWdg5e/ASeNXhaq084
F80Em9cWHkEwiFwfGYIaX/ECgYEAnUqfPyi0LaX4a6RAY/vrz5Yiy8DErI8aQsfl
ZiOWMUQVrPQaMNVGUY6GoLY8zDOwFpM8bgrL4h7wYHUkJWnqqxoVQDjwK4vBEclq
unDcyK58Sw8AEwURBye0kft/sSUhcaEqpCGt3+CTnx3A8GOM2yIZDEaGNRqxyGvn
yjzePYECgYBqr0OQJUDUOGcL+V5z8wCaLN2bbWK/bukaGe3ZU7W2kWhNvVL0SBNC
dUcxLp5ELOj8wFHmTvZv1q2aszIY2CQhnc1qGYCh34UAK14DkTvSYie9XcGPhkWK
UC9VLorkPbPAZipbBs9Dt079ub3HRyeXX4b0VHHKzu2XVGXVEOXO6A==
-----END RSA PRIVATE KEY-----
================================================
FILE: docker/sftp/id_rsa.pub
================================================
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRzHuNK74VlAkWbPT6+VujUj5rCVrG+plSx64+9f9gDfZzeESevQ6kQyF78++ziOT8GI+nkJ8hMV7rKTFS82W/GH/0Q6ydQVoWt/IuFifdOJxbqW/NImYnu3J1b8vTOkw+WIi1uO1ci3Ec7KhVyXbT4NP4tlaVFGee5INMLU5zq+8uizBviU5M1UHaRXtb26tTpb46KVwRHCONCmIgziaTCDZ1fRo9ZLmjQ321iLf6TqBK2qMN9xPDcvy1NZ4NxatuW8VqScV8HOR5fH11PKsBjDeKLctlDqEDlQry1SDBRVOFOH5AGHOGafXguN+nV9re+CWNj3KflnD4OuFu2fhP your_email@example.com
================================================
FILE: docker/sftp/ssh_host_ed25519_key
================================================
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACCPTP3g9O8X9Ir5KwrSVkYRqgCZFFt4i7oHBKyy31HiCQAAALBMCYEjTAmB
IwAAAAtzc2gtZWQyNTUxOQAAACCPTP3g9O8X9Ir5KwrSVkYRqgCZFFt4i7oHBKyy31HiCQ
AAAEAsRUm7efqtLmLIUwhaHzPnKf+hUubVc+xr49XcILRW5Y9M/eD07xf0ivkrCtJWRhGq
AJkUW3iLugcErLLfUeIJAAAAKmZyYW5rZGVqb25nZUBGcmFuay1kZS1Kb25nZXMtTUJQMj
AxNS5sb2NhbAECAw==
-----END OPENSSH PRIVATE KEY-----
================================================
FILE: docker/sftp/ssh_host_ed25519_key.pub
================================================
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII9M/eD07xf0ivkrCtJWRhGqAJkUW3iLugcErLLfUeIJ frankdejonge@Frank-de-Jonges-MBP2015.local
================================================
FILE: docker/sftp/ssh_host_rsa_key
================================================
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
NhAAAAAwEAAQAAAgEA48TBtykiBjI65ZBWCpOHk86EZrIxBFxb5WpL0ujBSrp05UJ8eKeX
moEefhR4Vqll9Oi+ejmKXUvvkH1M638N35SFq+S3HR8TsMvgn5MaWO5HH3F5zzzLzyZjtZ
H/K6mroMhG1b/DZoG3s4A5jN1i2ByRJS9d65q+Txd9ItlEifiCs7YYDS/5CkMWp3I6+i64
CmARpMUvfVwpclFFIaxnlevymRIwSxzUoZxGuLYR2+QUx/4sbx41CIOLwEWrxf0oq9+DS3
eyp+av4itLo47ndLfNEFexUHSPE8IofGuKAIUkHrPMVwLoFT+FbIDVH6vnU9XawIpbHFev
ju2Mopb3DS9CKLLziQhBSxwCgsJmFyTqzkPZ3YeSsaz22o+vW98HRnp6TTtv+iefArNRPr
gikBdFEjK8jWwZWV6MyeU5u4fZngMghV3DOfBjTlhrONI50iicM1sNKEvUqzEqCL/fs+zX
V04aeDyNw+O6Xy1uIT+2Ozo5bHMoeyqqM33Zb7CB+polZQckFrXZAS59KLRYy2EpNQVzNY
UThywQDPk5fkJ1CczHwlAzLlNZdxnsG69+JYGWlqKu//dT4AJPuYNDVW7SxPec8+yYq7uu
DZnsnp8in/0tiyr4AS4fnAOszIndqRwwzqm13BVIDNJbdwgJTRDC+3wBT6hjnPC0ktACtN
sAAAdo+KBGYPigRmAAAAAHc3NoLXJzYQAAAgEA48TBtykiBjI65ZBWCpOHk86EZrIxBFxb
5WpL0ujBSrp05UJ8eKeXmoEefhR4Vqll9Oi+ejmKXUvvkH1M638N35SFq+S3HR8TsMvgn5
MaWO5HH3F5zzzLzyZjtZH/K6mroMhG1b/DZoG3s4A5jN1i2ByRJS9d65q+Txd9ItlEifiC
s7YYDS/5CkMWp3I6+i64CmARpMUvfVwpclFFIaxnlevymRIwSxzUoZxGuLYR2+QUx/4sbx
41CIOLwEWrxf0oq9+DS3eyp+av4itLo47ndLfNEFexUHSPE8IofGuKAIUkHrPMVwLoFT+F
bIDVH6vnU9XawIpbHFevju2Mopb3DS9CKLLziQhBSxwCgsJmFyTqzkPZ3YeSsaz22o+vW9
8HRnp6TTtv+iefArNRPrgikBdFEjK8jWwZWV6MyeU5u4fZngMghV3DOfBjTlhrONI50iic
M1sNKEvUqzEqCL/fs+zXV04aeDyNw+O6Xy1uIT+2Ozo5bHMoeyqqM33Zb7CB+polZQckFr
XZAS59KLRYy2EpNQVzNYUThywQDPk5fkJ1CczHwlAzLlNZdxnsG69+JYGWlqKu//dT4AJP
uYNDVW7SxPec8+yYq7uuDZnsnp8in/0tiyr4AS4fnAOszIndqRwwzqm13BVIDNJbdwgJTR
DC+3wBT6hjnPC0ktACtNsAAAADAQABAAACAQCESra9HK3/bVNaHNBsyi2YAv5R67OetcpG
YMvzj28daVkWA9zp82WRvucoEdmndDKc4kYoFZ2w/LcDdFOmAKDdOJW/NlPJHVDBgllQNg
+6kYNL1wwJ+2ThR4noXwkXoi/mbgz+f6gNtNAu+Q30LG4J2eXP9EgX3UQmCh2LjShK/sVj
fiNQHYoHlNnmnel1gIcyt4Pn8QPZSxtjo6KEoW9025uHntHf/rnduDg3dsC+uCX91zqVu7
TP4h/cqFrR322tDmBjB/4DmXCU69K+B/WVjGAV2ulJMrobns0HHysDjFFjZ8kKzMxh8wga
8mVXRPBSeEbbSEENIDz+xijGEusga3EK3I4LMNaAN47oulrVM++oF1pHL74jiEAC1SGHTv
0ZOZdCDmdmPyxCC9ruvJmQb3IbvoVyROcrHd4AvYSKnRqE2ljzLnFHG+Qp13meoXNz8Bup
NO9ra+HJYWu6QgqHnPvwrIP5NdmwwSRWunglOvVSO2c11X7Li/NBkLLLk9wX1w9awAPfTf
nUGCFnKi+1ianXD94zEG0FyXImN1eRK2lgs8ul6wGr3qV5B3sv7llgyWF5ZKLLj9iQ79M5
OKret5evybFbEbIOWbRTTcREfmSLqcKv2g6ZJpBNI8z4bz0jgc4hEVcMJlyiH52v3yMePh
f7tXvv62Ws9IHLNhnSkQAAAQEApXAZ5Ns3aUOuCVvK3mEUWuQEsh/aM+rdrthX+zrw7Ma7
kw2O6zh6VviP5BV+8qG7sx1KyK9b0u+/qGDy4pMHN8SjNq1W+pTZ4FHkA/1UKI0kTvTwq/
WQvO1pBxLbx77gl5FuBjysPljnvMh3+l9QOFX4iwjFoyep1FgammuD4hnSR1yWuF3H/2xc
sb75LnXdaEDm5ALyauTwmzdufTCaQmq19y8hZ5Kbhxd/Pao160tm38n2gektN458AbGxri
8tHSwpj4+3PSOwFeFalxk/jO8daye4PluoqK+O60Z5NWqdDDaBdvxEMcj9sddUQaQ0BzmK
Fci3Hl6vS9TzBUWNOQAAAQEA/hDQkIH3ZIdZYBWIsinTKoztovUZHfHZihkBxcR0IbKZkn
7HGgtNtv1QshSvpzMp7HsAT45SqzIbN4ldNh0aSRmoOb2/pVNVNeJrWR0B+aS9hya6bAOY
AiZlfMMo+2cU+gJzGD50tlu7ge4hT3YkN3vmoDyJZ2p7DxOioQQ10D5Pno8ZGh8kZX3t5d
F+5GwPnsYXsi/iMgw18ggPTBPwAP+TeUsdP8Ae93h33LgCOkUfXeq/HmQ0jTUT84lxd9tW
2N/w6P/2SYa85oFVa2Y+u0qRJOLf6zJKTOH/ERV9IJYGLRzyxybQuo4iqw9V+kP13KkC2w
6vh2RcmEwXmxXRyQAAAQEA5YCwEjBqYLlPzQ3lOf75SbaT5+CIiyNy0P0ZUvpBl+URJkGC
m8/AA3naWylEP/oUJRUkv21qXB8EEL62404nhNVtfDGX1mpht8H4N9lcXAdNFg1SWmqnqb
58PTODzM4n9CVjqnqqlTGQi1ph77RLDVXc29UooopLtYFsttiFBvv8Cs/cKx3t1ECQRklU
hjPSdijeRTtv5x604uXt6mObYIGMAC87OSP4iaY00Dz17lnJ9m9UK5BCX+44sIEf3IQ9Ad
jfaeN5UoUTv+y/fB419baC+dlIR9+SaEFqNgMHQqpyZApWMhCf99oNQhJ9XDjVEtKtXnhc
5Lh4tYyNGMsDgwAAACpmcmFua2Rlam9uZ2VARnJhbmstZGUtSm9uZ2VzLU1CUDIwMTUubG
9jYWwBAgMEBQYH
-----END OPENSSH PRIVATE KEY-----
================================================
FILE: docker/sftp/ssh_host_rsa_key.pub
================================================
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDjxMG3KSIGMjrlkFYKk4eTzoRmsjEEXFvlakvS6MFKunTlQnx4p5eagR5+FHhWqWX06L56OYpdS++QfUzrfw3flIWr5LcdHxOwy+CfkxpY7kcfcXnPPMvPJmO1kf8rqaugyEbVv8NmgbezgDmM3WLYHJElL13rmr5PF30i2USJ+IKzthgNL/kKQxancjr6LrgKYBGkxS99XClyUUUhrGeV6/KZEjBLHNShnEa4thHb5BTH/ixvHjUIg4vARavF/Sir34NLd7Kn5q/iK0ujjud0t80QV7FQdI8Twih8a4oAhSQes8xXAugVP4VsgNUfq+dT1drAilscV6+O7YyilvcNL0IosvOJCEFLHAKCwmYXJOrOQ9ndh5KxrPbaj69b3wdGenpNO2/6J58Cs1E+uCKQF0USMryNbBlZXozJ5Tm7h9meAyCFXcM58GNOWGs40jnSKJwzWw0oS9SrMSoIv9+z7NdXThp4PI3D47pfLW4hP7Y7Ojlscyh7KqozfdlvsIH6miVlByQWtdkBLn0otFjLYSk1BXM1hROHLBAM+Tl+QnUJzMfCUDMuU1l3Gewbr34lgZaWoq7/91PgAk+5g0NVbtLE95zz7Jiru64NmeyenyKf/S2LKvgBLh+cA6zMid2pHDDOqbXcFUgM0lt3CAlNEML7fAFPqGOc8LSS0AK02w== frankdejonge@Frank-de-Jonges-MBP2015.local
================================================
FILE: docker/sftp/sshd_custom_configs.sh
================================================
#!/bin/bash
cat <<'EOF' >> /etc/ssh/sshd_config
KexAlgorithms curve25519-sha256
Ciphers aes256-gcm@openssh.com
MACs hmac-sha2-256-etm@openssh.com
HostKeyAlgorithms ssh-ed25519
EOF
================================================
FILE: docker/sftp/unknown.key
================================================
-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEAxZQy1FnuqhoZsiqvFanN5tZRL6GVeCBeoLroGBEggJYVDLw2
nPaXCDOaDoVzs1PkLoKhttA6g4l4eHZgDL1b6QTxRnDbkdOxKJHGEapk+WZk23yq
QmU4thDdmgj0clJfS3aTbF7hGWpxrv9zB5saWwgWB/fBeYXu+kXuuO1UJF+F0QGt
6TVVMc6o3NaI7Cq1qoMvt7XGN5LegwfNWRGxaKTL1yn7pkljhkPLOoGedBemieya
aQhV+dwzOVeBAXGKSmtiUwto8OMYl/suptMdo+91cLMRJkgXNQ5gA+4niqja5HGW
2Dw8vXja2Spp4+gWWh5OpXwd2pNnJScNINL30wIDAQABAoIBAHzBD7s/sdAcPN9f
zj+qgUVhS8/8gilgnv90JPqVTeWDXnU1HnLLzR+znXHP1/eCYBDyEPQi1N+bXMML
U6iXpEIlCcfFmQ6iETmhmeQrqChF/CcOt17HFSD400PgpaDN3DgE/h8uZYmryW6L
A3HpAKI8H9UWHkcCR5wlrg98Y2W3AacHMwRG8da7QxLcGd10+1lFLuPXTu7LwDf0
u/+Jq9IPwm8WemGiuuOEQ06r7y1s929pA7fWSsn8DlVBIFnPHCtGKpiDfAyEHmCh
/LUU6Z5Cl9186sGohu5hWUA+qTMAHw9IR0TbWxF+tAEX8qL5z/qH/surRvzfeRCi
BooQB4ECgYEA99xz7AWMUzu6J4FSm29M2AS989Yz0XAgGIYs69pxoaRR5azRffY+
F6wjBpeSI5BElu1DAublhv5hpCgdcw6X2EfMnKvP1jtSbv+DOdR0p80LVAnG7tb3
stSgNwMsGdBpTCghHKUvOUGDU3D3c9O8jCrrJd9Unu4gj2/xbVuC45MCgYEAzBER
XeuKuwrN0HmLCZ7AJ1aq7A60UKJjMicdXD5n95ePwHQ32s+hmrcGfUOjcZoWc1Dp
yZaZvqFDYu52ZJ1QbNZqjBQJ8sjERSxNlV8DZe1I3ohXvsRQ4I6g8/HRgSl0CVqg
+LeJA7n18+SrhhCWhY+dggV1m4qCNcWI7FCQwsECgYEAw+V7xT35U0twbIq8lFba
QB03WEGiwNRCub9KP7ptdtjdVY5KIKj/GEyXfj1LZko+u56YCPIe1Ju25jxCUk5l
Wq4cnHL6mBJYq5vMxmcRMBJR8sCrdtd1++QrIG+kal6a6nMJAI/ZjAIoXkl5ehUN
/yZopY0mX1pLZ7KM+OaLw3sCgYEAvaACllbA9Gvmspmu5IKbJjL34yEK137+VGVa
eBQZgk5ZK0oTeQXFssHuisomf/Lid8exZzzFowmxV6YlZ/ty96ALJB2e3PdIwsqX
UX0X6EgllXv2pXNBgFmpIOYNe0ts4yBPQq8x57+O2FMePBb/+B5rC55NGfsMYjEr
ugRncEECgYB9Wrnp4/WpTwcSJTJXWB+FJaMmcXqbFLk/AI1HbRXJFmR/ssK1gugQ
Y3vevvRMSJkYkd1Zmbp9x8ZAGKzR/yeT8lQ6gY/xYq1fYLKKKtzHdwTMCVIVDgSf
/v9Y8mLylcgr7ZjEo8xMTnb98ozSSFPBn2jq8c31AsQxXaIMK0BE7g==
-----END RSA PRIVATE KEY-----
================================================
FILE: docker/sftp/users.conf
================================================
foo:pass:1001:100:upload
bar:pass:1001:100:upload
================================================
FILE: docker-compose.yml
================================================
---
version: "3"
services:
# sabredav:
# image: php:8.1-alpine3.15
# restart: always
# volumes:
# - ./:/var/www/html/
# ports:
# - "4040:4040"
# command: php -S 0.0.0.0:4040 /var/www/html/src/WebDAV/resources/server.php
# webdav:
# image: bytemark/webdav
# restart: always
# ports:
# - "4080:80"
# environment:
# AUTH_TYPE: Digest
# USERNAME: alice
# PASSWORD: secret1234
# ANONYMOUS_METHODS: 'GET,OPTIONS'
sftp:
container_name: sftp
restart: always
image: atmoz/sftp:alpine
volumes:
- ./docker/sftp/users.conf:/etc/sftp/users.conf
- ./docker/sftp/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key
- ./docker/sftp/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key
- ./docker/sftp/id_rsa.pub:/home/bar/.ssh/keys/id_rsa.pub
ports:
- "2222:22"
sftp_eddsa_only:
container_name: sftp_eddsa_only
restart: always
image: atmoz/sftp:alpine
volumes:
- ./docker/sftp/users.conf:/etc/sftp/users.conf
- ./docker/sftp/sshd_custom_configs.sh:/etc/sftp.d/sshd_custom_configs.sh
- ./docker/sftp/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key
ports:
- "2223:22"
# ftp:
# container_name: ftp
# restart: always
# image: delfer/alpine-ftp-server
# environment:
# USERS: 'foo|pass|/home/foo/upload'
# ADDRESS: 'localhost'
# ports:
# - "2121:21"
# - "21000-21010:21000-21010"
azurite:
image: mcr.microsoft.com/azure-storage/azurite
profiles: ["azure"]
restart: unless-stopped
healthcheck:
test:
wget --no-verbose --tries=1 --spider http://localhost:10000/ || exit 1
interval: 10s
timeout: 10s
retries: 3
start_period: 10s
ports:
- 10000:10000
- 10001:10001
- 10002:10002
================================================
FILE: fixtures/adapter.template.ts
================================================
import {Readable} from "stream";
import {
StorageAdapter,
ChecksumOptions,
CreateDirectoryOptions,
FileContents, MimeTypeOptions,
PublicUrlOptions,
StatEntry, TemporaryUrlOptions,
WriteOptions,
PathPrefixer,
CopyFileOptions,
MoveFileOptions,
} from "@flystorage/file-storage";
export type AdapterFileStorageOptions = {
prefix?: string,
}
export class AdapterFileStorage implements StorageAdapter {
private readonly prefixer: PathPrefixer;
constructor(
private readonly options: AdapterFileStorageOptions = {},
) {
this.prefixer = new PathPrefixer(options.prefix || '');
}
async write(path: string, contents: Readable, options: WriteOptions): Promise<void> {
throw new Error('Not implemented');
}
async read(path: string): Promise<FileContents> {
throw new Error('Not implemented');
}
async deleteFile(path: string): Promise<void> {
throw new Error('Not implemented');
}
async createDirectory(path: string, options: CreateDirectoryOptions): Promise<void> {
throw new Error('Not implemented');
}
async stat(path: string): Promise<StatEntry> {
throw new Error('Not implemented');
}
list(path: string, options: {deep: boolean}): AsyncGenerator<StatEntry> {
throw new Error('Not implemented');
}
async changeVisibility(path: string, visibility: string): Promise<void> {
throw new Error('Not implemented');
}
async visibility(path: string): Promise<string> {
throw new Error('Not implemented');
}
async deleteDirectory(path: string): Promise<void> {
throw new Error('Not implemented');
}
async fileExists(path: string): Promise<boolean> {
throw new Error('Not implemented');
}
async directoryExists(path: string): Promise<boolean> {
throw new Error('Not implemented');
}
async publicUrl(path: string, options: PublicUrlOptions): Promise<string> {
throw new Error('Not implemented');
}
async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string> {
throw new Error('Not implemented');
}
async checksum(path: string, options: ChecksumOptions): Promise<string> {
throw new Error('Not implemented');
}
async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
throw new Error('Not implemented');
}
async lastModified(path: string): Promise<number> {
throw new Error('Not implemented');
}
async fileSize(path: string): Promise<number> {
throw new Error('Not implemented');
}
async copyFile(from: string, to: string, options: CopyFileOptions): Promise<void> {
throw new Error('Not implemented');
}
async moveFile(from: string, to: string, options: MoveFileOptions): Promise<void> {
throw new Error('Not implemented');
}
}
================================================
FILE: package.json
================================================
{
"private": true,
"type": "module",
"scripts": {
"build": "npm run compile -ws --if-present && npm run patch -ws --if-present",
"build:only": "npm run compile -ws --if-present",
"clean:build": "npm run clean && npm run build",
"watch": "concurrently npm:watch:*",
"watch:file-storage": "npm run watch -w ./packages/file-storage",
"watch:stream-mime-type": "npm run watch -w ./packages/stream-mime-type",
"watch:local": "npm run watch -w ./packages/local-fs",
"watch:aws-s3": "npm run watch -w ./packages/aws-s3",
"clean": "rm -rf ./packages/*/dist/",
"ts": "node --import tsx/esm",
"lint": "tsc --noEmit --incremental false",
"test": "npm run lint && npm run vitest",
"vitest": "vitest run"
},
"workspaces": [
"./packages/stream-mime-type",
"./packages/file-storage",
"./packages/*"
],
"devDependencies": {
"@swc/core": "^1.15.18",
"@types/express": "^5.0.6",
"@types/mime-types": "^3.0.1",
"@types/multer": "^2.1.0",
"@types/node": "^25.3.5",
"@vitest/ui": "^4.0.18",
"concurrently": "^9.2.1",
"cross-env": "^10.1.0",
"express": "^5.2.1",
"node-fetch": "^3.3.2",
"rimraf": "^6.1.3",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vite": "^7.3.1",
"vite-tsconfig-paths": "^6.1.1",
"vitest": "^4.0.14"
},
"dependencies": {
"dotenv": "^17.3.1"
}
}
================================================
FILE: packages/aws-s3/.npmignore
================================================
src
!dist
tsconfig.json
================================================
FILE: packages/aws-s3/CHANGELOG.md
================================================
# `@flystorage/aws-s3`
## 1.2.1
- Upgraded dependencies
## 1.2.0
## Changes
- Make tests run on Bun
## 1.1.1
## Changes
- Do not send a body when creating a directory to prevent hanging requests.
## 1.1.0
## Added
- AbortSignal Support
## 1.0.1
### Fixes
- Bug: fixed not being able to use write options
## 1.0.0
### Major Changes
- First 1.0.0 release
================================================
FILE: packages/aws-s3/LICENSE
================================================
Copyright (c) 2023-2024 Frank de Jonge
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: packages/aws-s3/README.md
================================================
<img src="https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg" width="50px" height="50px" />
# Flystorage adapter for AWS S3
This package contains the Flystorage adapter for AWS S3 using the V3 SDK.
## Installation
Install all the required packages
```bash
npm install --save @flystorage/file-storage @flystorage/aws-s3 @aws-sdk/client-s3
```
## Usage
```typescript
import {FileStorage} from '@flystorage/file-storage';
import {AwsS3StorageAdapter} from '@flystorage/aws-s3';
import {S3Client} from '@aws-sdk/client-s3';
const client = new S3Client();
const adapter = new AwsS3StorageAdapter(client, {
bucket: '{your-bucket-name}',
prefix: '{optional-path-prefix}',
});
const storage = new FileStorage(adapter);
```
> ⚠️ Always use the FileStorage, it is essential for security and a good developer
> experience. Do not use the adapter directly.
================================================
FILE: packages/aws-s3/package.json
================================================
{
"name": "@flystorage/aws-s3",
"type": "module",
"version": "1.2.0",
"dependencies": {
"@aws-sdk/client-s3": "^3.1004.0",
"@aws-sdk/lib-storage": "^3.1004.0",
"@aws-sdk/s3-request-presigner": "^3.1004.0",
"@flystorage/file-storage": "^1.1.0",
"@flystorage/stream-mime-type": "^1.0.0",
"file-type": "^21.3.1",
"mime-types": "^3.0.2"
},
"description": "",
"main": "./dist/cjs/index.js",
"types": "./dist/types/index.d.ts",
"exports": {
".": {
"import": {
"types": "./dist/types/index.d.ts",
"default": "./dist/esm/index.js"
},
"require": {
"types": "./dist/types/index.d.ts",
"default": "./dist/cjs/index.js"
}
}
},
"scripts": {
"compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json",
"compile:esm": "tsc --outDir ./dist/esm/ --declaration false",
"compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node",
"compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly",
"watch": "tsc --watch"
},
"keywords": [
"s3",
"file",
"storage",
"flystorage",
"filesystem"
],
"author": "Frank de Jonge (https://frankdejonge.nl)",
"homepage": "https://flystorage.dev/adapter/aws-s3/",
"repository": {
"type": "git",
"url": "git+https://github.com/duna-oss/flystorage.git",
"directory": "packages/aws-s3"
},
"license": "MIT"
}
================================================
FILE: packages/aws-s3/src/aws-s3-file-storage.test.ts
================================================
import {S3Client} from '@aws-sdk/client-s3';
import {
FileStorage,
readableToString,
Visibility,
closeReadable,
UploadRequestHeaders,
UnableToWriteFile,
UnableToReadFile,
} from '@flystorage/file-storage';
import {BinaryToTextEncoding, createHash, randomBytes} from 'crypto';
import * as https from 'https';
import {AwsS3StorageAdapter} from './aws-s3-storage-adapter.js';
import {createReadStream} from "node:fs";
import * as path from "node:path";
import 'dotenv/config';
import {PassThrough} from 'node:stream';
let client: S3Client;
let storage: FileStorage;
let bucket = 'flysystem-check';
const testSegment = randomBytes(10).toString('hex');
describe('aws-s3 file storage', () => {
beforeAll(() => {
const [major] = process.versions.node.split('.').map(Number);
const versionToBucketMapping = [[20, 'a'], [21, 'b'], [22, 'c'], [23, 'd']] as [number, string][];
const bucketSuffix = versionToBucketMapping.find(([version]) => version === major);
if (bucketSuffix) {
bucket = `flystorage-${bucketSuffix[1]}`;
}
client = new S3Client();
});
beforeEach(async () => {
const secondSegment = randomBytes(10).toString('hex');
storage = new FileStorage(new AwsS3StorageAdapter(client, {
bucket: bucket,
prefix: `storage/${testSegment}/${secondSegment}`,
}));
});
afterAll(async () => {
await new FileStorage(new AwsS3StorageAdapter(client, {
bucket: bucket,
prefix: 'storage',
})).deleteDirectory(testSegment);
client.destroy();
});
test('uploading using a prepared request', async () => {
const request = await storage.prepareUpload('prepared/request-file.txt', {
expiresAt: Date.now() + 60 * 1000,
headers: {
'Content-Type': 'text/plain',
}
});
await naivelyMakeRequestFile(
request.url,
request.headers,
request.method,
'this is the contents',
);
const contents = await storage.readToString('prepared/request-file.txt');
expect(contents).toEqual('this is the contents');
});
test('writing and reading a file', async () => {
await storage.write('path+name.txt', 'this is the contents');
expect(await storage.readToString('path+name.txt')).toEqual('this is the contents');
});
test('non deep and deep listing should have consistent and similar results', async () => {
await storage.write('file_1.txt', 'contents');
await storage.write('file_2.txt', 'contents');
await storage.createDirectory('directory_1');
await storage.createDirectory('directory_2');
const non_deep_listing = await storage.list('/', {deep: false}).toArray();
const deep_listing = await storage.list('/', {deep: true}).toArray();
expect(non_deep_listing).toHaveLength(4);
expect(deep_listing).toHaveLength(4);
expect(non_deep_listing).toEqual(deep_listing);
});
test('root listing should work without root-slash', async () => {
const rootStorage = new FileStorage(new AwsS3StorageAdapter(client, {
bucket: 'flystorage-root-check',
}));
await rootStorage.write('test-file.txt', 'contents');
await rootStorage.createDirectory('test-directory');
const non_deep_listing = await rootStorage.list('', {deep: false}).toArray();
const deep_listing = await rootStorage.list('', {deep: true}).toArray();
expect(non_deep_listing).toHaveLength(2);
expect(deep_listing).toHaveLength(2);
expect(non_deep_listing.map(item => item.path)).toEqual(deep_listing.map(item => item.path));
});
test('root listing should work with root-slash', async () => {
const rootStorage = new FileStorage(new AwsS3StorageAdapter(client, {
bucket: 'flystorage-root-check',
}));
await rootStorage.write('test-file.txt', 'contents');
await rootStorage.createDirectory('test-directory');
const non_deep_listing = await rootStorage.list('/', {deep: false}).toArray();
const deep_listing = await rootStorage.list('/', {deep: true}).toArray();
expect(non_deep_listing).toHaveLength(2);
expect(deep_listing).toHaveLength(2);
expect(non_deep_listing.map(item => item.path)).toEqual(deep_listing.map(item => item.path));
});
test('moving a file', async () => {
await storage.write('from+here.txt', 'this');
await storage.moveFile('from+here.txt', 'to+there.txt');
expect(await storage.fileExists('from+here.txt')).toEqual(false);
expect(await storage.readToString('to+there.txt')).toEqual('this');
});
test('copying a file', async () => {
await storage.write('from+this.txt', 'this');
await storage.copyFile('from+this.txt', 'to+that.txt');
expect(await storage.fileExists('from+this.txt')).toEqual(true);
expect(await storage.readToString('to+that.txt')).toEqual('this');
});
test('trying to copy a file that does not exist', async () => {
await expect(storage.copyFile('404.txt', 'to.txt')).rejects.toThrow();
});
test('timing out when writing a file', async () => {
const writeStream = new PassThrough();
writeStream.write('something');
setTimeout(() => {
writeStream.end('this');
}, 150);
await expect(storage.write('somewhere.txt', writeStream, {
timeout: 10,
})).rejects.toThrow();
})
test('trying to read a file that does not exist', async () => {
let was404 = false;
try {
await storage.read('404.txt');
} catch (err) {
if (err instanceof UnableToReadFile) {
was404 = err.wasFileNotFound;
}
}
expect(was404).toEqual(true);
});
test('trying to move a file that does not exist', async () => {
await expect(storage.moveFile('404.txt', 'to.txt')).rejects.toThrow();
});
test('you can download public files using a public URL', async () => {
await storage.write('public+file.txt', 'contents of the public file', {
visibility: Visibility.PUBLIC,
});
const url = await storage.publicUrl('public+file.txt');
const contents = await naivelyDownloadFile(url);
expect(contents).toEqual('contents of the public file');
});
test('private files can only be downloaded using a temporary URL', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(naivelyDownloadFile(await storage.publicUrl('private+file.txt'))).rejects.toThrow();
await expect(naivelyDownloadFile(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000})
)).resolves.toEqual('contents of the private file');
});
test('getting a temporary url for a non-existing-file', async () => {
await expect(
storage.temporaryUrl('this-does-not-exist.txt', {expiresAt: Date.now() + 60 * 1000})
).resolves.toContain('this-does-not-exist.txt');
});
describe('response headers', () => {
test('fetches file with Content-Disposition header when specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Disposition': 'attachment; filename="private+file.txt"'} }),
'Content-Disposition'
)).resolves.toEqual('attachment; filename="private+file.txt"');
});
test('fetches file without Content-Disposition header when not specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),
'Content-Disposition'
)).resolves.toBeUndefined();
});
test('fetches file with Cache-Control header when specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Cache-Control': 'none'} }),
'Cache-Control'
)).resolves.toEqual('none');
});
test('fetches file without Cache-Control header when not specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),
'Cache-Control'
)).resolves.toBeUndefined();
});
test('fetches file with Content-Encoding header when specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Encoding': 'br'} }),
'Content-Encoding'
)).resolves.toEqual('br');
});
test('fetches file without Content-Encoding header when not specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),
'Content-Encoding'
)).resolves.toBeUndefined();
});
test('fetches file with Content-Language header when specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Language': 'en-US'} }),
'Content-Language'
)).resolves.toEqual('en-US');
});
test('fetches file without Content-Language header when not specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),
'Content-Language'
)).resolves.toBeUndefined();
});
test('fetches file with Content-Type header when specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Type': 'image/jpeg+special'} }),
'Content-Type'
)).resolves.toEqual('image/jpeg+special');
});
test('fetches file with Expires header when specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Expires': 'Sat, 21 Oct 2023 07:28:00 GMT'} }),
'Expires'
)).resolves.toEqual('Sat, 21 Oct 2023 07:28:00 GMT');
});
test('fetches file without Expires header when not specified in the options', async () => {
await storage.write('private+file.txt', 'contents of the private file', {
visibility: Visibility.PRIVATE,
});
await expect(responseHeaderValue(
await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),
'Expires'
)).resolves.toBeUndefined();
});
});
test('retrieving the size of a file', async () => {
const contents = 'this is the contents of the file';
await storage.write('something+file.txt', contents);
expect(await storage.fileSize('something+file.txt')).toEqual(contents.length);
});
test('writing a png and fetching its mime-type', async () => {
const handle = createReadStream(path.resolve(process.cwd(), 'fixtures/screenshot.png'));
await storage.write('image.png', handle);
closeReadable(handle);
const mimeType = await storage.mimeType('image.png');
expect(mimeType).toEqual('image/png');
});
test('fetching the last modified', async () => {
await storage.write('test.txt', 'contents');
const lastModified = await storage.lastModified('test.txt');
expect(lastModified).toBeGreaterThan(Date.now() - 5000);
expect(lastModified).toBeLessThan(Date.now() + 5000);
});
test('it can request checksums', async () => {
function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string {
return createHash(algo).update(input).digest(encoding);
}
const contents = 'this is for the checksum';
await storage.write('path.txt', contents);
const expectedChecksum = hashString(contents, 'md5');
const checksum = await storage.checksum('path.txt', {
algo: 'etag',
});
expect(checksum).toEqual(expectedChecksum);
});
test('it can request sha256 checksums', async () => {
function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string {
return createHash(algo).update(input).digest(encoding);
}
const contents = 'this is for the checksum';
await storage.write('path.txt', contents, {
ChecksumAlgorithm: 'SHA256'
});
const expectedChecksum = hashString(contents, 'sha256', 'base64');
const checksum = await storage.checksum('path.txt', {
algo: 'sha256',
});
expect(checksum).toEqual(expectedChecksum);
});
test('uploads can be aborted', async () => {
const reason = new Error('Because I say so');
const controller = new AbortController();
const options = {
abortSignal: controller.signal,
} as const;
await expect((async () => {
const promise = storage.write('cache.txt', 'some content', {
abortSignal: controller.signal,
});
controller.abort(reason);
return promise;
})()).rejects.toThrow(UnableToWriteFile.because(
'Because I say so',
{cause: reason, context: {path: 'cache.txt', options}},
));
})
test('it handles custom Cache-Control header', async() => {
await storage.write('cache.txt', 'some content', {cacheControl: "max-age=7200, public", visibility: Visibility.PUBLIC});
const url = await storage.publicUrl('cache.txt')
const res = await fetch(url);
expect(res.headers.get("Cache-Control")).toEqual("max-age=7200, public")
});
});
function naivelyDownloadFile(url: string): Promise<string> {
return new Promise((resolve, reject) => {
https.get(url, async res => {
if (res.statusCode !== 200) {
reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));
} else {
resolve(await readableToString(res));
}
});
});
}
function naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise<void> {
return new Promise((resolve, reject) => {
const req = https.request(url, {
method: method,
headers: {
...headers,
'Content-Length': new Blob([data]).size,
}
}, async res => {
let responseBody = '';
res.on('data', (chunk) => {
responseBody += chunk;
});
res.on('end', () => {
const statusCode = res.statusCode ?? 500;
if (statusCode <= 200 && statusCode >= 299) {
reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));
} else {
resolve();
}
});
});
req.on('error', (err) => {
reject(err);
});
req.write(data);
req.end();
});
}
function responseHeaderValue(url: string, header: string): Promise<string | undefined> {
return new Promise((resolve, reject) => {
https.get(url, res => {
if (res.statusCode !== 200) {
reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));
} else {
resolve(
res.headers[header]?.toString()
?? res.headers[header.toLowerCase()]?.toString()
?? undefined
);
}
});
});
}
================================================
FILE: packages/aws-s3/src/aws-s3-storage-adapter.ts
================================================
import {
_Object,
CommonPrefix,
CopyObjectCommand,
CopyObjectRequest,
DeleteObjectCommand,
DeleteObjectsCommand,
GetObjectAclCommand,
GetObjectAclOutput,
GetObjectCommand,
GetObjectCommandInput,
GetObjectCommandOutput,
HeadObjectCommand,
ListObjectsV2Command,
ListObjectsV2Output,
ObjectCannedACL,
PutObjectAclCommand,
PutObjectCommand,
PutObjectCommandInput,
S3Client,
S3ServiceException,
} from '@aws-sdk/client-s3';
import {Configuration, Upload} from '@aws-sdk/lib-storage';
import {getSignedUrl} from '@aws-sdk/s3-request-presigner';
import {posix} from 'node:path';
import {
AdapterListOptions,
ChecksumIsNotAvailable,
ChecksumOptions,
closeReadable,
CopyFileOptions,
CreateDirectoryOptions,
FileContents,
FileWasNotFound,
MimeTypeOptions,
MiscellaneousOptions,
MoveFileOptions,
normalizeExpiryToMilliseconds,
PathPrefixer,
PublicUrlOptions,
StatEntry,
StorageAdapter,
TemporaryUrlOptions,
UploadRequest,
UploadRequestHeaders,
UploadRequestOptions,
Visibility,
WriteOptions,
} from '@flystorage/file-storage';
import {resolveMimeType} from '@flystorage/stream-mime-type';
import {Readable} from 'stream';
import {lookup} from 'mime-types';
type PutObjectOptions = Omit<PutObjectCommandInput, 'Bucket' | 'Key' | 'Body'>;
export type WriteOptionsForS3 = Omit<PutObjectOptions, 'ACL' | 'ContentLength'>;
const possibleChecksumAlgos = ['SHA1', 'SHA256', 'CRC32', 'CRC32C', 'ETAG'] as const;
type ChecksumAlgo = typeof possibleChecksumAlgos[number];
function isSupportedAlgo(algo: string): algo is ChecksumAlgo {
return possibleChecksumAlgos.includes(algo as ChecksumAlgo);
}
export type AwsS3StorageAdapterOptions = Readonly<{
bucket: string,
prefix?: string,
region?: string,
publicUrlOptions?: PublicUrlOptions,
uploadRequestOptions?: UploadRequestOptions,
putObjectOptions?: PutObjectOptions,
uploadConfiguration?: Partial<Omit<Configuration, 'abortController'>>,
defaultChecksumAlgo?: ChecksumAlgo,
}>;
export type AwsPublicUrlOptions = PublicUrlOptions & {
bucket: string,
region?: string,
forcePathStyle?: boolean,
baseUrl?: string,
}
export type AwsPublicUrlGenerator = {
publicUrl(path: string, options: AwsPublicUrlOptions): Promise<string>;
};
export class DefaultAwsPublicUrlGenerator implements AwsPublicUrlGenerator {
async publicUrl(path: string, options: AwsPublicUrlOptions): Promise<string> {
const baseUrl = options.baseUrl ?? 'https://{subdomain}.amazonaws.com/{uri}';
const subdomain = options.forcePathStyle !== true
? `${options.bucket}.s3`
: options.region === undefined
? 's3'
: `s3-${options.region}`;
const uri = options.forcePathStyle !== true
? encodePath(path)
: `${options.bucket}/${encodePath(path)}`;
return baseUrl.replace('{subdomain}', subdomain).replace('{uri}', uri);
}
}
/**
* BC extension
*/
export class HostStyleAwsPublicUrlGenerator extends DefaultAwsPublicUrlGenerator {
}
export type TimestampResolver = () => number;
type AclOptions = Pick<CopyObjectRequest, 'ACL'>;
/**
* Some commands need URI encoded paths to work ¯\_(ツ)_/¯
*/
function encodePath(path: string): string {
return path.split('/').map(encodeURIComponent).join('/');
}
function maybeAbort(signal?: AbortSignal) {
if (signal?.aborted) {
throw signal.reason;
}
}
export class AwsS3StorageAdapter implements StorageAdapter {
private readonly prefixer: PathPrefixer;
constructor(
private readonly client: S3Client,
private readonly options: AwsS3StorageAdapterOptions,
private readonly publicUrlGenerator: AwsPublicUrlGenerator = new DefaultAwsPublicUrlGenerator(),
private readonly timestampResolver: TimestampResolver = () => Date.now(),
) {
this.prefixer = new PathPrefixer(options.prefix ?? '', '/', (...paths) => {
const path = posix.join(...paths);
if (path === "." || path === "/") {
// 1) https://nodejs.org/api/path.html#pathjoinpaths
// Zero-length path segments are ignored. If the joined path string is a zero-length string then '.' will be
// returned, representing the current working directory.
// 2) In S3 we use delimiter:"/". In that case we need to remove the root-slash in order to list the
// root-directory contents.
return "";
} else {
return path;
}
});
}
async copyFile(from: string, to: string, options: CopyFileOptions): Promise<void> {
maybeAbort(options.abortSignal);
let visibility: string | undefined = options.visibility;
if (visibility === undefined && options.retainVisibility) {
visibility = await this.visibility(from, options);
maybeAbort(options.abortSignal);
}
let acl: AclOptions = (visibility !== undefined && options.useVisibility !== false)
? {ACL: this.visibilityToAcl(visibility)}
: {};
await this.client.send(new CopyObjectCommand({
Bucket: this.options.bucket,
CopySource: posix.join('/', this.options.bucket, encodePath(this.prefixer.prefixFilePath(from))),
Key: this.prefixer.prefixFilePath(to),
...acl,
}), {abortSignal: options.abortSignal});
}
async moveFile(from: string, to: string, options: MoveFileOptions): Promise<void> {
await this.copyFile(from, to, options);
await this.deleteFile(from, options);
}
async prepareUpload(path: string, options: UploadRequestOptions): Promise<UploadRequest> {
maybeAbort(options.abortSignal);
const expiry = normalizeExpiryToMilliseconds(options.expiresAt);
const now = (this.timestampResolver)();
const putObjectParams: PutObjectCommandInput = {
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
};
const headers: UploadRequestHeaders = {};
const contentType = options['Content-Type'] ?? options.contentType;
if (typeof contentType === 'string') {
putObjectParams.ContentType = contentType;
headers['Content-Type'] = contentType;
}
const url = await getSignedUrl(this.client, new PutObjectCommand(putObjectParams), {
expiresIn: Math.floor((expiry - now) / 1000),
});
return {
url,
method: 'PUT',
provider: 'aws-s3',
headers,
};
}
async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string> {
maybeAbort(options.abortSignal);
const expiry = normalizeExpiryToMilliseconds(options.expiresAt);
const now = (this.timestampResolver)();
const getObjectParams: GetObjectCommandInput = {
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
};
if (options.responseHeaders) {
if (options.responseHeaders['Cache-Control']) {
getObjectParams.ResponseCacheControl = options.responseHeaders['Cache-Control'];
}
if (options.responseHeaders['Content-Disposition']) {
getObjectParams.ResponseContentDisposition = options.responseHeaders['Content-Disposition'];
}
if (options.responseHeaders['Content-Encoding']) {
getObjectParams.ResponseContentEncoding = options.responseHeaders['Content-Encoding'];
}
if (options.responseHeaders['Content-Language']) {
getObjectParams.ResponseContentLanguage = options.responseHeaders['Content-Language'];
}
if (options.responseHeaders['Content-Type']) {
getObjectParams.ResponseContentType = options.responseHeaders['Content-Type'];
}
if (options.responseHeaders['Expires']) {
getObjectParams.ResponseExpires = new Date(options.responseHeaders['Expires']);
}
}
return await getSignedUrl(this.client, new GetObjectCommand(getObjectParams), {
expiresIn: Math.floor((expiry - now) / 1000),
});
}
async lastModified(path: string, options: MiscellaneousOptions): Promise<number> {
const stat = await this.stat(path, options);
if (stat.lastModifiedMs === undefined) {
throw new Error('Last modified is not available in stat');
}
return stat.lastModifiedMs;
}
async fileSize(path: string, options: MiscellaneousOptions): Promise<number> {
const stat = await this.stat(path, options);
if (stat.isFile === false) {
throw new Error('Path is not a file');
}
if (stat.size === undefined) {
throw new Error('File size is not available in stat.')
}
return stat.size;
}
async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
const response = await this.stat(path, options);
if (!response.isFile) {
throw new Error(`Path "${path} is not a file.`);
}
if (response.mimeType) {
return response.mimeType;
}
if (options.disallowFallback) {
throw new Error('Mime-type not available via HeadObject');
}
maybeAbort(options.abortSignal);
const method = options.fallbackMethod ?? 'path';
const mimeType = method === 'path'
? lookup(path)
: await this.lookupMimeTypeFromStream(path, options);
if (mimeType === undefined || mimeType === false) {
throw new Error('Unable to resolve mime-type');
}
return mimeType;
}
async visibility(path: string, options: MiscellaneousOptions): Promise<string> {
maybeAbort(options.abortSignal);
const response: GetObjectAclOutput = await this.client.send(new GetObjectAclCommand({
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
}), {
abortSignal: options.abortSignal,
});
const publicRead = response.Grants?.some(grant =>
grant.Grantee?.URI === 'http://acs.amazonaws.com/groups/global/AllUsers'
&& grant.Permission === 'READ'
) ?? false;
return publicRead ? Visibility.PUBLIC : Visibility.PRIVATE;
}
async* list(path: string, options: AdapterListOptions): AsyncGenerator<StatEntry, any, unknown> {
const listing = this.listObjects(path, {
deep: options.deep,
includePrefixes: true,
includeSelf: false,
abortSignal: options.abortSignal,
});
for await (const {type, item} of listing) {
if (type === 'prefix') {
yield {
type: 'directory',
isFile: false,
isDirectory: true,
path: this.prefixer.stripDirectoryPath(item.Prefix!),
};
} else {
const path = item.Key!;
if (path.endsWith('/')) {
yield {
type: 'directory',
isFile: false,
isDirectory: true,
path: this.prefixer.stripDirectoryPath(path),
};
} else {
yield {
type: 'file',
isFile: true,
isDirectory: false,
path: this.prefixer.stripFilePath(path),
size: item.Size ?? 0,
lastModifiedMs: item.LastModified?.getTime(),
};
}
}
}
}
async * listObjects(
path: string,
options: {
deep: boolean,
includePrefixes: boolean,
includeSelf: boolean,
maxKeys?: number,
abortSignal?: AbortSignal,
},
): AsyncGenerator<{ type: 'prefix', item: CommonPrefix } | { type: 'object', item: _Object }, any, unknown> {
maybeAbort(options.abortSignal);
const prefix = this.prefixer.prefixDirectoryPath(path);
let collectedKeys = 0;
let shouldContinue = true;
let continuationToken: string | undefined = undefined;
while (shouldContinue && (options.maxKeys === undefined || collectedKeys < options.maxKeys)) {
maybeAbort(options.abortSignal);
const response: ListObjectsV2Output = await this.client.send(new ListObjectsV2Command({
Bucket: this.options.bucket,
Prefix: prefix,
Delimiter: options.deep ? undefined : '/',
ContinuationToken: continuationToken,
MaxKeys: options.maxKeys,
}));
continuationToken = response.NextContinuationToken;
shouldContinue = response.IsTruncated ?? false;
const prefixes = options.includePrefixes ? response.CommonPrefixes ?? [] : [];
for (const item of prefixes) {
if ((!options.includeSelf && item.Prefix === prefix) || item.Prefix === undefined) {
continue;
}
collectedKeys++;
yield {type: 'prefix', item};
}
for (const item of response.Contents ?? []) {
if ((!options.includeSelf && item.Key === prefix) || item.Key === undefined) {
// not interested in itself
// not interested in empty prefixes
continue;
}
collectedKeys++;
yield {type: 'object', item};
}
}
}
async read(path: string, options: MiscellaneousOptions): Promise<FileContents> {
maybeAbort(options.abortSignal);
let response: GetObjectCommandOutput;
try {
response = await this.client.send(new GetObjectCommand({
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
}), {
abortSignal: options.abortSignal,
});
} catch (err) {
if (err instanceof S3ServiceException && err.$metadata.httpStatusCode === 404) {
throw FileWasNotFound.atLocation(path, {
context: {path, options},
cause: err,
});
}
throw err;
}
if (response.Body instanceof Readable || response.Body instanceof ReadableStream) {
return response.Body;
}
throw new Error('No response body was provided');
}
async stat(path: string, options: MiscellaneousOptions): Promise<StatEntry> {
maybeAbort(options.abortSignal);
const response = await this.client.send(new HeadObjectCommand({
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
}), {
abortSignal: options.abortSignal,
});
return {
path,
type: 'file',
isDirectory: false,
isFile: true,
size: response.ContentLength ?? 0,
lastModifiedMs: response.LastModified?.getTime(),
mimeType: response.ContentType,
};
}
async createDirectory(path: string, options: CreateDirectoryOptions): Promise<void> {
const key = this.prefixer.prefixDirectoryPath(path);
const abortSignal = options.abortSignal;
const abortController = new AbortController();
if (abortSignal) {
if (abortSignal.aborted) {
throw abortSignal.reason;
}
abortSignal.addEventListener('abort', () => {
abortController.abort(abortSignal.reason);
});
}
const params = this.createPutObjectParams(key, '', {
ContentLength: 0,
ACL: options.directoryVisibility ? this.visibilityToAcl(options.directoryVisibility) : undefined,
});
maybeAbort(abortSignal);
await this.client.send(new PutObjectCommand(params), {
abortSignal,
});
}
async deleteDirectory(path: string): Promise<void> {
// @ts-ignore because we know it will only be objects
let itemsToDelete: AsyncGenerator<{ item: _Object }> = this.listObjects(path, {
deep: true,
includeSelf: true,
includePrefixes: false,
});
const flush = async (keys: { Key: string }[]) => this.client.send(new DeleteObjectsCommand({
Bucket: this.options.bucket,
Delete: {
Objects: keys,
},
}));
let bucket: { Key: string }[] = [];
let promises: Promise<any>[] = [];
for await (const {item} of itemsToDelete) {
bucket.push({Key: item.Key!});
if (bucket.length > 1000) {
promises.push(flush(bucket));
bucket = [];
}
}
if (bucket.length > 0) {
promises.push(flush(bucket));
}
await Promise.all(promises);
}
async write(path: string, contents: Readable, options: WriteOptions): Promise<void> {
let mimeType = options.mimeType;
if (mimeType === undefined) {
[mimeType, contents] = await resolveMimeType(path, contents);
}
const writeOptions: PutObjectOptions = {
ACL: options.visibility ? this.visibilityToAcl(options.visibility) : undefined,
ContentType: mimeType,
ContentLength: options.size,
CacheControl: options.cacheControl,
}
for (const option of Object.keys(options)) {
if (isWriteOptionKey(option)) {
const resolver = (writeOptionResolvers as any)[option];
const value = options[option];
if (resolver(value)) {
(writeOptions as any)[option] = value;
}
}
}
const abortController = new AbortController();
if (options.abortSignal) {
const abortSignal = options.abortSignal;
if (abortSignal.aborted) {
throw abortSignal.reason;
}
abortSignal.addEventListener('abort', () => {
abortController.abort(abortSignal.reason);
});
}
const upload = new Upload({
client: this.client,
params: this.createPutObjectParams(
this.prefixer.prefixFilePath(path),
contents,
writeOptions,
),
abortController,
...this.options.uploadConfiguration,
});
await upload.done();
}
private createPutObjectParams(
key: string,
contents: Readable | '',
options: PutObjectOptions,
): PutObjectCommandInput {
const params: PutObjectCommandInput = {
Bucket: this.options.bucket,
Key: key,
...Object.assign({}, this.options.putObjectOptions, options),
};
if (contents !== '') {
params.Body = contents;
}
return params;
}
async deleteFile(path: string, options: MiscellaneousOptions): Promise<void> {
maybeAbort(options.abortSignal);
const key = this.prefixer.prefixFilePath(path);
await this.client.send(new DeleteObjectCommand({
Bucket: this.options.bucket,
Key: key,
}), {
abortSignal: options.abortSignal,
});
}
private visibilityToAcl(visibility: string): ObjectCannedACL {
if (visibility === Visibility.PUBLIC) {
return 'public-read';
} else if (visibility === Visibility.PRIVATE) {
return 'private';
}
throw new Error(`Unrecognized visibility provided; ${visibility}`);
}
async changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise<void> {
maybeAbort(options.abortSignal);
await this.client.send(new PutObjectAclCommand({
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
ACL: this.visibilityToAcl(visibility),
}), {
abortSignal: options.abortSignal,
});
}
async fileExists(path: string, options: MiscellaneousOptions): Promise<boolean> {
maybeAbort(options.abortSignal);
try {
await this.client.send(new HeadObjectCommand({
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
}), {
abortSignal: options.abortSignal,
});
return true;
} catch (e) {
if (e instanceof S3ServiceException && e.$metadata.httpStatusCode === 404) {
return false;
}
throw e;
}
}
async directoryExists(path: string, options: MiscellaneousOptions): Promise<boolean> {
const listing = this.listObjects(path, {
deep: true,
includePrefixes: true,
includeSelf: true,
maxKeys: 1,
abortSignal: options.abortSignal,
});
for await (const _item of listing) {
return true;
}
return false;
}
async publicUrl(path: string, options: PublicUrlOptions): Promise<string> {
maybeAbort(options.abortSignal);
return this.publicUrlGenerator.publicUrl(this.prefixer.prefixFilePath(path), {
bucket: this.options.bucket,
...options,
...this.options.publicUrlOptions,
});
}
async checksum(path: string, options: ChecksumOptions): Promise<string> {
maybeAbort(options.abortSignal);
const algo = (options.algo || this.options.defaultChecksumAlgo || 'SHA256').toUpperCase();
if (!isSupportedAlgo(algo)) {
throw ChecksumIsNotAvailable.checksumNotSupported(algo);
}
const responseKey = algo === 'ETAG' ? 'ETag' : `Checksum${algo}` as const;
const response = await this.client.send(new HeadObjectCommand({
Bucket: this.options.bucket,
Key: this.prefixer.prefixFilePath(path),
...algo === 'ETAG' ? {} : {ChecksumMode: 'ENABLED'},
}), {
abortSignal: options.abortSignal,
});
const checksum = response[responseKey];
if (checksum === undefined) {
throw new Error(`Unable to retrieve checksum with algo ${algo}`);
}
return checksum.replace(/^"(.+)"$/, '$1');
}
private async lookupMimeTypeFromStream(path: string, options: MimeTypeOptions) {
const [mimetype, stream] = await resolveMimeType(path, Readable.from(await this.read(path, options)));
await closeReadable(stream);
return mimetype;
}
}
/**
* BC export
*
* @deprecated
*/
export class AwsS3FileStorage extends AwsS3StorageAdapter {}
type ResolversForWriteOptions = {
[K in keyof WriteOptionsForS3]-?: (value: any) => value is WriteOptionsForS3[K]
}
function isWriteOptionKey(key: string): key is string & keyof ResolversForWriteOptions {
return Object.hasOwn(writeOptionResolvers, key);
}
export const writeOptionResolvers: ResolversForWriteOptions = {
ChecksumCRC64NVME: function (value: any): value is PutObjectOptions['ChecksumCRC64NVME'] {
return typeof value === 'string';
},
IfMatch: function (value: any): value is PutObjectOptions['IfMatch'] {
return typeof value === 'string';
},
WriteOffsetBytes: function (value: any): value is PutObjectOptions['WriteOffsetBytes'] {
return typeof value === 'string';
},
ChecksumSHA1: function (value: any): value is PutObjectOptions['ChecksumSHA1'] {
return typeof value === 'string';
},
ChecksumSHA256: function (value: any): value is PutObjectOptions['ChecksumSHA256'] {
return typeof value === 'string';
},
ChecksumCRC32: function (value: any): value is PutObjectOptions['ChecksumCRC32'] {
return typeof value === 'string';
},
ChecksumCRC32C: function (value: any): value is PutObjectOptions['ChecksumCRC32C'] {
return typeof value === 'string';
},
CacheControl: function (value: any): value is PutObjectOptions['CacheControl'] {
return typeof value === 'string';
},
ContentDisposition: function (value: any): value is PutObjectOptions['ContentDisposition'] {
return typeof value === 'string';
},
ContentEncoding: function (value: any): value is PutObjectOptions['ContentEncoding'] {
return typeof value === 'string';
},
ContentLanguage: function (value: any): value is PutObjectOptions['ContentLanguage'] {
return typeof value === 'string';
},
ContentMD5: function (value: any): value is PutObjectOptions['ContentMD5'] {
return typeof value === 'string';
},
ContentType: function (value: any): value is PutObjectOptions['ContentType'] {
return typeof value === 'string';
},
ChecksumAlgorithm: function (value: any): value is PutObjectOptions['ChecksumAlgorithm'] {
return typeof value === 'string';
},
Expires: function (value: any): value is PutObjectOptions['Expires'] {
return value instanceof Date;
},
GrantFullControl: function (value: any): value is PutObjectOptions['GrantFullControl'] {
return typeof value === 'string';
},
GrantRead: function (value: any): value is PutObjectOptions['GrantRead'] {
return typeof value === 'string';
},
GrantReadACP: function (value: any): value is PutObjectOptions['GrantReadACP'] {
return typeof value === 'string';
},
GrantWriteACP: function (value: any): value is PutObjectOptions['GrantWriteACP'] {
return typeof value === 'string';
},
Metadata: function (value: any): value is PutObjectOptions['Metadata'] {
return typeof value === 'object';
},
ServerSideEncryption: function (value: any): value is PutObjectOptions['ServerSideEncryption'] {
return typeof value === 'string';
},
StorageClass: function (value: any): value is PutObjectOptions['StorageClass'] {
return typeof value === 'string';
},
WebsiteRedirectLocation: function (value: any): value is PutObjectOptions['WebsiteRedirectLocation'] {
return typeof value === 'string';
},
SSECustomerAlgorithm: function (value: any): value is PutObjectOptions['SSECustomerAlgorithm'] {
return typeof value === 'string';
},
SSECustomerKey: function (value: any): value is PutObjectOptions['SSECustomerKey'] {
return typeof value === 'string';
},
SSECustomerKeyMD5: function (value: any): value is PutObjectOptions['SSECustomerKeyMD5'] {
return typeof value === 'string';
},
SSEKMSKeyId: function (value: any): value is PutObjectOptions['SSEKMSKeyId'] {
return typeof value === 'string';
},
SSEKMSEncryptionContext: function (value: any): value is PutObjectOptions['SSEKMSEncryptionContext'] {
return typeof value === 'string';
},
BucketKeyEnabled: function (value: any): value is PutObjectOptions['BucketKeyEnabled'] {
return typeof value === 'string';
},
RequestPayer: function (value: any): value is PutObjectOptions['RequestPayer'] {
return typeof value === 'string';
},
Tagging: function (value: any): value is PutObjectOptions['Tagging'] {
return typeof value === 'string';
},
ObjectLockMode: function (value: any): value is PutObjectOptions['ObjectLockMode'] {
return typeof value === 'string';
},
ObjectLockRetainUntilDate: function (value: any): value is PutObjectOptions['ObjectLockRetainUntilDate'] {
return value instanceof Date;
},
ObjectLockLegalHoldStatus: function (value: any): value is PutObjectOptions['ObjectLockLegalHoldStatus'] {
return typeof value === 'string';
},
ExpectedBucketOwner: function (value: any): value is PutObjectOptions['ExpectedBucketOwner'] {
return typeof value === 'string';
},
IfNoneMatch: function (value: any): value is WriteOptionsForS3['IfNoneMatch'] {
return typeof value === 'string';
},
};
================================================
FILE: packages/aws-s3/src/index.ts
================================================
export * from './aws-s3-storage-adapter.js';
================================================
FILE: packages/aws-s3/tsconfig.json
================================================
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist/"
},
"include": [
"src/**/*.ts"
]
}
================================================
FILE: packages/azure-storage-blob/.npmignore
================================================
src
!dist
tsconfig.json
================================================
FILE: packages/azure-storage-blob/CHANGELOG.md
================================================
# `@flystorage/azure-storage-blob`
## 1.2.0
## Changes
- Support Bun runtime
- Updated dependencies
- 404 detection for reads
## 1.1.0
## Added
- AbortSignal Support
## 1.0.0
### Major Changes
- First 1.0.0 release
================================================
FILE: packages/azure-storage-blob/LICENSE
================================================
Copyright (c) 2023-2024 Frank de Jonge
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: packages/azure-storage-blob/README.md
================================================
<img src="https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg" width="50px" height="50px" />
# Flystorage adapter for Azure Storage Blob
This package contains the Flystorage adapter for Azure Storage Blob
## Installation
Install all the required packages
```bash
npm install --save @flystorage/file-storage @flystorage/azure-storage-blob @azure/storage-blob
```
## Usage
```typescript
import {FileStorage} from '@flystorage/file-storage';
import {AzureStorageBlobStorageAdapter} from '@flystorage/azure-storage-blob';
const blobService = BlobServiceClient.fromConnectionString(process.env.AZURE_DSN!);
const container = blobService.getContainerClient('flysystem');
const adapter = new AzureStorageBlobStorageAdapter(container);
const storage = new FileStorage(adapter);
```
> ⚠️ Always use the FileStorage, it is essential for security and a good developer
> experience. Do not use the adapter directly.
================================================
FILE: packages/azure-storage-blob/package.json
================================================
{
"name": "@flystorage/azure-storage-blob",
"type": "module",
"version": "1.2.0",
"dependencies": {
"@azure/storage-blob": "^12.31.0",
"@flystorage/file-storage": "^1.1.0",
"@flystorage/stream-mime-type": "^1.0.0"
},
"description": "",
"main": "./dist/cjs/index.js",
"types": "./dist/types/index.d.ts",
"exports": {
".": {
"import": {
"types": "./dist/types/index.d.ts",
"default": "./dist/esm/index.js"
},
"require": {
"types": "./dist/types/index.d.ts",
"default": "./dist/cjs/index.js"
}
}
},
"scripts": {
"compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json",
"compile:esm": "tsc --outDir ./dist/esm/ --declaration false",
"compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node",
"compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly",
"watch": "tsc --watch"
},
"keywords": [
"s3",
"file",
"storage",
"flystorage",
"filesystem"
],
"author": "Frank de Jonge (https://frankdejonge.nl)",
"homepage": "https://flystorage.dev/adapter/azure-storage-blob/",
"repository": {
"type": "git",
"url": "git+https://github.com/duna-oss/flystorage.git",
"directory": "packages/aws-s3"
},
"license": "MIT"
}
================================================
FILE: packages/azure-storage-blob/src/azure-storage-blob.test.ts
================================================
import {BlobServiceClient} from "@azure/storage-blob";
import {AzureStorageBlobStorageAdapter} from "./azure-storage-blob.js";
import {randomBytes} from "crypto";
import {
FileStorage,
UploadRequestHeaders,
Visibility,
readableToString,
UnableToReadFile,
} from '@flystorage/file-storage';
import fetch from "node-fetch";
import { Readable } from "node:stream";
import https from 'https';
import 'dotenv/config';
const runSegment = process.env.AZURE_PREFIX ?? randomBytes(10).toString('hex');
describe('AzureStorageBlobStorageAdapter', () => {
const blobService = BlobServiceClient.fromConnectionString(
process.env.AZURE_DSN || "UseDevelopmentStorage=true;"
);
const container = blobService.getContainerClient('flysystem');
let storage: FileStorage;
beforeAll(async () => {
await container.createIfNotExists({ access: "container" });
});
beforeEach(() => {
const testSegment = randomBytes(10).toString('hex');
const adapter = new AzureStorageBlobStorageAdapter(
container,
{
prefix: `flystorage/${runSegment}/${testSegment}`,
}
);
storage = new FileStorage(adapter);
});
afterAll(async () => {
const adapter = new AzureStorageBlobStorageAdapter(container);
storage = new FileStorage(adapter);
await storage.deleteDirectory(`flystorage/${runSegment}`);
container
});
test('non deep and deep listing should have consistent and similar results', async () => {
await storage.write('file_1.txt', 'contents');
await storage.write('file_2.txt', 'contents');
await storage.write('directory_1/file.txt', 'contents');
await storage.write('directory_2/file.tx', 'contents');
const non_deep_listing = await storage.list('/', {deep: false}).toArray();
const deep_listing = await storage.list('/', {deep: true}).toArray();
expect(non_deep_listing).toHaveLength(4);
expect(deep_listing).toHaveLength(6);
});
test('it can request a checksum', async () => {
const contents = 'this is for the checksum';
await storage.write('path.txt', contents);
const checksum = await storage.checksum('path.txt', {
algo: 'etag',
});
expect(typeof checksum).toEqual('string');
expect(checksum.length).toBeGreaterThan(5);
});
test('reading a file that was written', async () => {
await storage.write('path.txt', 'content in azure');
const content = await storage.readToString('path.txt');
expect(content).toEqual('content in azure');
});
test('trying to read a file that does not exist', async () => {
let was404 = false;
try {
await storage.readToString('404.txt');
} catch (err) {
if (err instanceof UnableToReadFile) {
was404 = err.wasFileNotFound;
}
}
expect(was404).toEqual(true);
});
test('trying to see if a non-existing file exists', async () => {
expect(await storage.fileExists('404.txt')).toEqual(false);
});
test('trying to see if an existing file exists', async () => {
await storage.write('existing.txt', 'contents');
expect(await storage.fileExists('existing.txt')).toEqual(true);
});
test('deleting an existing file', async () => {
await storage.write('existing.txt', 'contents');
expect(await storage.fileExists('existing.txt')).toEqual(true);
await storage.deleteFile('existing.txt');
expect(await storage.fileExists('existing.txt')).toEqual(false);
});
test('deleting a non-existing file is OK', async () => {
await storage.deleteFile('404.txt');
});
test('copying a file', async () => {
await storage.write('file.txt', 'copied');
await storage.copyFile('file.txt', 'new-file.txt');
expect(await storage.fileExists('file.txt')).toEqual(true);
expect(await storage.fileExists('new-file.txt')).toEqual(true);
expect(await storage.readToString('new-file.txt')).toEqual('copied');
});
test('moving a file', async () => {
await storage.write('file.txt', 'moved');
await storage.moveFile('file.txt', 'new-file.txt');
expect(await storage.fileExists('file.txt')).toEqual(false);
expect(await storage.fileExists('new-file.txt')).toEqual(true);
expect(await storage.readToString('new-file.txt')).toEqual('moved');
});
test('setting visibility always fails', async () => {
await storage.write('existing.txt', 'yes');
await expect(storage.changeVisibility('existing.txt', Visibility.PRIVATE)).rejects.toThrow();
await expect(storage.changeVisibility('404.txt', Visibility.PUBLIC)).rejects.toThrow();
});
test('listing entries in a directory, shallow', async () => {
await storage.write('outside/path.txt', 'test');
await storage.write('inside/a.txt', 'test');
await storage.write('inside/b.txt', 'test');
await storage.write('inside/c/a.txt', 'test');
const listing = await storage.list('inside').toArray();
expect(listing).toHaveLength(3);
expect(listing[0].type).toEqual('file');
expect(listing[1].type).toEqual('file');
expect(listing[2].type).toEqual('directory');
expect(listing[0].path).toEqual('inside/a.txt');
expect(listing[1].path).toEqual('inside/b.txt');
expect(listing[2].path).toEqual('inside/c');
});
test('listing entries in a directory, deep', async () => {
await storage.write('outside/path.txt', 'test');
await storage.write('inside/a.txt', 'test');
await storage.write('inside/b.txt', 'test');
await storage.write('inside/c/a.txt', 'test');
const listing = await storage.list('inside', {deep: true}).toArray();
expect(listing).toHaveLength(4);
expect(listing[0].type).toEqual('file');
expect(listing[1].type).toEqual('file');
expect(listing[2].type).toEqual('directory');
expect(listing[3].type).toEqual('file');
expect(listing[0].path).toEqual('inside/a.txt');
expect(listing[1].path).toEqual('inside/b.txt');
expect(listing[2].path).toEqual('inside/c');
expect(listing[3].path).toEqual('inside/c/a.txt');
});
test('deleting a full directory', async () => {
await storage.write('directory/a.txt', 'test');
await storage.write('directory/b.txt', 'test');
await storage.write('directory/c/a.txt', 'test');
await storage.deleteDirectory('directory');
const listing = await storage.list('directory', {deep: true}).toArray();
expect(listing).toEqual([]);
});
test('checking if a directory exists', async () => {
await storage.write('directory/a.txt', 'test');
await storage.write('directory/b.txt', 'test');
await storage.write('directory/c/a.txt', 'test');
expect(await storage.directoryExists('directory')).toEqual(true);
expect(await storage.directoryExists('directory/c')).toEqual(true);
expect(await storage.directoryExists('directory/a')).toEqual(false);
});
test('accessing a file though public URL', async () => {
await storage.write('something.txt', 'something');
const url = await storage.publicUrl('something.txt');
const contents = await naivelyDownloadFile(url);
expect(contents).toEqual('something');
});
test('accessing a file though temporary URL', async () => {
await storage.write('something.txt', 'something');
const url = await storage.temporaryUrl('something.txt', {
expiresAt: Date.now() + 600000,
});
const contents = await naivelyDownloadFile(url);
expect(contents).toEqual('something');
});
test('uploading using a prepared request', async () => {
const request = await storage.prepareUpload('prepared/request-file.txt', {
expiresAt: Date.now() + 60 * 1000,
headers: {
'Content-Type': 'text/plain',
}
});
await naivelyMakeRequestFile(
request.url,
request.headers,
request.method,
'this is the contents',
);
const contents = await storage.readToString('prepared/request-file.txt');
expect(contents).toEqual('this is the contents');
});
test('setting cache-control headers', async () => {
await storage.write('cachecontrol.txt', 'something', { cacheControl: 'max-age=3200, public' });
const url = await storage.publicUrl('cachecontrol.txt');
const res = await fetch(url);
expect(res.headers.get("Cache-Control")).toEqual('max-age=3200, public');
});
});
async function naivelyDownloadFile(url: string): Promise<string> {
const res = await fetch(url);
if (res.status !== 200 || !res.body) {
throw new Error(`Not able to download the file from ${url}, response status [${res.status}]`);
} else {
return await readableToString(Readable.from(res.body));
}
}
function naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise<void> {
return new Promise((resolve, reject) => {
const req = https.request(url, {
method: method,
headers: {
...headers,
'Content-Length': new Blob([data]).size,
}
}, async res => {
let responseBody = '';
res.on('data', (chunk) => {
responseBody += chunk;
});
res.on('end', () => {
const statusCode = res.statusCode ?? 500;
if (statusCode <= 200 && statusCode >= 299) {
reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));
} else {
resolve();
}
});
});
req.on('error', (err) => {
reject(err);
});
req.write(data);
req.end();
});
}
================================================
FILE: packages/azure-storage-blob/src/azure-storage-blob.ts
================================================
import {Readable} from 'stream';
import {
ChecksumIsNotAvailable,
ChecksumOptions,
CopyFileOptions,
FileContents,
FileWasNotFound,
ListOptions,
MimeTypeOptions,
MiscellaneousOptions,
MoveFileOptions,
normalizeExpiryToDate,
PathPrefixer,
PublicUrlOptions,
StatEntry,
StorageAdapter,
TemporaryUrlOptions,
UploadRequest,
UploadRequestHeaders,
UploadRequestOptions,
WriteOptions,
} from '@flystorage/file-storage';
import {
BlobDownloadResponseParsed,
BlobGenerateSasUrlOptions,
BlobGetPropertiesResponse,
BlobProperties,
BlobSASPermissions,
ContainerClient,
} from '@azure/storage-blob';
import {resolveMimeType} from '@flystorage/stream-mime-type';
import {dirname} from 'node:path';
export type AzureStorageBlobStorageAdapterOptions = {
prefix?: string,
uploadMaxConcurrency?: number,
ignoreVisibility?: boolean,
ignoredVisibilityResponse?: string,
deleteDirBatchSize?: number,
temporaryUrlOptions?: TemporaryUrlOptions,
}
function maybeAbort(signal?: AbortSignal) {
if (signal?.aborted) {
throw signal.reason;
}
}
export class AzureStorageBlobStorageAdapter implements StorageAdapter {
private readonly prefixer: PathPrefixer;
constructor(
private readonly container: ContainerClient,
private readonly options: AzureStorageBlobStorageAdapterOptions = {},
) {
this.prefixer = new PathPrefixer(options.prefix || '');
}
async copyFile(from: string, to: string, options: CopyFileOptions): Promise<void> {
const fromUrl = this.blockClient(from).url;
maybeAbort(options.abortSignal);
await this.blockClient(to).syncCopyFromURL(fromUrl, {abortSignal: options.abortSignal});
}
async moveFile(from: string, to: string, options: MoveFileOptions): Promise<void> {
await this.copyFile(from, to, options);
await this.deleteFile(from, options);
}
async write(path: string, contents: Readable, options: WriteOptions): Promise<void> {
let mimeType = options.mimeType;
let stream = contents;
maybeAbort(options.abortSignal);
if (mimeType === undefined) {
[mimeType, stream] = await this.resolveMimetype(path, contents, options);
}
maybeAbort(options.abortSignal);
const blob = this.blockClient(path);
await blob.uploadStream(
stream,
options.size,
this.options.uploadMaxConcurrency,
{
abortSignal: options.abortSignal,
blobHTTPHeaders: {
blobContentType: mimeType,
blobCacheControl: options.cacheControl
},
},
);
}
private blockClient(path: string) {
return this.container.getBlockBlobClient(this.prefixer.prefixFilePath(path));
}
async read(path: string, options: MiscellaneousOptions): Promise<FileContents> {
maybeAbort(options.abortSignal);
const blob = this.blockClient(path);
let response: BlobDownloadResponseParsed;
try {
response = await blob.download(undefined, undefined, {
abortSignal: options.abortSignal,
});
} catch (err) {
if ((err as any).statusCode === 404) {
throw FileWasNotFound.atLocation(path, {
context: {path, options},
cause: err,
})
}
throw err;
}
if (!response.readableStreamBody) {
throw new Error('No readable stream body in response.');
}
return response.readableStreamBody;
}
async deleteFile(path: string, options: MiscellaneousOptions): Promise<void> {
const blob = this.blockClient(path);
maybeAbort(options.abortSignal);
await blob.deleteIfExists({
abortSignal: options.abortSignal,
});
}
async createDirectory(): Promise<void> {
// no-op, directories do not exist.
}
async stat(path: string, options: {abortSignal?: AbortSignal} = {}): Promise<StatEntry> {
maybeAbort(options.abortSignal);
const blob = this.blockClient(path);
const properties = await blob.getProperties({
abortSignal: options.abortSignal,
});
return this.mapToStatEntry(path, properties);
}
private mapToStatEntry(path: string, properties: BlobGetPropertiesResponse | BlobProperties): StatEntry {
return {
type: 'file',
isFile: true,
isDirectory: false,
path,
mimeType: properties.contentType,
size: properties.contentLength,
lastModifiedMs: properties.lastModified?.getTime(),
};
}
list(path: string, options: ListOptions): AsyncGenerator<StatEntry> {
return options.deep
? this.listDeep(path, options)
: this.listShallow(path, options);
}
async *listDeep(path: string, options: ListOptions): AsyncGenerator<StatEntry> {
maybeAbort(options?.abortSignal);
const directories = new Set<string>();
const listing = this.container.listBlobsFlat({
prefix: this.prefixer.prefixDirectoryPath(path),
abortSignal: options.abortSignal,
});
const listedPath = path;
for await (const item of listing) {
maybeAbort(options?.abortSignal);
const path = this.prefixer.stripFilePath(item.name);
let parentDir = dirname(path);
while(!['.', '', listedPath].includes(parentDir)) {
if (directories.has(parentDir)) {
break;
}
yield {
type: 'directory',
isFile: false,
isDirectory: true,
path: parentDir,
};
directories.add(parentDir);
parentDir = dirname(parentDir);
}
yield this.mapToStatEntry(path, item.properties);
}
}
async *listShallow(path: string, options: ListOptions): AsyncGenerator<StatEntry> {
maybeAbort(options?.abortSignal);
const listing = this.container.listBlobsByHierarchy('/', {
prefix: this.prefixer.prefixDirectoryPath(path),
abortSignal: options.abortSignal,
});
for await (const item of listing) {
maybeAbort(options?.abortSignal);
if (item.kind === 'blob') {
yield this.mapToStatEntry(
this.prefixer.stripFilePath(item.name),
item.properties
)
} else {
yield {
path: this.prefixer.stripDirectoryPath(item.name),
type: 'directory',
isFile: false,
isDirectory: true,
}
}
}
}
async changeVisibility(path: string, visibility: string): Promise<void> {
if (this.options.ignoreVisibility !== true) {
throw new Error('Not supported by this adapter');
}
}
async visibility(path: string): Promise<string> {
if (this.options.ignoreVisibility !== true) {
throw new Error('Not implemented');
}
// default to indicating it ss public because we cannot know if the default is private
return this.options.ignoredVisibilityResponse ?? 'public';
}
async deleteDirectory(path: string, options: MiscellaneousOptions): Promise<void> {
let deletes: Promise<any>[] = [];
const batchSize = this.options.deleteDirBatchSize ?? 10;
for await (const item of this.list(path, {deep: true})) {
if (item.isFile) {
deletes.push(this.deleteFile(item.path, options));
}
if (deletes.length >= batchSize) {
await Promise.all(deletes);
deletes = [];
}
}
await Promise.all(deletes);
}
async fileExists(path: string, options: MiscellaneousOptions): Promise<boolean> {
maybeAbort(options.abortSignal);
return await this.blockClient(path).exists({
abortSignal: options.abortSignal,
})
}
async directoryExists(path: string, options: MiscellaneousOptions): Promise<boolean> {
maybeAbort(options.abortSignal);
const listing = this.container.listBlobsFlat({
prefix: this.prefixer.prefixDirectoryPath(path),
abortSignal: options.abortSignal,
}).byPage({
maxPageSize: 1,
});
return (await listing.next()).value.segment.blobItems.length > 0;
}
async publicUrl(path: string, options?: PublicUrlOptions): Promise<string> {
return this.blockClient(path).url;
}
async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string> {
return await this.blockClient(path).generateSasUrl({
expiresOn: normalizeExpiryToDate(options.expiresAt),
permissions: BlobSASPermissions.parse('r'),
...(this.options.temporaryUrlOptions ?? {}),
});
}
async prepareUpload(path: string, options: UploadRequestOptions): Promise<UploadRequest> {
const headers: UploadRequestHeaders = {};
headers['x-ms-blob-type'] = options['x-ms-blob-type'] ?? 'BlockBlob';
const config: BlobGenerateSasUrlOptions = {
expiresOn: normalizeExpiryToDate(options.expiresAt),
permissions: BlobSASPermissions.parse('w'),
...(this.options.temporaryUrlOptions ?? {}),
};
const contentType = options['Content-Type'] ?? options.contentType;
if (typeof contentType === 'string') {
config.contentType = contentType;
headers['Content-Type'] = contentType;
}
const url = await this.blockClient(path).generateSasUrl(config);
return {method: 'PUT', provider: 'azure-storage-blob', url, headers};
}
async checksum(path: string, options: ChecksumOptions): Promise<string> {
maybeAbort(options?.abortSignal);
const algo = options.algo ?? 'etag';
if (algo !== 'etag') {
throw ChecksumIsNotAvailable.checksumNotSupported(algo);
}
const blob = this.blockClient(path);
const properties = await blob.getProperties({abortSignal: options.abortSignal});
const etag = properties.etag;
if (etag === undefined) {
throw new Error('Etag is not defined on blob properties.');
}
return etag;
}
async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
const stat = await this.stat(path, options);
if (stat.isDirectory) {
throw new Error('Path is not a file. No mimetype available.');
}
if (stat.mimeType === undefined) {
throw new Error('Mime-type not found for file.');
}
return stat.mimeType;
}
async lastModified(path: string): Promise<number> {
const stat = await this.stat(path);
if (stat.isDirectory) {
throw new Error('Path is not a file. No last modified available.');
}
if (stat.lastModifiedMs === undefined) {
throw new Error('Last modified not found for file.');
}
return stat.lastModifiedMs;
}
async fileSize(path: string): Promise<number> {
const stat = await this.stat(path);
if (stat.isDirectory) {
throw new Error('Path is not a file. No file size available.');
}
if (stat.size === undefined) {
throw new Error('File size not found for file.');
}
return stat.size;
}
private async resolveMimetype(path: string, contents: Readable, options: WriteOptions): Promise<[string, Readable]> {
if (options.mimeType) {
return [options.mimeType, contents];
}
const [mimeType, stream] = await resolveMimeType(path, contents);
return [mimeType ?? 'application/octet-stream', stream];
}
}
/**
* BC export
*
* @deprecated
*/
export class AzureStorageBlobFileStorage extends AzureStorageBlobStorageAdapter {}
================================================
FILE: packages/azure-storage-blob/src/index.ts
================================================
export * from './azure-storage-blob.js';
================================================
FILE: packages/azure-storage-blob/tsconfig.json
================================================
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist/"
},
"include": [
"src/**/*.ts"
]
}
================================================
FILE: packages/chaos/.npmignore
================================================
src
!dist
tsconfig.json
================================================
FILE: packages/chaos/LICENSE
================================================
Copyright (c) 2023-2024 Frank de Jonge
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: packages/chaos/README.md
================================================
<img src="https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg" width="50px" height="50px" />
# Flystorage adapter for chaos engineering
This package contains an adapter decorator that causes instrumented failures.
## Installation
Install all the required packages
```bash
npm install --save @flystorage/file-storage @flystorage/chaos
```
## Setup
```typescript
import {FileStorage} from '@flystorage/file-storage';
import {ChaosStorageAdapterDecorator, TriggeredErrors} from '@flystorage/chaos';
const strategy = new TriggeredErrors();
const adapter = new ChaosStorageAdapterDecorator(
createActualAdapter(),
strategy,
);
const storage = new FileStorage(adapter);
```
> ⚠️ Always use the FileStorage, it is essential for security and a good developer
> experience. Do not use the adapter directly.
## Usage
```typescript
import {TriggeredErrors} from '@flystorage/chaos';
const strategy = new TriggeredErrors();
// error on all write calls
strategy.on('write', () => new Error());
// error on first 2 stat calls
strategy.on('stat', () => new Error(), {times: 2});
// error after first 2 deleteFile calls
strategy.on('deleteFile', () => new Error(), {after: 2});
// error on 2nd and 3rd call to any method
strategy.on('*', () => new Error(), {after: 1, times: 2});
```
================================================
FILE: packages/chaos/changelog.md
================================================
# `@flystorage/chaos`
## 1.1.0
### Changes
- Added AbortSignal support
## 1.0.0
### Major Changes
- First 1.0.0 release
================================================
FILE: packages/chaos/package.json
================================================
{
"name": "@flystorage/chaos",
"type": "module",
"version": "1.1.0",
"dependencies": {
"@flystorage/file-storage": "^1.1.0"
},
"description": "A storage adapter decorator with the ability to stage errors.",
"main": "./dist/cjs/index.js",
"types": "./dist/types/index.d.ts",
"exports": {
".": {
"import": {
"types": "./dist/types/index.d.ts",
"default": "./dist/esm/index.js"
},
"require": {
"types": "./dist/types/index.d.ts",
"default": "./dist/cjs/index.js"
}
}
},
"scripts": {
"compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json",
"compile:esm": "tsc --outDir ./dist/esm/ --declaration false",
"compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node",
"compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly",
"watch": "tsc --watch"
},
"keywords": [
"testing",
"chaos",
"resillience",
"flystorage",
"filesystem"
],
"author": "Frank de Jonge (https://frankdejonge.nl)",
"homepage": "https://flystorage.dev/tools/chaos/",
"repository": {
"type": "git",
"url": "git+https://github.com/duna-oss/flystorage.git",
"directory": "packages/chaos"
},
"license": "MIT"
}
================================================
FILE: packages/chaos/src/index.test.ts
================================================
import { FileStorage } from "@flystorage/file-storage";
import {InMemoryStorageAdapter} from '@flystorage/in-memory';
import {AlwaysThrowError, ChaosStorageAdapterDecorator, NeverThrowError, TriggeredErrors} from './index.js';
describe('chaos adapter decorator', () => {
describe('triggered strategy', () => {
const strategy = new TriggeredErrors();
const adapter = new InMemoryStorageAdapter();
afterEach(() => {
strategy.clearTriggers();
adapter.deleteEverything();
});
const storage = new FileStorage(
new ChaosStorageAdapterDecorator(
adapter,
strategy,
),
);
test('trigger immediately on write', async () => {
strategy.on('write', () => new Error('Oh no...'));
await expect(storage.write('path.txt', 'context')).rejects.toThrow();
});
test('trigger immediately after 3 writes', async () => {
strategy.on('write', () => new Error('Oh no...'), {after: 3});
await storage.write('path.txt', 'context');
await storage.write('path.txt', 'context');
await storage.write('path.txt', 'context');
await expect(storage.write('path.txt', 'context')).rejects.toThrow();
});
test('trigger for 2 stat calls after 1 successful call', async () => {
await storage.write('path.txt', 'contents');
strategy.on('stat', () => new Error('Oh no...'), {times: 2, after: 1});
await storage.stat('path.txt');
await expect(storage.stat('path.txt')).rejects.toThrow();
await expect(storage.stat('path.txt')).rejects.toThrow();
await storage.stat('path.txt');
});
test('triggering on any method call with: *', async () => {
await storage.write('path.txt', 'contents');
strategy.on('*', () => new Error('Oh no...'), {times: 2, after: 1});
await storage.stat('path.txt');
await expect(storage.fileSize('path.txt')).rejects.toThrow();
await expect(storage.mimeType('path.txt')).rejects.toThrow();
await storage.deleteFile('path.txt');
});
});
describe('always throwing an error', () => {
const strategy = new AlwaysThrowError(() => new Error('Oh no...'));
const adapter = new InMemoryStorageAdapter();
afterEach(() => adapter.deleteEverything());
const storage = new FileStorage(
new ChaosStorageAdapterDecorator(
adapter,
strategy,
),
);
test('on any call', async () => {
await expect(storage.write('path.txt', 'contents')).rejects.toThrow();
});
});
describe('never throwing an error', () => {
const strategy = new NeverThrowError();
const adapter = new InMemoryStorageAdapter();
afterEach(() => adapter.deleteEverything());
const storage = new FileStorage(
new ChaosStorageAdapterDecorator(
adapter,
strategy,
),
);
test('on any call', async () => {
await storage.write('path.txt', 'contents');
expect(await storage.readToString('path.txt')).toEqual('contents');
});
});
});
================================================
FILE: packages/chaos/src/index.ts
================================================
import {
ChecksumOptions,
CopyFileOptions,
CreateDirectoryOptions,
FileContents,
MimeTypeOptions,
MiscellaneousOptions,
MoveFileOptions,
StatEntry,
StorageAdapter,
TemporaryUrlOptions,
WriteOptions,
} from '@flystorage/file-storage';
import {Readable} from 'stream';
export type AnyAdapterMethodName = keyof StorageAdapter;
export interface ChaosStrategy {
maybeGoNuts(method: AnyAdapterMethodName): void;
}
export class AlwaysThrowError implements ChaosStrategy {
constructor(private readonly newError: () => Error) {
}
maybeGoNuts(method: keyof StorageAdapter): void {
throw (this.newError)();
}
}
export class NeverThrowError implements ChaosStrategy {
maybeGoNuts(method: keyof StorageAdapter): void {
// do not do anything
}
}
export class TriggeredErrors implements ChaosStrategy {
private triggers: {
[I in AnyAdapterMethodName | '*']?: {
after: number,
times: number,
newError: () => unknown,
}
} = {};
on(method: AnyAdapterMethodName | '*', newError: () => unknown, options: {after?: number, times?: number} = {}) {
this.triggers[method] = {
after: options.after ?? 0,
times: options.times ?? Number.MAX_SAFE_INTEGER,
newError,
};
}
clearTriggers(): void {
this.triggers = {};
}
maybeGoNuts(method: keyof StorageAdapter): void {
const trigger = this.triggers[method] ?? this.triggers['*'];
if (trigger === undefined) {
return;
}
if (trigger.after > 0) {
trigger.after--;
} else if (trigger.times > 0) {
trigger.times--;
throw (trigger.newError)();
}
}
}
export class ChaosStorageAdapterDecorator implements StorageAdapter {
constructor(
private readonly storage: StorageAdapter,
private readonly chaos: ChaosStrategy,
) {
}
write(path: string, contents: Readable, options: WriteOptions): Promise<void> {
this.chaos.maybeGoNuts('write');
return this.storage.write(path, contents, options);
}
read(path: string, options: MiscellaneousOptions): Promise<FileContents> {
this.chaos.maybeGoNuts('read');
return this.storage.read(path, options);
}
deleteFile(path: string, options: MiscellaneousOptions): Promise<void> {
this.chaos.maybeGoNuts('deleteFile');
return this.storage.deleteFile(path, options);
}
createDirectory(path: string, options: CreateDirectoryOptions): Promise<void> {
this.chaos.maybeGoNuts('createDirectory');
return this.storage.createDirectory(path, options);
}
copyFile(from: string, to: string, options: CopyFileOptions): Promise<void> {
this.chaos.maybeGoNuts('copyFile');
return this.storage.copyFile(from, to, options);
}
moveFile(from: string, to: string, options: MoveFileOptions): Promise<void> {
this.chaos.maybeGoNuts('moveFile');
return this.storage.moveFile(from, to, options);
}
stat(path: string, options: MiscellaneousOptions): Promise<StatEntry> {
this.chaos.maybeGoNuts('stat');
return this.storage.stat(path, options);
}
list(path: string, options: { deep: boolean; }): AsyncGenerator<StatEntry, any, unknown> {
this.chaos.maybeGoNuts('list');
return this.storage.list(path, options);
}
changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise<void> {
this.chaos.maybeGoNuts('changeVisibility');
return this.storage.changeVisibility(path, visibility, options);
}
visibility(path: string, options: MiscellaneousOptions): Promise<string> {
this.chaos.maybeGoNuts('visibility');
return this.storage.visibility(path, options);
}
deleteDirectory(path: string, options: MiscellaneousOptions): Promise<void> {
this.chaos.maybeGoNuts('deleteDirectory');
return this.storage.deleteDirectory(path, options);
}
fileExists(path: string, options: MiscellaneousOptions): Promise<boolean> {
this.chaos.maybeGoNuts('fileExists');
return this.storage.fileExists(path, options);
}
directoryExists(path: string, options: MiscellaneousOptions): Promise<boolean> {
this.chaos.maybeGoNuts('directoryExists');
return this.storage.directoryExists(path, options);
}
publicUrl(path: string, options: MiscellaneousOptions): Promise<string> {
this.chaos.maybeGoNuts('publicUrl');
return this.storage.publicUrl(path, options);
}
temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string> {
this.chaos.maybeGoNuts('publicUrl');
return this.storage.publicUrl(path, options);
}
checksum(path: string, options: ChecksumOptions): Promise<string> {
this.chaos.maybeGoNuts('checksum');
return this.storage.checksum(path, options);
}
mimeType(path: string, options: MimeTypeOptions): Promise<string> {
this.chaos.maybeGoNuts('mimeType');
return this.storage.mimeType(path, options);
}
lastModified(path: string, options: MiscellaneousOptions): Promise<number> {
this.chaos.maybeGoNuts('lastModified');
return this.storage.lastModified(path, options);
}
fileSize(path: string, options: MiscellaneousOptions): Promise<number> {
this.chaos.maybeGoNuts('fileSize');
return this.storage.fileSize(path, options);
}
}
================================================
FILE: packages/chaos/tsconfig.json
================================================
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist/"
},
"include": [
"src/**/*.ts"
]
}
================================================
FILE: packages/dynamic-import/.npmignore
================================================
index.test.ts
tsconfig.json
src/
================================================
FILE: packages/dynamic-import/CHANGELOG.md
================================================
# `@flystorage/dynamic-import`
## 1.0.0
### Major Changes
- First 1.0.0 release
================================================
FILE: packages/dynamic-import/README.md
================================================
<img src="https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg" width="50px" height="50px" />
# Utility for dynamically importing ESM code
The TypeScript compiler turns dynamic import statements into require calls. This breaks importing
ESM packages in CommonJS code.
This utility helps to work around this by provided a JavaScript function to do the actual importing.
## Installation
Install all the required packages
```bash
npm install --save @flystorage/dynamic-import
```
## Usage
```typescript
const {dynamicallyImport} = require('@flystorage/dynamic-import');
async function useFileType() {
const {fileTypeFromBuffer} = await dynamicallyImport<typeof import('file-type')>('file-type');
}
useFileType();
```
================================================
FILE: packages/dynamic-import/index.d.ts
================================================
export declare function dynamicallyImport<Pkg>(path: string): Promise<Pkg>;
================================================
FILE: packages/dynamic-import/index.js
================================================
async function dynamicallyImport(path) {
return await import(path);
};
exports.dynamicallyImport = dynamicallyImport;
================================================
FILE: packages/dynamic-import/index.test.ts
================================================
import {dynamicallyImport} from './index.js';
// @ts-ignore
type FileType = typeof import('file-type');
describe('dynamic-import', () => {
test('dynamic-import', async () => {
const fileType = await dynamicallyImport<FileType>('file-type');
expect(fileType.supportedExtensions.has('jpg')).toEqual(true);
});
})
================================================
FILE: packages/dynamic-import/package.json
================================================
{
"name": "@flystorage/dynamic-import",
"type": "commonjs",
"version": "1.0.0",
"description": "Utility to dynamically import ESM packages from CommonJS without tsc messing things up.",
"main": "./index.js",
"types": "./index.d.ts",
"exports": {
".": {
"import": {
"types": "./index.d.ts",
"default": "./index.js"
},
"require": {
"types": "./index.d.ts",
"default": "./index.js"
}
}
},
"scripts": {},
"author": "Frank de Jonge (https://frankdejonge.nl)",
"repository": {
"type": "git",
"url": "git+https://github.com/duna-oss/flystorage.git",
"directory": "packages/dynamic-import"
},
"keywords": [],
"license": "MIT"
}
================================================
FILE: packages/dynamic-import/src/index.ts
================================================
//
export * from '../index.js';
================================================
FILE: packages/file-storage/.npmignore
================================================
src
!dist
tsconfig.json
================================================
FILE: packages/file-storage/CHANGELOG.md
================================================
# `@flystorage/file-storage`
## 1.2.2
### Fixes
- Fixed typo in timeout options name
## 1.2.1
### Fixes
- Prevent unprefixed root paths to become '/' for directory listings.
## 1.2.0
### Added
- Ability to ignore visibility using configuration
- Support Bun runtime
## 1.1.0
### Added
- AbortSignal Support
- Normalised over stream errors for reads
- Introduce way to detect failed reads because of missing files.
## 1.0.1
### Fixes
- Corrected error message for `UnableToGetChecksum` error.
## 1.0.0
### Major Changes
- First 1.0.0 release
================================================
FILE: packages/file-storage/LICENSE
================================================
Copyright (c) 2023-2024 Frank de Jonge
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: packages/file-storage/README.md
================================================
<img src="https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg" width="50px" height="50px" />
# Flystorage
Flystorage is a file storage abstraction for NodeJS and TypeScript. It is an 80/20 solution
that is built around a set of goals:
- Provide a straight-forward API that is easy to use.
- Allow application code to be unaware WHERE files are stored.
- Pragmatically smooth over underlying storage differences.
- Expose an async/await based API, promises all the way.
- Abstract over file permissions using "visibility".
- Actually tested using real integrations, _mocks are not welcome_.
- Stand on the shoulders of giants, use official vendor packages when possible.
### What is Flystorage NOT:
Flystorage is meant to be used in cases for generic file storage use-cases. It's not an API for
any specific filesystem. It's a generalised solution and will not implement feature only
specific to one particular storage implementation. There will be use-cases that are not catered
to, simply because they cannot be abstracted over in a reasonable manner.
## Capabilities
- Write files using string | buffer | readable/stream
- Read files as stream, string, or Uint8Array
- List the contents of a directory/prefix, (shallow and deep).
- Metadata, such as; mime-type, last modified timestamp, and file size.
- Expose or calculate checksums for files.
- Delete files without failing when they don't exist.
- Set permissions using abstracted visibility
- Delete directories (and any files it contains)
- Generate public URLs.
- Generate temporary (signed) URLs.
- Secure direct uploads from the browser.
- Moving files
- Copying files
## Adapters
- [Local Filesystem](https://www.npmjs.com/package/@flystorage/local-fs)
- [AWS S3 (using the V3 SDK)](https://www.npmjs.com/package/@flystorage/aws-s3)
- [Azure Blob Storage](https://www.npmjs.com/package/@flystorage/azure-storage-blob)
- [Test implementation (in-memory)](https://www.npmjs.com/package/@flystorage/in-memory)
- [Google Cloud Storage](https://www.npmjs.com/package/@flystorage/google-cloud-storage)
- [Chaos adapter decorator](https://www.npmjs.com/package/@flystorage/chaos)
## Usage
Install the main package and any adapters you might need:
```bash
npm i -S @flystorage/file-storage
# for using AWS S3
npm i -S @flystorage/aws-s3
# for using the local filesystem
npm i -S @flystorage/local-fs
```
## Local Usage
```typescript
import {resolve} from 'node:path';
import {createReadStream} from 'node:fs';
import {FileStorage, Visibility} from '@flystorage/file-storage';
import {LocalStorageAdapter} from '@flystorage/local-fs';
/**
* SETUP
**/
const rootDirectory = resolve(process.cwd(), 'my-files');
const storage = new FileStorage(new LocalStorageAdapter(rootDirectory));
/**
* USAGE
**/
// Write using a string
await storage.write('write-from-a-string.txt', 'file contents');
// Write using a stream
const stream = createReadStream(resolve(process.cwd(), 'test-files/picture.png'));
await storage.write('picture.png', stream);
// Write with visibility (permissions).
await storage.write('public.txt', 'debug', {
visibility: Visibility.PUBLIC, // mode: 0o644
});
await storage.write('private.txt', 'debug', {
visibility: Visibility.PRIVATE, // mode: 0o600
});
// List directory contents
const contentsAsAsyncGenerator = storage.list('', {deep: true});
for await (const item of contentsAsAsyncGenerator) {
console.log(item.path);
if (item.isFile) {
// do something with the file
} else if (item.isDirectory) {
// do something with the directory
}
}
// Delete a file
await storage.deleteFile('some-file.txt');
// Delete a directory (with all contents)
await storage.deleteDirectory('some-directory');
```
## Author
Flystorage is built by the maintainer of [Flysystem](https://flysystem.thephpleague.com), a
filesystem abstraction for PHP. This brings along over
a decade of filesystem abstraction experience.
================================================
FILE: packages/file-storage/package.json
================================================
{
"name": "@flystorage/file-storage",
"type": "module",
"version": "1.2.2",
"description": "File-storage abstraction: multiple filesystems, one API.",
"main": "./dist/cjs/index.js",
"types": "./dist/types/index.d.ts",
"exports": {
".": {
"import": {
"types": "./dist/types/index.d.ts",
"default": "./dist/esm/index.js"
},
"require": {
"types": "./dist/types/index.d.ts",
"default": "./dist/cjs/index.js"
}
}
},
"scripts": {
"compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json",
"compile:esm": "tsc --outDir ./dist/esm/ --declaration false",
"compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node",
"compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly",
"watch": "tsc --watch"
},
"author": "Frank de Jonge (https://frankdejonge.nl)",
"repository": {
"type": "git",
"url": "git+https://github.com/duna-oss/flystorage.git",
"directory": "packages/file-storage"
},
"homepage": "https://flystorage.dev/",
"keywords": [
"fs",
"file",
"files",
"filesystem",
"filesystems",
"storage"
],
"license": "MIT"
}
================================================
FILE: packages/file-storage/src/checksum-from-stream.ts
================================================
import {BinaryToTextEncoding, createHash} from 'crypto';
import {Readable} from 'node:stream';
import {TextEncoder} from 'util';
const encoder = new TextEncoder();
export async function checksumFromStream(
stream: Readable,
options: { algo?: string; encoding?: BinaryToTextEncoding }
): Promise<string> {
return new Promise((resolve, reject) => {
const hash = createHash(options.algo ?? 'md5');
stream.on('error', err => reject(err));
stream.on('data', (chunk: Uint8Array | string | number) => {
const type = typeof chunk;
if (type === 'string') {
chunk = encoder.encode(chunk as string);
} else if (type === 'number') {
chunk = new Uint8Array([chunk as number]);
}
hash.update(chunk as Uint8Array);
});
stream.on('end', () => resolve(hash.digest(options.encoding ?? 'hex')));
});
}
================================================
FILE: packages/file-storage/src/errors.ts
================================================
export type ErrorContext = { [index: string]: any };
export type OperationError = Error & {
readonly code: string,
readonly context: ErrorContext,
}
export function errorToMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
export abstract class FlystorageError extends Error implements OperationError {
readonly code: string = 'unknown_error';
constructor(
message: string,
public readonly context: ErrorContext = {},
cause: unknown = undefined,
) {
const options = cause === undefined ? undefined : {cause};
// @ts-ignore TS2554
super(message, options);
}
}
/**
* Thrown when the checksum algo is not supported or not pre-computed. This error
* is thrown with the intention of falling back to computing it based on a file read.
*/
export class ChecksumIsNotAvailable extends FlystorageError {
public readonly code = 'flystorage.checksum_not_supported';
constructor(
message: string,
public readonly algo: string,
context: ErrorContext = {},
cause: unknown = undefined,
) {
super(message, context, cause);
}
static checksumNotSupported = (algo: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
} = {}) => new ChecksumIsNotAvailable(
`Checksum algo "${algo}" is not supported`,
algo,
{...context, algo},
cause,
);
static isErrorOfType(error: unknown): error is ChecksumIsNotAvailable {
return (typeof error === 'object' && (error as any).code === 'flystorage.checksum_not_supported');
}
}
export class UnableToGetChecksum extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_checksum';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetChecksum(
`Unable to get checksum for file. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToGetMimeType extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_mimetype';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetMimeType(
`Unable to get mime-type. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToGetLastModified extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_last_modified';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetLastModified(
`Unable to get last modified. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToGetFileSize extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_file_size';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetFileSize(
`Unable to get file size. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToWriteFile extends FlystorageError {
public readonly code = 'flystorage.unable_to_write_file';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToWriteFile(
`Unable to write the file. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToReadFile extends FlystorageError {
public readonly code = 'flystorage.unable_to_read_file';
constructor(
public readonly wasFileNotFound: boolean,
message: string,
public readonly context: ErrorContext = {},
cause: unknown = undefined,
) {
super(message, context, cause);
}
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToReadFile(
false,
`Unable to read the file. Reason: ${reason}`,
context,
cause,
);
static becauseFileWasNotFound = (error: FileWasNotFound) => new UnableToReadFile(
true,
`Unable to read the file. Reason: ${error.message}`,
error.context,
error,
);
}
export class FileWasNotFound extends FlystorageError {
public readonly code = 'flystorage.file_was_not_found';
static atLocation = (location: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new FileWasNotFound(
`File was not found at location: ${location}`,
context,
cause,
);
}
export function isFileWasNotFound(error: unknown): error is FileWasNotFound {
return (typeof error === 'object' && (error as any).code === 'flystorage.file_was_not_found');
}
export class UnableToSetVisibility extends FlystorageError {
public readonly code = 'flystorage.unable_to_set_visibility';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToSetVisibility(
`Unable to set visibility. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToGetVisibility extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_visibility';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetVisibility(
`Unable to get visibility. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToGetPublicUrl extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_public_url';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetPublicUrl(
`Unable to get public URL. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToGetTemporaryUrl extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_temporary_url';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetTemporaryUrl(
`Unable to get temporary URL. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToPrepareUploadRequest extends FlystorageError {
public readonly code = 'flystorage.unable_to_prepare_upload_request';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetTemporaryUrl(
`Unable to prepare upload request. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToCopyFile extends FlystorageError {
public readonly code = 'flystorage.unable_to_copy_file';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToCopyFile(
`Unable to copy file. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToMoveFile extends FlystorageError {
public readonly code = 'flystorage.unable_to_move_file';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToMoveFile(
`Unable to move file. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToGetStat extends FlystorageError {
public readonly code = 'flystorage.unable_to_get_stat';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetStat(
`Unable to get stat. Reason: ${reason}`,
context,
cause,
);
static noFileStatResolved = ({context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToGetStat(
`Stat was not a file.`,
context,
cause,
);
}
export class UnableToCreateDirectory extends FlystorageError {
public readonly code = 'flystorage.unable_to_create_directory';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToCreateDirectory(
`Unable to create directory. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToDeleteDirectory extends FlystorageError {
public readonly code = 'flystorage.unable_to_delete_directory';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToDeleteDirectory(
`Unable to delete directory. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToDeleteFile extends FlystorageError {
public readonly code = 'flystorage.unable_to_delete_file';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToDeleteFile(
`Unable to delete file. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToCheckFileExistence extends FlystorageError {
public readonly code = 'flystorage.unable_to_check_file_existence';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToCheckFileExistence(
`Unable to check file existence. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToCheckDirectoryExistence extends FlystorageError {
public readonly code = 'flystorage.unable_to_check_directory_existence';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToCheckDirectoryExistence(
`Unable to check directory existence. Reason: ${reason}`,
context,
cause,
);
}
export class UnableToListDirectory extends FlystorageError {
public readonly code = 'flystorage.unable_to_list_directory_contents';
static because = (reason: string, {context = {}, cause = undefined}: {
context?: ErrorContext,
cause?: unknown
}) => new UnableToListDirectory(
`Unable to list directory contents. Reason: ${reason}`,
context,
cause,
);
}
================================================
FILE: packages/file-storage/src/file-storage.test.ts
================================================
import {FileStorage, UploadRequest, UploadRequestOptions} from './file-storage.js';
import {InMemoryStorageAdapter} from '@flystorage/in-memory';
import {createHash} from 'crypto';
describe('FileStorage', () => {
test('calculating a checksum through a fallback', async () => {
const hash = createHash('md5');
hash.update('contents');
const expectedChecksum = hash.digest('hex');
const storage = new FileStorage(new InMemoryStorageAdapter());
await storage.write('something.txt', 'contents');
const checksum = await storage.checksum('something.txt', {algo: 'md5'});
expect(checksum).toEqual(expectedChecksum);
});
test('supplying a prepared upload strategy', async () => {
const storage = new FileStorage(
new InMemoryStorageAdapter(),
undefined,
{
preparedUploadStrategy: {
async prepareUpload(path: string, options: UploadRequestOptions): Promise<UploadRequest> {
return {
method: 'POST',
url: `https://here.com/${path}`,
headers: options.headers ?? {},
};
}
}
}
);
const request = await storage.prepareUpload('here.txt', {
expiresAt: 0,
headers: {
'content-type': 'application/json',
}
});
expect(request).toEqual({
method: 'POST',
url: 'https://here.com/here.txt',
headers: {
'content-type': 'application/json',
},
});
});
});
================================================
FILE: packages/file-storage/src/file-storage.ts
================================================
import {BinaryToTextEncoding} from 'crypto';
import {Readable} from 'stream';
import {checksumFromStream} from './checksum-from-stream.js';
import {PathNormalizer, PathNormalizerV1} from './path-normalizer.js';
import {TextEncoder} from 'util';
import {
ChecksumIsNotAvailable,
errorToMessage,
isFileWasNotFound,
UnableToCheckDirectoryExistence,
UnableToCheckFileExistence,
UnableToCopyFile,
UnableToCreateDirectory,
UnableToDeleteDirectory,
UnableToDeleteFile,
UnableToGetChecksum,
UnableToGetFileSize,
UnableToGetLastModified,
UnableToGetMimeType,
UnableToGetPublicUrl,
UnableToGetStat,
UnableToGetTemporaryUrl,
UnableToGetVisibility,
UnableToListDirectory,
UnableToMoveFile,
UnableToPrepareUploadRequest,
UnableToReadFile,
UnableToSetVisibility,
UnableToWriteFile,
} from './errors.js';
import {PassThrough} from 'node:stream';
export type CommonStatInfo = Readonly<{
path: string,
lastModifiedMs?: number,
visibility?: string,
}>;
export type FileInfo = Readonly<{
type: 'file',
size?: number,
isFile: true,
isDirectory: false,
mimeType?: string,
} & CommonStatInfo>;
export type DirectoryInfo = Readonly<{
type: 'directory',
isFile: false,
isDirectory: true,
} & CommonStatInfo>;
export function isFile(stat: StatEntry): stat is FileInfo {
return stat.isFile;
}
export function isDirectory(stat: StatEntry): stat is DirectoryInfo {
return stat.isDirectory;
}
export type StatEntry = FileInfo | DirectoryInfo;
export type AdapterListOptions = ListOptions & { deep: boolean };
export interface StorageAdapter {
write(path: string, contents: Readable, options: WriteOptions): Promise<void>;
read(path: string, options: MiscellaneousOptions): Promise<FileContents>;
deleteFile(path: string, options: MiscellaneousOptions): Promise<void>;
createDirectory(path: string, options: CreateDirectoryOptions): Promise<void>;
copyFile(from: string, to: string, options: CopyFileOptions): Promise<void>;
moveFile(from: string, to: string, options: MoveFileOptions): Promise<void>;
stat(path: string, options: MiscellaneousOptions): Promise<StatEntry>;
list(path: string, options: AdapterListOptions): AsyncGenerator<StatEntry>;
changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise<void>;
visibility(path: string, options: MiscellaneousOptions): Promise<string>;
deleteDirectory(path: string, options: MiscellaneousOptions): Promise<void>;
fileExists(path: string, options: MiscellaneousOptions): Promise<boolean>;
directoryExists(path: string, options: MiscellaneousOptions): Promise<boolean>;
publicUrl(path: string, options: PublicUrlOptions): Promise<string>;
temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string>;
prepareUpload?(path: string, options: UploadRequestOptions): Promise<UploadRequest>;
checksum(path: string, options: ChecksumOptions): Promise<string>;
mimeType(path: string, options: MimeTypeOptions): Promise<string>;
lastModified(path: string, options: MiscellaneousOptions): Promise<number>;
fileSize(path: string, options: MiscellaneousOptions): Promise<number>;
}
export class DirectoryListing implements AsyncIterable<StatEntry> {
constructor(
private readonly listing: AsyncGenerator<StatEntry>,
private readonly path: string,
private readonly deep: boolean,
) {
}
async toArray(sorted: boolean = true): Promise<StatEntry[]> {
const items = [];
for await (const item of this.listing) {
items.push(item);
}
return sorted ? items.sort((a, b) => naturalSorting.compare(a.path, b.path)) : items;
}
filter(filter: (entry: StatEntry) => boolean): DirectoryListing {
const listing = this.listing;
const filtered = (async function* () {
for await (const entry of listing) {
if (filter(entry)) {
yield entry;
}
}
})();
return new DirectoryListing(filtered, this.path, this.deep);
}
async* [Symbol.asyncIterator]() {
try {
for await (const item of this.listing) {
yield item;
}
} catch (error) {
throw UnableToListDirectory.because(
errorToMessage(error),
{cause: error, context: {path: this.path, deep: this.deep}},
);
}
}
}
export type FileContents = Iterable<any> | AsyncIterable<any> | NodeJS.ReadableStream | Readable | string;
export type TimeoutOptions = { timeout?: number };
export type MiscellaneousOptions = TimeoutOptions & {
[option: string]: any,
abortSignal?: AbortSignal,
}
export type MimeTypeOptions = MiscellaneousOptions & {
disallowFallback?: boolean,
fallbackMethod?: 'contents' | 'path',
}
export type VisibilityFallback = {
strategy: 'ignore',
stagedVisibilityResponse?: string,
} | {
strategy: 'error',
errorMessage?: string,
}
export type VisibilityOptions = MiscellaneousOptions & {
visibility?: string,
directoryVisibility?: string,
retainVisibility?: boolean,
} & ({
useVisibility: true,
} | {
useVisibility: false,
visibilityFallback: VisibilityFallback,
} | {});
export type WriteOptions = VisibilityOptions & MiscellaneousOptions & {
mimeType?: string,
size?: number,
cacheControl?: string,
};
export type CreateDirectoryOptions = MiscellaneousOptions & Pick<VisibilityOptions, 'directoryVisibility'> & {};
export type PublicUrlOptions = MiscellaneousOptions & {};
export type UploadRequestOptions = MiscellaneousOptions & {
expiresAt: ExpiresAt,
contentType?: string,
headers?: UploadRequestHeaders,
};
export type CopyFileOptions = MiscellaneousOptions & VisibilityOptions & {
retainVisibility?: boolean,
};
export type MoveFileOptions = MiscellaneousOptions & VisibilityOptions & {
retainVisibility?: boolean,
};
export type ListOptions = MiscellaneousOptions & { deep?: boolean };
export type TemporaryUrlOptions = MiscellaneousOptions & {
expiresAt: ExpiresAt,
responseHeaders?: { [header: string]: string },
};
export type ChecksumOptions = MiscellaneousOptions & {
algo?: string,
encoding?: BinaryToTextEncoding,
}
export type ConfigurationOptions = {
visibility?: VisibilityOptions,
writes?: WriteOptions,
moves?: MoveFileOptions,
copies?: CopyFileOptions,
publicUrls?: PublicUrlOptions,
temporaryUrls?: TemporaryUrlOptions,
uploadRequest?: UploadRequestOptions,
checksums?: ChecksumOptions,
mimeTypes?: MimeTypeOptions,
preparedUploadStrategy?: PreparedUploadStrategy,
timeout?: TimeoutOptions,
list?: ListOptions,
}
export function toReadable(contents: FileContents): Readable {
if (contents instanceof Readable) {
return contents;
}
return Readable.from(contents);
}
const naturalSorting = new Intl.Collator(undefined, {
numeric: true,
sensitivity: 'base',
});
function instrumentAbortSignal<Options extends MiscellaneousOptions>(options: Options): Options {
let abortSignal = options.abortSignal;
if (options.timeout !== undefined) {
const timeoutAbort = AbortSignal.timeout(options.timeout);
if (options.abortSignal) {
const originalAbortSignal = options.abortSignal;
abortSignal = AbortSignal.any([
originalAbortSignal,
timeoutAbort,
]);
} else {
abortSignal = timeoutAbort;
}
}
if (abortSignal?.aborted) {
throw abortSignal.reason;
}
return {...options, abortSignal};
}
export class FileStorage {
constructor(
private readonly adapter: StorageAdapter,
private readonly pathNormalizer: PathNormalizer = new PathNormalizerV1(),
private readonly options: ConfigurationOptions = {},
) {
}
public async write(path: string, contents: FileContents, options: WriteOptions = {}): Promise<void> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.writes, ...options});
try {
const body = toReadable(contents);
await this.adapter.write(
this.pathNormalizer.normalizePath(path),
body,
options,
);
await closeReadable(body);
} catch (error) {
throw UnableToWriteFile.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
public async read(path: string, options: MiscellaneousOptions = {}): Promise<Readable> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
const stream = Readable.from(
await this.adapter.read(this.pathNormalizer.normalizePath(path), options),
);
const streamOut = new PassThrough();
stream.on('error', (error) => {
stream.unpipe(streamOut);
streamOut.destroy(
isFileWasNotFound(error)
? UnableToReadFile.becauseFileWasNotFound(error)
: error,
);
});
stream.pipe(streamOut);
return streamOut;
} catch (error) {
if (isFileWasNotFound(error)) {
throw UnableToReadFile.becauseFileWasNotFound(error);
}
throw UnableToReadFile.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public async readToString(path: string, options: MiscellaneousOptions = {}): Promise<string> {
return await readableToString(await this.read(path, options));
}
public async readToUint8Array(path: string, options: MiscellaneousOptions = {}): Promise<Uint8Array> {
return await readableToUint8Array(await this.read(path, options));
}
public async readToBuffer(path: string, options: MiscellaneousOptions = {}): Promise<Buffer> {
return Buffer.from(await this.readToUint8Array(path, options));
}
public async deleteFile(path: string, options: MiscellaneousOptions = {}): Promise<void> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
await this.adapter.deleteFile(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToDeleteFile.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public async createDirectory(path: string, options: CreateDirectoryOptions = {}): Promise<void> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...options});
try {
return await this.adapter.createDirectory(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToCreateDirectory.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
public async deleteDirectory(path: string, options: MiscellaneousOptions = {}): Promise<void> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
return await this.adapter.deleteDirectory(this.pathNormalizer.normalizePath(path), options);
} catch (error) {
throw UnableToDeleteDirectory.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public async stat(path: string, options: MiscellaneousOptions = {}): Promise<StatEntry> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
return await this.adapter.stat(this.pathNormalizer.normalizePath(path), options);
} catch (error) {
throw UnableToGetStat.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public async moveFile(from: string, to: string, options: MoveFileOptions = {}): Promise<void> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.moves, ...options});
try {
await this.adapter.moveFile(
this.pathNormalizer.normalizePath(from),
this.pathNormalizer.normalizePath(to),
options,
);
} catch (error) {
throw UnableToMoveFile.because(
errorToMessage(error),
{cause: error, context: {from, to}},
);
}
}
public async copyFile(from: string, to: string, options: CopyFileOptions = {}): Promise<void> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.copies, ...options});
try {
await this.adapter.copyFile(
this.pathNormalizer.normalizePath(from),
this.pathNormalizer.normalizePath(to),
options,
);
} catch (error) {
throw UnableToCopyFile.because(
errorToMessage(error),
{cause: error, context: {from, to}},
);
}
}
public async changeVisibility(path: string, visibility: string, options: VisibilityOptions = {}): Promise<void> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
if (options.useVisibility === false) {
const fallback: VisibilityFallback = options.visibilityFallback;
if (fallback.strategy === 'ignore') {
return;
} else if (fallback.strategy === 'error') {
throw UnableToSetVisibility.because(fallback.errorMessage ?? 'Configured not to use visibility', {
context: {path, visibility},
});
}
}
try {
return await this.adapter.changeVisibility(this.pathNormalizer.normalizePath(path), visibility, options);
} catch (error) {
throw UnableToSetVisibility.because(
errorToMessage(error),
{cause: error, context: {path, visibility}},
);
}
}
public async visibility(path: string, options: VisibilityOptions = {}): Promise<string> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...options});
if (options.useVisibility === false) {
const fallback: VisibilityFallback = options.visibilityFallback;
if (fallback.strategy === 'ignore') {
return fallback.stagedVisibilityResponse ?? 'unknown';
} else if (fallback.strategy === 'error') {
throw UnableToGetVisibility.because(fallback.errorMessage ?? 'Configured not to use visibility', {
context: {path},
});
}
}
try {
return await this.adapter.visibility(this.pathNormalizer.normalizePath(path), options);
} catch (error) {
throw UnableToGetVisibility.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public async fileExists(path: string, options: MiscellaneousOptions = {}): Promise<boolean> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
return await this.adapter.fileExists(this.pathNormalizer.normalizePath(path), options);
} catch (error) {
throw UnableToCheckFileExistence.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public list(path: string, options: ListOptions = {}): DirectoryListing {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.list, ...options});
const adapterOptions: AdapterListOptions = {
...options,
deep: options.deep ?? false,
};
return new DirectoryListing(
this.adapter.list(this.pathNormalizer.normalizePath(path), adapterOptions),
path,
adapterOptions.deep,
);
}
public async statFile(path: string, options: MiscellaneousOptions = {}): Promise<FileInfo> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
const stat = await this.stat(path, options);
if (isFile(stat)) {
return stat;
}
throw UnableToGetStat.noFileStatResolved({context: {path}});
}
public async directoryExists(path: string, options: MiscellaneousOptions = {}): Promise<boolean> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
return await this.adapter.directoryExists(this.pathNormalizer.normalizePath(path), options);
} catch (error) {
throw UnableToCheckDirectoryExistence.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public async publicUrl(path: string, options: PublicUrlOptions = {}): Promise<string> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.publicUrls, ...options});
try {
return await this.adapter.publicUrl(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToGetPublicUrl.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
public async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.temporaryUrls, ...options});
try {
return await this.adapter.temporaryUrl(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToGetTemporaryUrl.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
public async prepareUpload(path: string, options: UploadRequestOptions): Promise<UploadRequest> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.uploadRequest, ...options});
if (this.options.preparedUploadStrategy !== undefined) {
try {
return this.options.preparedUploadStrategy.prepareUpload(path, options);
} catch (error) {
throw UnableToPrepareUploadRequest.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
if (typeof this.adapter.prepareUpload !== 'function') {
throw new Error('The used adapter does not support prepared uploads.');
}
try {
return await this.adapter.prepareUpload(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToPrepareUploadRequest.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
public async checksum(path: string, options: ChecksumOptions = {}): Promise<string> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.checksums, ...options});
try {
return await this.adapter.checksum(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
if (ChecksumIsNotAvailable.isErrorOfType(error)) {
return this.calculateChecksum(path, options);
}
throw UnableToGetChecksum.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
public async mimeType(path: string, options: MimeTypeOptions = {}): Promise<string> {
options = instrumentAbortSignal({...this.options.timeout, ...this.options.mimeTypes, ...options});
try {
return await this.adapter.mimeType(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToGetMimeType.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
public async lastModified(path: string, options: MiscellaneousOptions = {}): Promise<number> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
return await this.adapter.lastModified(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToGetLastModified.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
public async fileSize(path: string, options: MiscellaneousOptions = {}): Promise<number> {
options = instrumentAbortSignal({...this.options.timeout, ...options});
try {
return await this.adapter.fileSize(
this.pathNormalizer.normalizePath(path),
options,
);
} catch (error) {
throw UnableToGetFileSize.because(
errorToMessage(error),
{cause: error, context: {path}},
);
}
}
private async calculateChecksum(path: string, options: ChecksumOptions): Promise<string> {
try {
return await checksumFromStream(await this.read(path, options), options);
} catch (error) {
throw UnableToGetChecksum.because(
errorToMessage(error),
{cause: error, context: {path, options}},
);
}
}
}
export type TimestampMs = number;
export type ExpiresAt = Date | TimestampMs;
export function normalizeExpiryToDate(expiresAt: ExpiresAt): Date {
return expiresAt instanceof Date ? expiresAt : new Date(expiresAt);
}
export function normalizeExpiryToMilliseconds(expiresAt: ExpiresAt): number {
return expiresAt instanceof Date ? expiresAt.getTime() : expiresAt;
}
export async function closeReadable(body: Readable) {
if (body.closed || body.destroyed) {
return;
}
await new Promise<void>((resolve, reject) => {
body.on('error', reject);
body.on('close', (err: any) => {
err ? reject(err) : resolve();
});
body.destroy();
});
}
const decoder = new TextDecoder();
export async function readableToString(stream: Readable): Promise<string> {
const contents = decoder.decode(await readableToUint8Array(stream));
await closeReadable(stream);
return contents;
}
export async function readableToBuffer(stream: Readable): Promise<Buffer> {
return new Promise<Buffer>((resolve, reject) => {
const buffers: Buffer[] = [];
stream.on('data', chunk => buffers.push(Buffer.from(chunk)));
stream.on('end', () => resolve(Buffer.concat(buffers)));
stream.on('finish', () => resolve(Buffer.concat(buffers)));
stream.on('error', err => reject(err));
});
}
const encoder = new TextEncoder();
export function readableToUint8Array(stream: Readable): Promise<Uint8Array> {
return new Promise((resolve, reject) => {
const parts: Uint8Array[] = [];
stream.on('data', (chunk: Uint8Array | string | number) => {
const type = typeof chunk;
if (type === 'string') {
chunk = encoder.encode(chunk as string);
} else if (type === 'number') {
chunk = new Uint8Array([chunk as number]);
}
parts.push(chunk as Uint8Array);
});
stream.on('error', reject);
stream.on('end', () => resolve(concatUint8Arrays(parts)));
});
}
function concatUint8Arrays(input: Uint8Array[]): Uint8Array {
const length = input.reduce((l, a) => l + (a.byteLength), 0);
const output = new Uint8Array(length);
let position = 0;
input.forEach(i => {
output.set(i, position);
position += i.byteLength;
});
return output;
}
export type UploadRequestHeaders = Record<string, string | ReadonlyArray<string>>;
export type UploadRequest = {
url: string,
provider?: string,
method: 'PUT' | 'POST'
headers: UploadRequestHeaders,
}
export interface PreparedUploadStrategy {
prepareUpload(path: string, options: UploadRequestOptions): Promise<UploadRequest>;
}
export class PreparedUploadsAreNotSupported implements PreparedUploadStrategy {
prepareUpload(): Promise<UploadRequest> {
throw new Error('The used adapter does not support prepared uploads.');
}
}
================================================
FILE: packages/file-storage/src/index.ts
================================================
export * from './checksum-from-stream.js';
export * from './file-storage.js';
export * from './errors.js';
export * from './path-normalizer.js';
export * from './path-prefixer.js'
export * from './portable-visibility.js';
================================================
FILE: packages/file-storage/src/path-normalizer.test.ts
================================================
import {CorruptedPathDetected, PathNormalizerV1, PathTraversalDetected} from './path-normalizer.js';
describe('PathNormalizerV1', () => {
const normalizer = new PathNormalizerV1();
test.each([
['.', ''],
['/path/to/dir/.', 'path/to/dir'],
['/dirname/', 'dirname'],
['dirname/..', ''],
['dirname/../', ''],
['dirname./', 'dirname.'],
['dirname/./', 'dirname'],
['dirname/.', 'dirname'],
['./dir/../././', ''],
['/something/deep/../../dirname', 'dirname'],
['00004869/files/other/10-75..stl', '00004869/files/other/10-75..stl'],
['/dirname//subdir///subsubdir', 'dirname/subdir/subsubdir'],
['example/path/..txt', 'example/path/..txt'],
['/example/../path.txt', 'path.txt'],
])('normalizing "%s" to "%s"', (path, expected) => {
expect(normalizer.normalizePath(path)).toEqual(expected);
});
test.each([
['something/../../../hehe'],
['/something/../../..'],
['..'],
])('detecting path traversal on "%s"', (path) => {
expect(() => normalizer.normalizePath(path)).toThrow(PathTraversalDetected.forPath(path));
});
test.each([
['some\0/path.txt'],
['s\x09i.php'],
])('detecting funky whitespace on "%s"', (path) => {
expect(() => normalizer.normalizePath(path)).toThrow(CorruptedPathDetected.unexpectedWhitespace(path));
});
})
================================================
FILE: packages/file-storage/src/path-normalizer.ts
================================================
import {join} from 'node:path';
export interface PathNormalizer {
normalizePath(path: string): string
}
const funkyWhiteSpaceRegex = new RegExp('\\p{C}+', 'u');
export class PathNormalizerV1 implements PathNormalizer {
normalizePath(path: string): string {
if (funkyWhiteSpaceRegex.test(path)) {
throw CorruptedPathDetected.unexpectedWhitespace(path);
}
const normalized = join(...(path.split('/')));
if (normalized.indexOf('../') !== -1 || normalized == '..') {
throw PathTraversalDetected.forPath(path);
}
return normalized === '.' ? '' : normalized;
}
}
export class CorruptedPathDetected extends Error {
static unexpectedWhitespace = (path: string) => new CorruptedPathDetected(
`Corrupted path detected with unexpected whitespace: ${path}`
);
}
export class PathTraversalDetected extends Error {
static forPath = (path: string) => new PathTraversalDetected(
`Path traversal detected for: ${path}`
);
}
================================================
FILE: packages/file-storage/src/path-prefixer.test.ts
================================================
import {PathPrefixer} from './path-prefixer.js';
describe('PathPrefixer', () => {
describe.each([
['with', '/prefix/'],
['without', '/prefix'],
])('prefixing %s a trailing slash prefix', (_name, prefix: string) => {
const prefixer = new PathPrefixer(prefix);
test('prefixing a file path with a leading slash', () => {
expect(prefixer.prefixFilePath('/file.txt')).toEqual('/prefix/file.txt');
});
test('prefixing a file path without a leading slash', () => {
expect(prefixer.prefixFilePath('file.txt')).toEqual('/prefix/file.txt');
});
test('prefixing a directory path with a leading slash', () => {
expect(prefixer.prefixDirectoryPath('/dirname')).toEqual('/prefix/dirname/');
});
test('prefixing a directory path with a trailing slash', () => {
expect(prefixer.prefixDirectoryPath('dirname/')).toEqual('/prefix/dirname/');
});
});
describe.each([
['with', '/prefix/'],
['without', '/prefix'],
])('stripping %s a trailing slash prefix', (_name, prefix: string) => {
const prefixer = new PathPrefixer(prefix);
test('stripping a file path', () => {
expect(prefixer.stripFilePath('/prefix/file.txt')).toEqual('file.txt');
});
test('stripping a directory path without a trailing slash', () => {
expect(prefixer.stripDirectoryPath('/prefix/dirname')).toEqual('dirname');
});
test('prefixing a directory path with a trailing slash', () => {
expect(prefixer.stripDirectoryPath('/prefix/dirname/')).toEqual('dirname');
});
});
describe('prefixing with an empty prefix', () => {
const prefixer = new PathPrefixer();
test('prefixing with a file path with leading slash', () => {
expect(prefixer.prefixFilePath('/file.txt')).toEqual('/file.txt');
});
test('prefixing with a directory path with leading slash', () => {
expect(prefixer.prefixDirectoryPath('/directory')).toEqual('/directory/');
});
})
});
================================================
FILE: packages/file-storage/src/path-prefixer.ts
================================================
import {type join, posix} from 'node:path';
export class PathPrefixer {
private readonly prefix: string = '';
constructor(
prefix: string = '',
private readonly separator: string = '/',
private readonly joinFunc: typeof join = posix.join,
) {
if (prefix.length > 0) {
this.prefix = this.joinFunc(prefix, this.separator);
}
}
prefixFilePath(path: string): string {
return this.prefix.length > 0 ? this.joinFunc(this.prefix, path): path;
}
prefixDirectoryPath(path: string): string {
let fullPath = this.prefix.length > 0
? this.joinFunc(this.prefix, path)
: path;
if (fullPath.length > 0 && !fullPath.endsWith(this.separator)) {
fullPath = `${fullPath}${this.separator}`;
}
return fullPath;
}
stripFilePath(path: string): string {
return path.substring(this.prefix.length);
}
stripDirectoryPath(path: string): string {
return this.stripFilePath(path).replace(/\/+$/g, '');
}
}
================================================
FILE: packages/file-storage/src/portable-visibility.ts
================================================
export enum Visibility {
PUBLIC = 'public',
PRIVATE = 'private',
}
================================================
FILE: packages/file-storage/src/readable-convertion.test.ts
================================================
import {Readable} from "node:stream";
import {readableToString} from "./file-storage.js";
describe('converting readables', () => {
test('converting a stream to a string', async () => {
expect(true).toEqual(true);
const readable = Readable.from('something');
const result = await readableToString(readable);
expect(result).toEqual('something');
});
});
================================================
FILE: packages/file-storage/src/utilities.test.ts
================================================
import {Readable} from 'stream';
import {readableToBuffer} from './file-storage.js';
describe('utilities', () => {
describe('readableToBuffer', () => {
test('converting to and from a buffer', async () => {
const input = Buffer.from('buffer contents');
const inputReadable = Readable.from(input);
const output = await readableToBuffer(inputReadable);
expect(output.toString()).toEqual(input.toString());
})
})
});
================================================
FILE: packages/file-storage/tsconfig.json
================================================
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist/"
},
"include": [
"src/**/*.ts"
]
}
================================================
FILE: packages/google-cloud-storage/.npmignore
================================================
src
!dist
tsconfig.json
================================================
FILE: packages/google-cloud-storage/CHANGELOG.md
================================================
# `@flystorage/google-cloud-storage`
## 1.2.0
### Changes
- Upgraded dependencies
- Remove not-used import
## 1.1.0
### Changes
- Improved stream errors
- Added way to detect reads failed due to missing files
## 1.0.0
### Major Changes
- First 1.0.0 release
================================================
FILE: packages/google-cloud-storage/LICENSE
================================================
Copyright (c) 2023-2024 Frank de Jonge
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: packages/google-cloud-storage/README.md
================================================
<img src="https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg" width="50px" height="50px" />
# Flystorage adapter for AWS S3
This package contains the Flystorage adapter for Google Cloud Storage
## Installation
Install all the required packages
```bash
npm install --save @flystorage/file-storage @flystorage/google-cloud-storage @google-cloud/storage
```
## Usage
```typescript
import {FileStorage} from '@flystorage/file-storage';
import {GoogleCloudStorageStorageAdapter} from '@flystorage/google-cloud-storage';
import {Storage} from '@google-cloud/storage';
const client = new Storage();
const bucket = client.bucket('{bucket-name}}', {
userProject: '{user-project}}',
});
const adapter = new GoogleCloudStorageStorageAdapter(bucket, {
prefix: '{optional-path-prefix}',
});
const storage = new FileStorage(adapter);
```
> ⚠️ Always use the FileStorage, it is essential for security and a good developer
> experience. Do not use the adapter directly.
## Visibility
Setting an retrieving visibility is only meaningful for legacy buckets. To use this functionality
with Flystorage, pass the legacy visibility handling to the constructor:
```typescript
import {GoogleCloudStorageStorageAdapter, LegacyVisibilityHandling} from '@flystorage/google-cloud-storage';
const adapter = new GoogleCloudStorageStorageAdapter(bucket, {
prefix: '{optional-path-prefix}',
}, new LegacyVisibilityHandling(
'allUsers', // acl entity, optional
'publicRead', // acl for Visibility.PUBLIC, optional,
'projectPrivate', // acl for Visibility.PRIVATE, optional,
));
```
================================================
FILE: packages/google-cloud-storage/package.json
================================================
{
"name": "@flystorage/google-cloud-storage",
"type": "module",
"version": "1.2.0",
"dependencies": {
"@flystorage/file-storage": "^1.1.0",
"@flystorage/stream-mime-type": "^1.0.0",
"@google-cloud/storage": "^7.19.0",
"file-type": "^21.3.1",
"mime-types": "^3.0.2"
},
"description": "",
"main": "./dist/cjs/index.js",
"types": "./dist/types/index.d.ts",
"exports": {
".": {
"import": {
"types": "./dist/types/index.d.ts",
"default": "./dist/esm/index.js"
},
"require": {
"types": "./dist/types/index.d.ts",
"default": "./dist/cjs/index.js"
}
}
},
"scripts": {
"compile": "rm -rf ./dist/ && concurrently npm:compile:* && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json",
"compile:esm": "tsc --outDir ./dist/esm/ --declaration false",
"compile:cjs": "tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node",
"compile:types": "tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly",
"watch": "tsc --watch"
},
"keywords": [
"google",
"file",
"storage",
"flystorage",
"filesystem"
],
"author": "Frank de Jonge (https://frankdejonge.nl)",
"homepage": "https://flystorage.dev/adapter/google-cloud-storage/",
"repository": {
"type": "git",
"url": "git+https://github.com/duna-oss/flystorage.git",
"directory": "packages/google-cloud-storage"
},
"license": "MIT"
}
================================================
FILE: packages/google-cloud-storage/src/google-cloud-storage.test.ts
================================================
import { Storage } from '@google-cloud/storage';
import {GoogleCloudStorageAdapter} from './google-cloud-storage.js';
import {
FileStorage,
UploadRequestHeaders,
readableToString,
UnableToReadFile, Visibility,
} from '@flystorage/file-storage';
import {randomBytes} from 'crypto';
import {resolve} from 'node:path';
import * as https from 'https';
import {LegacyVisibilityHandling} from './visibility-handling.js';
const testSegment = randomBytes(10).toString('hex');
let adapter: GoogleCloudStorageAdapter;
let storage: FileStorage;
let cleanupStorage: FileStorage;
describe('GoogleCloudStorageAdapter', () => {
const googleStorage = new Storage({keyFilename: resolve(process.cwd(), 'google-cloud-service-account.json')});
const bucket = googleStorage.bucket('no-acl-bucket-for-ci', {
userProject: 'flysystem-testing',
});
const legacyBucket = googleStorage.bucket('flysystem', {
userProject: 'flysystem-testing',
});
beforeAll(async () => {
const cleanupAdapter = new GoogleCloudStorageAdapter(bucket, {
prefix: `flystorage`,
});
cleanupStorage = new FileStorage(cleanupAdapter);
await cleanupStorage.deleteDirectory('non-existing-directory');
})
afterAll(async () => {
await cleanupStorage.deleteDirectory(testSegment);
});
describe('base API', () => {
beforeEach(() => {
const secondSegment = randomBytes(10).toString('hex');
adapter = new GoogleCloudStorageAdapter(bucket, {
prefix: `flystorage/${testSegment}/${secondSegment}`,
});
storage = new FileStorage(adapter);
});
test('reading a file that was written', async () => {
await storage.write('path.txt', 'content in azure');
const content = await storage.readToString('path.txt');
expect(content).toEqual('content in azure');
expect(await storage.fileExists('path.txt')).toEqual(true);
});
test('deleting a directory', async () => {
await storage.createDirectory('directory/a');
await storage.write('directory/b.txt', 'contents');
await storage.deleteDirectory('directory');
expect(await storage.fileExists('directory/b.txt')).toEqual(false);
});
test('statting a file', async () => {
await storage.write('directory/b.txt', 'contents');
const stat = await storage.statFile('directory/b.txt');
expect(stat.path).toEqual('directory/b.txt');
expect(typeof stat.lastModifiedMs).toEqual('number');
});
test('trying to read a file that does not exist', async () => {
let was404 = false;
try {
await storage.readToString('404.txt');
} catch (err) {
if (err instanceof UnableToReadFile) {
was404 = err.wasFileNotFound;
} else {
throw err;
}
}
expect(was404).toEqual(true);
});
test('checking if a directory exists by prefix', async () => {
await storage.write('directory/file.txt', 'contents');
expect(await storage.directoryExists('directory')).toEqual(true);
});
test('getting a crc32c checksum', async () => {
await storage.write('directory/file.txt', 'contents');
expect(await storage.checksum('directory/file.txt', {
algo: 'crc32c',
})).toEqual('E2fhmA==');
});
test('getting an md5 checksum', async () => {
await storage.write('directory/file.txt', 'contents');
expect(await storage.checksum('directory/file.txt', {
algo: 'md5',
})).toEqual('mL99jBV4Two9YyBEQeHiqg==');
});
test('getting an sha1 checksum', async () => {
await storage.write('directory/file.txt', 'contents');
expect(await storage.checksum('directory/file.txt', {
algo: 'sha1',
})).toEqual('4a756ca07e9487f482465a99e8286abc86ba4dc7');
});
test('checking if a created directory', async () => {
await storage.createDirectory('directory/here');
expect(await storage.directoryExists('directory/here')).toEqual(true);
expect(await storage.directoryExists('directory')).toEqual(true);
});
test('non deep and deep listing should have consistent and similar results', async () => {
await storage.write('file_1.txt', 'contents');
await storage.write('file_2.txt', 'contents');
await storage.createDirectory('directory_1');
await storage.createDirectory('directory_2');
const non_deep_listing = await storage.list('/', {deep: false}).toArray();
const deep_listing = await storage.list('/', {deep: true}).toArray();
expect(non_deep_listing).toHaveLength(4);
expect(deep_listing).toHaveLength(4);
expect(non_deep_listing).toEqual(deep_listing);
});
});
describe('visibility for legacy buckets', () => {
beforeEach(() => {
const secondSegment = randomBytes(10).toString('hex');
adapter = new GoogleCloudStorageAdapter(legacyBucket, {
prefix: `flystorage/${testSegment}/${secondSegment}`,
}, new LegacyVisibilityHandling());
storage = new FileStorage(adapter);
});
test('getting the visibility of a public file', async () => {
await storage.write('path.txt', 'contents', {
visibility: Visibility.PUBLIC,
});
expect(await storage.visibility('path.txt')).toEqual('public');
});
test('getting the visibility of a private file', async () => {
await storage.write('path.txt', 'contents', {
visibility: Visibility.PRIVATE,
});
expect(await storage.visibility('path.txt')).toEqual('private');
});
test('changing the visibility of a private file', async () => {
await storage.write('path.txt', 'contents', {
visibility: Visibility.PRIVATE,
});
await storage.changeVisibility('path.txt', Visibility.PUBLIC)
expect(await storage.visibility('path.txt')).toEqual('public');
});
test('using a public URL', async () => {
await storage.write('path.txt', 'public contents', {
visibility: Visibility.PUBLIC,
});
const url = await storage.publicUrl('path.txt');
const contents = await naivelyDownloadFile(url);
expect(contents).toEqual('public contents');
});
test('with a custom Cache-Control Header', async () => {
await storage.write('cache.txt', 'contents', {visibility: Visibility.PUBLIC, cacheControl: "max-age=9999, public"});
const url = await storage.publicUrl('cache.txt');
const res = await fetch(url);
expect (res.headers.get('Cache-Control')).toEqual('max-age=9999, public');
})
test('using a temporary URL', async () => {
await storage.write('path.txt', 'private contents', {
visibility: Visibility.PRIVATE,
});
const url = await storage.temporaryUrl('path.txt', {
expiresAt: Date.now() + 15 * 1000,
});
const contents = await naivelyDownloadFile(url);
expect(contents).toEqual('private contents');
});
test('uploading using a prepared request', async () => {
const request = await storage.prepareUpload('prepared/request-file.txt', {
expiresAt: Date.now() + 60 * 1000,
headers: {
'Content-Type': 'text/plain',
}
});
await naivelyMakeRequestFile(
request.url,
request.headers,
request.method,
'this is the contents',
);
const contents = await storage.readToString('prepared/request-file.txt');
expect(contents).toEqual('this is the contents');
});
});
});
function naivelyDownloadFile(url: string): Promise<string> {
return new Promise((resolve, reject) => {
https.get(url, async res => {
if (res.statusCode !== 200) {
reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));
} else {
resolve(await readableToString(res));
}
});
});
}
function naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise<void> {
return new Promise((resolve, reject) => {
const req = https.request(url, {
method: method,
headers: {
...headers,
'Content-Length': new Blob([data]).size,
}
}, async res => {
let responseBody = '';
res.on('data', (chunk) => {
responseBody += chunk;
});
res.on('end', () => {
const statusCode = res.statusCode ?? 500;
if (statusCode <= 200 && statusCode >= 299) {
reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));
} else {
resolve();
}
});
});
req.on('error', (err) => {
reject(err);
});
req.write(data);
req.end();
});
}
================================================
FILE: packages/google-cloud-storage/src/google-cloud-storage.ts
================================================
import {
ChecksumIsNotAvailable,
ChecksumOptions,
CopyFileOptions,
CreateDirectoryOptions,
FileContents,
FileWasNotFound,
MimeTypeOptions,
MoveFileOptions,
PathPrefixer,
PublicUrlOptions,
StatEntry,
StorageAdapter,
TemporaryUrlOptions,
UploadRequest,
UploadRequestHeaders,
UploadRequestOptions,
WriteOptions,
} from '@flystorage/file-storage';
import {Readable} from 'stream';
import {Bucket, GetFilesOptions, File, GetSignedUrlConfig, ApiError} from '@google-cloud/storage';
import {resolveMimeType} from '@flystorage/stream-mime-type';
import {pipeline} from 'node:stream/promises';
import {
UniformBucketLevelAccessVisibilityHandling,
VisibilityHandlingForGoogleCloudStorage,
} from './visibility-handling.js';
import {PassThrough} from 'node:stream';
export type GoogleCloudStorageAdapterOptions = {
prefix?: string,
}
export class GoogleCloudStorageAdapter implements StorageAdapter {
private readonly prefixer: PathPrefixer;
constructor(
private readonly bucket: Bucket,
readonly options: GoogleCloudStorageAdapterOptions = {},
readonly visibilityHandling: VisibilityHandlingForGoogleCloudStorage = new UniformBucketLevelAccessVisibilityHandling()
) {
this.prefixer = new PathPrefixer(options.prefix ?? '');
}
async write(path: string, contents: Readable, options: WriteOptions): Promise<void> {
let mimeType = options.mimeType;
if (mimeType === undefined) {
[mimeType, contents] = await resolveMimeType(path, contents);
}
const writeStream = this.bucket.file(this.prefixer.prefixFilePath(path))
.createWriteStream({
contentType: mimeType,
predefinedAcl: options.visibility
? this.visibilityHandling.visibilityToPredefinedAcl(options.visibility)
: undefined,
metadata: options.cacheControl? {cacheControl: options.cacheControl} : undefined,
});
await pipeline(contents, writeStream);
}
async read(path: string): Promise<FileContents> {
const readStream = this.bucket.file(this.prefixer.prefixFilePath(path)).createReadStream();
readStream.on('error', err => {
readStream.unpipe(errorHandler);
if (err instanceof ApiError && err.code === 404) {
errorHandler.destroy(FileWasNotFound.atLocation(path, {
context: {path},
cause: err,
}))
} else {
errorHandler.destroy(err);
}
});
const errorHandler = new PassThrough();
readStream.pipe(errorHandler);
return errorHandler;
}
async deleteFile(path: string): Promise<void> {
await this.bucket.file(this.prefixer.prefixFilePath(path)).delete({
ignoreNotFound: true,
});
}
async createDirectory(path: string, options: CreateDirectoryOptions): Promise<void> {
await this.bucket.file(this.prefixer.prefixDirectoryPath(path)).save('');
}
async copyFile(from: string, to: string, options: CopyFileOptions): Promise<void> {
await this.bucket.file(this.prefixer.prefixFilePath(from)).copy(
this.bucket.file(this.prefixer.prefixFilePath(to)),
);
}
async moveFile(from: string, to: string, options: MoveFileOptions): Promise<void> {
await this.copyFile(from, to, {});
await this.deleteFile(from);
}
async stat(path: string): Promise<StatEntry> {
const [metadata] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getMetadata();
return this.mapToStatEntry(metadata);
}
async* list(path: string, options: { deep: boolean; }): AsyncGenerator<StatEntry, any, unknown> {
let response: File[];
let query: GetFilesOptions | null = {
autoPaginate: false,
delimiter: options.deep ? undefined : '/',
includeTrailingDelimiter: options.deep ? undefined : true,
prefix: this.prefixer.prefixDirectoryPath(path),
}
while (query !== null) {
[response, query] = await this.bucket.getFiles(query);
for (const item of response) {
yield this.mapToStatEntry(item.metadata);
}
}
}
private mapToStatEntry(file: File['metadata']): StatEntry {
if (file.name!.endsWith('/')) {
return {
type: 'directory',
isFile: false,
isDirectory: true,
path: this.prefixer.stripDirectoryPath(file.name!),
};
}
return {
type: 'file',
isFile: true,
isDirectory: false,
path: this.prefixer.stripFilePath(file.name!),
lastModifiedMs: file.updated ? new Date(file.updated).getTime() : undefined,
mimeType: file.contentType,
};
}
async changeVisibility(path: string, visibility: string): Promise<void> {
await this.visibilityHandling.changeVisibility(
this.bucket.file(this.prefixer.prefixFilePath(path)),
visibility,
);
}
async visibility(path: string): Promise<string> {
return await this.visibilityHandling.determineVisibility(
this.bucket.file(this.prefixer.prefixFilePath(path)),
);
}
async deleteDirectory(path: string): P
gitextract_ft0asde_/ ├── .github/ │ ├── actions/ │ │ └── install-dependencies/ │ │ └── action.yml │ ├── dependabot.yml │ └── workflows/ │ ├── automerge-dependabot.yml │ ├── compile.yml │ ├── qa-ubuntu-bun.yml │ ├── qa-ubuntu-node.yml │ ├── qa-windows-node.yml │ └── shared-tests.yml ├── .gitignore ├── bin/ │ └── watch.ts ├── docker/ │ └── sftp/ │ ├── id_rsa │ ├── id_rsa.pub │ ├── ssh_host_ed25519_key │ ├── ssh_host_ed25519_key.pub │ ├── ssh_host_rsa_key │ ├── ssh_host_rsa_key.pub │ ├── sshd_custom_configs.sh │ ├── unknown.key │ └── users.conf ├── docker-compose.yml ├── fixtures/ │ └── adapter.template.ts ├── package.json ├── packages/ │ ├── aws-s3/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── aws-s3-file-storage.test.ts │ │ │ ├── aws-s3-storage-adapter.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── azure-storage-blob/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── azure-storage-blob.test.ts │ │ │ ├── azure-storage-blob.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── chaos/ │ │ ├── .npmignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── changelog.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── index.test.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── dynamic-import/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── index.d.ts │ │ ├── index.js │ │ ├── index.test.ts │ │ ├── package.json │ │ └── src/ │ │ └── index.ts │ ├── file-storage/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── checksum-from-stream.ts │ │ │ ├── errors.ts │ │ │ ├── file-storage.test.ts │ │ │ ├── file-storage.ts │ │ │ ├── index.ts │ │ │ ├── path-normalizer.test.ts │ │ │ ├── path-normalizer.ts │ │ │ ├── path-prefixer.test.ts │ │ │ ├── path-prefixer.ts │ │ │ ├── portable-visibility.ts │ │ │ ├── readable-convertion.test.ts │ │ │ └── utilities.test.ts │ │ └── tsconfig.json │ ├── google-cloud-storage/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── google-cloud-storage.test.ts │ │ │ ├── google-cloud-storage.ts │ │ │ ├── index.ts │ │ │ └── visibility-handling.ts │ │ └── tsconfig.json │ ├── in-memory/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── in-memory-file-storage.test.ts │ │ │ ├── in-memory-file-storage.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ ├── local-fs/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── index.ts │ │ │ ├── local-file-storage.test.ts │ │ │ ├── local-file-storage.ts │ │ │ └── unix-visibility.ts │ │ └── tsconfig.json │ ├── multer-storage/ │ │ ├── .npmignore │ │ ├── CHANGELOG.md │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── index.test.ts │ │ │ └── index.ts │ │ └── tsconfig.json │ └── stream-mime-type/ │ ├── .npmignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── package.json │ ├── src/ │ │ ├── index.ts │ │ ├── stream-mime-type.test.ts │ │ └── stream-mime-type.ts │ └── tsconfig.json ├── packed/ │ ├── .gitignore │ ├── cjs/ │ │ ├── index.js │ │ ├── index.test.js │ │ └── package.json │ └── esm/ │ ├── index.js │ └── package.json ├── tsconfig.build.json ├── tsconfig.json └── vitest.config.ts
SYMBOL INDEX (379 symbols across 24 files)
FILE: fixtures/adapter.template.ts
type AdapterFileStorageOptions (line 15) | type AdapterFileStorageOptions = {
class AdapterFileStorage (line 19) | class AdapterFileStorage implements StorageAdapter {
method constructor (line 22) | constructor(
method write (line 28) | async write(path: string, contents: Readable, options: WriteOptions): ...
method read (line 32) | async read(path: string): Promise<FileContents> {
method deleteFile (line 35) | async deleteFile(path: string): Promise<void> {
method createDirectory (line 38) | async createDirectory(path: string, options: CreateDirectoryOptions): ...
method stat (line 41) | async stat(path: string): Promise<StatEntry> {
method list (line 44) | list(path: string, options: {deep: boolean}): AsyncGenerator<StatEntry> {
method changeVisibility (line 47) | async changeVisibility(path: string, visibility: string): Promise<void> {
method visibility (line 50) | async visibility(path: string): Promise<string> {
method deleteDirectory (line 53) | async deleteDirectory(path: string): Promise<void> {
method fileExists (line 56) | async fileExists(path: string): Promise<boolean> {
method directoryExists (line 59) | async directoryExists(path: string): Promise<boolean> {
method publicUrl (line 62) | async publicUrl(path: string, options: PublicUrlOptions): Promise<stri...
method temporaryUrl (line 65) | async temporaryUrl(path: string, options: TemporaryUrlOptions): Promis...
method checksum (line 68) | async checksum(path: string, options: ChecksumOptions): Promise<string> {
method mimeType (line 71) | async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
method lastModified (line 74) | async lastModified(path: string): Promise<number> {
method fileSize (line 77) | async fileSize(path: string): Promise<number> {
method copyFile (line 80) | async copyFile(from: string, to: string, options: CopyFileOptions): Pr...
method moveFile (line 83) | async moveFile(from: string, to: string, options: MoveFileOptions): Pr...
FILE: packages/aws-s3/src/aws-s3-file-storage.test.ts
function hashString (line 358) | function hashString(input: string, algo: string, encoding: BinaryToTextE...
function hashString (line 374) | function hashString(input: string, algo: string, encoding: BinaryToTextE...
function naivelyDownloadFile (line 420) | function naivelyDownloadFile(url: string): Promise<string> {
function naivelyMakeRequestFile (line 432) | function naivelyMakeRequestFile(url: string, headers: UploadRequestHeade...
function responseHeaderValue (line 464) | function responseHeaderValue(url: string, header: string): Promise<strin...
FILE: packages/aws-s3/src/aws-s3-storage-adapter.ts
type PutObjectOptions (line 54) | type PutObjectOptions = Omit<PutObjectCommandInput, 'Bucket' | 'Key' | '...
type WriteOptionsForS3 (line 55) | type WriteOptionsForS3 = Omit<PutObjectOptions, 'ACL' | 'ContentLength'>;
type ChecksumAlgo (line 57) | type ChecksumAlgo = typeof possibleChecksumAlgos[number];
function isSupportedAlgo (line 59) | function isSupportedAlgo(algo: string): algo is ChecksumAlgo {
type AwsS3StorageAdapterOptions (line 63) | type AwsS3StorageAdapterOptions = Readonly<{
type AwsPublicUrlOptions (line 74) | type AwsPublicUrlOptions = PublicUrlOptions & {
type AwsPublicUrlGenerator (line 81) | type AwsPublicUrlGenerator = {
class DefaultAwsPublicUrlGenerator (line 85) | class DefaultAwsPublicUrlGenerator implements AwsPublicUrlGenerator {
method publicUrl (line 86) | async publicUrl(path: string, options: AwsPublicUrlOptions): Promise<s...
class HostStyleAwsPublicUrlGenerator (line 104) | class HostStyleAwsPublicUrlGenerator extends DefaultAwsPublicUrlGenerator {
type TimestampResolver (line 107) | type TimestampResolver = () => number;
type AclOptions (line 108) | type AclOptions = Pick<CopyObjectRequest, 'ACL'>;
function encodePath (line 113) | function encodePath(path: string): string {
function maybeAbort (line 117) | function maybeAbort(signal?: AbortSignal) {
class AwsS3StorageAdapter (line 123) | class AwsS3StorageAdapter implements StorageAdapter {
method constructor (line 126) | constructor(
method copyFile (line 148) | async copyFile(from: string, to: string, options: CopyFileOptions): Pr...
method moveFile (line 168) | async moveFile(from: string, to: string, options: MoveFileOptions): Pr...
method prepareUpload (line 173) | async prepareUpload(path: string, options: UploadRequestOptions): Prom...
method temporaryUrl (line 203) | async temporaryUrl(path: string, options: TemporaryUrlOptions): Promis...
method lastModified (line 244) | async lastModified(path: string, options: MiscellaneousOptions): Promi...
method fileSize (line 254) | async fileSize(path: string, options: MiscellaneousOptions): Promise<n...
method mimeType (line 268) | async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
method visibility (line 296) | async visibility(path: string, options: MiscellaneousOptions): Promise...
method list (line 313) | async* list(path: string, options: AdapterListOptions): AsyncGenerator...
method listObjects (line 353) | async * listObjects(
method read (line 405) | async read(path: string, options: MiscellaneousOptions): Promise<FileC...
method stat (line 435) | async stat(path: string, options: MiscellaneousOptions): Promise<StatE...
method createDirectory (line 455) | async createDirectory(path: string, options: CreateDirectoryOptions): ...
method deleteDirectory (line 480) | async deleteDirectory(path: string): Promise<void> {
method write (line 514) | async write(path: string, contents: Readable, options: WriteOptions): ...
method createPutObjectParams (line 566) | private createPutObjectParams(
method deleteFile (line 584) | async deleteFile(path: string, options: MiscellaneousOptions): Promise...
method visibilityToAcl (line 595) | private visibilityToAcl(visibility: string): ObjectCannedACL {
method changeVisibility (line 605) | async changeVisibility(path: string, visibility: string, options: Misc...
method fileExists (line 616) | async fileExists(path: string, options: MiscellaneousOptions): Promise...
method directoryExists (line 636) | async directoryExists(path: string, options: MiscellaneousOptions): Pr...
method publicUrl (line 652) | async publicUrl(path: string, options: PublicUrlOptions): Promise<stri...
method checksum (line 661) | async checksum(path: string, options: ChecksumOptions): Promise<string> {
method lookupMimeTypeFromStream (line 688) | private async lookupMimeTypeFromStream(path: string, options: MimeType...
class AwsS3FileStorage (line 701) | class AwsS3FileStorage extends AwsS3StorageAdapter {}
type ResolversForWriteOptions (line 703) | type ResolversForWriteOptions = {
function isWriteOptionKey (line 707) | function isWriteOptionKey(key: string): key is string & keyof ResolversF...
FILE: packages/azure-storage-blob/src/azure-storage-blob.test.ts
function naivelyDownloadFile (line 247) | async function naivelyDownloadFile(url: string): Promise<string> {
function naivelyMakeRequestFile (line 257) | function naivelyMakeRequestFile(url: string, headers: UploadRequestHeade...
FILE: packages/azure-storage-blob/src/azure-storage-blob.ts
type AzureStorageBlobStorageAdapterOptions (line 35) | type AzureStorageBlobStorageAdapterOptions = {
function maybeAbort (line 44) | function maybeAbort(signal?: AbortSignal) {
class AzureStorageBlobStorageAdapter (line 50) | class AzureStorageBlobStorageAdapter implements StorageAdapter {
method constructor (line 53) | constructor(
method copyFile (line 60) | async copyFile(from: string, to: string, options: CopyFileOptions): Pr...
method moveFile (line 65) | async moveFile(from: string, to: string, options: MoveFileOptions): Pr...
method write (line 70) | async write(path: string, contents: Readable, options: WriteOptions): ...
method blockClient (line 97) | private blockClient(path: string) {
method read (line 101) | async read(path: string, options: MiscellaneousOptions): Promise<FileC...
method deleteFile (line 128) | async deleteFile(path: string, options: MiscellaneousOptions): Promise...
method createDirectory (line 136) | async createDirectory(): Promise<void> {
method stat (line 140) | async stat(path: string, options: {abortSignal?: AbortSignal} = {}): P...
method mapToStatEntry (line 151) | private mapToStatEntry(path: string, properties: BlobGetPropertiesResp...
method list (line 163) | list(path: string, options: ListOptions): AsyncGenerator<StatEntry> {
method listDeep (line 170) | async *listDeep(path: string, options: ListOptions): AsyncGenerator<St...
method listShallow (line 204) | async *listShallow(path: string, options: ListOptions): AsyncGenerator...
method changeVisibility (line 231) | async changeVisibility(path: string, visibility: string): Promise<void> {
method visibility (line 236) | async visibility(path: string): Promise<string> {
method deleteDirectory (line 244) | async deleteDirectory(path: string, options: MiscellaneousOptions): Pr...
method fileExists (line 262) | async fileExists(path: string, options: MiscellaneousOptions): Promise...
method directoryExists (line 268) | async directoryExists(path: string, options: MiscellaneousOptions): Pr...
method publicUrl (line 279) | async publicUrl(path: string, options?: PublicUrlOptions): Promise<str...
method temporaryUrl (line 282) | async temporaryUrl(path: string, options: TemporaryUrlOptions): Promis...
method prepareUpload (line 290) | async prepareUpload(path: string, options: UploadRequestOptions): Prom...
method checksum (line 312) | async checksum(path: string, options: ChecksumOptions): Promise<string> {
method mimeType (line 331) | async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
method lastModified (line 345) | async lastModified(path: string): Promise<number> {
method fileSize (line 359) | async fileSize(path: string): Promise<number> {
method resolveMimetype (line 373) | private async resolveMimetype(path: string, contents: Readable, option...
class AzureStorageBlobFileStorage (line 389) | class AzureStorageBlobFileStorage extends AzureStorageBlobStorageAdapter {}
FILE: packages/chaos/src/index.ts
type AnyAdapterMethodName (line 16) | type AnyAdapterMethodName = keyof StorageAdapter;
type ChaosStrategy (line 18) | interface ChaosStrategy {
class AlwaysThrowError (line 22) | class AlwaysThrowError implements ChaosStrategy {
method constructor (line 23) | constructor(private readonly newError: () => Error) {
method maybeGoNuts (line 26) | maybeGoNuts(method: keyof StorageAdapter): void {
class NeverThrowError (line 31) | class NeverThrowError implements ChaosStrategy {
method maybeGoNuts (line 32) | maybeGoNuts(method: keyof StorageAdapter): void {
class TriggeredErrors (line 37) | class TriggeredErrors implements ChaosStrategy {
method on (line 46) | on(method: AnyAdapterMethodName | '*', newError: () => unknown, option...
method clearTriggers (line 54) | clearTriggers(): void {
method maybeGoNuts (line 58) | maybeGoNuts(method: keyof StorageAdapter): void {
class ChaosStorageAdapterDecorator (line 75) | class ChaosStorageAdapterDecorator implements StorageAdapter {
method constructor (line 76) | constructor(
method write (line 82) | write(path: string, contents: Readable, options: WriteOptions): Promis...
method read (line 88) | read(path: string, options: MiscellaneousOptions): Promise<FileContent...
method deleteFile (line 94) | deleteFile(path: string, options: MiscellaneousOptions): Promise<void> {
method createDirectory (line 100) | createDirectory(path: string, options: CreateDirectoryOptions): Promis...
method copyFile (line 106) | copyFile(from: string, to: string, options: CopyFileOptions): Promise<...
method moveFile (line 112) | moveFile(from: string, to: string, options: MoveFileOptions): Promise<...
method stat (line 118) | stat(path: string, options: MiscellaneousOptions): Promise<StatEntry> {
method list (line 124) | list(path: string, options: { deep: boolean; }): AsyncGenerator<StatEn...
method changeVisibility (line 130) | changeVisibility(path: string, visibility: string, options: Miscellane...
method visibility (line 136) | visibility(path: string, options: MiscellaneousOptions): Promise<strin...
method deleteDirectory (line 142) | deleteDirectory(path: string, options: MiscellaneousOptions): Promise<...
method fileExists (line 148) | fileExists(path: string, options: MiscellaneousOptions): Promise<boole...
method directoryExists (line 154) | directoryExists(path: string, options: MiscellaneousOptions): Promise<...
method publicUrl (line 160) | publicUrl(path: string, options: MiscellaneousOptions): Promise<string> {
method temporaryUrl (line 166) | temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<stri...
method checksum (line 172) | checksum(path: string, options: ChecksumOptions): Promise<string> {
method mimeType (line 178) | mimeType(path: string, options: MimeTypeOptions): Promise<string> {
method lastModified (line 184) | lastModified(path: string, options: MiscellaneousOptions): Promise<num...
method fileSize (line 190) | fileSize(path: string, options: MiscellaneousOptions): Promise<number> {
FILE: packages/dynamic-import/index.js
function dynamicallyImport (line 1) | async function dynamicallyImport(path) {
FILE: packages/dynamic-import/index.test.ts
type FileType (line 4) | type FileType = typeof import('file-type');
FILE: packages/file-storage/src/checksum-from-stream.ts
function checksumFromStream (line 7) | async function checksumFromStream(
FILE: packages/file-storage/src/errors.ts
type ErrorContext (line 1) | type ErrorContext = { [index: string]: any };
type OperationError (line 3) | type OperationError = Error & {
function errorToMessage (line 8) | function errorToMessage(error: unknown): string {
method constructor (line 15) | constructor(
class ChecksumIsNotAvailable (line 30) | class ChecksumIsNotAvailable extends FlystorageError {
method constructor (line 33) | constructor(
method isErrorOfType (line 52) | static isErrorOfType(error: unknown): error is ChecksumIsNotAvailable {
class UnableToGetChecksum (line 57) | class UnableToGetChecksum extends FlystorageError {
class UnableToGetMimeType (line 70) | class UnableToGetMimeType extends FlystorageError {
class UnableToGetLastModified (line 83) | class UnableToGetLastModified extends FlystorageError {
class UnableToGetFileSize (line 96) | class UnableToGetFileSize extends FlystorageError {
class UnableToWriteFile (line 109) | class UnableToWriteFile extends FlystorageError {
class UnableToReadFile (line 122) | class UnableToReadFile extends FlystorageError {
method constructor (line 125) | constructor(
class FileWasNotFound (line 152) | class FileWasNotFound extends FlystorageError {
function isFileWasNotFound (line 165) | function isFileWasNotFound(error: unknown): error is FileWasNotFound {
class UnableToSetVisibility (line 169) | class UnableToSetVisibility extends FlystorageError {
class UnableToGetVisibility (line 182) | class UnableToGetVisibility extends FlystorageError {
class UnableToGetPublicUrl (line 195) | class UnableToGetPublicUrl extends FlystorageError {
class UnableToGetTemporaryUrl (line 208) | class UnableToGetTemporaryUrl extends FlystorageError {
class UnableToPrepareUploadRequest (line 221) | class UnableToPrepareUploadRequest extends FlystorageError {
class UnableToCopyFile (line 234) | class UnableToCopyFile extends FlystorageError {
class UnableToMoveFile (line 247) | class UnableToMoveFile extends FlystorageError {
class UnableToGetStat (line 260) | class UnableToGetStat extends FlystorageError {
class UnableToCreateDirectory (line 282) | class UnableToCreateDirectory extends FlystorageError {
class UnableToDeleteDirectory (line 295) | class UnableToDeleteDirectory extends FlystorageError {
class UnableToDeleteFile (line 308) | class UnableToDeleteFile extends FlystorageError {
class UnableToCheckFileExistence (line 321) | class UnableToCheckFileExistence extends FlystorageError {
class UnableToCheckDirectoryExistence (line 334) | class UnableToCheckDirectoryExistence extends FlystorageError {
class UnableToListDirectory (line 347) | class UnableToListDirectory extends FlystorageError {
FILE: packages/file-storage/src/file-storage.test.ts
method prepareUpload (line 25) | async prepareUpload(path: string, options: UploadRequestOptions): Promis...
FILE: packages/file-storage/src/file-storage.ts
type CommonStatInfo (line 33) | type CommonStatInfo = Readonly<{
type FileInfo (line 39) | type FileInfo = Readonly<{
type DirectoryInfo (line 47) | type DirectoryInfo = Readonly<{
function isFile (line 53) | function isFile(stat: StatEntry): stat is FileInfo {
function isDirectory (line 57) | function isDirectory(stat: StatEntry): stat is DirectoryInfo {
type StatEntry (line 61) | type StatEntry = FileInfo | DirectoryInfo;
type AdapterListOptions (line 63) | type AdapterListOptions = ListOptions & { deep: boolean };
type StorageAdapter (line 65) | interface StorageAdapter {
class DirectoryListing (line 107) | class DirectoryListing implements AsyncIterable<StatEntry> {
method constructor (line 108) | constructor(
method toArray (line 115) | async toArray(sorted: boolean = true): Promise<StatEntry[]> {
method filter (line 124) | filter(filter: (entry: StatEntry) => boolean): DirectoryListing {
method [Symbol.asyncIterator] (line 137) | async* [Symbol.asyncIterator]() {
type FileContents (line 151) | type FileContents = Iterable<any> | AsyncIterable<any> | NodeJS.Readable...
type TimeoutOptions (line 153) | type TimeoutOptions = { timeout?: number };
type MiscellaneousOptions (line 155) | type MiscellaneousOptions = TimeoutOptions & {
type MimeTypeOptions (line 160) | type MimeTypeOptions = MiscellaneousOptions & {
type VisibilityFallback (line 165) | type VisibilityFallback = {
type VisibilityOptions (line 173) | type VisibilityOptions = MiscellaneousOptions & {
type WriteOptions (line 184) | type WriteOptions = VisibilityOptions & MiscellaneousOptions & {
type CreateDirectoryOptions (line 189) | type CreateDirectoryOptions = MiscellaneousOptions & Pick<VisibilityOpti...
type PublicUrlOptions (line 190) | type PublicUrlOptions = MiscellaneousOptions & {};
type UploadRequestOptions (line 191) | type UploadRequestOptions = MiscellaneousOptions & {
type CopyFileOptions (line 196) | type CopyFileOptions = MiscellaneousOptions & VisibilityOptions & {
type MoveFileOptions (line 199) | type MoveFileOptions = MiscellaneousOptions & VisibilityOptions & {
type ListOptions (line 202) | type ListOptions = MiscellaneousOptions & { deep?: boolean };
type TemporaryUrlOptions (line 203) | type TemporaryUrlOptions = MiscellaneousOptions & {
type ChecksumOptions (line 208) | type ChecksumOptions = MiscellaneousOptions & {
type ConfigurationOptions (line 213) | type ConfigurationOptions = {
function toReadable (line 228) | function toReadable(contents: FileContents): Readable {
function instrumentAbortSignal (line 241) | function instrumentAbortSignal<Options extends MiscellaneousOptions>(opt...
class FileStorage (line 265) | class FileStorage {
method constructor (line 266) | constructor(
method write (line 273) | public async write(path: string, contents: FileContents, options: Writ...
method read (line 292) | public async read(path: string, options: MiscellaneousOptions = {}): P...
method readToString (line 325) | public async readToString(path: string, options: MiscellaneousOptions ...
method readToUint8Array (line 329) | public async readToUint8Array(path: string, options: MiscellaneousOpti...
method readToBuffer (line 333) | public async readToBuffer(path: string, options: MiscellaneousOptions ...
method deleteFile (line 337) | public async deleteFile(path: string, options: MiscellaneousOptions = ...
method createDirectory (line 353) | public async createDirectory(path: string, options: CreateDirectoryOpt...
method deleteDirectory (line 369) | public async deleteDirectory(path: string, options: MiscellaneousOptio...
method stat (line 382) | public async stat(path: string, options: MiscellaneousOptions = {}): P...
method moveFile (line 395) | public async moveFile(from: string, to: string, options: MoveFileOptio...
method copyFile (line 412) | public async copyFile(from: string, to: string, options: CopyFileOptio...
method changeVisibility (line 429) | public async changeVisibility(path: string, visibility: string, option...
method visibility (line 454) | public async visibility(path: string, options: VisibilityOptions = {})...
method fileExists (line 479) | public async fileExists(path: string, options: MiscellaneousOptions = ...
method list (line 492) | public list(path: string, options: ListOptions = {}): DirectoryListing {
method statFile (line 507) | public async statFile(path: string, options: MiscellaneousOptions = {}...
method directoryExists (line 518) | public async directoryExists(path: string, options: MiscellaneousOptio...
method publicUrl (line 531) | public async publicUrl(path: string, options: PublicUrlOptions = {}): ...
method temporaryUrl (line 547) | public async temporaryUrl(path: string, options: TemporaryUrlOptions):...
method prepareUpload (line 563) | public async prepareUpload(path: string, options: UploadRequestOptions...
method checksum (line 594) | public async checksum(path: string, options: ChecksumOptions = {}): Pr...
method mimeType (line 614) | public async mimeType(path: string, options: MimeTypeOptions = {}): Pr...
method lastModified (line 630) | public async lastModified(path: string, options: MiscellaneousOptions ...
method fileSize (line 646) | public async fileSize(path: string, options: MiscellaneousOptions = {}...
method calculateChecksum (line 662) | private async calculateChecksum(path: string, options: ChecksumOptions...
type TimestampMs (line 674) | type TimestampMs = number;
type ExpiresAt (line 675) | type ExpiresAt = Date | TimestampMs;
function normalizeExpiryToDate (line 677) | function normalizeExpiryToDate(expiresAt: ExpiresAt): Date {
function normalizeExpiryToMilliseconds (line 681) | function normalizeExpiryToMilliseconds(expiresAt: ExpiresAt): number {
function closeReadable (line 685) | async function closeReadable(body: Readable) {
function readableToString (line 701) | async function readableToString(stream: Readable): Promise<string> {
function readableToBuffer (line 708) | async function readableToBuffer(stream: Readable): Promise<Buffer> {
function readableToUint8Array (line 720) | function readableToUint8Array(stream: Readable): Promise<Uint8Array> {
function concatUint8Arrays (line 737) | function concatUint8Arrays(input: Uint8Array[]): Uint8Array {
type UploadRequestHeaders (line 749) | type UploadRequestHeaders = Record<string, string | ReadonlyArray<string>>;
type UploadRequest (line 751) | type UploadRequest = {
type PreparedUploadStrategy (line 758) | interface PreparedUploadStrategy {
class PreparedUploadsAreNotSupported (line 762) | class PreparedUploadsAreNotSupported implements PreparedUploadStrategy {
method prepareUpload (line 763) | prepareUpload(): Promise<UploadRequest> {
FILE: packages/file-storage/src/path-normalizer.ts
type PathNormalizer (line 3) | interface PathNormalizer {
class PathNormalizerV1 (line 9) | class PathNormalizerV1 implements PathNormalizer {
method normalizePath (line 10) | normalizePath(path: string): string {
class CorruptedPathDetected (line 25) | class CorruptedPathDetected extends Error {
class PathTraversalDetected (line 31) | class PathTraversalDetected extends Error {
FILE: packages/file-storage/src/path-prefixer.ts
class PathPrefixer (line 3) | class PathPrefixer {
method constructor (line 5) | constructor(
method prefixFilePath (line 15) | prefixFilePath(path: string): string {
method prefixDirectoryPath (line 19) | prefixDirectoryPath(path: string): string {
method stripFilePath (line 31) | stripFilePath(path: string): string {
method stripDirectoryPath (line 35) | stripDirectoryPath(path: string): string {
FILE: packages/file-storage/src/portable-visibility.ts
type Visibility (line 1) | enum Visibility {
FILE: packages/google-cloud-storage/src/google-cloud-storage.test.ts
function naivelyDownloadFile (line 233) | function naivelyDownloadFile(url: string): Promise<string> {
function naivelyMakeRequestFile (line 245) | function naivelyMakeRequestFile(url: string, headers: UploadRequestHeade...
FILE: packages/google-cloud-storage/src/google-cloud-storage.ts
type GoogleCloudStorageAdapterOptions (line 30) | type GoogleCloudStorageAdapterOptions = {
class GoogleCloudStorageAdapter (line 34) | class GoogleCloudStorageAdapter implements StorageAdapter {
method constructor (line 36) | constructor(
method write (line 44) | async write(path: string, contents: Readable, options: WriteOptions): ...
method read (line 63) | async read(path: string): Promise<FileContents> {
method deleteFile (line 84) | async deleteFile(path: string): Promise<void> {
method createDirectory (line 90) | async createDirectory(path: string, options: CreateDirectoryOptions): ...
method copyFile (line 94) | async copyFile(from: string, to: string, options: CopyFileOptions): Pr...
method moveFile (line 100) | async moveFile(from: string, to: string, options: MoveFileOptions): Pr...
method stat (line 105) | async stat(path: string): Promise<StatEntry> {
method list (line 111) | async* list(path: string, options: { deep: boolean; }): AsyncGenerator...
method mapToStatEntry (line 129) | private mapToStatEntry(file: File['metadata']): StatEntry {
method changeVisibility (line 149) | async changeVisibility(path: string, visibility: string): Promise<void> {
method visibility (line 156) | async visibility(path: string): Promise<string> {
method deleteDirectory (line 162) | async deleteDirectory(path: string): Promise<void> {
method fileExists (line 172) | async fileExists(path: string): Promise<boolean> {
method directoryExists (line 178) | async directoryExists(path: string): Promise<boolean> {
method publicUrl (line 194) | async publicUrl(path: string, options: PublicUrlOptions): Promise<stri...
method temporaryUrl (line 198) | async temporaryUrl(path: string, options: TemporaryUrlOptions): Promis...
method prepareUpload (line 207) | async prepareUpload(path: string, options: UploadRequestOptions): Prom...
method checksum (line 231) | async checksum(path: string, options: ChecksumOptions): Promise<string> {
method mimeType (line 243) | async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
method lastModified (line 253) | async lastModified(path: string): Promise<number> {
method fileSize (line 263) | async fileSize(path: string): Promise<number> {
class GoogleCloudFileStorage (line 280) | class GoogleCloudFileStorage extends GoogleCloudStorageAdapter {}
FILE: packages/google-cloud-storage/src/visibility-handling.ts
type VisibilityHandlingForGoogleCloudStorage (line 4) | interface VisibilityHandlingForGoogleCloudStorage {
class UniformBucketLevelAccessVisibilityHandling (line 12) | class UniformBucketLevelAccessVisibilityHandling implements VisibilityHa...
method constructor (line 13) | constructor(
method changeVisibility (line 21) | async changeVisibility(file: File, visibility: string): Promise<void> {
method determineVisibility (line 27) | async determineVisibility(file: File): Promise<string> {
method visibilityToPredefinedAcl (line 33) | visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefin...
class LegacyVisibilityHandling (line 43) | class LegacyVisibilityHandling implements VisibilityHandlingForGoogleClo...
method constructor (line 44) | constructor(
method changeVisibility (line 51) | async changeVisibility(file: File, visibility: string): Promise<void> {
method determineVisibility (line 64) | async determineVisibility(file: File): Promise<string> {
method visibilityToPredefinedAcl (line 78) | visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefin...
FILE: packages/in-memory/src/in-memory-file-storage.ts
type FileEntry (line 22) | type FileEntry = {
type DirectoryEntry (line 30) | type DirectoryEntry = {
type TimestampResolver (line 36) | type TimestampResolver = () => number;
function cloneBuffer (line 38) | function cloneBuffer(input: Buffer): Buffer {
class InMemoryStorageAdapter (line 45) | class InMemoryStorageAdapter implements StorageAdapter {
method constructor (line 48) | constructor(
method deleteEverything (line 53) | deleteEverything(): void {
method write (line 57) | async write(path: string, contents: Readable, options: WriteOptions): ...
method ensureParentDirsExist (line 69) | private ensureParentDirsExist(path: string, options: VisibilityOptions) {
method read (line 83) | async read(path: string): Promise<FileContents> {
method deleteFile (line 93) | async deleteFile(path: string): Promise<void> {
method createDirectory (line 97) | async createDirectory(path: string, options: CreateDirectoryOptions): ...
method copyFile (line 106) | async copyFile(from: string, to: string, options: CopyFileOptions): Pr...
method moveFile (line 119) | async moveFile(from: string, to: string, options: MoveFileOptions): Pr...
method stat (line 123) | async stat(path: string): Promise<StatEntry> {
method mapToStatEntry (line 132) | private mapToStatEntry(entry: DirectoryEntry | FileEntry): StatEntry {
method list (line 150) | async *list(path: string, options: { deep: boolean; }): AsyncGenerator...
method changeVisibility (line 168) | async changeVisibility(path: string, visibility: string): Promise<void> {
method visibility (line 180) | async visibility(path: string): Promise<string> {
method deleteDirectory (line 189) | async deleteDirectory(path: string): Promise<void> {
method fileExists (line 199) | async fileExists(path: string): Promise<boolean> {
method directoryExists (line 202) | async directoryExists(path: string): Promise<boolean> {
method publicUrl (line 205) | async publicUrl(path: string, options: PublicUrlOptions): Promise<stri...
method temporaryUrl (line 208) | async temporaryUrl(path: string, options: TemporaryUrlOptions): Promis...
method checksum (line 211) | async checksum(path: string, options: ChecksumOptions): Promise<string> {
method mimeType (line 220) | async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
method lastModified (line 229) | async lastModified(path: string): Promise<number> {
method fileSize (line 239) | async fileSize(path: string): Promise<number> {
function resolveMimeType (line 250) | async function resolveMimeType(
class InMemoryFileStorage (line 271) | class InMemoryFileStorage extends InMemoryStorageAdapter {}
FILE: packages/local-fs/src/local-file-storage.test.ts
method prepareUpload (line 55) | async prepareUpload(path) {
function hashString (line 349) | function hashString(input: string, algo: string, encoding: BinaryToTextE...
class FakeTemporaryUrlGenerator (line 365) | class FakeTemporaryUrlGenerator implements LocalTemporaryUrlGenerator {
method temporaryUrl (line 366) | async temporaryUrl(path: string, options: LocalTemporaryUrlOptions): P...
FILE: packages/local-fs/src/local-file-storage.ts
type LocalStorageAdapterOptions (line 30) | type LocalStorageAdapterOptions = {
type LocalPublicUrlOptions (line 36) | type LocalPublicUrlOptions = PublicUrlOptions & {
type LocalPublicUrlGenerator (line 40) | type LocalPublicUrlGenerator = {
class BaseUrlLocalPublicUrlGenerator (line 44) | class BaseUrlLocalPublicUrlGenerator implements LocalPublicUrlGenerator {
method publicUrl (line 45) | async publicUrl(path: string, options: LocalPublicUrlOptions): Promise...
type LocalTemporaryUrlOptions (line 60) | type LocalTemporaryUrlOptions = TemporaryUrlOptions & {
type LocalTemporaryUrlGenerator (line 64) | type LocalTemporaryUrlGenerator = {
class FailingLocalTemporaryUrlGenerator (line 68) | class FailingLocalTemporaryUrlGenerator implements LocalTemporaryUrlGene...
method temporaryUrl (line 69) | async temporaryUrl(): Promise<string> {
type FileTypePackage (line 74) | type FileTypePackage = typeof import('file-type');
function maybeAbort (line 78) | function maybeAbort(signal?: AbortSignal) {
class LocalStorageAdapter (line 84) | class LocalStorageAdapter implements StorageAdapter {
method constructor (line 87) | constructor(
method copyFile (line 99) | async copyFile(from: string, to: string, options: CopyFileOptions): Pr...
method moveFile (line 110) | async moveFile(from: string, to: string, options: MoveFileOptions): Pr...
method prepareUpload (line 122) | prepareUpload(path: string, options: UploadRequestOptions): Promise<Up...
method temporaryUrl (line 127) | temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<stri...
method publicUrl (line 132) | publicUrl(path: string, options: PublicUrlOptions): Promise<string> {
method mimeType (line 137) | async mimeType(path: string, options: MimeTypeOptions): Promise<string> {
method fileSize (line 171) | async fileSize(path: string, options: MiscellaneousOptions): Promise<n...
method lastModified (line 186) | async lastModified(path: string, options: MiscellaneousOptions): Promi...
method list (line 200) | async* list(path: string, {deep}: { deep: boolean }): AsyncGenerator<S...
method read (line 217) | async read(path: string, options: MiscellaneousOptions): Promise<Reada...
method write (line 239) | async write(path: string, contents: Readable, options: WriteOptions): ...
method deleteFile (line 268) | async deleteFile(path: string): Promise<void> {
method createDirectory (line 274) | async createDirectory(path: string, options: CreateDirectoryOptions): ...
method stat (line 283) | async stat(path: string, options: MiscellaneousOptions): Promise<StatE...
method doStat (line 289) | private async doStat(path: string, type: 'file' | 'directory' = 'file'...
method fileExists (line 298) | async fileExists(path: string, options: MiscellaneousOptions): Promise...
method deleteDirectory (line 314) | async deleteDirectory(path: string, options: MiscellaneousOptions): Pr...
method mapStatToEntry (line 323) | private mapStatToEntry(info: Stats | Dirent, path: string): StatEntry {
method changeVisibility (line 348) | async changeVisibility(path: string, visibility: string, options: Misc...
method visibility (line 357) | async visibility(path: string, options: MiscellaneousOptions): Promise...
method directoryExists (line 368) | async directoryExists(path: string, options: MiscellaneousOptions): Pr...
method ensureRootDirectoryExists (line 385) | private async ensureRootDirectoryExists(): Promise<void> {
method ensureParentDirectoryExists (line 395) | private async ensureParentDirectoryExists(path: string, options: Visib...
method checksum (line 405) | async checksum(path: string, options: ChecksumOptions): Promise<string> {
class LocalFileStorage (line 417) | class LocalFileStorage extends LocalStorageAdapter {}
FILE: packages/local-fs/src/unix-visibility.ts
type UnixVisibilityConversion (line 3) | interface UnixVisibilityConversion {
class PortableUnixVisibilityConversion (line 12) | class PortableUnixVisibilityConversion implements UnixVisibilityConversi...
method constructor (line 13) | constructor(
method defaultDirectoryPermissions (line 22) | get defaultDirectoryPermissions(): number {
method directoryPermissionsToVisibility (line 26) | directoryPermissionsToVisibility(permissions: number): string {
method filePermissionsToVisibility (line 34) | filePermissionsToVisibility(permissions: number): string {
method visibilityToDirectoryPermissions (line 42) | visibilityToDirectoryPermissions(visibility: string): number {
method visibilityToFilePermissions (line 52) | visibilityToFilePermissions(visibility: string): number {
FILE: packages/multer-storage/src/index.ts
type DestinationResolver (line 5) | type DestinationResolver = (action: 'handle' | 'remove', req: Request, f...
class FlystorageMulterStorageEngine (line 7) | class FlystorageMulterStorageEngine implements StorageEngine {
method constructor (line 8) | constructor(
method _handleFile (line 14) | _handleFile(req: Request, file: Express.Multer.File, callback: (error?...
method _removeFile (line 28) | _removeFile(req: Request, file: Express.Multer.File, callback: (error:...
FILE: packages/stream-mime-type/src/stream-mime-type.ts
function concatUint8Arrays (line 7) | function concatUint8Arrays(input: Uint8Array[]): Uint8Array {
function streamHead (line 19) | async function streamHead(stream: Readable, size: number): Promise<[Uint...
type FileTypePackage (line 50) | type FileTypePackage = typeof import('file-type');
function resolveMimeType (line 54) | async function resolveMimeType(
Condensed preview — 128 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (282K chars).
[
{
"path": ".github/actions/install-dependencies/action.yml",
"chars": 1098,
"preview": "name: 'Install NPM Dependencies'\ndescription: 'Install NPM Dependencies'\n\ninputs:\n cache-key:\n description: 'Cache k"
},
{
"path": ".github/dependabot.yml",
"chars": 273,
"preview": "\nversion: 2\nupdates:\n - package-ecosystem: \"npm\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n groups:\n"
},
{
"path": ".github/workflows/automerge-dependabot.yml",
"chars": 795,
"preview": "name: Auto-merge Dependabot PRs\n\non:\n pull_request_target:\n paths:\n - package.json\n - package-lock.json\n "
},
{
"path": ".github/workflows/compile.yml",
"chars": 803,
"preview": "name: Compile libraries\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\njobs:\n run-"
},
{
"path": ".github/workflows/qa-ubuntu-bun.yml",
"chars": 452,
"preview": "name: Ubuntu + Bun\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nconcurrency:\n gr"
},
{
"path": ".github/workflows/qa-ubuntu-node.yml",
"chars": 443,
"preview": "name: Ubuntu + Node\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nconcurrency:\n g"
},
{
"path": ".github/workflows/qa-windows-node.yml",
"chars": 421,
"preview": "name: Ubuntu + Windows\n\non:\n workflow_dispatch:\n \n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}-wi"
},
{
"path": ".github/workflows/shared-tests.yml",
"chars": 2892,
"preview": "name: Tests\n\non:\n workflow_call:\n secrets:\n GCS_KEY_CONTENTS:\n required: true\n AWS_ACCESS_KEY_ID:\n "
},
{
"path": ".gitignore",
"chars": 122,
"preview": "google-cloud-service-account.json\ndist\n.env\n**/*/dist\nnode_modules\nbin/*\n!bin/watch.ts\nfixtures/test-files/\n.node-versio"
},
{
"path": "bin/watch.ts",
"chars": 511,
"preview": "import {readdir} from 'node:fs/promises';\nimport path from 'node:path';\nimport {concurrently} from 'concurrently';\n\ncons"
},
{
"path": "docker/sftp/id_rsa",
"chars": 1679,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA0cx7jSu+FZQJFmz0+vlbo1I+awlaxvqZUseuPvX/YA32c3hE\nnr0OpEMhe/Pvs4jk/BiPp5C"
},
{
"path": "docker/sftp/id_rsa.pub",
"chars": 404,
"preview": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRzHuNK74VlAkWbPT6+VujUj5rCVrG+plSx64+9f9gDfZzeESevQ6kQyF78++ziOT8GI+nkJ8hMV7rKTFS"
},
{
"path": "docker/sftp/ssh_host_ed25519_key",
"chars": 444,
"preview": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\nQyNTUxOQAAACC"
},
{
"path": "docker/sftp/ssh_host_ed25519_key.pub",
"chars": 124,
"preview": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII9M/eD07xf0ivkrCtJWRhGqAJkUW3iLugcErLLfUeIJ frankdejonge@Frank-de-Jonges-MBP2015.lo"
},
{
"path": "docker/sftp/ssh_host_rsa_key",
"chars": 3422,
"preview": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn\nNhAAAAAwEAAQA"
},
{
"path": "docker/sftp/ssh_host_rsa_key.pub",
"chars": 768,
"preview": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDjxMG3KSIGMjrlkFYKk4eTzoRmsjEEXFvlakvS6MFKunTlQnx4p5eagR5+FHhWqWX06L56OYpdS++QfUzr"
},
{
"path": "docker/sftp/sshd_custom_configs.sh",
"chars": 183,
"preview": "#!/bin/bash\n\ncat <<'EOF' >> /etc/ssh/sshd_config\n\nKexAlgorithms curve25519-sha256\nCiphers aes256-gcm@openssh.com\nMACs hm"
},
{
"path": "docker/sftp/unknown.key",
"chars": 1679,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAxZQy1FnuqhoZsiqvFanN5tZRL6GVeCBeoLroGBEggJYVDLw2\nnPaXCDOaDoVzs1PkLoKhttA"
},
{
"path": "docker/sftp/users.conf",
"chars": 50,
"preview": "foo:pass:1001:100:upload\nbar:pass:1001:100:upload\n"
},
{
"path": "docker-compose.yml",
"chars": 1823,
"preview": "---\nversion: \"3\"\nservices:\n# sabredav:\n# image: php:8.1-alpine3.15\n# restart: always\n# volumes:\n# - ./:/v"
},
{
"path": "fixtures/adapter.template.ts",
"chars": 2929,
"preview": "import {Readable} from \"stream\";\nimport {\n StorageAdapter,\n ChecksumOptions,\n CreateDirectoryOptions,\n FileC"
},
{
"path": "package.json",
"chars": 1395,
"preview": "{\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build\": \"npm run compile -ws --if-present && npm run patch "
},
{
"path": "packages/aws-s3/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/aws-s3/CHANGELOG.md",
"chars": 369,
"preview": "# `@flystorage/aws-s3`\n\n## 1.2.1\n\n- Upgraded dependencies\n\n## 1.2.0\n\n## Changes\n\n- Make tests run on Bun\n\n## 1.1.1\n\n## C"
},
{
"path": "packages/aws-s3/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/aws-s3/README.md",
"chars": 886,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/aws-s3/package.json",
"chars": 1525,
"preview": "{\n \"name\": \"@flystorage/aws-s3\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@aws-sdk/client-s3\""
},
{
"path": "packages/aws-s3/src/aws-s3-file-storage.test.ts",
"chars": 18456,
"preview": "import {S3Client} from '@aws-sdk/client-s3';\nimport {\n FileStorage,\n readableToString,\n Visibility,\n closeRe"
},
{
"path": "packages/aws-s3/src/aws-s3-storage-adapter.ts",
"chars": 28838,
"preview": "import {\n _Object,\n CommonPrefix,\n CopyObjectCommand,\n CopyObjectRequest,\n DeleteObjectCommand,\n Delet"
},
{
"path": "packages/aws-s3/src/index.ts",
"chars": 44,
"preview": "export * from './aws-s3-storage-adapter.js';"
},
{
"path": "packages/aws-s3/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packages/azure-storage-blob/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/azure-storage-blob/CHANGELOG.md",
"chars": 223,
"preview": "# `@flystorage/azure-storage-blob`\n\n## 1.2.0\n\n## Changes\n\n- Support Bun runtime\n- Updated dependencies\n- 404 detection f"
},
{
"path": "packages/azure-storage-blob/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/azure-storage-blob/README.md",
"chars": 936,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/azure-storage-blob/package.json",
"chars": 1401,
"preview": "{\n \"name\": \"@flystorage/azure-storage-blob\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@azure/"
},
{
"path": "packages/azure-storage-blob/src/azure-storage-blob.test.ts",
"chars": 10296,
"preview": "import {BlobServiceClient} from \"@azure/storage-blob\";\nimport {AzureStorageBlobStorageAdapter} from \"./azure-storage-blo"
},
{
"path": "packages/azure-storage-blob/src/azure-storage-blob.ts",
"chars": 12422,
"preview": "import {Readable} from 'stream';\nimport {\n ChecksumIsNotAvailable,\n ChecksumOptions,\n CopyFileOptions,\n File"
},
{
"path": "packages/azure-storage-blob/src/index.ts",
"chars": 40,
"preview": "export * from './azure-storage-blob.js';"
},
{
"path": "packages/azure-storage-blob/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packages/chaos/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/chaos/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/chaos/README.md",
"chars": 1317,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/chaos/changelog.md",
"chars": 125,
"preview": "# `@flystorage/chaos`\n\n## 1.1.0\n\n### Changes\n\n- Added AbortSignal support\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 re"
},
{
"path": "packages/chaos/package.json",
"chars": 1358,
"preview": "{\n \"name\": \"@flystorage/chaos\",\n \"type\": \"module\",\n \"version\": \"1.1.0\",\n \"dependencies\": {\n \"@flystorage/file-sto"
},
{
"path": "packages/chaos/src/index.test.ts",
"chars": 3372,
"preview": "import { FileStorage } from \"@flystorage/file-storage\";\nimport {InMemoryStorageAdapter} from '@flystorage/in-memory';\nim"
},
{
"path": "packages/chaos/src/index.ts",
"chars": 5629,
"preview": "import {\n ChecksumOptions,\n CopyFileOptions,\n CreateDirectoryOptions,\n FileContents,\n MimeTypeOptions,\n "
},
{
"path": "packages/chaos/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packages/dynamic-import/.npmignore",
"chars": 32,
"preview": "index.test.ts\ntsconfig.json\nsrc/"
},
{
"path": "packages/dynamic-import/CHANGELOG.md",
"chars": 82,
"preview": "# `@flystorage/dynamic-import`\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release"
},
{
"path": "packages/dynamic-import/README.md",
"chars": 751,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Ut"
},
{
"path": "packages/dynamic-import/index.d.ts",
"chars": 76,
"preview": "export declare function dynamicallyImport<Pkg>(path: string): Promise<Pkg>;\n"
},
{
"path": "packages/dynamic-import/index.js",
"chars": 122,
"preview": "async function dynamicallyImport(path) {\n return await import(path);\n};\n\nexports.dynamicallyImport = dynamicallyImpor"
},
{
"path": "packages/dynamic-import/index.test.ts",
"chars": 337,
"preview": "import {dynamicallyImport} from './index.js';\n\n// @ts-ignore\ntype FileType = typeof import('file-type');\n\ndescribe('dyna"
},
{
"path": "packages/dynamic-import/package.json",
"chars": 723,
"preview": "{\n \"name\": \"@flystorage/dynamic-import\",\n \"type\": \"commonjs\",\n \"version\": \"1.0.0\",\n \"description\": \"Utility to dynam"
},
{
"path": "packages/dynamic-import/src/index.ts",
"chars": 31,
"preview": "//\nexport * from '../index.js';"
},
{
"path": "packages/file-storage/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/file-storage/CHANGELOG.md",
"chars": 558,
"preview": "# `@flystorage/file-storage`\n\n## 1.2.2\n\n### Fixes\n\n- Fixed typo in timeout options name\n\n## 1.2.1\n\n### Fixes\n\n- Prevent "
},
{
"path": "packages/file-storage/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/file-storage/README.md",
"chars": 3960,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/file-storage/package.json",
"chars": 1293,
"preview": "{\n \"name\": \"@flystorage/file-storage\",\n \"type\": \"module\",\n \"version\": \"1.2.2\",\n \"description\": \"File-storage abstrac"
},
{
"path": "packages/file-storage/src/checksum-from-stream.ts",
"chars": 932,
"preview": "import {BinaryToTextEncoding, createHash} from 'crypto';\nimport {Readable} from 'node:stream';\nimport {TextEncoder} from"
},
{
"path": "packages/file-storage/src/errors.ts",
"chars": 10821,
"preview": "export type ErrorContext = { [index: string]: any };\n\nexport type OperationError = Error & {\n readonly code: string,\n"
},
{
"path": "packages/file-storage/src/file-storage.test.ts",
"chars": 1724,
"preview": "import {FileStorage, UploadRequest, UploadRequestOptions} from './file-storage.js';\nimport {InMemoryStorageAdapter} from"
},
{
"path": "packages/file-storage/src/file-storage.ts",
"chars": 25591,
"preview": "import {BinaryToTextEncoding} from 'crypto';\nimport {Readable} from 'stream';\nimport {checksumFromStream} from './checks"
},
{
"path": "packages/file-storage/src/index.ts",
"chars": 222,
"preview": "export * from './checksum-from-stream.js';\nexport * from './file-storage.js';\nexport * from './errors.js';\nexport * from"
},
{
"path": "packages/file-storage/src/path-normalizer.test.ts",
"chars": 1439,
"preview": "import {CorruptedPathDetected, PathNormalizerV1, PathTraversalDetected} from './path-normalizer.js';\n\ndescribe('PathNorm"
},
{
"path": "packages/file-storage/src/path-normalizer.ts",
"chars": 1030,
"preview": "import {join} from 'node:path';\n\nexport interface PathNormalizer {\n normalizePath(path: string): string\n}\n\nconst funk"
},
{
"path": "packages/file-storage/src/path-prefixer.test.ts",
"chars": 2144,
"preview": "import {PathPrefixer} from './path-prefixer.js';\n\ndescribe('PathPrefixer', () => {\n describe.each([\n ['with', "
},
{
"path": "packages/file-storage/src/path-prefixer.ts",
"chars": 1074,
"preview": "import {type join, posix} from 'node:path';\n\nexport class PathPrefixer {\n private readonly prefix: string = '';\n c"
},
{
"path": "packages/file-storage/src/portable-visibility.ts",
"chars": 74,
"preview": "export enum Visibility {\n PUBLIC = 'public',\n PRIVATE = 'private',\n}"
},
{
"path": "packages/file-storage/src/readable-convertion.test.ts",
"chars": 394,
"preview": "import {Readable} from \"node:stream\";\nimport {readableToString} from \"./file-storage.js\";\n\ndescribe('converting readable"
},
{
"path": "packages/file-storage/src/utilities.test.ts",
"chars": 487,
"preview": "import {Readable} from 'stream';\nimport {readableToBuffer} from './file-storage.js';\n\ndescribe('utilities', () => {\n "
},
{
"path": "packages/file-storage/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packages/google-cloud-storage/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/google-cloud-storage/CHANGELOG.md",
"chars": 266,
"preview": "# `@flystorage/google-cloud-storage`\n\n## 1.2.0\n\n### Changes\n\n- Upgraded dependencies\n- Remove not-used import\n\n## 1.1.0\n"
},
{
"path": "packages/google-cloud-storage/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/google-cloud-storage/README.md",
"chars": 1610,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/google-cloud-storage/package.json",
"chars": 1481,
"preview": "{\n \"name\": \"@flystorage/google-cloud-storage\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@flys"
},
{
"path": "packages/google-cloud-storage/src/google-cloud-storage.test.ts",
"chars": 9849,
"preview": "import { Storage } from '@google-cloud/storage';\nimport {GoogleCloudStorageAdapter} from './google-cloud-storage.js';\nim"
},
{
"path": "packages/google-cloud-storage/src/google-cloud-storage.ts",
"chars": 9123,
"preview": "import {\n ChecksumIsNotAvailable,\n ChecksumOptions,\n CopyFileOptions,\n CreateDirectoryOptions,\n FileConte"
},
{
"path": "packages/google-cloud-storage/src/index.ts",
"chars": 84,
"preview": "export * from './google-cloud-storage.js';\nexport * from './visibility-handling.js';"
},
{
"path": "packages/google-cloud-storage/src/visibility-handling.ts",
"chars": 3005,
"preview": "import {Visibility} from '@flystorage/file-storage';\nimport {ApiError, File, PredefinedAcl} from '@google-cloud/storage'"
},
{
"path": "packages/google-cloud-storage/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packages/in-memory/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/in-memory/CHANGELOG.md",
"chars": 137,
"preview": "# `@flystorage/in-memory`\n\n## 1.1.0\n\n### Changes\n\n- Use a Buffer for internal storage\n\n## 1.0.0\n\n### Major Changes\n\n- Fi"
},
{
"path": "packages/in-memory/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/in-memory/README.md",
"chars": 715,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/in-memory/package.json",
"chars": 1380,
"preview": "{\n \"name\": \"@flystorage/in-memory\",\n \"type\": \"module\",\n \"version\": \"1.1.0\",\n \"dependencies\": {\n \"@flystorage/file"
},
{
"path": "packages/in-memory/src/in-memory-file-storage.test.ts",
"chars": 5933,
"preview": "import {FileStorage, Visibility} from \"@flystorage/file-storage\";\nimport {InMemoryStorageAdapter} from \"./in-memory-file"
},
{
"path": "packages/in-memory/src/in-memory-file-storage.ts",
"chars": 7771,
"preview": "import {\n ChecksumIsNotAvailable,\n ChecksumOptions,\n CopyFileOptions,\n CreateDirectoryOptions,\n FileConte"
},
{
"path": "packages/in-memory/src/index.ts",
"chars": 44,
"preview": "export * from './in-memory-file-storage.js';"
},
{
"path": "packages/in-memory/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packages/local-fs/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/local-fs/CHANGELOG.md",
"chars": 119,
"preview": "# `@flystorage/local-fs`\n\n## 1.1.0\n\n## Added\n\n- AbortSignal Support\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release"
},
{
"path": "packages/local-fs/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/local-fs/README.md",
"chars": 794,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/local-fs/package.json",
"chars": 1530,
"preview": "{\n \"name\": \"@flystorage/local-fs\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@flystorage/dynam"
},
{
"path": "packages/local-fs/src/index.ts",
"chars": 78,
"preview": "export * from './local-file-storage.js';\nexport * from './unix-visibility.js';"
},
{
"path": "packages/local-fs/src/local-file-storage.test.ts",
"chars": 12855,
"preview": "import {\n FileInfo,\n FileStorage,\n normalizeExpiryToMilliseconds, UnableToReadFile,\n UploadRequest,\n Visi"
},
{
"path": "packages/local-fs/src/local-file-storage.ts",
"chars": 14070,
"preview": "import {\n checksumFromStream,\n ChecksumOptions,\n CreateDirectoryOptions,\n PathPrefixer,\n PublicUrlOptions"
},
{
"path": "packages/local-fs/src/unix-visibility.ts",
"chars": 2106,
"preview": "import {Visibility} from '@flystorage/file-storage';\n\nexport interface UnixVisibilityConversion {\n visibilityToFilePe"
},
{
"path": "packages/local-fs/tsconfig.json",
"chars": 136,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"./sr"
},
{
"path": "packages/multer-storage/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/multer-storage/CHANGELOG.md",
"chars": 181,
"preview": "# `@flystorage/multer-storage`\n\n## 1.2.0\n\n### Changes\n\n- Force security upgrades\n\n## 1.1.0\n\n### Changes\n\n- Unblock secur"
},
{
"path": "packages/multer-storage/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/multer-storage/README.md",
"chars": 1051,
"preview": "<img src=\"https://raw.githubusercontent.com/duna-oss/flystorage/main/flystorage.svg\" width=\"50px\" height=\"50px\" />\n\n# Fl"
},
{
"path": "packages/multer-storage/package.json",
"chars": 1452,
"preview": "{\n \"name\": \"@flystorage/multer-storage\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@flystorage"
},
{
"path": "packages/multer-storage/src/index.test.ts",
"chars": 2541,
"preview": "import express, {Response, Request} from 'express';\nimport {FlystorageMulterStorageEngine} from './index.js';\nimport {Lo"
},
{
"path": "packages/multer-storage/src/index.ts",
"chars": 1416,
"preview": "import {StorageEngine} from 'multer';\nimport type {Request} from 'express';\nimport {FileStorage} from '@flystorage/file-"
},
{
"path": "packages/multer-storage/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packages/stream-mime-type/.npmignore",
"chars": 23,
"preview": "src\n!dist\ntsconfig.json"
},
{
"path": "packages/stream-mime-type/CHANGELOG.md",
"chars": 140,
"preview": "# `@flystorage/stream-mime-type`\n\n## 1.1.0\n\n### Minor Changes\n\n- Upgrade to file-type 20\n\n## 1.0.0\n\n### Major Changes\n\n-"
},
{
"path": "packages/stream-mime-type/LICENSE",
"chars": 1062,
"preview": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "packages/stream-mime-type/package.json",
"chars": 1427,
"preview": "{\n \"name\": \"@flystorage/stream-mime-type\",\n \"type\": \"module\",\n \"version\": \"1.1.0\",\n \"dependencies\": {\n \"@flystora"
},
{
"path": "packages/stream-mime-type/src/index.ts",
"chars": 38,
"preview": "export * from './stream-mime-type.js';"
},
{
"path": "packages/stream-mime-type/src/stream-mime-type.test.ts",
"chars": 1315,
"preview": "import * as fs from 'node:fs';\nimport path from 'node:path';\n\nimport {Readable} from 'stream';\nimport {resolveMimeType} "
},
{
"path": "packages/stream-mime-type/src/stream-mime-type.ts",
"chars": 2424,
"preview": "import {Readable} from 'node:stream';\nimport {parse} from 'node:path';\nimport {lookup as mimeTimeForExt} from 'mime-type"
},
{
"path": "packages/stream-mime-type/tsconfig.json",
"chars": 134,
"preview": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/"
},
{
"path": "packed/.gitignore",
"chars": 50,
"preview": "*.tgz\n*/node_modules/\n*/package-lock.json\n*/files/"
},
{
"path": "packed/cjs/index.js",
"chars": 554,
"preview": "const {FileStorage} = require('@flystorage/file-storage');\nconst {LocalStorageAdapter} = require('@flystorage/local-fs')"
},
{
"path": "packed/cjs/index.test.js",
"chars": 499,
"preview": "const {FileStorage} = require('@flystorage/file-storage');\nconst {LocalStorageAdapter} = require('@flystorage/local-fs')"
},
{
"path": "packed/cjs/package.json",
"chars": 564,
"preview": "{\n \"private\": true,\n \"type\": \"commonjs\",\n \"name\": \"packed-cjs\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": "
},
{
"path": "packed/esm/index.js",
"chars": 496,
"preview": "import {FileStorage} from '@flystorage/file-storage';\nimport {LocalStorageAdapter} from '@flystorage/local-fs';\nimport {"
},
{
"path": "packed/esm/package.json",
"chars": 513,
"preview": "{\n \"private\": true,\n \"type\": \"module\",\n \"name\": \"packed-esm\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": \"i"
},
{
"path": "tsconfig.build.json",
"chars": 319,
"preview": "{\n \"exclude\": [\n \"**/*.test.ts\",\n \"node_modules\",\n \"dist\"\n ],\n \"compilerOptions\": {\n \"esModuleInterop\": f"
},
{
"path": "tsconfig.json",
"chars": 572,
"preview": "{\n \"exclude\": [\n \"node_modules\",\n \"dist\",\n \"**/dist/**/*.js\"\n ],\n \"include\": [\n \"./**/*.ts\"\n ],\n \"compi"
},
{
"path": "vitest.config.ts",
"chars": 1111,
"preview": "import { defineConfig } from 'vitest/config';\nimport tsconfigPaths from 'vite-tsconfig-paths';\nimport path from 'node:pa"
}
]
About this extraction
This page contains the full source code of the duna-oss/flystorage GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 128 files (258.2 KB), approximately 65.7k tokens, and a symbol index with 379 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.