[ { "path": ".github/actions/install-dependencies/action.yml", "content": "name: 'Install NPM Dependencies'\ndescription: 'Install NPM Dependencies'\n\ninputs:\n cache-key:\n description: 'Cache key'\n required: true\n cache-restore-keys:\n description: 'Cache restore keys'\n required: true\noutputs:\n cache-hit:\n description: 'Whether or not the cache was hit'\n value: ${{ steps.npm-cache.outputs.cache-hit }}\nruns:\n using: 'composite'\n steps:\n - run: mkdir ~/.npm-cache\n shell: sh\n name: Create NPM cache directory\n - name: Load NPM cache\n id: npm-cache\n uses: actions/cache/restore@v3\n with:\n path: |\n ~/.npm-cache\n node_modules\n key: ${{ inputs.cache-key }}\n restore-keys: ${{ inputs.cache-restore-keys }}\n - name: Install NPM dependencies\n if: steps.npm-cache.outputs.cache-hit != 'true'\n run: npm ci --prefer-offline --no-audit\n shell: sh\n - name: Save NPM cache\n if: steps.npm-cache.outputs.cache-hit != 'true'\n uses: actions/cache/save@v3\n with:\n key: ${{ inputs.cache-key }}\n path: |\n ~/.npm-cache\n node_modules\n" }, { "path": ".github/dependabot.yml", "content": "\nversion: 2\nupdates:\n - package-ecosystem: \"npm\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n groups:\n dependencies:\n patterns:\n - \"*\"\n\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n" }, { "path": ".github/workflows/automerge-dependabot.yml", "content": "name: Auto-merge Dependabot PRs\n\non:\n pull_request_target:\n paths:\n - package.json\n - package-lock.json\n branches:\n - main\n\npermissions:\n pull-requests: write\n contents: write\n\njobs:\n merge-dependabot-pr:\n name: Merge dependabot PR\n runs-on: ubuntu-latest\n if: ${{ github.actor == 'dependabot[bot]' }}\n steps:\n - name: Fetch Dependabot metadata\n id: meta\n uses: dependabot/fetch-metadata@v2\n with:\n github-token: ${{ secrets.GITHUB_TOKEN }}\n - name: Merge PR\n if: ${{ steps.meta.outputs.update-type != 'version-update:semver-major' }}\n run: gh pr merge --auto --squash \"$PR_URL\"\n env:\n PR_URL: ${{ github.event.pull_request.html_url }}\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/compile.yml", "content": "name: Compile libraries\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\njobs:\n run-tests:\n runs-on: ${{ matrix.platform }}\n strategy:\n matrix:\n platform: [ubuntu-latest]\n node: [22] # latest LTS version\n name: Compile libraries\n steps:\n - name: Checkout the repository\n uses: actions/checkout@v6\n - name: Install Node.js\n uses: actions/setup-node@v6\n with:\n node-version: ${{ matrix.node }}\n - uses: ./.github/actions/install-dependencies\n name: Install NPM dependencies\n with:\n cache-key: npm-${{ matrix.node }}-${{ hashFiles('package-lock.json') }}\n cache-restore-keys: npm-caches-${{ matrix.node }}\n - name: Run build\n run: npm run build:only\n" }, { "path": ".github/workflows/qa-ubuntu-bun.yml", "content": "name: Ubuntu + Bun\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}-ubuntu-bun\n cancel-in-progress: true\n\njobs:\n tests:\n uses: ./.github/workflows/shared-tests.yml\n secrets: inherit\n with:\n os: ubuntu-latest\n runtime: bun\n # as string for inputs workaround not accepting arrays\n versions: '[\"1.2\"]'\n experimental: true" }, { "path": ".github/workflows/qa-ubuntu-node.yml", "content": "name: Ubuntu + Node\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}-ubuntu-node\n cancel-in-progress: true\n\njobs:\n tests:\n uses: ./.github/workflows/shared-tests.yml\n secrets: inherit\n with:\n os: ubuntu-latest\n runtime: node\n # as string for inputs workaround not accepting arrays\n versions: '[20, 21, 22, 23, 24]'" }, { "path": ".github/workflows/qa-windows-node.yml", "content": "name: Ubuntu + Windows\n\non:\n workflow_dispatch:\n \n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}-windows-node\n cancel-in-progress: true\n\njobs:\n tests:\n uses: ./.github/workflows/shared-tests.yml\n secrets: inherit\n with:\n experimental: true\n os: windows-latest\n runtime: node\n # as string for inputs workaround not accepting arrays\n versions: '[20, 21, 22, 23, 24]'" }, { "path": ".github/workflows/shared-tests.yml", "content": "name: Tests\n\non:\n workflow_call:\n secrets:\n GCS_KEY_CONTENTS:\n required: true\n AWS_ACCESS_KEY_ID:\n required: true\n AWS_SECRET_ACCESS_KEY:\n required: true\n AWS_REGION:\n required: true\n AZURE_DSN:\n required: true\n inputs:\n os:\n required: true\n type: string\n runtime:\n required: true\n type: string\n versions:\n required: true\n type: string\n experimental:\n required: false\n type: boolean\n default: false\n\n#permissions:\n# id-token: write # This is required for requesting the JWT\n# contents: read # This is required for actions/checkout\n# pull-requests: read # Required for change detection\n\njobs:\n run-tests:\n runs-on: ${{ inputs.os }}\n continue-on-error: ${{ inputs.experimental }}\n strategy:\n matrix:\n version: ${{ fromJSON(inputs.versions) }}\n name: ${{ inputs.os }} - ${{ inputs.runtime }} - ${{ matrix.version }}\n steps:\n - name: Checkout the repository\n uses: actions/checkout@v6\n\n - name: Install Node.js\n if: inputs.runtime == 'node'\n uses: actions/setup-node@v6\n with:\n node-version: ${{ matrix.version }}\n - uses: ./.github/actions/install-dependencies\n if: inputs.runtime == 'node'\n name: Install NPM dependencies\n with:\n cache-key: ${{inputs.os}}-${{inputs.runtime}}-npm-${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}\n cache-restore-keys: ${{inputs.os}}-${{inputs.runtime}}-npm-${{ matrix.node }}\n\n - uses: oven-sh/setup-bun@v2\n if: inputs.runtime == 'bun'\n with:\n bun-version: ${{ inputs.version }}\n - name: Install NPM dependencies using Bun\n if: inputs.runtime == 'bun'\n run: bun install\n\n - name: Configure GCP access\n run: |\n echo $CONTENTS > google-cloud-service-account.json\n env:\n CONTENTS: \"${{ secrets.GCS_KEY_CONTENTS }}\"\n\n - name: Run relevant tests\n run: \"npm run test\"\n if: inputs.runtime == 'node'\n env:\n AWS_ACCESS_KEY_ID: \"${{ secrets.AWS_ACCESS_KEY_ID }}\"\n AWS_SECRET_ACCESS_KEY: \"${{ secrets.AWS_SECRET_ACCESS_KEY }}\"\n AWS_REGION: \"${{ secrets.AWS_REGION }}\"\n AZURE_DSN: \"${{ secrets.AZURE_DSN }}\"\n - name: Run relevant tests using Bun\n run: |\n npm run build:only\n bun test packages\n if: inputs.runtime == 'bun'\n env:\n AWS_ACCESS_KEY_ID: \"${{ secrets.AWS_ACCESS_KEY_ID }}\"\n AWS_SECRET_ACCESS_KEY: \"${{ secrets.AWS_SECRET_ACCESS_KEY }}\"\n AWS_REGION: \"${{ secrets.AWS_REGION }}\"\n AZURE_DSN: \"${{ secrets.AZURE_DSN }}\"\n - name: Delete GCP access\n if: always()\n run: |\n node_modules/.bin/rimraf google-cloud-service-account.json\n\n" }, { "path": ".gitignore", "content": "google-cloud-service-account.json\ndist\n.env\n**/*/dist\nnode_modules\nbin/*\n!bin/watch.ts\nfixtures/test-files/\n.node-version\n" }, { "path": "bin/watch.ts", "content": "import {readdir} from 'node:fs/promises';\nimport path from 'node:path';\nimport {concurrently} from 'concurrently';\n\nconst directories = (await readdir(path.resolve(process.cwd(), 'packages'), {\n withFileTypes: true,\n}))\n .filter(item => item.isDirectory())\n .map(dir => path.parse(dir.name).name);\n\nconst commands = directories.map(directory => ({\n name: directory,\n command: `npm run watch -w ./packages/${directory} --if-present`\n}));\n\nconsole.log('starting', commands);\nconcurrently(commands);" }, { "path": "docker/sftp/id_rsa", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA0cx7jSu+FZQJFmz0+vlbo1I+awlaxvqZUseuPvX/YA32c3hE\nnr0OpEMhe/Pvs4jk/BiPp5CfITFe6ykxUvNlvxh/9EOsnUFaFrfyLhYn3TicW6lv\nzSJmJ7tydW/L0zpMPliItbjtXItxHOyoVcl20+DT+LZWlRRnnuSDTC1Oc6vvLosw\nb4lOTNVB2kV7W9urU6W+OilcERwjjQpiIM4mkwg2dX0aPWS5o0N9tYi3+k6gStqj\nDfcTw3L8tTWeDcWrblvFaknFfBzkeXx9dTyrAYw3ii3LZQ6hA5UK8tUgwUVThTh+\nQBhzhmn14Ljfp1fa3vgljY9yn5Zw+Drhbtn4TwIDAQABAoIBACaZstnEhJK/y/Q+\nU8yheITSKv3SmMsnbHJYnuyiojvwFbolFKsIKdt7JnwB48Zql4bylevEpiKbTNWD\nnLmgYsYIIfK1SNseHQ81BPAJz4faVJpg0FszywvgZyzIRv40KbcG3xBgV/vBBCzI\nNiiiiqRtJ1MJaWDAglgvvyCS7W5GjHue74o/wPugyvIi7zmMqMmZLg3zz/YF3zxU\nh3JGFnw/TlN3yDJzdAOFiLGpSRD/bMcBJyWc9Oa2LQj+Emkc7dojmB/IJ7wJ9iuo\nNi2/EJIQoFtVYxK6r+nRVuvT6/siMi7Ok4EL7PANVNVA7s7e+rmZX3NjDiGnvh5V\nDyxdT5ECgYEA/UFm3qxLiSQwNGuQLpCO9WR+4eeZPthNI1ySkcAnGyrmxX70ws7M\n5j1jOIYUhf8i1ALUu++sBfvd+OhD0yeRgFvqrCdeHdFnAdgOtQDUGfU9bh1YpEX0\n4A7zIxK7TBi26hXb/fTu7YwDtv58W1BtOOfo9ZKOWPu1ZQbEG5fp9pMCgYEA1BKF\nMRWshqBOSqRhyrl34wXTr6EI1XcwIDNyAGJnQFlbxWcCuPYCkF+ENZOe8D6soXiZ\ntTW6zSkM+UmsNgEYjD5b0eWgd0yUdrOi/Z4JZ16SlroeLYfbqVYY6NPE6XEuzZ9p\nXEB5WedUGIdU2JZQ422m7BxpJBIRtlbhG1Oi8NUCgYEA9Y3MaGs2ciqccrc4fW28\nr0JZpEAi3kRrxrWjh56ATF80kpmeSKSrFzK+WbfnfmT7KAX2rqKccNDdUNIjsUDU\nW1jEGVeyccbv0WHkIKxE+0ZF4daic+VAoV7dcExhPk9YS3AWdg5e/ASeNXhaq084\nF80Em9cWHkEwiFwfGYIaX/ECgYEAnUqfPyi0LaX4a6RAY/vrz5Yiy8DErI8aQsfl\nZiOWMUQVrPQaMNVGUY6GoLY8zDOwFpM8bgrL4h7wYHUkJWnqqxoVQDjwK4vBEclq\nunDcyK58Sw8AEwURBye0kft/sSUhcaEqpCGt3+CTnx3A8GOM2yIZDEaGNRqxyGvn\nyjzePYECgYBqr0OQJUDUOGcL+V5z8wCaLN2bbWK/bukaGe3ZU7W2kWhNvVL0SBNC\ndUcxLp5ELOj8wFHmTvZv1q2aszIY2CQhnc1qGYCh34UAK14DkTvSYie9XcGPhkWK\nUC9VLorkPbPAZipbBs9Dt079ub3HRyeXX4b0VHHKzu2XVGXVEOXO6A==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "docker/sftp/id_rsa.pub", "content": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRzHuNK74VlAkWbPT6+VujUj5rCVrG+plSx64+9f9gDfZzeESevQ6kQyF78++ziOT8GI+nkJ8hMV7rKTFS82W/GH/0Q6ydQVoWt/IuFifdOJxbqW/NImYnu3J1b8vTOkw+WIi1uO1ci3Ec7KhVyXbT4NP4tlaVFGee5INMLU5zq+8uizBviU5M1UHaRXtb26tTpb46KVwRHCONCmIgziaTCDZ1fRo9ZLmjQ321iLf6TqBK2qMN9xPDcvy1NZ4NxatuW8VqScV8HOR5fH11PKsBjDeKLctlDqEDlQry1SDBRVOFOH5AGHOGafXguN+nV9re+CWNj3KflnD4OuFu2fhP your_email@example.com\n" }, { "path": "docker/sftp/ssh_host_ed25519_key", "content": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\nQyNTUxOQAAACCPTP3g9O8X9Ir5KwrSVkYRqgCZFFt4i7oHBKyy31HiCQAAALBMCYEjTAmB\nIwAAAAtzc2gtZWQyNTUxOQAAACCPTP3g9O8X9Ir5KwrSVkYRqgCZFFt4i7oHBKyy31HiCQ\nAAAEAsRUm7efqtLmLIUwhaHzPnKf+hUubVc+xr49XcILRW5Y9M/eD07xf0ivkrCtJWRhGq\nAJkUW3iLugcErLLfUeIJAAAAKmZyYW5rZGVqb25nZUBGcmFuay1kZS1Kb25nZXMtTUJQMj\nAxNS5sb2NhbAECAw==\n-----END OPENSSH PRIVATE KEY-----\n" }, { "path": "docker/sftp/ssh_host_ed25519_key.pub", "content": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII9M/eD07xf0ivkrCtJWRhGqAJkUW3iLugcErLLfUeIJ frankdejonge@Frank-de-Jonges-MBP2015.local\n" }, { "path": "docker/sftp/ssh_host_rsa_key", "content": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn\nNhAAAAAwEAAQAAAgEA48TBtykiBjI65ZBWCpOHk86EZrIxBFxb5WpL0ujBSrp05UJ8eKeX\nmoEefhR4Vqll9Oi+ejmKXUvvkH1M638N35SFq+S3HR8TsMvgn5MaWO5HH3F5zzzLzyZjtZ\nH/K6mroMhG1b/DZoG3s4A5jN1i2ByRJS9d65q+Txd9ItlEifiCs7YYDS/5CkMWp3I6+i64\nCmARpMUvfVwpclFFIaxnlevymRIwSxzUoZxGuLYR2+QUx/4sbx41CIOLwEWrxf0oq9+DS3\neyp+av4itLo47ndLfNEFexUHSPE8IofGuKAIUkHrPMVwLoFT+FbIDVH6vnU9XawIpbHFev\nju2Mopb3DS9CKLLziQhBSxwCgsJmFyTqzkPZ3YeSsaz22o+vW98HRnp6TTtv+iefArNRPr\ngikBdFEjK8jWwZWV6MyeU5u4fZngMghV3DOfBjTlhrONI50iicM1sNKEvUqzEqCL/fs+zX\nV04aeDyNw+O6Xy1uIT+2Ozo5bHMoeyqqM33Zb7CB+polZQckFrXZAS59KLRYy2EpNQVzNY\nUThywQDPk5fkJ1CczHwlAzLlNZdxnsG69+JYGWlqKu//dT4AJPuYNDVW7SxPec8+yYq7uu\nDZnsnp8in/0tiyr4AS4fnAOszIndqRwwzqm13BVIDNJbdwgJTRDC+3wBT6hjnPC0ktACtN\nsAAAdo+KBGYPigRmAAAAAHc3NoLXJzYQAAAgEA48TBtykiBjI65ZBWCpOHk86EZrIxBFxb\n5WpL0ujBSrp05UJ8eKeXmoEefhR4Vqll9Oi+ejmKXUvvkH1M638N35SFq+S3HR8TsMvgn5\nMaWO5HH3F5zzzLzyZjtZH/K6mroMhG1b/DZoG3s4A5jN1i2ByRJS9d65q+Txd9ItlEifiC\ns7YYDS/5CkMWp3I6+i64CmARpMUvfVwpclFFIaxnlevymRIwSxzUoZxGuLYR2+QUx/4sbx\n41CIOLwEWrxf0oq9+DS3eyp+av4itLo47ndLfNEFexUHSPE8IofGuKAIUkHrPMVwLoFT+F\nbIDVH6vnU9XawIpbHFevju2Mopb3DS9CKLLziQhBSxwCgsJmFyTqzkPZ3YeSsaz22o+vW9\n8HRnp6TTtv+iefArNRPrgikBdFEjK8jWwZWV6MyeU5u4fZngMghV3DOfBjTlhrONI50iic\nM1sNKEvUqzEqCL/fs+zXV04aeDyNw+O6Xy1uIT+2Ozo5bHMoeyqqM33Zb7CB+polZQckFr\nXZAS59KLRYy2EpNQVzNYUThywQDPk5fkJ1CczHwlAzLlNZdxnsG69+JYGWlqKu//dT4AJP\nuYNDVW7SxPec8+yYq7uuDZnsnp8in/0tiyr4AS4fnAOszIndqRwwzqm13BVIDNJbdwgJTR\nDC+3wBT6hjnPC0ktACtNsAAAADAQABAAACAQCESra9HK3/bVNaHNBsyi2YAv5R67OetcpG\nYMvzj28daVkWA9zp82WRvucoEdmndDKc4kYoFZ2w/LcDdFOmAKDdOJW/NlPJHVDBgllQNg\n+6kYNL1wwJ+2ThR4noXwkXoi/mbgz+f6gNtNAu+Q30LG4J2eXP9EgX3UQmCh2LjShK/sVj\nfiNQHYoHlNnmnel1gIcyt4Pn8QPZSxtjo6KEoW9025uHntHf/rnduDg3dsC+uCX91zqVu7\nTP4h/cqFrR322tDmBjB/4DmXCU69K+B/WVjGAV2ulJMrobns0HHysDjFFjZ8kKzMxh8wga\n8mVXRPBSeEbbSEENIDz+xijGEusga3EK3I4LMNaAN47oulrVM++oF1pHL74jiEAC1SGHTv\n0ZOZdCDmdmPyxCC9ruvJmQb3IbvoVyROcrHd4AvYSKnRqE2ljzLnFHG+Qp13meoXNz8Bup\nNO9ra+HJYWu6QgqHnPvwrIP5NdmwwSRWunglOvVSO2c11X7Li/NBkLLLk9wX1w9awAPfTf\nnUGCFnKi+1ianXD94zEG0FyXImN1eRK2lgs8ul6wGr3qV5B3sv7llgyWF5ZKLLj9iQ79M5\nOKret5evybFbEbIOWbRTTcREfmSLqcKv2g6ZJpBNI8z4bz0jgc4hEVcMJlyiH52v3yMePh\nf7tXvv62Ws9IHLNhnSkQAAAQEApXAZ5Ns3aUOuCVvK3mEUWuQEsh/aM+rdrthX+zrw7Ma7\nkw2O6zh6VviP5BV+8qG7sx1KyK9b0u+/qGDy4pMHN8SjNq1W+pTZ4FHkA/1UKI0kTvTwq/\nWQvO1pBxLbx77gl5FuBjysPljnvMh3+l9QOFX4iwjFoyep1FgammuD4hnSR1yWuF3H/2xc\nsb75LnXdaEDm5ALyauTwmzdufTCaQmq19y8hZ5Kbhxd/Pao160tm38n2gektN458AbGxri\n8tHSwpj4+3PSOwFeFalxk/jO8daye4PluoqK+O60Z5NWqdDDaBdvxEMcj9sddUQaQ0BzmK\nFci3Hl6vS9TzBUWNOQAAAQEA/hDQkIH3ZIdZYBWIsinTKoztovUZHfHZihkBxcR0IbKZkn\n7HGgtNtv1QshSvpzMp7HsAT45SqzIbN4ldNh0aSRmoOb2/pVNVNeJrWR0B+aS9hya6bAOY\nAiZlfMMo+2cU+gJzGD50tlu7ge4hT3YkN3vmoDyJZ2p7DxOioQQ10D5Pno8ZGh8kZX3t5d\nF+5GwPnsYXsi/iMgw18ggPTBPwAP+TeUsdP8Ae93h33LgCOkUfXeq/HmQ0jTUT84lxd9tW\n2N/w6P/2SYa85oFVa2Y+u0qRJOLf6zJKTOH/ERV9IJYGLRzyxybQuo4iqw9V+kP13KkC2w\n6vh2RcmEwXmxXRyQAAAQEA5YCwEjBqYLlPzQ3lOf75SbaT5+CIiyNy0P0ZUvpBl+URJkGC\nm8/AA3naWylEP/oUJRUkv21qXB8EEL62404nhNVtfDGX1mpht8H4N9lcXAdNFg1SWmqnqb\n58PTODzM4n9CVjqnqqlTGQi1ph77RLDVXc29UooopLtYFsttiFBvv8Cs/cKx3t1ECQRklU\nhjPSdijeRTtv5x604uXt6mObYIGMAC87OSP4iaY00Dz17lnJ9m9UK5BCX+44sIEf3IQ9Ad\njfaeN5UoUTv+y/fB419baC+dlIR9+SaEFqNgMHQqpyZApWMhCf99oNQhJ9XDjVEtKtXnhc\n5Lh4tYyNGMsDgwAAACpmcmFua2Rlam9uZ2VARnJhbmstZGUtSm9uZ2VzLU1CUDIwMTUubG\n9jYWwBAgMEBQYH\n-----END OPENSSH PRIVATE KEY-----\n" }, { "path": "docker/sftp/ssh_host_rsa_key.pub", "content": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDjxMG3KSIGMjrlkFYKk4eTzoRmsjEEXFvlakvS6MFKunTlQnx4p5eagR5+FHhWqWX06L56OYpdS++QfUzrfw3flIWr5LcdHxOwy+CfkxpY7kcfcXnPPMvPJmO1kf8rqaugyEbVv8NmgbezgDmM3WLYHJElL13rmr5PF30i2USJ+IKzthgNL/kKQxancjr6LrgKYBGkxS99XClyUUUhrGeV6/KZEjBLHNShnEa4thHb5BTH/ixvHjUIg4vARavF/Sir34NLd7Kn5q/iK0ujjud0t80QV7FQdI8Twih8a4oAhSQes8xXAugVP4VsgNUfq+dT1drAilscV6+O7YyilvcNL0IosvOJCEFLHAKCwmYXJOrOQ9ndh5KxrPbaj69b3wdGenpNO2/6J58Cs1E+uCKQF0USMryNbBlZXozJ5Tm7h9meAyCFXcM58GNOWGs40jnSKJwzWw0oS9SrMSoIv9+z7NdXThp4PI3D47pfLW4hP7Y7Ojlscyh7KqozfdlvsIH6miVlByQWtdkBLn0otFjLYSk1BXM1hROHLBAM+Tl+QnUJzMfCUDMuU1l3Gewbr34lgZaWoq7/91PgAk+5g0NVbtLE95zz7Jiru64NmeyenyKf/S2LKvgBLh+cA6zMid2pHDDOqbXcFUgM0lt3CAlNEML7fAFPqGOc8LSS0AK02w== frankdejonge@Frank-de-Jonges-MBP2015.local\n" }, { "path": "docker/sftp/sshd_custom_configs.sh", "content": "#!/bin/bash\n\ncat <<'EOF' >> /etc/ssh/sshd_config\n\nKexAlgorithms curve25519-sha256\nCiphers aes256-gcm@openssh.com\nMACs hmac-sha2-256-etm@openssh.com\nHostKeyAlgorithms ssh-ed25519\n\nEOF\n" }, { "path": "docker/sftp/unknown.key", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAxZQy1FnuqhoZsiqvFanN5tZRL6GVeCBeoLroGBEggJYVDLw2\nnPaXCDOaDoVzs1PkLoKhttA6g4l4eHZgDL1b6QTxRnDbkdOxKJHGEapk+WZk23yq\nQmU4thDdmgj0clJfS3aTbF7hGWpxrv9zB5saWwgWB/fBeYXu+kXuuO1UJF+F0QGt\n6TVVMc6o3NaI7Cq1qoMvt7XGN5LegwfNWRGxaKTL1yn7pkljhkPLOoGedBemieya\naQhV+dwzOVeBAXGKSmtiUwto8OMYl/suptMdo+91cLMRJkgXNQ5gA+4niqja5HGW\n2Dw8vXja2Spp4+gWWh5OpXwd2pNnJScNINL30wIDAQABAoIBAHzBD7s/sdAcPN9f\nzj+qgUVhS8/8gilgnv90JPqVTeWDXnU1HnLLzR+znXHP1/eCYBDyEPQi1N+bXMML\nU6iXpEIlCcfFmQ6iETmhmeQrqChF/CcOt17HFSD400PgpaDN3DgE/h8uZYmryW6L\nA3HpAKI8H9UWHkcCR5wlrg98Y2W3AacHMwRG8da7QxLcGd10+1lFLuPXTu7LwDf0\nu/+Jq9IPwm8WemGiuuOEQ06r7y1s929pA7fWSsn8DlVBIFnPHCtGKpiDfAyEHmCh\n/LUU6Z5Cl9186sGohu5hWUA+qTMAHw9IR0TbWxF+tAEX8qL5z/qH/surRvzfeRCi\nBooQB4ECgYEA99xz7AWMUzu6J4FSm29M2AS989Yz0XAgGIYs69pxoaRR5azRffY+\nF6wjBpeSI5BElu1DAublhv5hpCgdcw6X2EfMnKvP1jtSbv+DOdR0p80LVAnG7tb3\nstSgNwMsGdBpTCghHKUvOUGDU3D3c9O8jCrrJd9Unu4gj2/xbVuC45MCgYEAzBER\nXeuKuwrN0HmLCZ7AJ1aq7A60UKJjMicdXD5n95ePwHQ32s+hmrcGfUOjcZoWc1Dp\nyZaZvqFDYu52ZJ1QbNZqjBQJ8sjERSxNlV8DZe1I3ohXvsRQ4I6g8/HRgSl0CVqg\n+LeJA7n18+SrhhCWhY+dggV1m4qCNcWI7FCQwsECgYEAw+V7xT35U0twbIq8lFba\nQB03WEGiwNRCub9KP7ptdtjdVY5KIKj/GEyXfj1LZko+u56YCPIe1Ju25jxCUk5l\nWq4cnHL6mBJYq5vMxmcRMBJR8sCrdtd1++QrIG+kal6a6nMJAI/ZjAIoXkl5ehUN\n/yZopY0mX1pLZ7KM+OaLw3sCgYEAvaACllbA9Gvmspmu5IKbJjL34yEK137+VGVa\neBQZgk5ZK0oTeQXFssHuisomf/Lid8exZzzFowmxV6YlZ/ty96ALJB2e3PdIwsqX\nUX0X6EgllXv2pXNBgFmpIOYNe0ts4yBPQq8x57+O2FMePBb/+B5rC55NGfsMYjEr\nugRncEECgYB9Wrnp4/WpTwcSJTJXWB+FJaMmcXqbFLk/AI1HbRXJFmR/ssK1gugQ\nY3vevvRMSJkYkd1Zmbp9x8ZAGKzR/yeT8lQ6gY/xYq1fYLKKKtzHdwTMCVIVDgSf\n/v9Y8mLylcgr7ZjEo8xMTnb98ozSSFPBn2jq8c31AsQxXaIMK0BE7g==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "docker/sftp/users.conf", "content": "foo:pass:1001:100:upload\nbar:pass:1001:100:upload\n" }, { "path": "docker-compose.yml", "content": "---\nversion: \"3\"\nservices:\n# sabredav:\n# image: php:8.1-alpine3.15\n# restart: always\n# volumes:\n# - ./:/var/www/html/\n# ports:\n# - \"4040:4040\"\n# command: php -S 0.0.0.0:4040 /var/www/html/src/WebDAV/resources/server.php\n# webdav:\n# image: bytemark/webdav\n# restart: always\n# ports:\n# - \"4080:80\"\n# environment:\n# AUTH_TYPE: Digest\n# USERNAME: alice\n# PASSWORD: secret1234\n# ANONYMOUS_METHODS: 'GET,OPTIONS'\n sftp:\n container_name: sftp\n restart: always\n image: atmoz/sftp:alpine\n volumes:\n - ./docker/sftp/users.conf:/etc/sftp/users.conf\n - ./docker/sftp/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key\n - ./docker/sftp/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key\n - ./docker/sftp/id_rsa.pub:/home/bar/.ssh/keys/id_rsa.pub\n ports:\n - \"2222:22\"\n sftp_eddsa_only:\n container_name: sftp_eddsa_only\n restart: always\n image: atmoz/sftp:alpine\n volumes:\n - ./docker/sftp/users.conf:/etc/sftp/users.conf\n - ./docker/sftp/sshd_custom_configs.sh:/etc/sftp.d/sshd_custom_configs.sh\n - ./docker/sftp/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key\n ports:\n - \"2223:22\"\n# ftp:\n# container_name: ftp\n# restart: always\n# image: delfer/alpine-ftp-server\n# environment:\n# USERS: 'foo|pass|/home/foo/upload'\n# ADDRESS: 'localhost'\n# ports:\n# - \"2121:21\"\n# - \"21000-21010:21000-21010\"\n\n azurite:\n image: mcr.microsoft.com/azure-storage/azurite\n profiles: [\"azure\"]\n restart: unless-stopped\n healthcheck:\n test:\n wget --no-verbose --tries=1 --spider http://localhost:10000/ || exit 1\n interval: 10s\n timeout: 10s\n retries: 3\n start_period: 10s\n ports:\n - 10000:10000\n - 10001:10001\n - 10002:10002\n" }, { "path": "fixtures/adapter.template.ts", "content": "import {Readable} from \"stream\";\nimport {\n StorageAdapter,\n ChecksumOptions,\n CreateDirectoryOptions,\n FileContents, MimeTypeOptions,\n PublicUrlOptions,\n StatEntry, TemporaryUrlOptions,\n WriteOptions,\n PathPrefixer,\n CopyFileOptions,\n MoveFileOptions,\n} from \"@flystorage/file-storage\";\n\nexport type AdapterFileStorageOptions = {\n prefix?: string,\n}\n\nexport class AdapterFileStorage implements StorageAdapter {\n private readonly prefixer: PathPrefixer;\n\n constructor(\n private readonly options: AdapterFileStorageOptions = {},\n ) {\n this.prefixer = new PathPrefixer(options.prefix || '');\n }\n\n async write(path: string, contents: Readable, options: WriteOptions): Promise {\n throw new Error('Not implemented');\n }\n\n async read(path: string): Promise {\n throw new Error('Not implemented');\n }\n async deleteFile(path: string): Promise {\n throw new Error('Not implemented');\n }\n async createDirectory(path: string, options: CreateDirectoryOptions): Promise {\n throw new Error('Not implemented');\n }\n async stat(path: string): Promise {\n throw new Error('Not implemented');\n }\n list(path: string, options: {deep: boolean}): AsyncGenerator {\n throw new Error('Not implemented');\n }\n async changeVisibility(path: string, visibility: string): Promise {\n throw new Error('Not implemented');\n }\n async visibility(path: string): Promise {\n throw new Error('Not implemented');\n }\n async deleteDirectory(path: string): Promise {\n throw new Error('Not implemented');\n }\n async fileExists(path: string): Promise {\n throw new Error('Not implemented');\n }\n async directoryExists(path: string): Promise {\n throw new Error('Not implemented');\n }\n async publicUrl(path: string, options: PublicUrlOptions): Promise {\n throw new Error('Not implemented');\n }\n async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n throw new Error('Not implemented');\n }\n async checksum(path: string, options: ChecksumOptions): Promise {\n throw new Error('Not implemented');\n }\n async mimeType(path: string, options: MimeTypeOptions): Promise {\n throw new Error('Not implemented');\n }\n async lastModified(path: string): Promise {\n throw new Error('Not implemented');\n }\n async fileSize(path: string): Promise {\n throw new Error('Not implemented');\n }\n async copyFile(from: string, to: string, options: CopyFileOptions): Promise {\n throw new Error('Not implemented');\n }\n async moveFile(from: string, to: string, options: MoveFileOptions): Promise {\n throw new Error('Not implemented');\n }\n}" }, { "path": "package.json", "content": "{\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build\": \"npm run compile -ws --if-present && npm run patch -ws --if-present\",\n \"build:only\": \"npm run compile -ws --if-present\",\n \"clean:build\": \"npm run clean && npm run build\",\n \"watch\": \"concurrently npm:watch:*\",\n \"watch:file-storage\": \"npm run watch -w ./packages/file-storage\",\n \"watch:stream-mime-type\": \"npm run watch -w ./packages/stream-mime-type\",\n \"watch:local\": \"npm run watch -w ./packages/local-fs\",\n \"watch:aws-s3\": \"npm run watch -w ./packages/aws-s3\",\n \"clean\": \"rm -rf ./packages/*/dist/\",\n \"ts\": \"node --import tsx/esm\",\n \"lint\": \"tsc --noEmit --incremental false\",\n \"test\": \"npm run lint && npm run vitest\",\n \"vitest\": \"vitest run\"\n },\n \"workspaces\": [\n \"./packages/stream-mime-type\",\n \"./packages/file-storage\",\n \"./packages/*\"\n ],\n \"devDependencies\": {\n \"@swc/core\": \"^1.15.18\",\n \"@types/express\": \"^5.0.6\",\n \"@types/mime-types\": \"^3.0.1\",\n \"@types/multer\": \"^2.1.0\",\n \"@types/node\": \"^25.3.5\",\n \"@vitest/ui\": \"^4.0.18\",\n \"concurrently\": \"^9.2.1\",\n \"cross-env\": \"^10.1.0\",\n \"express\": \"^5.2.1\",\n \"node-fetch\": \"^3.3.2\",\n \"rimraf\": \"^6.1.3\",\n \"tsx\": \"^4.21.0\",\n \"typescript\": \"^5.9.3\",\n \"vite\": \"^7.3.1\",\n \"vite-tsconfig-paths\": \"^6.1.1\",\n \"vitest\": \"^4.0.14\"\n },\n \"dependencies\": {\n \"dotenv\": \"^17.3.1\"\n }\n}\n" }, { "path": "packages/aws-s3/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/aws-s3/CHANGELOG.md", "content": "# `@flystorage/aws-s3`\n\n## 1.2.1\n\n- Upgraded dependencies\n\n## 1.2.0\n\n## Changes\n\n- Make tests run on Bun\n\n## 1.1.1\n\n## Changes\n\n- Do not send a body when creating a directory to prevent hanging requests.\n\n## 1.1.0\n\n## Added\n\n- AbortSignal Support\n\n## 1.0.1\n\n### Fixes\n\n- Bug: fixed not being able to use write options\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/aws-s3/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/aws-s3/README.md", "content": "\n\n# Flystorage adapter for AWS S3\n\nThis package contains the Flystorage adapter for AWS S3 using the V3 SDK.\n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/file-storage @flystorage/aws-s3 @aws-sdk/client-s3\n```\n\n## Usage\n\n```typescript\nimport {FileStorage} from '@flystorage/file-storage';\nimport {AwsS3StorageAdapter} from '@flystorage/aws-s3';\nimport {S3Client} from '@aws-sdk/client-s3';\n\nconst client = new S3Client();\nconst adapter = new AwsS3StorageAdapter(client, {\n bucket: '{your-bucket-name}',\n prefix: '{optional-path-prefix}',\n});\nconst storage = new FileStorage(adapter);\n```\n\n> ⚠️ Always use the FileStorage, it is essential for security and a good developer\n> experience. Do not use the adapter directly.\n\n" }, { "path": "packages/aws-s3/package.json", "content": "{\n \"name\": \"@flystorage/aws-s3\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@aws-sdk/client-s3\": \"^3.1004.0\",\n \"@aws-sdk/lib-storage\": \"^3.1004.0\",\n \"@aws-sdk/s3-request-presigner\": \"^3.1004.0\",\n \"@flystorage/file-storage\": \"^1.1.0\",\n \"@flystorage/stream-mime-type\": \"^1.0.0\",\n \"file-type\": \"^21.3.1\",\n \"mime-types\": \"^3.0.2\"\n },\n \"description\": \"\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"keywords\": [\n \"s3\",\n \"file\",\n \"storage\",\n \"flystorage\",\n \"filesystem\"\n ],\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/adapter/aws-s3/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/aws-s3\"\n },\n \"license\": \"MIT\"\n}\n" }, { "path": "packages/aws-s3/src/aws-s3-file-storage.test.ts", "content": "import {S3Client} from '@aws-sdk/client-s3';\nimport {\n FileStorage,\n readableToString,\n Visibility,\n closeReadable,\n UploadRequestHeaders,\n UnableToWriteFile,\n UnableToReadFile,\n} from '@flystorage/file-storage';\nimport {BinaryToTextEncoding, createHash, randomBytes} from 'crypto';\nimport * as https from 'https';\nimport {AwsS3StorageAdapter} from './aws-s3-storage-adapter.js';\nimport {createReadStream} from \"node:fs\";\nimport * as path from \"node:path\";\nimport 'dotenv/config';\nimport {PassThrough} from 'node:stream';\n\nlet client: S3Client;\nlet storage: FileStorage;\nlet bucket = 'flysystem-check';\nconst testSegment = randomBytes(10).toString('hex');\n\ndescribe('aws-s3 file storage', () => {\n beforeAll(() => {\n const [major] = process.versions.node.split('.').map(Number);\n const versionToBucketMapping = [[20, 'a'], [21, 'b'], [22, 'c'], [23, 'd']] as [number, string][];\n const bucketSuffix = versionToBucketMapping.find(([version]) => version === major);\n\n if (bucketSuffix) {\n bucket = `flystorage-${bucketSuffix[1]}`;\n }\n\n client = new S3Client();\n });\n\n beforeEach(async () => {\n const secondSegment = randomBytes(10).toString('hex');\n storage = new FileStorage(new AwsS3StorageAdapter(client, {\n bucket: bucket,\n prefix: `storage/${testSegment}/${secondSegment}`,\n }));\n });\n\n afterAll(async () => {\n await new FileStorage(new AwsS3StorageAdapter(client, {\n bucket: bucket,\n prefix: 'storage',\n })).deleteDirectory(testSegment);\n client.destroy();\n });\n\n test('uploading using a prepared request', async () => {\n const request = await storage.prepareUpload('prepared/request-file.txt', {\n expiresAt: Date.now() + 60 * 1000,\n headers: {\n 'Content-Type': 'text/plain',\n }\n });\n\n await naivelyMakeRequestFile(\n request.url,\n request.headers,\n request.method,\n 'this is the contents',\n );\n\n const contents = await storage.readToString('prepared/request-file.txt');\n\n expect(contents).toEqual('this is the contents');\n });\n\n test('writing and reading a file', async () => {\n await storage.write('path+name.txt', 'this is the contents');\n\n expect(await storage.readToString('path+name.txt')).toEqual('this is the contents');\n });\n\n test('non deep and deep listing should have consistent and similar results', async () => {\n await storage.write('file_1.txt', 'contents');\n await storage.write('file_2.txt', 'contents');\n await storage.createDirectory('directory_1');\n await storage.createDirectory('directory_2');\n\n const non_deep_listing = await storage.list('/', {deep: false}).toArray();\n const deep_listing = await storage.list('/', {deep: true}).toArray();\n\n expect(non_deep_listing).toHaveLength(4);\n expect(deep_listing).toHaveLength(4);\n expect(non_deep_listing).toEqual(deep_listing);\n });\n\n test('root listing should work without root-slash', async () => {\n const rootStorage = new FileStorage(new AwsS3StorageAdapter(client, {\n bucket: 'flystorage-root-check',\n }));\n\n await rootStorage.write('test-file.txt', 'contents');\n await rootStorage.createDirectory('test-directory');\n\n const non_deep_listing = await rootStorage.list('', {deep: false}).toArray();\n const deep_listing = await rootStorage.list('', {deep: true}).toArray();\n\n expect(non_deep_listing).toHaveLength(2);\n expect(deep_listing).toHaveLength(2);\n expect(non_deep_listing.map(item => item.path)).toEqual(deep_listing.map(item => item.path));\n });\n\n test('root listing should work with root-slash', async () => {\n const rootStorage = new FileStorage(new AwsS3StorageAdapter(client, {\n bucket: 'flystorage-root-check',\n }));\n\n await rootStorage.write('test-file.txt', 'contents');\n await rootStorage.createDirectory('test-directory');\n\n const non_deep_listing = await rootStorage.list('/', {deep: false}).toArray();\n const deep_listing = await rootStorage.list('/', {deep: true}).toArray();\n\n expect(non_deep_listing).toHaveLength(2);\n expect(deep_listing).toHaveLength(2);\n expect(non_deep_listing.map(item => item.path)).toEqual(deep_listing.map(item => item.path));\n });\n\n test('moving a file', async () => {\n await storage.write('from+here.txt', 'this');\n\n await storage.moveFile('from+here.txt', 'to+there.txt');\n\n expect(await storage.fileExists('from+here.txt')).toEqual(false);\n expect(await storage.readToString('to+there.txt')).toEqual('this');\n });\n\n test('copying a file', async () => {\n await storage.write('from+this.txt', 'this');\n\n await storage.copyFile('from+this.txt', 'to+that.txt');\n\n expect(await storage.fileExists('from+this.txt')).toEqual(true);\n expect(await storage.readToString('to+that.txt')).toEqual('this');\n });\n\n test('trying to copy a file that does not exist', async () => {\n await expect(storage.copyFile('404.txt', 'to.txt')).rejects.toThrow();\n });\n\n test('timing out when writing a file', async () => {\n const writeStream = new PassThrough();\n writeStream.write('something');\n\n setTimeout(() => {\n writeStream.end('this');\n }, 150);\n\n await expect(storage.write('somewhere.txt', writeStream, {\n timeout: 10,\n })).rejects.toThrow();\n\n })\n\n test('trying to read a file that does not exist', async () => {\n let was404 = false;\n\n try {\n await storage.read('404.txt');\n } catch (err) {\n if (err instanceof UnableToReadFile) {\n was404 = err.wasFileNotFound;\n }\n }\n\n expect(was404).toEqual(true);\n });\n\n test('trying to move a file that does not exist', async () => {\n await expect(storage.moveFile('404.txt', 'to.txt')).rejects.toThrow();\n });\n\n test('you can download public files using a public URL', async () => {\n await storage.write('public+file.txt', 'contents of the public file', {\n visibility: Visibility.PUBLIC,\n });\n\n const url = await storage.publicUrl('public+file.txt');\n const contents = await naivelyDownloadFile(url);\n\n expect(contents).toEqual('contents of the public file');\n });\n\n test('private files can only be downloaded using a temporary URL', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(naivelyDownloadFile(await storage.publicUrl('private+file.txt'))).rejects.toThrow();\n\n await expect(naivelyDownloadFile(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000})\n )).resolves.toEqual('contents of the private file');\n });\n\n test('getting a temporary url for a non-existing-file', async () => {\n await expect(\n storage.temporaryUrl('this-does-not-exist.txt', {expiresAt: Date.now() + 60 * 1000})\n ).resolves.toContain('this-does-not-exist.txt');\n });\n\n describe('response headers', () => {\n test('fetches file with Content-Disposition header when specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Disposition': 'attachment; filename=\"private+file.txt\"'} }),\n 'Content-Disposition'\n )).resolves.toEqual('attachment; filename=\"private+file.txt\"');\n });\n\n test('fetches file without Content-Disposition header when not specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),\n 'Content-Disposition'\n )).resolves.toBeUndefined();\n });\n\n test('fetches file with Cache-Control header when specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Cache-Control': 'none'} }),\n 'Cache-Control'\n )).resolves.toEqual('none');\n });\n\n test('fetches file without Cache-Control header when not specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),\n 'Cache-Control'\n )).resolves.toBeUndefined();\n });\n\n test('fetches file with Content-Encoding header when specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Encoding': 'br'} }),\n 'Content-Encoding'\n )).resolves.toEqual('br');\n });\n\n test('fetches file without Content-Encoding header when not specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),\n 'Content-Encoding'\n )).resolves.toBeUndefined();\n });\n\n test('fetches file with Content-Language header when specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Language': 'en-US'} }),\n 'Content-Language'\n )).resolves.toEqual('en-US');\n });\n\n test('fetches file without Content-Language header when not specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),\n 'Content-Language'\n )).resolves.toBeUndefined();\n });\n\n test('fetches file with Content-Type header when specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Content-Type': 'image/jpeg+special'} }),\n 'Content-Type'\n )).resolves.toEqual('image/jpeg+special');\n });\n\n test('fetches file with Expires header when specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000, responseHeaders: {'Expires': 'Sat, 21 Oct 2023 07:28:00 GMT'} }),\n 'Expires'\n )).resolves.toEqual('Sat, 21 Oct 2023 07:28:00 GMT');\n });\n\n test('fetches file without Expires header when not specified in the options', async () => {\n await storage.write('private+file.txt', 'contents of the private file', {\n visibility: Visibility.PRIVATE,\n });\n\n await expect(responseHeaderValue(\n await storage.temporaryUrl('private+file.txt', {expiresAt: Date.now() + 60 * 1000}),\n 'Expires'\n )).resolves.toBeUndefined();\n });\n });\n\n test('retrieving the size of a file', async () => {\n const contents = 'this is the contents of the file';\n await storage.write('something+file.txt', contents);\n\n expect(await storage.fileSize('something+file.txt')).toEqual(contents.length);\n });\n\n test('writing a png and fetching its mime-type', async () => {\n const handle = createReadStream(path.resolve(process.cwd(), 'fixtures/screenshot.png'));\n await storage.write('image.png', handle);\n closeReadable(handle);\n\n const mimeType = await storage.mimeType('image.png');\n\n expect(mimeType).toEqual('image/png');\n });\n\n test('fetching the last modified', async () => {\n await storage.write('test.txt', 'contents');\n\n const lastModified = await storage.lastModified('test.txt');\n\n expect(lastModified).toBeGreaterThan(Date.now() - 5000);\n expect(lastModified).toBeLessThan(Date.now() + 5000);\n });\n\n test('it can request checksums', async () => {\n function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string {\n return createHash(algo).update(input).digest(encoding);\n }\n\n const contents = 'this is for the checksum';\n await storage.write('path.txt', contents);\n const expectedChecksum = hashString(contents, 'md5');\n\n const checksum = await storage.checksum('path.txt', {\n algo: 'etag',\n });\n\n expect(checksum).toEqual(expectedChecksum);\n });\n\n test('it can request sha256 checksums', async () => {\n function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string {\n return createHash(algo).update(input).digest(encoding);\n }\n\n const contents = 'this is for the checksum';\n await storage.write('path.txt', contents, {\n ChecksumAlgorithm: 'SHA256'\n });\n const expectedChecksum = hashString(contents, 'sha256', 'base64');\n\n const checksum = await storage.checksum('path.txt', {\n algo: 'sha256',\n });\n\n expect(checksum).toEqual(expectedChecksum);\n });\n\n test('uploads can be aborted', async () => {\n const reason = new Error('Because I say so');\n const controller = new AbortController();\n const options = {\n abortSignal: controller.signal,\n } as const;\n\n await expect((async () => {\n const promise = storage.write('cache.txt', 'some content', {\n abortSignal: controller.signal,\n });\n\n controller.abort(reason);\n\n return promise;\n })()).rejects.toThrow(UnableToWriteFile.because(\n 'Because I say so',\n {cause: reason, context: {path: 'cache.txt', options}},\n ));\n })\n\n test('it handles custom Cache-Control header', async() => {\n await storage.write('cache.txt', 'some content', {cacheControl: \"max-age=7200, public\", visibility: Visibility.PUBLIC});\n const url = await storage.publicUrl('cache.txt')\n const res = await fetch(url);\n expect(res.headers.get(\"Cache-Control\")).toEqual(\"max-age=7200, public\")\n });\n});\n\nfunction naivelyDownloadFile(url: string): Promise {\n return new Promise((resolve, reject) => {\n https.get(url, async res => {\n if (res.statusCode !== 200) {\n reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));\n } else {\n resolve(await readableToString(res));\n }\n });\n });\n}\n\nfunction naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise {\n return new Promise((resolve, reject) => {\n const req = https.request(url, {\n method: method,\n headers: {\n ...headers,\n 'Content-Length': new Blob([data]).size,\n }\n }, async res => {\n let responseBody = '';\n res.on('data', (chunk) => {\n responseBody += chunk;\n });\n res.on('end', () => {\n const statusCode = res.statusCode ?? 500;\n\n if (statusCode <= 200 && statusCode >= 299) {\n reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));\n } else {\n resolve();\n }\n });\n });\n\n req.on('error', (err) => {\n reject(err);\n });\n req.write(data);\n req.end();\n });\n}\n\nfunction responseHeaderValue(url: string, header: string): Promise {\n return new Promise((resolve, reject) => {\n https.get(url, res => {\n if (res.statusCode !== 200) {\n reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));\n } else {\n resolve(\n res.headers[header]?.toString()\n ?? res.headers[header.toLowerCase()]?.toString()\n ?? undefined\n );\n }\n });\n });\n}\n" }, { "path": "packages/aws-s3/src/aws-s3-storage-adapter.ts", "content": "import {\n _Object,\n CommonPrefix,\n CopyObjectCommand,\n CopyObjectRequest,\n DeleteObjectCommand,\n DeleteObjectsCommand,\n GetObjectAclCommand,\n GetObjectAclOutput,\n GetObjectCommand,\n GetObjectCommandInput,\n GetObjectCommandOutput,\n HeadObjectCommand,\n ListObjectsV2Command,\n ListObjectsV2Output,\n ObjectCannedACL,\n PutObjectAclCommand,\n PutObjectCommand,\n PutObjectCommandInput,\n S3Client,\n S3ServiceException,\n} from '@aws-sdk/client-s3';\nimport {Configuration, Upload} from '@aws-sdk/lib-storage';\nimport {getSignedUrl} from '@aws-sdk/s3-request-presigner';\nimport {posix} from 'node:path';\nimport {\n AdapterListOptions,\n ChecksumIsNotAvailable,\n ChecksumOptions,\n closeReadable,\n CopyFileOptions,\n CreateDirectoryOptions,\n FileContents,\n FileWasNotFound,\n MimeTypeOptions,\n MiscellaneousOptions,\n MoveFileOptions,\n normalizeExpiryToMilliseconds,\n PathPrefixer,\n PublicUrlOptions,\n StatEntry,\n StorageAdapter,\n TemporaryUrlOptions,\n UploadRequest,\n UploadRequestHeaders,\n UploadRequestOptions,\n Visibility,\n WriteOptions,\n} from '@flystorage/file-storage';\nimport {resolveMimeType} from '@flystorage/stream-mime-type';\nimport {Readable} from 'stream';\nimport {lookup} from 'mime-types';\n\ntype PutObjectOptions = Omit;\nexport type WriteOptionsForS3 = Omit;\nconst possibleChecksumAlgos = ['SHA1', 'SHA256', 'CRC32', 'CRC32C', 'ETAG'] as const;\ntype ChecksumAlgo = typeof possibleChecksumAlgos[number];\n\nfunction isSupportedAlgo(algo: string): algo is ChecksumAlgo {\n return possibleChecksumAlgos.includes(algo as ChecksumAlgo);\n}\n\nexport type AwsS3StorageAdapterOptions = Readonly<{\n bucket: string,\n prefix?: string,\n region?: string,\n publicUrlOptions?: PublicUrlOptions,\n uploadRequestOptions?: UploadRequestOptions,\n putObjectOptions?: PutObjectOptions,\n uploadConfiguration?: Partial>,\n defaultChecksumAlgo?: ChecksumAlgo,\n}>;\n\nexport type AwsPublicUrlOptions = PublicUrlOptions & {\n bucket: string,\n region?: string,\n forcePathStyle?: boolean,\n baseUrl?: string,\n}\n\nexport type AwsPublicUrlGenerator = {\n publicUrl(path: string, options: AwsPublicUrlOptions): Promise;\n};\n\nexport class DefaultAwsPublicUrlGenerator implements AwsPublicUrlGenerator {\n async publicUrl(path: string, options: AwsPublicUrlOptions): Promise {\n const baseUrl = options.baseUrl ?? 'https://{subdomain}.amazonaws.com/{uri}';\n const subdomain = options.forcePathStyle !== true\n ? `${options.bucket}.s3`\n : options.region === undefined\n ? 's3'\n : `s3-${options.region}`;\n const uri = options.forcePathStyle !== true\n ? encodePath(path)\n : `${options.bucket}/${encodePath(path)}`;\n\n return baseUrl.replace('{subdomain}', subdomain).replace('{uri}', uri);\n }\n}\n\n/**\n * BC extension\n */\nexport class HostStyleAwsPublicUrlGenerator extends DefaultAwsPublicUrlGenerator {\n}\n\nexport type TimestampResolver = () => number;\ntype AclOptions = Pick;\n\n/**\n * Some commands need URI encoded paths to work ¯\\_(ツ)_/¯\n */\nfunction encodePath(path: string): string {\n return path.split('/').map(encodeURIComponent).join('/');\n}\n\nfunction maybeAbort(signal?: AbortSignal) {\n if (signal?.aborted) {\n throw signal.reason;\n }\n}\n\nexport class AwsS3StorageAdapter implements StorageAdapter {\n private readonly prefixer: PathPrefixer;\n\n constructor(\n private readonly client: S3Client,\n private readonly options: AwsS3StorageAdapterOptions,\n private readonly publicUrlGenerator: AwsPublicUrlGenerator = new DefaultAwsPublicUrlGenerator(),\n private readonly timestampResolver: TimestampResolver = () => Date.now(),\n ) {\n this.prefixer = new PathPrefixer(options.prefix ?? '', '/', (...paths) => {\n const path = posix.join(...paths);\n\n if (path === \".\" || path === \"/\") {\n // 1) https://nodejs.org/api/path.html#pathjoinpaths\n // Zero-length path segments are ignored. If the joined path string is a zero-length string then '.' will be\n // returned, representing the current working directory.\n // 2) In S3 we use delimiter:\"/\". In that case we need to remove the root-slash in order to list the\n // root-directory contents.\n return \"\";\n } else {\n return path;\n }\n });\n }\n\n async copyFile(from: string, to: string, options: CopyFileOptions): Promise {\n maybeAbort(options.abortSignal);\n let visibility: string | undefined = options.visibility;\n\n if (visibility === undefined && options.retainVisibility) {\n visibility = await this.visibility(from, options);\n maybeAbort(options.abortSignal);\n }\n\n let acl: AclOptions = (visibility !== undefined && options.useVisibility !== false)\n ? {ACL: this.visibilityToAcl(visibility)}\n : {};\n\n await this.client.send(new CopyObjectCommand({\n Bucket: this.options.bucket,\n CopySource: posix.join('/', this.options.bucket, encodePath(this.prefixer.prefixFilePath(from))),\n Key: this.prefixer.prefixFilePath(to),\n ...acl,\n }), {abortSignal: options.abortSignal});\n }\n async moveFile(from: string, to: string, options: MoveFileOptions): Promise {\n await this.copyFile(from, to, options);\n await this.deleteFile(from, options);\n }\n\n async prepareUpload(path: string, options: UploadRequestOptions): Promise {\n maybeAbort(options.abortSignal);\n const expiry = normalizeExpiryToMilliseconds(options.expiresAt);\n const now = (this.timestampResolver)();\n\n const putObjectParams: PutObjectCommandInput = {\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n };\n\n const headers: UploadRequestHeaders = {};\n const contentType = options['Content-Type'] ?? options.contentType;\n\n if (typeof contentType === 'string') {\n putObjectParams.ContentType = contentType;\n headers['Content-Type'] = contentType;\n }\n\n const url = await getSignedUrl(this.client, new PutObjectCommand(putObjectParams), {\n expiresIn: Math.floor((expiry - now) / 1000),\n });\n\n return {\n url,\n method: 'PUT',\n provider: 'aws-s3',\n headers,\n };\n }\n\n async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n maybeAbort(options.abortSignal);\n const expiry = normalizeExpiryToMilliseconds(options.expiresAt);\n const now = (this.timestampResolver)();\n\n const getObjectParams: GetObjectCommandInput = {\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n };\n\n if (options.responseHeaders) {\n if (options.responseHeaders['Cache-Control']) {\n getObjectParams.ResponseCacheControl = options.responseHeaders['Cache-Control'];\n }\n\n if (options.responseHeaders['Content-Disposition']) {\n getObjectParams.ResponseContentDisposition = options.responseHeaders['Content-Disposition'];\n }\n\n if (options.responseHeaders['Content-Encoding']) {\n getObjectParams.ResponseContentEncoding = options.responseHeaders['Content-Encoding'];\n }\n\n if (options.responseHeaders['Content-Language']) {\n getObjectParams.ResponseContentLanguage = options.responseHeaders['Content-Language'];\n }\n\n if (options.responseHeaders['Content-Type']) {\n getObjectParams.ResponseContentType = options.responseHeaders['Content-Type'];\n }\n\n if (options.responseHeaders['Expires']) {\n getObjectParams.ResponseExpires = new Date(options.responseHeaders['Expires']);\n }\n }\n\n return await getSignedUrl(this.client, new GetObjectCommand(getObjectParams), {\n expiresIn: Math.floor((expiry - now) / 1000),\n });\n }\n\n async lastModified(path: string, options: MiscellaneousOptions): Promise {\n const stat = await this.stat(path, options);\n\n if (stat.lastModifiedMs === undefined) {\n throw new Error('Last modified is not available in stat');\n }\n\n return stat.lastModifiedMs;\n }\n\n async fileSize(path: string, options: MiscellaneousOptions): Promise {\n const stat = await this.stat(path, options);\n\n if (stat.isFile === false) {\n throw new Error('Path is not a file');\n }\n\n if (stat.size === undefined) {\n throw new Error('File size is not available in stat.')\n }\n\n return stat.size;\n }\n\n async mimeType(path: string, options: MimeTypeOptions): Promise {\n const response = await this.stat(path, options);\n\n if (!response.isFile) {\n throw new Error(`Path \"${path} is not a file.`);\n }\n\n if (response.mimeType) {\n return response.mimeType;\n }\n\n if (options.disallowFallback) {\n throw new Error('Mime-type not available via HeadObject');\n }\n\n maybeAbort(options.abortSignal);\n const method = options.fallbackMethod ?? 'path';\n const mimeType = method === 'path'\n ? lookup(path)\n : await this.lookupMimeTypeFromStream(path, options);\n\n if (mimeType === undefined || mimeType === false) {\n throw new Error('Unable to resolve mime-type');\n }\n\n return mimeType;\n }\n\n async visibility(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n const response: GetObjectAclOutput = await this.client.send(new GetObjectAclCommand({\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n }), {\n abortSignal: options.abortSignal,\n });\n\n const publicRead = response.Grants?.some(grant =>\n grant.Grantee?.URI === 'http://acs.amazonaws.com/groups/global/AllUsers'\n && grant.Permission === 'READ'\n ) ?? false;\n\n return publicRead ? Visibility.PUBLIC : Visibility.PRIVATE;\n }\n\n async* list(path: string, options: AdapterListOptions): AsyncGenerator {\n const listing = this.listObjects(path, {\n deep: options.deep,\n includePrefixes: true,\n includeSelf: false,\n abortSignal: options.abortSignal,\n });\n\n for await (const {type, item} of listing) {\n if (type === 'prefix') {\n yield {\n type: 'directory',\n isFile: false,\n isDirectory: true,\n path: this.prefixer.stripDirectoryPath(item.Prefix!),\n };\n } else {\n const path = item.Key!;\n\n if (path.endsWith('/')) {\n yield {\n type: 'directory',\n isFile: false,\n isDirectory: true,\n path: this.prefixer.stripDirectoryPath(path),\n };\n } else {\n yield {\n type: 'file',\n isFile: true,\n isDirectory: false,\n path: this.prefixer.stripFilePath(path),\n size: item.Size ?? 0,\n lastModifiedMs: item.LastModified?.getTime(),\n };\n }\n }\n }\n }\n\n async * listObjects(\n path: string,\n options: {\n deep: boolean,\n includePrefixes: boolean,\n includeSelf: boolean,\n maxKeys?: number,\n abortSignal?: AbortSignal,\n },\n ): AsyncGenerator<{ type: 'prefix', item: CommonPrefix } | { type: 'object', item: _Object }, any, unknown> {\n maybeAbort(options.abortSignal);\n const prefix = this.prefixer.prefixDirectoryPath(path);\n let collectedKeys = 0;\n let shouldContinue = true;\n let continuationToken: string | undefined = undefined;\n\n while (shouldContinue && (options.maxKeys === undefined || collectedKeys < options.maxKeys)) {\n maybeAbort(options.abortSignal);\n const response: ListObjectsV2Output = await this.client.send(new ListObjectsV2Command({\n Bucket: this.options.bucket,\n Prefix: prefix,\n Delimiter: options.deep ? undefined : '/',\n ContinuationToken: continuationToken,\n MaxKeys: options.maxKeys,\n }));\n\n continuationToken = response.NextContinuationToken;\n shouldContinue = response.IsTruncated ?? false;\n const prefixes = options.includePrefixes ? response.CommonPrefixes ?? [] : [];\n\n for (const item of prefixes) {\n if ((!options.includeSelf && item.Prefix === prefix) || item.Prefix === undefined) {\n continue;\n }\n\n collectedKeys++;\n yield {type: 'prefix', item};\n }\n\n for (const item of response.Contents ?? []) {\n if ((!options.includeSelf && item.Key === prefix) || item.Key === undefined) {\n // not interested in itself\n // not interested in empty prefixes\n continue;\n }\n\n collectedKeys++;\n yield {type: 'object', item};\n }\n }\n }\n\n async read(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n\n let response: GetObjectCommandOutput;\n\n try {\n response = await this.client.send(new GetObjectCommand({\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n }), {\n abortSignal: options.abortSignal,\n });\n } catch (err) {\n if (err instanceof S3ServiceException && err.$metadata.httpStatusCode === 404) {\n throw FileWasNotFound.atLocation(path, {\n context: {path, options},\n cause: err,\n });\n }\n\n throw err;\n }\n\n if (response.Body instanceof Readable || response.Body instanceof ReadableStream) {\n return response.Body;\n }\n\n throw new Error('No response body was provided');\n }\n\n async stat(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n const response = await this.client.send(new HeadObjectCommand({\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n }), {\n abortSignal: options.abortSignal,\n });\n\n return {\n path,\n type: 'file',\n isDirectory: false,\n isFile: true,\n size: response.ContentLength ?? 0,\n lastModifiedMs: response.LastModified?.getTime(),\n mimeType: response.ContentType,\n };\n }\n\n async createDirectory(path: string, options: CreateDirectoryOptions): Promise {\n const key = this.prefixer.prefixDirectoryPath(path);\n const abortSignal = options.abortSignal;\n const abortController = new AbortController();\n\n if (abortSignal) {\n if (abortSignal.aborted) {\n throw abortSignal.reason;\n }\n\n abortSignal.addEventListener('abort', () => {\n abortController.abort(abortSignal.reason);\n });\n }\n const params = this.createPutObjectParams(key, '', {\n ContentLength: 0,\n ACL: options.directoryVisibility ? this.visibilityToAcl(options.directoryVisibility) : undefined,\n });\n\n maybeAbort(abortSignal);\n await this.client.send(new PutObjectCommand(params), {\n abortSignal,\n });\n }\n\n async deleteDirectory(path: string): Promise {\n // @ts-ignore because we know it will only be objects\n let itemsToDelete: AsyncGenerator<{ item: _Object }> = this.listObjects(path, {\n deep: true,\n includeSelf: true,\n includePrefixes: false,\n });\n\n const flush = async (keys: { Key: string }[]) => this.client.send(new DeleteObjectsCommand({\n Bucket: this.options.bucket,\n Delete: {\n Objects: keys,\n },\n }));\n\n let bucket: { Key: string }[] = [];\n let promises: Promise[] = [];\n\n for await (const {item} of itemsToDelete) {\n bucket.push({Key: item.Key!});\n\n if (bucket.length > 1000) {\n promises.push(flush(bucket));\n bucket = [];\n }\n }\n\n if (bucket.length > 0) {\n promises.push(flush(bucket));\n }\n\n await Promise.all(promises);\n }\n\n async write(path: string, contents: Readable, options: WriteOptions): Promise {\n let mimeType = options.mimeType;\n\n if (mimeType === undefined) {\n [mimeType, contents] = await resolveMimeType(path, contents);\n }\n\n const writeOptions: PutObjectOptions = {\n ACL: options.visibility ? this.visibilityToAcl(options.visibility) : undefined,\n ContentType: mimeType,\n ContentLength: options.size,\n CacheControl: options.cacheControl,\n }\n\n for (const option of Object.keys(options)) {\n if (isWriteOptionKey(option)) {\n const resolver = (writeOptionResolvers as any)[option];\n const value = options[option];\n\n if (resolver(value)) {\n (writeOptions as any)[option] = value;\n }\n }\n }\n\n const abortController = new AbortController();\n\n if (options.abortSignal) {\n const abortSignal = options.abortSignal;\n if (abortSignal.aborted) {\n throw abortSignal.reason;\n }\n\n abortSignal.addEventListener('abort', () => {\n abortController.abort(abortSignal.reason);\n });\n }\n\n const upload = new Upload({\n client: this.client,\n params: this.createPutObjectParams(\n this.prefixer.prefixFilePath(path),\n contents,\n writeOptions,\n ),\n abortController,\n ...this.options.uploadConfiguration,\n });\n\n await upload.done();\n }\n\n private createPutObjectParams(\n key: string,\n contents: Readable | '',\n options: PutObjectOptions,\n ): PutObjectCommandInput {\n const params: PutObjectCommandInput = {\n Bucket: this.options.bucket,\n Key: key,\n ...Object.assign({}, this.options.putObjectOptions, options),\n };\n\n if (contents !== '') {\n params.Body = contents;\n }\n\n return params;\n }\n\n async deleteFile(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n const key = this.prefixer.prefixFilePath(path);\n await this.client.send(new DeleteObjectCommand({\n Bucket: this.options.bucket,\n Key: key,\n }), {\n abortSignal: options.abortSignal,\n });\n }\n\n private visibilityToAcl(visibility: string): ObjectCannedACL {\n if (visibility === Visibility.PUBLIC) {\n return 'public-read';\n } else if (visibility === Visibility.PRIVATE) {\n return 'private';\n }\n\n throw new Error(`Unrecognized visibility provided; ${visibility}`);\n }\n\n async changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n await this.client.send(new PutObjectAclCommand({\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n ACL: this.visibilityToAcl(visibility),\n }), {\n abortSignal: options.abortSignal,\n });\n }\n\n async fileExists(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n try {\n await this.client.send(new HeadObjectCommand({\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n }), {\n abortSignal: options.abortSignal,\n });\n\n return true;\n } catch (e) {\n if (e instanceof S3ServiceException && e.$metadata.httpStatusCode === 404) {\n return false;\n }\n\n throw e;\n }\n }\n\n async directoryExists(path: string, options: MiscellaneousOptions): Promise {\n const listing = this.listObjects(path, {\n deep: true,\n includePrefixes: true,\n includeSelf: true,\n maxKeys: 1,\n abortSignal: options.abortSignal,\n });\n\n for await (const _item of listing) {\n return true;\n }\n\n return false;\n }\n\n async publicUrl(path: string, options: PublicUrlOptions): Promise {\n maybeAbort(options.abortSignal);\n return this.publicUrlGenerator.publicUrl(this.prefixer.prefixFilePath(path), {\n bucket: this.options.bucket,\n ...options,\n ...this.options.publicUrlOptions,\n });\n }\n\n async checksum(path: string, options: ChecksumOptions): Promise {\n maybeAbort(options.abortSignal);\n const algo = (options.algo || this.options.defaultChecksumAlgo || 'SHA256').toUpperCase();\n\n if (!isSupportedAlgo(algo)) {\n throw ChecksumIsNotAvailable.checksumNotSupported(algo);\n }\n\n const responseKey = algo === 'ETAG' ? 'ETag' : `Checksum${algo}` as const;\n\n const response = await this.client.send(new HeadObjectCommand({\n Bucket: this.options.bucket,\n Key: this.prefixer.prefixFilePath(path),\n ...algo === 'ETAG' ? {} : {ChecksumMode: 'ENABLED'},\n }), {\n abortSignal: options.abortSignal,\n });\n\n const checksum = response[responseKey];\n\n if (checksum === undefined) {\n throw new Error(`Unable to retrieve checksum with algo ${algo}`);\n }\n\n return checksum.replace(/^\"(.+)\"$/, '$1');\n }\n\n private async lookupMimeTypeFromStream(path: string, options: MimeTypeOptions) {\n const [mimetype, stream] = await resolveMimeType(path, Readable.from(await this.read(path, options)));\n await closeReadable(stream);\n\n return mimetype;\n }\n}\n\n/**\n * BC export\n *\n * @deprecated\n */\nexport class AwsS3FileStorage extends AwsS3StorageAdapter {}\n\ntype ResolversForWriteOptions = {\n [K in keyof WriteOptionsForS3]-?: (value: any) => value is WriteOptionsForS3[K]\n}\n\nfunction isWriteOptionKey(key: string): key is string & keyof ResolversForWriteOptions {\n return Object.hasOwn(writeOptionResolvers, key);\n}\n\nexport const writeOptionResolvers: ResolversForWriteOptions = {\n ChecksumCRC64NVME: function (value: any): value is PutObjectOptions['ChecksumCRC64NVME'] {\n return typeof value === 'string';\n },\n IfMatch: function (value: any): value is PutObjectOptions['IfMatch'] {\n return typeof value === 'string';\n },\n WriteOffsetBytes: function (value: any): value is PutObjectOptions['WriteOffsetBytes'] {\n return typeof value === 'string';\n },\n ChecksumSHA1: function (value: any): value is PutObjectOptions['ChecksumSHA1'] {\n return typeof value === 'string';\n },\n ChecksumSHA256: function (value: any): value is PutObjectOptions['ChecksumSHA256'] {\n return typeof value === 'string';\n },\n ChecksumCRC32: function (value: any): value is PutObjectOptions['ChecksumCRC32'] {\n return typeof value === 'string';\n },\n ChecksumCRC32C: function (value: any): value is PutObjectOptions['ChecksumCRC32C'] {\n return typeof value === 'string';\n },\n CacheControl: function (value: any): value is PutObjectOptions['CacheControl'] {\n return typeof value === 'string';\n },\n ContentDisposition: function (value: any): value is PutObjectOptions['ContentDisposition'] {\n return typeof value === 'string';\n },\n ContentEncoding: function (value: any): value is PutObjectOptions['ContentEncoding'] {\n return typeof value === 'string';\n },\n ContentLanguage: function (value: any): value is PutObjectOptions['ContentLanguage'] {\n return typeof value === 'string';\n },\n ContentMD5: function (value: any): value is PutObjectOptions['ContentMD5'] {\n return typeof value === 'string';\n },\n ContentType: function (value: any): value is PutObjectOptions['ContentType'] {\n return typeof value === 'string';\n },\n ChecksumAlgorithm: function (value: any): value is PutObjectOptions['ChecksumAlgorithm'] {\n return typeof value === 'string';\n },\n Expires: function (value: any): value is PutObjectOptions['Expires'] {\n return value instanceof Date;\n },\n GrantFullControl: function (value: any): value is PutObjectOptions['GrantFullControl'] {\n return typeof value === 'string';\n },\n GrantRead: function (value: any): value is PutObjectOptions['GrantRead'] {\n return typeof value === 'string';\n },\n GrantReadACP: function (value: any): value is PutObjectOptions['GrantReadACP'] {\n return typeof value === 'string';\n },\n GrantWriteACP: function (value: any): value is PutObjectOptions['GrantWriteACP'] {\n return typeof value === 'string';\n },\n Metadata: function (value: any): value is PutObjectOptions['Metadata'] {\n return typeof value === 'object';\n },\n ServerSideEncryption: function (value: any): value is PutObjectOptions['ServerSideEncryption'] {\n return typeof value === 'string';\n },\n StorageClass: function (value: any): value is PutObjectOptions['StorageClass'] {\n return typeof value === 'string';\n },\n WebsiteRedirectLocation: function (value: any): value is PutObjectOptions['WebsiteRedirectLocation'] {\n return typeof value === 'string';\n },\n SSECustomerAlgorithm: function (value: any): value is PutObjectOptions['SSECustomerAlgorithm'] {\n return typeof value === 'string';\n },\n SSECustomerKey: function (value: any): value is PutObjectOptions['SSECustomerKey'] {\n return typeof value === 'string';\n },\n SSECustomerKeyMD5: function (value: any): value is PutObjectOptions['SSECustomerKeyMD5'] {\n return typeof value === 'string';\n },\n SSEKMSKeyId: function (value: any): value is PutObjectOptions['SSEKMSKeyId'] {\n return typeof value === 'string';\n },\n SSEKMSEncryptionContext: function (value: any): value is PutObjectOptions['SSEKMSEncryptionContext'] {\n return typeof value === 'string';\n },\n BucketKeyEnabled: function (value: any): value is PutObjectOptions['BucketKeyEnabled'] {\n return typeof value === 'string';\n },\n RequestPayer: function (value: any): value is PutObjectOptions['RequestPayer'] {\n return typeof value === 'string';\n },\n Tagging: function (value: any): value is PutObjectOptions['Tagging'] {\n return typeof value === 'string';\n },\n ObjectLockMode: function (value: any): value is PutObjectOptions['ObjectLockMode'] {\n return typeof value === 'string';\n },\n ObjectLockRetainUntilDate: function (value: any): value is PutObjectOptions['ObjectLockRetainUntilDate'] {\n return value instanceof Date;\n },\n ObjectLockLegalHoldStatus: function (value: any): value is PutObjectOptions['ObjectLockLegalHoldStatus'] {\n return typeof value === 'string';\n },\n ExpectedBucketOwner: function (value: any): value is PutObjectOptions['ExpectedBucketOwner'] {\n return typeof value === 'string';\n },\n IfNoneMatch: function (value: any): value is WriteOptionsForS3['IfNoneMatch'] {\n return typeof value === 'string';\n },\n};\n" }, { "path": "packages/aws-s3/src/index.ts", "content": "export * from './aws-s3-storage-adapter.js';" }, { "path": "packages/aws-s3/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packages/azure-storage-blob/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/azure-storage-blob/CHANGELOG.md", "content": "# `@flystorage/azure-storage-blob`\n\n## 1.2.0\n\n## Changes\n\n- Support Bun runtime\n- Updated dependencies\n- 404 detection for reads\n\n## 1.1.0\n\n## Added\n\n- AbortSignal Support\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/azure-storage-blob/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/azure-storage-blob/README.md", "content": "\n\n# Flystorage adapter for Azure Storage Blob\n\nThis package contains the Flystorage adapter for Azure Storage Blob\n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/file-storage @flystorage/azure-storage-blob @azure/storage-blob\n```\n\n## Usage\n\n```typescript\nimport {FileStorage} from '@flystorage/file-storage';\nimport {AzureStorageBlobStorageAdapter} from '@flystorage/azure-storage-blob';\n\nconst blobService = BlobServiceClient.fromConnectionString(process.env.AZURE_DSN!);\nconst container = blobService.getContainerClient('flysystem');\nconst adapter = new AzureStorageBlobStorageAdapter(container);\nconst storage = new FileStorage(adapter);\n```\n\n> ⚠️ Always use the FileStorage, it is essential for security and a good developer\n> experience. Do not use the adapter directly.\n\n" }, { "path": "packages/azure-storage-blob/package.json", "content": "{\n \"name\": \"@flystorage/azure-storage-blob\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@azure/storage-blob\": \"^12.31.0\",\n \"@flystorage/file-storage\": \"^1.1.0\",\n \"@flystorage/stream-mime-type\": \"^1.0.0\"\n },\n \"description\": \"\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"keywords\": [\n \"s3\",\n \"file\",\n \"storage\",\n \"flystorage\",\n \"filesystem\"\n ],\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/adapter/azure-storage-blob/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/aws-s3\"\n },\n \"license\": \"MIT\"\n}" }, { "path": "packages/azure-storage-blob/src/azure-storage-blob.test.ts", "content": "import {BlobServiceClient} from \"@azure/storage-blob\";\nimport {AzureStorageBlobStorageAdapter} from \"./azure-storage-blob.js\";\nimport {randomBytes} from \"crypto\";\nimport {\n FileStorage,\n UploadRequestHeaders,\n Visibility,\n readableToString,\n UnableToReadFile,\n} from '@flystorage/file-storage';\nimport fetch from \"node-fetch\";\nimport { Readable } from \"node:stream\";\nimport https from 'https';\nimport 'dotenv/config';\n\nconst runSegment = process.env.AZURE_PREFIX ?? randomBytes(10).toString('hex');\n\ndescribe('AzureStorageBlobStorageAdapter', () => {\n const blobService = BlobServiceClient.fromConnectionString(\n process.env.AZURE_DSN || \"UseDevelopmentStorage=true;\"\n );\n const container = blobService.getContainerClient('flysystem');\n let storage: FileStorage;\n\n beforeAll(async () => {\n await container.createIfNotExists({ access: \"container\" });\n });\n\n beforeEach(() => {\n const testSegment = randomBytes(10).toString('hex');\n const adapter = new AzureStorageBlobStorageAdapter(\n container,\n {\n prefix: `flystorage/${runSegment}/${testSegment}`,\n }\n );\n storage = new FileStorage(adapter);\n });\n\n afterAll(async () => {\n const adapter = new AzureStorageBlobStorageAdapter(container);\n storage = new FileStorage(adapter);\n await storage.deleteDirectory(`flystorage/${runSegment}`);\n container\n });\n\n test('non deep and deep listing should have consistent and similar results', async () => {\n await storage.write('file_1.txt', 'contents');\n await storage.write('file_2.txt', 'contents');\n await storage.write('directory_1/file.txt', 'contents');\n await storage.write('directory_2/file.tx', 'contents');\n\n const non_deep_listing = await storage.list('/', {deep: false}).toArray();\n const deep_listing = await storage.list('/', {deep: true}).toArray();\n\n expect(non_deep_listing).toHaveLength(4);\n expect(deep_listing).toHaveLength(6);\n });\n\n test('it can request a checksum', async () => {\n const contents = 'this is for the checksum';\n await storage.write('path.txt', contents);\n\n const checksum = await storage.checksum('path.txt', {\n algo: 'etag',\n });\n\n expect(typeof checksum).toEqual('string');\n expect(checksum.length).toBeGreaterThan(5);\n });\n\n test('reading a file that was written', async () => {\n await storage.write('path.txt', 'content in azure');\n const content = await storage.readToString('path.txt');\n\n expect(content).toEqual('content in azure');\n });\n\n test('trying to read a file that does not exist', async () => {\n let was404 = false;\n\n try {\n await storage.readToString('404.txt');\n } catch (err) {\n if (err instanceof UnableToReadFile) {\n was404 = err.wasFileNotFound;\n }\n }\n\n expect(was404).toEqual(true);\n });\n\n test('trying to see if a non-existing file exists', async () => {\n expect(await storage.fileExists('404.txt')).toEqual(false);\n });\n\n test('trying to see if an existing file exists', async () => {\n await storage.write('existing.txt', 'contents');\n\n expect(await storage.fileExists('existing.txt')).toEqual(true);\n });\n\n test('deleting an existing file', async () => {\n await storage.write('existing.txt', 'contents');\n\n expect(await storage.fileExists('existing.txt')).toEqual(true);\n\n await storage.deleteFile('existing.txt');\n\n expect(await storage.fileExists('existing.txt')).toEqual(false);\n });\n\n test('deleting a non-existing file is OK', async () => {\n await storage.deleteFile('404.txt');\n });\n\n test('copying a file', async () => {\n await storage.write('file.txt', 'copied');\n\n await storage.copyFile('file.txt', 'new-file.txt');\n\n expect(await storage.fileExists('file.txt')).toEqual(true);\n expect(await storage.fileExists('new-file.txt')).toEqual(true);\n expect(await storage.readToString('new-file.txt')).toEqual('copied');\n });\n\n test('moving a file', async () => {\n await storage.write('file.txt', 'moved');\n\n await storage.moveFile('file.txt', 'new-file.txt');\n\n expect(await storage.fileExists('file.txt')).toEqual(false);\n expect(await storage.fileExists('new-file.txt')).toEqual(true);\n expect(await storage.readToString('new-file.txt')).toEqual('moved');\n });\n\n test('setting visibility always fails', async () => {\n await storage.write('existing.txt', 'yes');\n await expect(storage.changeVisibility('existing.txt', Visibility.PRIVATE)).rejects.toThrow();\n await expect(storage.changeVisibility('404.txt', Visibility.PUBLIC)).rejects.toThrow();\n });\n\n test('listing entries in a directory, shallow', async () => {\n await storage.write('outside/path.txt', 'test');\n await storage.write('inside/a.txt', 'test');\n await storage.write('inside/b.txt', 'test');\n await storage.write('inside/c/a.txt', 'test');\n\n const listing = await storage.list('inside').toArray();\n expect(listing).toHaveLength(3);\n expect(listing[0].type).toEqual('file');\n expect(listing[1].type).toEqual('file');\n expect(listing[2].type).toEqual('directory');\n expect(listing[0].path).toEqual('inside/a.txt');\n expect(listing[1].path).toEqual('inside/b.txt');\n expect(listing[2].path).toEqual('inside/c');\n });\n\n test('listing entries in a directory, deep', async () => {\n await storage.write('outside/path.txt', 'test');\n await storage.write('inside/a.txt', 'test');\n await storage.write('inside/b.txt', 'test');\n await storage.write('inside/c/a.txt', 'test');\n\n const listing = await storage.list('inside', {deep: true}).toArray();\n expect(listing).toHaveLength(4);\n expect(listing[0].type).toEqual('file');\n expect(listing[1].type).toEqual('file');\n expect(listing[2].type).toEqual('directory');\n expect(listing[3].type).toEqual('file');\n expect(listing[0].path).toEqual('inside/a.txt');\n expect(listing[1].path).toEqual('inside/b.txt');\n expect(listing[2].path).toEqual('inside/c');\n expect(listing[3].path).toEqual('inside/c/a.txt');\n });\n\n test('deleting a full directory', async () => {\n await storage.write('directory/a.txt', 'test');\n await storage.write('directory/b.txt', 'test');\n await storage.write('directory/c/a.txt', 'test');\n\n await storage.deleteDirectory('directory');\n\n const listing = await storage.list('directory', {deep: true}).toArray();\n\n expect(listing).toEqual([]);\n });\n\n test('checking if a directory exists', async () => {\n await storage.write('directory/a.txt', 'test');\n await storage.write('directory/b.txt', 'test');\n await storage.write('directory/c/a.txt', 'test');\n\n expect(await storage.directoryExists('directory')).toEqual(true);\n expect(await storage.directoryExists('directory/c')).toEqual(true);\n expect(await storage.directoryExists('directory/a')).toEqual(false);\n });\n\n test('accessing a file though public URL', async () => {\n await storage.write('something.txt', 'something');\n\n const url = await storage.publicUrl('something.txt');\n const contents = await naivelyDownloadFile(url);\n\n expect(contents).toEqual('something');\n });\n\n test('accessing a file though temporary URL', async () => {\n await storage.write('something.txt', 'something');\n\n const url = await storage.temporaryUrl('something.txt', {\n expiresAt: Date.now() + 600000,\n });\n const contents = await naivelyDownloadFile(url);\n\n expect(contents).toEqual('something');\n });\n\n test('uploading using a prepared request', async () => {\n const request = await storage.prepareUpload('prepared/request-file.txt', {\n expiresAt: Date.now() + 60 * 1000,\n headers: {\n 'Content-Type': 'text/plain',\n }\n });\n\n await naivelyMakeRequestFile(\n request.url,\n request.headers,\n request.method,\n 'this is the contents',\n );\n\n const contents = await storage.readToString('prepared/request-file.txt');\n\n expect(contents).toEqual('this is the contents');\n });\n\n test('setting cache-control headers', async () => {\n await storage.write('cachecontrol.txt', 'something', { cacheControl: 'max-age=3200, public' });\n const url = await storage.publicUrl('cachecontrol.txt');\n const res = await fetch(url);\n expect(res.headers.get(\"Cache-Control\")).toEqual('max-age=3200, public');\n });\n});\n\nasync function naivelyDownloadFile(url: string): Promise {\n const res = await fetch(url);\n\n if (res.status !== 200 || !res.body) {\n throw new Error(`Not able to download the file from ${url}, response status [${res.status}]`);\n } else {\n return await readableToString(Readable.from(res.body));\n }\n}\n\nfunction naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise {\n return new Promise((resolve, reject) => {\n const req = https.request(url, {\n method: method,\n headers: {\n ...headers,\n 'Content-Length': new Blob([data]).size,\n }\n }, async res => {\n let responseBody = '';\n res.on('data', (chunk) => {\n responseBody += chunk;\n });\n res.on('end', () => {\n const statusCode = res.statusCode ?? 500;\n\n if (statusCode <= 200 && statusCode >= 299) {\n reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));\n } else {\n resolve();\n }\n });\n });\n\n req.on('error', (err) => {\n reject(err);\n });\n req.write(data);\n req.end();\n });\n}" }, { "path": "packages/azure-storage-blob/src/azure-storage-blob.ts", "content": "import {Readable} from 'stream';\nimport {\n ChecksumIsNotAvailable,\n ChecksumOptions,\n CopyFileOptions,\n FileContents,\n FileWasNotFound,\n ListOptions,\n MimeTypeOptions,\n MiscellaneousOptions,\n MoveFileOptions,\n normalizeExpiryToDate,\n PathPrefixer,\n PublicUrlOptions,\n StatEntry,\n StorageAdapter,\n TemporaryUrlOptions,\n UploadRequest,\n UploadRequestHeaders,\n UploadRequestOptions,\n WriteOptions,\n} from '@flystorage/file-storage';\nimport {\n BlobDownloadResponseParsed,\n BlobGenerateSasUrlOptions,\n BlobGetPropertiesResponse,\n BlobProperties,\n BlobSASPermissions,\n ContainerClient,\n} from '@azure/storage-blob';\nimport {resolveMimeType} from '@flystorage/stream-mime-type';\nimport {dirname} from 'node:path';\n\n\nexport type AzureStorageBlobStorageAdapterOptions = {\n prefix?: string,\n uploadMaxConcurrency?: number,\n ignoreVisibility?: boolean,\n ignoredVisibilityResponse?: string,\n deleteDirBatchSize?: number,\n temporaryUrlOptions?: TemporaryUrlOptions,\n}\n\nfunction maybeAbort(signal?: AbortSignal) {\n if (signal?.aborted) {\n throw signal.reason;\n }\n}\n\nexport class AzureStorageBlobStorageAdapter implements StorageAdapter {\n private readonly prefixer: PathPrefixer;\n\n constructor(\n private readonly container: ContainerClient,\n private readonly options: AzureStorageBlobStorageAdapterOptions = {},\n ) {\n this.prefixer = new PathPrefixer(options.prefix || '');\n }\n\n async copyFile(from: string, to: string, options: CopyFileOptions): Promise {\n const fromUrl = this.blockClient(from).url;\n maybeAbort(options.abortSignal);\n await this.blockClient(to).syncCopyFromURL(fromUrl, {abortSignal: options.abortSignal});\n }\n async moveFile(from: string, to: string, options: MoveFileOptions): Promise {\n await this.copyFile(from, to, options);\n await this.deleteFile(from, options);\n }\n\n async write(path: string, contents: Readable, options: WriteOptions): Promise {\n let mimeType = options.mimeType;\n let stream = contents;\n\n maybeAbort(options.abortSignal);\n\n if (mimeType === undefined) {\n [mimeType, stream] = await this.resolveMimetype(path, contents, options);\n }\n\n maybeAbort(options.abortSignal);\n\n const blob = this.blockClient(path);\n await blob.uploadStream(\n stream,\n options.size,\n this.options.uploadMaxConcurrency,\n {\n abortSignal: options.abortSignal,\n blobHTTPHeaders: {\n blobContentType: mimeType,\n blobCacheControl: options.cacheControl\n },\n },\n );\n }\n\n private blockClient(path: string) {\n return this.container.getBlockBlobClient(this.prefixer.prefixFilePath(path));\n }\n\n async read(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n const blob = this.blockClient(path);\n let response: BlobDownloadResponseParsed;\n\n try {\n response = await blob.download(undefined, undefined, {\n abortSignal: options.abortSignal,\n });\n } catch (err) {\n if ((err as any).statusCode === 404) {\n throw FileWasNotFound.atLocation(path, {\n context: {path, options},\n cause: err,\n })\n }\n\n throw err;\n }\n\n if (!response.readableStreamBody) {\n throw new Error('No readable stream body in response.');\n }\n\n return response.readableStreamBody;\n }\n\n async deleteFile(path: string, options: MiscellaneousOptions): Promise {\n const blob = this.blockClient(path);\n maybeAbort(options.abortSignal);\n await blob.deleteIfExists({\n abortSignal: options.abortSignal,\n });\n }\n\n async createDirectory(): Promise {\n // no-op, directories do not exist.\n }\n\n async stat(path: string, options: {abortSignal?: AbortSignal} = {}): Promise {\n maybeAbort(options.abortSignal);\n\n const blob = this.blockClient(path);\n const properties = await blob.getProperties({\n abortSignal: options.abortSignal,\n });\n\n return this.mapToStatEntry(path, properties);\n }\n\n private mapToStatEntry(path: string, properties: BlobGetPropertiesResponse | BlobProperties): StatEntry {\n return {\n type: 'file',\n isFile: true,\n isDirectory: false,\n path,\n mimeType: properties.contentType,\n size: properties.contentLength,\n lastModifiedMs: properties.lastModified?.getTime(),\n };\n }\n\n list(path: string, options: ListOptions): AsyncGenerator {\n return options.deep\n ? this.listDeep(path, options)\n : this.listShallow(path, options);\n\n }\n\n async *listDeep(path: string, options: ListOptions): AsyncGenerator {\n maybeAbort(options?.abortSignal);\n const directories = new Set();\n const listing = this.container.listBlobsFlat({\n prefix: this.prefixer.prefixDirectoryPath(path),\n abortSignal: options.abortSignal,\n });\n const listedPath = path;\n\n for await (const item of listing) {\n maybeAbort(options?.abortSignal);\n const path = this.prefixer.stripFilePath(item.name);\n let parentDir = dirname(path);\n\n while(!['.', '', listedPath].includes(parentDir)) {\n if (directories.has(parentDir)) {\n break;\n }\n\n yield {\n type: 'directory',\n isFile: false,\n isDirectory: true,\n path: parentDir,\n };\n\n directories.add(parentDir);\n parentDir = dirname(parentDir);\n }\n\n yield this.mapToStatEntry(path, item.properties);\n }\n }\n\n async *listShallow(path: string, options: ListOptions): AsyncGenerator {\n maybeAbort(options?.abortSignal);\n\n const listing = this.container.listBlobsByHierarchy('/', {\n prefix: this.prefixer.prefixDirectoryPath(path),\n abortSignal: options.abortSignal,\n });\n\n for await (const item of listing) {\n maybeAbort(options?.abortSignal);\n\n if (item.kind === 'blob') {\n yield this.mapToStatEntry(\n this.prefixer.stripFilePath(item.name),\n item.properties\n )\n } else {\n yield {\n path: this.prefixer.stripDirectoryPath(item.name),\n type: 'directory',\n isFile: false,\n isDirectory: true,\n }\n }\n }\n }\n\n async changeVisibility(path: string, visibility: string): Promise {\n if (this.options.ignoreVisibility !== true) {\n throw new Error('Not supported by this adapter');\n }\n }\n async visibility(path: string): Promise {\n if (this.options.ignoreVisibility !== true) {\n throw new Error('Not implemented');\n }\n\n // default to indicating it ss public because we cannot know if the default is private\n return this.options.ignoredVisibilityResponse ?? 'public';\n }\n async deleteDirectory(path: string, options: MiscellaneousOptions): Promise {\n let deletes: Promise[] = [];\n const batchSize = this.options.deleteDirBatchSize ?? 10;\n\n for await (const item of this.list(path, {deep: true})) {\n if (item.isFile) {\n deletes.push(this.deleteFile(item.path, options));\n }\n\n if (deletes.length >= batchSize) {\n await Promise.all(deletes);\n deletes = [];\n }\n\n }\n\n await Promise.all(deletes);\n }\n async fileExists(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n return await this.blockClient(path).exists({\n abortSignal: options.abortSignal,\n })\n }\n async directoryExists(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n const listing = this.container.listBlobsFlat({\n prefix: this.prefixer.prefixDirectoryPath(path),\n abortSignal: options.abortSignal,\n }).byPage({\n maxPageSize: 1,\n });\n\n return (await listing.next()).value.segment.blobItems.length > 0;\n }\n async publicUrl(path: string, options?: PublicUrlOptions): Promise {\n return this.blockClient(path).url;\n }\n async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n return await this.blockClient(path).generateSasUrl({\n expiresOn: normalizeExpiryToDate(options.expiresAt),\n permissions: BlobSASPermissions.parse('r'),\n ...(this.options.temporaryUrlOptions ?? {}),\n });\n }\n\n async prepareUpload(path: string, options: UploadRequestOptions): Promise {\n const headers: UploadRequestHeaders = {};\n headers['x-ms-blob-type'] = options['x-ms-blob-type'] ?? 'BlockBlob';\n const config: BlobGenerateSasUrlOptions = {\n expiresOn: normalizeExpiryToDate(options.expiresAt),\n permissions: BlobSASPermissions.parse('w'),\n ...(this.options.temporaryUrlOptions ?? {}),\n };\n\n\n const contentType = options['Content-Type'] ?? options.contentType;\n\n if (typeof contentType === 'string') {\n config.contentType = contentType;\n headers['Content-Type'] = contentType;\n }\n\n const url = await this.blockClient(path).generateSasUrl(config);\n\n return {method: 'PUT', provider: 'azure-storage-blob', url, headers};\n }\n\n async checksum(path: string, options: ChecksumOptions): Promise {\n maybeAbort(options?.abortSignal);\n const algo = options.algo ?? 'etag';\n\n if (algo !== 'etag') {\n throw ChecksumIsNotAvailable.checksumNotSupported(algo);\n }\n\n const blob = this.blockClient(path);\n const properties = await blob.getProperties({abortSignal: options.abortSignal});\n const etag = properties.etag;\n\n if (etag === undefined) {\n throw new Error('Etag is not defined on blob properties.');\n }\n\n return etag;\n }\n\n async mimeType(path: string, options: MimeTypeOptions): Promise {\n const stat = await this.stat(path, options);\n\n if (stat.isDirectory) {\n throw new Error('Path is not a file. No mimetype available.');\n }\n\n if (stat.mimeType === undefined) {\n throw new Error('Mime-type not found for file.');\n }\n\n return stat.mimeType;\n }\n\n async lastModified(path: string): Promise {\n const stat = await this.stat(path);\n\n if (stat.isDirectory) {\n throw new Error('Path is not a file. No last modified available.');\n }\n\n if (stat.lastModifiedMs === undefined) {\n throw new Error('Last modified not found for file.');\n }\n\n return stat.lastModifiedMs;\n }\n\n async fileSize(path: string): Promise {\n const stat = await this.stat(path);\n\n if (stat.isDirectory) {\n throw new Error('Path is not a file. No file size available.');\n }\n\n if (stat.size === undefined) {\n throw new Error('File size not found for file.');\n }\n\n return stat.size;\n }\n\n private async resolveMimetype(path: string, contents: Readable, options: WriteOptions): Promise<[string, Readable]> {\n if (options.mimeType) {\n return [options.mimeType, contents];\n }\n\n const [mimeType, stream] = await resolveMimeType(path, contents);\n\n return [mimeType ?? 'application/octet-stream', stream];\n }\n}\n\n/**\n * BC export\n *\n * @deprecated\n */\nexport class AzureStorageBlobFileStorage extends AzureStorageBlobStorageAdapter {}" }, { "path": "packages/azure-storage-blob/src/index.ts", "content": "export * from './azure-storage-blob.js';" }, { "path": "packages/azure-storage-blob/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packages/chaos/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/chaos/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/chaos/README.md", "content": "\n\n# Flystorage adapter for chaos engineering\n\nThis package contains an adapter decorator that causes instrumented failures.\n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/file-storage @flystorage/chaos\n```\n\n## Setup\n\n```typescript\nimport {FileStorage} from '@flystorage/file-storage';\nimport {ChaosStorageAdapterDecorator, TriggeredErrors} from '@flystorage/chaos';\n\nconst strategy = new TriggeredErrors();\nconst adapter = new ChaosStorageAdapterDecorator(\n createActualAdapter(),\n strategy,\n);\nconst storage = new FileStorage(adapter);\n```\n\n> ⚠️ Always use the FileStorage, it is essential for security and a good developer\n> experience. Do not use the adapter directly.\n \n## Usage\n\n```typescript\n\nimport {TriggeredErrors} from '@flystorage/chaos';\n\nconst strategy = new TriggeredErrors();\n\n// error on all write calls\nstrategy.on('write', () => new Error());\n\n// error on first 2 stat calls\nstrategy.on('stat', () => new Error(), {times: 2});\n\n// error after first 2 deleteFile calls\nstrategy.on('deleteFile', () => new Error(), {after: 2});\n\n// error on 2nd and 3rd call to any method\nstrategy.on('*', () => new Error(), {after: 1, times: 2});\n```\n\n" }, { "path": "packages/chaos/changelog.md", "content": "# `@flystorage/chaos`\n\n## 1.1.0\n\n### Changes\n\n- Added AbortSignal support\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/chaos/package.json", "content": "{\n \"name\": \"@flystorage/chaos\",\n \"type\": \"module\",\n \"version\": \"1.1.0\",\n \"dependencies\": {\n \"@flystorage/file-storage\": \"^1.1.0\"\n },\n \"description\": \"A storage adapter decorator with the ability to stage errors.\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"keywords\": [\n \"testing\",\n \"chaos\",\n \"resillience\",\n \"flystorage\",\n \"filesystem\"\n ],\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/tools/chaos/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/chaos\"\n },\n \"license\": \"MIT\"\n}" }, { "path": "packages/chaos/src/index.test.ts", "content": "import { FileStorage } from \"@flystorage/file-storage\";\nimport {InMemoryStorageAdapter} from '@flystorage/in-memory';\nimport {AlwaysThrowError, ChaosStorageAdapterDecorator, NeverThrowError, TriggeredErrors} from './index.js';\n\ndescribe('chaos adapter decorator', () => {\n describe('triggered strategy', () => {\n const strategy = new TriggeredErrors();\n const adapter = new InMemoryStorageAdapter();\n\n afterEach(() => {\n strategy.clearTriggers();\n adapter.deleteEverything();\n });\n\n const storage = new FileStorage(\n new ChaosStorageAdapterDecorator(\n adapter,\n strategy,\n ),\n );\n\n test('trigger immediately on write', async () => {\n strategy.on('write', () => new Error('Oh no...'));\n\n await expect(storage.write('path.txt', 'context')).rejects.toThrow();\n });\n\n test('trigger immediately after 3 writes', async () => {\n strategy.on('write', () => new Error('Oh no...'), {after: 3});\n\n await storage.write('path.txt', 'context');\n await storage.write('path.txt', 'context');\n await storage.write('path.txt', 'context');\n await expect(storage.write('path.txt', 'context')).rejects.toThrow();\n });\n\n test('trigger for 2 stat calls after 1 successful call', async () => {\n await storage.write('path.txt', 'contents');\n strategy.on('stat', () => new Error('Oh no...'), {times: 2, after: 1});\n\n await storage.stat('path.txt');\n await expect(storage.stat('path.txt')).rejects.toThrow();\n await expect(storage.stat('path.txt')).rejects.toThrow();\n await storage.stat('path.txt');\n });\n\n test('triggering on any method call with: *', async () => {\n await storage.write('path.txt', 'contents');\n strategy.on('*', () => new Error('Oh no...'), {times: 2, after: 1});\n\n await storage.stat('path.txt');\n await expect(storage.fileSize('path.txt')).rejects.toThrow();\n await expect(storage.mimeType('path.txt')).rejects.toThrow();\n await storage.deleteFile('path.txt');\n });\n });\n\n describe('always throwing an error', () => {\n const strategy = new AlwaysThrowError(() => new Error('Oh no...'));\n const adapter = new InMemoryStorageAdapter();\n\n afterEach(() => adapter.deleteEverything());\n\n const storage = new FileStorage(\n new ChaosStorageAdapterDecorator(\n adapter,\n strategy,\n ),\n );\n\n test('on any call', async () => {\n await expect(storage.write('path.txt', 'contents')).rejects.toThrow();\n });\n });\n\n describe('never throwing an error', () => {\n const strategy = new NeverThrowError();\n const adapter = new InMemoryStorageAdapter();\n\n afterEach(() => adapter.deleteEverything());\n\n const storage = new FileStorage(\n new ChaosStorageAdapterDecorator(\n adapter,\n strategy,\n ),\n );\n\n test('on any call', async () => {\n await storage.write('path.txt', 'contents');\n\n expect(await storage.readToString('path.txt')).toEqual('contents');\n });\n });\n});" }, { "path": "packages/chaos/src/index.ts", "content": "import {\n ChecksumOptions,\n CopyFileOptions,\n CreateDirectoryOptions,\n FileContents,\n MimeTypeOptions,\n MiscellaneousOptions,\n MoveFileOptions,\n StatEntry,\n StorageAdapter,\n TemporaryUrlOptions,\n WriteOptions,\n} from '@flystorage/file-storage';\nimport {Readable} from 'stream';\n\nexport type AnyAdapterMethodName = keyof StorageAdapter;\n\nexport interface ChaosStrategy {\n maybeGoNuts(method: AnyAdapterMethodName): void;\n}\n\nexport class AlwaysThrowError implements ChaosStrategy {\n constructor(private readonly newError: () => Error) {\n }\n\n maybeGoNuts(method: keyof StorageAdapter): void {\n throw (this.newError)();\n }\n}\n\nexport class NeverThrowError implements ChaosStrategy {\n maybeGoNuts(method: keyof StorageAdapter): void {\n // do not do anything\n }\n}\n\nexport class TriggeredErrors implements ChaosStrategy {\n private triggers: {\n [I in AnyAdapterMethodName | '*']?: {\n after: number,\n times: number,\n newError: () => unknown,\n }\n } = {};\n\n on(method: AnyAdapterMethodName | '*', newError: () => unknown, options: {after?: number, times?: number} = {}) {\n this.triggers[method] = {\n after: options.after ?? 0,\n times: options.times ?? Number.MAX_SAFE_INTEGER,\n newError,\n };\n }\n\n clearTriggers(): void {\n this.triggers = {};\n }\n\n maybeGoNuts(method: keyof StorageAdapter): void {\n const trigger = this.triggers[method] ?? this.triggers['*'];\n\n if (trigger === undefined) {\n return;\n }\n\n if (trigger.after > 0) {\n trigger.after--;\n } else if (trigger.times > 0) {\n trigger.times--;\n\n throw (trigger.newError)();\n }\n }\n}\n\nexport class ChaosStorageAdapterDecorator implements StorageAdapter {\n constructor(\n private readonly storage: StorageAdapter,\n private readonly chaos: ChaosStrategy,\n ) {\n }\n\n write(path: string, contents: Readable, options: WriteOptions): Promise {\n this.chaos.maybeGoNuts('write');\n\n return this.storage.write(path, contents, options);\n }\n\n read(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('read');\n\n return this.storage.read(path, options);\n }\n\n deleteFile(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('deleteFile');\n\n return this.storage.deleteFile(path, options);\n }\n\n createDirectory(path: string, options: CreateDirectoryOptions): Promise {\n this.chaos.maybeGoNuts('createDirectory');\n\n return this.storage.createDirectory(path, options);\n }\n\n copyFile(from: string, to: string, options: CopyFileOptions): Promise {\n this.chaos.maybeGoNuts('copyFile');\n\n return this.storage.copyFile(from, to, options);\n }\n\n moveFile(from: string, to: string, options: MoveFileOptions): Promise {\n this.chaos.maybeGoNuts('moveFile');\n\n return this.storage.moveFile(from, to, options);\n }\n\n stat(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('stat');\n\n return this.storage.stat(path, options);\n }\n\n list(path: string, options: { deep: boolean; }): AsyncGenerator {\n this.chaos.maybeGoNuts('list');\n\n return this.storage.list(path, options);\n }\n\n changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('changeVisibility');\n\n return this.storage.changeVisibility(path, visibility, options);\n }\n\n visibility(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('visibility');\n\n return this.storage.visibility(path, options);\n }\n\n deleteDirectory(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('deleteDirectory');\n\n return this.storage.deleteDirectory(path, options);\n }\n\n fileExists(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('fileExists');\n\n return this.storage.fileExists(path, options);\n }\n\n directoryExists(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('directoryExists');\n\n return this.storage.directoryExists(path, options);\n }\n\n publicUrl(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('publicUrl');\n\n return this.storage.publicUrl(path, options);\n }\n\n temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n this.chaos.maybeGoNuts('publicUrl');\n\n return this.storage.publicUrl(path, options);\n }\n\n checksum(path: string, options: ChecksumOptions): Promise {\n this.chaos.maybeGoNuts('checksum');\n\n return this.storage.checksum(path, options);\n }\n\n mimeType(path: string, options: MimeTypeOptions): Promise {\n this.chaos.maybeGoNuts('mimeType');\n\n return this.storage.mimeType(path, options);\n }\n\n lastModified(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('lastModified');\n\n return this.storage.lastModified(path, options);\n }\n\n fileSize(path: string, options: MiscellaneousOptions): Promise {\n this.chaos.maybeGoNuts('fileSize');\n\n return this.storage.fileSize(path, options);\n }\n\n}" }, { "path": "packages/chaos/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packages/dynamic-import/.npmignore", "content": "index.test.ts\ntsconfig.json\nsrc/" }, { "path": "packages/dynamic-import/CHANGELOG.md", "content": "# `@flystorage/dynamic-import`\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/dynamic-import/README.md", "content": "\n\n# Utility for dynamically importing ESM code\n\nThe TypeScript compiler turns dynamic import statements into require calls. This breaks importing\nESM packages in CommonJS code.\n\nThis utility helps to work around this by provided a JavaScript function to do the actual importing. \n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/dynamic-import\n```\n\n## Usage\n\n```typescript\nconst {dynamicallyImport} = require('@flystorage/dynamic-import');\n\nasync function useFileType() {\n const {fileTypeFromBuffer} = await dynamicallyImport('file-type');\n}\n\nuseFileType();\n```\n" }, { "path": "packages/dynamic-import/index.d.ts", "content": "export declare function dynamicallyImport(path: string): Promise;\n" }, { "path": "packages/dynamic-import/index.js", "content": "async function dynamicallyImport(path) {\n return await import(path);\n};\n\nexports.dynamicallyImport = dynamicallyImport;" }, { "path": "packages/dynamic-import/index.test.ts", "content": "import {dynamicallyImport} from './index.js';\n\n// @ts-ignore\ntype FileType = typeof import('file-type');\n\ndescribe('dynamic-import', () => {\n test('dynamic-import', async () => {\n const fileType = await dynamicallyImport('file-type');\n\n expect(fileType.supportedExtensions.has('jpg')).toEqual(true);\n });\n})" }, { "path": "packages/dynamic-import/package.json", "content": "{\n \"name\": \"@flystorage/dynamic-import\",\n \"type\": \"commonjs\",\n \"version\": \"1.0.0\",\n \"description\": \"Utility to dynamically import ESM packages from CommonJS without tsc messing things up.\",\n \"main\": \"./index.js\",\n \"types\": \"./index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./index.d.ts\",\n \"default\": \"./index.js\"\n },\n \"require\": {\n \"types\": \"./index.d.ts\",\n \"default\": \"./index.js\"\n }\n }\n },\n \"scripts\": {},\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/dynamic-import\"\n },\n \"keywords\": [],\n \"license\": \"MIT\"\n}" }, { "path": "packages/dynamic-import/src/index.ts", "content": "//\nexport * from '../index.js';" }, { "path": "packages/file-storage/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/file-storage/CHANGELOG.md", "content": "# `@flystorage/file-storage`\n\n## 1.2.2\n\n### Fixes\n\n- Fixed typo in timeout options name\n\n## 1.2.1\n\n### Fixes\n\n- Prevent unprefixed root paths to become '/' for directory listings.\n\n## 1.2.0\n\n### Added\n\n- Ability to ignore visibility using configuration\n- Support Bun runtime\n\n## 1.1.0\n\n### Added\n\n- AbortSignal Support\n- Normalised over stream errors for reads\n- Introduce way to detect failed reads because of missing files.\n\n## 1.0.1\n\n### Fixes\n\n- Corrected error message for `UnableToGetChecksum` error.\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/file-storage/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/file-storage/README.md", "content": "\n\n# Flystorage\nFlystorage is a file storage abstraction for NodeJS and TypeScript. It is an 80/20 solution\nthat is built around a set of goals:\n\n- Provide a straight-forward API that is easy to use.\n- Allow application code to be unaware WHERE files are stored.\n- Pragmatically smooth over underlying storage differences.\n- Expose an async/await based API, promises all the way.\n- Abstract over file permissions using \"visibility\".\n- Actually tested using real integrations, _mocks are not welcome_.\n- Stand on the shoulders of giants, use official vendor packages when possible.\n\n### What is Flystorage NOT:\nFlystorage is meant to be used in cases for generic file storage use-cases. It's not an API for\nany specific filesystem. It's a generalised solution and will not implement feature only\nspecific to one particular storage implementation. There will be use-cases that are not catered\nto, simply because they cannot be abstracted over in a reasonable manner.\n\n## Capabilities\n\n- Write files using string | buffer | readable/stream\n- Read files as stream, string, or Uint8Array\n- List the contents of a directory/prefix, (shallow and deep).\n- Metadata, such as; mime-type, last modified timestamp, and file size.\n- Expose or calculate checksums for files.\n- Delete files without failing when they don't exist.\n- Set permissions using abstracted visibility\n- Delete directories (and any files it contains)\n- Generate public URLs.\n- Generate temporary (signed) URLs.\n- Secure direct uploads from the browser.\n- Moving files\n- Copying files\n\n\n## Adapters\n- [Local Filesystem](https://www.npmjs.com/package/@flystorage/local-fs)\n- [AWS S3 (using the V3 SDK)](https://www.npmjs.com/package/@flystorage/aws-s3)\n- [Azure Blob Storage](https://www.npmjs.com/package/@flystorage/azure-storage-blob)\n- [Test implementation (in-memory)](https://www.npmjs.com/package/@flystorage/in-memory)\n- [Google Cloud Storage](https://www.npmjs.com/package/@flystorage/google-cloud-storage)\n- [Chaos adapter decorator](https://www.npmjs.com/package/@flystorage/chaos)\n\n## Usage\nInstall the main package and any adapters you might need:\n\n```bash\nnpm i -S @flystorage/file-storage\n\n# for using AWS S3\nnpm i -S @flystorage/aws-s3\n\n# for using the local filesystem\nnpm i -S @flystorage/local-fs\n```\n\n## Local Usage\n```typescript\nimport {resolve} from 'node:path';\nimport {createReadStream} from 'node:fs';\nimport {FileStorage, Visibility} from '@flystorage/file-storage';\nimport {LocalStorageAdapter} from '@flystorage/local-fs';\n\n/**\n * SETUP\n **/\n\nconst rootDirectory = resolve(process.cwd(), 'my-files');\nconst storage = new FileStorage(new LocalStorageAdapter(rootDirectory));\n\n/**\n * USAGE\n **/\n\n// Write using a string\nawait storage.write('write-from-a-string.txt', 'file contents');\n\n// Write using a stream\nconst stream = createReadStream(resolve(process.cwd(), 'test-files/picture.png'));\nawait storage.write('picture.png', stream);\n\n// Write with visibility (permissions).\nawait storage.write('public.txt', 'debug', {\n visibility: Visibility.PUBLIC, // mode: 0o644\n});\nawait storage.write('private.txt', 'debug', {\n visibility: Visibility.PRIVATE, // mode: 0o600\n});\n\n// List directory contents\nconst contentsAsAsyncGenerator = storage.list('', {deep: true});\n\nfor await (const item of contentsAsAsyncGenerator) {\n console.log(item.path);\n\n if (item.isFile) {\n // do something with the file\n } else if (item.isDirectory) {\n // do something with the directory\n }\n}\n\n// Delete a file\nawait storage.deleteFile('some-file.txt');\n\n// Delete a directory (with all contents)\nawait storage.deleteDirectory('some-directory');\n```\n\n## Author\nFlystorage is built by the maintainer of [Flysystem](https://flysystem.thephpleague.com), a\nfilesystem abstraction for PHP. This brings along over\na decade of filesystem abstraction experience.\n" }, { "path": "packages/file-storage/package.json", "content": "{\n \"name\": \"@flystorage/file-storage\",\n \"type\": \"module\",\n \"version\": \"1.2.2\",\n \"description\": \"File-storage abstraction: multiple filesystems, one API.\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/file-storage\"\n },\n \"homepage\": \"https://flystorage.dev/\",\n \"keywords\": [\n \"fs\",\n \"file\",\n \"files\",\n \"filesystem\",\n \"filesystems\",\n \"storage\"\n ],\n \"license\": \"MIT\"\n}" }, { "path": "packages/file-storage/src/checksum-from-stream.ts", "content": "import {BinaryToTextEncoding, createHash} from 'crypto';\nimport {Readable} from 'node:stream';\nimport {TextEncoder} from 'util';\n\nconst encoder = new TextEncoder();\n\nexport async function checksumFromStream(\n stream: Readable,\n options: { algo?: string; encoding?: BinaryToTextEncoding }\n): Promise {\n return new Promise((resolve, reject) => {\n const hash = createHash(options.algo ?? 'md5');\n stream.on('error', err => reject(err));\n stream.on('data', (chunk: Uint8Array | string | number) => {\n const type = typeof chunk;\n if (type === 'string') {\n chunk = encoder.encode(chunk as string);\n } else if (type === 'number') {\n chunk = new Uint8Array([chunk as number]);\n }\n hash.update(chunk as Uint8Array);\n });\n stream.on('end', () => resolve(hash.digest(options.encoding ?? 'hex')));\n });\n}" }, { "path": "packages/file-storage/src/errors.ts", "content": "export type ErrorContext = { [index: string]: any };\n\nexport type OperationError = Error & {\n readonly code: string,\n readonly context: ErrorContext,\n}\n\nexport function errorToMessage(error: unknown): string {\n return error instanceof Error ? error.message : String(error);\n}\n\nexport abstract class FlystorageError extends Error implements OperationError {\n readonly code: string = 'unknown_error';\n\n constructor(\n message: string,\n public readonly context: ErrorContext = {},\n cause: unknown = undefined,\n ) {\n const options = cause === undefined ? undefined : {cause};\n // @ts-ignore TS2554\n super(message, options);\n }\n}\n\n/**\n * Thrown when the checksum algo is not supported or not pre-computed. This error\n * is thrown with the intention of falling back to computing it based on a file read.\n */\nexport class ChecksumIsNotAvailable extends FlystorageError {\n public readonly code = 'flystorage.checksum_not_supported';\n\n constructor(\n message: string,\n public readonly algo: string,\n context: ErrorContext = {},\n cause: unknown = undefined,\n ) {\n super(message, context, cause);\n }\n\n static checksumNotSupported = (algo: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n } = {}) => new ChecksumIsNotAvailable(\n `Checksum algo \"${algo}\" is not supported`,\n algo,\n {...context, algo},\n cause,\n );\n\n static isErrorOfType(error: unknown): error is ChecksumIsNotAvailable {\n return (typeof error === 'object' && (error as any).code === 'flystorage.checksum_not_supported');\n }\n}\n\nexport class UnableToGetChecksum extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_checksum';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetChecksum(\n `Unable to get checksum for file. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToGetMimeType extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_mimetype';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetMimeType(\n `Unable to get mime-type. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToGetLastModified extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_last_modified';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetLastModified(\n `Unable to get last modified. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToGetFileSize extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_file_size';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetFileSize(\n `Unable to get file size. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToWriteFile extends FlystorageError {\n public readonly code = 'flystorage.unable_to_write_file';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToWriteFile(\n `Unable to write the file. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToReadFile extends FlystorageError {\n public readonly code = 'flystorage.unable_to_read_file';\n\n constructor(\n public readonly wasFileNotFound: boolean,\n message: string,\n public readonly context: ErrorContext = {},\n cause: unknown = undefined,\n ) {\n super(message, context, cause);\n }\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToReadFile(\n false,\n `Unable to read the file. Reason: ${reason}`,\n context,\n cause,\n );\n\n static becauseFileWasNotFound = (error: FileWasNotFound) => new UnableToReadFile(\n true,\n `Unable to read the file. Reason: ${error.message}`,\n error.context,\n error,\n );\n}\n\nexport class FileWasNotFound extends FlystorageError {\n public readonly code = 'flystorage.file_was_not_found';\n\n static atLocation = (location: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new FileWasNotFound(\n `File was not found at location: ${location}`,\n context,\n cause,\n );\n}\n\nexport function isFileWasNotFound(error: unknown): error is FileWasNotFound {\n return (typeof error === 'object' && (error as any).code === 'flystorage.file_was_not_found');\n}\n\nexport class UnableToSetVisibility extends FlystorageError {\n public readonly code = 'flystorage.unable_to_set_visibility';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToSetVisibility(\n `Unable to set visibility. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToGetVisibility extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_visibility';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetVisibility(\n `Unable to get visibility. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToGetPublicUrl extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_public_url';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetPublicUrl(\n `Unable to get public URL. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToGetTemporaryUrl extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_temporary_url';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetTemporaryUrl(\n `Unable to get temporary URL. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToPrepareUploadRequest extends FlystorageError {\n public readonly code = 'flystorage.unable_to_prepare_upload_request';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetTemporaryUrl(\n `Unable to prepare upload request. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToCopyFile extends FlystorageError {\n public readonly code = 'flystorage.unable_to_copy_file';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToCopyFile(\n `Unable to copy file. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToMoveFile extends FlystorageError {\n public readonly code = 'flystorage.unable_to_move_file';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToMoveFile(\n `Unable to move file. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToGetStat extends FlystorageError {\n public readonly code = 'flystorage.unable_to_get_stat';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetStat(\n `Unable to get stat. Reason: ${reason}`,\n context,\n cause,\n );\n\n static noFileStatResolved = ({context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToGetStat(\n `Stat was not a file.`,\n context,\n cause,\n );\n}\n\nexport class UnableToCreateDirectory extends FlystorageError {\n public readonly code = 'flystorage.unable_to_create_directory';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToCreateDirectory(\n `Unable to create directory. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToDeleteDirectory extends FlystorageError {\n public readonly code = 'flystorage.unable_to_delete_directory';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToDeleteDirectory(\n `Unable to delete directory. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToDeleteFile extends FlystorageError {\n public readonly code = 'flystorage.unable_to_delete_file';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToDeleteFile(\n `Unable to delete file. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToCheckFileExistence extends FlystorageError {\n public readonly code = 'flystorage.unable_to_check_file_existence';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToCheckFileExistence(\n `Unable to check file existence. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToCheckDirectoryExistence extends FlystorageError {\n public readonly code = 'flystorage.unable_to_check_directory_existence';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToCheckDirectoryExistence(\n `Unable to check directory existence. Reason: ${reason}`,\n context,\n cause,\n );\n}\n\nexport class UnableToListDirectory extends FlystorageError {\n public readonly code = 'flystorage.unable_to_list_directory_contents';\n\n static because = (reason: string, {context = {}, cause = undefined}: {\n context?: ErrorContext,\n cause?: unknown\n }) => new UnableToListDirectory(\n `Unable to list directory contents. Reason: ${reason}`,\n context,\n cause,\n );\n}" }, { "path": "packages/file-storage/src/file-storage.test.ts", "content": "import {FileStorage, UploadRequest, UploadRequestOptions} from './file-storage.js';\nimport {InMemoryStorageAdapter} from '@flystorage/in-memory';\nimport {createHash} from 'crypto';\n\ndescribe('FileStorage', () => {\n test('calculating a checksum through a fallback', async () => {\n const hash = createHash('md5');\n hash.update('contents');\n const expectedChecksum = hash.digest('hex');\n\n const storage = new FileStorage(new InMemoryStorageAdapter());\n\n await storage.write('something.txt', 'contents');\n\n const checksum = await storage.checksum('something.txt', {algo: 'md5'});\n expect(checksum).toEqual(expectedChecksum);\n });\n\n test('supplying a prepared upload strategy', async () => {\n const storage = new FileStorage(\n new InMemoryStorageAdapter(),\n undefined,\n {\n preparedUploadStrategy: {\n async prepareUpload(path: string, options: UploadRequestOptions): Promise {\n return {\n method: 'POST',\n url: `https://here.com/${path}`,\n headers: options.headers ?? {},\n };\n }\n }\n }\n );\n\n const request = await storage.prepareUpload('here.txt', {\n expiresAt: 0,\n headers: {\n 'content-type': 'application/json',\n }\n });\n\n expect(request).toEqual({\n method: 'POST',\n url: 'https://here.com/here.txt',\n headers: {\n 'content-type': 'application/json',\n },\n });\n });\n});" }, { "path": "packages/file-storage/src/file-storage.ts", "content": "import {BinaryToTextEncoding} from 'crypto';\nimport {Readable} from 'stream';\nimport {checksumFromStream} from './checksum-from-stream.js';\nimport {PathNormalizer, PathNormalizerV1} from './path-normalizer.js';\nimport {TextEncoder} from 'util';\nimport {\n ChecksumIsNotAvailable,\n errorToMessage,\n isFileWasNotFound,\n UnableToCheckDirectoryExistence,\n UnableToCheckFileExistence,\n UnableToCopyFile,\n UnableToCreateDirectory,\n UnableToDeleteDirectory,\n UnableToDeleteFile,\n UnableToGetChecksum,\n UnableToGetFileSize,\n UnableToGetLastModified,\n UnableToGetMimeType,\n UnableToGetPublicUrl,\n UnableToGetStat,\n UnableToGetTemporaryUrl,\n UnableToGetVisibility,\n UnableToListDirectory,\n UnableToMoveFile,\n UnableToPrepareUploadRequest,\n UnableToReadFile,\n UnableToSetVisibility,\n UnableToWriteFile,\n} from './errors.js';\nimport {PassThrough} from 'node:stream';\n\nexport type CommonStatInfo = Readonly<{\n path: string,\n lastModifiedMs?: number,\n visibility?: string,\n}>;\n\nexport type FileInfo = Readonly<{\n type: 'file',\n size?: number,\n isFile: true,\n isDirectory: false,\n mimeType?: string,\n} & CommonStatInfo>;\n\nexport type DirectoryInfo = Readonly<{\n type: 'directory',\n isFile: false,\n isDirectory: true,\n} & CommonStatInfo>;\n\nexport function isFile(stat: StatEntry): stat is FileInfo {\n return stat.isFile;\n}\n\nexport function isDirectory(stat: StatEntry): stat is DirectoryInfo {\n return stat.isDirectory;\n}\n\nexport type StatEntry = FileInfo | DirectoryInfo;\n\nexport type AdapterListOptions = ListOptions & { deep: boolean };\n\nexport interface StorageAdapter {\n write(path: string, contents: Readable, options: WriteOptions): Promise;\n\n read(path: string, options: MiscellaneousOptions): Promise;\n\n deleteFile(path: string, options: MiscellaneousOptions): Promise;\n\n createDirectory(path: string, options: CreateDirectoryOptions): Promise;\n\n copyFile(from: string, to: string, options: CopyFileOptions): Promise;\n\n moveFile(from: string, to: string, options: MoveFileOptions): Promise;\n\n stat(path: string, options: MiscellaneousOptions): Promise;\n\n list(path: string, options: AdapterListOptions): AsyncGenerator;\n\n changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise;\n\n visibility(path: string, options: MiscellaneousOptions): Promise;\n\n deleteDirectory(path: string, options: MiscellaneousOptions): Promise;\n\n fileExists(path: string, options: MiscellaneousOptions): Promise;\n\n directoryExists(path: string, options: MiscellaneousOptions): Promise;\n\n publicUrl(path: string, options: PublicUrlOptions): Promise;\n\n temporaryUrl(path: string, options: TemporaryUrlOptions): Promise;\n\n prepareUpload?(path: string, options: UploadRequestOptions): Promise;\n\n checksum(path: string, options: ChecksumOptions): Promise;\n\n mimeType(path: string, options: MimeTypeOptions): Promise;\n\n lastModified(path: string, options: MiscellaneousOptions): Promise;\n\n fileSize(path: string, options: MiscellaneousOptions): Promise;\n}\n\nexport class DirectoryListing implements AsyncIterable {\n constructor(\n private readonly listing: AsyncGenerator,\n private readonly path: string,\n private readonly deep: boolean,\n ) {\n }\n\n async toArray(sorted: boolean = true): Promise {\n const items = [];\n for await (const item of this.listing) {\n items.push(item);\n }\n\n return sorted ? items.sort((a, b) => naturalSorting.compare(a.path, b.path)) : items;\n }\n\n filter(filter: (entry: StatEntry) => boolean): DirectoryListing {\n const listing = this.listing;\n const filtered = (async function* () {\n for await (const entry of listing) {\n if (filter(entry)) {\n yield entry;\n }\n }\n })();\n\n return new DirectoryListing(filtered, this.path, this.deep);\n }\n\n async* [Symbol.asyncIterator]() {\n try {\n for await (const item of this.listing) {\n yield item;\n }\n } catch (error) {\n throw UnableToListDirectory.because(\n errorToMessage(error),\n {cause: error, context: {path: this.path, deep: this.deep}},\n );\n }\n }\n}\n\nexport type FileContents = Iterable | AsyncIterable | NodeJS.ReadableStream | Readable | string;\n\nexport type TimeoutOptions = { timeout?: number };\n\nexport type MiscellaneousOptions = TimeoutOptions & {\n [option: string]: any,\n abortSignal?: AbortSignal,\n}\n\nexport type MimeTypeOptions = MiscellaneousOptions & {\n disallowFallback?: boolean,\n fallbackMethod?: 'contents' | 'path',\n}\n\nexport type VisibilityFallback = {\n strategy: 'ignore',\n stagedVisibilityResponse?: string,\n} | {\n strategy: 'error',\n errorMessage?: string,\n}\n\nexport type VisibilityOptions = MiscellaneousOptions & {\n visibility?: string,\n directoryVisibility?: string,\n retainVisibility?: boolean,\n} & ({\n useVisibility: true,\n} | {\n useVisibility: false,\n visibilityFallback: VisibilityFallback,\n} | {});\n\nexport type WriteOptions = VisibilityOptions & MiscellaneousOptions & {\n mimeType?: string,\n size?: number,\n cacheControl?: string,\n};\nexport type CreateDirectoryOptions = MiscellaneousOptions & Pick & {};\nexport type PublicUrlOptions = MiscellaneousOptions & {};\nexport type UploadRequestOptions = MiscellaneousOptions & {\n expiresAt: ExpiresAt,\n contentType?: string,\n headers?: UploadRequestHeaders,\n};\nexport type CopyFileOptions = MiscellaneousOptions & VisibilityOptions & {\n retainVisibility?: boolean,\n};\nexport type MoveFileOptions = MiscellaneousOptions & VisibilityOptions & {\n retainVisibility?: boolean,\n};\nexport type ListOptions = MiscellaneousOptions & { deep?: boolean };\nexport type TemporaryUrlOptions = MiscellaneousOptions & {\n expiresAt: ExpiresAt,\n responseHeaders?: { [header: string]: string },\n};\n\nexport type ChecksumOptions = MiscellaneousOptions & {\n algo?: string,\n encoding?: BinaryToTextEncoding,\n}\n\nexport type ConfigurationOptions = {\n visibility?: VisibilityOptions,\n writes?: WriteOptions,\n moves?: MoveFileOptions,\n copies?: CopyFileOptions,\n publicUrls?: PublicUrlOptions,\n temporaryUrls?: TemporaryUrlOptions,\n uploadRequest?: UploadRequestOptions,\n checksums?: ChecksumOptions,\n mimeTypes?: MimeTypeOptions,\n preparedUploadStrategy?: PreparedUploadStrategy,\n timeout?: TimeoutOptions,\n list?: ListOptions,\n}\n\nexport function toReadable(contents: FileContents): Readable {\n if (contents instanceof Readable) {\n return contents;\n }\n\n return Readable.from(contents);\n}\n\nconst naturalSorting = new Intl.Collator(undefined, {\n numeric: true,\n sensitivity: 'base',\n});\n\nfunction instrumentAbortSignal(options: Options): Options {\n let abortSignal = options.abortSignal;\n\n if (options.timeout !== undefined) {\n const timeoutAbort = AbortSignal.timeout(options.timeout);\n\n if (options.abortSignal) {\n const originalAbortSignal = options.abortSignal;\n abortSignal = AbortSignal.any([\n originalAbortSignal,\n timeoutAbort,\n ]);\n } else {\n abortSignal = timeoutAbort;\n }\n }\n\n if (abortSignal?.aborted) {\n throw abortSignal.reason;\n }\n\n return {...options, abortSignal};\n}\n\nexport class FileStorage {\n constructor(\n private readonly adapter: StorageAdapter,\n private readonly pathNormalizer: PathNormalizer = new PathNormalizerV1(),\n private readonly options: ConfigurationOptions = {},\n ) {\n }\n\n public async write(path: string, contents: FileContents, options: WriteOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.writes, ...options});\n\n try {\n const body = toReadable(contents);\n await this.adapter.write(\n this.pathNormalizer.normalizePath(path),\n body,\n options,\n );\n await closeReadable(body);\n } catch (error) {\n throw UnableToWriteFile.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n public async read(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n const stream = Readable.from(\n await this.adapter.read(this.pathNormalizer.normalizePath(path), options),\n );\n\n const streamOut = new PassThrough();\n stream.on('error', (error) => {\n stream.unpipe(streamOut);\n\n streamOut.destroy(\n isFileWasNotFound(error)\n ? UnableToReadFile.becauseFileWasNotFound(error)\n : error,\n );\n });\n stream.pipe(streamOut);\n\n return streamOut;\n } catch (error) {\n if (isFileWasNotFound(error)) {\n throw UnableToReadFile.becauseFileWasNotFound(error);\n }\n\n throw UnableToReadFile.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public async readToString(path: string, options: MiscellaneousOptions = {}): Promise {\n return await readableToString(await this.read(path, options));\n }\n\n public async readToUint8Array(path: string, options: MiscellaneousOptions = {}): Promise {\n return await readableToUint8Array(await this.read(path, options));\n }\n\n public async readToBuffer(path: string, options: MiscellaneousOptions = {}): Promise {\n return Buffer.from(await this.readToUint8Array(path, options));\n }\n\n public async deleteFile(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n await this.adapter.deleteFile(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToDeleteFile.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public async createDirectory(path: string, options: CreateDirectoryOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...options});\n\n try {\n return await this.adapter.createDirectory(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToCreateDirectory.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n public async deleteDirectory(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n return await this.adapter.deleteDirectory(this.pathNormalizer.normalizePath(path), options);\n } catch (error) {\n throw UnableToDeleteDirectory.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public async stat(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n return await this.adapter.stat(this.pathNormalizer.normalizePath(path), options);\n } catch (error) {\n throw UnableToGetStat.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public async moveFile(from: string, to: string, options: MoveFileOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.moves, ...options});\n\n try {\n await this.adapter.moveFile(\n this.pathNormalizer.normalizePath(from),\n this.pathNormalizer.normalizePath(to),\n options,\n );\n } catch (error) {\n throw UnableToMoveFile.because(\n errorToMessage(error),\n {cause: error, context: {from, to}},\n );\n }\n }\n\n public async copyFile(from: string, to: string, options: CopyFileOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...this.options.copies, ...options});\n\n try {\n await this.adapter.copyFile(\n this.pathNormalizer.normalizePath(from),\n this.pathNormalizer.normalizePath(to),\n options,\n );\n } catch (error) {\n throw UnableToCopyFile.because(\n errorToMessage(error),\n {cause: error, context: {from, to}},\n );\n }\n }\n\n public async changeVisibility(path: string, visibility: string, options: VisibilityOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n if (options.useVisibility === false) {\n const fallback: VisibilityFallback = options.visibilityFallback;\n\n if (fallback.strategy === 'ignore') {\n return;\n } else if (fallback.strategy === 'error') {\n throw UnableToSetVisibility.because(fallback.errorMessage ?? 'Configured not to use visibility', {\n context: {path, visibility},\n });\n }\n }\n\n try {\n return await this.adapter.changeVisibility(this.pathNormalizer.normalizePath(path), visibility, options);\n } catch (error) {\n throw UnableToSetVisibility.because(\n errorToMessage(error),\n {cause: error, context: {path, visibility}},\n );\n }\n }\n\n public async visibility(path: string, options: VisibilityOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.visibility, ...options});\n\n if (options.useVisibility === false) {\n const fallback: VisibilityFallback = options.visibilityFallback;\n\n if (fallback.strategy === 'ignore') {\n return fallback.stagedVisibilityResponse ?? 'unknown';\n } else if (fallback.strategy === 'error') {\n throw UnableToGetVisibility.because(fallback.errorMessage ?? 'Configured not to use visibility', {\n context: {path},\n });\n }\n }\n\n try {\n return await this.adapter.visibility(this.pathNormalizer.normalizePath(path), options);\n } catch (error) {\n throw UnableToGetVisibility.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public async fileExists(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n return await this.adapter.fileExists(this.pathNormalizer.normalizePath(path), options);\n } catch (error) {\n throw UnableToCheckFileExistence.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public list(path: string, options: ListOptions = {}): DirectoryListing {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.list, ...options});\n\n const adapterOptions: AdapterListOptions = {\n ...options,\n deep: options.deep ?? false,\n };\n\n return new DirectoryListing(\n this.adapter.list(this.pathNormalizer.normalizePath(path), adapterOptions),\n path,\n adapterOptions.deep,\n );\n }\n\n public async statFile(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n const stat = await this.stat(path, options);\n\n if (isFile(stat)) {\n return stat;\n }\n\n throw UnableToGetStat.noFileStatResolved({context: {path}});\n }\n\n public async directoryExists(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n return await this.adapter.directoryExists(this.pathNormalizer.normalizePath(path), options);\n } catch (error) {\n throw UnableToCheckDirectoryExistence.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public async publicUrl(path: string, options: PublicUrlOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.publicUrls, ...options});\n\n try {\n return await this.adapter.publicUrl(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToGetPublicUrl.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n public async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.temporaryUrls, ...options});\n\n try {\n return await this.adapter.temporaryUrl(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToGetTemporaryUrl.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n public async prepareUpload(path: string, options: UploadRequestOptions): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.uploadRequest, ...options});\n\n if (this.options.preparedUploadStrategy !== undefined) {\n try {\n return this.options.preparedUploadStrategy.prepareUpload(path, options);\n } catch (error) {\n throw UnableToPrepareUploadRequest.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n if (typeof this.adapter.prepareUpload !== 'function') {\n throw new Error('The used adapter does not support prepared uploads.');\n }\n\n try {\n return await this.adapter.prepareUpload(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToPrepareUploadRequest.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n public async checksum(path: string, options: ChecksumOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.checksums, ...options});\n\n try {\n return await this.adapter.checksum(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n if (ChecksumIsNotAvailable.isErrorOfType(error)) {\n return this.calculateChecksum(path, options);\n }\n\n throw UnableToGetChecksum.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n public async mimeType(path: string, options: MimeTypeOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...this.options.mimeTypes, ...options});\n\n try {\n return await this.adapter.mimeType(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToGetMimeType.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n\n public async lastModified(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n return await this.adapter.lastModified(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToGetLastModified.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n public async fileSize(path: string, options: MiscellaneousOptions = {}): Promise {\n options = instrumentAbortSignal({...this.options.timeout, ...options});\n\n try {\n return await this.adapter.fileSize(\n this.pathNormalizer.normalizePath(path),\n options,\n );\n } catch (error) {\n throw UnableToGetFileSize.because(\n errorToMessage(error),\n {cause: error, context: {path}},\n );\n }\n }\n\n private async calculateChecksum(path: string, options: ChecksumOptions): Promise {\n try {\n return await checksumFromStream(await this.read(path, options), options);\n } catch (error) {\n throw UnableToGetChecksum.because(\n errorToMessage(error),\n {cause: error, context: {path, options}},\n );\n }\n }\n}\n\nexport type TimestampMs = number;\nexport type ExpiresAt = Date | TimestampMs;\n\nexport function normalizeExpiryToDate(expiresAt: ExpiresAt): Date {\n return expiresAt instanceof Date ? expiresAt : new Date(expiresAt);\n}\n\nexport function normalizeExpiryToMilliseconds(expiresAt: ExpiresAt): number {\n return expiresAt instanceof Date ? expiresAt.getTime() : expiresAt;\n}\n\nexport async function closeReadable(body: Readable) {\n if (body.closed || body.destroyed) {\n return;\n }\n\n await new Promise((resolve, reject) => {\n body.on('error', reject);\n body.on('close', (err: any) => {\n err ? reject(err) : resolve();\n });\n body.destroy();\n });\n}\n\nconst decoder = new TextDecoder();\n\nexport async function readableToString(stream: Readable): Promise {\n const contents = decoder.decode(await readableToUint8Array(stream));\n await closeReadable(stream);\n\n return contents;\n}\n\nexport async function readableToBuffer(stream: Readable): Promise {\n return new Promise((resolve, reject) => {\n const buffers: Buffer[] = [];\n stream.on('data', chunk => buffers.push(Buffer.from(chunk)));\n stream.on('end', () => resolve(Buffer.concat(buffers)));\n stream.on('finish', () => resolve(Buffer.concat(buffers)));\n stream.on('error', err => reject(err));\n });\n}\n\nconst encoder = new TextEncoder();\n\nexport function readableToUint8Array(stream: Readable): Promise {\n return new Promise((resolve, reject) => {\n const parts: Uint8Array[] = [];\n stream.on('data', (chunk: Uint8Array | string | number) => {\n const type = typeof chunk;\n if (type === 'string') {\n chunk = encoder.encode(chunk as string);\n } else if (type === 'number') {\n chunk = new Uint8Array([chunk as number]);\n }\n parts.push(chunk as Uint8Array);\n });\n stream.on('error', reject);\n stream.on('end', () => resolve(concatUint8Arrays(parts)));\n });\n}\n\nfunction concatUint8Arrays(input: Uint8Array[]): Uint8Array {\n const length = input.reduce((l, a) => l + (a.byteLength), 0);\n const output = new Uint8Array(length);\n let position = 0;\n input.forEach(i => {\n output.set(i, position);\n position += i.byteLength;\n });\n\n return output;\n}\n\nexport type UploadRequestHeaders = Record>;\n\nexport type UploadRequest = {\n url: string,\n provider?: string,\n method: 'PUT' | 'POST'\n headers: UploadRequestHeaders,\n}\n\nexport interface PreparedUploadStrategy {\n prepareUpload(path: string, options: UploadRequestOptions): Promise;\n}\n\nexport class PreparedUploadsAreNotSupported implements PreparedUploadStrategy {\n prepareUpload(): Promise {\n throw new Error('The used adapter does not support prepared uploads.');\n }\n}\n\n" }, { "path": "packages/file-storage/src/index.ts", "content": "export * from './checksum-from-stream.js';\nexport * from './file-storage.js';\nexport * from './errors.js';\nexport * from './path-normalizer.js';\nexport * from './path-prefixer.js'\nexport * from './portable-visibility.js';\n" }, { "path": "packages/file-storage/src/path-normalizer.test.ts", "content": "import {CorruptedPathDetected, PathNormalizerV1, PathTraversalDetected} from './path-normalizer.js';\n\ndescribe('PathNormalizerV1', () => {\n const normalizer = new PathNormalizerV1();\n test.each([\n ['.', ''],\n ['/path/to/dir/.', 'path/to/dir'],\n ['/dirname/', 'dirname'],\n ['dirname/..', ''],\n ['dirname/../', ''],\n ['dirname./', 'dirname.'],\n ['dirname/./', 'dirname'],\n ['dirname/.', 'dirname'],\n ['./dir/../././', ''],\n ['/something/deep/../../dirname', 'dirname'],\n ['00004869/files/other/10-75..stl', '00004869/files/other/10-75..stl'],\n ['/dirname//subdir///subsubdir', 'dirname/subdir/subsubdir'],\n ['example/path/..txt', 'example/path/..txt'],\n ['/example/../path.txt', 'path.txt'],\n ])('normalizing \"%s\" to \"%s\"', (path, expected) => {\n expect(normalizer.normalizePath(path)).toEqual(expected);\n });\n\n test.each([\n ['something/../../../hehe'],\n ['/something/../../..'],\n ['..'],\n ])('detecting path traversal on \"%s\"', (path) => {\n expect(() => normalizer.normalizePath(path)).toThrow(PathTraversalDetected.forPath(path));\n });\n\n test.each([\n ['some\\0/path.txt'],\n ['s\\x09i.php'],\n ])('detecting funky whitespace on \"%s\"', (path) => {\n expect(() => normalizer.normalizePath(path)).toThrow(CorruptedPathDetected.unexpectedWhitespace(path));\n });\n})" }, { "path": "packages/file-storage/src/path-normalizer.ts", "content": "import {join} from 'node:path';\n\nexport interface PathNormalizer {\n normalizePath(path: string): string\n}\n\nconst funkyWhiteSpaceRegex = new RegExp('\\\\p{C}+', 'u');\n\nexport class PathNormalizerV1 implements PathNormalizer {\n normalizePath(path: string): string {\n if (funkyWhiteSpaceRegex.test(path)) {\n throw CorruptedPathDetected.unexpectedWhitespace(path);\n }\n\n const normalized = join(...(path.split('/')));\n\n if (normalized.indexOf('../') !== -1 || normalized == '..') {\n throw PathTraversalDetected.forPath(path);\n }\n\n return normalized === '.' ? '' : normalized;\n }\n}\n\nexport class CorruptedPathDetected extends Error {\n static unexpectedWhitespace = (path: string) => new CorruptedPathDetected(\n `Corrupted path detected with unexpected whitespace: ${path}`\n );\n}\n\nexport class PathTraversalDetected extends Error {\n static forPath = (path: string) => new PathTraversalDetected(\n `Path traversal detected for: ${path}`\n );\n}" }, { "path": "packages/file-storage/src/path-prefixer.test.ts", "content": "import {PathPrefixer} from './path-prefixer.js';\n\ndescribe('PathPrefixer', () => {\n describe.each([\n ['with', '/prefix/'],\n ['without', '/prefix'],\n ])('prefixing %s a trailing slash prefix', (_name, prefix: string) => {\n const prefixer = new PathPrefixer(prefix);\n\n test('prefixing a file path with a leading slash', () => {\n expect(prefixer.prefixFilePath('/file.txt')).toEqual('/prefix/file.txt');\n });\n\n test('prefixing a file path without a leading slash', () => {\n expect(prefixer.prefixFilePath('file.txt')).toEqual('/prefix/file.txt');\n });\n\n test('prefixing a directory path with a leading slash', () => {\n expect(prefixer.prefixDirectoryPath('/dirname')).toEqual('/prefix/dirname/');\n });\n\n test('prefixing a directory path with a trailing slash', () => {\n expect(prefixer.prefixDirectoryPath('dirname/')).toEqual('/prefix/dirname/');\n });\n });\n\n describe.each([\n ['with', '/prefix/'],\n ['without', '/prefix'],\n ])('stripping %s a trailing slash prefix', (_name, prefix: string) => {\n const prefixer = new PathPrefixer(prefix);\n\n test('stripping a file path', () => {\n expect(prefixer.stripFilePath('/prefix/file.txt')).toEqual('file.txt');\n });\n\n test('stripping a directory path without a trailing slash', () => {\n expect(prefixer.stripDirectoryPath('/prefix/dirname')).toEqual('dirname');\n });\n\n test('prefixing a directory path with a trailing slash', () => {\n expect(prefixer.stripDirectoryPath('/prefix/dirname/')).toEqual('dirname');\n });\n });\n\n describe('prefixing with an empty prefix', () => {\n const prefixer = new PathPrefixer();\n\n test('prefixing with a file path with leading slash', () => {\n expect(prefixer.prefixFilePath('/file.txt')).toEqual('/file.txt');\n });\n\n test('prefixing with a directory path with leading slash', () => {\n expect(prefixer.prefixDirectoryPath('/directory')).toEqual('/directory/');\n });\n })\n});" }, { "path": "packages/file-storage/src/path-prefixer.ts", "content": "import {type join, posix} from 'node:path';\n\nexport class PathPrefixer {\n private readonly prefix: string = '';\n constructor(\n prefix: string = '',\n private readonly separator: string = '/',\n private readonly joinFunc: typeof join = posix.join,\n ) {\n if (prefix.length > 0) {\n this.prefix = this.joinFunc(prefix, this.separator);\n }\n }\n\n prefixFilePath(path: string): string {\n return this.prefix.length > 0 ? this.joinFunc(this.prefix, path): path;\n }\n\n prefixDirectoryPath(path: string): string {\n let fullPath = this.prefix.length > 0\n ? this.joinFunc(this.prefix, path)\n : path;\n\n if (fullPath.length > 0 && !fullPath.endsWith(this.separator)) {\n fullPath = `${fullPath}${this.separator}`;\n }\n\n return fullPath;\n }\n\n stripFilePath(path: string): string {\n return path.substring(this.prefix.length);\n }\n\n stripDirectoryPath(path: string): string {\n return this.stripFilePath(path).replace(/\\/+$/g, '');\n }\n}" }, { "path": "packages/file-storage/src/portable-visibility.ts", "content": "export enum Visibility {\n PUBLIC = 'public',\n PRIVATE = 'private',\n}" }, { "path": "packages/file-storage/src/readable-convertion.test.ts", "content": "import {Readable} from \"node:stream\";\nimport {readableToString} from \"./file-storage.js\";\n\ndescribe('converting readables', () => {\n test('converting a stream to a string', async () => {\n expect(true).toEqual(true);\n const readable = Readable.from('something');\n\n const result = await readableToString(readable);\n\n expect(result).toEqual('something');\n });\n});" }, { "path": "packages/file-storage/src/utilities.test.ts", "content": "import {Readable} from 'stream';\nimport {readableToBuffer} from './file-storage.js';\n\ndescribe('utilities', () => {\n describe('readableToBuffer', () => {\n test('converting to and from a buffer', async () => {\n const input = Buffer.from('buffer contents');\n const inputReadable = Readable.from(input);\n\n const output = await readableToBuffer(inputReadable);\n\n expect(output.toString()).toEqual(input.toString());\n })\n })\n});" }, { "path": "packages/file-storage/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packages/google-cloud-storage/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/google-cloud-storage/CHANGELOG.md", "content": "# `@flystorage/google-cloud-storage`\n\n## 1.2.0\n\n### Changes\n\n- Upgraded dependencies\n- Remove not-used import\n\n## 1.1.0\n\n### Changes\n\n- Improved stream errors\n- Added way to detect reads failed due to missing files\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/google-cloud-storage/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/google-cloud-storage/README.md", "content": "\n\n# Flystorage adapter for AWS S3\n\nThis package contains the Flystorage adapter for Google Cloud Storage\n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/file-storage @flystorage/google-cloud-storage @google-cloud/storage\n```\n\n## Usage\n\n```typescript\nimport {FileStorage} from '@flystorage/file-storage';\nimport {GoogleCloudStorageStorageAdapter} from '@flystorage/google-cloud-storage';\nimport {Storage} from '@google-cloud/storage';\n\nconst client = new Storage();\nconst bucket = client.bucket('{bucket-name}}', {\n userProject: '{user-project}}',\n});\nconst adapter = new GoogleCloudStorageStorageAdapter(bucket, {\n prefix: '{optional-path-prefix}',\n});\nconst storage = new FileStorage(adapter);\n```\n\n> ⚠️ Always use the FileStorage, it is essential for security and a good developer\n> experience. Do not use the adapter directly.\n\n## Visibility\n\nSetting an retrieving visibility is only meaningful for legacy buckets. To use this functionality\nwith Flystorage, pass the legacy visibility handling to the constructor:\n\n```typescript\nimport {GoogleCloudStorageStorageAdapter, LegacyVisibilityHandling} from '@flystorage/google-cloud-storage';\n\nconst adapter = new GoogleCloudStorageStorageAdapter(bucket, {\n prefix: '{optional-path-prefix}',\n}, new LegacyVisibilityHandling(\n 'allUsers', // acl entity, optional\n 'publicRead', // acl for Visibility.PUBLIC, optional,\n 'projectPrivate', // acl for Visibility.PRIVATE, optional,\n));\n```\n\n" }, { "path": "packages/google-cloud-storage/package.json", "content": "{\n \"name\": \"@flystorage/google-cloud-storage\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@flystorage/file-storage\": \"^1.1.0\",\n \"@flystorage/stream-mime-type\": \"^1.0.0\",\n \"@google-cloud/storage\": \"^7.19.0\",\n \"file-type\": \"^21.3.1\",\n \"mime-types\": \"^3.0.2\"\n },\n \"description\": \"\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"keywords\": [\n \"google\",\n \"file\",\n \"storage\",\n \"flystorage\",\n \"filesystem\"\n ],\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/adapter/google-cloud-storage/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/google-cloud-storage\"\n },\n \"license\": \"MIT\"\n}\n" }, { "path": "packages/google-cloud-storage/src/google-cloud-storage.test.ts", "content": "import { Storage } from '@google-cloud/storage';\nimport {GoogleCloudStorageAdapter} from './google-cloud-storage.js';\nimport {\n FileStorage,\n UploadRequestHeaders,\n readableToString,\n UnableToReadFile, Visibility,\n} from '@flystorage/file-storage';\nimport {randomBytes} from 'crypto';\nimport {resolve} from 'node:path';\nimport * as https from 'https';\nimport {LegacyVisibilityHandling} from './visibility-handling.js';\n\nconst testSegment = randomBytes(10).toString('hex');\nlet adapter: GoogleCloudStorageAdapter;\nlet storage: FileStorage;\nlet cleanupStorage: FileStorage;\n\ndescribe('GoogleCloudStorageAdapter', () => {\n const googleStorage = new Storage({keyFilename: resolve(process.cwd(), 'google-cloud-service-account.json')});\n const bucket = googleStorage.bucket('no-acl-bucket-for-ci', {\n userProject: 'flysystem-testing',\n });\n const legacyBucket = googleStorage.bucket('flysystem', {\n userProject: 'flysystem-testing',\n });\n\n beforeAll(async () => {\n const cleanupAdapter = new GoogleCloudStorageAdapter(bucket, {\n prefix: `flystorage`,\n });\n cleanupStorage = new FileStorage(cleanupAdapter);\n await cleanupStorage.deleteDirectory('non-existing-directory');\n })\n\n afterAll(async () => {\n await cleanupStorage.deleteDirectory(testSegment);\n });\n\n describe('base API', () => {\n beforeEach(() => {\n const secondSegment = randomBytes(10).toString('hex');\n\n adapter = new GoogleCloudStorageAdapter(bucket, {\n prefix: `flystorage/${testSegment}/${secondSegment}`,\n });\n storage = new FileStorage(adapter);\n });\n\n test('reading a file that was written', async () => {\n await storage.write('path.txt', 'content in azure');\n const content = await storage.readToString('path.txt');\n\n expect(content).toEqual('content in azure');\n expect(await storage.fileExists('path.txt')).toEqual(true);\n });\n\n test('deleting a directory', async () => {\n await storage.createDirectory('directory/a');\n await storage.write('directory/b.txt', 'contents');\n\n await storage.deleteDirectory('directory');\n\n expect(await storage.fileExists('directory/b.txt')).toEqual(false);\n });\n\n test('statting a file', async () => {\n await storage.write('directory/b.txt', 'contents');\n\n const stat = await storage.statFile('directory/b.txt');\n\n expect(stat.path).toEqual('directory/b.txt');\n expect(typeof stat.lastModifiedMs).toEqual('number');\n });\n\n test('trying to read a file that does not exist', async () => {\n let was404 = false;\n\n try {\n await storage.readToString('404.txt');\n } catch (err) {\n if (err instanceof UnableToReadFile) {\n was404 = err.wasFileNotFound;\n } else {\n throw err;\n }\n }\n\n expect(was404).toEqual(true);\n });\n\n test('checking if a directory exists by prefix', async () => {\n await storage.write('directory/file.txt', 'contents');\n\n expect(await storage.directoryExists('directory')).toEqual(true);\n });\n\n test('getting a crc32c checksum', async () => {\n await storage.write('directory/file.txt', 'contents');\n\n expect(await storage.checksum('directory/file.txt', {\n algo: 'crc32c',\n })).toEqual('E2fhmA==');\n });\n\n test('getting an md5 checksum', async () => {\n await storage.write('directory/file.txt', 'contents');\n\n expect(await storage.checksum('directory/file.txt', {\n algo: 'md5',\n })).toEqual('mL99jBV4Two9YyBEQeHiqg==');\n });\n\n test('getting an sha1 checksum', async () => {\n await storage.write('directory/file.txt', 'contents');\n\n expect(await storage.checksum('directory/file.txt', {\n algo: 'sha1',\n })).toEqual('4a756ca07e9487f482465a99e8286abc86ba4dc7');\n });\n\n test('checking if a created directory', async () => {\n await storage.createDirectory('directory/here');\n\n expect(await storage.directoryExists('directory/here')).toEqual(true);\n expect(await storage.directoryExists('directory')).toEqual(true);\n });\n\n test('non deep and deep listing should have consistent and similar results', async () => {\n await storage.write('file_1.txt', 'contents');\n await storage.write('file_2.txt', 'contents');\n await storage.createDirectory('directory_1');\n await storage.createDirectory('directory_2');\n\n const non_deep_listing = await storage.list('/', {deep: false}).toArray();\n const deep_listing = await storage.list('/', {deep: true}).toArray();\n\n expect(non_deep_listing).toHaveLength(4);\n expect(deep_listing).toHaveLength(4);\n expect(non_deep_listing).toEqual(deep_listing);\n });\n });\n\n describe('visibility for legacy buckets', () => {\n beforeEach(() => {\n const secondSegment = randomBytes(10).toString('hex');\n\n adapter = new GoogleCloudStorageAdapter(legacyBucket, {\n prefix: `flystorage/${testSegment}/${secondSegment}`,\n }, new LegacyVisibilityHandling());\n storage = new FileStorage(adapter);\n });\n\n test('getting the visibility of a public file', async () => {\n await storage.write('path.txt', 'contents', {\n visibility: Visibility.PUBLIC,\n });\n\n expect(await storage.visibility('path.txt')).toEqual('public');\n });\n\n test('getting the visibility of a private file', async () => {\n await storage.write('path.txt', 'contents', {\n visibility: Visibility.PRIVATE,\n });\n\n expect(await storage.visibility('path.txt')).toEqual('private');\n });\n\n test('changing the visibility of a private file', async () => {\n await storage.write('path.txt', 'contents', {\n visibility: Visibility.PRIVATE,\n });\n\n await storage.changeVisibility('path.txt', Visibility.PUBLIC)\n\n expect(await storage.visibility('path.txt')).toEqual('public');\n });\n\n test('using a public URL', async () => {\n await storage.write('path.txt', 'public contents', {\n visibility: Visibility.PUBLIC,\n });\n\n const url = await storage.publicUrl('path.txt');\n const contents = await naivelyDownloadFile(url);\n\n expect(contents).toEqual('public contents');\n });\n\n test('with a custom Cache-Control Header', async () => {\n await storage.write('cache.txt', 'contents', {visibility: Visibility.PUBLIC, cacheControl: \"max-age=9999, public\"});\n const url = await storage.publicUrl('cache.txt');\n const res = await fetch(url);\n expect (res.headers.get('Cache-Control')).toEqual('max-age=9999, public');\n })\n\n test('using a temporary URL', async () => {\n await storage.write('path.txt', 'private contents', {\n visibility: Visibility.PRIVATE,\n });\n\n const url = await storage.temporaryUrl('path.txt', {\n expiresAt: Date.now() + 15 * 1000,\n });\n const contents = await naivelyDownloadFile(url);\n\n expect(contents).toEqual('private contents');\n });\n\n test('uploading using a prepared request', async () => {\n const request = await storage.prepareUpload('prepared/request-file.txt', {\n expiresAt: Date.now() + 60 * 1000,\n headers: {\n 'Content-Type': 'text/plain',\n }\n });\n\n await naivelyMakeRequestFile(\n request.url,\n request.headers,\n request.method,\n 'this is the contents',\n );\n\n const contents = await storage.readToString('prepared/request-file.txt');\n\n expect(contents).toEqual('this is the contents');\n });\n });\n});\n\nfunction naivelyDownloadFile(url: string): Promise {\n return new Promise((resolve, reject) => {\n https.get(url, async res => {\n if (res.statusCode !== 200) {\n reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));\n } else {\n resolve(await readableToString(res));\n }\n });\n });\n}\n\nfunction naivelyMakeRequestFile(url: string, headers: UploadRequestHeaders, method: string, data: string): Promise {\n return new Promise((resolve, reject) => {\n const req = https.request(url, {\n method: method,\n headers: {\n ...headers,\n 'Content-Length': new Blob([data]).size,\n }\n }, async res => {\n let responseBody = '';\n res.on('data', (chunk) => {\n responseBody += chunk;\n });\n res.on('end', () => {\n const statusCode = res.statusCode ?? 500;\n\n if (statusCode <= 200 && statusCode >= 299) {\n reject(new Error(`Not able to download the file from ${url}, response status [${res.statusCode}]`));\n } else {\n resolve();\n }\n });\n });\n\n req.on('error', (err) => {\n reject(err);\n });\n req.write(data);\n req.end();\n });\n}" }, { "path": "packages/google-cloud-storage/src/google-cloud-storage.ts", "content": "import {\n ChecksumIsNotAvailable,\n ChecksumOptions,\n CopyFileOptions,\n CreateDirectoryOptions,\n FileContents,\n FileWasNotFound,\n MimeTypeOptions,\n MoveFileOptions,\n PathPrefixer,\n PublicUrlOptions,\n StatEntry,\n StorageAdapter,\n TemporaryUrlOptions,\n UploadRequest,\n UploadRequestHeaders,\n UploadRequestOptions,\n WriteOptions,\n} from '@flystorage/file-storage';\nimport {Readable} from 'stream';\nimport {Bucket, GetFilesOptions, File, GetSignedUrlConfig, ApiError} from '@google-cloud/storage';\nimport {resolveMimeType} from '@flystorage/stream-mime-type';\nimport {pipeline} from 'node:stream/promises';\nimport {\n UniformBucketLevelAccessVisibilityHandling,\n VisibilityHandlingForGoogleCloudStorage,\n} from './visibility-handling.js';\nimport {PassThrough} from 'node:stream';\n\nexport type GoogleCloudStorageAdapterOptions = {\n prefix?: string,\n}\n\nexport class GoogleCloudStorageAdapter implements StorageAdapter {\n private readonly prefixer: PathPrefixer;\n constructor(\n private readonly bucket: Bucket,\n readonly options: GoogleCloudStorageAdapterOptions = {},\n readonly visibilityHandling: VisibilityHandlingForGoogleCloudStorage = new UniformBucketLevelAccessVisibilityHandling()\n ) {\n this.prefixer = new PathPrefixer(options.prefix ?? '');\n }\n\n async write(path: string, contents: Readable, options: WriteOptions): Promise {\n let mimeType = options.mimeType;\n\n if (mimeType === undefined) {\n [mimeType, contents] = await resolveMimeType(path, contents);\n }\n\n const writeStream = this.bucket.file(this.prefixer.prefixFilePath(path))\n .createWriteStream({\n contentType: mimeType,\n predefinedAcl: options.visibility\n ? this.visibilityHandling.visibilityToPredefinedAcl(options.visibility)\n : undefined,\n metadata: options.cacheControl? {cacheControl: options.cacheControl} : undefined,\n });\n\n await pipeline(contents, writeStream);\n }\n\n async read(path: string): Promise {\n const readStream = this.bucket.file(this.prefixer.prefixFilePath(path)).createReadStream();\n\n readStream.on('error', err => {\n readStream.unpipe(errorHandler);\n if (err instanceof ApiError && err.code === 404) {\n errorHandler.destroy(FileWasNotFound.atLocation(path, {\n context: {path},\n cause: err,\n }))\n } else {\n errorHandler.destroy(err);\n }\n });\n\n const errorHandler = new PassThrough();\n readStream.pipe(errorHandler);\n\n return errorHandler;\n }\n\n async deleteFile(path: string): Promise {\n await this.bucket.file(this.prefixer.prefixFilePath(path)).delete({\n ignoreNotFound: true,\n });\n }\n\n async createDirectory(path: string, options: CreateDirectoryOptions): Promise {\n await this.bucket.file(this.prefixer.prefixDirectoryPath(path)).save('');\n }\n\n async copyFile(from: string, to: string, options: CopyFileOptions): Promise {\n await this.bucket.file(this.prefixer.prefixFilePath(from)).copy(\n this.bucket.file(this.prefixer.prefixFilePath(to)),\n );\n }\n\n async moveFile(from: string, to: string, options: MoveFileOptions): Promise {\n await this.copyFile(from, to, {});\n await this.deleteFile(from);\n }\n\n async stat(path: string): Promise {\n const [metadata] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getMetadata();\n\n return this.mapToStatEntry(metadata);\n }\n\n async* list(path: string, options: { deep: boolean; }): AsyncGenerator {\n let response: File[];\n let query: GetFilesOptions | null = {\n autoPaginate: false,\n delimiter: options.deep ? undefined : '/',\n includeTrailingDelimiter: options.deep ? undefined : true,\n prefix: this.prefixer.prefixDirectoryPath(path),\n }\n\n while (query !== null) {\n [response, query] = await this.bucket.getFiles(query);\n\n for (const item of response) {\n yield this.mapToStatEntry(item.metadata);\n }\n }\n }\n\n private mapToStatEntry(file: File['metadata']): StatEntry {\n if (file.name!.endsWith('/')) {\n return {\n type: 'directory',\n isFile: false,\n isDirectory: true,\n path: this.prefixer.stripDirectoryPath(file.name!),\n };\n }\n\n return {\n type: 'file',\n isFile: true,\n isDirectory: false,\n path: this.prefixer.stripFilePath(file.name!),\n lastModifiedMs: file.updated ? new Date(file.updated).getTime() : undefined,\n mimeType: file.contentType,\n };\n }\n\n async changeVisibility(path: string, visibility: string): Promise {\n await this.visibilityHandling.changeVisibility(\n this.bucket.file(this.prefixer.prefixFilePath(path)),\n visibility,\n );\n }\n\n async visibility(path: string): Promise {\n return await this.visibilityHandling.determineVisibility(\n this.bucket.file(this.prefixer.prefixFilePath(path)),\n );\n }\n\n async deleteDirectory(path: string): Promise {\n const prefix = this.prefixer.prefixDirectoryPath(path);\n await this.bucket.deleteFiles({\n prefix,\n });\n await this.bucket.file(prefix).delete({\n ignoreNotFound: true,\n });\n }\n\n async fileExists(path: string): Promise {\n const [exists] = await this.bucket.file(this.prefixer.prefixFilePath(path)).exists();\n\n return exists;\n }\n\n async directoryExists(path: string): Promise {\n const [exists] = await this.bucket.file(this.prefixer.prefixDirectoryPath(path)).exists();\n\n if (exists) {\n return true;\n }\n\n const [response] = await this.bucket.getFiles({\n autoPaginate: false,\n maxResults: 1,\n prefix: this.prefixer.prefixDirectoryPath(path),\n });\n\n return response.length > 0;\n }\n\n async publicUrl(path: string, options: PublicUrlOptions): Promise {\n return this.bucket.file(this.prefixer.prefixFilePath(path)).publicUrl();\n }\n\n async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n const [response] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getSignedUrl({\n action: 'read',\n expires: options.expiresAt,\n });\n\n return response;\n }\n\n async prepareUpload(path: string, options: UploadRequestOptions): Promise {\n const headers: UploadRequestHeaders = {};\n const config: GetSignedUrlConfig = {\n action: 'write',\n expires: options.expiresAt,\n };\n\n const contentType = options['Content-Type'] ?? options.contentType;\n\n if (typeof contentType === 'string') {\n config.contentType = contentType;\n headers['Content-Type'] = contentType;\n }\n\n const [url] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getSignedUrl(config);\n\n return {\n url,\n headers,\n method: 'PUT',\n provider: 'google-cloud-storage',\n };\n }\n\n async checksum(path: string, options: ChecksumOptions): Promise {\n const algo = options.algo ?? 'md5';\n\n if (algo !== 'md5' && algo !== 'crc32c') {\n throw ChecksumIsNotAvailable.checksumNotSupported(algo);\n }\n\n const [metadata] = await this.bucket.file(this.prefixer.prefixFilePath(path)).getMetadata();\n\n return algo === 'crc32c' ? metadata.crc32c! : metadata.md5Hash!;\n }\n\n async mimeType(path: string, options: MimeTypeOptions): Promise {\n const stat = await this.stat(path);\n\n if (stat.type !== 'file' || stat.mimeType === undefined) {\n throw new Error('Unable to resolve mime-type, not available in stat entry.');\n }\n\n return stat.mimeType;\n }\n\n async lastModified(path: string): Promise {\n const stat = await this.stat(path);\n\n if (stat.type !== 'file' || stat.lastModifiedMs === undefined) {\n throw new Error('Unable to resolve last modified time, not available in stat entry.');\n }\n\n return stat.lastModifiedMs;\n }\n\n async fileSize(path: string): Promise {\n const stat = await this.stat(path);\n\n if (stat.type !== 'file' || stat.size === undefined) {\n throw new Error('Unable to resolve file size, not available in stat entry.');\n }\n\n return stat.size;\n }\n\n}\n\n/**\n * BC export\n *\n * @deprecated\n */\nexport class GoogleCloudFileStorage extends GoogleCloudStorageAdapter {}\n" }, { "path": "packages/google-cloud-storage/src/index.ts", "content": "export * from './google-cloud-storage.js';\nexport * from './visibility-handling.js';" }, { "path": "packages/google-cloud-storage/src/visibility-handling.ts", "content": "import {Visibility} from '@flystorage/file-storage';\nimport {ApiError, File, PredefinedAcl} from '@google-cloud/storage';\n\nexport interface VisibilityHandlingForGoogleCloudStorage {\n changeVisibility(file: File, visibility: string): Promise;\n\n determineVisibility(file: File): Promise;\n\n visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefined,\n}\n\nexport class UniformBucketLevelAccessVisibilityHandling implements VisibilityHandlingForGoogleCloudStorage {\n constructor(\n private readonly errorOnChange: boolean = false,\n private readonly errorOnDetermine: boolean = false,\n private readonly fakeVisibility: string = 'unknown',\n ) {\n\n }\n\n async changeVisibility(file: File, visibility: string): Promise {\n if (this.errorOnChange) {\n throw new Error('Unable to set visibility when using uniform bucket level access control.');\n }\n // ignored, no-op\n }\n async determineVisibility(file: File): Promise {\n if (this.errorOnDetermine) {\n throw new Error('Unable to determine visibility when using uniform bucket level access control.');\n }\n return 'unknown';\n }\n visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefined {\n if (this.errorOnChange) {\n throw new Error('Unable to set visibility when using uniform bucket level access control.');\n }\n\n return undefined;\n }\n\n}\n\nexport class LegacyVisibilityHandling implements VisibilityHandlingForGoogleCloudStorage {\n constructor(\n private readonly entity: string = 'allUsers',\n private readonly publicAcl: PredefinedAcl = 'publicRead',\n private readonly privateAcl: PredefinedAcl = 'projectPrivate',\n ) {\n }\n\n async changeVisibility(file: File, visibility: string): Promise {\n if (visibility === Visibility.PRIVATE) {\n await file.acl.delete({\n entity: this.entity,\n })\n } else if (visibility === Visibility.PUBLIC) {\n await file.acl.update({\n entity: this.entity,\n role: 'READER',\n })\n }\n }\n\n async determineVisibility(file: File): Promise {\n try {\n const [, metadata] = await file.acl.get({entity: 'allUsers'});\n\n return metadata.role === 'READER' ? Visibility.PUBLIC : Visibility.PRIVATE;\n } catch (error) {\n if (!(error instanceof ApiError) || error.response?.statusCode !== 404) {\n throw error;\n }\n\n return Visibility.PRIVATE;\n }\n }\n\n visibilityToPredefinedAcl(visibility: string): PredefinedAcl | undefined {\n if (visibility === Visibility.PUBLIC) {\n return this.publicAcl;\n } else if (visibility === Visibility.PRIVATE) {\n return this.privateAcl;\n }\n\n throw new Error(`Not able to set visibility ${visibility}, no mapping known.`);\n }\n}" }, { "path": "packages/google-cloud-storage/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packages/in-memory/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/in-memory/CHANGELOG.md", "content": "# `@flystorage/in-memory`\n\n## 1.1.0\n\n### Changes\n\n- Use a Buffer for internal storage\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/in-memory/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/in-memory/README.md", "content": "\n\n# Flystorage adapter using memory\n\nThis package contains the Flystorage adapter that uses only memory\n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/file-storage @flystorage/in-memory\n```\n\n## Usage\n\n```typescript\nimport {FileStorage} from '@flystorage/file-storage';\nimport {InMemoryStorageAdapter} from '@flystorage/in-memory';\n\nconst adapter = new InMemoryStorageAdapter();\nconst storage = new FileStorage(adapter);\n```\n\n> ⚠️ Always use the FileStorage, it is essential for security and a good developer\n> experience. Do not use the adapter directly.\n\n" }, { "path": "packages/in-memory/package.json", "content": "{\n \"name\": \"@flystorage/in-memory\",\n \"type\": \"module\",\n \"version\": \"1.1.0\",\n \"dependencies\": {\n \"@flystorage/file-storage\": \"^1.1.0\",\n \"file-type\": \"^21.3.1\",\n \"mime-types\": \"^3.0.2\"\n },\n \"description\": \"\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"keywords\": [\n \"test double\",\n \"memory\",\n \"file\",\n \"storage\",\n \"flystorage\",\n \"filesystem\"\n ],\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/adapter/in-memory/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/in-memory\"\n },\n \"license\": \"MIT\"\n}" }, { "path": "packages/in-memory/src/in-memory-file-storage.test.ts", "content": "import {FileStorage, Visibility} from \"@flystorage/file-storage\";\nimport {InMemoryStorageAdapter} from \"./in-memory-file-storage.js\";\n\ndescribe('InMemoryStorageAdapter', () => {\n const adapter = new InMemoryStorageAdapter();\n const storage = new FileStorage(adapter);\n\n beforeEach(() => {\n adapter.deleteEverything();\n });\n\n test('non deep and deep listing should have consistent and similar results', async () => {\n await storage.write('file_1.txt', 'contents');\n await storage.write('file_2.txt', 'contents');\n await storage.createDirectory('directory_1');\n await storage.createDirectory('directory_2');\n\n const non_deep_listing = await storage.list('/', {deep: false}).toArray();\n const deep_listing = await storage.list('/', {deep: true}).toArray();\n\n expect(non_deep_listing).toHaveLength(4);\n expect(deep_listing).toHaveLength(4);\n expect(non_deep_listing).toEqual(deep_listing);\n });\n\n test('reading a file that was written', async () => {\n await storage.write('path.txt', 'content in azure');\n const content = await storage.readToString('path.txt');\n\n expect(content).toEqual('content in azure');\n });\n\n test('trying to read a file that does not exist', async () => {\n await expect(storage.readToString('404.tx')).rejects.toThrow();\n });\n\n test('trying to see if a non-existing file exists', async () => {\n expect(await storage.fileExists('404.txt')).toEqual(false);\n });\n\n test('trying to see if an existing file exists', async () => {\n await storage.write('existing.txt', 'contents');\n\n expect(await storage.fileExists('existing.txt')).toEqual(true);\n });\n\n test('deleting an existing file', async () => {\n await storage.write('existing.txt', 'contents');\n\n expect(await storage.fileExists('existing.txt')).toEqual(true);\n\n await storage.deleteFile('existing.txt');\n\n expect(await storage.fileExists('existing.txt')).toEqual(false);\n });\n\n test('deleting a non-existing file is OK', async () => {\n await expect(storage.deleteFile('404.txt'));\n });\n\n test('copying a file', async () => {\n await storage.write('file.txt', 'copied');\n\n await storage.copyFile('file.txt', 'new-file.txt');\n\n expect(await storage.fileExists('file.txt')).toEqual(true);\n expect(await storage.fileExists('new-file.txt')).toEqual(true);\n expect(await storage.readToString('new-file.txt')).toEqual('copied');\n });\n\n test('moving a file', async () => {\n await storage.write('file.txt', 'moved');\n\n await storage.moveFile('file.txt', 'new-file.txt');\n\n expect(await storage.fileExists('file.txt')).toEqual(false);\n expect(await storage.fileExists('new-file.txt')).toEqual(true);\n expect(await storage.readToString('new-file.txt')).toEqual('moved');\n });\n\n test('setting visibility always fails', async () => {\n await storage.write('exsiting.txt', 'yes');\n await expect(storage.changeVisibility('existing.txt', Visibility.PUBLIC)).rejects.toThrow();\n await expect(storage.changeVisibility('404.txt', Visibility.PRIVATE)).rejects.toThrow();\n });\n\n test('listing entries in a directory, shallow', async () => {\n await storage.write('outside/path.txt', 'test');\n await storage.write('inside/a.txt', 'test');\n await storage.write('inside/b.txt', 'test');\n await storage.write('inside/c/a.txt', 'test');\n\n const listing = await storage.list('inside').toArray();\n expect(listing).toHaveLength(3);\n expect(listing[0].type).toEqual('file');\n expect(listing[1].type).toEqual('file');\n expect(listing[2].type).toEqual('directory');\n expect(listing[0].path).toEqual('inside/a.txt');\n expect(listing[1].path).toEqual('inside/b.txt');\n expect(listing[2].path).toEqual('inside/c');\n });\n\n test('listing entries in a directory, deep', async () => {\n await storage.write('outside/path.txt', 'test');\n await storage.write('inside/a.txt', 'test');\n await storage.write('inside/b.txt', 'test');\n await storage.write('inside/c/a.txt', 'test');\n\n const listing = await storage.list('inside', {deep: true}).toArray();\n expect(listing).toHaveLength(4);\n expect(listing[0].type).toEqual('file');\n expect(listing[1].type).toEqual('file');\n expect(listing[2].type).toEqual('directory');\n expect(listing[3].type).toEqual('file');\n expect(listing[0].path).toEqual('inside/a.txt');\n expect(listing[1].path).toEqual('inside/b.txt');\n expect(listing[2].path).toEqual('inside/c');\n expect(listing[3].path).toEqual('inside/c/a.txt');\n });\n\n test('deleting a full directory', async () => {\n await storage.write('directory/a.txt', 'test');\n await storage.write('directory/b.txt', 'test');\n await storage.write('directory/c/a.txt', 'test');\n\n await storage.deleteDirectory('directory');\n\n const listing = await storage.list('directory', {deep: true}).toArray();\n\n expect(listing).toEqual([]);\n });\n\n test('checking if a directory exists', async () => {\n await storage.write('directory/a.txt', 'test');\n await storage.write('directory/b.txt', 'test');\n await storage.write('directory/c/a.txt', 'test');\n\n expect(await storage.directoryExists('directory')).toEqual(true);\n expect(await storage.directoryExists('directory/c')).toEqual(true);\n expect(await storage.directoryExists('directory/a')).toEqual(false);\n });\n\n test('getting a md5 checksum of a file', async () => {\n await storage.write('this.txt', 'test');\n const checksum = await storage.checksum('this.txt', {algo: 'md5'});\n\n expect(checksum).toEqual('098f6bcd4621d373cade4e832627b4f6');\n });\n});" }, { "path": "packages/in-memory/src/in-memory-file-storage.ts", "content": "import {\n ChecksumIsNotAvailable,\n ChecksumOptions,\n CopyFileOptions,\n CreateDirectoryOptions,\n FileContents,\n MimeTypeOptions,\n MoveFileOptions,\n PublicUrlOptions,\n StatEntry,\n StorageAdapter,\n TemporaryUrlOptions,\n VisibilityOptions,\n WriteOptions,\n readableToBuffer,\n readableToString,\n} from '@flystorage/file-storage';\nimport {Readable} from \"node:stream\";\nimport {dirname, parse} from 'node:path'\nimport {lookup as mimeTimeForExt} from \"mime-types\";\n\ntype FileEntry = {\n type: 'file',\n path: string,\n contents: Buffer,\n lastModifiedMs: number,\n visibility?: string,\n}\n\ntype DirectoryEntry = {\n type: 'directory',\n path: string,\n visibility?: string,\n}\n\nexport type TimestampResolver = () => number;\n\nfunction cloneBuffer(input: Buffer): Buffer {\n const output = Buffer.alloc(input.length);\n input.copy(output);\n\n return output;\n}\n\nexport class InMemoryStorageAdapter implements StorageAdapter {\n private entries: Map = new Map;\n\n constructor(\n private readonly timestampResolver: TimestampResolver = () => Date.now(),\n ) {\n }\n\n deleteEverything(): void {\n this.entries = new Map;\n }\n\n async write(path: string, contents: Readable, options: WriteOptions): Promise {\n this.ensureParentDirsExist(path, options);\n\n this.entries.set(path, {\n type: 'file',\n path,\n contents: await readableToBuffer(contents),\n lastModifiedMs: this.timestampResolver(),\n visibility: options.visibility,\n })\n }\n\n private ensureParentDirsExist(path: string, options: VisibilityOptions) {\n let parentDir = dirname(path);\n\n while (!['.', ''].includes(parentDir) && !this.entries.has(parentDir)) {\n this.entries.set(parentDir, {\n path: parentDir,\n type: 'directory',\n visibility: options.directoryVisibility,\n });\n\n parentDir = dirname(parentDir);\n }\n }\n\n async read(path: string): Promise {\n const file = this.entries.get(path);\n\n if (file?.type !== 'file') {\n throw new Error(`Path \"${path}\" is not a file`);\n }\n\n return Readable.from(cloneBuffer(file.contents));\n }\n\n async deleteFile(path: string): Promise {\n this.entries.delete(path);\n }\n\n async createDirectory(path: string, options: CreateDirectoryOptions): Promise {\n path = path.replace(/\\/+$/g, '');\n\n this.entries.set(path, {\n path,\n type: 'directory',\n visibility: options.directoryVisibility,\n });\n }\n async copyFile(from: string, to: string, options: CopyFileOptions): Promise {\n const source = this.entries.get(from);\n\n if (source?.type !== 'file') {\n throw new Error(`Source file ${from} does not exist`);\n }\n\n this.ensureParentDirsExist(to, options);\n this.entries.set(to, {\n ...source,\n path: to,\n });\n }\n async moveFile(from: string, to: string, options: MoveFileOptions): Promise {\n await this.copyFile(from, to, options);\n await this.deleteFile(from);\n }\n async stat(path: string): Promise {\n const entry = this.entries.get(path);\n\n if (entry === undefined) {\n throw new Error(`No entry found for path \"${path}\"`);\n }\n\n return this.mapToStatEntry(entry);\n }\n private mapToStatEntry(entry: DirectoryEntry | FileEntry): StatEntry {\n if (entry.type === 'directory') {\n return {\n ...entry,\n isDirectory: true,\n isFile: false,\n };\n }\n\n return {\n type: 'file',\n path: entry.path,\n visibility: entry.visibility,\n isFile: true,\n isDirectory: false,\n lastModifiedMs: entry.lastModifiedMs,\n }\n }\n async *list(path: string, options: { deep: boolean; }): AsyncGenerator {\n const entries = this.entries.values();\n const prefix = path === '' || path === '/'\n ? ''\n : `${path.replace(/\\/+$/g, '')}/`;\n\n for (const entry of entries) {\n if (!entry.path.startsWith(prefix)) {\n continue;\n }\n\n if (options.deep !== true && entry.path.indexOf('/', prefix.length) !== -1) {\n continue;\n }\n\n yield this.mapToStatEntry(entry);\n }\n }\n async changeVisibility(path: string, visibility: string): Promise {\n const entry = this.entries.get(path);\n\n if (entry?.type !== 'file') {\n throw new Error(`Path ${path} is not a file`);\n }\n\n this.entries.set(path, {\n ...entry,\n visibility,\n });\n }\n async visibility(path: string): Promise {\n const entry = this.entries.get(path);\n\n if (entry === undefined) {\n throw new Error(`Path ${path} does not exist`);\n }\n\n return entry.visibility ?? 'public';\n }\n async deleteDirectory(path: string): Promise {\n const entries = this.entries.values();\n const prefix = `${path.replace(/\\/+$/g, '')}/`;\n\n for (const entry of entries) {\n if (entry.path.startsWith(prefix) || entry.path === path) {\n this.entries.delete(entry.path);\n }\n }\n }\n async fileExists(path: string): Promise {\n return this.entries.get(path)?.type === 'file';\n }\n async directoryExists(path: string): Promise {\n return this.entries.get(path)?.type === 'directory';\n }\n async publicUrl(path: string, options: PublicUrlOptions): Promise {\n throw new Error(\"Method not implemented.\");\n }\n async temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n throw new Error(\"Method not implemented.\");\n }\n async checksum(path: string, options: ChecksumOptions): Promise {\n if (this.entries.get(path)?.type !== 'file') {\n // throw new Error(`File ${path} does not exists`);\n }\n\n throw ChecksumIsNotAvailable.checksumNotSupported(options.algo ?? 'unknown', {\n context: {path},\n });\n }\n async mimeType(path: string, options: MimeTypeOptions): Promise {\n const entry = this.entries.get(path);\n\n if (entry?.type !== 'file') {\n throw new Error(`File ${path} does not exist`);\n }\n\n return await resolveMimeType(path, Buffer.from(entry.contents));\n }\n async lastModified(path: string): Promise {\n const entry = this.entries.get(path);\n\n if (entry?.type !== 'file') {\n throw new Error(`File ${path} does not exist`);\n }\n\n return entry.lastModifiedMs;\n }\n\n async fileSize(path: string): Promise {\n const entry = this.entries.get(path);\n\n if (entry?.type !== 'file') {\n throw new Error(`File ${path} does not exist`);\n }\n\n return Buffer.byteLength(entry.contents);\n }\n}\n\nexport async function resolveMimeType(\n filename: string,\n contents: Uint8Array,\n): Promise {\n const {fileTypeFromBuffer} = await import('file-type');\n const lookup = await fileTypeFromBuffer(contents);\n\n if (lookup) {\n return lookup.mime;\n }\n\n const {ext} = parse(filename);\n\n return mimeTimeForExt(ext) || 'application/octet-stream';\n}\n\n/**\n * BC export\n *\n * @deprecated\n */\nexport class InMemoryFileStorage extends InMemoryStorageAdapter {}" }, { "path": "packages/in-memory/src/index.ts", "content": "export * from './in-memory-file-storage.js';" }, { "path": "packages/in-memory/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packages/local-fs/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/local-fs/CHANGELOG.md", "content": "# `@flystorage/local-fs`\n\n## 1.1.0\n\n## Added\n\n- AbortSignal Support\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/local-fs/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/local-fs/README.md", "content": "\n\n# Flystorage adapter for the local filesystem\n\nThis package contains the Flystorage adapter for the local filesystem.\n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/file-storage @flystorage/local-fs\n```\n\n## Usage\n\n```typescript\nimport {FileStorage} from '@flystorage/file-storage';\nimport {LocalStorageAdapter} from '@flystorage/local-fs';\n\nconst rootDirectory = resolve(process.cwd(), 'my-files');\nconst adapter = new LocalStorageAdapter(rootDirectory);\nconst storage = new FileStorage(adapter);\n```\n\n> ⚠️ Always use the FileStorage, it is essential for security and a good developer\n> experience. Do not use the adapter directly.\n\n" }, { "path": "packages/local-fs/package.json", "content": "{\n \"name\": \"@flystorage/local-fs\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@flystorage/dynamic-import\": \"^1.0.0\",\n \"@flystorage/file-storage\": \"^1.1.0\",\n \"file-type\": \"^21.3.1\",\n \"mime-types\": \"^3.0.2\"\n },\n \"description\": \"\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"patch\": \"gsed -i 's/dynamicallyImport(/import(/g' dist/esm/local-file-storage.js\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"keywords\": [\n \"fs\",\n \"file\",\n \"storage\",\n \"flystorage\",\n \"filesystem\"\n ],\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/adapter/local-fs/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/local-fs\"\n },\n \"engines\": {\n \"node\": \">=20.1.0\"\n },\n \"license\": \"MIT\"\n}\n" }, { "path": "packages/local-fs/src/index.ts", "content": "export * from './local-file-storage.js';\nexport * from './unix-visibility.js';" }, { "path": "packages/local-fs/src/local-file-storage.test.ts", "content": "import {\n FileInfo,\n FileStorage,\n normalizeExpiryToMilliseconds, UnableToReadFile,\n UploadRequest,\n Visibility,\n} from '@flystorage/file-storage';\nimport {BinaryToTextEncoding, createHash, randomBytes} from 'crypto';\nimport * as path from 'node:path';\nimport {LocalStorageAdapter, LocalTemporaryUrlGenerator, LocalTemporaryUrlOptions} from './local-file-storage.js';\nimport {execSync} from 'child_process';\nimport {createReadStream} from 'node:fs';\nimport {closeReadable} from \"@flystorage/file-storage\";\n\nconst rootDirectory = path.resolve(process.cwd(), 'fixtures/test-files');\nlet storage: FileStorage;\n\ndescribe('LocalStorageAdapter', () => {\n beforeEach(() => {\n const testSegment = randomBytes(10).toString('hex');\n storage = new FileStorage(\n new LocalStorageAdapter(\n path.join(rootDirectory, testSegment),\n {\n publicUrlOptions: {\n baseUrl: 'https://default.com/',\n }\n }\n ),\n );\n })\n beforeAll(() => {\n execSync(`rm -rf ${rootDirectory}/*`);\n });\n\n test('preparing uploads is not supported by default', async () => {\n await expect(storage.prepareUpload('somewhere', {\n expiresAt: Date.now() + 10000,\n })).rejects.toThrowError();\n });\n\n test('you can use your own upload preparer to support preparing uploads', async () => {\n const storage = new FileStorage(\n new LocalStorageAdapter(\n rootDirectory,\n {\n publicUrlOptions: {\n baseUrl: 'https://default.com/',\n },\n },\n undefined,\n undefined,\n undefined,\n {\n async prepareUpload(path) {\n return {\n method: 'PUT',\n url: `https://fixates.url.com/${path}`,\n headers: {},\n }\n },\n },\n ),\n );\n\n const request = await storage.prepareUpload('somewhere/here.txt', {\n expiresAt: 0,\n });\n\n expect(request).toEqual({\n method: 'PUT',\n url: 'https://fixates.url.com/somewhere/here.txt',\n headers: {},\n } satisfies UploadRequest);\n });\n\n test('deleting a file', async () => {\n await storage.write('text.txt', 'contents');\n\n expect(await storage.fileExists('text.txt')).toEqual(true);\n\n await storage.deleteFile('text.txt');\n\n expect(await storage.fileExists('text.txt')).toEqual(false);\n });\n\n test('deleting a directory', async () => {\n await storage.write('directory/text.txt', 'contents');\n\n await storage.deleteDirectory('directory');\n\n expect(await storage.fileExists('directory/text.txt')).toEqual(false);\n expect(await storage.directoryExists('directory')).toEqual(false);\n });\n\n test('files written with private visibility have private visibility', async () => {\n await storage.write('test.txt', 'contents', {\n visibility: Visibility.PRIVATE,\n });\n\n const stat = await storage.stat('test.txt');\n\n expect(stat.visibility).toEqual(Visibility.PRIVATE);\n });\n\n test('writing a png and fetching its mime-type', async () => {\n const handle = createReadStream(path.resolve(process.cwd(), 'fixtures/screenshot.png'));\n await storage.write('image.png', handle);\n closeReadable(handle);\n\n const mimeType = await storage.mimeType('image.png');\n\n expect(mimeType).toEqual('image/png');\n });\n\n test('retrieving the size of a file', async () => {\n const contents = 'this is the contents of the file';\n await storage.write('something.txt', contents);\n\n expect(await storage.fileSize('something.txt')).toEqual(contents.length);\n });\n\n describe('stat for (public) files', () => {\n let fileInfo: FileInfo;\n beforeEach(async () => {\n await storage.write('test.txt', 'contents', {\n visibility: Visibility.PUBLIC,\n });\n\n fileInfo = await storage.statFile('test.txt');\n });\n\n test('it exposes file size', async () => {\n expect(fileInfo.size).toEqual(8);\n });\n\n test('it exposes a type', async () => {\n expect(fileInfo.type).toEqual('file');\n });\n\n test('it exposes (public) visibility', async () => {\n expect(fileInfo.visibility).toEqual(Visibility.PUBLIC);\n });\n });\n\n test('listing the contents of a directory, shallow', async () => {\n await storage.write('file-1.txt', 'contents');\n await storage.write('file-2.txt', 'contents');\n\n const listing = await storage.list('/').toArray();\n\n expect(listing).toHaveLength(2);\n expect(listing.map(l => l.type)).toEqual(['file', 'file']);\n expect(listing.map(l => l.path).sort()).toEqual(['file-1.txt', 'file-2.txt']);\n });\n\n test('file file visibility can be changes', async () => {\n await storage.write('file.txt', 'contents', {\n visibility: Visibility.PUBLIC,\n });\n\n expect((await storage.stat('file.txt')).visibility).toEqual(Visibility.PUBLIC);\n\n await storage.changeVisibility('file.txt', Visibility.PRIVATE);\n\n expect((await storage.statFile('file.txt')).visibility).toEqual(Visibility.PRIVATE);\n });\n\n test('checking if a non-existing file exists', async () => {\n expect(await storage.fileExists('non-existing-file.txt')).toEqual(false);\n });\n\n test('checking if an existing file exists', async () => {\n await storage.write('existing-file.txt', 'contents');\n\n expect(await storage.fileExists('existing-file.txt')).toEqual(true);\n });\n\n test('checking if a directory is an existing file', async () => {\n await storage.createDirectory('existing-file.txt');\n\n expect(await storage.fileExists('existing-file.txt')).toEqual(false);\n });\n\n test('checking if a non-existing directory exists', async () => {\n expect(await storage.directoryExists('does-not-exist')).toEqual(false);\n });\n\n test('checking if a file is a directory', async () => {\n await storage.write('location', 'nothing');\n\n expect(await storage.directoryExists('location')).toEqual(false);\n });\n\n test('checking if an existing directory exists', async () => {\n await storage.createDirectory('location');\n\n expect(await storage.directoryExists('location')).toEqual(true);\n });\n\n test('writing a file implicitly creates parent directories', async () => {\n await storage.write('deeply/nested/directory.txt', 'contents');\n\n const listing = await storage.list('/', {deep: true}).toArray();\n\n expect(listing).toHaveLength(3);\n expect(listing.map(l => l.type).sort()).toEqual(['directory', 'directory', 'file']);\n });\n\n test('non deep and deep listing should have consistent and similar results', async () => {\n await storage.write('file_1.txt', 'contents');\n await storage.write('file_2.txt', 'contents');\n await storage.createDirectory('directory_1');\n await storage.createDirectory('directory_2');\n\n const non_deep_listing = await storage.list('/', {deep: false}).toArray();\n const deep_listing = await storage.list('/', {deep: true}).toArray();\n\n expect(non_deep_listing).toHaveLength(4);\n expect(deep_listing).toHaveLength(4);\n expect(non_deep_listing).toEqual(deep_listing);\n });\n\n test('filtering out directories for a deep listing', async () => {\n await storage.write('deeply/nested/directory.txt', 'contents');\n\n const listing = await storage.list('/', {deep: true})\n .filter(entry => entry.isFile)\n .toArray();\n\n expect(listing).toHaveLength(1);\n expect(listing.map(l => l.type).sort()).toEqual(['file']);\n });\n\n test('generating a public urls works when the base URL is provided in the constructor', async () => {\n const url = await storage.publicUrl('some/path.txt');\n\n expect(url).toEqual('https://default.com/some/path.txt');\n });\n\n test('generating a public urls works when the base URL is provided as an option', async () => {\n const url = await storage.publicUrl('/some/path.txt', {\n baseUrl: 'https://example.org/with-prefix/',\n });\n\n expect(url).toEqual('https://example.org/with-prefix/some/path.txt');\n });\n\n test('moving a file', async () => {\n await storage.write('move-from.txt', 'this');\n\n expect(await storage.readToString('move-from.txt')).toEqual('this');\n\n await storage.moveFile('move-from.txt', 'move-to.txt');\n\n expect(await storage.fileExists('move-from.txt')).toEqual(false);\n expect(await storage.readToString('move-to.txt')).toEqual('this');\n });\n\n test('copying a file', async () => {\n await storage.write('from.txt', 'this');\n\n await storage.copyFile('from.txt', 'to.txt');\n\n expect(await storage.fileExists('from.txt')).toEqual(true);\n expect(await storage.readToString('to.txt')).toEqual('this');\n });\n\n test('trying to copy a file that does not exist', async () => {\n await expect(storage.copyFile('from.txt', 'to.txt')).rejects.toThrow();\n });\n\n test('trying to read a file that does not exist', async () => {\n let was404 = false;\n\n try {\n await storage.readToString('404.txt');\n } catch (err) {\n if (err instanceof UnableToReadFile) {\n was404 = err.wasFileNotFound;\n }\n }\n\n expect(was404).toEqual(true);\n });\n\n test('trying to move a file that does not exist', async () => {\n await expect(storage.moveFile('from.txt', 'to.txt')).rejects.toThrow();\n });\n\n test('moving a file between directories', async () => {\n await storage.write('dir-a/here.txt', 'this');\n\n await storage.moveFile('dir-a/here.txt', 'dir-b/there.txt');\n\n expect(await storage.fileExists('dir-a/here.txt')).toEqual(false);\n expect(await storage.readToString('dir-b/there.txt')).toEqual('this');\n });\n\n test('copying a file between directories', async () => {\n await storage.write('dir-a/here.txt', 'this');\n\n await storage.copyFile('dir-a/here.txt', 'dir-b/there.txt');\n\n expect(await storage.fileExists('dir-a/here.txt')).toEqual(true);\n expect(await storage.readToString('dir-b/there.txt')).toEqual('this');\n });\n\n test('generating a public urls failed when the base URL is undefined', async () => {\n const url = storage.publicUrl('/some/path.txt', {\n baseUrl: undefined\n });\n\n await expect(url).rejects.toThrow();\n });\n\n test('generating a temporary URL fails when no generator is configured', async () => {\n await expect(storage.temporaryUrl('path.txt', {\n expiresAt: new Date(),\n })).rejects.toThrow();\n });\n\n test('generating a temporary URL works when the generator is configured', async () => {\n const storage = new FileStorage(\n new LocalStorageAdapter(\n rootDirectory,\n {\n publicUrlOptions: {\n baseUrl: 'https://default.com/',\n },\n temporaryUrlOptions: {\n baseUrl: 'https://secret.com/'\n }\n },\n undefined,\n undefined,\n new FakeTemporaryUrlGenerator(),\n ),\n );\n\n const now = Date.now();\n\n await expect(storage.temporaryUrl('fake.txt', {\n expiresAt: now,\n })).resolves.toEqual(`https://secret.com/fake.txt?ts=${now}`);\n });\n\n test('it can calculate checksums', async () => {\n function hashString(input: string, algo: string, encoding: BinaryToTextEncoding = 'hex'): string {\n return createHash(algo).update(input).digest(encoding);\n }\n\n const contents = 'this is for the checksum';\n await storage.write('path.txt', contents);\n const expectedChecksum = hashString(contents, 'md5');\n\n const checksum = await storage.checksum('path.txt', {\n algo: 'md5',\n });\n\n expect(checksum).toEqual(expectedChecksum);\n });\n});\n\nclass FakeTemporaryUrlGenerator implements LocalTemporaryUrlGenerator {\n async temporaryUrl(path: string, options: LocalTemporaryUrlOptions): Promise {\n return `${options.baseUrl}${path}?ts=${normalizeExpiryToMilliseconds(options.expiresAt)}`;\n }\n}" }, { "path": "packages/local-fs/src/local-file-storage.ts", "content": "import {\n checksumFromStream,\n ChecksumOptions,\n CreateDirectoryOptions,\n PathPrefixer,\n PublicUrlOptions,\n StatEntry,\n StorageAdapter,\n TemporaryUrlOptions,\n WriteOptions,\n CopyFileOptions,\n MoveFileOptions,\n VisibilityOptions,\n MimeTypeOptions,\n UploadRequestOptions,\n UploadRequest,\n MiscellaneousOptions, FileWasNotFound,\n} from '@flystorage/file-storage';\nimport {lookup} from \"mime-types\";\nimport {createReadStream, createWriteStream, Dirent, Stats} from 'node:fs';\nimport {chmod, mkdir, opendir, rm, stat, rename, copyFile} from 'node:fs/promises';\nimport {join, extname, dirname, sep} from 'node:path';\nimport {Readable} from 'stream';\nimport {pipeline} from 'stream/promises';\nimport {PortableUnixVisibilityConversion, UnixVisibilityConversion} from './unix-visibility.js';\nimport {dynamicallyImport} from '@flystorage/dynamic-import';\nimport {PreparedUploadsAreNotSupported, PreparedUploadStrategy} from '@flystorage/file-storage';\nimport {PassThrough} from 'node:stream';\n\nexport type LocalStorageAdapterOptions = {\n rootDirectoryVisibility?: string,\n publicUrlOptions?: LocalPublicUrlOptions,\n temporaryUrlOptions?: Omit,\n};\n\nexport type LocalPublicUrlOptions = PublicUrlOptions & {\n baseUrl?: string,\n}\n\nexport type LocalPublicUrlGenerator = {\n publicUrl(path: string, options: LocalPublicUrlOptions): Promise;\n}\n\nexport class BaseUrlLocalPublicUrlGenerator implements LocalPublicUrlGenerator {\n async publicUrl(path: string, options: LocalPublicUrlOptions): Promise {\n if (options.baseUrl === undefined) {\n throw new Error('No base URL defined for public URL generation');\n }\n\n const base = options.baseUrl.endsWith('/') ? options.baseUrl : `${options.baseUrl}/`;\n\n if (sep === '\\\\' && path.includes(sep)) {\n path = path.replace(sep, '/');\n }\n\n return `${base}${path}`;\n }\n}\n\nexport type LocalTemporaryUrlOptions = TemporaryUrlOptions & {\n baseUrl?: string,\n}\n\nexport type LocalTemporaryUrlGenerator = {\n temporaryUrl(path: string, options: LocalTemporaryUrlOptions): Promise;\n}\n\nexport class FailingLocalTemporaryUrlGenerator implements LocalTemporaryUrlGenerator {\n async temporaryUrl(): Promise {\n throw new Error('No temporary URL generator provided');\n }\n}\n\ntype FileTypePackage = typeof import('file-type');\nlet fileTypeImport: Promise | undefined;\nlet fileTypes: FileTypePackage | undefined = undefined;\n\nfunction maybeAbort(signal?: AbortSignal) {\n if (signal?.aborted) {\n throw signal.reason;\n }\n}\n\nexport class LocalStorageAdapter implements StorageAdapter {\n private prefixer: PathPrefixer;\n\n constructor(\n readonly rootDir: string,\n private readonly options: LocalStorageAdapterOptions = {},\n private readonly visibilityConversion: UnixVisibilityConversion = new PortableUnixVisibilityConversion(),\n private readonly publicUrlGenerator: LocalPublicUrlGenerator = new BaseUrlLocalPublicUrlGenerator(),\n private readonly temporaryUrlGenerator: LocalTemporaryUrlGenerator = new FailingLocalTemporaryUrlGenerator(),\n private readonly uploadPreparer: PreparedUploadStrategy = new PreparedUploadsAreNotSupported(),\n ) {\n this.rootDir = join(this.rootDir, sep);\n this.prefixer = new PathPrefixer(this.rootDir, sep, join);\n }\n\n async copyFile(from: string, to: string, options: CopyFileOptions): Promise {\n maybeAbort(options.abortSignal);\n await this.ensureRootDirectoryExists();\n maybeAbort(options.abortSignal);\n await this.ensureParentDirectoryExists(to, options);\n maybeAbort(options.abortSignal);\n await copyFile(\n this.prefixer.prefixFilePath(from),\n this.prefixer.prefixFilePath(to),\n );\n }\n async moveFile(from: string, to: string, options: MoveFileOptions): Promise {\n maybeAbort(options.abortSignal);\n await this.ensureRootDirectoryExists();\n maybeAbort(options.abortSignal);\n await this.ensureParentDirectoryExists(to, options);\n maybeAbort(options.abortSignal);\n await rename(\n this.prefixer.prefixFilePath(from),\n this.prefixer.prefixFilePath(to),\n );\n }\n\n prepareUpload(path: string, options: UploadRequestOptions): Promise {\n maybeAbort(options.abortSignal);\n return this.uploadPreparer.prepareUpload(path, options);\n }\n\n temporaryUrl(path: string, options: TemporaryUrlOptions): Promise {\n maybeAbort(options.abortSignal);\n return this.temporaryUrlGenerator.temporaryUrl(path, {...this.options.temporaryUrlOptions, ...options});\n }\n\n publicUrl(path: string, options: PublicUrlOptions): Promise {\n maybeAbort(options.abortSignal);\n return this.publicUrlGenerator.publicUrl(path, {...this.options.publicUrlOptions, ...options});\n }\n\n async mimeType(path: string, options: MimeTypeOptions): Promise {\n maybeAbort(options.abortSignal);\n if (fileTypeImport === undefined) {\n fileTypeImport = dynamicallyImport('file-type');\n }\n\n if (fileTypes === undefined) {\n fileTypes = await fileTypeImport;\n maybeAbort(options.abortSignal);\n }\n\n const {fileTypeFromFile, supportedExtensions} = fileTypes;\n const extension = extname(path);\n\n if (!supportedExtensions.has(extension as any)) {\n const mimetype = lookup(extension);\n\n if (mimetype === false) {\n throw new Error('Unable to resolve mime-type');\n }\n\n return mimetype;\n }\n\n const location = this.prefixer.prefixFilePath(path);\n const result = await fileTypeFromFile(location);\n\n if (result === undefined) {\n throw new Error('Unable to resolve mime-type');\n }\n\n return result.mime;\n }\n\n async fileSize(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n const stat = await this.doStat(path, 'file');\n\n if (!stat.isFile) {\n throw new Error(`Path ${path} is not a file.`);\n }\n\n if (stat.size === undefined) {\n throw new Error('Stat unexpectedly did not return file size.');\n }\n\n return stat.size;\n }\n\n async lastModified(path: string, options: MiscellaneousOptions): Promise {\n const stat = await this.doStat(path, 'file');\n\n if (!stat.isFile) {\n throw new Error(`Path ${path} is not a file.`);\n }\n\n if (stat.lastModifiedMs === undefined) {\n throw new Error('Stat unexpectedly did not return last modified.');\n }\n\n return stat.lastModifiedMs;\n }\n\n async* list(path: string, {deep}: { deep: boolean }): AsyncGenerator {\n let entries = await opendir(this.prefixer.prefixDirectoryPath(path), {\n recursive: deep,\n });\n\n for await (const item of entries) {\n const itemPath = join(item.parentPath, item.name);\n\n yield this.mapStatToEntry(\n item,\n item.isFile()\n ? this.prefixer.stripFilePath(itemPath)\n : this.prefixer.stripDirectoryPath(itemPath)\n );\n }\n }\n\n async read(path: string, options: MiscellaneousOptions): Promise {\n const readStream = createReadStream(this.prefixer.prefixFilePath(path));\n const errorProxy = new PassThrough();\n\n readStream.on('error', error => {\n readStream.unpipe(errorProxy);\n\n if ((error as any).message?.includes('ENOENT')) {\n errorProxy.destroy(FileWasNotFound.atLocation(path, {\n cause: error,\n context: {path, options},\n }));\n } else {\n errorProxy.destroy(error);\n }\n });\n\n readStream.pipe(errorProxy);\n\n return errorProxy;\n }\n\n async write(path: string, contents: Readable, options: WriteOptions): Promise {\n maybeAbort(options?.abortSignal);\n await this.ensureRootDirectoryExists();\n maybeAbort(options?.abortSignal);\n await this.ensureParentDirectoryExists(path, options);\n maybeAbort(options?.abortSignal);\n\n const writeStream = createWriteStream(\n this.prefixer.prefixFilePath(path),\n {\n flags: 'w+',\n mode: options.visibility\n ? this.visibilityConversion.visibilityToFilePermissions(options.visibility)\n : undefined,\n },\n );\n\n if (options.abortSignal) {\n const signal = options.abortSignal;\n\n signal.addEventListener('abort', event => {\n contents.destroy(signal.reason);\n writeStream.destroy(signal.reason);\n });\n }\n\n await pipeline(contents, writeStream);\n }\n\n async deleteFile(path: string): Promise {\n await rm(this.prefixer.prefixFilePath(path), {\n force: true,\n });\n }\n\n async createDirectory(path: string, options: CreateDirectoryOptions): Promise {\n await mkdir(this.prefixer.prefixDirectoryPath(path), {\n recursive: true,\n mode: options.directoryVisibility\n ? this.visibilityConversion.visibilityToDirectoryPermissions(options.directoryVisibility)\n : undefined,\n });\n }\n\n async stat(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n\n return this.doStat(path, 'file');\n }\n\n private async doStat(path: string, type: 'file' | 'directory' = 'file'): Promise {\n return this.mapStatToEntry(\n await stat(type === 'file'\n ? this.prefixer.prefixFilePath(path)\n : this.prefixer.prefixDirectoryPath(path)),\n path,\n );\n }\n\n async fileExists(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n\n try {\n const stat = await this.doStat(path, 'file');\n\n return stat.isFile;\n } catch (e) {\n if (typeof e === 'object' && (e as any).code === 'ENOENT') {\n return false;\n }\n\n throw e;\n }\n }\n\n async deleteDirectory(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n\n await rm(this.prefixer.prefixDirectoryPath(path), {\n recursive: true,\n force: true,\n });\n }\n\n private mapStatToEntry(info: Stats | Dirent, path: string): StatEntry {\n if (!info.isFile() && !info.isDirectory()) {\n throw new Error('Unsupported file entry encountered...');\n }\n\n const isDirent = info instanceof Dirent;\n\n return info.isFile() ? {\n path,\n type: 'file',\n isFile: true,\n isDirectory: false,\n visibility: isDirent ? undefined : this.visibilityConversion.filePermissionsToVisibility(info.mode & 0o777),\n lastModifiedMs: isDirent ? undefined : info.mtimeMs,\n size: isDirent ? undefined : info.size,\n } : {\n path,\n type: 'directory',\n isFile: false,\n isDirectory: true,\n visibility: isDirent ? undefined : this.visibilityConversion.directoryPermissionsToVisibility(info.mode & 0o777),\n lastModifiedMs: isDirent ? undefined : info.mtimeMs,\n };\n }\n\n async changeVisibility(path: string, visibility: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n\n await chmod(\n this.prefixer.prefixFilePath(path),\n this.visibilityConversion.visibilityToFilePermissions(visibility),\n );\n }\n\n async visibility(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n const stat = await this.doStat(path, 'file');\n\n if (!stat.visibility) {\n throw new Error('Unable to determine visibility');\n }\n\n return stat.visibility;\n }\n\n async directoryExists(path: string, options: MiscellaneousOptions): Promise {\n maybeAbort(options.abortSignal);\n try {\n const stat = await this.doStat(path, 'directory');\n\n return stat.isDirectory;\n } catch (e) {\n if (typeof e === 'object' && ['ENOTDIR', 'ENOENT'].includes((e as any).code)) {\n return false;\n }\n\n throw e;\n }\n }\n\n private rootDirectoryCreation: Promise | undefined = undefined;\n\n private async ensureRootDirectoryExists(): Promise {\n if (this.rootDirectoryCreation === undefined) {\n this.rootDirectoryCreation = this.createDirectory('', {\n directoryVisibility: this.options.rootDirectoryVisibility ?? this.visibilityConversion.defaultDirectoryVisibility,\n });\n }\n\n return await this.rootDirectoryCreation;\n }\n\n private async ensureParentDirectoryExists(path: string, options: VisibilityOptions) {\n const directoryName = dirname(path);\n\n if (directoryName !== '.' && directoryName !== sep) {\n await this.createDirectory(directoryName, {\n directoryVisibility: options.directoryVisibility,\n });\n }\n }\n\n async checksum(path: string, options: ChecksumOptions): Promise {\n maybeAbort(options.abortSignal);\n\n return checksumFromStream(await this.read(path, options), options);\n }\n}\n\n/**\n * BC export\n *\n * @deprecated\n */\nexport class LocalFileStorage extends LocalStorageAdapter {}" }, { "path": "packages/local-fs/src/unix-visibility.ts", "content": "import {Visibility} from '@flystorage/file-storage';\n\nexport interface UnixVisibilityConversion {\n visibilityToFilePermissions(visibility: string): number;\n visibilityToDirectoryPermissions(visibility: string): number;\n filePermissionsToVisibility(permissions: number): string;\n directoryPermissionsToVisibility(permissions: number): string;\n defaultDirectoryPermissions: number;\n defaultDirectoryVisibility: string;\n}\n\nexport class PortableUnixVisibilityConversion implements UnixVisibilityConversion {\n constructor(\n private readonly filePublic: number = 0o644,\n private readonly filePrivate: number = 0o600,\n private readonly directoryPublic: number = 0o755,\n private readonly directoryPrivate: number = 0o700,\n public readonly defaultDirectoryVisibility: Visibility = Visibility.PUBLIC,\n ) {\n }\n\n get defaultDirectoryPermissions(): number {\n return this.visibilityToDirectoryPermissions(this.defaultDirectoryVisibility);\n }\n\n directoryPermissionsToVisibility(permissions: number): string {\n if (permissions === this.directoryPrivate) {\n return Visibility.PRIVATE;\n }\n\n return Visibility.PUBLIC;\n }\n\n filePermissionsToVisibility(permissions: number): string {\n if (permissions === this.filePrivate) {\n return Visibility.PRIVATE;\n }\n\n return Visibility.PUBLIC;\n }\n\n visibilityToDirectoryPermissions(visibility: string): number {\n if (visibility === Visibility.PUBLIC) {\n return this.directoryPublic;\n } else if (visibility === Visibility.PRIVATE) {\n return this.directoryPrivate;\n }\n\n throw new Error(`Unsupported visibility was provided: ${visibility}`);\n }\n\n visibilityToFilePermissions(visibility: string): number {\n if (visibility === Visibility.PUBLIC) {\n return this.filePublic;\n } else if (visibility === Visibility.PRIVATE) {\n return this.filePrivate;\n }\n\n throw new Error(`Unsupported visibility was provided: ${visibility}`);\n }\n\n}" }, { "path": "packages/local-fs/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"./src/**/*.ts\"\n ]\n}" }, { "path": "packages/multer-storage/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/multer-storage/CHANGELOG.md", "content": "# `@flystorage/multer-storage`\n\n## 1.2.0\n\n### Changes\n\n- Force security upgrades\n\n## 1.1.0\n\n### Changes\n\n- Unblock security fixes\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/multer-storage/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/multer-storage/README.md", "content": "\n\n# Flystorage for multer\n\nThis package contains the Flystorage bindings for [multer](https://github.com/expressjs/multer).\n\nThis allows multer to upload to any of the supported Flystorage filesystems.\n\n## Installation\n\nInstall all the required packages\n\n```bash\nnpm install --save @flystorage/file-storage @flystorage/multer-storage\n```\n\n## Usage\n\n```typescript\nimport {FileStorage} from '@flystorage/file-storage';\nimport {FlystorageMulterStorageEngine} from '@flystorage/multer-storage';\nimport multer from 'multer';\n\nconst adapter = createYourAdapter();\nconst fileStorage = new FileStorage(adapter);\n\nconst storage = new FlystorageMulterStorageEngine(\n uploadStorage,\n async (action, _req: express.Request, file: Express.Multer.File) => {\n if (action === 'handle') {\n return file.originalname;\n } else {\n return file.destination;\n }\n }\n);\n\nconst uploader = multer({storage});\n```\n\n" }, { "path": "packages/multer-storage/package.json", "content": "{\n \"name\": \"@flystorage/multer-storage\",\n \"type\": \"module\",\n \"version\": \"1.2.0\",\n \"dependencies\": {\n \"@flystorage/file-storage\": \"^1.0.0\",\n \"@flystorage/stream-mime-type\": \"^1.0.0\",\n \"multer\": \"^2.1.1\"\n },\n \"description\": \"\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"keywords\": [\n \"s3\",\n \"file\",\n \"storage\",\n \"flystorage\",\n \"filesystem\"\n ],\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/tools/multer/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/aws-s3\"\n },\n \"license\": \"MIT\",\n \"devDependencies\": {\n \"body-parser\": \"^2.2.2\",\n \"form-data\": \"^4.0.5\"\n }\n}\n" }, { "path": "packages/multer-storage/src/index.test.ts", "content": "import express, {Response, Request} from 'express';\nimport {FlystorageMulterStorageEngine} from './index.js';\nimport {LocalStorageAdapter} from \"@flystorage/local-fs\";\nimport {FileStorage} from \"@flystorage/file-storage\";\nimport {resolve} from \"node:path\";\nimport FormData from \"form-data\";\nimport bodyParser from \"body-parser\";\nimport multer from 'multer';\nimport fetch from 'node-fetch';\nimport {IncomingMessage} from 'node:http';\n\ndescribe('FlystorageMulterStorageEngine', () => {\n test('it can process uploaded files', async () => {\n const uploadStorage = new FileStorage(\n new LocalStorageAdapter(\n resolve(process.cwd(), 'fixtures/test-files'),\n )\n );\n const fixtureStorage = new FileStorage(\n new LocalStorageAdapter(\n resolve(process.cwd(), 'fixtures'),\n )\n );\n\n const app = express();\n app.use(bodyParser.urlencoded({ extended: false }));\n const storage = new FlystorageMulterStorageEngine(\n uploadStorage,\n async (action, _req: express.Request, file: Express.Multer.File) => {\n if (action === 'handle') {\n return file.originalname;\n } else {\n return file.destination;\n }\n }\n );\n\n const uploader = multer({storage});\n app.post('/upload', uploader.single('image'), (_: Request, res: Response) => {\n res.status(200).json({\n status: 'OK',\n });\n })\n const server = app.listen(5555);\n await new Promise(resolve => setTimeout(resolve, 100));\n const formData = new FormData();\n const stat = await fixtureStorage.statFile('screenshot.png');\n formData.append('image', await fixtureStorage.read('screenshot.png'), {\n knownLength: stat.size,\n contentType: await fixtureStorage.mimeType('screenshot.png'),\n filename: 'screenshot-uploaded.png',\n });\n const response = await new Promise((resolve, reject) => {\n formData.submit('http://localhost:5555/upload', (error, response) => {\n if (error) {\n reject(error);\n } else {\n resolve(response);\n }\n })\n });\n\n expect(response.statusCode).toEqual(200);\n server.close();\n\n expect(await uploadStorage.fileExists('screenshot-uploaded.png')).toEqual(true);\n })\n})" }, { "path": "packages/multer-storage/src/index.ts", "content": "import {StorageEngine} from 'multer';\nimport type {Request} from 'express';\nimport {FileStorage} from '@flystorage/file-storage';\n\nexport type DestinationResolver = (action: 'handle' | 'remove', req: Request, file: Express.Multer.File) => Promise;\n\nexport class FlystorageMulterStorageEngine implements StorageEngine {\n constructor(\n private readonly storage: FileStorage,\n private readonly destinationResolver: DestinationResolver,\n ) {\n }\n\n _handleFile(req: Request, file: Express.Multer.File, callback: (error?: any, info?: Partial) => void): void {\n (async () => {\n const destination = await (this.destinationResolver)('handle', req, file);\n await this.storage.write(destination, file.stream, {\n size: file.size,\n mimeType: file.mimetype,\n });\n\n return destination;\n })()\n .then(destination => callback(null, {destination}))\n .catch(error => callback(error));\n }\n\n _removeFile(req: Request, file: Express.Multer.File, callback: (error: (Error | null)) => void): void {\n (async () => {\n const destination = await (this.destinationResolver)('remove', req, file);\n await this.storage.deleteFile(destination);\n })()\n .then(() => callback(null))\n .catch(error => callback(error));\n }\n}" }, { "path": "packages/multer-storage/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packages/stream-mime-type/.npmignore", "content": "src\n!dist\ntsconfig.json" }, { "path": "packages/stream-mime-type/CHANGELOG.md", "content": "# `@flystorage/stream-mime-type`\n\n## 1.1.0\n\n### Minor Changes\n\n- Upgrade to file-type 20\n\n## 1.0.0\n\n### Major Changes\n\n- First 1.0.0 release" }, { "path": "packages/stream-mime-type/LICENSE", "content": "Copyright (c) 2023-2024 Frank de Jonge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is furnished\nto do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "packages/stream-mime-type/package.json", "content": "{\n \"name\": \"@flystorage/stream-mime-type\",\n \"type\": \"module\",\n \"version\": \"1.1.0\",\n \"dependencies\": {\n \"@flystorage/dynamic-import\": \"^1.0.0\",\n \"file-type\": \"^21.3.1\",\n \"mime-types\": \"^3.0.2\"\n },\n \"description\": \"Get the mime-type of a readable stream, non-destructive\",\n \"main\": \"./dist/cjs/index.js\",\n \"types\": \"./dist/types/index.d.ts\",\n \"exports\": {\n \".\": {\n \"import\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/esm/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/types/index.d.ts\",\n \"default\": \"./dist/cjs/index.js\"\n }\n }\n },\n \"scripts\": {\n \"compile\": \"rm -rf ./dist/ && concurrently npm:compile:* && echo '{\\\"type\\\": \\\"commonjs\\\"}' > ./dist/cjs/package.json\",\n \"patch\": \"gsed -i 's/dynamicallyImport(/import(/g' dist/esm/stream-mime-type.js\",\n \"compile:esm\": \"tsc --outDir ./dist/esm/ --declaration false\",\n \"compile:cjs\": \"tsc --outDir ./dist/cjs/ --declaration false --module commonjs --moduleResolution node\",\n \"compile:types\": \"tsc --outDir ./dist/types/ --declaration --emitDeclarationOnly\",\n \"watch\": \"tsc --watch\"\n },\n \"author\": \"Frank de Jonge (https://frankdejonge.nl)\",\n \"homepage\": \"https://flystorage.dev/tools/stream-mime-type/\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/duna-oss/flystorage.git\",\n \"directory\": \"packages/stream-mime-type\"\n },\n \"license\": \"MIT\"\n}\n" }, { "path": "packages/stream-mime-type/src/index.ts", "content": "export * from './stream-mime-type.js';" }, { "path": "packages/stream-mime-type/src/stream-mime-type.test.ts", "content": "import * as fs from 'node:fs';\nimport path from 'node:path';\n\nimport {Readable} from 'stream';\nimport {resolveMimeType} from './stream-mime-type.js';\n\ndescribe('resolveMimeType', () => {\n let stream: Readable;\n let mime: string | undefined;\n\n afterEach(() => {\n stream?.closed === false && stream.destroy();\n mime = undefined;\n });\n\n test('resolving the mime-type of a file', async () => {\n stream = fs.createReadStream(path.resolve(process.cwd(), 'fixtures/screenshot.png'));\n\n [mime, stream] = await resolveMimeType('screenshot.png', stream);\n\n expect(mime).toEqual('image/png');\n\n stream.closed || stream.destroy();\n });\n\n test('when no mime-type can be resolved, use an extension based lookup', async () => {\n stream = Readable.from(Buffer.from(''));\n\n [mime, stream] = await resolveMimeType('screenshot.jpg', stream);\n\n expect(mime).toEqual('image/jpeg');\n });\n\n test('when no mime-type can be resolved, the fallback is used', async () => {\n stream = Readable.from(Buffer.from(''));\n\n [mime, stream] = await resolveMimeType('screenshot.unkown', stream, 'application/this-is-not-known');\n\n expect(mime).toEqual('application/this-is-not-known');\n\n stream.closed || stream.destroy();\n });\n});" }, { "path": "packages/stream-mime-type/src/stream-mime-type.ts", "content": "import {Readable} from 'node:stream';\nimport {parse} from 'node:path';\nimport {lookup as mimeTimeForExt} from 'mime-types';\nimport {PassThrough} from 'node:stream';\nimport {dynamicallyImport} from '@flystorage/dynamic-import';\n\nfunction concatUint8Arrays(input: Uint8Array[]): Uint8Array {\n const length = input.reduce((l, a) => l + a.byteLength, 0);\n const output = new Uint8Array(length);\n let position = 0;\n input.forEach(i => {\n output.set(i, position);\n position += i.byteLength;\n });\n\n return output;\n}\n\nexport async function streamHead(stream: Readable, size: number): Promise<[Uint8Array, Readable]> {\n return new Promise((resolve, reject) => {\n const tunnel = new PassThrough();\n const outputStream = new PassThrough();\n let readBytes = 0;\n let buffers: Uint8Array[] = [];\n let resolved = false;\n\n tunnel.once('error', reject);\n tunnel.on('data', (chunk: Uint8Array) => {\n if (!resolved) {\n readBytes += chunk.byteLength;\n buffers.push(chunk);\n\n if (readBytes >= size) {\n resolved = true;\n const head = concatUint8Arrays(buffers);\n buffers = [];\n resolve([head, outputStream]);\n }\n }\n });\n tunnel.once('end', () => {\n if (!resolved) {\n resolve([concatUint8Arrays(buffers), outputStream]);\n }\n });\n stream.pipe(tunnel).pipe(outputStream);\n });\n}\n\ntype FileTypePackage = typeof import('file-type');\nlet fileTypeImport: Promise | undefined;\nlet fileTypes: FileTypePackage | undefined = undefined;\n\nexport async function resolveMimeType(\n filename: string,\n stream: Readable,\n fallback: string | undefined = undefined\n): Promise<[string|undefined, Readable]> {\n const [head, readable] = await streamHead(stream, 4100);\n\n if (fileTypeImport === undefined) {\n fileTypeImport = dynamicallyImport('file-type');\n }\n\n if (fileTypes === undefined) {\n fileTypes = await fileTypeImport;\n }\n\n\n const {fileTypeFromBuffer} = fileTypes;\n const lookup = await fileTypeFromBuffer(head);\n\n if (lookup) {\n return [lookup.mime, readable];\n }\n\n const {ext} = parse(filename);\n\n return [mimeTimeForExt(ext) || fallback, readable];\n}" }, { "path": "packages/stream-mime-type/tsconfig.json", "content": "{\n \"extends\": \"../../tsconfig.build.json\",\n \"compilerOptions\": {\n \"outDir\": \"./dist/\"\n },\n \"include\": [\n \"src/**/*.ts\"\n ]\n}" }, { "path": "packed/.gitignore", "content": "*.tgz\n*/node_modules/\n*/package-lock.json\n*/files/" }, { "path": "packed/cjs/index.js", "content": "const {FileStorage} = require('@flystorage/file-storage');\nconst {LocalStorageAdapter} = require('@flystorage/local-fs');\nconst {resolve} = require(\"path\");\n\nconst storage = new FileStorage(\n new LocalStorageAdapter(\n resolve(__dirname, 'files'),\n )\n);\n\n(async () => {\n await storage.write('some/deep/file.txt', 'contents');\n console.log(await storage.mimeType('path.svg'));\n const mimetype = await storage.mimeType('screenshot.png');\n console.log(mimetype);\n console.log(await storage.list('', {deep: true}).toArray());\n})();" }, { "path": "packed/cjs/index.test.js", "content": "const {FileStorage} = require('@flystorage/file-storage');\nconst {LocalStorageAdapter} = require('@flystorage/local-fs');\n\ndescribe('running inside jest', () => {\n test('testing it', async () => {\n const fs = new FileStorage(\n new LocalStorageAdapter(\n `${__dirname}/files`,\n ),\n );\n\n await fs.write('jest/file.txt', 'contents');\n\n const listing = await fs.list('jest').toArray();\n\n expect(listing.length).toBe(1);\n })\n})" }, { "path": "packed/cjs/package.json", "content": "{\n \"private\": true,\n \"type\": \"commonjs\",\n \"name\": \"packed-cjs\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"dependencies\": {\n \"@flystorage/file-storage\": \"file:~/Sites/filestorage/packed/flystorage-file-storage-0.1.7.tgz\",\n \"@flystorage/local-fs\": \"file:~/Sites/filestorage/packed/flystorage-local-fs-0.1.9.tgz\",\n \"@flystorage/stream-mime-type\": \"file:~/Sites/filestorage/packed/flystorage-stream-mime-type-0.1.6.tgz\"\n },\n \"keywords\": [],\n \"author\": \"\",\n \"license\": \"ISC\",\n \"devDependencies\": {\n \"jest\": \"^30.0.0\"\n }\n}\n" }, { "path": "packed/esm/index.js", "content": "import {FileStorage} from '@flystorage/file-storage';\nimport {LocalStorageAdapter} from '@flystorage/local-fs';\nimport {join} from 'path';\n\nconst storage = new FileStorage(\n new LocalStorageAdapter(\n join(process.cwd(), 'files'),\n )\n);\n\nawait storage.write('some/deep/file.txt', 'contents');\nconsole.log(await storage.mimeType('path.svg'));\nconst mimetype = await storage.mimeType('screenshot.png');\nconsole.log(mimetype);\nconsole.log(await storage.list('', {deep: true}).toArray());" }, { "path": "packed/esm/package.json", "content": "{\n \"private\": true,\n \"type\": \"module\",\n \"name\": \"packed-esm\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"dependencies\": {\n \"@flystorage/file-storage\": \"file:~/Sites/filestorage/packed/flystorage-file-storage-0.1.7.tgz\",\n \"@flystorage/local-fs\": \"file:~/Sites/filestorage/packed/flystorage-local-fs-0.1.10.tgz\",\n \"@flystorage/stream-mime-type\": \"file:~/Sites/filestorage/packed/flystorage-stream-mime-type-0.1.7.tgz\"\n },\n \"keywords\": [],\n \"author\": \"\",\n \"license\": \"ISC\"\n}\n" }, { "path": "tsconfig.build.json", "content": "{\n \"exclude\": [\n \"**/*.test.ts\",\n \"node_modules\",\n \"dist\"\n ],\n \"compilerOptions\": {\n \"esModuleInterop\": false,\n \"declaration\": true,\n \"skipLibCheck\": true,\n \"lib\": [\"es2023\"],\n \"module\": \"node16\",\n \"target\": \"es2022\",\n \"strict\": true,\n \"forceConsistentCasingInFileNames\": true,\n }\n}" }, { "path": "tsconfig.json", "content": "{\n \"exclude\": [\n \"node_modules\",\n \"dist\",\n \"**/dist/**/*.js\"\n ],\n \"include\": [\n \"./**/*.ts\"\n ],\n \"compilerOptions\": {\n \"declaration\": true,\n \"skipLibCheck\": true,\n \"lib\": [\"es2023\"],\n \"forceConsistentCasingInFileNames\": true,\n \"target\": \"ES2020\",\n \"module\": \"NodeNext\",\n \"strict\": true,\n \"moduleResolution\": \"NodeNext\",\n \"esModuleInterop\": true,\n \"allowSyntheticDefaultImports\": true,\n \"types\": [\n \"node\",\n \"vitest/globals\"\n ],\n \"paths\": {\n \"@flystorage/*\": [\"./packages/*/src/index.ts\"]\n }\n },\n\n}" }, { "path": "vitest.config.ts", "content": "import { defineConfig } from 'vitest/config';\nimport tsconfigPaths from 'vite-tsconfig-paths';\nimport path from 'node:path';\n\nexport default defineConfig({\n plugins: [tsconfigPaths()],\n resolve: {\n alias: [\n {\n find: /^@flystorage\\/(.*)$/,\n replacement: path.join(new URL(import.meta.url).pathname, '../packages', '$1', 'src/index.ts'),\n }\n ]\n },\n test: {\n testTimeout: 10_000,\n include: ['packages/**/*.{test,spec}.?(c|m)[jt]s?(x)'],\n globals: true,\n clearMocks: false,\n setupFiles: ['dotenv/config'],\n\n // profiling\n // pool: 'forks',\n // poolOptions: {\n // forks: {\n // execArgv: [\n // '--cpu-prof',\n // '--cpu-prof-dir=test-runner-profile',\n // // '--heap-prof',\n // // '--heap-prof-dir=test-runner-profile'\n // ],\n //\n // // To generate a single profile\n // singleFork: true,\n // },\n // },\n },\n});" } ]