[
  {
    "path": ".gitignore",
    "content": "*/node_modules\n*/.next"
  },
  {
    "path": "README.md",
    "content": "## Multipart + Presigned URL upload to AWS S3/Minio via the browser\n\n### Motivation\n\nI created this demo repo because documentation for multipart uploading of large files using presigned URLs was very scant.\n\nI wanted to create a solution to allow users to upload files directly from the browser to AWS S3 (or any S3-compliant storage server). This worked great when I used AWS SDK's getSignedUrl API to generate a temporary URL that the browser could upload the file to. \n\nHowever, I hit a snag when dealing with files > 5GB because the pre-signed URL only allows for a maximum file size of 5GB to be uploaded at one go. As such, this repo demonstrates the use of multipart + presigned URLs to upload large files to an AWS S3-compliant storage service.\n\n### Components used in this demo\n\n* Frontend Server: React (Next.js)\n* Backend Server: Node.js (Express), using the AWS JS SDK\n* Storage Server: Minio (but this can easily be switched out to AWS S3)\n\n### How to run\n\n* Clone the repo and change directory into the repo\n* Open three different terminal windows.\n\n**Storage Server**\n\nIn window 1, run:\n```\n# Set up the Minio server (ignore this if you are using AWS S3)\n# Minio docs: https://docs.minio.io/docs/minio-quickstart-guide\nminio server /data\n```\n> Note: Set the S3-compliant bucket policy as appropriate to allow the right access\n\n**Backend Server**\n\nReplace the following code in `backend/server.js` with your AWS S3 or S3-compliant storage server config.\n\n```\nconst s3  = new AWS.S3({\n  accessKeyId: '<ACCESS_KEY_ID>' , // Replace with your access key id\n  secretAccessKey: '<SECRET_ACCESS_KEY>' , // Replace with your secret access key\n  endpoint: 'http://127.0.0.1:9000' ,\n  s3ForcePathStyle: true, // needed with minio?\n  signatureVersion: 'v4'\n});\n```\n\nNote: If you are using AWS S3, follow the docs on the AWS website to instantiate a new AWS S3 client.\n\nIn window 2, run:\n```\ncd backend\nnpm install\nnode server.js\n```\n\n**Frontend Server**\n\nIn window 3, run:\n```\ncd frontend\nnpm install\nnpm run dev\n```\n\n**Upload File**\n\nGo to `http://localhost:3000` in your browser window and upload a file.\n"
  },
  {
    "path": "backend/package.json",
    "content": "{\n  \"name\": \"aws-sdk-multipart-presigned-upload-backend\",\n  \"version\": \"1.0.0\",\n  \"description\": \"\",\n  \"main\": \"index.js\",\n  \"scripts\": {\n    \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n  },\n  \"keywords\": [],\n  \"author\": \"\",\n  \"license\": \"ISC\",\n  \"dependencies\": {\n    \"aws-sdk\": \"^2.375.0\",\n    \"bluebird\": \"^3.5.3\",\n    \"body-parser\": \"^1.18.3\",\n    \"express\": \"^4.16.4\",\n    \"uuid\": \"^3.3.2\"\n  }\n}\n"
  },
  {
    "path": "backend/server.js",
    "content": "const express = require('express')\nconst app = express()\nconst BluebirdPromise = require('bluebird')\nconst AWS = require('aws-sdk')\nconst bodyParser = require('body-parser')\n\napp.use(bodyParser.json())\n\nconst port = 4000\nconst BUCKET_NAME = \"vaultgovsg\"\n\nconst s3  = new AWS.S3({\n\taccessKeyId: '<ACCESS_KEY_ID>' , // Replace with your access key id\n\tsecretAccessKey: '<SECRET_ACCESS_KEY>' , // Replace with your secret access key\n\tendpoint: 'http://127.0.0.1:9000' ,\n\ts3ForcePathStyle: true, // needed with minio?\n\tsignatureVersion: 'v4'\n});\n\napp.use(function(req, res, next) {\n  res.header(\"Access-Control-Allow-Origin\", \"*\");\n  res.header(\"Access-Control-Allow-Headers\", \"Origin, X-Requested-With, Content-Type, Accept\");\n  next();\n});\n\napp.get('/', (req, res, next) => {\n\tres.send('Hello World!')\n})\n\napp.get('/start-upload', async (req, res, next) => {\n\ttry {\n\t\tlet params = {\n\t\t\tBucket: BUCKET_NAME,\n\t\t\tKey: req.query.fileName,\n\t\t\tContentType: req.query.fileType\n\t\t}\n\t\tlet createUploadPromised = BluebirdPromise.promisify(s3.createMultipartUpload.bind(s3))\n\t\tlet uploadData = await createUploadPromised(params)\n\t\tres.send({uploadId: uploadData.UploadId})\n\t} catch(err) {\n\t\tconsole.log(err)\n\t}\n})\n\napp.get('/get-upload-url', async (req, res, next) => {\n\ttry {\n\t\tlet params = {\n\t\t\tBucket: BUCKET_NAME,\n\t\t\tKey: req.query.fileName,\n\t\t\tPartNumber: req.query.partNumber,\n\t\t\tUploadId: req.query.uploadId\n\t\t}\n\t\tconsole.log(params)\n\t    let uploadPartPromised = BluebirdPromise.promisify(s3.getSignedUrl.bind(s3))\n\t    let presignedUrl = await uploadPartPromised('uploadPart', params)\n\t\tres.send({presignedUrl})\n\t} catch(err) {\n\t\tconsole.log(err)\n\t}\n})\n\napp.post('/complete-upload', async (req, res, next) => {\n\ttry {\n\t\tconsole.log(req.body, ': body')\n\t\tlet params = {\n\t\t\tBucket: BUCKET_NAME,\n\t\t\tKey: req.body.params.fileName,\n\t\t\tMultipartUpload: {\n\t\t\t\tParts: req.body.params.parts\n\t\t\t},\n\t\t\tUploadId: req.body.params.uploadId\n\t\t}\n\t\tconsole.log(params)\n\t    let completeUploadPromised = BluebirdPromise.promisify(s3.completeMultipartUpload.bind(s3))\n\t    let data = await completeUploadPromised(params)\n\t\tres.send({data})\n\t} catch(err) {\n\t\tconsole.log(err)\n\t}\n})\n\napp.listen(port, () => console.log(`Example app listening on port ${port}!`))"
  },
  {
    "path": "frontend/package.json",
    "content": "{\n  \"name\": \"aws-sdk-multipart-presigned-upload-frontend\",\n  \"version\": \"1.0.0\",\n  \"description\": \"\",\n  \"main\": \"index.js\",\n  \"scripts\": {\n    \"dev\": \"next\",\n    \"build\": \"next build\",\n    \"start\": \"next start\"\n  },\n  \"keywords\": [],\n  \"author\": \"\",\n  \"license\": \"ISC\",\n  \"dependencies\": {\n    \"next\": \"^7.0.2\",\n    \"react\": \"^16.6.3\",\n    \"react-dom\": \"^16.6.3\"\n  }\n}\n"
  },
  {
    "path": "frontend/pages/index.js",
    "content": "import React, { Component } from 'react'\nimport axios from 'axios'\n\nexport default class Index extends Component {\n\tconstructor(props) {\n\t\tsuper(props)\n    this.state = {\n      selectedFile: null,\n      uploadId: '',\n      fileName: '',\n      backendUrl: 'http://localhost:4000'\n    }\n\t}\n\n// ===============================================\n// The fileChangedHandler obtains the file specified in the\n// input field in the form.\n// ===============================================\n\n  async fileChangedHandler(event) {\n    try {\n      // console.log('Inside fileChangedHandler')\n      let selectedFile = event.target.files[0]\n      let fileName = selectedFile.name\n      this.setState({ selectedFile })\n      this.setState({ fileName })\n    } catch (err) {\n      console.error(err, err.message)\n    }\n  }\n\n// ===============================================\n// The startUpload function obtains an uploadId generated in the backend\n// server by the AWS S3 SDK. This uploadId will be used subsequently for uploading\n// the individual chunks of the selectedFile.\n// ===============================================\n\n  async startUpload(event) {\n    try {\n      // console.log('Inside startUpload')\n      event.preventDefault()\n\n      console.log(this.state.selectedFile.type + ' FileType')\n      let resp = await axios.get(`${this.state.backendUrl}/start-upload`, {\n        params: {\n          fileName: this.state.fileName,\n          fileType: this.state.selectedFile.type\n        }\n      })\n\n      let {uploadId} = resp.data\n      this.setState({uploadId})\n\n      this.uploadMultipartFile()\n    } catch(err) {\n      console.log(err)\n    }\n  }\n\n// ===============================================\n// The uploadMultipartFile function splits the selectedFile into chunks\n// of 10MB and does the following:\n// (1) call the backend server for a presigned url for each part,\n// (2) uploads them, and\n// (3) upon completion of all responses, sends a completeMultipartUpload call to the backend server.\n//\n// Note: the AWS SDK can only split one file into 10,000 separate uploads.\n// This means that, each uploaded part being 10MB, each file has a max size of \n// 100GB.\n// ===============================================\n\n  async uploadMultipartFile() {\n    try {\n      console.log('Inside uploadMultipartFile')\n      const FILE_CHUNK_SIZE = 10000000 // 10MB\n      const fileSize = this.state.selectedFile.size\n      const NUM_CHUNKS = Math.floor(fileSize / FILE_CHUNK_SIZE) + 1\n      let promisesArray = []\n      let start, end, blob\n\n      for (let index = 1; index < NUM_CHUNKS + 1; index++) {\n        start = (index - 1)*FILE_CHUNK_SIZE\n        end = (index)*FILE_CHUNK_SIZE\n        blob = (index < NUM_CHUNKS) ? this.state.selectedFile.slice(start, end) : this.state.selectedFile.slice(start)\n\n        // (1) Generate presigned URL for each part\n        let getUploadUrlResp = await axios.get(`${this.state.backendUrl}/get-upload-url`, {\n          params: {\n            fileName: this.state.fileName,\n            partNumber: index,\n            uploadId: this.state.uploadId\n          }\n        })\n\n        let { presignedUrl } = getUploadUrlResp.data\n        console.log('   Presigned URL ' + index + ': ' + presignedUrl + ' filetype ' + this.state.selectedFile.type)\n\n        // (2) Puts each file part into the storage server\n        let uploadResp = axios.put(\n          presignedUrl,\n          blob,\n          { headers: { 'Content-Type': this.state.selectedFile.type } }\n        )\n        // console.log('   Upload no ' + index + '; Etag: ' + uploadResp.headers.etag)\n        promisesArray.push(uploadResp)\n      }\n\n      let resolvedArray = await Promise.all(promisesArray)\n      console.log(resolvedArray, ' resolvedAr')\n\n      let uploadPartsArray = []\n      resolvedArray.forEach((resolvedPromise, index) => {\n        uploadPartsArray.push({\n          ETag: resolvedPromise.headers.etag,\n          PartNumber: index + 1\n        })\n      })\n\n      // (3) Calls the CompleteMultipartUpload endpoint in the backend server\n\n      let completeUploadResp = await axios.post(`${this.state.backendUrl}/complete-upload`, {\n        params: {\n          fileName: this.state.fileName,\n          parts: uploadPartsArray,\n          uploadId: this.state.uploadId\n        }\n      })\n\n      console.log(completeUploadResp.data, ' Stuff')\n\n    } catch(err) {\n      console.log(err)\n    }\n  }\n\n\trender() {\n\t\treturn (\n      <div>\n        <form onSubmit={this.startUpload.bind(this)}>\n          <div>\n            <p>Upload Dataset:</p>\n            <input type='file' id='file' onChange={this.fileChangedHandler.bind(this)} />\n            <button type='submit'>\n              Upload\n            </button>\n          </div>\n        </form>\n      </div>\n\t\t)\n\t}\n}"
  }
]