Full Code of prestonlimlianjie/aws-s3-multipart-presigned-upload for AI

Repository: prestonlimlianjie/aws-s3-multipart-presigned-upload
Branch: master
Commit: 3615aee50ea9
Files: 6
Total size: 9.7 KB

Directory structure:
gitextract_ylb0pit2/

├── .gitignore
├── README.md
├── backend/
│   ├── package.json
│   └── server.js
└── frontend/
    ├── package.json
    └── pages/
        └── index.js

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
*/node_modules
*/.next

================================================
FILE: README.md
================================================
## Multipart + Presigned URL upload to AWS S3/Minio via the browser

### Motivation

I created this demo repo because documentation for multipart uploading of large files using presigned URLs was very scant.

I wanted to create a solution to allow users to upload files directly from the browser to AWS S3 (or any S3-compliant storage server). This worked great when I used AWS SDK's getSignedUrl API to generate a temporary URL that the browser could upload the file to. 

However, I hit a snag when dealing with files > 5GB because the pre-signed URL only allows for a maximum file size of 5GB to be uploaded at one go. As such, this repo demonstrates the use of multipart + presigned URLs to upload large files to an AWS S3-compliant storage service.

### Components used in this demo

* Frontend Server: React (Next.js)
* Backend Server: Node.js (Express), using the AWS JS SDK
* Storage Server: Minio (but this can easily be switched out to AWS S3)

### How to run

* Clone the repo and change directory into the repo
* Open three different terminal windows.

**Storage Server**

In window 1, run:
```
# Set up the Minio server (ignore this if you are using AWS S3)
# Minio docs: https://docs.minio.io/docs/minio-quickstart-guide
minio server /data
```
> Note: Set the S3-compliant bucket policy as appropriate to allow the right access

**Backend Server**

Replace the following code in `backend/server.js` with your AWS S3 or S3-compliant storage server config.

```
const s3  = new AWS.S3({
  accessKeyId: '<ACCESS_KEY_ID>' , // Replace with your access key id
  secretAccessKey: '<SECRET_ACCESS_KEY>' , // Replace with your secret access key
  endpoint: 'http://127.0.0.1:9000' ,
  s3ForcePathStyle: true, // needed with minio?
  signatureVersion: 'v4'
});
```

Note: If you are using AWS S3, follow the docs on the AWS website to instantiate a new AWS S3 client.

In window 2, run:
```
cd backend
npm install
node server.js
```

**Frontend Server**

In window 3, run:
```
cd frontend
npm install
npm run dev
```

**Upload File**

Go to `http://localhost:3000` in your browser window and upload a file.


================================================
FILE: backend/package.json
================================================
{
  "name": "aws-sdk-multipart-presigned-upload-backend",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "aws-sdk": "^2.375.0",
    "bluebird": "^3.5.3",
    "body-parser": "^1.18.3",
    "express": "^4.16.4",
    "uuid": "^3.3.2"
  }
}


================================================
FILE: backend/server.js
================================================
const express = require('express')
const app = express()
const BluebirdPromise = require('bluebird')
const AWS = require('aws-sdk')
const bodyParser = require('body-parser')

app.use(bodyParser.json())

const port = 4000
const BUCKET_NAME = "vaultgovsg"

const s3  = new AWS.S3({
	accessKeyId: '<ACCESS_KEY_ID>' , // Replace with your access key id
	secretAccessKey: '<SECRET_ACCESS_KEY>' , // Replace with your secret access key
	endpoint: 'http://127.0.0.1:9000' ,
	s3ForcePathStyle: true, // needed with minio?
	signatureVersion: 'v4'
});

app.use(function(req, res, next) {
  res.header("Access-Control-Allow-Origin", "*");
  res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
  next();
});

app.get('/', (req, res, next) => {
	res.send('Hello World!')
})

app.get('/start-upload', async (req, res, next) => {
	try {
		let params = {
			Bucket: BUCKET_NAME,
			Key: req.query.fileName,
			ContentType: req.query.fileType
		}
		let createUploadPromised = BluebirdPromise.promisify(s3.createMultipartUpload.bind(s3))
		let uploadData = await createUploadPromised(params)
		res.send({uploadId: uploadData.UploadId})
	} catch(err) {
		console.log(err)
	}
})

app.get('/get-upload-url', async (req, res, next) => {
	try {
		let params = {
			Bucket: BUCKET_NAME,
			Key: req.query.fileName,
			PartNumber: req.query.partNumber,
			UploadId: req.query.uploadId
		}
		console.log(params)
	    let uploadPartPromised = BluebirdPromise.promisify(s3.getSignedUrl.bind(s3))
	    let presignedUrl = await uploadPartPromised('uploadPart', params)
		res.send({presignedUrl})
	} catch(err) {
		console.log(err)
	}
})

app.post('/complete-upload', async (req, res, next) => {
	try {
		console.log(req.body, ': body')
		let params = {
			Bucket: BUCKET_NAME,
			Key: req.body.params.fileName,
			MultipartUpload: {
				Parts: req.body.params.parts
			},
			UploadId: req.body.params.uploadId
		}
		console.log(params)
	    let completeUploadPromised = BluebirdPromise.promisify(s3.completeMultipartUpload.bind(s3))
	    let data = await completeUploadPromised(params)
		res.send({data})
	} catch(err) {
		console.log(err)
	}
})

app.listen(port, () => console.log(`Example app listening on port ${port}!`))

================================================
FILE: frontend/package.json
================================================
{
  "name": "aws-sdk-multipart-presigned-upload-frontend",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "dev": "next",
    "build": "next build",
    "start": "next start"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "next": "^7.0.2",
    "react": "^16.6.3",
    "react-dom": "^16.6.3"
  }
}


================================================
FILE: frontend/pages/index.js
================================================
import React, { Component } from 'react'
import axios from 'axios'

export default class Index extends Component {
	constructor(props) {
		super(props)
    this.state = {
      selectedFile: null,
      uploadId: '',
      fileName: '',
      backendUrl: 'http://localhost:4000'
    }
	}

// ===============================================
// The fileChangedHandler obtains the file specified in the
// input field in the form.
// ===============================================

  async fileChangedHandler(event) {
    try {
      // console.log('Inside fileChangedHandler')
      let selectedFile = event.target.files[0]
      let fileName = selectedFile.name
      this.setState({ selectedFile })
      this.setState({ fileName })
    } catch (err) {
      console.error(err, err.message)
    }
  }

// ===============================================
// The startUpload function obtains an uploadId generated in the backend
// server by the AWS S3 SDK. This uploadId will be used subsequently for uploading
// the individual chunks of the selectedFile.
// ===============================================

  async startUpload(event) {
    try {
      // console.log('Inside startUpload')
      event.preventDefault()

      console.log(this.state.selectedFile.type + ' FileType')
      let resp = await axios.get(`${this.state.backendUrl}/start-upload`, {
        params: {
          fileName: this.state.fileName,
          fileType: this.state.selectedFile.type
        }
      })

      let {uploadId} = resp.data
      this.setState({uploadId})

      this.uploadMultipartFile()
    } catch(err) {
      console.log(err)
    }
  }

// ===============================================
// The uploadMultipartFile function splits the selectedFile into chunks
// of 10MB and does the following:
// (1) call the backend server for a presigned url for each part,
// (2) uploads them, and
// (3) upon completion of all responses, sends a completeMultipartUpload call to the backend server.
//
// Note: the AWS SDK can only split one file into 10,000 separate uploads.
// This means that, each uploaded part being 10MB, each file has a max size of 
// 100GB.
// ===============================================

  async uploadMultipartFile() {
    try {
      console.log('Inside uploadMultipartFile')
      const FILE_CHUNK_SIZE = 10000000 // 10MB
      const fileSize = this.state.selectedFile.size
      const NUM_CHUNKS = Math.floor(fileSize / FILE_CHUNK_SIZE) + 1
      let promisesArray = []
      let start, end, blob

      for (let index = 1; index < NUM_CHUNKS + 1; index++) {
        start = (index - 1)*FILE_CHUNK_SIZE
        end = (index)*FILE_CHUNK_SIZE
        blob = (index < NUM_CHUNKS) ? this.state.selectedFile.slice(start, end) : this.state.selectedFile.slice(start)

        // (1) Generate presigned URL for each part
        let getUploadUrlResp = await axios.get(`${this.state.backendUrl}/get-upload-url`, {
          params: {
            fileName: this.state.fileName,
            partNumber: index,
            uploadId: this.state.uploadId
          }
        })

        let { presignedUrl } = getUploadUrlResp.data
        console.log('   Presigned URL ' + index + ': ' + presignedUrl + ' filetype ' + this.state.selectedFile.type)

        // (2) Puts each file part into the storage server
        let uploadResp = axios.put(
          presignedUrl,
          blob,
          { headers: { 'Content-Type': this.state.selectedFile.type } }
        )
        // console.log('   Upload no ' + index + '; Etag: ' + uploadResp.headers.etag)
        promisesArray.push(uploadResp)
      }

      let resolvedArray = await Promise.all(promisesArray)
      console.log(resolvedArray, ' resolvedAr')

      let uploadPartsArray = []
      resolvedArray.forEach((resolvedPromise, index) => {
        uploadPartsArray.push({
          ETag: resolvedPromise.headers.etag,
          PartNumber: index + 1
        })
      })

      // (3) Calls the CompleteMultipartUpload endpoint in the backend server

      let completeUploadResp = await axios.post(`${this.state.backendUrl}/complete-upload`, {
        params: {
          fileName: this.state.fileName,
          parts: uploadPartsArray,
          uploadId: this.state.uploadId
        }
      })

      console.log(completeUploadResp.data, ' Stuff')

    } catch(err) {
      console.log(err)
    }
  }

	render() {
		return (
      <div>
        <form onSubmit={this.startUpload.bind(this)}>
          <div>
            <p>Upload Dataset:</p>
            <input type='file' id='file' onChange={this.fileChangedHandler.bind(this)} />
            <button type='submit'>
              Upload
            </button>
          </div>
        </form>
      </div>
		)
	}
}